From 7822cef1ac6658e4d45134764a566623d306e26f Mon Sep 17 00:00:00 2001 From: Sean Parkinson Date: Fri, 25 Jan 2019 12:03:08 +1000 Subject: [PATCH] Pull out x86_64 ASM into separate files --- configure.ac | 392 +- src/include.am | 14 + wolfcrypt/benchmark/benchmark.c | 2 + wolfcrypt/src/aes.c | 3154 +--- wolfcrypt/src/aes_gcm_asm.S | 8367 ++++++++++ wolfcrypt/src/chacha.c | 1226 +- wolfcrypt/src/chacha_asm.S | 1314 ++ wolfcrypt/src/poly1305.c | 865 +- wolfcrypt/src/poly1305_asm.S | 986 ++ wolfcrypt/src/sha256.c | 1747 +- wolfcrypt/src/sha256_asm.S | 22437 ++++++++++++++++++++++++++ wolfcrypt/src/sha512.c | 1850 +-- wolfcrypt/src/sha512_asm.S | 10545 ++++++++++++ wolfcrypt/src/sp_x86_64.c | 25810 +---------------------------- wolfcrypt/src/sp_x86_64_asm.S | 25934 ++++++++++++++++++++++++++++++ wolfcrypt/test/test.c | 4 +- wolfssl/wolfcrypt/sha256.h | 3 - 17 files changed, 70054 insertions(+), 34596 deletions(-) create mode 100644 wolfcrypt/src/aes_gcm_asm.S create mode 100644 wolfcrypt/src/chacha_asm.S create mode 100644 wolfcrypt/src/poly1305_asm.S create mode 100644 wolfcrypt/src/sha256_asm.S create mode 100644 wolfcrypt/src/sha512_asm.S create mode 100644 wolfcrypt/src/sp_x86_64_asm.S diff --git a/configure.ac b/configure.ac index 2cb42ebbee..e066dacfb3 100644 --- a/configure.ac +++ b/configure.ac @@ -1063,6 +1063,7 @@ then fi AM_CONDITIONAL([BUILD_AESNI], [test "x$ENABLED_AESNI" = "xyes"]) +AM_CONDITIONAL([BUILD_INTELASM], [test "x$ENABLED_INTELASM" = "xyes"]) # Linux af_alg @@ -3554,191 +3555,6 @@ then fi fi -# Single Precision maths implementation -AC_ARG_ENABLE([sp], - [AS_HELP_STRING([--enable-sp],[Enable Single Precision maths implementation (default: disabled)])], - [ ENABLED_SP=$enableval ], - [ ENABLED_SP=no ], - ) - -ENABLED_SP_RSA=no -ENABLED_SP_DH=no -ENABLED_SP_ECC=no -for v in `echo $ENABLED_SP | tr "," " "` -do - case $v in - small) - AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_SP_SMALL" - ENABLED_SP_RSA=yes - ENABLED_SP_DH=yes - ENABLED_SP_ECC=yes - ;; - yes) - ENABLED_SP_RSA=yes - ENABLED_SP_DH=yes - ENABLED_SP_ECC=yes - ;; - no) - ;; - - smallec256 | smallp256 | small256) - ENABLED_SP_ECC=yes - AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_SP_SMALL" - ;; - ec256 | p256 | 256) - ENABLED_SP_ECC=yes - ;; - - small2048) - AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_SP_SMALL" - ENABLED_SP_RSA=yes - ENABLED_SP_DH=yes - AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_SP_NO_3072" - ;; - 2048) - ENABLED_SP_RSA=yes - ENABLED_SP_DH=yes - AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_SP_NO_3072" - ;; - - smallrsa2048) - AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_SP_SMALL" - ENABLED_SP_RSA=yes - AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_SP_NO_3072" - ;; - rsa2048) - ENABLED_SP_RSA=yes - AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_SP_NO_3072" - ;; - - small3072) - AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_SP_SMALL" - ENABLED_SP_RSA=yes - ENABLED_SP_DH=yes - AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_SP_NO_2048" - ;; - 3072) - ENABLED_SP_RSA=yes - ENABLED_SP_DH=yes - AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_SP_NO_2048" - ;; - - smallrsa3072) - AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_SP_SMALL" - ENABLED_SP_RSA=yes - AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_SP_NO_2048" - ;; - rsa3072) - ENABLED_SP_RSA=yes - AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_SP_NO_2048" - ;; - - *) - AC_MSG_ERROR([Invalid choice of Single Precision length in bits [256, 2048, 3072]: $ENABLED_SP.]) - break;; - esac -done - -ENABLED_SP=no -if test "$ENABLED_RSA" = "yes" && test "$ENABLED_SP_RSA" = "yes"; then - ENABLED_SP=yes - AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_HAVE_SP_RSA" -fi -if test "$ENABLED_DH" = "yes" && test "$ENABLED_SP_DH" = "yes"; then - ENABLED_SP=yes - AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_HAVE_SP_DH" -fi -if test "$ENABLED_ECC" = "yes" && test "$ENABLED_SP_ECC" = "yes"; then - ENABLED_SP=yes - AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_HAVE_SP_ECC" -fi - - -AC_ARG_ENABLE([sp-asm], - [AS_HELP_STRING([--enable-sp-asm],[Enable Single Precision assembly implementation (default: disabled)])], - [ ENABLED_SP_ASM=$enableval ], - [ ENABLED_SP_ASM=no ], - ) -if test "$ENABLED_SP_ASM" = "yes"; then - if test "$ENABLED_SP" = "no"; then - AC_MSG_ERROR([Must have SP enabled: --enable-sp]) - fi - if test "$ENABLED_ASM" = "no"; then - AC_MSG_ERROR([Assembly code turned off]) - fi - - AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_SP_ASM" - case $host_cpu in - *aarch64*) - AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_SP_ARM64_ASM" - ENABLED_SP_ARM64_ASM=yes - ;; - *arm*) - if test $host_alias = "thumb"; then - AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_SP_ARM_THUMB_ASM -mthumb -march=armv6" - ENABLED_SP_ARM_THUMB_ASM=yes - else - if test $host_alias = "cortex"; then - AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_SP_ARM_CORTEX_ASM" - ENABLED_SP_ARM_CORTEX_ASM=yes - else - AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_SP_ARM32_ASM" - ENABLED_SP_ARM32_ASM=yes - fi - fi - ;; - *x86_64*) - AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_SP_X86_64_ASM" - ENABLED_SP_X86_64_ASM=yes - ;; - *) - AC_MSG_ERROR([ASM not available for CPU. Supported CPUs: x86_64, aarch64, arm]) - ;; - esac -fi - -AC_ARG_ENABLE([sp-math], - [AS_HELP_STRING([--enable-sp-math],[Enable Single Precision math implementation only (default: disabled)])], - [ ENABLED_SP_MATH=$enableval ], - [ ENABLED_SP_MATH=no ], - ) -if test "$ENABLED_SP_MATH" = "yes"; then - if test "$ENABLED_SP" = "no"; then - AC_MSG_ERROR([Must have SP enabled: --enable-sp]) - fi - if test "$ENABLED_ECCCUSTCURVES" = "yes"; then - AC_MSG_ERROR([Cannot use single precision math and custom curves]) - fi - if test "$ENABLED_OPENSSLEXTRA" = "yes"; then - AC_MSG_ERROR([Cannot use single precision math and OpenSSL extra]) - fi - if test "$ENABLED_DSA" = "yes"; then - AC_MSG_ERROR([Cannot use single precision math and DSA]) - fi - if test "$ENABLED_SRP" = "yes"; then - AC_MSG_ERROR([Cannot use single precision math and SRP]) - fi - if test "$ENABLED_SP_RSA" = "no" && test "$ENABLED_RSA" = "yes"; then - AC_MSG_ERROR([Cannot use Single Precision maths without RSA with RSA]) - fi - if test "$ENABLED_SP_DH" = "no" && test "$ENABLED_DH" = "yes"; then - AC_MSG_ERROR([Cannot use Single Precision maths without DH with DH]) - fi -fi -if test "$ENABLED_SP_MATH" = "yes"; then - AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_SP_MATH" -fi - -AM_CONDITIONAL([BUILD_SP], [test "x$ENABLED_SP" = "xyes"]) -AM_CONDITIONAL([BUILD_SP_C], [test "x$ENABLED_SP" = "xyes" && test "x$ENABLED_SP_ASM" = "xno" ]) -AM_CONDITIONAL([BUILD_SP_ARM64], [test "x$ENABLED_SP_ARM64_ASM" = "xyes" ]) -AM_CONDITIONAL([BUILD_SP_ARM32], [test "x$ENABLED_SP_ARM32_ASM" = "xyes" ]) -AM_CONDITIONAL([BUILD_SP_ARM_THUMB], [test "x$ENABLED_SP_ARM_THUMB_ASM" = "xyes" ]) -AM_CONDITIONAL([BUILD_SP_ARM_CORTEX], [test "x$ENABLED_SP_ARM_CORTEX_ASM" = "xyes" ]) -AM_CONDITIONAL([BUILD_SP_X86_64], [test "x$ENABLED_SP_X86_64_ASM" = "xyes" ]) -AM_CONDITIONAL([BUILD_SP_INT], [test "x$ENABLED_SP_MATH" = "xyes" ]) - - # set fastmath default FASTMATH_DEFAULT=no @@ -4016,6 +3832,212 @@ AC_ARG_WITH([intelqa], ) AM_CONDITIONAL([BUILD_INTEL_QA], [test "x$ENABLED_INTEL_QA" = "xyes"]) +# Single Precision maths implementation +AC_ARG_ENABLE([sp], + [AS_HELP_STRING([--enable-sp],[Enable Single Precision maths implementation (default: disabled)])], + [ ENABLED_SP=$enableval ], + [ ENABLED_SP=no ], + ) + +ENABLED_SP_RSA=no +ENABLED_SP_DH=no +ENABLED_SP_ECC=no +for v in `echo $ENABLED_SP | tr "," " "` +do + case $v in + small) + AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_SP_SMALL" + ENABLED_SP_RSA=yes + ENABLED_SP_DH=yes + ENABLED_SP_ECC=yes + ;; + yes) + ENABLED_SP_RSA=yes + ENABLED_SP_DH=yes + ENABLED_SP_ECC=yes + ;; + no) + ;; + + smallec256 | smallp256 | small256) + ENABLED_SP_ECC=yes + AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_SP_SMALL" + AM_CCASFLAGS="$AM_CCASFLAGS -DWOLFSSL_SP_SMALL" + ;; + ec256 | p256 | 256) + ENABLED_SP_ECC=yes + ;; + + small2048) + AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_SP_SMALL" + AM_CCASFLAGS="$AM_CCASFLAGS -DWOLFSSL_SP_SMALL" + ENABLED_SP_RSA=yes + ENABLED_SP_DH=yes + AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_SP_NO_3072" + AM_CCASFLAGS="$AM_CCASFLAGS -DWOLFSSL_SP_NO_3072" + ;; + 2048) + ENABLED_SP_RSA=yes + ENABLED_SP_DH=yes + AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_SP_NO_3072" + AM_CCASFLAGS="$AM_CCASFLAGS -DWOLFSSL_SP_NO_3072" + ;; + + smallrsa2048) + AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_SP_SMALL" + AM_CCASFLAGS="$AM_CCASFLAGS -DWOLFSSL_SP_SMALL" + ENABLED_SP_RSA=yes + AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_SP_NO_3072" + AM_CCASFLAGS="$AM_CCASFLAGS -DWOLFSSL_SP_NO_3072" + ;; + rsa2048) + ENABLED_SP_RSA=yes + AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_SP_NO_3072" + AM_CCASFLAGS="$AM_CCASFLAGS -DWOLFSSL_SP_NO_3072" + ;; + + small3072) + AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_SP_SMALL" + AM_CCASFLAGS="$AM_CCASFLAGS -DWOLFSSL_SP_SMALL" + ENABLED_SP_RSA=yes + ENABLED_SP_DH=yes + AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_SP_NO_2048" + AM_CCASFLAGS="$AM_CCASFLAGS -DWOLFSSL_SP_NO_2048" + ;; + 3072) + ENABLED_SP_RSA=yes + ENABLED_SP_DH=yes + AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_SP_NO_2048" + AM_CCASFLAGS="$AM_CCASFLAGS -DWOLFSSL_SP_NO_2048" + ;; + + smallrsa3072) + AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_SP_SMALL" + AM_CCASFLAGS="$AM_CCASFLAGS -DWOLFSSL_SP_SMALL" + ENABLED_SP_RSA=yes + AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_SP_NO_2048" + AM_CCASFLAGS="$AM_CCASFLAGS -DWOLFSSL_SP_NO_2048" + ;; + rsa3072) + ENABLED_SP_RSA=yes + AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_SP_NO_2048" + AM_CCASFLAGS="$AM_CCASFLAGS -DWOLFSSL_SP_NO_2048" + ;; + + *) + AC_MSG_ERROR([Invalid choice of Single Precision length in bits [256, 2048, 3072]: $ENABLED_SP.]) + break;; + esac +done + +ENABLED_SP=no +if test "$ENABLED_RSA" = "yes" && test "$ENABLED_SP_RSA" = "yes"; then + ENABLED_SP=yes + AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_HAVE_SP_RSA" + AM_CCASFLAGS="$AM_CCASFLAGS -DWOLFSSL_HAVE_SP_RSA" +fi +if test "$ENABLED_DH" = "yes" && test "$ENABLED_SP_DH" = "yes"; then + ENABLED_SP=yes + AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_HAVE_SP_DH" + AM_CCASFLAGS="$AM_CCASFLAGS -DWOLFSSL_HAVE_SP_DH" +fi +if test "$ENABLED_ECC" = "yes" && test "$ENABLED_SP_ECC" = "yes"; then + ENABLED_SP=yes + AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_HAVE_SP_ECC" + AM_CCASFLAGS="$AM_CCASFLAGS -DWOLFSSL_HAVE_SP_ECC" +fi + + +AC_ARG_ENABLE([sp-asm], + [AS_HELP_STRING([--enable-sp-asm],[Enable Single Precision assembly implementation (default: disabled)])], + [ ENABLED_SP_ASM=$enableval ], + [ ENABLED_SP_ASM=no ], + ) +if test "$ENABLED_SP_ASM" = "yes"; then + if test "$ENABLED_SP" = "no"; then + AC_MSG_ERROR([Must have SP enabled: --enable-sp]) + fi + if test "$ENABLED_ASM" = "no"; then + AC_MSG_ERROR([Assembly code turned off]) + fi + + AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_SP_ASM" + AM_CCASFLAGS="$AM_CCASFLAGS -DWOLFSSL_SP_ASM" + case $host_cpu in + *aarch64*) + AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_SP_ARM64_ASM" + AM_CCASFLAGS="$AM_CCASFLAGS -DWOLFSSL_SP_ARM64_ASM" + ENABLED_SP_ARM64_ASM=yes + ;; + *arm*) + if test $host_alias = "thumb"; then + AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_SP_ARM_THUMB_ASM -mthumb -march=armv6" + AM_CCASFLAGS="$AM_CCASFLAGS -DWOLFSSL_SP_ARM_THUMB_ASM" + ENABLED_SP_ARM_THUMB_ASM=yes + else + if test $host_alias = "cortex"; then + AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_SP_ARM_CORTEX_ASM" + AM_CCASFLAGS="$AM_CCASFLAGS -DWOLFSSL_SP_ARM_CORTEX_ASM" + ENABLED_SP_ARM_CORTEX_ASM=yes + else + AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_SP_ARM32_ASM" + AM_CCASFLAGS="$AM_CCASFLAGS -DWOLFSSL_SP_ARM32_ASM" + ENABLED_SP_ARM32_ASM=yes + fi + fi + ;; + *x86_64*) + AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_SP_X86_64_ASM" + AM_CCASFLAGS="$AM_CCASFLAGS -DWOLFSSL_SP_X86_64_ASM" + ENABLED_SP_X86_64_ASM=yes + ;; + *) + AC_MSG_ERROR([ASM not available for CPU. Supported CPUs: x86_64, aarch64, arm]) + ;; + esac +fi + +AC_ARG_ENABLE([sp-math], + [AS_HELP_STRING([--enable-sp-math],[Enable Single Precision math implementation only (default: disabled)])], + [ ENABLED_SP_MATH=$enableval ], + [ ENABLED_SP_MATH=no ], + ) +if test "$ENABLED_SP_MATH" = "yes"; then + if test "$ENABLED_SP" = "no"; then + AC_MSG_ERROR([Must have SP enabled: --enable-sp]) + fi + if test "$ENABLED_ECCCUSTCURVES" = "yes"; then + AC_MSG_ERROR([Cannot use single precision math and custom curves]) + fi + if test "$ENABLED_OPENSSLEXTRA" = "yes"; then + AC_MSG_ERROR([Cannot use single precision math and OpenSSL extra]) + fi + if test "$ENABLED_DSA" = "yes"; then + AC_MSG_ERROR([Cannot use single precision math and DSA]) + fi + if test "$ENABLED_SRP" = "yes"; then + AC_MSG_ERROR([Cannot use single precision math and SRP]) + fi + if test "$ENABLED_SP_RSA" = "no" && test "$ENABLED_RSA" = "yes"; then + AC_MSG_ERROR([Cannot use P256 single precision only math and RSA]) + fi + if test "$ENABLED_SP_DH" = "no" && test "$ENABLED_DH" = "yes"; then + AC_MSG_ERROR([Cannot use P256 single precision only math and DH]) + fi +fi +if test "$ENABLED_SP_MATH" = "yes"; then + AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_SP_MATH" +fi + +AM_CONDITIONAL([BUILD_SP], [test "x$ENABLED_SP" = "xyes"]) +AM_CONDITIONAL([BUILD_SP_C], [test "x$ENABLED_SP" = "xyes" && test "x$ENABLED_SP_ASM" = "xno" ]) +AM_CONDITIONAL([BUILD_SP_ARM64], [test "x$ENABLED_SP_ARM64_ASM" = "xyes" ]) +AM_CONDITIONAL([BUILD_SP_ARM32], [test "x$ENABLED_SP_ARM32_ASM" = "xyes" ]) +AM_CONDITIONAL([BUILD_SP_ARM_THUMB], [test "x$ENABLED_SP_ARM_THUMB_ASM" = "xyes" ]) +AM_CONDITIONAL([BUILD_SP_ARM_CORTEX], [test "x$ENABLED_SP_ARM_CORTEX_ASM" = "xyes" ]) +AM_CONDITIONAL([BUILD_SP_X86_64], [test "x$ENABLED_SP_X86_64_ASM" = "xyes" ]) +AM_CONDITIONAL([BUILD_SP_INT], [test "x$ENABLED_SP_MATH" = "xyes" ]) + # Fast RSA using Intel IPP ippdir="${srcdir}/IPP" ipplib="lib" # if autoconf guesses 32bit system changes lib directory diff --git a/src/include.am b/src/include.am index 9a4b4bce9c..9105f913a9 100644 --- a/src/include.am +++ b/src/include.am @@ -145,6 +145,9 @@ if BUILD_ARMASM src_libwolfssl_la_SOURCES += wolfcrypt/src/port/arm/armv8-sha256.c else src_libwolfssl_la_SOURCES += wolfcrypt/src/sha256.c +if BUILD_INTELASM +src_libwolfssl_la_SOURCES += wolfcrypt/src/sha256_asm.S +endif endif endif @@ -179,6 +182,7 @@ src_libwolfssl_la_SOURCES += wolfcrypt/src/sp_c64.c endif if BUILD_SP_X86_64 src_libwolfssl_la_SOURCES += wolfcrypt/src/sp_x86_64.c +src_libwolfssl_la_SOURCES += wolfcrypt/src/sp_x86_64_asm.S endif if BUILD_SP_ARM32 src_libwolfssl_la_SOURCES += wolfcrypt/src/sp_arm32.c @@ -230,6 +234,9 @@ endif if !BUILD_FIPS_V2 if BUILD_SHA512 src_libwolfssl_la_SOURCES += wolfcrypt/src/sha512.c +if BUILD_INTELASM +src_libwolfssl_la_SOURCES += wolfcrypt/src/sha512_asm.S +endif endif endif @@ -267,6 +274,9 @@ endif if BUILD_POLY1305 src_libwolfssl_la_SOURCES += wolfcrypt/src/poly1305.c +if BUILD_INTELASM +src_libwolfssl_la_SOURCES += wolfcrypt/src/poly1305_asm.S +endif endif if BUILD_RC4 @@ -293,6 +303,7 @@ endif if !BUILD_FIPS_V2 if BUILD_AESNI src_libwolfssl_la_SOURCES += wolfcrypt/src/aes_asm.S +src_libwolfssl_la_SOURCES += wolfcrypt/src/aes_gcm_asm.S endif endif @@ -322,6 +333,9 @@ endif if BUILD_CHACHA src_libwolfssl_la_SOURCES += wolfcrypt/src/chacha.c +if BUILD_INTELASM +src_libwolfssl_la_SOURCES += wolfcrypt/src/chacha_asm.S +endif if BUILD_POLY1305 src_libwolfssl_la_SOURCES += wolfcrypt/src/chacha20_poly1305.c endif diff --git a/wolfcrypt/benchmark/benchmark.c b/wolfcrypt/benchmark/benchmark.c index 9a540dd4de..9674aeb578 100644 --- a/wolfcrypt/benchmark/benchmark.c +++ b/wolfcrypt/benchmark/benchmark.c @@ -4141,7 +4141,9 @@ exit_rsa_verify: } FREE_ARRAY_DYNAMIC(enc, BENCH_MAX_PENDING, HEAP_HINT); +#if !defined(WOLFSSL_RSA_VERIFY_INLINE) && !defined(WOLFSSL_RSA_PUBLIC_ONLY) FREE_ARRAY_DYNAMIC(out, BENCH_MAX_PENDING, HEAP_HINT); +#endif FREE_VAR(message, HEAP_HINT); } diff --git a/wolfcrypt/src/aes.c b/wolfcrypt/src/aes.c index cacd18970d..43bfe9e339 100644 --- a/wolfcrypt/src/aes.c +++ b/wolfcrypt/src/aes.c @@ -3504,3132 +3504,54 @@ int wc_AesGcmSetKey(Aes* aes, const byte* key, word32 len) #define HAVE_INTEL_AVX2 #endif /* USE_INTEL_SPEEDUP */ -#ifdef _MSC_VER - #define S(w,z) ((char)((unsigned long long)(w) >> (8*(7-(z))) & 0xFF)) - #define M128_INIT(x,y) { S((x),7), S((x),6), S((x),5), S((x),4), \ - S((x),3), S((x),2), S((x),1), S((x),0), \ - S((y),7), S((y),6), S((y),5), S((y),4), \ - S((y),3), S((y),2), S((y),1), S((y),0) } -#else - #define M128_INIT(x,y) { (x), (y) } -#endif - -static const __m128i MOD2_128 = M128_INIT(0x1, - (long long int)0xc200000000000000UL); - - -/* See Intel® Carry-Less Multiplication Instruction - * and its Usage for Computing the GCM Mode White Paper - * by Shay Gueron, Intel Mobility Group, Israel Development Center; - * and Michael E. Kounavis, Intel Labs, Circuits and Systems Research */ - - -/* Figure 9. AES-GCM – Encrypt With Single Block Ghash at a Time */ - -static const __m128i ONE = M128_INIT(0x0, 0x1); -#ifndef AES_GCM_AESNI_NO_UNROLL -static const __m128i TWO = M128_INIT(0x0, 0x2); -static const __m128i THREE = M128_INIT(0x0, 0x3); -static const __m128i FOUR = M128_INIT(0x0, 0x4); -static const __m128i FIVE = M128_INIT(0x0, 0x5); -static const __m128i SIX = M128_INIT(0x0, 0x6); -static const __m128i SEVEN = M128_INIT(0x0, 0x7); -static const __m128i EIGHT = M128_INIT(0x0, 0x8); -#endif -static const __m128i BSWAP_EPI64 = M128_INIT(0x0001020304050607, 0x08090a0b0c0d0e0f); -static const __m128i BSWAP_MASK = M128_INIT(0x08090a0b0c0d0e0f, 0x0001020304050607); - - #ifndef _MSC_VER -#define _VAR(a) "" #a "" -#define VAR(a) _VAR(a) - -#define HR %%xmm14 -#define XR %%xmm15 -#define KR %%ebx -#define KR64 %%rbx -#if !defined(AES_GCM_AESNI_NO_UNROLL) && !defined(AES_GCM_AVX1_NO_UNROLL) -#define CTR1 128(%%rsp) -#define TR 144(%%rsp) -#define HTR %%rsp -#define STACK_OFFSET 160 -#else -#define CTR1 (%%rsp) -#define TR 16(%%rsp) -#define STACK_OFFSET 32 -#endif - -#define AESENC() \ - "aesenc %%xmm12, %%xmm4\n\t" \ - "aesenc %%xmm12, %%xmm5\n\t" \ - "aesenc %%xmm12, %%xmm6\n\t" \ - "aesenc %%xmm12, %%xmm7\n\t" \ - "aesenc %%xmm12, %%xmm8\n\t" \ - "aesenc %%xmm12, %%xmm9\n\t" \ - "aesenc %%xmm12, %%xmm10\n\t" \ - "aesenc %%xmm12, %%xmm11\n\t" - -#define AESENC_SET(o) \ - "movdqa " #o "(%[KEY]), %%xmm12\n\t" \ - AESENC() - -#define AESENC_CTR() \ - "movdqu " VAR(CTR1) ", %%xmm4\n\t" \ - "movdqa %[BSWAP_EPI64], %%xmm1\n\t" \ - "movdqu %%xmm4, %%xmm0\n\t" \ - "pshufb %%xmm1, %%xmm4\n\t" \ - "movdqa %%xmm0, %%xmm5\n\t" \ - "paddd %[ONE], %%xmm5\n\t" \ - "pshufb %%xmm1, %%xmm5\n\t" \ - "movdqa %%xmm0, %%xmm6\n\t" \ - "paddd %[TWO], %%xmm6\n\t" \ - "pshufb %%xmm1, %%xmm6\n\t" \ - "movdqa %%xmm0, %%xmm7\n\t" \ - "paddd %[THREE], %%xmm7\n\t" \ - "pshufb %%xmm1, %%xmm7\n\t" \ - "movdqa %%xmm0, %%xmm8\n\t" \ - "paddd %[FOUR], %%xmm8\n\t" \ - "pshufb %%xmm1, %%xmm8\n\t" \ - "movdqa %%xmm0, %%xmm9\n\t" \ - "paddd %[FIVE], %%xmm9\n\t" \ - "pshufb %%xmm1, %%xmm9\n\t" \ - "movdqa %%xmm0, %%xmm10\n\t" \ - "paddd %[SIX], %%xmm10\n\t" \ - "pshufb %%xmm1, %%xmm10\n\t" \ - "movdqa %%xmm0, %%xmm11\n\t" \ - "paddd %[SEVEN], %%xmm11\n\t" \ - "pshufb %%xmm1, %%xmm11\n\t" \ - "paddd %[EIGHT], %%xmm0\n\t" - -#define AESENC_XOR() \ - "movdqa (%[KEY]), %%xmm12\n\t" \ - "movdqu %%xmm0, " VAR(CTR1) "\n\t" \ - "pxor %%xmm12, %%xmm4\n\t" \ - "pxor %%xmm12, %%xmm5\n\t" \ - "pxor %%xmm12, %%xmm6\n\t" \ - "pxor %%xmm12, %%xmm7\n\t" \ - "pxor %%xmm12, %%xmm8\n\t" \ - "pxor %%xmm12, %%xmm9\n\t" \ - "pxor %%xmm12, %%xmm10\n\t" \ - "pxor %%xmm12, %%xmm11\n\t" - -/* Encrypt and carry-less multiply for AVX1. */ -#define AESENC_PCLMUL_1(src, o1, o2, o3) \ - "movdqu " #o3 "(" VAR(HTR) "), %%xmm12\n\t" \ - "movdqu " #o2 "(" #src "), %%xmm0\n\t" \ - "aesenc " #o1 "(%[KEY]), %%xmm4\n\t" \ - "pshufb %[BSWAP_MASK], %%xmm0\n\t" \ - "pxor %%xmm2, %%xmm0\n\t" \ - "pshufd $0x4e, %%xmm12, %%xmm1\n\t" \ - "pshufd $0x4e, %%xmm0, %%xmm14\n\t" \ - "pxor %%xmm12, %%xmm1\n\t" \ - "pxor %%xmm0, %%xmm14\n\t" \ - "movdqa %%xmm0, %%xmm3\n\t" \ - "pclmulqdq $0x11, %%xmm12, %%xmm3\n\t" \ - "aesenc " #o1 "(%[KEY]), %%xmm5\n\t" \ - "aesenc " #o1 "(%[KEY]), %%xmm6\n\t" \ - "movdqa %%xmm0, %%xmm2\n\t" \ - "pclmulqdq $0x00, %%xmm12, %%xmm2\n\t" \ - "aesenc " #o1 "(%[KEY]), %%xmm7\n\t" \ - "aesenc " #o1 "(%[KEY]), %%xmm8\n\t" \ - "pclmulqdq $0x00, %%xmm14, %%xmm1\n\t" \ - "aesenc " #o1 "(%[KEY]), %%xmm9\n\t" \ - "aesenc " #o1 "(%[KEY]), %%xmm10\n\t" \ - "aesenc " #o1 "(%[KEY]), %%xmm11\n\t" \ - "pxor %%xmm2, %%xmm1\n\t" \ - "pxor %%xmm3, %%xmm1\n\t" \ - -#define AESENC_PCLMUL_N(src, o1, o2, o3) \ - "movdqu " #o3 "(" VAR(HTR) "), %%xmm12\n\t" \ - "movdqu " #o2 "(" #src" ), %%xmm0\n\t" \ - "pshufd $0x4e, %%xmm12, %%xmm13\n\t" \ - "pshufb %[BSWAP_MASK], %%xmm0\n\t" \ - "aesenc " #o1 "(%[KEY]), %%xmm4\n\t" \ - "pxor %%xmm12, %%xmm13\n\t" \ - "pshufd $0x4e, %%xmm0, %%xmm14\n\t" \ - "pxor %%xmm0, %%xmm14\n\t" \ - "movdqa %%xmm0, %%xmm15\n\t" \ - "pclmulqdq $0x11, %%xmm12, %%xmm15\n\t" \ - "aesenc " #o1 "(%[KEY]), %%xmm5\n\t" \ - "aesenc " #o1 "(%[KEY]), %%xmm6\n\t" \ - "pclmulqdq $0x00, %%xmm0, %%xmm12\n\t" \ - "aesenc " #o1 "(%[KEY]), %%xmm7\n\t" \ - "aesenc " #o1 "(%[KEY]), %%xmm8\n\t" \ - "pclmulqdq $0x00, %%xmm14, %%xmm13\n\t" \ - "aesenc " #o1 "(%[KEY]), %%xmm9\n\t" \ - "aesenc " #o1 "(%[KEY]), %%xmm10\n\t" \ - "aesenc " #o1 "(%[KEY]), %%xmm11\n\t" \ - "pxor %%xmm12, %%xmm1\n\t" \ - "pxor %%xmm12, %%xmm2\n\t" \ - "pxor %%xmm15, %%xmm1\n\t" \ - "pxor %%xmm15, %%xmm3\n\t" \ - "pxor %%xmm13, %%xmm1\n\t" \ - -#define AESENC_PCLMUL_L(o) \ - "movdqa %%xmm1, %%xmm14\n\t" \ - "psrldq $8, %%xmm1\n\t" \ - "pslldq $8, %%xmm14\n\t" \ - "aesenc " #o "(%[KEY]), %%xmm4\n\t" \ - "pxor %%xmm14, %%xmm2\n\t" \ - "pxor %%xmm1, %%xmm3\n\t" \ - "movdqa %%xmm2, %%xmm12\n\t" \ - "movdqa %%xmm2, %%xmm13\n\t" \ - "movdqa %%xmm2, %%xmm14\n\t" \ - "aesenc " #o "(%[KEY]), %%xmm5\n\t" \ - "pslld $31, %%xmm12\n\t" \ - "pslld $30, %%xmm13\n\t" \ - "pslld $25, %%xmm14\n\t" \ - "aesenc " #o "(%[KEY]), %%xmm6\n\t" \ - "pxor %%xmm13, %%xmm12\n\t" \ - "pxor %%xmm14, %%xmm12\n\t" \ - "aesenc " #o "(%[KEY]), %%xmm7\n\t" \ - "movdqa %%xmm12, %%xmm13\n\t" \ - "pslldq $12, %%xmm12\n\t" \ - "psrldq $4, %%xmm13\n\t" \ - "aesenc " #o "(%[KEY]), %%xmm8\n\t" \ - "pxor %%xmm12, %%xmm2\n\t" \ - "movdqa %%xmm2, %%xmm14\n\t" \ - "movdqa %%xmm2, %%xmm1\n\t" \ - "movdqa %%xmm2, %%xmm0\n\t" \ - "aesenc " #o "(%[KEY]), %%xmm9\n\t" \ - "psrld $1, %%xmm14\n\t" \ - "psrld $2, %%xmm1\n\t" \ - "psrld $7, %%xmm0\n\t" \ - "aesenc " #o "(%[KEY]), %%xmm10\n\t" \ - "pxor %%xmm1, %%xmm14\n\t" \ - "pxor %%xmm0, %%xmm14\n\t" \ - "aesenc " #o "(%[KEY]), %%xmm11\n\t" \ - "pxor %%xmm13, %%xmm14\n\t" \ - "pxor %%xmm14, %%xmm2\n\t" \ - "pxor %%xmm3, %%xmm2\n\t" \ - -/* Encrypt and carry-less multiply with last key. */ -#define AESENC_LAST(in, out) \ - "aesenclast %%xmm12, %%xmm4\n\t" \ - "aesenclast %%xmm12, %%xmm5\n\t" \ - "movdqu (" #in "),%%xmm0\n\t" \ - "movdqu 16(" #in "),%%xmm1\n\t" \ - "pxor %%xmm0, %%xmm4\n\t" \ - "pxor %%xmm1, %%xmm5\n\t" \ - "movdqu %%xmm4, (" #out ")\n\t" \ - "movdqu %%xmm5, 16(" #out ")\n\t" \ - "aesenclast %%xmm12, %%xmm6\n\t" \ - "aesenclast %%xmm12, %%xmm7\n\t" \ - "movdqu 32(" #in "),%%xmm0\n\t" \ - "movdqu 48(" #in "),%%xmm1\n\t" \ - "pxor %%xmm0, %%xmm6\n\t" \ - "pxor %%xmm1, %%xmm7\n\t" \ - "movdqu %%xmm6, 32(" #out ")\n\t" \ - "movdqu %%xmm7, 48(" #out ")\n\t" \ - "aesenclast %%xmm12, %%xmm8\n\t" \ - "aesenclast %%xmm12, %%xmm9\n\t" \ - "movdqu 64(" #in "),%%xmm0\n\t" \ - "movdqu 80(" #in "),%%xmm1\n\t" \ - "pxor %%xmm0, %%xmm8\n\t" \ - "pxor %%xmm1, %%xmm9\n\t" \ - "movdqu %%xmm8, 64(" #out ")\n\t" \ - "movdqu %%xmm9, 80(" #out ")\n\t" \ - "aesenclast %%xmm12, %%xmm10\n\t" \ - "aesenclast %%xmm12, %%xmm11\n\t" \ - "movdqu 96(" #in "),%%xmm0\n\t" \ - "movdqu 112(" #in "),%%xmm1\n\t" \ - "pxor %%xmm0, %%xmm10\n\t" \ - "pxor %%xmm1, %%xmm11\n\t" \ - "movdqu %%xmm10, 96(" #out ")\n\t" \ - "movdqu %%xmm11, 112(" #out ")\n\t" - -#define _AESENC_AVX(r) \ - "aesenc 16(%[KEY]), " #r "\n\t" \ - "aesenc 32(%[KEY]), " #r "\n\t" \ - "aesenc 48(%[KEY]), " #r "\n\t" \ - "aesenc 64(%[KEY]), " #r "\n\t" \ - "aesenc 80(%[KEY]), " #r "\n\t" \ - "aesenc 96(%[KEY]), " #r "\n\t" \ - "aesenc 112(%[KEY]), " #r "\n\t" \ - "aesenc 128(%[KEY]), " #r "\n\t" \ - "aesenc 144(%[KEY]), " #r "\n\t" \ - "cmpl $11, %[nr]\n\t" \ - "movdqa 160(%[KEY]), %%xmm5\n\t" \ - "jl %=f\n\t" \ - "aesenc %%xmm5, " #r "\n\t" \ - "aesenc 176(%[KEY]), " #r "\n\t" \ - "cmpl $13, %[nr]\n\t" \ - "movdqa 192(%[KEY]), %%xmm5\n\t" \ - "jl %=f\n\t" \ - "aesenc %%xmm5, " #r "\n\t" \ - "aesenc 208(%[KEY]), " #r "\n\t" \ - "movdqa 224(%[KEY]), %%xmm5\n\t" \ - "%=:\n\t" \ - "aesenclast %%xmm5, " #r "\n\t" -#define AESENC_AVX(r) \ - _AESENC_AVX(r) - -#define AESENC_BLOCK(in, out) \ - "movdqu " VAR(CTR1) ", %%xmm4\n\t" \ - "movdqu %%xmm4, %%xmm5\n\t" \ - "pshufb %[BSWAP_EPI64], %%xmm4\n\t" \ - "paddd %[ONE], %%xmm5\n\t" \ - "pxor (%[KEY]), %%xmm4\n\t" \ - "movdqu %%xmm5, " VAR(CTR1) "\n\t" \ - AESENC_AVX(%%xmm4) \ - "movdqu (" #in "), %%xmm5\n\t" \ - "pxor %%xmm5, %%xmm4\n\t" \ - "movdqu %%xmm4, (" #out ")\n\t" \ - "pshufb %[BSWAP_MASK], %%xmm4\n\t" \ - "pxor %%xmm4, " VAR(XR) "\n\t" - -#define _AESENC_GFMUL(in, out, H, X) \ - "movdqu " VAR(CTR1) ", %%xmm4\n\t" \ - "movdqu %%xmm4, %%xmm5\n\t" \ - "pshufb %[BSWAP_EPI64], %%xmm4\n\t" \ - "paddd %[ONE], %%xmm5\n\t" \ - "pxor (%[KEY]), %%xmm4\n\t" \ - "movdqu %%xmm5, " VAR(CTR1) "\n\t" \ - "movdqa " #X ", %%xmm6\n\t" \ - "pclmulqdq $0x10, " #H ", %%xmm6\n\t" \ - "aesenc 16(%[KEY]), %%xmm4\n\t" \ - "aesenc 32(%[KEY]), %%xmm4\n\t" \ - "movdqa " #X ", %%xmm7\n\t" \ - "pclmulqdq $0x01, " #H ", %%xmm7\n\t" \ - "aesenc 48(%[KEY]), %%xmm4\n\t" \ - "aesenc 64(%[KEY]), %%xmm4\n\t" \ - "movdqa " #X ", %%xmm8\n\t" \ - "pclmulqdq $0x00, " #H ", %%xmm8\n\t" \ - "aesenc 80(%[KEY]), %%xmm4\n\t" \ - "movdqa " #X ", %%xmm1\n\t" \ - "pclmulqdq $0x11, " #H ", %%xmm1\n\t" \ - "aesenc 96(%[KEY]), %%xmm4\n\t" \ - "pxor %%xmm7, %%xmm6\n\t" \ - "movdqa %%xmm6, %%xmm2\n\t" \ - "psrldq $8, %%xmm6\n\t" \ - "pslldq $8, %%xmm2\n\t" \ - "aesenc 112(%[KEY]), %%xmm4\n\t" \ - "movdqa %%xmm1, %%xmm3\n\t" \ - "pxor %%xmm8, %%xmm2\n\t" \ - "pxor %%xmm6, %%xmm3\n\t" \ - "movdqa %[MOD2_128], %%xmm0\n\t" \ - "movdqa %%xmm2, %%xmm7\n\t" \ - "pclmulqdq $0x10, %%xmm0, %%xmm7\n\t" \ - "aesenc 128(%[KEY]), %%xmm4\n\t" \ - "pshufd $0x4e, %%xmm2, %%xmm6\n\t" \ - "pxor %%xmm7, %%xmm6\n\t" \ - "movdqa %%xmm6, %%xmm7\n\t" \ - "pclmulqdq $0x10, %%xmm0, %%xmm7\n\t" \ - "aesenc 144(%[KEY]), %%xmm4\n\t" \ - "pshufd $0x4e, %%xmm6, " VAR(XR) "\n\t" \ - "pxor %%xmm7, " VAR(XR) "\n\t" \ - "pxor %%xmm3, " VAR(XR) "\n\t" \ - "cmpl $11, %[nr]\n\t" \ - "movdqu 160(%[KEY]), %%xmm5\n\t" \ - "jl %=f\n\t" \ - "aesenc %%xmm5, %%xmm4\n\t" \ - "aesenc 176(%[KEY]), %%xmm4\n\t" \ - "cmpl $13, %[nr]\n\t" \ - "movdqu 192(%[KEY]), %%xmm5\n\t" \ - "jl %=f\n\t" \ - "aesenc %%xmm5, %%xmm4\n\t" \ - "aesenc 208(%[KEY]), %%xmm4\n\t" \ - "movdqa 224(%[KEY]), %%xmm5\n\t" \ - "%=:\n\t" \ - "aesenclast %%xmm5, %%xmm4\n\t" \ - "movdqu (" #in "), %%xmm5\n\t" \ - "pxor %%xmm5, %%xmm4\n\t" \ - "movdqu %%xmm4, (" #out ")\n\t" -#define AESENC_GFMUL(in, out, H, X) \ - _AESENC_GFMUL(in, out, H, X) - -#define _GHASH_GFMUL_AVX(r, r2, a, b) \ - "pshufd $0x4e, "#a", %%xmm1\n\t" \ - "pshufd $0x4e, "#b", %%xmm2\n\t" \ - "movdqa "#b", %%xmm3\n\t" \ - "movdqa "#b", %%xmm0\n\t" \ - "pclmulqdq $0x11, "#a", %%xmm3\n\t" \ - "pclmulqdq $0x00, "#a", %%xmm0\n\t" \ - "pxor "#a", %%xmm1\n\t" \ - "pxor "#b", %%xmm2\n\t" \ - "pclmulqdq $0x00, %%xmm2, %%xmm1\n\t" \ - "pxor %%xmm0, %%xmm1\n\t" \ - "pxor %%xmm3, %%xmm1\n\t" \ - "movdqa %%xmm1, %%xmm2\n\t" \ - "movdqa %%xmm0, "#r2"\n\t" \ - "movdqa %%xmm3, " #r "\n\t" \ - "pslldq $8, %%xmm2\n\t" \ - "psrldq $8, %%xmm1\n\t" \ - "pxor %%xmm2, "#r2"\n\t" \ - "pxor %%xmm1, " #r "\n\t" -#define GHASH_GFMUL_AVX(r, r2, a, b) \ - _GHASH_GFMUL_AVX(r, r2, a, b) - -#define _GHASH_GFMUL_XOR_AVX(r, r2, a, b) \ - "pshufd $0x4e, "#a", %%xmm1\n\t" \ - "pshufd $0x4e, "#b", %%xmm2\n\t" \ - "movdqa "#b", %%xmm3\n\t" \ - "movdqa "#b", %%xmm0\n\t" \ - "pclmulqdq $0x11, "#a", %%xmm3\n\t" \ - "pclmulqdq $0x00, "#a", %%xmm0\n\t" \ - "pxor "#a", %%xmm1\n\t" \ - "pxor "#b", %%xmm2\n\t" \ - "pclmulqdq $0x00, %%xmm2, %%xmm1\n\t" \ - "pxor %%xmm0, %%xmm1\n\t" \ - "pxor %%xmm3, %%xmm1\n\t" \ - "movdqa %%xmm1, %%xmm2\n\t" \ - "pxor %%xmm0, "#r2"\n\t" \ - "pxor %%xmm3, " #r "\n\t" \ - "pslldq $8, %%xmm2\n\t" \ - "psrldq $8, %%xmm1\n\t" \ - "pxor %%xmm2, "#r2"\n\t" \ - "pxor %%xmm1, " #r "\n\t" -#define GHASH_GFMUL_XOR_AVX(r, r2, a, b) \ - _GHASH_GFMUL_XOR_AVX(r, r2, a, b) - -#define GHASH_MID_AVX(r, r2) \ - "movdqa "#r2", %%xmm0\n\t" \ - "movdqa " #r ", %%xmm1\n\t" \ - "psrld $31, %%xmm0\n\t" \ - "psrld $31, %%xmm1\n\t" \ - "pslld $1, "#r2"\n\t" \ - "pslld $1, " #r "\n\t" \ - "movdqa %%xmm0, %%xmm2\n\t" \ - "pslldq $4, %%xmm0\n\t" \ - "psrldq $12, %%xmm2\n\t" \ - "pslldq $4, %%xmm1\n\t" \ - "por %%xmm2, " #r "\n\t" \ - "por %%xmm0, "#r2"\n\t" \ - "por %%xmm1, " #r "\n\t" - -#define _GHASH_GFMUL_RED_AVX(r, a, b) \ - "pshufd $0x4e, "#a", %%xmm5\n\t" \ - "pshufd $0x4e, "#b", %%xmm6\n\t" \ - "movdqa "#b", %%xmm7\n\t" \ - "movdqa "#b", %%xmm4\n\t" \ - "pclmulqdq $0x11, "#a", %%xmm7\n\t" \ - "pclmulqdq $0x00, "#a", %%xmm4\n\t" \ - "pxor "#a", %%xmm5\n\t" \ - "pxor "#b", %%xmm6\n\t" \ - "pclmulqdq $0x00, %%xmm6, %%xmm5\n\t" \ - "pxor %%xmm4, %%xmm5\n\t" \ - "pxor %%xmm7, %%xmm5\n\t" \ - "movdqa %%xmm5, %%xmm6\n\t" \ - "movdqa %%xmm7, " #r "\n\t" \ - "pslldq $8, %%xmm6\n\t" \ - "psrldq $8, %%xmm5\n\t" \ - "pxor %%xmm6, %%xmm4\n\t" \ - "pxor %%xmm5, " #r "\n\t" \ - "movdqa %%xmm4, %%xmm8\n\t" \ - "movdqa %%xmm4, %%xmm9\n\t" \ - "movdqa %%xmm4, %%xmm10\n\t" \ - "pslld $31, %%xmm8\n\t" \ - "pslld $30, %%xmm9\n\t" \ - "pslld $25, %%xmm10\n\t" \ - "pxor %%xmm9, %%xmm8\n\t" \ - "pxor %%xmm10, %%xmm8\n\t" \ - "movdqa %%xmm8, %%xmm9\n\t" \ - "psrldq $4, %%xmm9\n\t" \ - "pslldq $12, %%xmm8\n\t" \ - "pxor %%xmm8, %%xmm4\n\t" \ - "movdqa %%xmm4, %%xmm10\n\t" \ - "movdqa %%xmm4, %%xmm6\n\t" \ - "movdqa %%xmm4, %%xmm5\n\t" \ - "psrld $1, %%xmm10\n\t" \ - "psrld $2, %%xmm6\n\t" \ - "psrld $7, %%xmm5\n\t" \ - "pxor %%xmm6, %%xmm10\n\t" \ - "pxor %%xmm5, %%xmm10\n\t" \ - "pxor %%xmm9, %%xmm10\n\t" \ - "pxor %%xmm4, %%xmm10\n\t" \ - "pxor %%xmm10, " #r "\n\t" -#define GHASH_GFMUL_RED_AVX(r, a, b) \ - _GHASH_GFMUL_RED_AVX(r, a, b) - -#define GHASH_RED_AVX(r, r2) \ - "movdqa "#r2", %%xmm0\n\t" \ - "movdqa "#r2", %%xmm1\n\t" \ - "movdqa "#r2", %%xmm2\n\t" \ - "pslld $31, %%xmm0\n\t" \ - "pslld $30, %%xmm1\n\t" \ - "pslld $25, %%xmm2\n\t" \ - "pxor %%xmm1, %%xmm0\n\t" \ - "pxor %%xmm2, %%xmm0\n\t" \ - "movdqa %%xmm0, %%xmm1\n\t" \ - "psrldq $4, %%xmm1\n\t" \ - "pslldq $12, %%xmm0\n\t" \ - "pxor %%xmm0, "#r2"\n\t" \ - "movdqa "#r2", %%xmm2\n\t" \ - "movdqa "#r2", %%xmm3\n\t" \ - "movdqa "#r2", %%xmm0\n\t" \ - "psrld $1, %%xmm2\n\t" \ - "psrld $2, %%xmm3\n\t" \ - "psrld $7, %%xmm0\n\t" \ - "pxor %%xmm3, %%xmm2\n\t" \ - "pxor %%xmm0, %%xmm2\n\t" \ - "pxor %%xmm1, %%xmm2\n\t" \ - "pxor "#r2", %%xmm2\n\t" \ - "pxor %%xmm2, " #r "\n\t" - -#define GHASH_GFMUL_RED_XOR_AVX(r, r2, a, b) \ - GHASH_GFMUL_XOR_AVX(r, r2, a, b) \ - GHASH_RED_AVX(r, r2) - -#define GHASH_FULL_AVX(r, r2, a, b) \ - GHASH_GFMUL_AVX(r, r2, a, b) \ - GHASH_MID_AVX(r, r2) \ - GHASH_RED_AVX(r, r2) - -#define CALC_IV_12() \ - "# Calculate values when IV is 12 bytes\n\t" \ - "# Set counter based on IV\n\t" \ - "movl $0x01000000, %%ecx\n\t" \ - "pinsrq $0, 0(%%rax), %%xmm13\n\t" \ - "pinsrd $2, 8(%%rax), %%xmm13\n\t" \ - "pinsrd $3, %%ecx, %%xmm13\n\t" \ - "# H = Encrypt X(=0) and T = Encrypt counter\n\t" \ - "movdqu %%xmm13, %%xmm1\n\t" \ - "movdqa 0(%[KEY]), " VAR(HR) "\n\t" \ - "pxor " VAR(HR) ", %%xmm1\n\t" \ - "movdqa 16(%[KEY]), %%xmm12\n\t" \ - "aesenc %%xmm12, " VAR(HR) "\n\t" \ - "aesenc %%xmm12, %%xmm1\n\t" \ - "movdqa 32(%[KEY]), %%xmm12\n\t" \ - "aesenc %%xmm12, " VAR(HR) "\n\t" \ - "aesenc %%xmm12, %%xmm1\n\t" \ - "movdqa 48(%[KEY]), %%xmm12\n\t" \ - "aesenc %%xmm12, " VAR(HR) "\n\t" \ - "aesenc %%xmm12, %%xmm1\n\t" \ - "movdqa 64(%[KEY]), %%xmm12\n\t" \ - "aesenc %%xmm12, " VAR(HR) "\n\t" \ - "aesenc %%xmm12, %%xmm1\n\t" \ - "movdqa 80(%[KEY]), %%xmm12\n\t" \ - "aesenc %%xmm12, " VAR(HR) "\n\t" \ - "aesenc %%xmm12, %%xmm1\n\t" \ - "movdqa 96(%[KEY]), %%xmm12\n\t" \ - "aesenc %%xmm12, " VAR(HR) "\n\t" \ - "aesenc %%xmm12, %%xmm1\n\t" \ - "movdqa 112(%[KEY]), %%xmm12\n\t" \ - "aesenc %%xmm12, " VAR(HR) "\n\t" \ - "aesenc %%xmm12, %%xmm1\n\t" \ - "movdqa 128(%[KEY]), %%xmm12\n\t" \ - "aesenc %%xmm12, " VAR(HR) "\n\t" \ - "aesenc %%xmm12, %%xmm1\n\t" \ - "movdqa 144(%[KEY]), %%xmm12\n\t" \ - "aesenc %%xmm12, " VAR(HR) "\n\t" \ - "aesenc %%xmm12, %%xmm1\n\t" \ - "cmpl $11, %[nr]\n\t" \ - "movdqa 160(%[KEY]), %%xmm12\n\t" \ - "jl 31f\n\t" \ - "aesenc %%xmm12, " VAR(HR) "\n\t" \ - "aesenc %%xmm12, %%xmm1\n\t" \ - "movdqa 176(%[KEY]), %%xmm12\n\t" \ - "aesenc %%xmm12, " VAR(HR) "\n\t" \ - "aesenc %%xmm12, %%xmm1\n\t" \ - "cmpl $13, %[nr]\n\t" \ - "movdqa 192(%[KEY]), %%xmm12\n\t" \ - "jl 31f\n\t" \ - "aesenc %%xmm12, " VAR(HR) "\n\t" \ - "aesenc %%xmm12, %%xmm1\n\t" \ - "movdqu 208(%[KEY]), %%xmm12\n\t" \ - "aesenc %%xmm12, " VAR(HR) "\n\t" \ - "aesenc %%xmm12, %%xmm1\n\t" \ - "movdqu 224(%[KEY]), %%xmm12\n\t" \ - "31:\n\t" \ - "aesenclast %%xmm12, " VAR(HR) "\n\t" \ - "aesenclast %%xmm12, %%xmm1\n\t" \ - "pshufb %[BSWAP_MASK], " VAR(HR) "\n\t" \ - "movdqu %%xmm1, " VAR(TR) "\n\t" \ - "jmp 39f\n\t" - -#define CALC_IV() \ - "# Calculate values when IV is not 12 bytes\n\t" \ - "# H = Encrypt X(=0)\n\t" \ - "movdqa 0(%[KEY]), " VAR(HR) "\n\t" \ - AESENC_AVX(HR) \ - "pshufb %[BSWAP_MASK], " VAR(HR) "\n\t" \ - "# Calc counter\n\t" \ - "# Initialization vector\n\t" \ - "cmpl $0, %%edx\n\t" \ - "movq $0, %%rcx\n\t" \ - "je 45f\n\t" \ - "cmpl $16, %%edx\n\t" \ - "jl 44f\n\t" \ - "andl $0xfffffff0, %%edx\n\t" \ - "\n" \ - "43:\n\t" \ - "movdqu (%%rax,%%rcx,1), %%xmm4\n\t" \ - "pshufb %[BSWAP_MASK], %%xmm4\n\t" \ - "pxor %%xmm4, %%xmm13\n\t" \ - GHASH_FULL_AVX(%%xmm13, %%xmm12, %%xmm13, HR) \ - "addl $16, %%ecx\n\t" \ - "cmpl %%edx, %%ecx\n\t" \ - "jl 43b\n\t" \ - "movl %[ibytes], %%edx\n\t" \ - "cmpl %%edx, %%ecx\n\t" \ - "je 45f\n\t" \ - "\n" \ - "44:\n\t" \ - "subq $16, %%rsp\n\t" \ - "pxor %%xmm4, %%xmm4\n\t" \ - "xorl %%ebx, %%ebx\n\t" \ - "movdqu %%xmm4, (%%rsp)\n\t" \ - "42:\n\t" \ - "movzbl (%%rax,%%rcx,1), %%r13d\n\t" \ - "movb %%r13b, (%%rsp,%%rbx,1)\n\t" \ - "incl %%ecx\n\t" \ - "incl %%ebx\n\t" \ - "cmpl %%edx, %%ecx\n\t" \ - "jl 42b\n\t" \ - "movdqu (%%rsp), %%xmm4\n\t" \ - "addq $16, %%rsp\n\t" \ - "pshufb %[BSWAP_MASK], %%xmm4\n\t" \ - "pxor %%xmm4, %%xmm13\n\t" \ - GHASH_FULL_AVX(%%xmm13, %%xmm12, %%xmm13, HR) \ - "\n" \ - "45:\n\t" \ - "# T = Encrypt counter\n\t" \ - "pxor %%xmm0, %%xmm0\n\t" \ - "shll $3, %%edx\n\t" \ - "pinsrq $0, %%rdx, %%xmm0\n\t" \ - "pxor %%xmm0, %%xmm13\n\t" \ - GHASH_FULL_AVX(%%xmm13, %%xmm12, %%xmm13, HR) \ - "pshufb %[BSWAP_MASK], %%xmm13\n\t" \ - "# Encrypt counter\n\t" \ - "movdqa 0(%[KEY]), %%xmm4\n\t" \ - "pxor %%xmm13, %%xmm4\n\t" \ - AESENC_AVX(%%xmm4) \ - "movdqu %%xmm4, " VAR(TR) "\n\t" - -#define CALC_AAD() \ - "# Additional authentication data\n\t" \ - "movl %[abytes], %%edx\n\t" \ - "cmpl $0, %%edx\n\t" \ - "je 25f\n\t" \ - "movq %[addt], %%rax\n\t" \ - "xorl %%ecx, %%ecx\n\t" \ - "cmpl $16, %%edx\n\t" \ - "jl 24f\n\t" \ - "andl $0xfffffff0, %%edx\n\t" \ - "\n" \ - "23:\n\t" \ - "movdqu (%%rax,%%rcx,1), %%xmm4\n\t" \ - "pshufb %[BSWAP_MASK], %%xmm4\n\t" \ - "pxor %%xmm4, " VAR(XR) "\n\t" \ - GHASH_FULL_AVX(XR, %%xmm12, XR, HR) \ - "addl $16, %%ecx\n\t" \ - "cmpl %%edx, %%ecx\n\t" \ - "jl 23b\n\t" \ - "movl %[abytes], %%edx\n\t" \ - "cmpl %%edx, %%ecx\n\t" \ - "je 25f\n\t" \ - "\n" \ - "24:\n\t" \ - "subq $16, %%rsp\n\t" \ - "pxor %%xmm4, %%xmm4\n\t" \ - "xorl %%ebx, %%ebx\n\t" \ - "movdqu %%xmm4, (%%rsp)\n\t" \ - "22:\n\t" \ - "movzbl (%%rax,%%rcx,1), %%r13d\n\t" \ - "movb %%r13b, (%%rsp,%%rbx,1)\n\t" \ - "incl %%ecx\n\t" \ - "incl %%ebx\n\t" \ - "cmpl %%edx, %%ecx\n\t" \ - "jl 22b\n\t" \ - "movdqu (%%rsp), %%xmm4\n\t" \ - "addq $16, %%rsp\n\t" \ - "pshufb %[BSWAP_MASK], %%xmm4\n\t" \ - "pxor %%xmm4, " VAR(XR) "\n\t" \ - GHASH_FULL_AVX(XR, %%xmm12, XR, HR) \ - "\n" \ - "25:\n\t" - -#define CALC_HT_8_AVX() \ - "movdqa " VAR(XR) ", %%xmm2\n\t" \ - "# H ^ 1\n\t" \ - "movdqu " VAR(HR) ", 0(" VAR(HTR) ")\n\t" \ - "# H ^ 2\n\t" \ - GHASH_GFMUL_RED_AVX(%%xmm0, HR, HR) \ - "movdqu %%xmm0 , 16(" VAR(HTR) ")\n\t" \ - "# H ^ 3\n\t" \ - GHASH_GFMUL_RED_AVX(%%xmm1, HR, %%xmm0) \ - "movdqu %%xmm1 , 32(" VAR(HTR) ")\n\t" \ - "# H ^ 4\n\t" \ - GHASH_GFMUL_RED_AVX(%%xmm3, %%xmm0, %%xmm0) \ - "movdqu %%xmm3 , 48(" VAR(HTR) ")\n\t" \ - "# H ^ 5\n\t" \ - GHASH_GFMUL_RED_AVX(%%xmm12, %%xmm0, %%xmm1) \ - "movdqu %%xmm12, 64(" VAR(HTR) ")\n\t" \ - "# H ^ 6\n\t" \ - GHASH_GFMUL_RED_AVX(%%xmm12, %%xmm1, %%xmm1) \ - "movdqu %%xmm12, 80(" VAR(HTR) ")\n\t" \ - "# H ^ 7\n\t" \ - GHASH_GFMUL_RED_AVX(%%xmm12, %%xmm1, %%xmm3) \ - "movdqu %%xmm12, 96(" VAR(HTR) ")\n\t" \ - "# H ^ 8\n\t" \ - GHASH_GFMUL_RED_AVX(%%xmm12, %%xmm3, %%xmm3) \ - "movdqu %%xmm12, 112(" VAR(HTR) ")\n\t" - -#define AESENC_128_GHASH_AVX(src, o) \ - "leaq (%[in]," VAR(KR64) ",1), %%rcx\n\t" \ - "leaq (%[out]," VAR(KR64) ",1), %%rdx\n\t" \ - /* src is either %%rcx or %%rdx */ \ - AESENC_CTR() \ - AESENC_XOR() \ - AESENC_PCLMUL_1(src, 16, o-128, 112) \ - AESENC_PCLMUL_N(src, 32, o-112, 96) \ - AESENC_PCLMUL_N(src, 48, o -96, 80) \ - AESENC_PCLMUL_N(src, 64, o -80, 64) \ - AESENC_PCLMUL_N(src, 80, o -64, 48) \ - AESENC_PCLMUL_N(src, 96, o -48, 32) \ - AESENC_PCLMUL_N(src, 112, o -32, 16) \ - AESENC_PCLMUL_N(src, 128, o -16, 0) \ - AESENC_PCLMUL_L(144) \ - "cmpl $11, %[nr]\n\t" \ - "movdqa 160(%[KEY]), %%xmm12\n\t" \ - "jl 4f\n\t" \ - AESENC() \ - AESENC_SET(176) \ - "cmpl $13, %[nr]\n\t" \ - "movdqa 192(%[KEY]), %%xmm12\n\t" \ - "jl 4f\n\t" \ - AESENC() \ - AESENC_SET(208) \ - "movdqa 224(%[KEY]), %%xmm12\n\t" \ - "\n" \ -"4:\n\t" \ - AESENC_LAST(%%rcx, %%rdx) - -#define AESENC_LAST15_ENC_AVX() \ - "movl %[nbytes], %%ecx\n\t" \ - "movl %%ecx, %%edx\n\t" \ - "andl $0x0f, %%ecx\n\t" \ - "jz 55f\n\t" \ - "movdqu " VAR(CTR1) ", %%xmm13\n\t" \ - "pshufb %[BSWAP_EPI64], %%xmm13\n\t" \ - "pxor 0(%[KEY]), %%xmm13\n\t" \ - AESENC_AVX(%%xmm13) \ - "subq $16, %%rsp\n\t" \ - "xorl %%ecx, %%ecx\n\t" \ - "movdqu %%xmm13, (%%rsp)\n\t" \ - "\n" \ - "51:\n\t" \ - "movzbl (%[in]," VAR(KR64) ",1), %%r13d\n\t" \ - "xorb (%%rsp,%%rcx,1), %%r13b\n\t" \ - "movb %%r13b, (%[out]," VAR(KR64) ",1)\n\t" \ - "movb %%r13b, (%%rsp,%%rcx,1)\n\t" \ - "incl " VAR(KR) "\n\t" \ - "incl %%ecx\n\t" \ - "cmpl %%edx, " VAR(KR) "\n\t" \ - "jl 51b\n\t" \ - "xorq %%r13, %%r13\n\t" \ - "cmpl $16, %%ecx\n\t" \ - "je 53f\n\t" \ - "\n" \ - "52:\n\t" \ - "movb %%r13b, (%%rsp,%%rcx,1)\n\t" \ - "incl %%ecx\n\t" \ - "cmpl $16, %%ecx\n\t" \ - "jl 52b\n\t" \ - "53:\n\t" \ - "movdqu (%%rsp), %%xmm13\n\t" \ - "addq $16, %%rsp\n\t" \ - "pshufb %[BSWAP_MASK], %%xmm13\n\t" \ - "pxor %%xmm13, " VAR(XR) "\n\t" \ - GHASH_GFMUL_RED_AVX(XR, HR, XR) \ - -#define AESENC_LAST15_DEC_AVX() \ - "movl %[nbytes], %%ecx\n\t" \ - "movl %%ecx, %%edx\n\t" \ - "andl $0x0f, %%ecx\n\t" \ - "jz 55f\n\t" \ - "movdqu " VAR(CTR1) ", %%xmm13\n\t" \ - "pshufb %[BSWAP_EPI64], %%xmm13\n\t" \ - "pxor 0(%[KEY]), %%xmm13\n\t" \ - AESENC_AVX(%%xmm13) \ - "subq $32, %%rsp\n\t" \ - "xorl %%ecx, %%ecx\n\t" \ - "movdqu %%xmm13, (%%rsp)\n\t" \ - "pxor %%xmm0, %%xmm0\n\t" \ - "movdqu %%xmm0, 16(%%rsp)\n\t" \ - "\n" \ - "51:\n\t" \ - "movzbl (%[in]," VAR(KR64) ",1), %%r13d\n\t" \ - "movb %%r13b, 16(%%rsp,%%rcx,1)\n\t" \ - "xorb (%%rsp,%%rcx,1), %%r13b\n\t" \ - "movb %%r13b, (%[out]," VAR(KR64) ",1)\n\t" \ - "incl " VAR(KR) "\n\t" \ - "incl %%ecx\n\t" \ - "cmpl %%edx, " VAR(KR) "\n\t" \ - "jl 51b\n\t" \ - "53:\n\t" \ - "movdqu 16(%%rsp), %%xmm13\n\t" \ - "addq $32, %%rsp\n\t" \ - "pshufb %[BSWAP_MASK], %%xmm13\n\t" \ - "pxor %%xmm13, " VAR(XR) "\n\t" \ - GHASH_GFMUL_RED_AVX(XR, HR, XR) \ - -#define CALC_TAG() \ - "movl %[nbytes], %%edx\n\t" \ - "movl %[abytes], %%ecx\n\t" \ - "shlq $3, %%rdx\n\t" \ - "shlq $3, %%rcx\n\t" \ - "pinsrq $0, %%rdx, %%xmm0\n\t" \ - "pinsrq $1, %%rcx, %%xmm0\n\t" \ - "pxor %%xmm0, " VAR(XR) "\n\t" \ - GHASH_GFMUL_RED_AVX(XR, HR, XR) \ - "pshufb %[BSWAP_MASK], " VAR(XR) "\n\t" \ - "movdqu " VAR(TR) ", %%xmm0\n\t" \ - "pxor " VAR(XR) ", %%xmm0\n\t" \ - -#define STORE_TAG() \ - "cmpl $16, %[tbytes]\n\t" \ - "je 71f\n\t" \ - "xorq %%rcx, %%rcx\n\t" \ - "movdqu %%xmm0, (%%rsp)\n\t" \ - "73:\n\t" \ - "movzbl (%%rsp,%%rcx,1), %%r13d\n\t" \ - "movb %%r13b, (%[tag],%%rcx,1)\n\t" \ - "incl %%ecx\n\t" \ - "cmpl %[tbytes], %%ecx\n\t" \ - "jne 73b\n\t" \ - "jmp 72f\n\t" \ - "\n" \ - "71:\n\t" \ - "movdqu %%xmm0, (%[tag])\n\t" \ - "\n" \ - "72:\n\t" - -#define CMP_TAG() \ - "cmpl $16, %[tbytes]\n\t" \ - "je 71f\n\t" \ - "subq $16, %%rsp\n\t" \ - "xorq %%rcx, %%rcx\n\t" \ - "xorq %%rax, %%rax\n\t" \ - "movdqu %%xmm0, (%%rsp)\n\t" \ - "\n" \ - "73:\n\t" \ - "movzbl (%%rsp,%%rcx,1), %%r13d\n\t" \ - "xorb (%[tag],%%rcx,1), %%r13b\n\t" \ - "orb %%r13b, %%al\n\t" \ - "incl %%ecx\n\t" \ - "cmpl %[tbytes], %%ecx\n\t" \ - "jne 73b\n\t" \ - "cmpb $0x00, %%al\n\t" \ - "sete %%al\n\t" \ - "addq $16, %%rsp\n\t" \ - "xorq %%rcx, %%rcx\n\t" \ - "jmp 72f\n\t" \ - "\n" \ - "71:\n\t" \ - "movdqu (%[tag]), %%xmm1\n\t" \ - "pcmpeqb %%xmm1, %%xmm0\n\t" \ - "pmovmskb %%xmm0, %%edx\n\t" \ - "# %%edx == 0xFFFF then return 1 else => return 0\n\t" \ - "xorl %%eax, %%eax\n\t" \ - "cmpl $0xffff, %%edx\n\t" \ - "sete %%al\n\t" \ - "\n" \ - "72:\n\t" \ - "movl %%eax, (%[res])\n\t" - -static void AES_GCM_encrypt(const unsigned char *in, unsigned char *out, - const unsigned char* addt, - const unsigned char* ivec, unsigned char *tag, - unsigned int nbytes, unsigned int abytes, - unsigned int ibytes, unsigned int tbytes, - const unsigned char* key, int nr) -{ - register const unsigned char* iv asm("rax") = ivec; - register unsigned int ivLen asm("ebx") = ibytes; - - __asm__ __volatile__ ( - "subq $" VAR(STACK_OFFSET) ", %%rsp\n\t" - /* Counter is xmm13 */ - "pxor %%xmm13, %%xmm13\n\t" - "pxor " VAR(XR) ", " VAR(XR) "\n\t" - "movl %[ibytes], %%edx\n\t" - "cmpl $12, %%edx\n\t" - "jne 35f\n\t" - CALC_IV_12() - "\n" - "35:\n\t" - CALC_IV() - "\n" - "39:\n\t" - - CALC_AAD() - - "# Calculate counter and H\n\t" - "pshufb %[BSWAP_EPI64], %%xmm13\n\t" - "movdqa " VAR(HR) ", %%xmm5\n\t" - "paddd %[ONE], %%xmm13\n\t" - "movdqa " VAR(HR) ", %%xmm4\n\t" - "movdqu %%xmm13, " VAR(CTR1) "\n\t" - "psrlq $63, %%xmm5\n\t" - "psllq $1, %%xmm4\n\t" - "pslldq $8, %%xmm5\n\t" - "por %%xmm5, %%xmm4\n\t" - "pshufd $0xff, " VAR(HR) ", " VAR(HR) "\n\t" - "psrad $31, " VAR(HR) "\n\t" - "pand %[MOD2_128], " VAR(HR) "\n\t" - "pxor %%xmm4, " VAR(HR) "\n\t" - - "xorl " VAR(KR) ", " VAR(KR) "\n\t" - -#if !defined(AES_GCM_AESNI_NO_UNROLL) && !defined(AES_GCM_AVX1_NO_UNROLL) - "cmpl $128, %[nbytes]\n\t" - "movl %[nbytes], %%r13d\n\t" - "jl 5f\n\t" - "andl $0xffffff80, %%r13d\n\t" - - CALC_HT_8_AVX() - - "# First 128 bytes of input\n\t" - AESENC_CTR() - AESENC_XOR() - AESENC_SET(16) - AESENC_SET(32) - AESENC_SET(48) - AESENC_SET(64) - AESENC_SET(80) - AESENC_SET(96) - AESENC_SET(112) - AESENC_SET(128) - AESENC_SET(144) - "cmpl $11, %[nr]\n\t" - "movdqa 160(%[KEY]), %%xmm12\n\t" - "jl 1f\n\t" - AESENC() - AESENC_SET(176) - "cmpl $13, %[nr]\n\t" - "movdqa 192(%[KEY]), %%xmm12\n\t" - "jl 1f\n\t" - AESENC() - AESENC_SET(208) - "movdqa 224(%[KEY]), %%xmm12\n\t" - "\n" - "1:\n\t" - AESENC_LAST(%[in], %[out]) - - "cmpl $128, %%r13d\n\t" - "movl $128, " VAR(KR) "\n\t" - "jle 2f\n\t" - - "# More 128 bytes of input\n\t" - "\n" - "3:\n\t" - AESENC_128_GHASH_AVX(%%rdx, 0) - "addl $128, " VAR(KR) "\n\t" - "cmpl %%r13d, " VAR(KR) "\n\t" - "jl 3b\n\t" - "\n" - "2:\n\t" - "movdqa %[BSWAP_MASK], %%xmm13\n\t" - "pshufb %%xmm13, %%xmm4\n\t" - "pshufb %%xmm13, %%xmm5\n\t" - "pshufb %%xmm13, %%xmm6\n\t" - "pshufb %%xmm13, %%xmm7\n\t" - "pxor %%xmm2, %%xmm4\n\t" - "pshufb %%xmm13, %%xmm8\n\t" - "pshufb %%xmm13, %%xmm9\n\t" - "pshufb %%xmm13, %%xmm10\n\t" - "pshufb %%xmm13, %%xmm11\n\t" - - "movdqu 112(" VAR(HTR) "), %%xmm12\n\t" - GHASH_GFMUL_AVX(XR, %%xmm13, %%xmm4, %%xmm12) - "movdqu 96(" VAR(HTR) "), %%xmm12\n\t" - GHASH_GFMUL_XOR_AVX(XR, %%xmm13, %%xmm5, %%xmm12) - "movdqu 80(" VAR(HTR) "), %%xmm12\n\t" - GHASH_GFMUL_XOR_AVX(XR, %%xmm13, %%xmm6, %%xmm12) - "movdqu 64(" VAR(HTR) "), %%xmm12\n\t" - GHASH_GFMUL_XOR_AVX(XR, %%xmm13, %%xmm7, %%xmm12) - "movdqu 48(" VAR(HTR) "), %%xmm12\n\t" - GHASH_GFMUL_XOR_AVX(XR, %%xmm13, %%xmm8, %%xmm12) - "movdqu 32(" VAR(HTR) "), %%xmm12\n\t" - GHASH_GFMUL_XOR_AVX(XR, %%xmm13, %%xmm9, %%xmm12) - "movdqu 16(" VAR(HTR) "), %%xmm12\n\t" - GHASH_GFMUL_XOR_AVX(XR, %%xmm13, %%xmm10, %%xmm12) - "movdqu (" VAR(HTR) "), %%xmm12\n\t" - GHASH_GFMUL_RED_XOR_AVX(XR, %%xmm13, %%xmm11, %%xmm12) - - "movdqu 0(" VAR(HTR) "), " VAR(HR) "\n\t" - "\n" - "5:\n\t" - "movl %[nbytes], %%edx\n\t" - "cmpl %%edx, " VAR(KR) "\n\t" - "jge 55f\n\t" -#endif - - "movl %[nbytes], %%r13d\n\t" - "andl $0xfffffff0, %%r13d\n\t" - "cmpl %%r13d, " VAR(KR) "\n\t" - "jge 14f\n\t" - - "leaq (%[in]," VAR(KR64) ",1), %%rcx\n\t" - "leaq (%[out]," VAR(KR64) ",1), %%rdx\n\t" - AESENC_BLOCK(%%rcx, %%rdx) - "addl $16, " VAR(KR) "\n\t" - "cmpl %%r13d, " VAR(KR) "\n\t" - "jge 13f\n\t" - "\n" - "12:\n\t" - "leaq (%[in]," VAR(KR64) ",1), %%rcx\n\t" - "leaq (%[out]," VAR(KR64) ",1), %%rdx\n\t" - AESENC_GFMUL(%%rcx, %%rdx, HR, XR) - "pshufb %[BSWAP_MASK], %%xmm4\n\t" - "pxor %%xmm4, " VAR(XR) "\n\t" - "addl $16, " VAR(KR) "\n\t" - "cmpl %%r13d, " VAR(KR) "\n\t" - "jl 12b\n\t" - "\n" - "13:\n\t" - GHASH_GFMUL_RED_AVX(XR, HR, XR) - "\n" - "14:\n\t" - - AESENC_LAST15_ENC_AVX() - "\n" - "55:\n\t" - - CALC_TAG() - STORE_TAG() - "addq $" VAR(STACK_OFFSET) ", %%rsp\n\t" - - : - : [KEY] "r" (key), - [in] "r" (in), [out] "r" (out), [nr] "r" (nr), - [nbytes] "r" (nbytes), [abytes] "r" (abytes), [addt] "r" (addt), - [ivec] "r" (iv), [ibytes] "r" (ivLen), [tbytes] "r" (tbytes), - [tag] "r" (tag), - [BSWAP_MASK] "m" (BSWAP_MASK), - [BSWAP_EPI64] "m" (BSWAP_EPI64), - [ONE] "m" (ONE), -#if !defined(AES_GCM_AESNI_NO_UNROLL) && !defined(AES_GCM_AVX1_NO_UNROLL) - [TWO] "m" (TWO), [THREE] "m" (THREE), [FOUR] "m" (FOUR), - [FIVE] "m" (FIVE), [SIX] "m" (SIX), [SEVEN] "m" (SEVEN), - [EIGHT] "m" (EIGHT), -#endif - [MOD2_128] "m" (MOD2_128) - : "xmm15", "xmm14", "xmm13", "xmm12", - "xmm0", "xmm1", "xmm2", "xmm3", "memory", - "xmm4", "xmm5", "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", - "rcx", "rdx", "r13" - ); -} - +void AES_GCM_encrypt(const unsigned char *in, unsigned char *out, + const unsigned char* addt, const unsigned char* ivec, + unsigned char *tag, unsigned int nbytes, + unsigned int abytes, unsigned int ibytes, + unsigned int tbytes, const unsigned char* key, int nr) + XASM_LINK("AES_GCM_encrypt"); #ifdef HAVE_INTEL_AVX1 -/* Encrypt with key in xmm12. */ -#define VAESENC() \ - "vaesenc %%xmm12, %%xmm4, %%xmm4\n\t" \ - "vaesenc %%xmm12, %%xmm5, %%xmm5\n\t" \ - "vaesenc %%xmm12, %%xmm6, %%xmm6\n\t" \ - "vaesenc %%xmm12, %%xmm7, %%xmm7\n\t" \ - "vaesenc %%xmm12, %%xmm8, %%xmm8\n\t" \ - "vaesenc %%xmm12, %%xmm9, %%xmm9\n\t" \ - "vaesenc %%xmm12, %%xmm10, %%xmm10\n\t" \ - "vaesenc %%xmm12, %%xmm11, %%xmm11\n\t" - -#define VAESENC_SET(o) \ - "vmovdqa "#o"(%[KEY]), %%xmm12\n\t" \ - VAESENC() - -#define VAESENC_CTR() \ - "vmovdqu " VAR(CTR1) ", %%xmm0\n\t" \ - "vmovdqa %[BSWAP_EPI64], %%xmm1\n\t" \ - "vpshufb %%xmm1, %%xmm0, %%xmm4\n\t" \ - "vpaddd %[ONE], %%xmm0, %%xmm5\n\t" \ - "vpshufb %%xmm1, %%xmm5, %%xmm5\n\t" \ - "vpaddd %[TWO], %%xmm0, %%xmm6\n\t" \ - "vpshufb %%xmm1, %%xmm6, %%xmm6\n\t" \ - "vpaddd %[THREE], %%xmm0, %%xmm7\n\t" \ - "vpshufb %%xmm1, %%xmm7, %%xmm7\n\t" \ - "vpaddd %[FOUR], %%xmm0, %%xmm8\n\t" \ - "vpshufb %%xmm1, %%xmm8, %%xmm8\n\t" \ - "vpaddd %[FIVE], %%xmm0, %%xmm9\n\t" \ - "vpshufb %%xmm1, %%xmm9, %%xmm9\n\t" \ - "vpaddd %[SIX], %%xmm0, %%xmm10\n\t" \ - "vpshufb %%xmm1, %%xmm10, %%xmm10\n\t" \ - "vpaddd %[SEVEN], %%xmm0, %%xmm11\n\t" \ - "vpshufb %%xmm1, %%xmm11, %%xmm11\n\t" \ - "vpaddd %[EIGHT], %%xmm0, %%xmm0\n\t" - -#define VAESENC_XOR() \ - "vmovdqa (%[KEY]), %%xmm12\n\t" \ - "vmovdqu %%xmm0, " VAR(CTR1) "\n\t" \ - "vpxor %%xmm12, %%xmm4, %%xmm4\n\t" \ - "vpxor %%xmm12, %%xmm5, %%xmm5\n\t" \ - "vpxor %%xmm12, %%xmm6, %%xmm6\n\t" \ - "vpxor %%xmm12, %%xmm7, %%xmm7\n\t" \ - "vpxor %%xmm12, %%xmm8, %%xmm8\n\t" \ - "vpxor %%xmm12, %%xmm9, %%xmm9\n\t" \ - "vpxor %%xmm12, %%xmm10, %%xmm10\n\t" \ - "vpxor %%xmm12, %%xmm11, %%xmm11\n\t" - -#define VAESENC_128() \ - VAESENC_CTR() \ - VAESENC_XOR() \ - VAESENC_SET(16) \ - VAESENC_SET(32) \ - VAESENC_SET(48) \ - VAESENC_SET(64) \ - VAESENC_SET(80) \ - VAESENC_SET(96) \ - VAESENC_SET(112) \ - VAESENC_SET(128) \ - VAESENC_SET(144) \ - "cmpl $11, %[nr]\n\t" \ - "vmovdqa 160(%[KEY]), %%xmm12\n\t" \ - "jl 1f\n\t" \ - VAESENC() \ - VAESENC_SET(176) \ - "cmpl $13, %[nr]\n\t" \ - "vmovdqa 192(%[KEY]), %%xmm12\n\t" \ - "jl 1f\n\t" \ - VAESENC() \ - VAESENC_SET(208) \ - "vmovdqa 224(%[KEY]), %%xmm12\n\t" \ - "\n" \ -"1:\n\t" \ - VAESENC_LAST(%[in], %[out]) - -/* Encrypt and carry-less multiply for AVX1. */ -#define VAESENC_PCLMUL_1(src, o1, o2, o3) \ - "vmovdqu " #o3 "(" VAR(HTR) "), %%xmm12\n\t" \ - "vmovdqu " #o2 "(" #src "), %%xmm0\n\t" \ - "vaesenc " #o1 "(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vpshufb %[BSWAP_MASK], %%xmm0, %%xmm0\n\t" \ - "vpxor %%xmm2, %%xmm0, %%xmm0\n\t" \ - "vpshufd $0x4e, %%xmm12, %%xmm1\n\t" \ - "vpshufd $0x4e, %%xmm0, %%xmm14\n\t" \ - "vpxor %%xmm12, %%xmm1, %%xmm1\n\t" \ - "vpxor %%xmm0, %%xmm14, %%xmm14\n\t" \ - "vpclmulqdq $0x11, %%xmm12, %%xmm0, %%xmm3\n\t" \ - "vaesenc " #o1 "(%[KEY]), %%xmm5, %%xmm5\n\t" \ - "vaesenc " #o1 "(%[KEY]), %%xmm6, %%xmm6\n\t" \ - "vpclmulqdq $0x00, %%xmm12, %%xmm0, %%xmm2\n\t" \ - "vaesenc " #o1 "(%[KEY]), %%xmm7, %%xmm7\n\t" \ - "vaesenc " #o1 "(%[KEY]), %%xmm8, %%xmm8\n\t" \ - "vpclmulqdq $0x00, %%xmm14, %%xmm1, %%xmm1\n\t" \ - "vaesenc " #o1 "(%[KEY]), %%xmm9, %%xmm9\n\t" \ - "vaesenc " #o1 "(%[KEY]), %%xmm10, %%xmm10\n\t" \ - "vaesenc " #o1 "(%[KEY]), %%xmm11, %%xmm11\n\t" \ - "vpxor %%xmm2, %%xmm1, %%xmm1\n\t" \ - "vpxor %%xmm3, %%xmm1, %%xmm1\n\t" \ - -#define VAESENC_PCLMUL_N(src, o1, o2, o3) \ - "vmovdqu " #o3 "(" VAR(HTR) "), %%xmm12\n\t" \ - "vmovdqu " #o2 "(" #src "), %%xmm0\n\t" \ - "vpshufd $0x4e, %%xmm12, %%xmm13\n\t" \ - "vpshufb %[BSWAP_MASK], %%xmm0, %%xmm0\n\t" \ - "vaesenc " #o1 "(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vpxor %%xmm12, %%xmm13, %%xmm13\n\t" \ - "vpshufd $0x4e, %%xmm0, %%xmm14\n\t" \ - "vpxor %%xmm0, %%xmm14, %%xmm14\n\t" \ - "vpclmulqdq $0x11, %%xmm12, %%xmm0, %%xmm15\n\t" \ - "vaesenc " #o1 "(%[KEY]), %%xmm5, %%xmm5\n\t" \ - "vaesenc " #o1 "(%[KEY]), %%xmm6, %%xmm6\n\t" \ - "vpclmulqdq $0x00, %%xmm12, %%xmm0, %%xmm12\n\t" \ - "vaesenc " #o1 "(%[KEY]), %%xmm7, %%xmm7\n\t" \ - "vaesenc " #o1 "(%[KEY]), %%xmm8, %%xmm8\n\t" \ - "vpclmulqdq $0x00, %%xmm14, %%xmm13, %%xmm13\n\t" \ - "vaesenc " #o1 "(%[KEY]), %%xmm9, %%xmm9\n\t" \ - "vaesenc " #o1 "(%[KEY]), %%xmm10, %%xmm10\n\t" \ - "vaesenc " #o1 "(%[KEY]), %%xmm11, %%xmm11\n\t" \ - "vpxor %%xmm12, %%xmm1, %%xmm1\n\t" \ - "vpxor %%xmm12, %%xmm2, %%xmm2\n\t" \ - "vpxor %%xmm15, %%xmm1, %%xmm1\n\t" \ - "vpxor %%xmm15, %%xmm3, %%xmm3\n\t" \ - "vpxor %%xmm13, %%xmm1, %%xmm1\n\t" \ - -#define VAESENC_PCLMUL_L(o) \ - "vpslldq $8, %%xmm1, %%xmm14\n\t" \ - "vpsrldq $8, %%xmm1, %%xmm1\n\t" \ - "vaesenc "#o"(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vpxor %%xmm14, %%xmm2, %%xmm2\n\t" \ - "vpxor %%xmm1, %%xmm3, %%xmm3\n\t" \ - "vaesenc "#o"(%[KEY]), %%xmm5, %%xmm5\n\t" \ - "vpslld $31, %%xmm2, %%xmm12\n\t" \ - "vpslld $30, %%xmm2, %%xmm13\n\t" \ - "vpslld $25, %%xmm2, %%xmm14\n\t" \ - "vaesenc "#o"(%[KEY]), %%xmm6, %%xmm6\n\t" \ - "vpxor %%xmm13, %%xmm12, %%xmm12\n\t" \ - "vpxor %%xmm14, %%xmm12, %%xmm12\n\t" \ - "vaesenc "#o"(%[KEY]), %%xmm7, %%xmm7\n\t" \ - "vpsrldq $4, %%xmm12, %%xmm13\n\t" \ - "vpslldq $12, %%xmm12, %%xmm12\n\t" \ - "vaesenc "#o"(%[KEY]), %%xmm8, %%xmm8\n\t" \ - "vpxor %%xmm12, %%xmm2, %%xmm2\n\t" \ - "vpsrld $1, %%xmm2, %%xmm14\n\t" \ - "vaesenc "#o"(%[KEY]), %%xmm9, %%xmm9\n\t" \ - "vpsrld $2, %%xmm2, %%xmm1\n\t" \ - "vpsrld $7, %%xmm2, %%xmm0\n\t" \ - "vaesenc "#o"(%[KEY]), %%xmm10, %%xmm10\n\t" \ - "vpxor %%xmm1, %%xmm14, %%xmm14\n\t" \ - "vpxor %%xmm0, %%xmm14, %%xmm14\n\t" \ - "vaesenc "#o"(%[KEY]), %%xmm11, %%xmm11\n\t" \ - "vpxor %%xmm13, %%xmm14, %%xmm14\n\t" \ - "vpxor %%xmm14, %%xmm2, %%xmm2\n\t" \ - "vpxor %%xmm3, %%xmm2, %%xmm2\n\t" \ - - -/* Encrypt and carry-less multiply with last key. */ -#define VAESENC_LAST(in, out) \ - "vaesenclast %%xmm12, %%xmm4, %%xmm4\n\t" \ - "vaesenclast %%xmm12, %%xmm5, %%xmm5\n\t" \ - "vmovdqu (" #in "), %%xmm0\n\t" \ - "vmovdqu 16(" #in "), %%xmm1\n\t" \ - "vpxor %%xmm0, %%xmm4, %%xmm4\n\t" \ - "vpxor %%xmm1, %%xmm5, %%xmm5\n\t" \ - "vmovdqu %%xmm4, (" #out ")\n\t" \ - "vmovdqu %%xmm5, 16(" #out ")\n\t" \ - "vaesenclast %%xmm12, %%xmm6, %%xmm6\n\t" \ - "vaesenclast %%xmm12, %%xmm7, %%xmm7\n\t" \ - "vmovdqu 32(" #in "), %%xmm0\n\t" \ - "vmovdqu 48(" #in "), %%xmm1\n\t" \ - "vpxor %%xmm0, %%xmm6, %%xmm6\n\t" \ - "vpxor %%xmm1, %%xmm7, %%xmm7\n\t" \ - "vmovdqu %%xmm6, 32(" #out ")\n\t" \ - "vmovdqu %%xmm7, 48(" #out ")\n\t" \ - "vaesenclast %%xmm12, %%xmm8, %%xmm8\n\t" \ - "vaesenclast %%xmm12, %%xmm9, %%xmm9\n\t" \ - "vmovdqu 64(" #in "), %%xmm0\n\t" \ - "vmovdqu 80(" #in "), %%xmm1\n\t" \ - "vpxor %%xmm0, %%xmm8, %%xmm8\n\t" \ - "vpxor %%xmm1, %%xmm9, %%xmm9\n\t" \ - "vmovdqu %%xmm8, 64(" #out ")\n\t" \ - "vmovdqu %%xmm9, 80(" #out ")\n\t" \ - "vaesenclast %%xmm12, %%xmm10, %%xmm10\n\t" \ - "vaesenclast %%xmm12, %%xmm11, %%xmm11\n\t" \ - "vmovdqu 96(" #in "), %%xmm0\n\t" \ - "vmovdqu 112(" #in "), %%xmm1\n\t" \ - "vpxor %%xmm0, %%xmm10, %%xmm10\n\t" \ - "vpxor %%xmm1, %%xmm11, %%xmm11\n\t" \ - "vmovdqu %%xmm10, 96(" #out ")\n\t" \ - "vmovdqu %%xmm11, 112(" #out ")\n\t" - -#define VAESENC_BLOCK() \ - "vmovdqu " VAR(CTR1) ", %%xmm5\n\t" \ - "vpshufb %[BSWAP_EPI64], %%xmm5, %%xmm4\n\t" \ - "vpaddd %[ONE], %%xmm5, %%xmm5\n\t" \ - "vmovdqu %%xmm5, " VAR(CTR1) "\n\t" \ - "vpxor (%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vaesenc 16(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vaesenc 32(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vaesenc 48(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vaesenc 64(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vaesenc 80(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vaesenc 96(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vaesenc 112(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vaesenc 128(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vaesenc 144(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "cmpl $11, %[nr]\n\t" \ - "vmovdqa 160(%[KEY]), %%xmm5\n\t" \ - "jl %=f\n\t" \ - "vaesenc %%xmm5, %%xmm4, %%xmm4\n\t" \ - "vaesenc 176(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "cmpl $13, %[nr]\n\t" \ - "vmovdqa 192(%[KEY]), %%xmm5\n\t" \ - "jl %=f\n\t" \ - "vaesenc %%xmm5, %%xmm4, %%xmm4\n\t" \ - "vaesenc 208(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vmovdqa 224(%[KEY]), %%xmm5\n\t" \ - "%=:\n\t" \ - "vaesenclast %%xmm5, %%xmm4, %%xmm4\n\t" \ - "vmovdqu (%[in]," VAR(KR64) ",1), %%xmm5\n\t" \ - "vpxor %%xmm5, %%xmm4, %%xmm4\n\t" \ - "vmovdqu %%xmm4, (%[out]," VAR(KR64) ",1)\n\t" \ - "vpshufb %[BSWAP_MASK], %%xmm4, %%xmm4\n\t" \ - "vpxor %%xmm4, " VAR(XR) ", " VAR(XR) "\n\t" - -#define _VAESENC_GFMUL(in, H, X) \ - "vmovdqu " VAR(CTR1) ", %%xmm5\n\t" \ - "vpshufb %[BSWAP_EPI64], %%xmm5, %%xmm4\n\t" \ - "vpaddd %[ONE], %%xmm5, %%xmm5\n\t" \ - "vmovdqu %%xmm5, " VAR(CTR1) "\n\t" \ - "vpxor (%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vpclmulqdq $0x10, " #H ", " #X ", %%xmm6\n\t" \ - "vaesenc 16(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vaesenc 32(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vpclmulqdq $0x01, " #H ", " #X ", %%xmm7\n\t" \ - "vaesenc 48(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vaesenc 64(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vpclmulqdq $0x00, " #H ", " #X ", %%xmm8\n\t" \ - "vaesenc 80(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vpclmulqdq $0x11, " #H ", " #X ", %%xmm1\n\t" \ - "vaesenc 96(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vpxor %%xmm7, %%xmm6, %%xmm6\n\t" \ - "vpslldq $8, %%xmm6, %%xmm2\n\t" \ - "vpsrldq $8, %%xmm6, %%xmm6\n\t" \ - "vaesenc 112(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vpxor %%xmm8, %%xmm2, %%xmm2\n\t" \ - "vpxor %%xmm6, %%xmm1, %%xmm3\n\t" \ - "vmovdqa %[MOD2_128], %%xmm0\n\t" \ - "vpclmulqdq $0x10, %%xmm0, %%xmm2, %%xmm7\n\t" \ - "vaesenc 128(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vpshufd $0x4e, %%xmm2, %%xmm6\n\t" \ - "vpxor %%xmm7, %%xmm6, %%xmm6\n\t" \ - "vpclmulqdq $0x10, %%xmm0, %%xmm6, %%xmm7\n\t" \ - "vaesenc 144(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vpshufd $0x4e, %%xmm6, %%xmm6\n\t" \ - "vpxor %%xmm7, %%xmm6, %%xmm6\n\t" \ - "vpxor %%xmm3, %%xmm6, " VAR(XR) "\n\t" \ - "cmpl $11, %[nr]\n\t" \ - "vmovdqa 160(%[KEY]), %%xmm5\n\t" \ - "jl 1f\n\t" \ - "vaesenc %%xmm5, %%xmm4, %%xmm4\n\t" \ - "vaesenc 176(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "cmpl $13, %[nr]\n\t" \ - "vmovdqa 192(%[KEY]), %%xmm5\n\t" \ - "jl 1f\n\t" \ - "vaesenc %%xmm5, %%xmm4, %%xmm4\n\t" \ - "vaesenc 208(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vmovdqa 224(%[KEY]), %%xmm5\n\t" \ - "1:\n\t" \ - "vaesenclast %%xmm5, %%xmm4, %%xmm4\n\t" \ - "vmovdqu " #in ", %%xmm0\n\t" \ - "vpxor %%xmm0, %%xmm4, %%xmm4\n\t" \ - "vmovdqu %%xmm4, (%[out]," VAR(KR64) ",1)\n\t" -#define VAESENC_GFMUL(in, H, X) \ - _VAESENC_GFMUL(in, H, X) - - -#define _GHASH_GFMUL_AVX1(r, r2, a, b) \ - "vpshufd $0x4e, "#a", %%xmm1\n\t" \ - "vpshufd $0x4e, "#b", %%xmm2\n\t" \ - "vpclmulqdq $0x11, "#a", "#b", %%xmm3\n\t" \ - "vpclmulqdq $0x00, "#a", "#b", %%xmm0\n\t" \ - "vpxor "#a", %%xmm1, %%xmm1\n\t" \ - "vpxor "#b", %%xmm2, %%xmm2\n\t" \ - "vpclmulqdq $0x00, %%xmm2, %%xmm1, %%xmm1\n\t" \ - "vpxor %%xmm0, %%xmm1, %%xmm1\n\t" \ - "vpxor %%xmm3, %%xmm1, %%xmm1\n\t" \ - "vmovdqa %%xmm0, "#r2"\n\t" \ - "vmovdqa %%xmm3, " #r "\n\t" \ - "vpslldq $8, %%xmm1, %%xmm2\n\t" \ - "vpsrldq $8, %%xmm1, %%xmm1\n\t" \ - "vpxor %%xmm2, "#r2", "#r2"\n\t" \ - "vpxor %%xmm1, " #r ", " #r "\n\t" -#define GHASH_GFMUL_AVX1(r, r2, a, b) \ - _GHASH_GFMUL_AVX1(r, r2, a, b) - -#define _GHASH_GFMUL_XOR_AVX1(r, r2, a, b) \ - "vpshufd $0x4e, "#a", %%xmm1\n\t" \ - "vpshufd $0x4e, "#b", %%xmm2\n\t" \ - "vpclmulqdq $0x11, "#a", "#b", %%xmm3\n\t" \ - "vpclmulqdq $0x00, "#a", "#b", %%xmm0\n\t" \ - "vpxor "#a", %%xmm1, %%xmm1\n\t" \ - "vpxor "#b", %%xmm2, %%xmm2\n\t" \ - "vpclmulqdq $0x00, %%xmm2, %%xmm1, %%xmm1\n\t" \ - "vpxor %%xmm0, %%xmm1, %%xmm1\n\t" \ - "vpxor %%xmm3, %%xmm1, %%xmm1\n\t" \ - "vpxor %%xmm0, "#r2", "#r2"\n\t" \ - "vpxor %%xmm3, " #r ", " #r "\n\t" \ - "vpslldq $8, %%xmm1, %%xmm2\n\t" \ - "vpsrldq $8, %%xmm1, %%xmm1\n\t" \ - "vpxor %%xmm2, "#r2", "#r2"\n\t" \ - "vpxor %%xmm1, " #r ", " #r "\n\t" -#define GHASH_GFMUL_XOR_AVX1(r, r2, a, b) \ - _GHASH_GFMUL_XOR_AVX1(r, r2, a, b) - -#define GHASH_MID_AVX1(r, r2) \ - "vpsrld $31, "#r2", %%xmm0\n\t" \ - "vpsrld $31, " #r ", %%xmm1\n\t" \ - "vpslld $1, "#r2", "#r2"\n\t" \ - "vpslld $1, " #r ", " #r "\n\t" \ - "vpsrldq $12, %%xmm0, %%xmm2\n\t" \ - "vpslldq $4, %%xmm0, %%xmm0\n\t" \ - "vpslldq $4, %%xmm1, %%xmm1\n\t" \ - "vpor %%xmm2, " #r ", " #r "\n\t" \ - "vpor %%xmm0, "#r2", "#r2"\n\t" \ - "vpor %%xmm1, " #r ", " #r "\n\t" - -#define _GHASH_GFMUL_RED_AVX1(r, a, b) \ - "vpshufd $0x4e, "#a", %%xmm5\n\t" \ - "vpshufd $0x4e, "#b", %%xmm6\n\t" \ - "vpclmulqdq $0x11, "#a", "#b", %%xmm7\n\t" \ - "vpclmulqdq $0x00, "#a", "#b", %%xmm4\n\t" \ - "vpxor "#a", %%xmm5, %%xmm5\n\t" \ - "vpxor "#b", %%xmm6, %%xmm6\n\t" \ - "vpclmulqdq $0x00, %%xmm6, %%xmm5, %%xmm5\n\t" \ - "vpxor %%xmm4, %%xmm5, %%xmm5\n\t" \ - "vpxor %%xmm7, %%xmm5, %%xmm5\n\t" \ - "vpslldq $8, %%xmm5, %%xmm6\n\t" \ - "vpsrldq $8, %%xmm5, %%xmm5\n\t" \ - "vpxor %%xmm6, %%xmm4, %%xmm4\n\t" \ - "vpxor %%xmm5, %%xmm7, " #r "\n\t" \ - "vpslld $31, %%xmm4, %%xmm8\n\t" \ - "vpslld $30, %%xmm4, %%xmm9\n\t" \ - "vpslld $25, %%xmm4, %%xmm10\n\t" \ - "vpxor %%xmm9, %%xmm8, %%xmm8\n\t" \ - "vpxor %%xmm10, %%xmm8, %%xmm8\n\t" \ - "vpsrldq $4, %%xmm8, %%xmm9\n\t" \ - "vpslldq $12, %%xmm8, %%xmm8\n\t" \ - "vpxor %%xmm8, %%xmm4, %%xmm4\n\t" \ - "vpsrld $1, %%xmm4, %%xmm10\n\t" \ - "vpsrld $2, %%xmm4, %%xmm6\n\t" \ - "vpsrld $7, %%xmm4, %%xmm5\n\t" \ - "vpxor %%xmm6, %%xmm10, %%xmm10\n\t" \ - "vpxor %%xmm5, %%xmm10, %%xmm10\n\t" \ - "vpxor %%xmm9, %%xmm10, %%xmm10\n\t" \ - "vpxor %%xmm4, %%xmm10, %%xmm10\n\t" \ - "vpxor %%xmm10, " #r ", " #r "\n\t" -#define GHASH_GFMUL_RED_AVX1(r, a, b) \ - _GHASH_GFMUL_RED_AVX1(r, a, b) - -#define _GHASH_GFSQR_RED_AVX1(r, a) \ - "vpclmulqdq $0x00, "#a", "#a", %%xmm4\n\t" \ - "vpclmulqdq $0x11, "#a", "#a", " #r "\n\t" \ - "vpslld $31, %%xmm4, %%xmm8\n\t" \ - "vpslld $30, %%xmm4, %%xmm9\n\t" \ - "vpslld $25, %%xmm4, %%xmm10\n\t" \ - "vpxor %%xmm9, %%xmm8, %%xmm8\n\t" \ - "vpxor %%xmm10, %%xmm8, %%xmm8\n\t" \ - "vpsrldq $4, %%xmm8, %%xmm9\n\t" \ - "vpslldq $12, %%xmm8, %%xmm8\n\t" \ - "vpxor %%xmm8, %%xmm4, %%xmm4\n\t" \ - "vpsrld $1, %%xmm4, %%xmm10\n\t" \ - "vpsrld $2, %%xmm4, %%xmm6\n\t" \ - "vpsrld $7, %%xmm4, %%xmm5\n\t" \ - "vpxor %%xmm6, %%xmm10, %%xmm10\n\t" \ - "vpxor %%xmm5, %%xmm10, %%xmm10\n\t" \ - "vpxor %%xmm9, %%xmm10, %%xmm10\n\t" \ - "vpxor %%xmm4, %%xmm10, %%xmm10\n\t" \ - "vpxor %%xmm10, " #r ", " #r "\n\t" -#define GHASH_GFSQR_RED_AVX1(r, a) \ - _GHASH_GFSQR_RED_AVX1(r, a) - -#define GHASH_RED_AVX1(r, r2) \ - "vpslld $31, "#r2", %%xmm0\n\t" \ - "vpslld $30, "#r2", %%xmm1\n\t" \ - "vpslld $25, "#r2", %%xmm2\n\t" \ - "vpxor %%xmm1, %%xmm0, %%xmm0\n\t" \ - "vpxor %%xmm2, %%xmm0, %%xmm0\n\t" \ - "vmovdqa %%xmm0, %%xmm1\n\t" \ - "vpsrldq $4, %%xmm1, %%xmm1\n\t" \ - "vpslldq $12, %%xmm0, %%xmm0\n\t" \ - "vpxor %%xmm0, "#r2", "#r2"\n\t" \ - "vpsrld $1, "#r2", %%xmm2\n\t" \ - "vpsrld $2, "#r2", %%xmm3\n\t" \ - "vpsrld $7, "#r2", %%xmm0\n\t" \ - "vpxor %%xmm3, %%xmm2, %%xmm2\n\t" \ - "vpxor %%xmm0, %%xmm2, %%xmm2\n\t" \ - "vpxor %%xmm1, %%xmm2, %%xmm2\n\t" \ - "vpxor "#r2", %%xmm2, %%xmm2\n\t" \ - "vpxor %%xmm2, " #r ", " #r "\n\t" - -#define GHASH_GFMUL_RED_XOR_AVX1(r, r2, a, b) \ - GHASH_GFMUL_XOR_AVX1(r, r2, a, b) \ - GHASH_RED_AVX1(r, r2) - -#define GHASH_FULL_AVX1(r, r2, a, b) \ - GHASH_GFMUL_AVX1(r, r2, a, b) \ - GHASH_MID_AVX1(r, r2) \ - GHASH_RED_AVX1(r, r2) - -#define CALC_IV_12_AVX1() \ - "# Calculate values when IV is 12 bytes\n\t" \ - "# Set counter based on IV\n\t" \ - "movl $0x01000000, %%ecx\n\t" \ - "vpinsrq $0, 0(%%rax), %%xmm13, %%xmm13\n\t" \ - "vpinsrd $2, 8(%%rax), %%xmm13, %%xmm13\n\t" \ - "vpinsrd $3, %%ecx, %%xmm13, %%xmm13\n\t" \ - "# H = Encrypt X(=0) and T = Encrypt counter\n\t" \ - "vmovdqa 0(%[KEY]), " VAR(HR) "\n\t" \ - "vpxor " VAR(HR) ", %%xmm13, %%xmm1\n\t" \ - "vmovdqa 16(%[KEY]), %%xmm12\n\t" \ - "vaesenc %%xmm12, " VAR(HR) ", " VAR(HR) "\n\t" \ - "vaesenc %%xmm12, %%xmm1, %%xmm1\n\t" \ - "vmovdqa 32(%[KEY]), %%xmm12\n\t" \ - "vaesenc %%xmm12, " VAR(HR) ", " VAR(HR) "\n\t" \ - "vaesenc %%xmm12, %%xmm1, %%xmm1\n\t" \ - "vmovdqa 48(%[KEY]), %%xmm12\n\t" \ - "vaesenc %%xmm12, " VAR(HR) ", " VAR(HR) "\n\t" \ - "vaesenc %%xmm12, %%xmm1, %%xmm1\n\t" \ - "vmovdqa 64(%[KEY]), %%xmm12\n\t" \ - "vaesenc %%xmm12, " VAR(HR) ", " VAR(HR) "\n\t" \ - "vaesenc %%xmm12, %%xmm1, %%xmm1\n\t" \ - "vmovdqa 80(%[KEY]), %%xmm12\n\t" \ - "vaesenc %%xmm12, " VAR(HR) ", " VAR(HR) "\n\t" \ - "vaesenc %%xmm12, %%xmm1, %%xmm1\n\t" \ - "vmovdqa 96(%[KEY]), %%xmm12\n\t" \ - "vaesenc %%xmm12, " VAR(HR) ", " VAR(HR) "\n\t" \ - "vaesenc %%xmm12, %%xmm1, %%xmm1\n\t" \ - "vmovdqa 112(%[KEY]), %%xmm12\n\t" \ - "vaesenc %%xmm12, " VAR(HR) ", " VAR(HR) "\n\t" \ - "vaesenc %%xmm12, %%xmm1, %%xmm1\n\t" \ - "vmovdqa 128(%[KEY]), %%xmm12\n\t" \ - "vaesenc %%xmm12, " VAR(HR) ", " VAR(HR) "\n\t" \ - "vaesenc %%xmm12, %%xmm1, %%xmm1\n\t" \ - "vmovdqa 144(%[KEY]), %%xmm12\n\t" \ - "vaesenc %%xmm12, " VAR(HR) ", " VAR(HR) "\n\t" \ - "vaesenc %%xmm12, %%xmm1, %%xmm1\n\t" \ - "cmpl $11, %[nr]\n\t" \ - "vmovdqa 160(%[KEY]), %%xmm12\n\t" \ - "jl 31f\n\t" \ - "vaesenc %%xmm12, " VAR(HR) ", " VAR(HR) "\n\t" \ - "vaesenc %%xmm12, %%xmm1, %%xmm1\n\t" \ - "vmovdqa 176(%[KEY]), %%xmm12\n\t" \ - "vaesenc %%xmm12, " VAR(HR) ", " VAR(HR) "\n\t" \ - "vaesenc %%xmm12, %%xmm1, %%xmm1\n\t" \ - "cmpl $13, %[nr]\n\t" \ - "vmovdqa 192(%[KEY]), %%xmm12\n\t" \ - "jl 31f\n\t" \ - "vaesenc %%xmm12, " VAR(HR) ", " VAR(HR) "\n\t" \ - "vaesenc %%xmm12, %%xmm1, %%xmm1\n\t" \ - "vmovdqa 208(%[KEY]), %%xmm12\n\t" \ - "vaesenc %%xmm12, " VAR(HR) ", " VAR(HR) "\n\t" \ - "vaesenc %%xmm12, %%xmm1, %%xmm1\n\t" \ - "vmovdqu 224(%[KEY]), %%xmm12\n\t" \ - "31:\n\t" \ - "vaesenclast %%xmm12, " VAR(HR) ", " VAR(HR) "\n\t" \ - "vaesenclast %%xmm12, %%xmm1, %%xmm1\n\t" \ - "vpshufb %[BSWAP_MASK], " VAR(HR) ", " VAR(HR) "\n\t" \ - "vmovdqu %%xmm1, " VAR(TR) "\n\t" \ - "jmp 39f\n\t" - -#define CALC_IV_AVX1() \ - "# Calculate values when IV is not 12 bytes\n\t" \ - "# H = Encrypt X(=0)\n\t" \ - "vmovdqa 0(%[KEY]), " VAR(HR) "\n\t" \ - VAESENC_AVX(HR) \ - "vpshufb %[BSWAP_MASK], " VAR(HR) ", " VAR(HR) "\n\t" \ - "# Calc counter\n\t" \ - "# Initialization vector\n\t" \ - "cmpl $0, %%edx\n\t" \ - "movq $0, %%rcx\n\t" \ - "je 45f\n\t" \ - "cmpl $16, %%edx\n\t" \ - "jl 44f\n\t" \ - "andl $0xfffffff0, %%edx\n\t" \ - "\n" \ - "43:\n\t" \ - "vmovdqu (%%rax,%%rcx,1), %%xmm4\n\t" \ - "vpshufb %[BSWAP_MASK], %%xmm4, %%xmm4\n\t" \ - "vpxor %%xmm4, %%xmm13, %%xmm13\n\t" \ - GHASH_FULL_AVX1(%%xmm13, %%xmm12, %%xmm13, HR) \ - "addl $16, %%ecx\n\t" \ - "cmpl %%edx, %%ecx\n\t" \ - "jl 43b\n\t" \ - "movl %[ibytes], %%edx\n\t" \ - "cmpl %%edx, %%ecx\n\t" \ - "je 45f\n\t" \ - "\n" \ - "44:\n\t" \ - "subq $16, %%rsp\n\t" \ - "vpxor %%xmm4, %%xmm4, %%xmm4\n\t" \ - "xorl %%ebx, %%ebx\n\t" \ - "vmovdqu %%xmm4, (%%rsp)\n\t" \ - "42:\n\t" \ - "movzbl (%%rax,%%rcx,1), %%r13d\n\t" \ - "movb %%r13b, (%%rsp,%%rbx,1)\n\t" \ - "incl %%ecx\n\t" \ - "incl %%ebx\n\t" \ - "cmpl %%edx, %%ecx\n\t" \ - "jl 42b\n\t" \ - "vmovdqu (%%rsp), %%xmm4\n\t" \ - "addq $16, %%rsp\n\t" \ - "vpshufb %[BSWAP_MASK], %%xmm4, %%xmm4\n\t" \ - "vpxor %%xmm4, %%xmm13, %%xmm13\n\t" \ - GHASH_FULL_AVX1(%%xmm13, %%xmm12, %%xmm13, HR) \ - "\n" \ - "45:\n\t" \ - "# T = Encrypt counter\n\t" \ - "vpxor %%xmm0, %%xmm0, %%xmm0\n\t" \ - "shll $3, %%edx\n\t" \ - "vpinsrq $0, %%rdx, %%xmm0, %%xmm0\n\t" \ - "vpxor %%xmm0, %%xmm13, %%xmm13\n\t" \ - GHASH_FULL_AVX1(%%xmm13, %%xmm12, %%xmm13, HR) \ - "vpshufb %[BSWAP_MASK], %%xmm13, %%xmm13\n\t" \ - "# Encrypt counter\n\t" \ - "vmovdqa 0(%[KEY]), %%xmm4\n\t" \ - "vpxor %%xmm13, %%xmm4, %%xmm4\n\t" \ - VAESENC_AVX(%%xmm4) \ - "vmovdqu %%xmm4, " VAR(TR) "\n\t" - -#define CALC_AAD_AVX1() \ - "# Additional authentication data\n\t" \ - "movl %[abytes], %%edx\n\t" \ - "cmpl $0, %%edx\n\t" \ - "je 25f\n\t" \ - "movq %[addt], %%rax\n\t" \ - "xorl %%ecx, %%ecx\n\t" \ - "cmpl $16, %%edx\n\t" \ - "jl 24f\n\t" \ - "andl $0xfffffff0, %%edx\n\t" \ - "\n" \ - "23:\n\t" \ - "vmovdqu (%%rax,%%rcx,1), %%xmm4\n\t" \ - "vpshufb %[BSWAP_MASK], %%xmm4, %%xmm4\n\t" \ - "vpxor %%xmm4, " VAR(XR) ", " VAR(XR) "\n\t" \ - GHASH_FULL_AVX1(XR, %%xmm12, XR, HR) \ - "addl $16, %%ecx\n\t" \ - "cmpl %%edx, %%ecx\n\t" \ - "jl 23b\n\t" \ - "movl %[abytes], %%edx\n\t" \ - "cmpl %%edx, %%ecx\n\t" \ - "je 25f\n\t" \ - "\n" \ - "24:\n\t" \ - "subq $16, %%rsp\n\t" \ - "vpxor %%xmm4, %%xmm4, %%xmm4\n\t" \ - "xorl %%ebx, %%ebx\n\t" \ - "vmovdqu %%xmm4, (%%rsp)\n\t" \ - "22:\n\t" \ - "movzbl (%%rax,%%rcx,1), %%r13d\n\t" \ - "movb %%r13b, (%%rsp,%%rbx,1)\n\t" \ - "incl %%ecx\n\t" \ - "incl %%ebx\n\t" \ - "cmpl %%edx, %%ecx\n\t" \ - "jl 22b\n\t" \ - "vmovdqu (%%rsp), %%xmm4\n\t" \ - "addq $16, %%rsp\n\t" \ - "vpshufb %[BSWAP_MASK], %%xmm4, %%xmm4\n\t" \ - "vpxor %%xmm4, " VAR(XR) ", " VAR(XR) "\n\t" \ - GHASH_FULL_AVX1(XR, %%xmm12, XR, HR) \ - "\n" \ - "25:\n\t" - -#define CALC_HT_8_AVX1() \ - "vmovdqa " VAR(XR) ", %%xmm2\n\t" \ - "# H ^ 1\n\t" \ - "vmovdqu " VAR(HR) ", 0(" VAR(HTR) ")\n\t" \ - "# H ^ 2\n\t" \ - GHASH_GFSQR_RED_AVX1(%%xmm0, HR) \ - "vmovdqu %%xmm0 , 16(" VAR(HTR) ")\n\t" \ - "# H ^ 3\n\t" \ - GHASH_GFMUL_RED_AVX1(%%xmm1, HR, %%xmm0) \ - "vmovdqu %%xmm1 , 32(" VAR(HTR) ")\n\t" \ - "# H ^ 4\n\t" \ - GHASH_GFSQR_RED_AVX1(%%xmm3, %%xmm0) \ - "vmovdqu %%xmm3 , 48(" VAR(HTR) ")\n\t" \ - "# H ^ 5\n\t" \ - GHASH_GFMUL_RED_AVX1(%%xmm12, %%xmm0, %%xmm1) \ - "vmovdqu %%xmm12, 64(" VAR(HTR) ")\n\t" \ - "# H ^ 6\n\t" \ - GHASH_GFSQR_RED_AVX1(%%xmm12, %%xmm1) \ - "vmovdqu %%xmm12, 80(" VAR(HTR) ")\n\t" \ - "# H ^ 7\n\t" \ - GHASH_GFMUL_RED_AVX1(%%xmm12, %%xmm1, %%xmm3) \ - "vmovdqu %%xmm12, 96(" VAR(HTR) ")\n\t" \ - "# H ^ 8\n\t" \ - GHASH_GFSQR_RED_AVX1(%%xmm12, %%xmm3) \ - "vmovdqu %%xmm12, 112(" VAR(HTR) ")\n\t" - -#define VAESENC_128_GHASH_AVX1(src, o) \ - "leaq (%[in]," VAR(KR64) ",1), %%rcx\n\t" \ - "leaq (%[out]," VAR(KR64) ",1), %%rdx\n\t" \ - /* src is either %%rcx or %%rdx */ \ - VAESENC_CTR() \ - VAESENC_XOR() \ - VAESENC_PCLMUL_1(src, 16, (o-128), 112) \ - VAESENC_PCLMUL_N(src, 32, (o-112), 96) \ - VAESENC_PCLMUL_N(src, 48, (o- 96), 80) \ - VAESENC_PCLMUL_N(src, 64, (o- 80), 64) \ - VAESENC_PCLMUL_N(src, 80, (o- 64), 48) \ - VAESENC_PCLMUL_N(src, 96, (o- 48), 32) \ - VAESENC_PCLMUL_N(src, 112, (o- 32), 16) \ - VAESENC_PCLMUL_N(src, 128, (o- 16), 0) \ - VAESENC_PCLMUL_L(144) \ - "cmpl $11, %[nr]\n\t" \ - "vmovdqa 160(%[KEY]), %%xmm12\n\t" \ - "jl 4f\n\t" \ - VAESENC() \ - VAESENC_SET(176) \ - "cmpl $13, %[nr]\n\t" \ - "vmovdqa 192(%[KEY]), %%xmm12\n\t" \ - "jl 4f\n\t" \ - VAESENC() \ - VAESENC_SET(208) \ - "vmovdqa 224(%[KEY]), %%xmm12\n\t" \ - "\n" \ -"4:\n\t" \ - VAESENC_LAST(%%rcx, %%rdx) - -#define _VAESENC_AVX(r) \ - "vaesenc 16(%[KEY]), " #r ", " #r "\n\t" \ - "vaesenc 32(%[KEY]), " #r ", " #r "\n\t" \ - "vaesenc 48(%[KEY]), " #r ", " #r "\n\t" \ - "vaesenc 64(%[KEY]), " #r ", " #r "\n\t" \ - "vaesenc 80(%[KEY]), " #r ", " #r "\n\t" \ - "vaesenc 96(%[KEY]), " #r ", " #r "\n\t" \ - "vaesenc 112(%[KEY]), " #r ", " #r "\n\t" \ - "vaesenc 128(%[KEY]), " #r ", " #r "\n\t" \ - "vaesenc 144(%[KEY]), " #r ", " #r "\n\t" \ - "cmpl $11, %[nr]\n\t" \ - "vmovdqa 160(%[KEY]), %%xmm5\n\t" \ - "jl %=f\n\t" \ - "vaesenc %%xmm5, " #r ", " #r "\n\t" \ - "vaesenc 176(%[KEY]), " #r ", " #r "\n\t" \ - "cmpl $13, %[nr]\n\t" \ - "vmovdqa 192(%[KEY]), %%xmm5\n\t" \ - "jl %=f\n\t" \ - "vaesenc %%xmm5, " #r ", " #r "\n\t" \ - "vaesenc 208(%[KEY]), " #r ", " #r "\n\t" \ - "vmovdqa 224(%[KEY]), %%xmm5\n\t" \ - "%=:\n\t" \ - "vaesenclast %%xmm5, " #r ", " #r "\n\t" -#define VAESENC_AVX(r) \ - _VAESENC_AVX(r) - -#define AESENC_LAST15_ENC_AVX1() \ - "movl %[nbytes], %%ecx\n\t" \ - "movl %%ecx, %%edx\n\t" \ - "andl $0x0f, %%ecx\n\t" \ - "jz 55f\n\t" \ - "vmovdqu " VAR(CTR1) ", %%xmm13\n\t" \ - "vpshufb %[BSWAP_EPI64], %%xmm13, %%xmm13\n\t" \ - "vpxor 0(%[KEY]), %%xmm13, %%xmm13\n\t" \ - VAESENC_AVX(%%xmm13) \ - "subq $16, %%rsp\n\t" \ - "xorl %%ecx, %%ecx\n\t" \ - "vmovdqu %%xmm13, (%%rsp)\n\t" \ - "\n" \ - "51:\n\t" \ - "movzbl (%[in]," VAR(KR64) ",1), %%r13d\n\t" \ - "xorb (%%rsp,%%rcx,1), %%r13b\n\t" \ - "movb %%r13b, (%[out]," VAR(KR64) ",1)\n\t" \ - "movb %%r13b, (%%rsp,%%rcx,1)\n\t" \ - "incl " VAR(KR) "\n\t" \ - "incl %%ecx\n\t" \ - "cmpl %%edx, " VAR(KR) "\n\t" \ - "jl 51b\n\t" \ - "xorq %%r13, %%r13\n\t" \ - "cmpl $16, %%ecx\n\t" \ - "je 53f\n\t" \ - "\n" \ - "52:\n\t" \ - "movb %%r13b, (%%rsp,%%rcx,1)\n\t" \ - "incl %%ecx\n\t" \ - "cmpl $16, %%ecx\n\t" \ - "jl 52b\n\t" \ - "53:\n\t" \ - "vmovdqu (%%rsp), %%xmm13\n\t" \ - "addq $16, %%rsp\n\t" \ - "vpshufb %[BSWAP_MASK], %%xmm13, %%xmm13\n\t" \ - "vpxor %%xmm13, " VAR(XR) ", " VAR(XR) "\n\t" \ - GHASH_GFMUL_RED_AVX1(XR, HR, XR) \ - -#define AESENC_LAST15_DEC_AVX1() \ - "movl %[nbytes], %%ecx\n\t" \ - "movl %%ecx, %%edx\n\t" \ - "andl $0x0f, %%ecx\n\t" \ - "jz 55f\n\t" \ - "vmovdqu " VAR(CTR1) ", %%xmm13\n\t" \ - "vpshufb %[BSWAP_EPI64], %%xmm13, %%xmm13\n\t" \ - "vpxor 0(%[KEY]), %%xmm13, %%xmm13\n\t" \ - VAESENC_AVX(%%xmm13) \ - "subq $32, %%rsp\n\t" \ - "xorl %%ecx, %%ecx\n\t" \ - "vmovdqu %%xmm13, (%%rsp)\n\t" \ - "vpxor %%xmm0, %%xmm0, %%xmm0\n\t" \ - "vmovdqu %%xmm0, 16(%%rsp)\n\t" \ - "\n" \ - "51:\n\t" \ - "movzbl (%[in]," VAR(KR64) ",1), %%r13d\n\t" \ - "movb %%r13b, 16(%%rsp,%%rcx,1)\n\t" \ - "xorb (%%rsp,%%rcx,1), %%r13b\n\t" \ - "movb %%r13b, (%[out]," VAR(KR64) ",1)\n\t" \ - "incl " VAR(KR) "\n\t" \ - "incl %%ecx\n\t" \ - "cmpl %%edx, " VAR(KR) "\n\t" \ - "jl 51b\n\t" \ - "53:\n\t" \ - "vmovdqu 16(%%rsp), %%xmm13\n\t" \ - "addq $32, %%rsp\n\t" \ - "vpshufb %[BSWAP_MASK], %%xmm13, %%xmm13\n\t" \ - "vpxor %%xmm13, " VAR(XR) ", " VAR(XR) "\n\t" \ - GHASH_GFMUL_RED_AVX1(XR, HR, XR) \ - -#define CALC_TAG_AVX1() \ - "movl %[nbytes], %%edx\n\t" \ - "movl %[abytes], %%ecx\n\t" \ - "shlq $3, %%rdx\n\t" \ - "shlq $3, %%rcx\n\t" \ - "vpinsrq $0, %%rdx, %%xmm0, %%xmm0\n\t" \ - "vpinsrq $1, %%rcx, %%xmm0, %%xmm0\n\t" \ - "vpxor %%xmm0, " VAR(XR) ", " VAR(XR) "\n\t" \ - GHASH_GFMUL_RED_AVX1(XR, HR, XR) \ - "vpshufb %[BSWAP_MASK], " VAR(XR) ", " VAR(XR) "\n\t" \ - "vpxor " VAR(TR) ", " VAR(XR) ", %%xmm0\n\t" \ - -#define STORE_TAG_AVX() \ - "cmpl $16, %[tbytes]\n\t" \ - "je 71f\n\t" \ - "xorq %%rcx, %%rcx\n\t" \ - "vmovdqu %%xmm0, (%%rsp)\n\t" \ - "73:\n\t" \ - "movzbl (%%rsp,%%rcx,1), %%r13d\n\t" \ - "movb %%r13b, (%[tag],%%rcx,1)\n\t" \ - "incl %%ecx\n\t" \ - "cmpl %[tbytes], %%ecx\n\t" \ - "jne 73b\n\t" \ - "jmp 72f\n\t" \ - "\n" \ - "71:\n\t" \ - "vmovdqu %%xmm0, (%[tag])\n\t" \ - "\n" \ - "72:\n\t" - -#define CMP_TAG_AVX() \ - "cmpl $16, %[tbytes]\n\t" \ - "je 71f\n\t" \ - "subq $16, %%rsp\n\t" \ - "xorq %%rcx, %%rcx\n\t" \ - "xorq %%rax, %%rax\n\t" \ - "vmovdqu %%xmm0, (%%rsp)\n\t" \ - "\n" \ - "73:\n\t" \ - "movzbl (%%rsp,%%rcx,1), %%r13d\n\t" \ - "xorb (%[tag],%%rcx,1), %%r13b\n\t" \ - "orb %%r13b, %%al\n\t" \ - "incl %%ecx\n\t" \ - "cmpl %[tbytes], %%ecx\n\t" \ - "jne 73b\n\t" \ - "cmpb $0x00, %%al\n\t" \ - "sete %%al\n\t" \ - "addq $16, %%rsp\n\t" \ - "jmp 72f\n\t" \ - "\n" \ - "71:\n\t" \ - "vmovdqu (%[tag]), %%xmm1\n\t" \ - "vpcmpeqb %%xmm1, %%xmm0, %%xmm0\n\t" \ - "vpmovmskb %%xmm0, %%edx\n\t" \ - "# %%edx == 0xFFFF then return 1 else => return 0\n\t" \ - "xorl %%eax, %%eax\n\t" \ - "cmpl $0xffff, %%edx\n\t" \ - "sete %%al\n\t" \ - "\n" \ - "72:\n\t" \ - "movl %%eax, (%[res])\n\t" - -static void AES_GCM_encrypt_avx1(const unsigned char *in, unsigned char *out, - const unsigned char* addt, - const unsigned char* ivec, unsigned char *tag, - unsigned int nbytes, unsigned int abytes, - unsigned int ibytes, unsigned int tbytes, - const unsigned char* key, int nr) -{ - register const unsigned char* iv asm("rax") = ivec; - register unsigned int ivLen asm("ebx") = ibytes; - - __asm__ __volatile__ ( - "subq $" VAR(STACK_OFFSET) ", %%rsp\n\t" - /* Counter is xmm13 */ - "vpxor %%xmm13, %%xmm13, %%xmm13\n\t" - "vpxor " VAR(XR) ", " VAR(XR) ", " VAR(XR) "\n\t" - "movl %[ibytes], %%edx\n\t" - "cmpl $12, %%edx\n\t" - "jne 35f\n\t" - CALC_IV_12_AVX1() - "\n" - "35:\n\t" - CALC_IV_AVX1() - "\n" - "39:\n\t" - - CALC_AAD_AVX1() - - "# Calculate counter and H\n\t" - "vpsrlq $63, " VAR(HR) ", %%xmm5\n\t" - "vpsllq $1, " VAR(HR) ", %%xmm4\n\t" - "vpslldq $8, %%xmm5, %%xmm5\n\t" - "vpor %%xmm5, %%xmm4, %%xmm4\n\t" - "vpshufd $0xff, " VAR(HR) ", " VAR(HR) "\n\t" - "vpsrad $31, " VAR(HR) ", " VAR(HR) "\n\t" - "vpshufb %[BSWAP_EPI64], %%xmm13, %%xmm13\n\t" - "vpand %[MOD2_128], " VAR(HR) ", " VAR(HR) "\n\t" - "vpaddd %[ONE], %%xmm13, %%xmm13\n\t" - "vpxor %%xmm4, " VAR(HR) ", " VAR(HR) "\n\t" - "vmovdqu %%xmm13, " VAR(CTR1) "\n\t" - - "xorl " VAR(KR) ", " VAR(KR) "\n\t" - -#if !defined(AES_GCM_AESNI_NO_UNROLL) && !defined(AES_GCM_AVX1_NO_UNROLL) - "cmpl $128, %[nbytes]\n\t" - "movl %[nbytes], %%r13d\n\t" - "jl 5f\n\t" - "andl $0xffffff80, %%r13d\n\t" - - CALC_HT_8_AVX1() - - "# First 128 bytes of input\n\t" - VAESENC_128() - - "cmpl $128, %%r13d\n\t" - "movl $128, " VAR(KR) "\n\t" - "jle 2f\n\t" - - "# More 128 bytes of input\n\t" - "\n" - "3:\n\t" - VAESENC_128_GHASH_AVX1(%%rdx, 0) - "addl $128, " VAR(KR) "\n\t" - "cmpl %%r13d, " VAR(KR) "\n\t" - "jl 3b\n\t" - "\n" - "2:\n\t" - "vmovdqa %[BSWAP_MASK], %%xmm13\n\t" - "vpshufb %%xmm13, %%xmm4, %%xmm4\n\t" - "vpshufb %%xmm13, %%xmm5, %%xmm5\n\t" - "vpshufb %%xmm13, %%xmm6, %%xmm6\n\t" - "vpshufb %%xmm13, %%xmm7, %%xmm7\n\t" - "vpxor %%xmm2, %%xmm4, %%xmm4\n\t" - "vpshufb %%xmm13, %%xmm8, %%xmm8\n\t" - "vpshufb %%xmm13, %%xmm9, %%xmm9\n\t" - "vpshufb %%xmm13, %%xmm10, %%xmm10\n\t" - "vpshufb %%xmm13, %%xmm11, %%xmm11\n\t" - - "vmovdqu (" VAR(HTR) "), %%xmm12\n\t" - "vmovdqu 16(" VAR(HTR) "), %%xmm14\n\t" - GHASH_GFMUL_AVX1(XR, %%xmm13, %%xmm11, %%xmm12) - GHASH_GFMUL_XOR_AVX1(XR, %%xmm13, %%xmm10, %%xmm14) - "vmovdqu 32(" VAR(HTR) "), %%xmm12\n\t" - "vmovdqu 48(" VAR(HTR) "), %%xmm14\n\t" - GHASH_GFMUL_XOR_AVX1(XR, %%xmm13, %%xmm9, %%xmm12) - GHASH_GFMUL_XOR_AVX1(XR, %%xmm13, %%xmm8, %%xmm14) - "vmovdqu 64(" VAR(HTR) "), %%xmm12\n\t" - "vmovdqu 80(" VAR(HTR) "), %%xmm14\n\t" - GHASH_GFMUL_XOR_AVX1(XR, %%xmm13, %%xmm7, %%xmm12) - GHASH_GFMUL_XOR_AVX1(XR, %%xmm13, %%xmm6, %%xmm14) - "vmovdqu 96(" VAR(HTR) "), %%xmm12\n\t" - "vmovdqu 112(" VAR(HTR) "), %%xmm14\n\t" - GHASH_GFMUL_XOR_AVX1(XR, %%xmm13, %%xmm5, %%xmm12) - GHASH_GFMUL_RED_XOR_AVX1(XR, %%xmm13, %%xmm4, %%xmm14) - - "vmovdqu 0(" VAR(HTR) "), " VAR(HR) "\n\t" - "\n" - "5:\n\t" - "movl %[nbytes], %%edx\n\t" - "cmpl %%edx, " VAR(KR) "\n\t" - "jge 55f\n\t" -#endif - - "movl %[nbytes], %%r13d\n\t" - "andl $0xfffffff0, %%r13d\n\t" - "cmpl %%r13d, " VAR(KR) "\n\t" - "jge 14f\n\t" - - VAESENC_BLOCK() - "addl $16, " VAR(KR) "\n\t" - "cmpl %%r13d, " VAR(KR) "\n\t" - "jge 13f\n\t" - "\n" - "12:\n\t" - "vmovdqu (%[in]," VAR(KR64) ",1), %%xmm9\n\t" - VAESENC_GFMUL(%%xmm9, HR, XR) - "vpshufb %[BSWAP_MASK], %%xmm4, %%xmm4\n\t" - "addl $16, " VAR(KR) "\n\t" - "vpxor %%xmm4, " VAR(XR) ", " VAR(XR) "\n\t" - "cmpl %%r13d, " VAR(KR) "\n\t" - "jl 12b\n\t" - "\n" - "13:\n\t" - GHASH_GFMUL_RED_AVX1(XR, HR, XR) - "\n" - "14:\n\t" - - AESENC_LAST15_ENC_AVX1() - "\n" - "55:\n\t" - - CALC_TAG_AVX1() - STORE_TAG_AVX() - "addq $" VAR(STACK_OFFSET) ", %%rsp\n\t" - "vzeroupper\n\t" - - : - : [KEY] "r" (key), - [in] "r" (in), [out] "r" (out), [nr] "r" (nr), - [nbytes] "r" (nbytes), [abytes] "r" (abytes), [addt] "r" (addt), - [ivec] "r" (iv), [ibytes] "r" (ivLen), [tbytes] "r" (tbytes), - [tag] "r" (tag), - [BSWAP_MASK] "m" (BSWAP_MASK), - [BSWAP_EPI64] "m" (BSWAP_EPI64), - [ONE] "m" (ONE), -#if !defined(AES_GCM_AESNI_NO_UNROLL) && !defined(AES_GCM_AVX1_NO_UNROLL) - [TWO] "m" (TWO), [THREE] "m" (THREE), [FOUR] "m" (FOUR), - [FIVE] "m" (FIVE), [SIX] "m" (SIX), [SEVEN] "m" (SEVEN), - [EIGHT] "m" (EIGHT), -#endif - [MOD2_128] "m" (MOD2_128) - : "xmm15", "xmm14", "xmm13", "xmm12", - "xmm0", "xmm1", "xmm2", "xmm3", "memory", - "xmm4", "xmm5", "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", - "rcx", "rdx", "r13" - ); -} - +void AES_GCM_encrypt_avx1(const unsigned char *in, unsigned char *out, + const unsigned char* addt, const unsigned char* ivec, + unsigned char *tag, unsigned int nbytes, + unsigned int abytes, unsigned int ibytes, + unsigned int tbytes, const unsigned char* key, + int nr) + XASM_LINK("AES_GCM_encrypt_avx1"); #ifdef HAVE_INTEL_AVX2 -/* Encrypt and carry-less multiply for AVX2. */ -#define VAESENC_PCLMUL_AVX2_1(src, o1, o2, o3) \ - "vmovdqu " #o2 "(" #src "), %%xmm12\n\t" \ - "vmovdqa " #o1 "(%[KEY]), %%xmm0\n\t" \ - "vpshufb %[BSWAP_MASK], %%xmm12, %%xmm12\n\t" \ - "vmovdqu " #o3 "(" VAR(HTR) "), %%xmm13\n\t" \ - "vpxor %%xmm2, %%xmm12, %%xmm12\n\t" \ - "vpclmulqdq $0x10, %%xmm13, %%xmm12, %%xmm1\n\t" \ - "vpclmulqdq $0x01, %%xmm13, %%xmm12, %%xmm14\n\t" \ - "vpclmulqdq $0x00, %%xmm13, %%xmm12, %%xmm2\n\t" \ - "vpclmulqdq $0x11, %%xmm13, %%xmm12, %%xmm3\n\t" \ - "vaesenc %%xmm0, %%xmm4, %%xmm4\n\t" \ - "vaesenc %%xmm0, %%xmm5, %%xmm5\n\t" \ - "vaesenc %%xmm0, %%xmm6, %%xmm6\n\t" \ - "vaesenc %%xmm0, %%xmm7, %%xmm7\n\t" \ - "vaesenc %%xmm0, %%xmm8, %%xmm8\n\t" \ - "vaesenc %%xmm0, %%xmm9, %%xmm9\n\t" \ - "vaesenc %%xmm0, %%xmm10, %%xmm10\n\t" \ - "vaesenc %%xmm0, %%xmm11, %%xmm11\n\t" \ - -#define VAESENC_PCLMUL_AVX2_2(src, o1, o2, o3) \ - "vmovdqu " #o2 "(" #src "), %%xmm12\n\t" \ - "vmovdqu " #o3 "(" VAR(HTR) "), %%xmm0\n\t" \ - "vpshufb %[BSWAP_MASK], %%xmm12, %%xmm12\n\t" \ - "vpxor %%xmm14, %%xmm1, %%xmm1\n\t" \ - "vpclmulqdq $0x10, %%xmm0, %%xmm12, %%xmm13\n\t" \ - "vpclmulqdq $0x01, %%xmm0, %%xmm12, %%xmm14\n\t" \ - "vpclmulqdq $0x00, %%xmm0, %%xmm12, %%xmm15\n\t" \ - "vpclmulqdq $0x11, %%xmm0, %%xmm12, %%xmm12\n\t" \ - "vmovdqa " #o1 "(%[KEY]), %%xmm0\n\t" \ - "vpxor %%xmm13, %%xmm1, %%xmm1\n\t" \ - "vpxor %%xmm12, %%xmm3, %%xmm3\n\t" \ - "vaesenc %%xmm0, %%xmm4, %%xmm4\n\t" \ - "vaesenc %%xmm0, %%xmm5, %%xmm5\n\t" \ - "vaesenc %%xmm0, %%xmm6, %%xmm6\n\t" \ - "vaesenc %%xmm0, %%xmm7, %%xmm7\n\t" \ - "vaesenc %%xmm0, %%xmm8, %%xmm8\n\t" \ - "vaesenc %%xmm0, %%xmm9, %%xmm9\n\t" \ - "vaesenc %%xmm0, %%xmm10, %%xmm10\n\t" \ - "vaesenc %%xmm0, %%xmm11, %%xmm11\n\t" \ - -#define VAESENC_PCLMUL_AVX2_N(src, o1, o2, o3) \ - "vmovdqu " #o2 "(" #src "), %%xmm12\n\t" \ - "vmovdqu " #o3 "(" VAR(HTR) "), %%xmm0\n\t" \ - "vpshufb %[BSWAP_MASK], %%xmm12, %%xmm12\n\t" \ - "vpxor %%xmm14, %%xmm1, %%xmm1\n\t" \ - "vpxor %%xmm15, %%xmm2, %%xmm2\n\t" \ - "vpclmulqdq $0x10, %%xmm0, %%xmm12, %%xmm13\n\t" \ - "vpclmulqdq $0x01, %%xmm0, %%xmm12, %%xmm14\n\t" \ - "vpclmulqdq $0x00, %%xmm0, %%xmm12, %%xmm15\n\t" \ - "vpclmulqdq $0x11, %%xmm0, %%xmm12, %%xmm12\n\t" \ - "vmovdqa " #o1 "(%[KEY]), %%xmm0\n\t" \ - "vpxor %%xmm13, %%xmm1, %%xmm1\n\t" \ - "vpxor %%xmm12, %%xmm3, %%xmm3\n\t" \ - "vaesenc %%xmm0, %%xmm4, %%xmm4\n\t" \ - "vaesenc %%xmm0, %%xmm5, %%xmm5\n\t" \ - "vaesenc %%xmm0, %%xmm6, %%xmm6\n\t" \ - "vaesenc %%xmm0, %%xmm7, %%xmm7\n\t" \ - "vaesenc %%xmm0, %%xmm8, %%xmm8\n\t" \ - "vaesenc %%xmm0, %%xmm9, %%xmm9\n\t" \ - "vaesenc %%xmm0, %%xmm10, %%xmm10\n\t" \ - "vaesenc %%xmm0, %%xmm11, %%xmm11\n\t" \ - -#define VAESENC_PCLMUL_AVX2_L(o) \ - "vpxor %%xmm14, %%xmm1, %%xmm1\n\t" \ - "vpxor %%xmm15, %%xmm2, %%xmm2\n\t" \ - "vpslldq $8, %%xmm1, %%xmm12\n\t" \ - "vpsrldq $8, %%xmm1, %%xmm1\n\t" \ - "vmovdqa "#o"(%[KEY]), %%xmm15\n\t" \ - "vmovdqa %[MOD2_128], %%xmm0\n\t" \ - "vaesenc %%xmm15, %%xmm4, %%xmm4\n\t" \ - "vpxor %%xmm12, %%xmm2, %%xmm2\n\t" \ - "vpxor %%xmm1, %%xmm3, %%xmm3\n\t" \ - "vpclmulqdq $0x10, %%xmm0, %%xmm2, %%xmm14\n\t" \ - "vaesenc %%xmm15, %%xmm5, %%xmm5\n\t" \ - "vaesenc %%xmm15, %%xmm6, %%xmm6\n\t" \ - "vaesenc %%xmm15, %%xmm7, %%xmm7\n\t" \ - "vpshufd $0x4e, %%xmm2, %%xmm2\n\t" \ - "vpxor %%xmm14, %%xmm2, %%xmm2\n\t" \ - "vpclmulqdq $0x10, %%xmm0, %%xmm2, %%xmm14\n\t" \ - "vaesenc %%xmm15, %%xmm8, %%xmm8\n\t" \ - "vaesenc %%xmm15, %%xmm9, %%xmm9\n\t" \ - "vaesenc %%xmm15, %%xmm10, %%xmm10\n\t" \ - "vpshufd $0x4e, %%xmm2, %%xmm2\n\t" \ - "vpxor %%xmm14, %%xmm2, %%xmm2\n\t" \ - "vpxor %%xmm3, %%xmm2, %%xmm2\n\t" \ - "vaesenc %%xmm15, %%xmm11, %%xmm11\n\t" - -#define VAESENC_BLOCK_AVX2() \ - "vmovdqu " VAR(CTR1) ", %%xmm5\n\t" \ - "vpshufb %[BSWAP_EPI64], %%xmm5, %%xmm4\n\t" \ - "vpaddd %[ONE], %%xmm5, %%xmm5\n\t" \ - "vmovdqu %%xmm5, " VAR(CTR1) "\n\t" \ - "vpxor (%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vaesenc 16(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vaesenc 32(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vaesenc 48(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vaesenc 64(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vaesenc 80(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vaesenc 96(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vaesenc 112(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vaesenc 128(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vaesenc 144(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "cmpl $11, %[nr]\n\t" \ - "vmovdqa 160(%[KEY]), %%xmm5\n\t" \ - "jl %=f\n\t" \ - "vaesenc %%xmm5, %%xmm4, %%xmm4\n\t" \ - "vaesenc 176(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "cmpl $13, %[nr]\n\t" \ - "vmovdqa 192(%[KEY]), %%xmm5\n\t" \ - "jl %=f\n\t" \ - "vaesenc %%xmm5, %%xmm4, %%xmm4\n\t" \ - "vaesenc 208(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vmovdqa 224(%[KEY]), %%xmm5\n\t" \ - "%=:\n\t" \ - "vaesenclast %%xmm5, %%xmm4, %%xmm4\n\t" \ - "vmovdqu (%[in]," VAR(KR64) ",1), %%xmm5\n\t" \ - "vpxor %%xmm5, %%xmm4, %%xmm4\n\t" \ - "vmovdqu %%xmm4, (%[out]," VAR(KR64) ",1)\n\t" \ - "vpshufb %[BSWAP_MASK], %%xmm4, %%xmm4\n\t" \ - "vpxor %%xmm4, " VAR(XR) ", " VAR(XR) "\n\t" - -/* Karatsuba multiplication - slower - * H01 = H[1] ^ H[0] (top and bottom 64-bits XORed) - */ -#define _VAESENC_GFMUL_AVX2(in, H, X, ctr1, H01) \ - "vpxor (%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vaesenc 16(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vaesenc 32(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vaesenc 48(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vaesenc 64(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vaesenc 80(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vaesenc 96(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vaesenc 112(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vaesenc 128(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vaesenc 144(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "cmpl $11, %[nr]\n\t" \ - "vmovdqa 160(%[KEY]), %%xmm5\n\t" \ - "jl %=f\n\t" \ - "vaesenc %%xmm5, %%xmm4, %%xmm4\n\t" \ - "vaesenc 176(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "cmpl $13, %[nr]\n\t" \ - "vmovdqa 192(%[KEY]), %%xmm5\n\t" \ - "jl %=f\n\t" \ - "vaesenc %%xmm5, %%xmm4, %%xmm4\n\t" \ - "vaesenc 208(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vmovdqa 224(%[KEY]), %%xmm5\n\t" \ - "%=:\n\t" \ - "vaesenclast %%xmm5, %%xmm4, %%xmm4\n\t" \ - "vmovdqu " #in ", %%xmm0\n\t" \ - "vpxor %%xmm0, %%xmm4, %%xmm4\n\t" \ - \ - "vpsrldq $8, " #X ", %%xmm2\n\t" \ - "vpxor " #X ", %%xmm2, %%xmm2\n\t" \ - "vpclmulqdq $0x00, " #H ", " #X ", %%xmm5\n\t" \ - "vpclmulqdq $0x11, " #H ", " #X ", %%xmm8\n\t" \ - "vpclmulqdq $0x00, "#H01", %%xmm2, %%xmm7\n\t" \ - "vpxor %%xmm5, %%xmm7, %%xmm7\n\t" \ - "vpxor %%xmm8, %%xmm7, %%xmm7\n\t" \ - "vpslldq $8, %%xmm7, %%xmm6\n\t" \ - "vpsrldq $8, %%xmm7, %%xmm7\n\t" \ - "vpxor %%xmm7, %%xmm8, %%xmm8\n\t" \ - "vpxor %%xmm5, %%xmm6, %%xmm6\n\t" \ - \ - "vpclmulqdq $0x10, %[MOD2_128], %%xmm6, %%xmm5\n\t" \ - "vpshufd $0x4e, %%xmm6, %%xmm6\n\t" \ - "vpxor %%xmm5, %%xmm6, %%xmm6\n\t" \ - "vpclmulqdq $0x10, %[MOD2_128], %%xmm6, %%xmm5\n\t" \ - "vpshufd $0x4e, %%xmm6, %%xmm6\n\t" \ - "vpxor %%xmm8, %%xmm6, %%xmm6\n\t" \ - "vpxor %%xmm5, %%xmm6, " VAR(XR) "\n\t" -#define VAESENC_GFMUL_AVX2(in, H, X, ctr1) \ - _VAESENC_GFMUL_AVX2(in, H, X, ctr1) - -#define _VAESENC_GFMUL_SB_AVX2(in, H, X, ctr1) \ - "vpclmulqdq $0x10, " #H ", " #X ", %%xmm7\n\t" \ - "vpclmulqdq $0x01, " #H ", " #X ", %%xmm6\n\t" \ - "vpclmulqdq $0x00, " #H ", " #X ", %%xmm5\n\t" \ - "vpclmulqdq $0x11, " #H ", " #X ", %%xmm8\n\t" \ - "vpxor (%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vaesenc 16(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vpxor %%xmm6, %%xmm7, %%xmm7\n\t" \ - "vpslldq $8, %%xmm7, %%xmm6\n\t" \ - "vpsrldq $8, %%xmm7, %%xmm7\n\t" \ - "vaesenc 32(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vpxor %%xmm5, %%xmm6, %%xmm6\n\t" \ - "vpclmulqdq $0x10, %[MOD2_128], %%xmm6, %%xmm5\n\t" \ - "vaesenc 48(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vaesenc 64(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vaesenc 80(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vpshufd $0x4e, %%xmm6, %%xmm6\n\t" \ - "vpxor %%xmm5, %%xmm6, %%xmm6\n\t" \ - "vpclmulqdq $0x10, %[MOD2_128], %%xmm6, %%xmm5\n\t" \ - "vaesenc 96(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vaesenc 112(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vaesenc 128(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vpshufd $0x4e, %%xmm6, %%xmm6\n\t" \ - "vaesenc 144(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vpxor %%xmm7, %%xmm8, %%xmm8\n\t" \ - "vpxor %%xmm8, %%xmm6, %%xmm6\n\t" \ - "cmpl $11, %[nr]\n\t" \ - "vmovdqa 160(%[KEY]), %%xmm3\n\t" \ - "jl %=f\n\t" \ - "vaesenc %%xmm3, %%xmm4, %%xmm4\n\t" \ - "vaesenc 176(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "cmpl $13, %[nr]\n\t" \ - "vmovdqa 192(%[KEY]), %%xmm3\n\t" \ - "jl %=f\n\t" \ - "vaesenc %%xmm3, %%xmm4, %%xmm4\n\t" \ - "vaesenc 208(%[KEY]), %%xmm4, %%xmm4\n\t" \ - "vmovdqa 224(%[KEY]), %%xmm3\n\t" \ - "%=:\n\t" \ - "vaesenclast %%xmm3, %%xmm4, %%xmm4\n\t" \ - "vpxor %%xmm5, %%xmm6, " VAR(XR) "\n\t" \ - "vmovdqu " #in ", %%xmm5\n\t" \ - "vpxor %%xmm5, %%xmm4, %%xmm4\n\t" -#define VAESENC_GFMUL_SB_AVX2(in, H, X, ctr1) \ - _VAESENC_GFMUL_SB_AVX2(in, H, X, ctr1) - - -#define _GHASH_GFMUL_AVX2(r, r2, a, b) \ - "vpclmulqdq $0x10, "#a", "#b", %%xmm2\n\t" \ - "vpclmulqdq $0x01, "#a", "#b", %%xmm1\n\t" \ - "vpclmulqdq $0x00, "#a", "#b", %%xmm0\n\t" \ - "vpclmulqdq $0x11, "#a", "#b", %%xmm3\n\t" \ - "vpxor %%xmm1, %%xmm2, %%xmm2\n\t" \ - "vpslldq $8, %%xmm2, %%xmm1\n\t" \ - "vpsrldq $8, %%xmm2, %%xmm2\n\t" \ - "vpxor %%xmm1, %%xmm0, "#r2"\n\t" \ - "vpxor %%xmm2, %%xmm3, " #r "\n\t" -#define GHASH_GFMUL_AVX2(r, r2, a, b) \ - _GHASH_GFMUL_AVX2(r, r2, a, b) - -#define GHASH_MID_AVX2(r, r2) \ - "vpsrld $31, "#r2", %%xmm0\n\t" \ - "vpsrld $31, " #r ", %%xmm1\n\t" \ - "vpslld $1, "#r2", "#r2"\n\t" \ - "vpslld $1, " #r ", " #r "\n\t" \ - "vpsrldq $12, %%xmm0, %%xmm2\n\t" \ - "vpslldq $4, %%xmm0, %%xmm0\n\t" \ - "vpslldq $4, %%xmm1, %%xmm1\n\t" \ - "vpor %%xmm2, " #r ", " #r "\n\t" \ - "vpor %%xmm0, "#r2", "#r2"\n\t" \ - "vpor %%xmm1, " #r ", " #r "\n\t" - -#define _GHASH_GFMUL_RED_AVX2(r, a, b) \ - "vpclmulqdq $0x10, "#a", "#b", %%xmm7\n\t" \ - "vpclmulqdq $0x01, "#a", "#b", %%xmm6\n\t" \ - "vpclmulqdq $0x00, "#a", "#b", %%xmm5\n\t" \ - "vpxor %%xmm6, %%xmm7, %%xmm7\n\t" \ - "vpslldq $8, %%xmm7, %%xmm6\n\t" \ - "vpsrldq $8, %%xmm7, %%xmm7\n\t" \ - "vpxor %%xmm5, %%xmm6, %%xmm6\n\t" \ - "vpclmulqdq $0x11, "#a", "#b", %%xmm8\n\t" \ - "vpclmulqdq $0x10, %[MOD2_128], %%xmm6, %%xmm5\n\t" \ - "vpshufd $0x4e, %%xmm6, %%xmm6\n\t" \ - "vpxor %%xmm5, %%xmm6, %%xmm6\n\t" \ - "vpclmulqdq $0x10, %[MOD2_128], %%xmm6, %%xmm5\n\t" \ - "vpshufd $0x4e, %%xmm6, %%xmm6\n\t" \ - "vpxor %%xmm7, %%xmm8, %%xmm8\n\t" \ - "vpxor %%xmm8, %%xmm6, %%xmm6\n\t" \ - "vpxor %%xmm5, %%xmm6, " #r "\n\t" -#define GHASH_GFMUL_RED_AVX2(r, a, b) \ - _GHASH_GFMUL_RED_AVX2(r, a, b) - -#define _GHASH_GFSQR_RED2_AVX2(r, a, mod128) \ - "vpclmulqdq $0x00, "#a", "#a", %%xmm6\n\t" \ - "vpclmulqdq $0x11, "#a", "#a", %%xmm8\n\t" \ - "vpclmulqdq $0x10, "#mod128", %%xmm6, %%xmm5\n\t" \ - "vpshufd $0x4e, %%xmm6, %%xmm6\n\t" \ - "vpxor %%xmm5, %%xmm6, %%xmm6\n\t" \ - "vpclmulqdq $0x10, "#mod128", %%xmm6, %%xmm5\n\t" \ - "vpshufd $0x4e, %%xmm6, %%xmm6\n\t" \ - "vpxor %%xmm5, %%xmm6, %%xmm6\n\t" \ - "vpxor %%xmm6, %%xmm8, " #r "\n\t" -#define GHASH_GFSQR_RED2_AVX2(r, a, mod128) \ - _GHASH_GFSQR_RED2_AVX2(r, a, mod128) - -#define _GHASH_GFMUL_SQR_RED2_AVX2(rm, rs, a, b, mod128) \ - "vpclmulqdq $0x10, "#a", "#b", %%xmm7\n\t" \ - "vpclmulqdq $0x01, "#a", "#b", %%xmm6\n\t" \ - "vpclmulqdq $0x00, "#a", "#b", %%xmm5\n\t" \ - "vpclmulqdq $0x11, "#a", "#b", %%xmm8\n\t" \ - "vpclmulqdq $0x00, "#b", "#b", %%xmm9\n\t" \ - "vpclmulqdq $0x11, "#b", "#b", %%xmm10\n\t" \ - "vpxor %%xmm6, %%xmm7, %%xmm7\n\t" \ - "vpslldq $8, %%xmm7, %%xmm6\n\t" \ - "vpsrldq $8, %%xmm7, %%xmm7\n\t" \ - "vpxor %%xmm5, %%xmm6, %%xmm6\n\t" \ - "vpclmulqdq $0x10, "#mod128", %%xmm9, %%xmm4\n\t" \ - "vpclmulqdq $0x10, "#mod128", %%xmm6, %%xmm5\n\t" \ - "vpshufd $0x4e, %%xmm6, %%xmm6\n\t" \ - "vpshufd $0x4e, %%xmm9, %%xmm9\n\t" \ - "vpxor %%xmm5, %%xmm6, %%xmm6\n\t" \ - "vpxor %%xmm4, %%xmm9, %%xmm9\n\t" \ - "vpclmulqdq $0x10, "#mod128", %%xmm6, %%xmm5\n\t" \ - "vpclmulqdq $0x10, "#mod128", %%xmm9, %%xmm4\n\t" \ - "vpshufd $0x4e, %%xmm6, %%xmm6\n\t" \ - "vpshufd $0x4e, %%xmm9, %%xmm9\n\t" \ - "vpxor %%xmm7, %%xmm8, %%xmm8\n\t" \ - "vpxor %%xmm4, %%xmm9, %%xmm9\n\t" \ - "vpxor %%xmm8, %%xmm6, %%xmm6\n\t" \ - "vpxor %%xmm10, %%xmm9, "#rs"\n\t" \ - "vpxor %%xmm5, %%xmm6, "#rm"\n\t" -#define GHASH_GFMUL_SQR_RED2_AVX2(rm, rs, a, b, mod128) \ - _GHASH_GFMUL_SQR_RED2_AVX2(rm, rs, a, b, mod128) - -#define CALC_HT_8_AVX2() \ - "vmovdqa %[MOD2_128], %%xmm11\n\t" \ - "vmovdqa " VAR(XR) ", %%xmm2\n\t" \ - "# H ^ 1 and H ^ 2\n\t" \ - GHASH_GFSQR_RED2_AVX2(%%xmm0, HR, %%xmm11) \ - "vmovdqu " VAR(HR) ", 0(" VAR(HTR) ")\n\t" \ - "vmovdqu %%xmm0 , 16(" VAR(HTR) ")\n\t" \ - "# H ^ 3 and H ^ 4\n\t" \ - GHASH_GFMUL_SQR_RED2_AVX2(%%xmm1, %%xmm3, HR, %%xmm0, %%xmm11) \ - "vmovdqu %%xmm1 , 32(" VAR(HTR) ")\n\t" \ - "vmovdqu %%xmm3 , 48(" VAR(HTR) ")\n\t" \ - "# H ^ 5 and H ^ 6\n\t" \ - GHASH_GFMUL_SQR_RED2_AVX2(%%xmm12, %%xmm0, %%xmm0, %%xmm1, %%xmm11) \ - "vmovdqu %%xmm12, 64(" VAR(HTR) ")\n\t" \ - "vmovdqu %%xmm0 , 80(" VAR(HTR) ")\n\t" \ - "# H ^ 7 and H ^ 8\n\t" \ - GHASH_GFMUL_SQR_RED2_AVX2(%%xmm12, %%xmm0, %%xmm1, %%xmm3, %%xmm11) \ - "vmovdqu %%xmm12, 96(" VAR(HTR) ")\n\t" \ - "vmovdqu %%xmm0 , 112(" VAR(HTR) ")\n\t" - -#define _GHASH_RED_AVX2(r, r2) \ - "vmovdqa %[MOD2_128], %%xmm2\n\t" \ - "vpclmulqdq $0x10, %%xmm2, "#r2", %%xmm0\n\t" \ - "vpshufd $0x4e, "#r2", %%xmm1\n\t" \ - "vpxor %%xmm0, %%xmm1, %%xmm1\n\t" \ - "vpclmulqdq $0x10, %%xmm2, %%xmm1, %%xmm0\n\t" \ - "vpshufd $0x4e, %%xmm1, %%xmm1\n\t" \ - "vpxor %%xmm0, %%xmm1, %%xmm1\n\t" \ - "vpxor %%xmm1, " #r ", " #r "\n\t" -#define GHASH_RED_AVX2(r, r2) \ - _GHASH_RED_AVX2(r, r2) - -#define GHASH_FULL_AVX2(r, r2, a, b) \ - GHASH_GFMUL_AVX2(r, r2, a, b) \ - GHASH_MID_AVX2(r, r2) \ - GHASH_RED_AVX2(r, r2) - -#define _GFMUL_3V_AVX2(r, r2, r3, a, b) \ - "vpclmulqdq $0x10, "#a", "#b", "#r3"\n\t" \ - "vpclmulqdq $0x01, "#a", "#b", %%xmm1\n\t" \ - "vpclmulqdq $0x00, "#a", "#b", "#r2"\n\t" \ - "vpclmulqdq $0x11, "#a", "#b", " #r "\n\t" \ - "vpxor %%xmm1, "#r3", "#r3"\n\t" -#define GFMUL_3V_AVX2(r, r2, r3, a, b) \ - _GFMUL_3V_AVX2(r, r2, r3, a, b) - -#define _GFMUL_XOR_3V_AVX2(r, r2, r3, a, b) \ - "vpclmulqdq $0x10, "#a", "#b", %%xmm2\n\t" \ - "vpclmulqdq $0x01, "#a", "#b", %%xmm1\n\t" \ - "vpclmulqdq $0x00, "#a", "#b", %%xmm0\n\t" \ - "vpclmulqdq $0x11, "#a", "#b", %%xmm3\n\t" \ - "vpxor %%xmm1, %%xmm2, %%xmm2\n\t" \ - "vpxor %%xmm3, " #r ", " #r "\n\t" \ - "vpxor %%xmm2, "#r3", "#r3"\n\t" \ - "vpxor %%xmm0, "#r2", "#r2"\n\t" -#define GFMUL_XOR_3V_AVX2(r, r2, r3, a, b) \ - _GFMUL_XOR_3V_AVX2(r, r2, r3, a, b) - -#define GHASH_GFMUL_RED_8_AVX2() \ - "vmovdqu (" VAR(HTR) "), %%xmm12\n\t" \ - GFMUL_3V_AVX2(XR, %%xmm13, %%xmm14, %%xmm11, %%xmm12) \ - "vmovdqu 16(" VAR(HTR) "), %%xmm12\n\t" \ - GFMUL_XOR_3V_AVX2(XR, %%xmm13, %%xmm14, %%xmm10, %%xmm12) \ - "vmovdqu 32(" VAR(HTR) "), %%xmm11\n\t" \ - "vmovdqu 48(" VAR(HTR) "), %%xmm12\n\t" \ - GFMUL_XOR_3V_AVX2(XR, %%xmm13, %%xmm14, %%xmm9, %%xmm11) \ - GFMUL_XOR_3V_AVX2(XR, %%xmm13, %%xmm14, %%xmm8, %%xmm12) \ - "vmovdqu 64(" VAR(HTR) "), %%xmm11\n\t" \ - "vmovdqu 80(" VAR(HTR) "), %%xmm12\n\t" \ - GFMUL_XOR_3V_AVX2(XR, %%xmm13, %%xmm14, %%xmm7, %%xmm11) \ - GFMUL_XOR_3V_AVX2(XR, %%xmm13, %%xmm14, %%xmm6, %%xmm12) \ - "vmovdqu 96(" VAR(HTR) "), %%xmm11\n\t" \ - "vmovdqu 112(" VAR(HTR) "), %%xmm12\n\t" \ - GFMUL_XOR_3V_AVX2(XR, %%xmm13, %%xmm14, %%xmm5, %%xmm11) \ - GFMUL_XOR_3V_AVX2(XR, %%xmm13, %%xmm14, %%xmm4, %%xmm12) \ - "vpslldq $8, %%xmm14, %%xmm12\n\t" \ - "vpsrldq $8, %%xmm14, %%xmm14\n\t" \ - "vpxor %%xmm12, %%xmm13, %%xmm13\n\t" \ - "vpxor %%xmm14, " VAR(XR) ", " VAR(XR) "\n\t" \ - GHASH_RED_AVX2(XR, %%xmm13) - -#define CALC_IV_12_AVX2() \ - "# Calculate values when IV is 12 bytes\n\t" \ - "# Set counter based on IV\n\t" \ - "movl $0x01000000, %%ecx\n\t" \ - "vpinsrq $0, 0(%%rax), %%xmm13, %%xmm13\n\t" \ - "vpinsrd $2, 8(%%rax), %%xmm13, %%xmm13\n\t" \ - "vpinsrd $3, %%ecx, %%xmm13, %%xmm13\n\t" \ - "# H = Encrypt X(=0) and T = Encrypt counter\n\t" \ - "vmovdqa 0(%[KEY]), " VAR(HR) "\n\t" \ - "vmovdqa 16(%[KEY]), %%xmm12\n\t" \ - "vpxor " VAR(HR) ", %%xmm13, %%xmm1\n\t" \ - "vaesenc %%xmm12, " VAR(HR) ", " VAR(HR) "\n\t" \ - "vaesenc %%xmm12, %%xmm1, %%xmm1\n\t" \ - "vmovdqa 32(%[KEY]), %%xmm0\n\t" \ - "vmovdqa 48(%[KEY]), %%xmm12\n\t" \ - "vaesenc %%xmm0, " VAR(HR) ", " VAR(HR) "\n\t" \ - "vaesenc %%xmm0, %%xmm1, %%xmm1\n\t" \ - "vaesenc %%xmm12, " VAR(HR) ", " VAR(HR) "\n\t" \ - "vaesenc %%xmm12, %%xmm1, %%xmm1\n\t" \ - "vmovdqa 64(%[KEY]), %%xmm0\n\t" \ - "vmovdqa 80(%[KEY]), %%xmm12\n\t" \ - "vaesenc %%xmm0, " VAR(HR) ", " VAR(HR) "\n\t" \ - "vaesenc %%xmm0, %%xmm1, %%xmm1\n\t" \ - "vaesenc %%xmm12, " VAR(HR) ", " VAR(HR) "\n\t" \ - "vaesenc %%xmm12, %%xmm1, %%xmm1\n\t" \ - "vmovdqa 96(%[KEY]), %%xmm0\n\t" \ - "vmovdqa 112(%[KEY]), %%xmm12\n\t" \ - "vaesenc %%xmm0, " VAR(HR) ", " VAR(HR) "\n\t" \ - "vaesenc %%xmm0, %%xmm1, %%xmm1\n\t" \ - "vaesenc %%xmm12, " VAR(HR) ", " VAR(HR) "\n\t" \ - "vaesenc %%xmm12, %%xmm1, %%xmm1\n\t" \ - "vmovdqa 128(%[KEY]), %%xmm0\n\t" \ - "vmovdqa 144(%[KEY]), %%xmm12\n\t" \ - "vaesenc %%xmm0, " VAR(HR) ", " VAR(HR) "\n\t" \ - "vaesenc %%xmm0, %%xmm1, %%xmm1\n\t" \ - "vaesenc %%xmm12, " VAR(HR) ", " VAR(HR) "\n\t" \ - "vaesenc %%xmm12, %%xmm1, %%xmm1\n\t" \ - "cmpl $11, %[nr]\n\t" \ - "vmovdqa 160(%[KEY]), %%xmm0\n\t" \ - "jl 31f\n\t" \ - "vmovdqa 176(%[KEY]), %%xmm12\n\t" \ - "vaesenc %%xmm0, " VAR(HR) ", " VAR(HR) "\n\t" \ - "vaesenc %%xmm0, %%xmm1, %%xmm1\n\t" \ - "vaesenc %%xmm12, " VAR(HR) ", " VAR(HR) "\n\t" \ - "vaesenc %%xmm12, %%xmm1, %%xmm1\n\t" \ - "cmpl $13, %[nr]\n\t" \ - "vmovdqa 192(%[KEY]), %%xmm0\n\t" \ - "jl 31f\n\t" \ - "vmovdqa 208(%[KEY]), %%xmm12\n\t" \ - "vaesenc %%xmm0, " VAR(HR) ", " VAR(HR) "\n\t" \ - "vaesenc %%xmm0, %%xmm1, %%xmm1\n\t" \ - "vaesenc %%xmm12, " VAR(HR) ", " VAR(HR) "\n\t" \ - "vaesenc %%xmm12, %%xmm1, %%xmm1\n\t" \ - "vmovdqu 224(%[KEY]), %%xmm0\n\t" \ - "31:\n\t" \ - "vaesenclast %%xmm0, " VAR(HR) ", " VAR(HR) "\n\t" \ - "vaesenclast %%xmm0, %%xmm1, %%xmm1\n\t" \ - "vpshufb %[BSWAP_MASK], " VAR(HR) ", " VAR(HR) "\n\t" \ - "vmovdqu %%xmm1, " VAR(TR) "\n\t" \ - -#define CALC_IV_AVX2() \ - "# Calculate values when IV is not 12 bytes\n\t" \ - "# H = Encrypt X(=0)\n\t" \ - "vmovdqa 0(%[KEY]), " VAR(HR) "\n\t" \ - VAESENC_AVX(HR) \ - "vpshufb %[BSWAP_MASK], " VAR(HR) ", " VAR(HR) "\n\t" \ - "# Calc counter\n\t" \ - "# Initialization vector\n\t" \ - "cmpl $0, %%edx\n\t" \ - "movq $0, %%rcx\n\t" \ - "je 45f\n\t" \ - "cmpl $16, %%edx\n\t" \ - "jl 44f\n\t" \ - "andl $0xfffffff0, %%edx\n\t" \ - "\n" \ - "43:\n\t" \ - "vmovdqu (%%rax,%%rcx,1), %%xmm4\n\t" \ - "vpshufb %[BSWAP_MASK], %%xmm4, %%xmm4\n\t" \ - "vpxor %%xmm4, %%xmm13, %%xmm13\n\t" \ - GHASH_FULL_AVX2(%%xmm13, %%xmm12, %%xmm13, HR) \ - "addl $16, %%ecx\n\t" \ - "cmpl %%edx, %%ecx\n\t" \ - "jl 43b\n\t" \ - "movl %[ibytes], %%edx\n\t" \ - "cmpl %%edx, %%ecx\n\t" \ - "je 45f\n\t" \ - "\n" \ - "44:\n\t" \ - "subq $16, %%rsp\n\t" \ - "vpxor %%xmm4, %%xmm4, %%xmm4\n\t" \ - "xorl %%ebx, %%ebx\n\t" \ - "vmovdqu %%xmm4, (%%rsp)\n\t" \ - "42:\n\t" \ - "movzbl (%%rax,%%rcx,1), %%r13d\n\t" \ - "movb %%r13b, (%%rsp,%%rbx,1)\n\t" \ - "incl %%ecx\n\t" \ - "incl %%ebx\n\t" \ - "cmpl %%edx, %%ecx\n\t" \ - "jl 42b\n\t" \ - "vmovdqu (%%rsp), %%xmm4\n\t" \ - "addq $16, %%rsp\n\t" \ - "vpshufb %[BSWAP_MASK], %%xmm4, %%xmm4\n\t" \ - "vpxor %%xmm4, %%xmm13, %%xmm13\n\t" \ - GHASH_FULL_AVX2(%%xmm13, %%xmm12, %%xmm13, HR) \ - "\n" \ - "45:\n\t" \ - "# T = Encrypt counter\n\t" \ - "vpxor %%xmm0, %%xmm0, %%xmm0\n\t" \ - "shll $3, %%edx\n\t" \ - "vpinsrq $0, %%rdx, %%xmm0, %%xmm0\n\t" \ - "vpxor %%xmm0, %%xmm13, %%xmm13\n\t" \ - GHASH_FULL_AVX2(%%xmm13, %%xmm12, %%xmm13, HR) \ - "vpshufb %[BSWAP_MASK], %%xmm13, %%xmm13\n\t" \ - "# Encrypt counter\n\t" \ - "vmovdqa 0(%[KEY]), %%xmm4\n\t" \ - "vpxor %%xmm13, %%xmm4, %%xmm4\n\t" \ - VAESENC_AVX(%%xmm4) \ - "vmovdqu %%xmm4, " VAR(TR) "\n\t" - -#define CALC_AAD_AVX2() \ - "# Additional authentication data\n\t" \ - "movl %[abytes], %%edx\n\t" \ - "cmpl $0, %%edx\n\t" \ - "je 25f\n\t" \ - "movq %[addt], %%rax\n\t" \ - "xorl %%ecx, %%ecx\n\t" \ - "cmpl $16, %%edx\n\t" \ - "jl 24f\n\t" \ - "andl $0xfffffff0, %%edx\n\t" \ - "\n" \ - "23:\n\t" \ - "vmovdqu (%%rax,%%rcx,1), %%xmm4\n\t" \ - "vpshufb %[BSWAP_MASK], %%xmm4, %%xmm4\n\t" \ - "vpxor %%xmm4, " VAR(XR) ", " VAR(XR) "\n\t" \ - GHASH_FULL_AVX2(XR, %%xmm12, XR, HR) \ - "addl $16, %%ecx\n\t" \ - "cmpl %%edx, %%ecx\n\t" \ - "jl 23b\n\t" \ - "movl %[abytes], %%edx\n\t" \ - "cmpl %%edx, %%ecx\n\t" \ - "je 25f\n\t" \ - "\n" \ - "24:\n\t" \ - "subq $16, %%rsp\n\t" \ - "vpxor %%xmm4, %%xmm4, %%xmm4\n\t" \ - "xorl %%ebx, %%ebx\n\t" \ - "vmovdqu %%xmm4, (%%rsp)\n\t" \ - "22:\n\t" \ - "movzbl (%%rax,%%rcx,1), %%r13d\n\t" \ - "movb %%r13b, (%%rsp,%%rbx,1)\n\t" \ - "incl %%ecx\n\t" \ - "incl %%ebx\n\t" \ - "cmpl %%edx, %%ecx\n\t" \ - "jl 22b\n\t" \ - "vmovdqu (%%rsp), %%xmm4\n\t" \ - "addq $16, %%rsp\n\t" \ - "vpshufb %[BSWAP_MASK], %%xmm4, %%xmm4\n\t" \ - "vpxor %%xmm4, " VAR(XR) ", " VAR(XR) "\n\t" \ - GHASH_FULL_AVX2(XR, %%xmm12, XR, HR) \ - "\n" \ - "25:\n\t" - -#define VAESENC_128_GHASH_AVX2(src, o) \ - "leaq (%[in]," VAR(KR64) ",1), %%rcx\n\t" \ - "leaq (%[out]," VAR(KR64) ",1), %%rdx\n\t" \ - /* src is either %%rcx or %%rdx */ \ - VAESENC_CTR() \ - VAESENC_XOR() \ - VAESENC_PCLMUL_AVX2_1(src, 16, (o-128), 112) \ - VAESENC_PCLMUL_AVX2_2(src, 32, (o-112), 96) \ - VAESENC_PCLMUL_AVX2_N(src, 48, (o- 96), 80) \ - VAESENC_PCLMUL_AVX2_N(src, 64, (o- 80), 64) \ - VAESENC_PCLMUL_AVX2_N(src, 80, (o- 64), 48) \ - VAESENC_PCLMUL_AVX2_N(src, 96, (o- 48), 32) \ - VAESENC_PCLMUL_AVX2_N(src, 112, (o- 32), 16) \ - VAESENC_PCLMUL_AVX2_N(src, 128, (o- 16), 0) \ - VAESENC_PCLMUL_AVX2_L(144) \ - "cmpl $11, %[nr]\n\t" \ - "vmovdqa 160(%[KEY]), %%xmm12\n\t" \ - "jl 4f\n\t" \ - VAESENC() \ - VAESENC_SET(176) \ - "cmpl $13, %[nr]\n\t" \ - "vmovdqa 192(%[KEY]), %%xmm12\n\t" \ - "jl 4f\n\t" \ - VAESENC() \ - VAESENC_SET(208) \ - "vmovdqa 224(%[KEY]), %%xmm12\n\t" \ - "\n" \ -"4:\n\t" \ - VAESENC_LAST(%%rcx, %%rdx) - -#define AESENC_LAST15_ENC_AVX2() \ - "movl %[nbytes], %%ecx\n\t" \ - "movl %%ecx, %%edx\n\t" \ - "andl $0x0f, %%ecx\n\t" \ - "jz 55f\n\t" \ - "vmovdqu " VAR(CTR1) ", %%xmm13\n\t" \ - "vpshufb %[BSWAP_EPI64], %%xmm13, %%xmm13\n\t" \ - "vpxor 0(%[KEY]), %%xmm13, %%xmm13\n\t" \ - VAESENC_AVX(%%xmm13) \ - "subq $16, %%rsp\n\t" \ - "xorl %%ecx, %%ecx\n\t" \ - "vmovdqu %%xmm13, (%%rsp)\n\t" \ - "\n" \ - "51:\n\t" \ - "movzbl (%[in]," VAR(KR64) ",1), %%r13d\n\t" \ - "xorb (%%rsp,%%rcx,1), %%r13b\n\t" \ - "movb %%r13b, (%[out]," VAR(KR64) ",1)\n\t" \ - "movb %%r13b, (%%rsp,%%rcx,1)\n\t" \ - "incl " VAR(KR) "\n\t" \ - "incl %%ecx\n\t" \ - "cmpl %%edx, " VAR(KR) "\n\t" \ - "jl 51b\n\t" \ - "xorq %%r13, %%r13\n\t" \ - "cmpl $16, %%ecx\n\t" \ - "je 53f\n\t" \ - "\n" \ - "52:\n\t" \ - "movb %%r13b, (%%rsp,%%rcx,1)\n\t" \ - "incl %%ecx\n\t" \ - "cmpl $16, %%ecx\n\t" \ - "jl 52b\n\t" \ - "53:\n\t" \ - "vmovdqu (%%rsp), %%xmm13\n\t" \ - "addq $16, %%rsp\n\t" \ - "vpshufb %[BSWAP_MASK], %%xmm13, %%xmm13\n\t" \ - "vpxor %%xmm13, " VAR(XR) ", " VAR(XR) "\n\t" \ - GHASH_GFMUL_RED_AVX2(XR, HR, XR) \ - -#define AESENC_LAST15_DEC_AVX2() \ - "movl %[nbytes], %%ecx\n\t" \ - "movl %%ecx, %%edx\n\t" \ - "andl $0x0f, %%ecx\n\t" \ - "jz 55f\n\t" \ - "vmovdqu " VAR(CTR1) ", %%xmm13\n\t" \ - "vpshufb %[BSWAP_EPI64], %%xmm13, %%xmm13\n\t" \ - "vpxor 0(%[KEY]), %%xmm13, %%xmm13\n\t" \ - VAESENC_AVX(%%xmm13) \ - "subq $32, %%rsp\n\t" \ - "xorl %%ecx, %%ecx\n\t" \ - "vmovdqu %%xmm13, (%%rsp)\n\t" \ - "vpxor %%xmm0, %%xmm0, %%xmm0\n\t" \ - "vmovdqu %%xmm0, 16(%%rsp)\n\t" \ - "\n" \ - "51:\n\t" \ - "movzbl (%[in]," VAR(KR64) ",1), %%r13d\n\t" \ - "movb %%r13b, 16(%%rsp,%%rcx,1)\n\t" \ - "xorb (%%rsp,%%rcx,1), %%r13b\n\t" \ - "movb %%r13b, (%[out]," VAR(KR64) ",1)\n\t" \ - "incl " VAR(KR) "\n\t" \ - "incl %%ecx\n\t" \ - "cmpl %%edx, " VAR(KR) "\n\t" \ - "jl 51b\n\t" \ - "53:\n\t" \ - "vmovdqu 16(%%rsp), %%xmm13\n\t" \ - "addq $32, %%rsp\n\t" \ - "vpshufb %[BSWAP_MASK], %%xmm13, %%xmm13\n\t" \ - "vpxor %%xmm13, " VAR(XR) ", " VAR(XR) "\n\t" \ - GHASH_GFMUL_RED_AVX2(XR, HR, XR) \ - -#define CALC_TAG_AVX2() \ - "movl %[nbytes], %%edx\n\t" \ - "movl %[abytes], %%ecx\n\t" \ - "shlq $3, %%rdx\n\t" \ - "shlq $3, %%rcx\n\t" \ - "vpinsrq $0, %%rdx, %%xmm0, %%xmm0\n\t" \ - "vpinsrq $1, %%rcx, %%xmm0, %%xmm0\n\t" \ - "vpxor %%xmm0, " VAR(XR) ", " VAR(XR) "\n\t" \ - GHASH_GFMUL_RED_AVX2(XR, HR, XR) \ - "vpshufb %[BSWAP_MASK], " VAR(XR) ", " VAR(XR) "\n\t" \ - "vpxor " VAR(TR) ", " VAR(XR) ", %%xmm0\n\t" \ - - -static void AES_GCM_encrypt_avx2(const unsigned char *in, unsigned char *out, - const unsigned char* addt, - const unsigned char* ivec, unsigned char *tag, - unsigned int nbytes, unsigned int abytes, - unsigned int ibytes, unsigned int tbytes, - const unsigned char* key, int nr) -{ - register const unsigned char* iv asm("rax") = ivec; - register unsigned int ivLen asm("ebx") = ibytes; - - __asm__ __volatile__ ( - "subq $" VAR(STACK_OFFSET) ", %%rsp\n\t" - /* Counter is xmm13 */ - "vpxor %%xmm13, %%xmm13, %%xmm13\n\t" - "vpxor " VAR(XR) ", " VAR(XR) ", " VAR(XR) "\n\t" - "movl %[ibytes], %%edx\n\t" - "cmpl $12, %%edx\n\t" - "jne 35f\n\t" - CALC_IV_12_AVX2() - "jmp 39f\n\t" - "\n" - "35:\n\t" - CALC_IV_AVX2() - "\n" - "39:\n\t" - - CALC_AAD_AVX2() - - "# Calculate counter and H\n\t" - "vpsrlq $63, " VAR(HR) ", %%xmm5\n\t" - "vpsllq $1, " VAR(HR) ", %%xmm4\n\t" - "vpslldq $8, %%xmm5, %%xmm5\n\t" - "vpor %%xmm5, %%xmm4, %%xmm4\n\t" - "vpshufd $0xff, " VAR(HR) ", " VAR(HR) "\n\t" - "vpsrad $31, " VAR(HR) ", " VAR(HR) "\n\t" - "vpshufb %[BSWAP_EPI64], %%xmm13, %%xmm13\n\t" - "vpand %[MOD2_128], " VAR(HR) ", " VAR(HR) "\n\t" - "vpaddd %[ONE], %%xmm13, %%xmm13\n\t" - "vpxor %%xmm4, " VAR(HR) ", " VAR(HR) "\n\t" - "vmovdqu %%xmm13, " VAR(CTR1) "\n\t" - - "xorl " VAR(KR) ", " VAR(KR) "\n\t" - -#if !defined(AES_GCM_AESNI_NO_UNROLL) && !defined(AES_GCM_AVX2_NO_UNROLL) - "cmpl $128, %[nbytes]\n\t" - "movl %[nbytes], %%r13d\n\t" - "jl 5f\n\t" - "andl $0xffffff80, %%r13d\n\t" - - CALC_HT_8_AVX2() - - "# First 128 bytes of input\n\t" - VAESENC_128() - - "cmpl $128, %%r13d\n\t" - "movl $128, " VAR(KR) "\n\t" - "jle 2f\n\t" - - "# More 128 bytes of input\n\t" - "\n" - "3:\n\t" - VAESENC_128_GHASH_AVX2(%%rdx, 0) - "addl $128, " VAR(KR) "\n\t" - "cmpl %%r13d, " VAR(KR) "\n\t" - "jl 3b\n\t" - "\n" - "2:\n\t" - "vmovdqa %[BSWAP_MASK], %%xmm13\n\t" - "vpshufb %%xmm13, %%xmm4, %%xmm4\n\t" - "vpshufb %%xmm13, %%xmm5, %%xmm5\n\t" - "vpshufb %%xmm13, %%xmm6, %%xmm6\n\t" - "vpshufb %%xmm13, %%xmm7, %%xmm7\n\t" - "vpshufb %%xmm13, %%xmm8, %%xmm8\n\t" - "vpshufb %%xmm13, %%xmm9, %%xmm9\n\t" - "vpshufb %%xmm13, %%xmm10, %%xmm10\n\t" - "vpshufb %%xmm13, %%xmm11, %%xmm11\n\t" - "vpxor %%xmm2, %%xmm4, %%xmm4\n\t" - - GHASH_GFMUL_RED_8_AVX2() - - "vmovdqu 0(" VAR(HTR) "), " VAR(HR) "\n\t" - "\n" - "5:\n\t" - "movl %[nbytes], %%edx\n\t" - "cmpl %%edx, " VAR(KR) "\n\t" - "jge 55f\n\t" -#endif - - "movl %[nbytes], %%r13d\n\t" - "andl $0xfffffff0, %%r13d\n\t" - "cmpl %%r13d, " VAR(KR) "\n\t" - "jge 14f\n\t" - - VAESENC_BLOCK_AVX2() - "addl $16, " VAR(KR) "\n\t" - "cmpl %%r13d, " VAR(KR) "\n\t" - "jge 13f\n\t" - "vmovdqa %[MOD2_128], %%xmm0\n\t" - "\n" - "12:\n\t" - "vmovdqu (%[in]," VAR(KR64) ",1), %%xmm9\n\t" - "vmovdqu " VAR(CTR1) ", %%xmm5\n\t" - "vpshufb %[BSWAP_EPI64], %%xmm5, %%xmm4\n\t" - "vpaddd %[ONE], %%xmm5, %%xmm5\n\t" - "vmovdqu %%xmm5, " VAR(CTR1) "\n\t" - VAESENC_GFMUL_SB_AVX2(%%xmm9, HR, XR, CTR1) - "vmovdqu %%xmm4, (%[out]," VAR(KR64) ",1)\n\t" - "vpshufb %[BSWAP_MASK], %%xmm4, %%xmm4\n\t" - "addl $16, " VAR(KR) "\n\t" - "vpxor %%xmm4, " VAR(XR) ", " VAR(XR) "\n\t" - "cmpl %%r13d, " VAR(KR) "\n\t" - "jl 12b\n\t" - "\n" - "13:\n\t" - GHASH_GFMUL_RED_AVX2(XR, HR, XR) - "\n" - "14:\n\t" - - AESENC_LAST15_ENC_AVX2() - "\n" - "55:\n\t" - - CALC_TAG_AVX2() - STORE_TAG_AVX() - "addq $" VAR(STACK_OFFSET) ", %%rsp\n\t" - "vzeroupper\n\t" - - : - : [KEY] "r" (key), - [in] "r" (in), [out] "r" (out), [nr] "r" (nr), - [nbytes] "r" (nbytes), [abytes] "r" (abytes), [addt] "r" (addt), - [ivec] "r" (iv), [ibytes] "r" (ivLen), [tbytes] "r" (tbytes), - [tag] "r" (tag), - [BSWAP_MASK] "m" (BSWAP_MASK), - [BSWAP_EPI64] "m" (BSWAP_EPI64), - [ONE] "m" (ONE), -#if !defined(AES_GCM_AESNI_NO_UNROLL) && !defined(AES_GCM_AVX2_NO_UNROLL) - [TWO] "m" (TWO), [THREE] "m" (THREE), [FOUR] "m" (FOUR), - [FIVE] "m" (FIVE), [SIX] "m" (SIX), [SEVEN] "m" (SEVEN), - [EIGHT] "m" (EIGHT), -#endif - [MOD2_128] "m" (MOD2_128) - : "xmm15", "xmm14", "xmm13", "xmm12", - "xmm0", "xmm1", "xmm2", "xmm3", "memory", - "xmm4", "xmm5", "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", - "rcx", "rdx", "r13" - ); -} +void AES_GCM_encrypt_avx2(const unsigned char *in, unsigned char *out, + const unsigned char* addt, const unsigned char* ivec, + unsigned char *tag, unsigned int nbytes, + unsigned int abytes, unsigned int ibytes, + unsigned int tbytes, const unsigned char* key, + int nr) + XASM_LINK("AES_GCM_encrypt_avx2"); #endif /* HAVE_INTEL_AVX2 */ #endif /* HAVE_INTEL_AVX1 */ #ifdef HAVE_AES_DECRYPT -/* Figure 10. AES-GCM – Decrypt With Single Block Ghash at a Time */ - -static void AES_GCM_decrypt(const unsigned char *in, unsigned char *out, - const unsigned char* addt, - const unsigned char* ivec, const unsigned char *tag, - int nbytes, int abytes, int ibytes, int tbytes, - const unsigned char* key, int nr, int* res) -{ - register const unsigned char* iv asm("rax") = ivec; - register int ivLen asm("ebx") = ibytes; - register int tagLen asm("edx") = tbytes; - - __asm__ __volatile__ ( - "pushq %%rdx\n\t" - "subq $" VAR(STACK_OFFSET) ", %%rsp\n\t" - /* Counter is xmm13 */ - "pxor %%xmm13, %%xmm13\n\t" - "pxor %%xmm15, %%xmm15\n\t" - "movl %[ibytes], %%edx\n\t" - "cmpl $12, %%edx\n\t" - "jne 35f\n\t" - CALC_IV_12() - "\n" - "35:\n\t" - CALC_IV() - "\n" - "39:\n\t" - - CALC_AAD() - - "# Calculate counter and H\n\t" - "pshufb %[BSWAP_EPI64], %%xmm13\n\t" - "movdqa " VAR(HR) ", %%xmm5\n\t" - "paddd %[ONE], %%xmm13\n\t" - "movdqa " VAR(HR) ", %%xmm4\n\t" - "movdqu %%xmm13, " VAR(CTR1) "\n\t" - "psrlq $63, %%xmm5\n\t" - "psllq $1, %%xmm4\n\t" - "pslldq $8, %%xmm5\n\t" - "por %%xmm5, %%xmm4\n\t" - "pshufd $0xff, " VAR(HR) ", " VAR(HR) "\n\t" - "psrad $31, " VAR(HR) "\n\t" - "pand %[MOD2_128], " VAR(HR) "\n\t" - "pxor %%xmm4, " VAR(HR) "\n\t" - - "xorl " VAR(KR) ", " VAR(KR) "\n\t" - -#if !defined(AES_GCM_AESNI_NO_UNROLL) && !defined(AES_GCM_AVX1_NO_UNROLL) - "cmpl $128, %[nbytes]\n\t" - "jl 5f\n\t" - - CALC_HT_8_AVX() - - "movl %[nbytes], %%r13d\n\t" - "andl $0xffffff80, %%r13d\n\t" - "\n" - "2:\n\t" - AESENC_128_GHASH_AVX(%%rcx, 128) - "addl $128, " VAR(KR) "\n\t" - "cmpl %%r13d, " VAR(KR) "\n\t" - "jl 2b\n\t" - - "movdqa %%xmm2, " VAR(XR) "\n\t" - "movdqu (%%rsp), " VAR(HR) "\n\t" - "5:\n\t" - "movl %[nbytes], %%edx\n\t" - "cmpl %%edx, " VAR(KR) "\n\t" - "jge 55f\n\t" -#endif - "movl %[nbytes], %%r13d\n\t" - "andl $0xfffffff0, %%r13d\n\t" - "cmpl %%r13d, " VAR(KR) "\n\t" - "jge 13f\n\t" - - "\n" - "12:\n\t" - "leaq (%[in]," VAR(KR64) ",1), %%rcx\n\t" - "leaq (%[out]," VAR(KR64) ",1), %%rdx\n\t" - "movdqu (%%rcx), %%xmm1\n\t" - "movdqa " VAR(HR) ", %%xmm0\n\t" - "pshufb %[BSWAP_MASK], %%xmm1\n\t" - "pxor " VAR(XR) ", %%xmm1\n\t" - AESENC_GFMUL(%%rcx, %%rdx, %%xmm0, %%xmm1) - "addl $16, " VAR(KR) "\n\t" - "cmpl %%r13d, " VAR(KR) "\n\t" - "jl 12b\n\t" - "\n" - "13:\n\t" - - AESENC_LAST15_DEC_AVX() - "\n" - "55:\n\t" - - CALC_TAG() - "addq $" VAR(STACK_OFFSET) ", %%rsp\n\t" - "popq %%rdx\n\t" - CMP_TAG() - - : - : [KEY] "r" (key), - [in] "r" (in), [out] "r" (out), [nr] "r" (nr), - [nbytes] "r" (nbytes), [abytes] "r" (abytes), [addt] "r" (addt), - [ivec] "r" (iv), [ibytes] "r" (ivLen), [tbytes] "r" (tagLen), - [tag] "r" (tag), [res] "r" (res), - [BSWAP_MASK] "m" (BSWAP_MASK), - [BSWAP_EPI64] "m" (BSWAP_EPI64), - [ONE] "m" (ONE), -#if !defined(AES_GCM_AESNI_NO_UNROLL) && !defined(AES_GCM_AVX1_NO_UNROLL) - [TWO] "m" (TWO), [THREE] "m" (THREE), [FOUR] "m" (FOUR), - [FIVE] "m" (FIVE), [SIX] "m" (SIX), [SEVEN] "m" (SEVEN), - [EIGHT] "m" (EIGHT), -#endif - [MOD2_128] "m" (MOD2_128) - : "xmm15", "xmm14", "xmm13", "xmm12", - "xmm0", "xmm1", "xmm2", "xmm3", "memory", - "xmm4", "xmm5", "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", - "rcx", "r13" - ); -} - +void AES_GCM_decrypt(const unsigned char *in, unsigned char *out, + const unsigned char* addt, const unsigned char* ivec, + const unsigned char *tag, int nbytes, int abytes, + int ibytes, int tbytes, const unsigned char* key, int nr, + int* res) + XASM_LINK("AES_GCM_decrypt"); #ifdef HAVE_INTEL_AVX1 -static void AES_GCM_decrypt_avx1(const unsigned char *in, unsigned char *out, - const unsigned char* addt, - const unsigned char* ivec, - const unsigned char *tag, int nbytes, - int abytes, int ibytes, int tbytes, - const unsigned char* key, int nr, int* res) -{ - register const unsigned char* iv asm("rax") = ivec; - register int ivLen asm("ebx") = ibytes; - register int tagLen asm("edx") = tbytes; - - __asm__ __volatile__ ( - "pushq %%rdx\n\t" - "subq $" VAR(STACK_OFFSET) ", %%rsp\n\t" - /* Counter is xmm13 */ - "vpxor %%xmm13, %%xmm13, %%xmm13\n\t" - "vpxor %%xmm15, %%xmm15, %%xmm15\n\t" - "movl %[ibytes], %%edx\n\t" - "cmpl $12, %%edx\n\t" - "jne 35f\n\t" - CALC_IV_12_AVX1() - "\n" - "35:\n\t" - CALC_IV_AVX1() - "\n" - "39:\n\t" - - CALC_AAD_AVX1() - - "# Calculate counter and H\n\t" - "vpsrlq $63, " VAR(HR) ", %%xmm5\n\t" - "vpsllq $1, " VAR(HR) ", %%xmm4\n\t" - "vpslldq $8, %%xmm5, %%xmm5\n\t" - "vpor %%xmm5, %%xmm4, %%xmm4\n\t" - "vpshufd $0xff, " VAR(HR) ", " VAR(HR) "\n\t" - "vpsrad $31, " VAR(HR) ", " VAR(HR) "\n\t" - "vpshufb %[BSWAP_EPI64], %%xmm13, %%xmm13\n\t" - "vpand %[MOD2_128], " VAR(HR) ", " VAR(HR) "\n\t" - "vpaddd %[ONE], %%xmm13, %%xmm13\n\t" - "vpxor %%xmm4, " VAR(HR) ", " VAR(HR) "\n\t" - "vmovdqu %%xmm13, " VAR(CTR1) "\n\t" - - "xorl " VAR(KR) ", " VAR(KR) "\n\t" - -#if !defined(AES_GCM_AESNI_NO_UNROLL) && !defined(AES_GCM_AVX1_NO_UNROLL) - "cmpl $128, %[nbytes]\n\t" - "jl 5f\n\t" - - CALC_HT_8_AVX1() - - "movl %[nbytes], %%r13d\n\t" - "andl $0xffffff80, %%r13d\n\t" - "\n" - "2:\n\t" - VAESENC_128_GHASH_AVX1(%%rcx, 128) - "addl $128, " VAR(KR) "\n\t" - "cmpl %%r13d, " VAR(KR) "\n\t" - "jl 2b\n\t" - - "vmovdqa %%xmm2, " VAR(XR) "\n\t" - "vmovdqu (%%rsp), " VAR(HR) "\n\t" - "5:\n\t" - "movl %[nbytes], %%edx\n\t" - "cmpl %%edx, " VAR(KR) "\n\t" - "jge 55f\n\t" -#endif - "movl %[nbytes], %%r13d\n\t" - "andl $0xfffffff0, %%r13d\n\t" - "cmpl %%r13d, " VAR(KR) "\n\t" - "jge 13f\n\t" - - "\n" - "12:\n\t" - "vmovdqu (%[in]," VAR(KR64) ",1), %%xmm9\n\t" - "vmovdqa " VAR(HR) ", %%xmm0\n\t" - "vpshufb %[BSWAP_MASK], %%xmm9, %%xmm1\n\t" - "vpxor " VAR(XR) ", %%xmm1, %%xmm1\n\t" - VAESENC_GFMUL(%%xmm9, %%xmm0, %%xmm1) - "addl $16, " VAR(KR) "\n\t" - "cmpl %%r13d, " VAR(KR) "\n\t" - "jl 12b\n\t" - "\n" - "13:\n\t" - - AESENC_LAST15_DEC_AVX1() - "\n" - "55:\n\t" - - CALC_TAG_AVX1() - "addq $" VAR(STACK_OFFSET) ", %%rsp\n\t" - "popq %%rdx\n\t" - CMP_TAG_AVX() - "vzeroupper\n\t" - - : - : [KEY] "r" (key), - [in] "r" (in), [out] "r" (out), [nr] "r" (nr), - [nbytes] "r" (nbytes), [abytes] "r" (abytes), [addt] "r" (addt), - [ivec] "r" (iv), [ibytes] "r" (ivLen), [tbytes] "r" (tagLen), - [tag] "r" (tag), [res] "r" (res), - [BSWAP_MASK] "m" (BSWAP_MASK), - [BSWAP_EPI64] "m" (BSWAP_EPI64), - [ONE] "m" (ONE), -#if !defined(AES_GCM_AESNI_NO_UNROLL) && !defined(AES_GCM_AVX1_NO_UNROLL) - [TWO] "m" (TWO), [THREE] "m" (THREE), [FOUR] "m" (FOUR), - [FIVE] "m" (FIVE), [SIX] "m" (SIX), [SEVEN] "m" (SEVEN), - [EIGHT] "m" (EIGHT), -#endif - [MOD2_128] "m" (MOD2_128) - : "xmm15", "xmm14", "xmm13", "xmm12", - "xmm0", "xmm1", "xmm2", "xmm3", "memory", - "xmm4", "xmm5", "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", - "rcx", "r13" - ); -} - +void AES_GCM_decrypt_avx1(const unsigned char *in, unsigned char *out, + const unsigned char* addt, const unsigned char* ivec, + const unsigned char *tag, int nbytes, int abytes, + int ibytes, int tbytes, const unsigned char* key, + int nr, int* res) + XASM_LINK("AES_GCM_decrypt_avx1"); #ifdef HAVE_INTEL_AVX2 -static void AES_GCM_decrypt_avx2(const unsigned char *in, unsigned char *out, - const unsigned char* addt, - const unsigned char* ivec, - const unsigned char *tag, int nbytes, - int abytes, int ibytes, int tbytes, - const unsigned char* key, int nr, int* res) -{ - register const unsigned char* iv asm("rax") = ivec; - register int ivLen asm("ebx") = ibytes; - register int tagLen asm("edx") = tbytes; - - __asm__ __volatile__ ( - "pushq %%rdx\n\t" - "subq $" VAR(STACK_OFFSET) ", %%rsp\n\t" - /* Counter is xmm13 */ - "vpxor %%xmm13, %%xmm13, %%xmm13\n\t" - "vpxor %%xmm15, %%xmm15, %%xmm15\n\t" - "movl %[ibytes], %%edx\n\t" - "cmpl $12, %%edx\n\t" - "jne 35f\n\t" - CALC_IV_12_AVX2() - "jmp 39f\n\t" - "\n" - "35:\n\t" - CALC_IV_AVX2() - "\n" - "39:\n\t" - - CALC_AAD_AVX2() - - "# Calculate counter and H\n\t" - "vpsrlq $63, " VAR(HR) ", %%xmm5\n\t" - "vpsllq $1, " VAR(HR) ", %%xmm4\n\t" - "vpslldq $8, %%xmm5, %%xmm5\n\t" - "vpor %%xmm5, %%xmm4, %%xmm4\n\t" - "vpshufd $0xff, " VAR(HR) ", " VAR(HR) "\n\t" - "vpsrad $31, " VAR(HR) ", " VAR(HR) "\n\t" - "vpshufb %[BSWAP_EPI64], %%xmm13, %%xmm13\n\t" - "vpand %[MOD2_128], " VAR(HR) ", " VAR(HR) "\n\t" - "vpaddd %[ONE], %%xmm13, %%xmm13\n\t" - "vpxor %%xmm4, " VAR(HR) ", " VAR(HR) "\n\t" - "vmovdqu %%xmm13, " VAR(CTR1) "\n\t" - - "xorl " VAR(KR) ", " VAR(KR) "\n\t" - -#if !defined(AES_GCM_AESNI_NO_UNROLL) && !defined(AES_GCM_AVX2_NO_UNROLL) - "cmpl $128, %[nbytes]\n\t" - "jl 5f\n\t" - - CALC_HT_8_AVX2() - - "movl %[nbytes], %%r13d\n\t" - "andl $0xffffff80, %%r13d\n\t" - "\n" - "2:\n\t" - VAESENC_128_GHASH_AVX2(%%rcx, 128) - "addl $128, " VAR(KR) "\n\t" - "cmpl %%r13d, " VAR(KR) "\n\t" - "jl 2b\n\t" - - "vmovdqa %%xmm2, " VAR(XR) "\n\t" - "vmovdqu (%%rsp), " VAR(HR) "\n\t" - "5:\n\t" - "movl %[nbytes], %%edx\n\t" - "cmpl %%edx, " VAR(KR) "\n\t" - "jge 55f\n\t" -#endif - "movl %[nbytes], %%r13d\n\t" - "andl $0xfffffff0, %%r13d\n\t" - "cmpl %%r13d, " VAR(KR) "\n\t" - "jge 13f\n\t" - - "vmovdqa %[MOD2_128], %%xmm0\n\t" - "\n" - "12:\n\t" - "vmovdqu (%[in]," VAR(KR64) ",1), %%xmm9\n\t" - "vmovdqu " VAR(CTR1) ", %%xmm5\n\t" - "vpshufb %[BSWAP_MASK], %%xmm9, %%xmm1\n\t" - "vpshufb %[BSWAP_EPI64], %%xmm5, %%xmm4\n\t" - "vpaddd %[ONE], %%xmm5, %%xmm5\n\t" - "vpxor " VAR(XR) ", %%xmm1, %%xmm1\n\t" - "vmovdqu %%xmm5, " VAR(CTR1) "\n\t" - VAESENC_GFMUL_SB_AVX2(%%xmm9, HR, %%xmm1, CTR1) - "vmovdqu %%xmm4, (%[out]," VAR(KR64) ",1)\n\t" - "addl $16, " VAR(KR) "\n\t" - "cmpl %%r13d, " VAR(KR) "\n\t" - "jl 12b\n\t" - "\n" - "13:\n\t" - - AESENC_LAST15_DEC_AVX2() - "\n" - "55:\n\t" - - CALC_TAG_AVX2() - "addq $" VAR(STACK_OFFSET) ", %%rsp\n\t" - "popq %%rdx\n\t" - CMP_TAG_AVX() - "vzeroupper\n\t" - - : - : [KEY] "r" (key), - [in] "r" (in), [out] "r" (out), [nr] "r" (nr), - [nbytes] "r" (nbytes), [abytes] "r" (abytes), [addt] "r" (addt), - [ivec] "r" (iv), [ibytes] "r" (ivLen), [tbytes] "r" (tagLen), - [tag] "r" (tag), [res] "r" (res), - [BSWAP_MASK] "m" (BSWAP_MASK), - [BSWAP_EPI64] "m" (BSWAP_EPI64), - [ONE] "m" (ONE), -#if !defined(AES_GCM_AESNI_NO_UNROLL) && !defined(AES_GCM_AVX2_NO_UNROLL) - [TWO] "m" (TWO), [THREE] "m" (THREE), [FOUR] "m" (FOUR), - [FIVE] "m" (FIVE), [SIX] "m" (SIX), [SEVEN] "m" (SEVEN), - [EIGHT] "m" (EIGHT), -#endif - [MOD2_128] "m" (MOD2_128) - : "xmm15", "xmm14", "xmm13", "xmm12", - "xmm0", "xmm1", "xmm2", "xmm3", "memory", - "xmm4", "xmm5", "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", - "rcx", "r13" - ); -} +void AES_GCM_decrypt_avx2(const unsigned char *in, unsigned char *out, + const unsigned char* addt, const unsigned char* ivec, + const unsigned char *tag, int nbytes, int abytes, + int ibytes, int tbytes, const unsigned char* key, + int nr, int* res) + XASM_LINK("AES_GCM_decrypt_avx2"); #endif /* HAVE_INTEL_AVX2 */ #endif /* HAVE_INTEL_AVX1 */ #endif /* HAVE_AES_DECRYPT */ diff --git a/wolfcrypt/src/aes_gcm_asm.S b/wolfcrypt/src/aes_gcm_asm.S new file mode 100644 index 0000000000..f6f9f32743 --- /dev/null +++ b/wolfcrypt/src/aes_gcm_asm.S @@ -0,0 +1,8367 @@ +/* aes_gcm_asm + * + * Copyright (C) 2006-2018 wolfSSL Inc. + * + * This file is part of wolfSSL. + * + * wolfSSL is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * wolfSSL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA + */ + +#ifndef HAVE_INTEL_AVX1 +#define HAVE_INTEL_AVX1 +#endif /* HAVE_INTEL_AVX1 */ +#ifndef HAVE_INTEL_AVX2 +#define HAVE_INTEL_AVX2 +#endif /* HAVE_INTEL_AVX2 */ + +.align 16 +L_aes_gcm_one: +.quad 0x0, 0x1 +.align 16 +L_aes_gcm_two: +.quad 0x0, 0x2 +.align 16 +L_aes_gcm_three: +.quad 0x0, 0x3 +.align 16 +L_aes_gcm_four: +.quad 0x0, 0x4 +.align 16 +L_aes_gcm_five: +.quad 0x0, 0x5 +.align 16 +L_aes_gcm_six: +.quad 0x0, 0x6 +.align 16 +L_aes_gcm_seven: +.quad 0x0, 0x7 +.align 16 +L_aes_gcm_eight: +.quad 0x0, 0x8 +.align 16 +L_aes_gcm_bswap_epi64: +.quad 0x1020304050607, 0x8090a0b0c0d0e0f +.align 16 +L_aes_gcm_bswap_mask: +.quad 0x8090a0b0c0d0e0f, 0x1020304050607 +.align 16 +L_aes_gcm_mod2_128: +.quad 0x1, 0xc200000000000000 +.globl AES_GCM_encrypt +.type AES_GCM_encrypt,@function +.align 4 +AES_GCM_encrypt: + pushq %r13 + pushq %r12 + pushq %rbx + pushq %r14 + pushq %r15 + movq %rdx, %r12 + movq %rcx, %rax + movq 48(%rsp), %r11 + movq 56(%rsp), %rbx + movq 64(%rsp), %r14 + movq 72(%rsp), %r15 + movq 80(%rsp), %r10 + subq $0xa0, %rsp + pxor %xmm4, %xmm4 + pxor %xmm6, %xmm6 + cmpl $12, %ebx + movl %ebx, %edx + jne L_AES_GCM_encrypt_iv_not_12 + # # Calculate values when IV is 12 bytes + # Set counter based on IV + movl $0x1000000, %ecx + pinsrq $0x00, (%rax), %xmm4 + pinsrd $2, 8(%rax), %xmm4 + pinsrd $3, %ecx, %xmm4 + # H = Encrypt X(=0) and T = Encrypt counter + movdqa %xmm4, %xmm1 + movdqa (%r15), %xmm5 + pxor %xmm5, %xmm1 + movdqa 16(%r15), %xmm7 + aesenc %xmm7, %xmm5 + aesenc %xmm7, %xmm1 + movdqa 32(%r15), %xmm7 + aesenc %xmm7, %xmm5 + aesenc %xmm7, %xmm1 + movdqa 48(%r15), %xmm7 + aesenc %xmm7, %xmm5 + aesenc %xmm7, %xmm1 + movdqa 64(%r15), %xmm7 + aesenc %xmm7, %xmm5 + aesenc %xmm7, %xmm1 + movdqa 80(%r15), %xmm7 + aesenc %xmm7, %xmm5 + aesenc %xmm7, %xmm1 + movdqa 96(%r15), %xmm7 + aesenc %xmm7, %xmm5 + aesenc %xmm7, %xmm1 + movdqa 112(%r15), %xmm7 + aesenc %xmm7, %xmm5 + aesenc %xmm7, %xmm1 + movdqa 128(%r15), %xmm7 + aesenc %xmm7, %xmm5 + aesenc %xmm7, %xmm1 + movdqa 144(%r15), %xmm7 + aesenc %xmm7, %xmm5 + aesenc %xmm7, %xmm1 + cmpl $11, %r10d + movdqa 160(%r15), %xmm7 + jl L_AES_GCM_encrypt_calc_iv_12_last + aesenc %xmm7, %xmm5 + aesenc %xmm7, %xmm1 + movdqa 176(%r15), %xmm7 + aesenc %xmm7, %xmm5 + aesenc %xmm7, %xmm1 + cmpl $13, %r10d + movdqa 192(%r15), %xmm7 + jl L_AES_GCM_encrypt_calc_iv_12_last + aesenc %xmm7, %xmm5 + aesenc %xmm7, %xmm1 + movdqa 208(%r15), %xmm7 + aesenc %xmm7, %xmm5 + aesenc %xmm7, %xmm1 + movdqa 224(%r15), %xmm7 +L_AES_GCM_encrypt_calc_iv_12_last: + aesenclast %xmm7, %xmm5 + aesenclast %xmm7, %xmm1 + pshufb L_aes_gcm_bswap_mask(%rip), %xmm5 + movdqa %xmm1, 144(%rsp) + jmp L_AES_GCM_encrypt_iv_done +L_AES_GCM_encrypt_iv_not_12: + # Calculate values when IV is not 12 bytes + # H = Encrypt X(=0) + movdqa (%r15), %xmm5 + aesenc 16(%r15), %xmm5 + aesenc 32(%r15), %xmm5 + aesenc 48(%r15), %xmm5 + aesenc 64(%r15), %xmm5 + aesenc 80(%r15), %xmm5 + aesenc 96(%r15), %xmm5 + aesenc 112(%r15), %xmm5 + aesenc 128(%r15), %xmm5 + aesenc 144(%r15), %xmm5 + cmpl $11, %r10d + movdqa 160(%r15), %xmm9 + jl L_AES_GCM_encrypt_calc_iv_1_aesenc_avx_last + aesenc %xmm9, %xmm5 + aesenc 176(%r15), %xmm5 + cmpl $13, %r10d + movdqa 192(%r15), %xmm9 + jl L_AES_GCM_encrypt_calc_iv_1_aesenc_avx_last + aesenc %xmm9, %xmm5 + aesenc 208(%r15), %xmm5 + movdqa 224(%r15), %xmm9 +L_AES_GCM_encrypt_calc_iv_1_aesenc_avx_last: + aesenclast %xmm9, %xmm5 + pshufb L_aes_gcm_bswap_mask(%rip), %xmm5 + # Calc counter + # Initialization vector + cmpl $0x00, %edx + movq $0x00, %rcx + je L_AES_GCM_encrypt_calc_iv_done + cmpl $16, %edx + jl L_AES_GCM_encrypt_calc_iv_lt16 + andl $0xfffffff0, %edx +L_AES_GCM_encrypt_calc_iv_16_loop: + movdqu (%rax,%rcx,1), %xmm8 + pshufb L_aes_gcm_bswap_mask(%rip), %xmm8 + pxor %xmm8, %xmm4 + pshufd $0x4e, %xmm4, %xmm1 + pshufd $0x4e, %xmm5, %xmm2 + movdqa %xmm5, %xmm3 + movdqa %xmm5, %xmm0 + pclmulqdq $0x11, %xmm4, %xmm3 + pclmulqdq $0x00, %xmm4, %xmm0 + pxor %xmm4, %xmm1 + pxor %xmm5, %xmm2 + pclmulqdq $0x00, %xmm2, %xmm1 + pxor %xmm0, %xmm1 + pxor %xmm3, %xmm1 + movdqa %xmm1, %xmm2 + movdqa %xmm0, %xmm7 + movdqa %xmm3, %xmm4 + pslldq $8, %xmm2 + psrldq $8, %xmm1 + pxor %xmm2, %xmm7 + pxor %xmm1, %xmm4 + movdqa %xmm7, %xmm0 + movdqa %xmm4, %xmm1 + psrld $31, %xmm0 + psrld $31, %xmm1 + pslld $0x01, %xmm7 + pslld $0x01, %xmm4 + movdqa %xmm0, %xmm2 + pslldq $4, %xmm0 + psrldq $12, %xmm2 + pslldq $4, %xmm1 + por %xmm2, %xmm4 + por %xmm0, %xmm7 + por %xmm1, %xmm4 + movdqa %xmm7, %xmm0 + movdqa %xmm7, %xmm1 + movdqa %xmm7, %xmm2 + pslld $31, %xmm0 + pslld $30, %xmm1 + pslld $25, %xmm2 + pxor %xmm1, %xmm0 + pxor %xmm2, %xmm0 + movdqa %xmm0, %xmm1 + psrldq $4, %xmm1 + pslldq $12, %xmm0 + pxor %xmm0, %xmm7 + movdqa %xmm7, %xmm2 + movdqa %xmm7, %xmm3 + movdqa %xmm7, %xmm0 + psrld $0x01, %xmm2 + psrld $2, %xmm3 + psrld $7, %xmm0 + pxor %xmm3, %xmm2 + pxor %xmm0, %xmm2 + pxor %xmm1, %xmm2 + pxor %xmm7, %xmm2 + pxor %xmm2, %xmm4 + addl $16, %ecx + cmpl %edx, %ecx + jl L_AES_GCM_encrypt_calc_iv_16_loop + movl %ebx, %edx + cmpl %edx, %ecx + je L_AES_GCM_encrypt_calc_iv_done +L_AES_GCM_encrypt_calc_iv_lt16: + subq $16, %rsp + pxor %xmm8, %xmm8 + xorl %ebx, %ebx + movdqa %xmm8, (%rsp) +L_AES_GCM_encrypt_calc_iv_loop: + movzbl (%rax,%rcx,1), %r13d + movb %r13b, (%rsp,%rbx,1) + incl %ecx + incl %ebx + cmpl %edx, %ecx + jl L_AES_GCM_encrypt_calc_iv_loop + movdqa (%rsp), %xmm8 + addq $16, %rsp + pshufb L_aes_gcm_bswap_mask(%rip), %xmm8 + pxor %xmm8, %xmm4 + pshufd $0x4e, %xmm4, %xmm1 + pshufd $0x4e, %xmm5, %xmm2 + movdqa %xmm5, %xmm3 + movdqa %xmm5, %xmm0 + pclmulqdq $0x11, %xmm4, %xmm3 + pclmulqdq $0x00, %xmm4, %xmm0 + pxor %xmm4, %xmm1 + pxor %xmm5, %xmm2 + pclmulqdq $0x00, %xmm2, %xmm1 + pxor %xmm0, %xmm1 + pxor %xmm3, %xmm1 + movdqa %xmm1, %xmm2 + movdqa %xmm0, %xmm7 + movdqa %xmm3, %xmm4 + pslldq $8, %xmm2 + psrldq $8, %xmm1 + pxor %xmm2, %xmm7 + pxor %xmm1, %xmm4 + movdqa %xmm7, %xmm0 + movdqa %xmm4, %xmm1 + psrld $31, %xmm0 + psrld $31, %xmm1 + pslld $0x01, %xmm7 + pslld $0x01, %xmm4 + movdqa %xmm0, %xmm2 + pslldq $4, %xmm0 + psrldq $12, %xmm2 + pslldq $4, %xmm1 + por %xmm2, %xmm4 + por %xmm0, %xmm7 + por %xmm1, %xmm4 + movdqa %xmm7, %xmm0 + movdqa %xmm7, %xmm1 + movdqa %xmm7, %xmm2 + pslld $31, %xmm0 + pslld $30, %xmm1 + pslld $25, %xmm2 + pxor %xmm1, %xmm0 + pxor %xmm2, %xmm0 + movdqa %xmm0, %xmm1 + psrldq $4, %xmm1 + pslldq $12, %xmm0 + pxor %xmm0, %xmm7 + movdqa %xmm7, %xmm2 + movdqa %xmm7, %xmm3 + movdqa %xmm7, %xmm0 + psrld $0x01, %xmm2 + psrld $2, %xmm3 + psrld $7, %xmm0 + pxor %xmm3, %xmm2 + pxor %xmm0, %xmm2 + pxor %xmm1, %xmm2 + pxor %xmm7, %xmm2 + pxor %xmm2, %xmm4 +L_AES_GCM_encrypt_calc_iv_done: + # T = Encrypt counter + pxor %xmm0, %xmm0 + shll $3, %edx + pinsrq $0x00, %rdx, %xmm0 + pxor %xmm0, %xmm4 + pshufd $0x4e, %xmm4, %xmm1 + pshufd $0x4e, %xmm5, %xmm2 + movdqa %xmm5, %xmm3 + movdqa %xmm5, %xmm0 + pclmulqdq $0x11, %xmm4, %xmm3 + pclmulqdq $0x00, %xmm4, %xmm0 + pxor %xmm4, %xmm1 + pxor %xmm5, %xmm2 + pclmulqdq $0x00, %xmm2, %xmm1 + pxor %xmm0, %xmm1 + pxor %xmm3, %xmm1 + movdqa %xmm1, %xmm2 + movdqa %xmm0, %xmm7 + movdqa %xmm3, %xmm4 + pslldq $8, %xmm2 + psrldq $8, %xmm1 + pxor %xmm2, %xmm7 + pxor %xmm1, %xmm4 + movdqa %xmm7, %xmm0 + movdqa %xmm4, %xmm1 + psrld $31, %xmm0 + psrld $31, %xmm1 + pslld $0x01, %xmm7 + pslld $0x01, %xmm4 + movdqa %xmm0, %xmm2 + pslldq $4, %xmm0 + psrldq $12, %xmm2 + pslldq $4, %xmm1 + por %xmm2, %xmm4 + por %xmm0, %xmm7 + por %xmm1, %xmm4 + movdqa %xmm7, %xmm0 + movdqa %xmm7, %xmm1 + movdqa %xmm7, %xmm2 + pslld $31, %xmm0 + pslld $30, %xmm1 + pslld $25, %xmm2 + pxor %xmm1, %xmm0 + pxor %xmm2, %xmm0 + movdqa %xmm0, %xmm1 + psrldq $4, %xmm1 + pslldq $12, %xmm0 + pxor %xmm0, %xmm7 + movdqa %xmm7, %xmm2 + movdqa %xmm7, %xmm3 + movdqa %xmm7, %xmm0 + psrld $0x01, %xmm2 + psrld $2, %xmm3 + psrld $7, %xmm0 + pxor %xmm3, %xmm2 + pxor %xmm0, %xmm2 + pxor %xmm1, %xmm2 + pxor %xmm7, %xmm2 + pxor %xmm2, %xmm4 + pshufb L_aes_gcm_bswap_mask(%rip), %xmm4 + # Encrypt counter + movdqa (%r15), %xmm8 + pxor %xmm4, %xmm8 + aesenc 16(%r15), %xmm8 + aesenc 32(%r15), %xmm8 + aesenc 48(%r15), %xmm8 + aesenc 64(%r15), %xmm8 + aesenc 80(%r15), %xmm8 + aesenc 96(%r15), %xmm8 + aesenc 112(%r15), %xmm8 + aesenc 128(%r15), %xmm8 + aesenc 144(%r15), %xmm8 + cmpl $11, %r10d + movdqa 160(%r15), %xmm9 + jl L_AES_GCM_encrypt_calc_iv_2_aesenc_avx_last + aesenc %xmm9, %xmm8 + aesenc 176(%r15), %xmm8 + cmpl $13, %r10d + movdqa 192(%r15), %xmm9 + jl L_AES_GCM_encrypt_calc_iv_2_aesenc_avx_last + aesenc %xmm9, %xmm8 + aesenc 208(%r15), %xmm8 + movdqa 224(%r15), %xmm9 +L_AES_GCM_encrypt_calc_iv_2_aesenc_avx_last: + aesenclast %xmm9, %xmm8 + movdqa %xmm8, 144(%rsp) +L_AES_GCM_encrypt_iv_done: + # Additional authentication data + movl %r11d, %edx + cmpl $0x00, %edx + je L_AES_GCM_encrypt_calc_aad_done + xorl %ecx, %ecx + cmpl $16, %edx + jl L_AES_GCM_encrypt_calc_aad_lt16 + andl $0xfffffff0, %edx +L_AES_GCM_encrypt_calc_aad_16_loop: + movdqu (%r12,%rcx,1), %xmm8 + pshufb L_aes_gcm_bswap_mask(%rip), %xmm8 + pxor %xmm8, %xmm6 + pshufd $0x4e, %xmm6, %xmm1 + pshufd $0x4e, %xmm5, %xmm2 + movdqa %xmm5, %xmm3 + movdqa %xmm5, %xmm0 + pclmulqdq $0x11, %xmm6, %xmm3 + pclmulqdq $0x00, %xmm6, %xmm0 + pxor %xmm6, %xmm1 + pxor %xmm5, %xmm2 + pclmulqdq $0x00, %xmm2, %xmm1 + pxor %xmm0, %xmm1 + pxor %xmm3, %xmm1 + movdqa %xmm1, %xmm2 + movdqa %xmm0, %xmm7 + movdqa %xmm3, %xmm6 + pslldq $8, %xmm2 + psrldq $8, %xmm1 + pxor %xmm2, %xmm7 + pxor %xmm1, %xmm6 + movdqa %xmm7, %xmm0 + movdqa %xmm6, %xmm1 + psrld $31, %xmm0 + psrld $31, %xmm1 + pslld $0x01, %xmm7 + pslld $0x01, %xmm6 + movdqa %xmm0, %xmm2 + pslldq $4, %xmm0 + psrldq $12, %xmm2 + pslldq $4, %xmm1 + por %xmm2, %xmm6 + por %xmm0, %xmm7 + por %xmm1, %xmm6 + movdqa %xmm7, %xmm0 + movdqa %xmm7, %xmm1 + movdqa %xmm7, %xmm2 + pslld $31, %xmm0 + pslld $30, %xmm1 + pslld $25, %xmm2 + pxor %xmm1, %xmm0 + pxor %xmm2, %xmm0 + movdqa %xmm0, %xmm1 + psrldq $4, %xmm1 + pslldq $12, %xmm0 + pxor %xmm0, %xmm7 + movdqa %xmm7, %xmm2 + movdqa %xmm7, %xmm3 + movdqa %xmm7, %xmm0 + psrld $0x01, %xmm2 + psrld $2, %xmm3 + psrld $7, %xmm0 + pxor %xmm3, %xmm2 + pxor %xmm0, %xmm2 + pxor %xmm1, %xmm2 + pxor %xmm7, %xmm2 + pxor %xmm2, %xmm6 + addl $16, %ecx + cmpl %edx, %ecx + jl L_AES_GCM_encrypt_calc_aad_16_loop + movl %r11d, %edx + cmpl %edx, %ecx + je L_AES_GCM_encrypt_calc_aad_done +L_AES_GCM_encrypt_calc_aad_lt16: + subq $16, %rsp + pxor %xmm8, %xmm8 + xorl %ebx, %ebx + movdqa %xmm8, (%rsp) +L_AES_GCM_encrypt_calc_aad_loop: + movzbl (%r12,%rcx,1), %r13d + movb %r13b, (%rsp,%rbx,1) + incl %ecx + incl %ebx + cmpl %edx, %ecx + jl L_AES_GCM_encrypt_calc_aad_loop + movdqa (%rsp), %xmm8 + addq $16, %rsp + pshufb L_aes_gcm_bswap_mask(%rip), %xmm8 + pxor %xmm8, %xmm6 + pshufd $0x4e, %xmm6, %xmm1 + pshufd $0x4e, %xmm5, %xmm2 + movdqa %xmm5, %xmm3 + movdqa %xmm5, %xmm0 + pclmulqdq $0x11, %xmm6, %xmm3 + pclmulqdq $0x00, %xmm6, %xmm0 + pxor %xmm6, %xmm1 + pxor %xmm5, %xmm2 + pclmulqdq $0x00, %xmm2, %xmm1 + pxor %xmm0, %xmm1 + pxor %xmm3, %xmm1 + movdqa %xmm1, %xmm2 + movdqa %xmm0, %xmm7 + movdqa %xmm3, %xmm6 + pslldq $8, %xmm2 + psrldq $8, %xmm1 + pxor %xmm2, %xmm7 + pxor %xmm1, %xmm6 + movdqa %xmm7, %xmm0 + movdqa %xmm6, %xmm1 + psrld $31, %xmm0 + psrld $31, %xmm1 + pslld $0x01, %xmm7 + pslld $0x01, %xmm6 + movdqa %xmm0, %xmm2 + pslldq $4, %xmm0 + psrldq $12, %xmm2 + pslldq $4, %xmm1 + por %xmm2, %xmm6 + por %xmm0, %xmm7 + por %xmm1, %xmm6 + movdqa %xmm7, %xmm0 + movdqa %xmm7, %xmm1 + movdqa %xmm7, %xmm2 + pslld $31, %xmm0 + pslld $30, %xmm1 + pslld $25, %xmm2 + pxor %xmm1, %xmm0 + pxor %xmm2, %xmm0 + movdqa %xmm0, %xmm1 + psrldq $4, %xmm1 + pslldq $12, %xmm0 + pxor %xmm0, %xmm7 + movdqa %xmm7, %xmm2 + movdqa %xmm7, %xmm3 + movdqa %xmm7, %xmm0 + psrld $0x01, %xmm2 + psrld $2, %xmm3 + psrld $7, %xmm0 + pxor %xmm3, %xmm2 + pxor %xmm0, %xmm2 + pxor %xmm1, %xmm2 + pxor %xmm7, %xmm2 + pxor %xmm2, %xmm6 +L_AES_GCM_encrypt_calc_aad_done: + # Calculate counter and H + pshufb L_aes_gcm_bswap_epi64(%rip), %xmm4 + movdqa %xmm5, %xmm9 + paddd L_aes_gcm_one(%rip), %xmm4 + movdqa %xmm5, %xmm8 + movdqa %xmm4, 128(%rsp) + psrlq $63, %xmm9 + psllq $0x01, %xmm8 + pslldq $8, %xmm9 + por %xmm9, %xmm8 + pshufd $0xff, %xmm5, %xmm5 + psrad $31, %xmm5 + pand L_aes_gcm_mod2_128(%rip), %xmm5 + pxor %xmm8, %xmm5 + xorq %rbx, %rbx + cmpl $0x80, %r9d + movl %r9d, %r13d + jl L_AES_GCM_encrypt_done_128 + andl $0xffffff80, %r13d + movdqa %xmm6, %xmm2 + # H ^ 1 + movdqa %xmm5, (%rsp) + # H ^ 2 + pshufd $0x4e, %xmm5, %xmm9 + pshufd $0x4e, %xmm5, %xmm10 + movdqa %xmm5, %xmm11 + movdqa %xmm5, %xmm8 + pclmulqdq $0x11, %xmm5, %xmm11 + pclmulqdq $0x00, %xmm5, %xmm8 + pxor %xmm5, %xmm9 + pxor %xmm5, %xmm10 + pclmulqdq $0x00, %xmm10, %xmm9 + pxor %xmm8, %xmm9 + pxor %xmm11, %xmm9 + movdqa %xmm9, %xmm10 + movdqa %xmm11, %xmm0 + pslldq $8, %xmm10 + psrldq $8, %xmm9 + pxor %xmm10, %xmm8 + pxor %xmm9, %xmm0 + movdqa %xmm8, %xmm12 + movdqa %xmm8, %xmm13 + movdqa %xmm8, %xmm14 + pslld $31, %xmm12 + pslld $30, %xmm13 + pslld $25, %xmm14 + pxor %xmm13, %xmm12 + pxor %xmm14, %xmm12 + movdqa %xmm12, %xmm13 + psrldq $4, %xmm13 + pslldq $12, %xmm12 + pxor %xmm12, %xmm8 + movdqa %xmm8, %xmm14 + movdqa %xmm8, %xmm10 + movdqa %xmm8, %xmm9 + psrld $0x01, %xmm14 + psrld $2, %xmm10 + psrld $7, %xmm9 + pxor %xmm10, %xmm14 + pxor %xmm9, %xmm14 + pxor %xmm13, %xmm14 + pxor %xmm8, %xmm14 + pxor %xmm14, %xmm0 + movdqa %xmm0, 16(%rsp) + # H ^ 3 + pshufd $0x4e, %xmm5, %xmm9 + pshufd $0x4e, %xmm0, %xmm10 + movdqa %xmm0, %xmm11 + movdqa %xmm0, %xmm8 + pclmulqdq $0x11, %xmm5, %xmm11 + pclmulqdq $0x00, %xmm5, %xmm8 + pxor %xmm5, %xmm9 + pxor %xmm0, %xmm10 + pclmulqdq $0x00, %xmm10, %xmm9 + pxor %xmm8, %xmm9 + pxor %xmm11, %xmm9 + movdqa %xmm9, %xmm10 + movdqa %xmm11, %xmm1 + pslldq $8, %xmm10 + psrldq $8, %xmm9 + pxor %xmm10, %xmm8 + pxor %xmm9, %xmm1 + movdqa %xmm8, %xmm12 + movdqa %xmm8, %xmm13 + movdqa %xmm8, %xmm14 + pslld $31, %xmm12 + pslld $30, %xmm13 + pslld $25, %xmm14 + pxor %xmm13, %xmm12 + pxor %xmm14, %xmm12 + movdqa %xmm12, %xmm13 + psrldq $4, %xmm13 + pslldq $12, %xmm12 + pxor %xmm12, %xmm8 + movdqa %xmm8, %xmm14 + movdqa %xmm8, %xmm10 + movdqa %xmm8, %xmm9 + psrld $0x01, %xmm14 + psrld $2, %xmm10 + psrld $7, %xmm9 + pxor %xmm10, %xmm14 + pxor %xmm9, %xmm14 + pxor %xmm13, %xmm14 + pxor %xmm8, %xmm14 + pxor %xmm14, %xmm1 + movdqa %xmm1, 32(%rsp) + # H ^ 4 + pshufd $0x4e, %xmm0, %xmm9 + pshufd $0x4e, %xmm0, %xmm10 + movdqa %xmm0, %xmm11 + movdqa %xmm0, %xmm8 + pclmulqdq $0x11, %xmm0, %xmm11 + pclmulqdq $0x00, %xmm0, %xmm8 + pxor %xmm0, %xmm9 + pxor %xmm0, %xmm10 + pclmulqdq $0x00, %xmm10, %xmm9 + pxor %xmm8, %xmm9 + pxor %xmm11, %xmm9 + movdqa %xmm9, %xmm10 + movdqa %xmm11, %xmm3 + pslldq $8, %xmm10 + psrldq $8, %xmm9 + pxor %xmm10, %xmm8 + pxor %xmm9, %xmm3 + movdqa %xmm8, %xmm12 + movdqa %xmm8, %xmm13 + movdqa %xmm8, %xmm14 + pslld $31, %xmm12 + pslld $30, %xmm13 + pslld $25, %xmm14 + pxor %xmm13, %xmm12 + pxor %xmm14, %xmm12 + movdqa %xmm12, %xmm13 + psrldq $4, %xmm13 + pslldq $12, %xmm12 + pxor %xmm12, %xmm8 + movdqa %xmm8, %xmm14 + movdqa %xmm8, %xmm10 + movdqa %xmm8, %xmm9 + psrld $0x01, %xmm14 + psrld $2, %xmm10 + psrld $7, %xmm9 + pxor %xmm10, %xmm14 + pxor %xmm9, %xmm14 + pxor %xmm13, %xmm14 + pxor %xmm8, %xmm14 + pxor %xmm14, %xmm3 + movdqa %xmm3, 48(%rsp) + # H ^ 5 + pshufd $0x4e, %xmm0, %xmm9 + pshufd $0x4e, %xmm1, %xmm10 + movdqa %xmm1, %xmm11 + movdqa %xmm1, %xmm8 + pclmulqdq $0x11, %xmm0, %xmm11 + pclmulqdq $0x00, %xmm0, %xmm8 + pxor %xmm0, %xmm9 + pxor %xmm1, %xmm10 + pclmulqdq $0x00, %xmm10, %xmm9 + pxor %xmm8, %xmm9 + pxor %xmm11, %xmm9 + movdqa %xmm9, %xmm10 + movdqa %xmm11, %xmm7 + pslldq $8, %xmm10 + psrldq $8, %xmm9 + pxor %xmm10, %xmm8 + pxor %xmm9, %xmm7 + movdqa %xmm8, %xmm12 + movdqa %xmm8, %xmm13 + movdqa %xmm8, %xmm14 + pslld $31, %xmm12 + pslld $30, %xmm13 + pslld $25, %xmm14 + pxor %xmm13, %xmm12 + pxor %xmm14, %xmm12 + movdqa %xmm12, %xmm13 + psrldq $4, %xmm13 + pslldq $12, %xmm12 + pxor %xmm12, %xmm8 + movdqa %xmm8, %xmm14 + movdqa %xmm8, %xmm10 + movdqa %xmm8, %xmm9 + psrld $0x01, %xmm14 + psrld $2, %xmm10 + psrld $7, %xmm9 + pxor %xmm10, %xmm14 + pxor %xmm9, %xmm14 + pxor %xmm13, %xmm14 + pxor %xmm8, %xmm14 + pxor %xmm14, %xmm7 + movdqa %xmm7, 64(%rsp) + # H ^ 6 + pshufd $0x4e, %xmm1, %xmm9 + pshufd $0x4e, %xmm1, %xmm10 + movdqa %xmm1, %xmm11 + movdqa %xmm1, %xmm8 + pclmulqdq $0x11, %xmm1, %xmm11 + pclmulqdq $0x00, %xmm1, %xmm8 + pxor %xmm1, %xmm9 + pxor %xmm1, %xmm10 + pclmulqdq $0x00, %xmm10, %xmm9 + pxor %xmm8, %xmm9 + pxor %xmm11, %xmm9 + movdqa %xmm9, %xmm10 + movdqa %xmm11, %xmm7 + pslldq $8, %xmm10 + psrldq $8, %xmm9 + pxor %xmm10, %xmm8 + pxor %xmm9, %xmm7 + movdqa %xmm8, %xmm12 + movdqa %xmm8, %xmm13 + movdqa %xmm8, %xmm14 + pslld $31, %xmm12 + pslld $30, %xmm13 + pslld $25, %xmm14 + pxor %xmm13, %xmm12 + pxor %xmm14, %xmm12 + movdqa %xmm12, %xmm13 + psrldq $4, %xmm13 + pslldq $12, %xmm12 + pxor %xmm12, %xmm8 + movdqa %xmm8, %xmm14 + movdqa %xmm8, %xmm10 + movdqa %xmm8, %xmm9 + psrld $0x01, %xmm14 + psrld $2, %xmm10 + psrld $7, %xmm9 + pxor %xmm10, %xmm14 + pxor %xmm9, %xmm14 + pxor %xmm13, %xmm14 + pxor %xmm8, %xmm14 + pxor %xmm14, %xmm7 + movdqa %xmm7, 80(%rsp) + # H ^ 7 + pshufd $0x4e, %xmm1, %xmm9 + pshufd $0x4e, %xmm3, %xmm10 + movdqa %xmm3, %xmm11 + movdqa %xmm3, %xmm8 + pclmulqdq $0x11, %xmm1, %xmm11 + pclmulqdq $0x00, %xmm1, %xmm8 + pxor %xmm1, %xmm9 + pxor %xmm3, %xmm10 + pclmulqdq $0x00, %xmm10, %xmm9 + pxor %xmm8, %xmm9 + pxor %xmm11, %xmm9 + movdqa %xmm9, %xmm10 + movdqa %xmm11, %xmm7 + pslldq $8, %xmm10 + psrldq $8, %xmm9 + pxor %xmm10, %xmm8 + pxor %xmm9, %xmm7 + movdqa %xmm8, %xmm12 + movdqa %xmm8, %xmm13 + movdqa %xmm8, %xmm14 + pslld $31, %xmm12 + pslld $30, %xmm13 + pslld $25, %xmm14 + pxor %xmm13, %xmm12 + pxor %xmm14, %xmm12 + movdqa %xmm12, %xmm13 + psrldq $4, %xmm13 + pslldq $12, %xmm12 + pxor %xmm12, %xmm8 + movdqa %xmm8, %xmm14 + movdqa %xmm8, %xmm10 + movdqa %xmm8, %xmm9 + psrld $0x01, %xmm14 + psrld $2, %xmm10 + psrld $7, %xmm9 + pxor %xmm10, %xmm14 + pxor %xmm9, %xmm14 + pxor %xmm13, %xmm14 + pxor %xmm8, %xmm14 + pxor %xmm14, %xmm7 + movdqa %xmm7, 96(%rsp) + # H ^ 8 + pshufd $0x4e, %xmm3, %xmm9 + pshufd $0x4e, %xmm3, %xmm10 + movdqa %xmm3, %xmm11 + movdqa %xmm3, %xmm8 + pclmulqdq $0x11, %xmm3, %xmm11 + pclmulqdq $0x00, %xmm3, %xmm8 + pxor %xmm3, %xmm9 + pxor %xmm3, %xmm10 + pclmulqdq $0x00, %xmm10, %xmm9 + pxor %xmm8, %xmm9 + pxor %xmm11, %xmm9 + movdqa %xmm9, %xmm10 + movdqa %xmm11, %xmm7 + pslldq $8, %xmm10 + psrldq $8, %xmm9 + pxor %xmm10, %xmm8 + pxor %xmm9, %xmm7 + movdqa %xmm8, %xmm12 + movdqa %xmm8, %xmm13 + movdqa %xmm8, %xmm14 + pslld $31, %xmm12 + pslld $30, %xmm13 + pslld $25, %xmm14 + pxor %xmm13, %xmm12 + pxor %xmm14, %xmm12 + movdqa %xmm12, %xmm13 + psrldq $4, %xmm13 + pslldq $12, %xmm12 + pxor %xmm12, %xmm8 + movdqa %xmm8, %xmm14 + movdqa %xmm8, %xmm10 + movdqa %xmm8, %xmm9 + psrld $0x01, %xmm14 + psrld $2, %xmm10 + psrld $7, %xmm9 + pxor %xmm10, %xmm14 + pxor %xmm9, %xmm14 + pxor %xmm13, %xmm14 + pxor %xmm8, %xmm14 + pxor %xmm14, %xmm7 + movdqa %xmm7, 112(%rsp) + # First 128 bytes of input + movdqa 128(%rsp), %xmm8 + movdqa L_aes_gcm_bswap_epi64(%rip), %xmm1 + movdqa %xmm8, %xmm0 + pshufb %xmm1, %xmm8 + movdqa %xmm0, %xmm9 + paddd L_aes_gcm_one(%rip), %xmm9 + pshufb %xmm1, %xmm9 + movdqa %xmm0, %xmm10 + paddd L_aes_gcm_two(%rip), %xmm10 + pshufb %xmm1, %xmm10 + movdqa %xmm0, %xmm11 + paddd L_aes_gcm_three(%rip), %xmm11 + pshufb %xmm1, %xmm11 + movdqa %xmm0, %xmm12 + paddd L_aes_gcm_four(%rip), %xmm12 + pshufb %xmm1, %xmm12 + movdqa %xmm0, %xmm13 + paddd L_aes_gcm_five(%rip), %xmm13 + pshufb %xmm1, %xmm13 + movdqa %xmm0, %xmm14 + paddd L_aes_gcm_six(%rip), %xmm14 + pshufb %xmm1, %xmm14 + movdqa %xmm0, %xmm15 + paddd L_aes_gcm_seven(%rip), %xmm15 + pshufb %xmm1, %xmm15 + paddd L_aes_gcm_eight(%rip), %xmm0 + movdqa (%r15), %xmm7 + movdqa %xmm0, 128(%rsp) + pxor %xmm7, %xmm8 + pxor %xmm7, %xmm9 + pxor %xmm7, %xmm10 + pxor %xmm7, %xmm11 + pxor %xmm7, %xmm12 + pxor %xmm7, %xmm13 + pxor %xmm7, %xmm14 + pxor %xmm7, %xmm15 + movdqa 16(%r15), %xmm7 + aesenc %xmm7, %xmm8 + aesenc %xmm7, %xmm9 + aesenc %xmm7, %xmm10 + aesenc %xmm7, %xmm11 + aesenc %xmm7, %xmm12 + aesenc %xmm7, %xmm13 + aesenc %xmm7, %xmm14 + aesenc %xmm7, %xmm15 + movdqa 32(%r15), %xmm7 + aesenc %xmm7, %xmm8 + aesenc %xmm7, %xmm9 + aesenc %xmm7, %xmm10 + aesenc %xmm7, %xmm11 + aesenc %xmm7, %xmm12 + aesenc %xmm7, %xmm13 + aesenc %xmm7, %xmm14 + aesenc %xmm7, %xmm15 + movdqa 48(%r15), %xmm7 + aesenc %xmm7, %xmm8 + aesenc %xmm7, %xmm9 + aesenc %xmm7, %xmm10 + aesenc %xmm7, %xmm11 + aesenc %xmm7, %xmm12 + aesenc %xmm7, %xmm13 + aesenc %xmm7, %xmm14 + aesenc %xmm7, %xmm15 + movdqa 64(%r15), %xmm7 + aesenc %xmm7, %xmm8 + aesenc %xmm7, %xmm9 + aesenc %xmm7, %xmm10 + aesenc %xmm7, %xmm11 + aesenc %xmm7, %xmm12 + aesenc %xmm7, %xmm13 + aesenc %xmm7, %xmm14 + aesenc %xmm7, %xmm15 + movdqa 80(%r15), %xmm7 + aesenc %xmm7, %xmm8 + aesenc %xmm7, %xmm9 + aesenc %xmm7, %xmm10 + aesenc %xmm7, %xmm11 + aesenc %xmm7, %xmm12 + aesenc %xmm7, %xmm13 + aesenc %xmm7, %xmm14 + aesenc %xmm7, %xmm15 + movdqa 96(%r15), %xmm7 + aesenc %xmm7, %xmm8 + aesenc %xmm7, %xmm9 + aesenc %xmm7, %xmm10 + aesenc %xmm7, %xmm11 + aesenc %xmm7, %xmm12 + aesenc %xmm7, %xmm13 + aesenc %xmm7, %xmm14 + aesenc %xmm7, %xmm15 + movdqa 112(%r15), %xmm7 + aesenc %xmm7, %xmm8 + aesenc %xmm7, %xmm9 + aesenc %xmm7, %xmm10 + aesenc %xmm7, %xmm11 + aesenc %xmm7, %xmm12 + aesenc %xmm7, %xmm13 + aesenc %xmm7, %xmm14 + aesenc %xmm7, %xmm15 + movdqa 128(%r15), %xmm7 + aesenc %xmm7, %xmm8 + aesenc %xmm7, %xmm9 + aesenc %xmm7, %xmm10 + aesenc %xmm7, %xmm11 + aesenc %xmm7, %xmm12 + aesenc %xmm7, %xmm13 + aesenc %xmm7, %xmm14 + aesenc %xmm7, %xmm15 + movdqa 144(%r15), %xmm7 + aesenc %xmm7, %xmm8 + aesenc %xmm7, %xmm9 + aesenc %xmm7, %xmm10 + aesenc %xmm7, %xmm11 + aesenc %xmm7, %xmm12 + aesenc %xmm7, %xmm13 + aesenc %xmm7, %xmm14 + aesenc %xmm7, %xmm15 + cmpl $11, %r10d + movdqa 160(%r15), %xmm7 + jl L_AES_GCM_encrypt_enc_done + aesenc %xmm7, %xmm8 + aesenc %xmm7, %xmm9 + aesenc %xmm7, %xmm10 + aesenc %xmm7, %xmm11 + aesenc %xmm7, %xmm12 + aesenc %xmm7, %xmm13 + aesenc %xmm7, %xmm14 + aesenc %xmm7, %xmm15 + movdqa 176(%r15), %xmm7 + aesenc %xmm7, %xmm8 + aesenc %xmm7, %xmm9 + aesenc %xmm7, %xmm10 + aesenc %xmm7, %xmm11 + aesenc %xmm7, %xmm12 + aesenc %xmm7, %xmm13 + aesenc %xmm7, %xmm14 + aesenc %xmm7, %xmm15 + cmpl $13, %r10d + movdqa 192(%r15), %xmm7 + jl L_AES_GCM_encrypt_enc_done + aesenc %xmm7, %xmm8 + aesenc %xmm7, %xmm9 + aesenc %xmm7, %xmm10 + aesenc %xmm7, %xmm11 + aesenc %xmm7, %xmm12 + aesenc %xmm7, %xmm13 + aesenc %xmm7, %xmm14 + aesenc %xmm7, %xmm15 + movdqa 208(%r15), %xmm7 + aesenc %xmm7, %xmm8 + aesenc %xmm7, %xmm9 + aesenc %xmm7, %xmm10 + aesenc %xmm7, %xmm11 + aesenc %xmm7, %xmm12 + aesenc %xmm7, %xmm13 + aesenc %xmm7, %xmm14 + aesenc %xmm7, %xmm15 + movdqa 224(%r15), %xmm7 +L_AES_GCM_encrypt_enc_done: + aesenclast %xmm7, %xmm8 + aesenclast %xmm7, %xmm9 + movdqu (%rdi), %xmm0 + movdqu 16(%rdi), %xmm1 + pxor %xmm0, %xmm8 + pxor %xmm1, %xmm9 + movdqu %xmm8, (%rsi) + movdqu %xmm9, 16(%rsi) + aesenclast %xmm7, %xmm10 + aesenclast %xmm7, %xmm11 + movdqu 32(%rdi), %xmm0 + movdqu 48(%rdi), %xmm1 + pxor %xmm0, %xmm10 + pxor %xmm1, %xmm11 + movdqu %xmm10, 32(%rsi) + movdqu %xmm11, 48(%rsi) + aesenclast %xmm7, %xmm12 + aesenclast %xmm7, %xmm13 + movdqu 64(%rdi), %xmm0 + movdqu 80(%rdi), %xmm1 + pxor %xmm0, %xmm12 + pxor %xmm1, %xmm13 + movdqu %xmm12, 64(%rsi) + movdqu %xmm13, 80(%rsi) + aesenclast %xmm7, %xmm14 + aesenclast %xmm7, %xmm15 + movdqu 96(%rdi), %xmm0 + movdqu 112(%rdi), %xmm1 + pxor %xmm0, %xmm14 + pxor %xmm1, %xmm15 + movdqu %xmm14, 96(%rsi) + movdqu %xmm15, 112(%rsi) + cmpl $0x80, %r13d + movl $0x80, %ebx + jle L_AES_GCM_encrypt_end_128 + # More 128 bytes of input +L_AES_GCM_encrypt_ghash_128: + leaq (%rdi,%rbx,1), %rcx + leaq (%rsi,%rbx,1), %rdx + movdqa 128(%rsp), %xmm8 + movdqa L_aes_gcm_bswap_epi64(%rip), %xmm1 + movdqa %xmm8, %xmm0 + pshufb %xmm1, %xmm8 + movdqa %xmm0, %xmm9 + paddd L_aes_gcm_one(%rip), %xmm9 + pshufb %xmm1, %xmm9 + movdqa %xmm0, %xmm10 + paddd L_aes_gcm_two(%rip), %xmm10 + pshufb %xmm1, %xmm10 + movdqa %xmm0, %xmm11 + paddd L_aes_gcm_three(%rip), %xmm11 + pshufb %xmm1, %xmm11 + movdqa %xmm0, %xmm12 + paddd L_aes_gcm_four(%rip), %xmm12 + pshufb %xmm1, %xmm12 + movdqa %xmm0, %xmm13 + paddd L_aes_gcm_five(%rip), %xmm13 + pshufb %xmm1, %xmm13 + movdqa %xmm0, %xmm14 + paddd L_aes_gcm_six(%rip), %xmm14 + pshufb %xmm1, %xmm14 + movdqa %xmm0, %xmm15 + paddd L_aes_gcm_seven(%rip), %xmm15 + pshufb %xmm1, %xmm15 + paddd L_aes_gcm_eight(%rip), %xmm0 + movdqa (%r15), %xmm7 + movdqa %xmm0, 128(%rsp) + pxor %xmm7, %xmm8 + pxor %xmm7, %xmm9 + pxor %xmm7, %xmm10 + pxor %xmm7, %xmm11 + pxor %xmm7, %xmm12 + pxor %xmm7, %xmm13 + pxor %xmm7, %xmm14 + pxor %xmm7, %xmm15 + movdqa 112(%rsp), %xmm7 + movdqu -128(%rdx), %xmm0 + aesenc 16(%r15), %xmm8 + pshufb L_aes_gcm_bswap_mask(%rip), %xmm0 + pxor %xmm2, %xmm0 + pshufd $0x4e, %xmm7, %xmm1 + pshufd $0x4e, %xmm0, %xmm5 + pxor %xmm7, %xmm1 + pxor %xmm0, %xmm5 + movdqa %xmm0, %xmm3 + pclmulqdq $0x11, %xmm7, %xmm3 + aesenc 16(%r15), %xmm9 + aesenc 16(%r15), %xmm10 + movdqa %xmm0, %xmm2 + pclmulqdq $0x00, %xmm7, %xmm2 + aesenc 16(%r15), %xmm11 + aesenc 16(%r15), %xmm12 + pclmulqdq $0x00, %xmm5, %xmm1 + aesenc 16(%r15), %xmm13 + aesenc 16(%r15), %xmm14 + aesenc 16(%r15), %xmm15 + pxor %xmm2, %xmm1 + pxor %xmm3, %xmm1 + movdqa 96(%rsp), %xmm7 + movdqu -112(%rdx), %xmm0 + pshufd $0x4e, %xmm7, %xmm4 + pshufb L_aes_gcm_bswap_mask(%rip), %xmm0 + aesenc 32(%r15), %xmm8 + pxor %xmm7, %xmm4 + pshufd $0x4e, %xmm0, %xmm5 + pxor %xmm0, %xmm5 + movdqa %xmm0, %xmm6 + pclmulqdq $0x11, %xmm7, %xmm6 + aesenc 32(%r15), %xmm9 + aesenc 32(%r15), %xmm10 + pclmulqdq $0x00, %xmm0, %xmm7 + aesenc 32(%r15), %xmm11 + aesenc 32(%r15), %xmm12 + pclmulqdq $0x00, %xmm5, %xmm4 + aesenc 32(%r15), %xmm13 + aesenc 32(%r15), %xmm14 + aesenc 32(%r15), %xmm15 + pxor %xmm7, %xmm1 + pxor %xmm7, %xmm2 + pxor %xmm6, %xmm1 + pxor %xmm6, %xmm3 + pxor %xmm4, %xmm1 + movdqa 80(%rsp), %xmm7 + movdqu -96(%rdx), %xmm0 + pshufd $0x4e, %xmm7, %xmm4 + pshufb L_aes_gcm_bswap_mask(%rip), %xmm0 + aesenc 48(%r15), %xmm8 + pxor %xmm7, %xmm4 + pshufd $0x4e, %xmm0, %xmm5 + pxor %xmm0, %xmm5 + movdqa %xmm0, %xmm6 + pclmulqdq $0x11, %xmm7, %xmm6 + aesenc 48(%r15), %xmm9 + aesenc 48(%r15), %xmm10 + pclmulqdq $0x00, %xmm0, %xmm7 + aesenc 48(%r15), %xmm11 + aesenc 48(%r15), %xmm12 + pclmulqdq $0x00, %xmm5, %xmm4 + aesenc 48(%r15), %xmm13 + aesenc 48(%r15), %xmm14 + aesenc 48(%r15), %xmm15 + pxor %xmm7, %xmm1 + pxor %xmm7, %xmm2 + pxor %xmm6, %xmm1 + pxor %xmm6, %xmm3 + pxor %xmm4, %xmm1 + movdqa 64(%rsp), %xmm7 + movdqu -80(%rdx), %xmm0 + pshufd $0x4e, %xmm7, %xmm4 + pshufb L_aes_gcm_bswap_mask(%rip), %xmm0 + aesenc 64(%r15), %xmm8 + pxor %xmm7, %xmm4 + pshufd $0x4e, %xmm0, %xmm5 + pxor %xmm0, %xmm5 + movdqa %xmm0, %xmm6 + pclmulqdq $0x11, %xmm7, %xmm6 + aesenc 64(%r15), %xmm9 + aesenc 64(%r15), %xmm10 + pclmulqdq $0x00, %xmm0, %xmm7 + aesenc 64(%r15), %xmm11 + aesenc 64(%r15), %xmm12 + pclmulqdq $0x00, %xmm5, %xmm4 + aesenc 64(%r15), %xmm13 + aesenc 64(%r15), %xmm14 + aesenc 64(%r15), %xmm15 + pxor %xmm7, %xmm1 + pxor %xmm7, %xmm2 + pxor %xmm6, %xmm1 + pxor %xmm6, %xmm3 + pxor %xmm4, %xmm1 + movdqa 48(%rsp), %xmm7 + movdqu -64(%rdx), %xmm0 + pshufd $0x4e, %xmm7, %xmm4 + pshufb L_aes_gcm_bswap_mask(%rip), %xmm0 + aesenc 80(%r15), %xmm8 + pxor %xmm7, %xmm4 + pshufd $0x4e, %xmm0, %xmm5 + pxor %xmm0, %xmm5 + movdqa %xmm0, %xmm6 + pclmulqdq $0x11, %xmm7, %xmm6 + aesenc 80(%r15), %xmm9 + aesenc 80(%r15), %xmm10 + pclmulqdq $0x00, %xmm0, %xmm7 + aesenc 80(%r15), %xmm11 + aesenc 80(%r15), %xmm12 + pclmulqdq $0x00, %xmm5, %xmm4 + aesenc 80(%r15), %xmm13 + aesenc 80(%r15), %xmm14 + aesenc 80(%r15), %xmm15 + pxor %xmm7, %xmm1 + pxor %xmm7, %xmm2 + pxor %xmm6, %xmm1 + pxor %xmm6, %xmm3 + pxor %xmm4, %xmm1 + movdqa 32(%rsp), %xmm7 + movdqu -48(%rdx), %xmm0 + pshufd $0x4e, %xmm7, %xmm4 + pshufb L_aes_gcm_bswap_mask(%rip), %xmm0 + aesenc 96(%r15), %xmm8 + pxor %xmm7, %xmm4 + pshufd $0x4e, %xmm0, %xmm5 + pxor %xmm0, %xmm5 + movdqa %xmm0, %xmm6 + pclmulqdq $0x11, %xmm7, %xmm6 + aesenc 96(%r15), %xmm9 + aesenc 96(%r15), %xmm10 + pclmulqdq $0x00, %xmm0, %xmm7 + aesenc 96(%r15), %xmm11 + aesenc 96(%r15), %xmm12 + pclmulqdq $0x00, %xmm5, %xmm4 + aesenc 96(%r15), %xmm13 + aesenc 96(%r15), %xmm14 + aesenc 96(%r15), %xmm15 + pxor %xmm7, %xmm1 + pxor %xmm7, %xmm2 + pxor %xmm6, %xmm1 + pxor %xmm6, %xmm3 + pxor %xmm4, %xmm1 + movdqa 16(%rsp), %xmm7 + movdqu -32(%rdx), %xmm0 + pshufd $0x4e, %xmm7, %xmm4 + pshufb L_aes_gcm_bswap_mask(%rip), %xmm0 + aesenc 112(%r15), %xmm8 + pxor %xmm7, %xmm4 + pshufd $0x4e, %xmm0, %xmm5 + pxor %xmm0, %xmm5 + movdqa %xmm0, %xmm6 + pclmulqdq $0x11, %xmm7, %xmm6 + aesenc 112(%r15), %xmm9 + aesenc 112(%r15), %xmm10 + pclmulqdq $0x00, %xmm0, %xmm7 + aesenc 112(%r15), %xmm11 + aesenc 112(%r15), %xmm12 + pclmulqdq $0x00, %xmm5, %xmm4 + aesenc 112(%r15), %xmm13 + aesenc 112(%r15), %xmm14 + aesenc 112(%r15), %xmm15 + pxor %xmm7, %xmm1 + pxor %xmm7, %xmm2 + pxor %xmm6, %xmm1 + pxor %xmm6, %xmm3 + pxor %xmm4, %xmm1 + movdqa (%rsp), %xmm7 + movdqu -16(%rdx), %xmm0 + pshufd $0x4e, %xmm7, %xmm4 + pshufb L_aes_gcm_bswap_mask(%rip), %xmm0 + aesenc 128(%r15), %xmm8 + pxor %xmm7, %xmm4 + pshufd $0x4e, %xmm0, %xmm5 + pxor %xmm0, %xmm5 + movdqa %xmm0, %xmm6 + pclmulqdq $0x11, %xmm7, %xmm6 + aesenc 128(%r15), %xmm9 + aesenc 128(%r15), %xmm10 + pclmulqdq $0x00, %xmm0, %xmm7 + aesenc 128(%r15), %xmm11 + aesenc 128(%r15), %xmm12 + pclmulqdq $0x00, %xmm5, %xmm4 + aesenc 128(%r15), %xmm13 + aesenc 128(%r15), %xmm14 + aesenc 128(%r15), %xmm15 + pxor %xmm7, %xmm1 + pxor %xmm7, %xmm2 + pxor %xmm6, %xmm1 + pxor %xmm6, %xmm3 + pxor %xmm4, %xmm1 + movdqa %xmm1, %xmm5 + psrldq $8, %xmm1 + pslldq $8, %xmm5 + aesenc 144(%r15), %xmm8 + pxor %xmm5, %xmm2 + pxor %xmm1, %xmm3 + movdqa %xmm2, %xmm7 + movdqa %xmm2, %xmm4 + movdqa %xmm2, %xmm5 + aesenc 144(%r15), %xmm9 + pslld $31, %xmm7 + pslld $30, %xmm4 + pslld $25, %xmm5 + aesenc 144(%r15), %xmm10 + pxor %xmm4, %xmm7 + pxor %xmm5, %xmm7 + aesenc 144(%r15), %xmm11 + movdqa %xmm7, %xmm4 + pslldq $12, %xmm7 + psrldq $4, %xmm4 + aesenc 144(%r15), %xmm12 + pxor %xmm7, %xmm2 + movdqa %xmm2, %xmm5 + movdqa %xmm2, %xmm1 + movdqa %xmm2, %xmm0 + aesenc 144(%r15), %xmm13 + psrld $0x01, %xmm5 + psrld $2, %xmm1 + psrld $7, %xmm0 + aesenc 144(%r15), %xmm14 + pxor %xmm1, %xmm5 + pxor %xmm0, %xmm5 + aesenc 144(%r15), %xmm15 + pxor %xmm4, %xmm5 + pxor %xmm5, %xmm2 + pxor %xmm3, %xmm2 + cmpl $11, %r10d + movdqa 160(%r15), %xmm7 + jl L_AES_GCM_encrypt_aesenc_128_ghash_avx_done + aesenc %xmm7, %xmm8 + aesenc %xmm7, %xmm9 + aesenc %xmm7, %xmm10 + aesenc %xmm7, %xmm11 + aesenc %xmm7, %xmm12 + aesenc %xmm7, %xmm13 + aesenc %xmm7, %xmm14 + aesenc %xmm7, %xmm15 + movdqa 176(%r15), %xmm7 + aesenc %xmm7, %xmm8 + aesenc %xmm7, %xmm9 + aesenc %xmm7, %xmm10 + aesenc %xmm7, %xmm11 + aesenc %xmm7, %xmm12 + aesenc %xmm7, %xmm13 + aesenc %xmm7, %xmm14 + aesenc %xmm7, %xmm15 + cmpl $13, %r10d + movdqa 192(%r15), %xmm7 + jl L_AES_GCM_encrypt_aesenc_128_ghash_avx_done + aesenc %xmm7, %xmm8 + aesenc %xmm7, %xmm9 + aesenc %xmm7, %xmm10 + aesenc %xmm7, %xmm11 + aesenc %xmm7, %xmm12 + aesenc %xmm7, %xmm13 + aesenc %xmm7, %xmm14 + aesenc %xmm7, %xmm15 + movdqa 208(%r15), %xmm7 + aesenc %xmm7, %xmm8 + aesenc %xmm7, %xmm9 + aesenc %xmm7, %xmm10 + aesenc %xmm7, %xmm11 + aesenc %xmm7, %xmm12 + aesenc %xmm7, %xmm13 + aesenc %xmm7, %xmm14 + aesenc %xmm7, %xmm15 + movdqa 224(%r15), %xmm7 +L_AES_GCM_encrypt_aesenc_128_ghash_avx_done: + aesenclast %xmm7, %xmm8 + aesenclast %xmm7, %xmm9 + movdqu (%rcx), %xmm0 + movdqu 16(%rcx), %xmm1 + pxor %xmm0, %xmm8 + pxor %xmm1, %xmm9 + movdqu %xmm8, (%rdx) + movdqu %xmm9, 16(%rdx) + aesenclast %xmm7, %xmm10 + aesenclast %xmm7, %xmm11 + movdqu 32(%rcx), %xmm0 + movdqu 48(%rcx), %xmm1 + pxor %xmm0, %xmm10 + pxor %xmm1, %xmm11 + movdqu %xmm10, 32(%rdx) + movdqu %xmm11, 48(%rdx) + aesenclast %xmm7, %xmm12 + aesenclast %xmm7, %xmm13 + movdqu 64(%rcx), %xmm0 + movdqu 80(%rcx), %xmm1 + pxor %xmm0, %xmm12 + pxor %xmm1, %xmm13 + movdqu %xmm12, 64(%rdx) + movdqu %xmm13, 80(%rdx) + aesenclast %xmm7, %xmm14 + aesenclast %xmm7, %xmm15 + movdqu 96(%rcx), %xmm0 + movdqu 112(%rcx), %xmm1 + pxor %xmm0, %xmm14 + pxor %xmm1, %xmm15 + movdqu %xmm14, 96(%rdx) + movdqu %xmm15, 112(%rdx) + addl $0x80, %ebx + cmpl %r13d, %ebx + jl L_AES_GCM_encrypt_ghash_128 +L_AES_GCM_encrypt_end_128: + movdqa L_aes_gcm_bswap_mask(%rip), %xmm4 + pshufb %xmm4, %xmm8 + pshufb %xmm4, %xmm9 + pshufb %xmm4, %xmm10 + pshufb %xmm4, %xmm11 + pxor %xmm2, %xmm8 + pshufb %xmm4, %xmm12 + pshufb %xmm4, %xmm13 + pshufb %xmm4, %xmm14 + pshufb %xmm4, %xmm15 + movdqa 112(%rsp), %xmm7 + pshufd $0x4e, %xmm8, %xmm1 + pshufd $0x4e, %xmm7, %xmm2 + movdqa %xmm7, %xmm3 + movdqa %xmm7, %xmm0 + pclmulqdq $0x11, %xmm8, %xmm3 + pclmulqdq $0x00, %xmm8, %xmm0 + pxor %xmm8, %xmm1 + pxor %xmm7, %xmm2 + pclmulqdq $0x00, %xmm2, %xmm1 + pxor %xmm0, %xmm1 + pxor %xmm3, %xmm1 + movdqa %xmm1, %xmm2 + movdqa %xmm0, %xmm4 + movdqa %xmm3, %xmm6 + pslldq $8, %xmm2 + psrldq $8, %xmm1 + pxor %xmm2, %xmm4 + pxor %xmm1, %xmm6 + movdqa 96(%rsp), %xmm7 + pshufd $0x4e, %xmm9, %xmm1 + pshufd $0x4e, %xmm7, %xmm2 + movdqa %xmm7, %xmm3 + movdqa %xmm7, %xmm0 + pclmulqdq $0x11, %xmm9, %xmm3 + pclmulqdq $0x00, %xmm9, %xmm0 + pxor %xmm9, %xmm1 + pxor %xmm7, %xmm2 + pclmulqdq $0x00, %xmm2, %xmm1 + pxor %xmm0, %xmm1 + pxor %xmm3, %xmm1 + movdqa %xmm1, %xmm2 + pxor %xmm0, %xmm4 + pxor %xmm3, %xmm6 + pslldq $8, %xmm2 + psrldq $8, %xmm1 + pxor %xmm2, %xmm4 + pxor %xmm1, %xmm6 + movdqa 80(%rsp), %xmm7 + pshufd $0x4e, %xmm10, %xmm1 + pshufd $0x4e, %xmm7, %xmm2 + movdqa %xmm7, %xmm3 + movdqa %xmm7, %xmm0 + pclmulqdq $0x11, %xmm10, %xmm3 + pclmulqdq $0x00, %xmm10, %xmm0 + pxor %xmm10, %xmm1 + pxor %xmm7, %xmm2 + pclmulqdq $0x00, %xmm2, %xmm1 + pxor %xmm0, %xmm1 + pxor %xmm3, %xmm1 + movdqa %xmm1, %xmm2 + pxor %xmm0, %xmm4 + pxor %xmm3, %xmm6 + pslldq $8, %xmm2 + psrldq $8, %xmm1 + pxor %xmm2, %xmm4 + pxor %xmm1, %xmm6 + movdqa 64(%rsp), %xmm7 + pshufd $0x4e, %xmm11, %xmm1 + pshufd $0x4e, %xmm7, %xmm2 + movdqa %xmm7, %xmm3 + movdqa %xmm7, %xmm0 + pclmulqdq $0x11, %xmm11, %xmm3 + pclmulqdq $0x00, %xmm11, %xmm0 + pxor %xmm11, %xmm1 + pxor %xmm7, %xmm2 + pclmulqdq $0x00, %xmm2, %xmm1 + pxor %xmm0, %xmm1 + pxor %xmm3, %xmm1 + movdqa %xmm1, %xmm2 + pxor %xmm0, %xmm4 + pxor %xmm3, %xmm6 + pslldq $8, %xmm2 + psrldq $8, %xmm1 + pxor %xmm2, %xmm4 + pxor %xmm1, %xmm6 + movdqa 48(%rsp), %xmm7 + pshufd $0x4e, %xmm12, %xmm1 + pshufd $0x4e, %xmm7, %xmm2 + movdqa %xmm7, %xmm3 + movdqa %xmm7, %xmm0 + pclmulqdq $0x11, %xmm12, %xmm3 + pclmulqdq $0x00, %xmm12, %xmm0 + pxor %xmm12, %xmm1 + pxor %xmm7, %xmm2 + pclmulqdq $0x00, %xmm2, %xmm1 + pxor %xmm0, %xmm1 + pxor %xmm3, %xmm1 + movdqa %xmm1, %xmm2 + pxor %xmm0, %xmm4 + pxor %xmm3, %xmm6 + pslldq $8, %xmm2 + psrldq $8, %xmm1 + pxor %xmm2, %xmm4 + pxor %xmm1, %xmm6 + movdqa 32(%rsp), %xmm7 + pshufd $0x4e, %xmm13, %xmm1 + pshufd $0x4e, %xmm7, %xmm2 + movdqa %xmm7, %xmm3 + movdqa %xmm7, %xmm0 + pclmulqdq $0x11, %xmm13, %xmm3 + pclmulqdq $0x00, %xmm13, %xmm0 + pxor %xmm13, %xmm1 + pxor %xmm7, %xmm2 + pclmulqdq $0x00, %xmm2, %xmm1 + pxor %xmm0, %xmm1 + pxor %xmm3, %xmm1 + movdqa %xmm1, %xmm2 + pxor %xmm0, %xmm4 + pxor %xmm3, %xmm6 + pslldq $8, %xmm2 + psrldq $8, %xmm1 + pxor %xmm2, %xmm4 + pxor %xmm1, %xmm6 + movdqa 16(%rsp), %xmm7 + pshufd $0x4e, %xmm14, %xmm1 + pshufd $0x4e, %xmm7, %xmm2 + movdqa %xmm7, %xmm3 + movdqa %xmm7, %xmm0 + pclmulqdq $0x11, %xmm14, %xmm3 + pclmulqdq $0x00, %xmm14, %xmm0 + pxor %xmm14, %xmm1 + pxor %xmm7, %xmm2 + pclmulqdq $0x00, %xmm2, %xmm1 + pxor %xmm0, %xmm1 + pxor %xmm3, %xmm1 + movdqa %xmm1, %xmm2 + pxor %xmm0, %xmm4 + pxor %xmm3, %xmm6 + pslldq $8, %xmm2 + psrldq $8, %xmm1 + pxor %xmm2, %xmm4 + pxor %xmm1, %xmm6 + movdqa (%rsp), %xmm7 + pshufd $0x4e, %xmm15, %xmm1 + pshufd $0x4e, %xmm7, %xmm2 + movdqa %xmm7, %xmm3 + movdqa %xmm7, %xmm0 + pclmulqdq $0x11, %xmm15, %xmm3 + pclmulqdq $0x00, %xmm15, %xmm0 + pxor %xmm15, %xmm1 + pxor %xmm7, %xmm2 + pclmulqdq $0x00, %xmm2, %xmm1 + pxor %xmm0, %xmm1 + pxor %xmm3, %xmm1 + movdqa %xmm1, %xmm2 + pxor %xmm0, %xmm4 + pxor %xmm3, %xmm6 + pslldq $8, %xmm2 + psrldq $8, %xmm1 + pxor %xmm2, %xmm4 + pxor %xmm1, %xmm6 + movdqa %xmm4, %xmm0 + movdqa %xmm4, %xmm1 + movdqa %xmm4, %xmm2 + pslld $31, %xmm0 + pslld $30, %xmm1 + pslld $25, %xmm2 + pxor %xmm1, %xmm0 + pxor %xmm2, %xmm0 + movdqa %xmm0, %xmm1 + psrldq $4, %xmm1 + pslldq $12, %xmm0 + pxor %xmm0, %xmm4 + movdqa %xmm4, %xmm2 + movdqa %xmm4, %xmm3 + movdqa %xmm4, %xmm0 + psrld $0x01, %xmm2 + psrld $2, %xmm3 + psrld $7, %xmm0 + pxor %xmm3, %xmm2 + pxor %xmm0, %xmm2 + pxor %xmm1, %xmm2 + pxor %xmm4, %xmm2 + pxor %xmm2, %xmm6 + movdqa (%rsp), %xmm5 +L_AES_GCM_encrypt_done_128: + movl %r9d, %edx + cmpl %edx, %ebx + jge L_AES_GCM_encrypt_done_enc + movl %r9d, %r13d + andl $0xfffffff0, %r13d + cmpl %r13d, %ebx + jge L_AES_GCM_encrypt_last_block_done + leaq (%rdi,%rbx,1), %rcx + leaq (%rsi,%rbx,1), %rdx + movdqa 128(%rsp), %xmm8 + movdqa %xmm8, %xmm9 + pshufb L_aes_gcm_bswap_epi64(%rip), %xmm8 + paddd L_aes_gcm_one(%rip), %xmm9 + pxor (%r15), %xmm8 + movdqa %xmm9, 128(%rsp) + aesenc 16(%r15), %xmm8 + aesenc 32(%r15), %xmm8 + aesenc 48(%r15), %xmm8 + aesenc 64(%r15), %xmm8 + aesenc 80(%r15), %xmm8 + aesenc 96(%r15), %xmm8 + aesenc 112(%r15), %xmm8 + aesenc 128(%r15), %xmm8 + aesenc 144(%r15), %xmm8 + cmpl $11, %r10d + movdqa 160(%r15), %xmm9 + jl L_AES_GCM_encrypt_aesenc_block_aesenc_avx_last + aesenc %xmm9, %xmm8 + aesenc 176(%r15), %xmm8 + cmpl $13, %r10d + movdqa 192(%r15), %xmm9 + jl L_AES_GCM_encrypt_aesenc_block_aesenc_avx_last + aesenc %xmm9, %xmm8 + aesenc 208(%r15), %xmm8 + movdqa 224(%r15), %xmm9 +L_AES_GCM_encrypt_aesenc_block_aesenc_avx_last: + aesenclast %xmm9, %xmm8 + movdqu (%rcx), %xmm9 + pxor %xmm9, %xmm8 + movdqu %xmm8, (%rdx) + pshufb L_aes_gcm_bswap_mask(%rip), %xmm8 + pxor %xmm8, %xmm6 + addl $16, %ebx + cmpl %r13d, %ebx + jge L_AES_GCM_encrypt_last_block_ghash +L_AES_GCM_encrypt_last_block_start: + leaq (%rdi,%rbx,1), %rcx + leaq (%rsi,%rbx,1), %rdx + movdqa 128(%rsp), %xmm8 + movdqa %xmm8, %xmm9 + pshufb L_aes_gcm_bswap_epi64(%rip), %xmm8 + paddd L_aes_gcm_one(%rip), %xmm9 + pxor (%r15), %xmm8 + movdqa %xmm9, 128(%rsp) + movdqa %xmm6, %xmm10 + pclmulqdq $16, %xmm5, %xmm10 + aesenc 16(%r15), %xmm8 + aesenc 32(%r15), %xmm8 + movdqa %xmm6, %xmm11 + pclmulqdq $0x01, %xmm5, %xmm11 + aesenc 48(%r15), %xmm8 + aesenc 64(%r15), %xmm8 + movdqa %xmm6, %xmm12 + pclmulqdq $0x00, %xmm5, %xmm12 + aesenc 80(%r15), %xmm8 + movdqa %xmm6, %xmm1 + pclmulqdq $0x11, %xmm5, %xmm1 + aesenc 96(%r15), %xmm8 + pxor %xmm11, %xmm10 + movdqa %xmm10, %xmm2 + psrldq $8, %xmm10 + pslldq $8, %xmm2 + aesenc 112(%r15), %xmm8 + movdqa %xmm1, %xmm3 + pxor %xmm12, %xmm2 + pxor %xmm10, %xmm3 + movdqa L_aes_gcm_mod2_128(%rip), %xmm0 + movdqa %xmm2, %xmm11 + pclmulqdq $16, %xmm0, %xmm11 + aesenc 128(%r15), %xmm8 + pshufd $0x4e, %xmm2, %xmm10 + pxor %xmm11, %xmm10 + movdqa %xmm10, %xmm11 + pclmulqdq $16, %xmm0, %xmm11 + aesenc 144(%r15), %xmm8 + pshufd $0x4e, %xmm10, %xmm6 + pxor %xmm11, %xmm6 + pxor %xmm3, %xmm6 + cmpl $11, %r10d + movdqa 160(%r15), %xmm9 + jl L_AES_GCM_encrypt_aesenc_gfmul_last + aesenc %xmm9, %xmm8 + aesenc 176(%r15), %xmm8 + cmpl $13, %r10d + movdqa 192(%r15), %xmm9 + jl L_AES_GCM_encrypt_aesenc_gfmul_last + aesenc %xmm9, %xmm8 + aesenc 208(%r15), %xmm8 + movdqa 224(%r15), %xmm9 +L_AES_GCM_encrypt_aesenc_gfmul_last: + aesenclast %xmm9, %xmm8 + movdqu (%rcx), %xmm9 + pxor %xmm9, %xmm8 + movdqu %xmm8, (%rdx) + pshufb L_aes_gcm_bswap_mask(%rip), %xmm8 + pxor %xmm8, %xmm6 + addl $16, %ebx + cmpl %r13d, %ebx + jl L_AES_GCM_encrypt_last_block_start +L_AES_GCM_encrypt_last_block_ghash: + pshufd $0x4e, %xmm5, %xmm9 + pshufd $0x4e, %xmm6, %xmm10 + movdqa %xmm6, %xmm11 + movdqa %xmm6, %xmm8 + pclmulqdq $0x11, %xmm5, %xmm11 + pclmulqdq $0x00, %xmm5, %xmm8 + pxor %xmm5, %xmm9 + pxor %xmm6, %xmm10 + pclmulqdq $0x00, %xmm10, %xmm9 + pxor %xmm8, %xmm9 + pxor %xmm11, %xmm9 + movdqa %xmm9, %xmm10 + movdqa %xmm11, %xmm6 + pslldq $8, %xmm10 + psrldq $8, %xmm9 + pxor %xmm10, %xmm8 + pxor %xmm9, %xmm6 + movdqa %xmm8, %xmm12 + movdqa %xmm8, %xmm13 + movdqa %xmm8, %xmm14 + pslld $31, %xmm12 + pslld $30, %xmm13 + pslld $25, %xmm14 + pxor %xmm13, %xmm12 + pxor %xmm14, %xmm12 + movdqa %xmm12, %xmm13 + psrldq $4, %xmm13 + pslldq $12, %xmm12 + pxor %xmm12, %xmm8 + movdqa %xmm8, %xmm14 + movdqa %xmm8, %xmm10 + movdqa %xmm8, %xmm9 + psrld $0x01, %xmm14 + psrld $2, %xmm10 + psrld $7, %xmm9 + pxor %xmm10, %xmm14 + pxor %xmm9, %xmm14 + pxor %xmm13, %xmm14 + pxor %xmm8, %xmm14 + pxor %xmm14, %xmm6 +L_AES_GCM_encrypt_last_block_done: + movl %r9d, %ecx + movl %ecx, %edx + andl $15, %ecx + jz L_AES_GCM_encrypt_aesenc_last15_enc_avx_done + movdqa 128(%rsp), %xmm4 + pshufb L_aes_gcm_bswap_epi64(%rip), %xmm4 + pxor (%r15), %xmm4 + aesenc 16(%r15), %xmm4 + aesenc 32(%r15), %xmm4 + aesenc 48(%r15), %xmm4 + aesenc 64(%r15), %xmm4 + aesenc 80(%r15), %xmm4 + aesenc 96(%r15), %xmm4 + aesenc 112(%r15), %xmm4 + aesenc 128(%r15), %xmm4 + aesenc 144(%r15), %xmm4 + cmpl $11, %r10d + movdqa 160(%r15), %xmm9 + jl L_AES_GCM_encrypt_aesenc_last15_enc_avx_aesenc_avx_last + aesenc %xmm9, %xmm4 + aesenc 176(%r15), %xmm4 + cmpl $13, %r10d + movdqa 192(%r15), %xmm9 + jl L_AES_GCM_encrypt_aesenc_last15_enc_avx_aesenc_avx_last + aesenc %xmm9, %xmm4 + aesenc 208(%r15), %xmm4 + movdqa 224(%r15), %xmm9 +L_AES_GCM_encrypt_aesenc_last15_enc_avx_aesenc_avx_last: + aesenclast %xmm9, %xmm4 + subq $16, %rsp + xorl %ecx, %ecx + movdqa %xmm4, (%rsp) +L_AES_GCM_encrypt_aesenc_last15_enc_avx_loop: + movzbl (%rdi,%rbx,1), %r13d + xorb (%rsp,%rcx,1), %r13b + movb %r13b, (%rsi,%rbx,1) + movb %r13b, (%rsp,%rcx,1) + incl %ebx + incl %ecx + cmpl %edx, %ebx + jl L_AES_GCM_encrypt_aesenc_last15_enc_avx_loop + xorq %r13, %r13 + cmpl $16, %ecx + je L_AES_GCM_encrypt_aesenc_last15_enc_avx_finish_enc +L_AES_GCM_encrypt_aesenc_last15_enc_avx_byte_loop: + movb %r13b, (%rsp,%rcx,1) + incl %ecx + cmpl $16, %ecx + jl L_AES_GCM_encrypt_aesenc_last15_enc_avx_byte_loop +L_AES_GCM_encrypt_aesenc_last15_enc_avx_finish_enc: + movdqa (%rsp), %xmm4 + addq $16, %rsp + pshufb L_aes_gcm_bswap_mask(%rip), %xmm4 + pxor %xmm4, %xmm6 + pshufd $0x4e, %xmm5, %xmm9 + pshufd $0x4e, %xmm6, %xmm10 + movdqa %xmm6, %xmm11 + movdqa %xmm6, %xmm8 + pclmulqdq $0x11, %xmm5, %xmm11 + pclmulqdq $0x00, %xmm5, %xmm8 + pxor %xmm5, %xmm9 + pxor %xmm6, %xmm10 + pclmulqdq $0x00, %xmm10, %xmm9 + pxor %xmm8, %xmm9 + pxor %xmm11, %xmm9 + movdqa %xmm9, %xmm10 + movdqa %xmm11, %xmm6 + pslldq $8, %xmm10 + psrldq $8, %xmm9 + pxor %xmm10, %xmm8 + pxor %xmm9, %xmm6 + movdqa %xmm8, %xmm12 + movdqa %xmm8, %xmm13 + movdqa %xmm8, %xmm14 + pslld $31, %xmm12 + pslld $30, %xmm13 + pslld $25, %xmm14 + pxor %xmm13, %xmm12 + pxor %xmm14, %xmm12 + movdqa %xmm12, %xmm13 + psrldq $4, %xmm13 + pslldq $12, %xmm12 + pxor %xmm12, %xmm8 + movdqa %xmm8, %xmm14 + movdqa %xmm8, %xmm10 + movdqa %xmm8, %xmm9 + psrld $0x01, %xmm14 + psrld $2, %xmm10 + psrld $7, %xmm9 + pxor %xmm10, %xmm14 + pxor %xmm9, %xmm14 + pxor %xmm13, %xmm14 + pxor %xmm8, %xmm14 + pxor %xmm14, %xmm6 +L_AES_GCM_encrypt_aesenc_last15_enc_avx_done: +L_AES_GCM_encrypt_done_enc: + movl %r9d, %edx + movl %r11d, %ecx + shlq $3, %rdx + shlq $3, %rcx + pinsrq $0x00, %rdx, %xmm0 + pinsrq $0x01, %rcx, %xmm0 + pxor %xmm0, %xmm6 + pshufd $0x4e, %xmm5, %xmm9 + pshufd $0x4e, %xmm6, %xmm10 + movdqa %xmm6, %xmm11 + movdqa %xmm6, %xmm8 + pclmulqdq $0x11, %xmm5, %xmm11 + pclmulqdq $0x00, %xmm5, %xmm8 + pxor %xmm5, %xmm9 + pxor %xmm6, %xmm10 + pclmulqdq $0x00, %xmm10, %xmm9 + pxor %xmm8, %xmm9 + pxor %xmm11, %xmm9 + movdqa %xmm9, %xmm10 + movdqa %xmm11, %xmm6 + pslldq $8, %xmm10 + psrldq $8, %xmm9 + pxor %xmm10, %xmm8 + pxor %xmm9, %xmm6 + movdqa %xmm8, %xmm12 + movdqa %xmm8, %xmm13 + movdqa %xmm8, %xmm14 + pslld $31, %xmm12 + pslld $30, %xmm13 + pslld $25, %xmm14 + pxor %xmm13, %xmm12 + pxor %xmm14, %xmm12 + movdqa %xmm12, %xmm13 + psrldq $4, %xmm13 + pslldq $12, %xmm12 + pxor %xmm12, %xmm8 + movdqa %xmm8, %xmm14 + movdqa %xmm8, %xmm10 + movdqa %xmm8, %xmm9 + psrld $0x01, %xmm14 + psrld $2, %xmm10 + psrld $7, %xmm9 + pxor %xmm10, %xmm14 + pxor %xmm9, %xmm14 + pxor %xmm13, %xmm14 + pxor %xmm8, %xmm14 + pxor %xmm14, %xmm6 + pshufb L_aes_gcm_bswap_mask(%rip), %xmm6 + movdqa 144(%rsp), %xmm0 + pxor %xmm6, %xmm0 + cmpl $16, %r14d + je L_AES_GCM_encrypt_store_tag_16 + xorq %rcx, %rcx + movdqa %xmm0, (%rsp) +L_AES_GCM_encrypt_store_tag_loop: + movzbl (%rsp,%rcx,1), %r13d + movb %r13b, (%r8,%rcx,1) + incl %ecx + cmpl %r14d, %ecx + jne L_AES_GCM_encrypt_store_tag_loop + jmp L_AES_GCM_encrypt_store_tag_done +L_AES_GCM_encrypt_store_tag_16: + movdqu %xmm0, (%r8) +L_AES_GCM_encrypt_store_tag_done: + addq $0xa0, %rsp + popq %r15 + popq %r14 + popq %rbx + popq %r12 + popq %r13 + repz retq +.size AES_GCM_encrypt,.-AES_GCM_encrypt +.globl AES_GCM_decrypt +.type AES_GCM_decrypt,@function +.align 4 +AES_GCM_decrypt: + pushq %r13 + pushq %r12 + pushq %rbx + pushq %r14 + pushq %r15 + pushq %rbp + movq %rdx, %r12 + movq %rcx, %rax + movq 56(%rsp), %r11 + movq 64(%rsp), %rbx + movq 72(%rsp), %r14 + movq 80(%rsp), %r15 + movq 88(%rsp), %r10 + movq 96(%rsp), %rbp + subq $0xa8, %rsp + pxor %xmm4, %xmm4 + pxor %xmm6, %xmm6 + cmpl $12, %ebx + movl %ebx, %edx + jne L_AES_GCM_decrypt_iv_not_12 + # # Calculate values when IV is 12 bytes + # Set counter based on IV + movl $0x1000000, %ecx + pinsrq $0x00, (%rax), %xmm4 + pinsrd $2, 8(%rax), %xmm4 + pinsrd $3, %ecx, %xmm4 + # H = Encrypt X(=0) and T = Encrypt counter + movdqa %xmm4, %xmm1 + movdqa (%r15), %xmm5 + pxor %xmm5, %xmm1 + movdqa 16(%r15), %xmm7 + aesenc %xmm7, %xmm5 + aesenc %xmm7, %xmm1 + movdqa 32(%r15), %xmm7 + aesenc %xmm7, %xmm5 + aesenc %xmm7, %xmm1 + movdqa 48(%r15), %xmm7 + aesenc %xmm7, %xmm5 + aesenc %xmm7, %xmm1 + movdqa 64(%r15), %xmm7 + aesenc %xmm7, %xmm5 + aesenc %xmm7, %xmm1 + movdqa 80(%r15), %xmm7 + aesenc %xmm7, %xmm5 + aesenc %xmm7, %xmm1 + movdqa 96(%r15), %xmm7 + aesenc %xmm7, %xmm5 + aesenc %xmm7, %xmm1 + movdqa 112(%r15), %xmm7 + aesenc %xmm7, %xmm5 + aesenc %xmm7, %xmm1 + movdqa 128(%r15), %xmm7 + aesenc %xmm7, %xmm5 + aesenc %xmm7, %xmm1 + movdqa 144(%r15), %xmm7 + aesenc %xmm7, %xmm5 + aesenc %xmm7, %xmm1 + cmpl $11, %r10d + movdqa 160(%r15), %xmm7 + jl L_AES_GCM_decrypt_calc_iv_12_last + aesenc %xmm7, %xmm5 + aesenc %xmm7, %xmm1 + movdqa 176(%r15), %xmm7 + aesenc %xmm7, %xmm5 + aesenc %xmm7, %xmm1 + cmpl $13, %r10d + movdqa 192(%r15), %xmm7 + jl L_AES_GCM_decrypt_calc_iv_12_last + aesenc %xmm7, %xmm5 + aesenc %xmm7, %xmm1 + movdqa 208(%r15), %xmm7 + aesenc %xmm7, %xmm5 + aesenc %xmm7, %xmm1 + movdqa 224(%r15), %xmm7 +L_AES_GCM_decrypt_calc_iv_12_last: + aesenclast %xmm7, %xmm5 + aesenclast %xmm7, %xmm1 + pshufb L_aes_gcm_bswap_mask(%rip), %xmm5 + movdqa %xmm1, 144(%rsp) + jmp L_AES_GCM_decrypt_iv_done +L_AES_GCM_decrypt_iv_not_12: + # Calculate values when IV is not 12 bytes + # H = Encrypt X(=0) + movdqa (%r15), %xmm5 + aesenc 16(%r15), %xmm5 + aesenc 32(%r15), %xmm5 + aesenc 48(%r15), %xmm5 + aesenc 64(%r15), %xmm5 + aesenc 80(%r15), %xmm5 + aesenc 96(%r15), %xmm5 + aesenc 112(%r15), %xmm5 + aesenc 128(%r15), %xmm5 + aesenc 144(%r15), %xmm5 + cmpl $11, %r10d + movdqa 160(%r15), %xmm9 + jl L_AES_GCM_decrypt_calc_iv_1_aesenc_avx_last + aesenc %xmm9, %xmm5 + aesenc 176(%r15), %xmm5 + cmpl $13, %r10d + movdqa 192(%r15), %xmm9 + jl L_AES_GCM_decrypt_calc_iv_1_aesenc_avx_last + aesenc %xmm9, %xmm5 + aesenc 208(%r15), %xmm5 + movdqa 224(%r15), %xmm9 +L_AES_GCM_decrypt_calc_iv_1_aesenc_avx_last: + aesenclast %xmm9, %xmm5 + pshufb L_aes_gcm_bswap_mask(%rip), %xmm5 + # Calc counter + # Initialization vector + cmpl $0x00, %edx + movq $0x00, %rcx + je L_AES_GCM_decrypt_calc_iv_done + cmpl $16, %edx + jl L_AES_GCM_decrypt_calc_iv_lt16 + andl $0xfffffff0, %edx +L_AES_GCM_decrypt_calc_iv_16_loop: + movdqu (%rax,%rcx,1), %xmm8 + pshufb L_aes_gcm_bswap_mask(%rip), %xmm8 + pxor %xmm8, %xmm4 + pshufd $0x4e, %xmm4, %xmm1 + pshufd $0x4e, %xmm5, %xmm2 + movdqa %xmm5, %xmm3 + movdqa %xmm5, %xmm0 + pclmulqdq $0x11, %xmm4, %xmm3 + pclmulqdq $0x00, %xmm4, %xmm0 + pxor %xmm4, %xmm1 + pxor %xmm5, %xmm2 + pclmulqdq $0x00, %xmm2, %xmm1 + pxor %xmm0, %xmm1 + pxor %xmm3, %xmm1 + movdqa %xmm1, %xmm2 + movdqa %xmm0, %xmm7 + movdqa %xmm3, %xmm4 + pslldq $8, %xmm2 + psrldq $8, %xmm1 + pxor %xmm2, %xmm7 + pxor %xmm1, %xmm4 + movdqa %xmm7, %xmm0 + movdqa %xmm4, %xmm1 + psrld $31, %xmm0 + psrld $31, %xmm1 + pslld $0x01, %xmm7 + pslld $0x01, %xmm4 + movdqa %xmm0, %xmm2 + pslldq $4, %xmm0 + psrldq $12, %xmm2 + pslldq $4, %xmm1 + por %xmm2, %xmm4 + por %xmm0, %xmm7 + por %xmm1, %xmm4 + movdqa %xmm7, %xmm0 + movdqa %xmm7, %xmm1 + movdqa %xmm7, %xmm2 + pslld $31, %xmm0 + pslld $30, %xmm1 + pslld $25, %xmm2 + pxor %xmm1, %xmm0 + pxor %xmm2, %xmm0 + movdqa %xmm0, %xmm1 + psrldq $4, %xmm1 + pslldq $12, %xmm0 + pxor %xmm0, %xmm7 + movdqa %xmm7, %xmm2 + movdqa %xmm7, %xmm3 + movdqa %xmm7, %xmm0 + psrld $0x01, %xmm2 + psrld $2, %xmm3 + psrld $7, %xmm0 + pxor %xmm3, %xmm2 + pxor %xmm0, %xmm2 + pxor %xmm1, %xmm2 + pxor %xmm7, %xmm2 + pxor %xmm2, %xmm4 + addl $16, %ecx + cmpl %edx, %ecx + jl L_AES_GCM_decrypt_calc_iv_16_loop + movl %ebx, %edx + cmpl %edx, %ecx + je L_AES_GCM_decrypt_calc_iv_done +L_AES_GCM_decrypt_calc_iv_lt16: + subq $16, %rsp + pxor %xmm8, %xmm8 + xorl %ebx, %ebx + movdqa %xmm8, (%rsp) +L_AES_GCM_decrypt_calc_iv_loop: + movzbl (%rax,%rcx,1), %r13d + movb %r13b, (%rsp,%rbx,1) + incl %ecx + incl %ebx + cmpl %edx, %ecx + jl L_AES_GCM_decrypt_calc_iv_loop + movdqa (%rsp), %xmm8 + addq $16, %rsp + pshufb L_aes_gcm_bswap_mask(%rip), %xmm8 + pxor %xmm8, %xmm4 + pshufd $0x4e, %xmm4, %xmm1 + pshufd $0x4e, %xmm5, %xmm2 + movdqa %xmm5, %xmm3 + movdqa %xmm5, %xmm0 + pclmulqdq $0x11, %xmm4, %xmm3 + pclmulqdq $0x00, %xmm4, %xmm0 + pxor %xmm4, %xmm1 + pxor %xmm5, %xmm2 + pclmulqdq $0x00, %xmm2, %xmm1 + pxor %xmm0, %xmm1 + pxor %xmm3, %xmm1 + movdqa %xmm1, %xmm2 + movdqa %xmm0, %xmm7 + movdqa %xmm3, %xmm4 + pslldq $8, %xmm2 + psrldq $8, %xmm1 + pxor %xmm2, %xmm7 + pxor %xmm1, %xmm4 + movdqa %xmm7, %xmm0 + movdqa %xmm4, %xmm1 + psrld $31, %xmm0 + psrld $31, %xmm1 + pslld $0x01, %xmm7 + pslld $0x01, %xmm4 + movdqa %xmm0, %xmm2 + pslldq $4, %xmm0 + psrldq $12, %xmm2 + pslldq $4, %xmm1 + por %xmm2, %xmm4 + por %xmm0, %xmm7 + por %xmm1, %xmm4 + movdqa %xmm7, %xmm0 + movdqa %xmm7, %xmm1 + movdqa %xmm7, %xmm2 + pslld $31, %xmm0 + pslld $30, %xmm1 + pslld $25, %xmm2 + pxor %xmm1, %xmm0 + pxor %xmm2, %xmm0 + movdqa %xmm0, %xmm1 + psrldq $4, %xmm1 + pslldq $12, %xmm0 + pxor %xmm0, %xmm7 + movdqa %xmm7, %xmm2 + movdqa %xmm7, %xmm3 + movdqa %xmm7, %xmm0 + psrld $0x01, %xmm2 + psrld $2, %xmm3 + psrld $7, %xmm0 + pxor %xmm3, %xmm2 + pxor %xmm0, %xmm2 + pxor %xmm1, %xmm2 + pxor %xmm7, %xmm2 + pxor %xmm2, %xmm4 +L_AES_GCM_decrypt_calc_iv_done: + # T = Encrypt counter + pxor %xmm0, %xmm0 + shll $3, %edx + pinsrq $0x00, %rdx, %xmm0 + pxor %xmm0, %xmm4 + pshufd $0x4e, %xmm4, %xmm1 + pshufd $0x4e, %xmm5, %xmm2 + movdqa %xmm5, %xmm3 + movdqa %xmm5, %xmm0 + pclmulqdq $0x11, %xmm4, %xmm3 + pclmulqdq $0x00, %xmm4, %xmm0 + pxor %xmm4, %xmm1 + pxor %xmm5, %xmm2 + pclmulqdq $0x00, %xmm2, %xmm1 + pxor %xmm0, %xmm1 + pxor %xmm3, %xmm1 + movdqa %xmm1, %xmm2 + movdqa %xmm0, %xmm7 + movdqa %xmm3, %xmm4 + pslldq $8, %xmm2 + psrldq $8, %xmm1 + pxor %xmm2, %xmm7 + pxor %xmm1, %xmm4 + movdqa %xmm7, %xmm0 + movdqa %xmm4, %xmm1 + psrld $31, %xmm0 + psrld $31, %xmm1 + pslld $0x01, %xmm7 + pslld $0x01, %xmm4 + movdqa %xmm0, %xmm2 + pslldq $4, %xmm0 + psrldq $12, %xmm2 + pslldq $4, %xmm1 + por %xmm2, %xmm4 + por %xmm0, %xmm7 + por %xmm1, %xmm4 + movdqa %xmm7, %xmm0 + movdqa %xmm7, %xmm1 + movdqa %xmm7, %xmm2 + pslld $31, %xmm0 + pslld $30, %xmm1 + pslld $25, %xmm2 + pxor %xmm1, %xmm0 + pxor %xmm2, %xmm0 + movdqa %xmm0, %xmm1 + psrldq $4, %xmm1 + pslldq $12, %xmm0 + pxor %xmm0, %xmm7 + movdqa %xmm7, %xmm2 + movdqa %xmm7, %xmm3 + movdqa %xmm7, %xmm0 + psrld $0x01, %xmm2 + psrld $2, %xmm3 + psrld $7, %xmm0 + pxor %xmm3, %xmm2 + pxor %xmm0, %xmm2 + pxor %xmm1, %xmm2 + pxor %xmm7, %xmm2 + pxor %xmm2, %xmm4 + pshufb L_aes_gcm_bswap_mask(%rip), %xmm4 + # Encrypt counter + movdqa (%r15), %xmm8 + pxor %xmm4, %xmm8 + aesenc 16(%r15), %xmm8 + aesenc 32(%r15), %xmm8 + aesenc 48(%r15), %xmm8 + aesenc 64(%r15), %xmm8 + aesenc 80(%r15), %xmm8 + aesenc 96(%r15), %xmm8 + aesenc 112(%r15), %xmm8 + aesenc 128(%r15), %xmm8 + aesenc 144(%r15), %xmm8 + cmpl $11, %r10d + movdqa 160(%r15), %xmm9 + jl L_AES_GCM_decrypt_calc_iv_2_aesenc_avx_last + aesenc %xmm9, %xmm8 + aesenc 176(%r15), %xmm8 + cmpl $13, %r10d + movdqa 192(%r15), %xmm9 + jl L_AES_GCM_decrypt_calc_iv_2_aesenc_avx_last + aesenc %xmm9, %xmm8 + aesenc 208(%r15), %xmm8 + movdqa 224(%r15), %xmm9 +L_AES_GCM_decrypt_calc_iv_2_aesenc_avx_last: + aesenclast %xmm9, %xmm8 + movdqa %xmm8, 144(%rsp) +L_AES_GCM_decrypt_iv_done: + # Additional authentication data + movl %r11d, %edx + cmpl $0x00, %edx + je L_AES_GCM_decrypt_calc_aad_done + xorl %ecx, %ecx + cmpl $16, %edx + jl L_AES_GCM_decrypt_calc_aad_lt16 + andl $0xfffffff0, %edx +L_AES_GCM_decrypt_calc_aad_16_loop: + movdqu (%r12,%rcx,1), %xmm8 + pshufb L_aes_gcm_bswap_mask(%rip), %xmm8 + pxor %xmm8, %xmm6 + pshufd $0x4e, %xmm6, %xmm1 + pshufd $0x4e, %xmm5, %xmm2 + movdqa %xmm5, %xmm3 + movdqa %xmm5, %xmm0 + pclmulqdq $0x11, %xmm6, %xmm3 + pclmulqdq $0x00, %xmm6, %xmm0 + pxor %xmm6, %xmm1 + pxor %xmm5, %xmm2 + pclmulqdq $0x00, %xmm2, %xmm1 + pxor %xmm0, %xmm1 + pxor %xmm3, %xmm1 + movdqa %xmm1, %xmm2 + movdqa %xmm0, %xmm7 + movdqa %xmm3, %xmm6 + pslldq $8, %xmm2 + psrldq $8, %xmm1 + pxor %xmm2, %xmm7 + pxor %xmm1, %xmm6 + movdqa %xmm7, %xmm0 + movdqa %xmm6, %xmm1 + psrld $31, %xmm0 + psrld $31, %xmm1 + pslld $0x01, %xmm7 + pslld $0x01, %xmm6 + movdqa %xmm0, %xmm2 + pslldq $4, %xmm0 + psrldq $12, %xmm2 + pslldq $4, %xmm1 + por %xmm2, %xmm6 + por %xmm0, %xmm7 + por %xmm1, %xmm6 + movdqa %xmm7, %xmm0 + movdqa %xmm7, %xmm1 + movdqa %xmm7, %xmm2 + pslld $31, %xmm0 + pslld $30, %xmm1 + pslld $25, %xmm2 + pxor %xmm1, %xmm0 + pxor %xmm2, %xmm0 + movdqa %xmm0, %xmm1 + psrldq $4, %xmm1 + pslldq $12, %xmm0 + pxor %xmm0, %xmm7 + movdqa %xmm7, %xmm2 + movdqa %xmm7, %xmm3 + movdqa %xmm7, %xmm0 + psrld $0x01, %xmm2 + psrld $2, %xmm3 + psrld $7, %xmm0 + pxor %xmm3, %xmm2 + pxor %xmm0, %xmm2 + pxor %xmm1, %xmm2 + pxor %xmm7, %xmm2 + pxor %xmm2, %xmm6 + addl $16, %ecx + cmpl %edx, %ecx + jl L_AES_GCM_decrypt_calc_aad_16_loop + movl %r11d, %edx + cmpl %edx, %ecx + je L_AES_GCM_decrypt_calc_aad_done +L_AES_GCM_decrypt_calc_aad_lt16: + subq $16, %rsp + pxor %xmm8, %xmm8 + xorl %ebx, %ebx + movdqa %xmm8, (%rsp) +L_AES_GCM_decrypt_calc_aad_loop: + movzbl (%r12,%rcx,1), %r13d + movb %r13b, (%rsp,%rbx,1) + incl %ecx + incl %ebx + cmpl %edx, %ecx + jl L_AES_GCM_decrypt_calc_aad_loop + movdqa (%rsp), %xmm8 + addq $16, %rsp + pshufb L_aes_gcm_bswap_mask(%rip), %xmm8 + pxor %xmm8, %xmm6 + pshufd $0x4e, %xmm6, %xmm1 + pshufd $0x4e, %xmm5, %xmm2 + movdqa %xmm5, %xmm3 + movdqa %xmm5, %xmm0 + pclmulqdq $0x11, %xmm6, %xmm3 + pclmulqdq $0x00, %xmm6, %xmm0 + pxor %xmm6, %xmm1 + pxor %xmm5, %xmm2 + pclmulqdq $0x00, %xmm2, %xmm1 + pxor %xmm0, %xmm1 + pxor %xmm3, %xmm1 + movdqa %xmm1, %xmm2 + movdqa %xmm0, %xmm7 + movdqa %xmm3, %xmm6 + pslldq $8, %xmm2 + psrldq $8, %xmm1 + pxor %xmm2, %xmm7 + pxor %xmm1, %xmm6 + movdqa %xmm7, %xmm0 + movdqa %xmm6, %xmm1 + psrld $31, %xmm0 + psrld $31, %xmm1 + pslld $0x01, %xmm7 + pslld $0x01, %xmm6 + movdqa %xmm0, %xmm2 + pslldq $4, %xmm0 + psrldq $12, %xmm2 + pslldq $4, %xmm1 + por %xmm2, %xmm6 + por %xmm0, %xmm7 + por %xmm1, %xmm6 + movdqa %xmm7, %xmm0 + movdqa %xmm7, %xmm1 + movdqa %xmm7, %xmm2 + pslld $31, %xmm0 + pslld $30, %xmm1 + pslld $25, %xmm2 + pxor %xmm1, %xmm0 + pxor %xmm2, %xmm0 + movdqa %xmm0, %xmm1 + psrldq $4, %xmm1 + pslldq $12, %xmm0 + pxor %xmm0, %xmm7 + movdqa %xmm7, %xmm2 + movdqa %xmm7, %xmm3 + movdqa %xmm7, %xmm0 + psrld $0x01, %xmm2 + psrld $2, %xmm3 + psrld $7, %xmm0 + pxor %xmm3, %xmm2 + pxor %xmm0, %xmm2 + pxor %xmm1, %xmm2 + pxor %xmm7, %xmm2 + pxor %xmm2, %xmm6 +L_AES_GCM_decrypt_calc_aad_done: + # Calculate counter and H + pshufb L_aes_gcm_bswap_epi64(%rip), %xmm4 + movdqa %xmm5, %xmm9 + paddd L_aes_gcm_one(%rip), %xmm4 + movdqa %xmm5, %xmm8 + movdqa %xmm4, 128(%rsp) + psrlq $63, %xmm9 + psllq $0x01, %xmm8 + pslldq $8, %xmm9 + por %xmm9, %xmm8 + pshufd $0xff, %xmm5, %xmm5 + psrad $31, %xmm5 + pand L_aes_gcm_mod2_128(%rip), %xmm5 + pxor %xmm8, %xmm5 + xorl %ebx, %ebx + cmpl $0x80, %r9d + movl %r9d, %r13d + jl L_AES_GCM_decrypt_done_128 + andl $0xffffff80, %r13d + movdqa %xmm6, %xmm2 + # H ^ 1 + movdqa %xmm5, (%rsp) + # H ^ 2 + pshufd $0x4e, %xmm5, %xmm9 + pshufd $0x4e, %xmm5, %xmm10 + movdqa %xmm5, %xmm11 + movdqa %xmm5, %xmm8 + pclmulqdq $0x11, %xmm5, %xmm11 + pclmulqdq $0x00, %xmm5, %xmm8 + pxor %xmm5, %xmm9 + pxor %xmm5, %xmm10 + pclmulqdq $0x00, %xmm10, %xmm9 + pxor %xmm8, %xmm9 + pxor %xmm11, %xmm9 + movdqa %xmm9, %xmm10 + movdqa %xmm11, %xmm0 + pslldq $8, %xmm10 + psrldq $8, %xmm9 + pxor %xmm10, %xmm8 + pxor %xmm9, %xmm0 + movdqa %xmm8, %xmm12 + movdqa %xmm8, %xmm13 + movdqa %xmm8, %xmm14 + pslld $31, %xmm12 + pslld $30, %xmm13 + pslld $25, %xmm14 + pxor %xmm13, %xmm12 + pxor %xmm14, %xmm12 + movdqa %xmm12, %xmm13 + psrldq $4, %xmm13 + pslldq $12, %xmm12 + pxor %xmm12, %xmm8 + movdqa %xmm8, %xmm14 + movdqa %xmm8, %xmm10 + movdqa %xmm8, %xmm9 + psrld $0x01, %xmm14 + psrld $2, %xmm10 + psrld $7, %xmm9 + pxor %xmm10, %xmm14 + pxor %xmm9, %xmm14 + pxor %xmm13, %xmm14 + pxor %xmm8, %xmm14 + pxor %xmm14, %xmm0 + movdqa %xmm0, 16(%rsp) + # H ^ 3 + pshufd $0x4e, %xmm5, %xmm9 + pshufd $0x4e, %xmm0, %xmm10 + movdqa %xmm0, %xmm11 + movdqa %xmm0, %xmm8 + pclmulqdq $0x11, %xmm5, %xmm11 + pclmulqdq $0x00, %xmm5, %xmm8 + pxor %xmm5, %xmm9 + pxor %xmm0, %xmm10 + pclmulqdq $0x00, %xmm10, %xmm9 + pxor %xmm8, %xmm9 + pxor %xmm11, %xmm9 + movdqa %xmm9, %xmm10 + movdqa %xmm11, %xmm1 + pslldq $8, %xmm10 + psrldq $8, %xmm9 + pxor %xmm10, %xmm8 + pxor %xmm9, %xmm1 + movdqa %xmm8, %xmm12 + movdqa %xmm8, %xmm13 + movdqa %xmm8, %xmm14 + pslld $31, %xmm12 + pslld $30, %xmm13 + pslld $25, %xmm14 + pxor %xmm13, %xmm12 + pxor %xmm14, %xmm12 + movdqa %xmm12, %xmm13 + psrldq $4, %xmm13 + pslldq $12, %xmm12 + pxor %xmm12, %xmm8 + movdqa %xmm8, %xmm14 + movdqa %xmm8, %xmm10 + movdqa %xmm8, %xmm9 + psrld $0x01, %xmm14 + psrld $2, %xmm10 + psrld $7, %xmm9 + pxor %xmm10, %xmm14 + pxor %xmm9, %xmm14 + pxor %xmm13, %xmm14 + pxor %xmm8, %xmm14 + pxor %xmm14, %xmm1 + movdqa %xmm1, 32(%rsp) + # H ^ 4 + pshufd $0x4e, %xmm0, %xmm9 + pshufd $0x4e, %xmm0, %xmm10 + movdqa %xmm0, %xmm11 + movdqa %xmm0, %xmm8 + pclmulqdq $0x11, %xmm0, %xmm11 + pclmulqdq $0x00, %xmm0, %xmm8 + pxor %xmm0, %xmm9 + pxor %xmm0, %xmm10 + pclmulqdq $0x00, %xmm10, %xmm9 + pxor %xmm8, %xmm9 + pxor %xmm11, %xmm9 + movdqa %xmm9, %xmm10 + movdqa %xmm11, %xmm3 + pslldq $8, %xmm10 + psrldq $8, %xmm9 + pxor %xmm10, %xmm8 + pxor %xmm9, %xmm3 + movdqa %xmm8, %xmm12 + movdqa %xmm8, %xmm13 + movdqa %xmm8, %xmm14 + pslld $31, %xmm12 + pslld $30, %xmm13 + pslld $25, %xmm14 + pxor %xmm13, %xmm12 + pxor %xmm14, %xmm12 + movdqa %xmm12, %xmm13 + psrldq $4, %xmm13 + pslldq $12, %xmm12 + pxor %xmm12, %xmm8 + movdqa %xmm8, %xmm14 + movdqa %xmm8, %xmm10 + movdqa %xmm8, %xmm9 + psrld $0x01, %xmm14 + psrld $2, %xmm10 + psrld $7, %xmm9 + pxor %xmm10, %xmm14 + pxor %xmm9, %xmm14 + pxor %xmm13, %xmm14 + pxor %xmm8, %xmm14 + pxor %xmm14, %xmm3 + movdqa %xmm3, 48(%rsp) + # H ^ 5 + pshufd $0x4e, %xmm0, %xmm9 + pshufd $0x4e, %xmm1, %xmm10 + movdqa %xmm1, %xmm11 + movdqa %xmm1, %xmm8 + pclmulqdq $0x11, %xmm0, %xmm11 + pclmulqdq $0x00, %xmm0, %xmm8 + pxor %xmm0, %xmm9 + pxor %xmm1, %xmm10 + pclmulqdq $0x00, %xmm10, %xmm9 + pxor %xmm8, %xmm9 + pxor %xmm11, %xmm9 + movdqa %xmm9, %xmm10 + movdqa %xmm11, %xmm7 + pslldq $8, %xmm10 + psrldq $8, %xmm9 + pxor %xmm10, %xmm8 + pxor %xmm9, %xmm7 + movdqa %xmm8, %xmm12 + movdqa %xmm8, %xmm13 + movdqa %xmm8, %xmm14 + pslld $31, %xmm12 + pslld $30, %xmm13 + pslld $25, %xmm14 + pxor %xmm13, %xmm12 + pxor %xmm14, %xmm12 + movdqa %xmm12, %xmm13 + psrldq $4, %xmm13 + pslldq $12, %xmm12 + pxor %xmm12, %xmm8 + movdqa %xmm8, %xmm14 + movdqa %xmm8, %xmm10 + movdqa %xmm8, %xmm9 + psrld $0x01, %xmm14 + psrld $2, %xmm10 + psrld $7, %xmm9 + pxor %xmm10, %xmm14 + pxor %xmm9, %xmm14 + pxor %xmm13, %xmm14 + pxor %xmm8, %xmm14 + pxor %xmm14, %xmm7 + movdqa %xmm7, 64(%rsp) + # H ^ 6 + pshufd $0x4e, %xmm1, %xmm9 + pshufd $0x4e, %xmm1, %xmm10 + movdqa %xmm1, %xmm11 + movdqa %xmm1, %xmm8 + pclmulqdq $0x11, %xmm1, %xmm11 + pclmulqdq $0x00, %xmm1, %xmm8 + pxor %xmm1, %xmm9 + pxor %xmm1, %xmm10 + pclmulqdq $0x00, %xmm10, %xmm9 + pxor %xmm8, %xmm9 + pxor %xmm11, %xmm9 + movdqa %xmm9, %xmm10 + movdqa %xmm11, %xmm7 + pslldq $8, %xmm10 + psrldq $8, %xmm9 + pxor %xmm10, %xmm8 + pxor %xmm9, %xmm7 + movdqa %xmm8, %xmm12 + movdqa %xmm8, %xmm13 + movdqa %xmm8, %xmm14 + pslld $31, %xmm12 + pslld $30, %xmm13 + pslld $25, %xmm14 + pxor %xmm13, %xmm12 + pxor %xmm14, %xmm12 + movdqa %xmm12, %xmm13 + psrldq $4, %xmm13 + pslldq $12, %xmm12 + pxor %xmm12, %xmm8 + movdqa %xmm8, %xmm14 + movdqa %xmm8, %xmm10 + movdqa %xmm8, %xmm9 + psrld $0x01, %xmm14 + psrld $2, %xmm10 + psrld $7, %xmm9 + pxor %xmm10, %xmm14 + pxor %xmm9, %xmm14 + pxor %xmm13, %xmm14 + pxor %xmm8, %xmm14 + pxor %xmm14, %xmm7 + movdqa %xmm7, 80(%rsp) + # H ^ 7 + pshufd $0x4e, %xmm1, %xmm9 + pshufd $0x4e, %xmm3, %xmm10 + movdqa %xmm3, %xmm11 + movdqa %xmm3, %xmm8 + pclmulqdq $0x11, %xmm1, %xmm11 + pclmulqdq $0x00, %xmm1, %xmm8 + pxor %xmm1, %xmm9 + pxor %xmm3, %xmm10 + pclmulqdq $0x00, %xmm10, %xmm9 + pxor %xmm8, %xmm9 + pxor %xmm11, %xmm9 + movdqa %xmm9, %xmm10 + movdqa %xmm11, %xmm7 + pslldq $8, %xmm10 + psrldq $8, %xmm9 + pxor %xmm10, %xmm8 + pxor %xmm9, %xmm7 + movdqa %xmm8, %xmm12 + movdqa %xmm8, %xmm13 + movdqa %xmm8, %xmm14 + pslld $31, %xmm12 + pslld $30, %xmm13 + pslld $25, %xmm14 + pxor %xmm13, %xmm12 + pxor %xmm14, %xmm12 + movdqa %xmm12, %xmm13 + psrldq $4, %xmm13 + pslldq $12, %xmm12 + pxor %xmm12, %xmm8 + movdqa %xmm8, %xmm14 + movdqa %xmm8, %xmm10 + movdqa %xmm8, %xmm9 + psrld $0x01, %xmm14 + psrld $2, %xmm10 + psrld $7, %xmm9 + pxor %xmm10, %xmm14 + pxor %xmm9, %xmm14 + pxor %xmm13, %xmm14 + pxor %xmm8, %xmm14 + pxor %xmm14, %xmm7 + movdqa %xmm7, 96(%rsp) + # H ^ 8 + pshufd $0x4e, %xmm3, %xmm9 + pshufd $0x4e, %xmm3, %xmm10 + movdqa %xmm3, %xmm11 + movdqa %xmm3, %xmm8 + pclmulqdq $0x11, %xmm3, %xmm11 + pclmulqdq $0x00, %xmm3, %xmm8 + pxor %xmm3, %xmm9 + pxor %xmm3, %xmm10 + pclmulqdq $0x00, %xmm10, %xmm9 + pxor %xmm8, %xmm9 + pxor %xmm11, %xmm9 + movdqa %xmm9, %xmm10 + movdqa %xmm11, %xmm7 + pslldq $8, %xmm10 + psrldq $8, %xmm9 + pxor %xmm10, %xmm8 + pxor %xmm9, %xmm7 + movdqa %xmm8, %xmm12 + movdqa %xmm8, %xmm13 + movdqa %xmm8, %xmm14 + pslld $31, %xmm12 + pslld $30, %xmm13 + pslld $25, %xmm14 + pxor %xmm13, %xmm12 + pxor %xmm14, %xmm12 + movdqa %xmm12, %xmm13 + psrldq $4, %xmm13 + pslldq $12, %xmm12 + pxor %xmm12, %xmm8 + movdqa %xmm8, %xmm14 + movdqa %xmm8, %xmm10 + movdqa %xmm8, %xmm9 + psrld $0x01, %xmm14 + psrld $2, %xmm10 + psrld $7, %xmm9 + pxor %xmm10, %xmm14 + pxor %xmm9, %xmm14 + pxor %xmm13, %xmm14 + pxor %xmm8, %xmm14 + pxor %xmm14, %xmm7 + movdqa %xmm7, 112(%rsp) +L_AES_GCM_decrypt_ghash_128: + leaq (%rdi,%rbx,1), %rcx + leaq (%rsi,%rbx,1), %rdx + movdqa 128(%rsp), %xmm8 + movdqa L_aes_gcm_bswap_epi64(%rip), %xmm1 + movdqa %xmm8, %xmm0 + pshufb %xmm1, %xmm8 + movdqa %xmm0, %xmm9 + paddd L_aes_gcm_one(%rip), %xmm9 + pshufb %xmm1, %xmm9 + movdqa %xmm0, %xmm10 + paddd L_aes_gcm_two(%rip), %xmm10 + pshufb %xmm1, %xmm10 + movdqa %xmm0, %xmm11 + paddd L_aes_gcm_three(%rip), %xmm11 + pshufb %xmm1, %xmm11 + movdqa %xmm0, %xmm12 + paddd L_aes_gcm_four(%rip), %xmm12 + pshufb %xmm1, %xmm12 + movdqa %xmm0, %xmm13 + paddd L_aes_gcm_five(%rip), %xmm13 + pshufb %xmm1, %xmm13 + movdqa %xmm0, %xmm14 + paddd L_aes_gcm_six(%rip), %xmm14 + pshufb %xmm1, %xmm14 + movdqa %xmm0, %xmm15 + paddd L_aes_gcm_seven(%rip), %xmm15 + pshufb %xmm1, %xmm15 + paddd L_aes_gcm_eight(%rip), %xmm0 + movdqa (%r15), %xmm7 + movdqa %xmm0, 128(%rsp) + pxor %xmm7, %xmm8 + pxor %xmm7, %xmm9 + pxor %xmm7, %xmm10 + pxor %xmm7, %xmm11 + pxor %xmm7, %xmm12 + pxor %xmm7, %xmm13 + pxor %xmm7, %xmm14 + pxor %xmm7, %xmm15 + movdqa 112(%rsp), %xmm7 + movdqu (%rcx), %xmm0 + aesenc 16(%r15), %xmm8 + pshufb L_aes_gcm_bswap_mask(%rip), %xmm0 + pxor %xmm2, %xmm0 + pshufd $0x4e, %xmm7, %xmm1 + pshufd $0x4e, %xmm0, %xmm5 + pxor %xmm7, %xmm1 + pxor %xmm0, %xmm5 + movdqa %xmm0, %xmm3 + pclmulqdq $0x11, %xmm7, %xmm3 + aesenc 16(%r15), %xmm9 + aesenc 16(%r15), %xmm10 + movdqa %xmm0, %xmm2 + pclmulqdq $0x00, %xmm7, %xmm2 + aesenc 16(%r15), %xmm11 + aesenc 16(%r15), %xmm12 + pclmulqdq $0x00, %xmm5, %xmm1 + aesenc 16(%r15), %xmm13 + aesenc 16(%r15), %xmm14 + aesenc 16(%r15), %xmm15 + pxor %xmm2, %xmm1 + pxor %xmm3, %xmm1 + movdqa 96(%rsp), %xmm7 + movdqu 16(%rcx), %xmm0 + pshufd $0x4e, %xmm7, %xmm4 + pshufb L_aes_gcm_bswap_mask(%rip), %xmm0 + aesenc 32(%r15), %xmm8 + pxor %xmm7, %xmm4 + pshufd $0x4e, %xmm0, %xmm5 + pxor %xmm0, %xmm5 + movdqa %xmm0, %xmm6 + pclmulqdq $0x11, %xmm7, %xmm6 + aesenc 32(%r15), %xmm9 + aesenc 32(%r15), %xmm10 + pclmulqdq $0x00, %xmm0, %xmm7 + aesenc 32(%r15), %xmm11 + aesenc 32(%r15), %xmm12 + pclmulqdq $0x00, %xmm5, %xmm4 + aesenc 32(%r15), %xmm13 + aesenc 32(%r15), %xmm14 + aesenc 32(%r15), %xmm15 + pxor %xmm7, %xmm1 + pxor %xmm7, %xmm2 + pxor %xmm6, %xmm1 + pxor %xmm6, %xmm3 + pxor %xmm4, %xmm1 + movdqa 80(%rsp), %xmm7 + movdqu 32(%rcx), %xmm0 + pshufd $0x4e, %xmm7, %xmm4 + pshufb L_aes_gcm_bswap_mask(%rip), %xmm0 + aesenc 48(%r15), %xmm8 + pxor %xmm7, %xmm4 + pshufd $0x4e, %xmm0, %xmm5 + pxor %xmm0, %xmm5 + movdqa %xmm0, %xmm6 + pclmulqdq $0x11, %xmm7, %xmm6 + aesenc 48(%r15), %xmm9 + aesenc 48(%r15), %xmm10 + pclmulqdq $0x00, %xmm0, %xmm7 + aesenc 48(%r15), %xmm11 + aesenc 48(%r15), %xmm12 + pclmulqdq $0x00, %xmm5, %xmm4 + aesenc 48(%r15), %xmm13 + aesenc 48(%r15), %xmm14 + aesenc 48(%r15), %xmm15 + pxor %xmm7, %xmm1 + pxor %xmm7, %xmm2 + pxor %xmm6, %xmm1 + pxor %xmm6, %xmm3 + pxor %xmm4, %xmm1 + movdqa 64(%rsp), %xmm7 + movdqu 48(%rcx), %xmm0 + pshufd $0x4e, %xmm7, %xmm4 + pshufb L_aes_gcm_bswap_mask(%rip), %xmm0 + aesenc 64(%r15), %xmm8 + pxor %xmm7, %xmm4 + pshufd $0x4e, %xmm0, %xmm5 + pxor %xmm0, %xmm5 + movdqa %xmm0, %xmm6 + pclmulqdq $0x11, %xmm7, %xmm6 + aesenc 64(%r15), %xmm9 + aesenc 64(%r15), %xmm10 + pclmulqdq $0x00, %xmm0, %xmm7 + aesenc 64(%r15), %xmm11 + aesenc 64(%r15), %xmm12 + pclmulqdq $0x00, %xmm5, %xmm4 + aesenc 64(%r15), %xmm13 + aesenc 64(%r15), %xmm14 + aesenc 64(%r15), %xmm15 + pxor %xmm7, %xmm1 + pxor %xmm7, %xmm2 + pxor %xmm6, %xmm1 + pxor %xmm6, %xmm3 + pxor %xmm4, %xmm1 + movdqa 48(%rsp), %xmm7 + movdqu 64(%rcx), %xmm0 + pshufd $0x4e, %xmm7, %xmm4 + pshufb L_aes_gcm_bswap_mask(%rip), %xmm0 + aesenc 80(%r15), %xmm8 + pxor %xmm7, %xmm4 + pshufd $0x4e, %xmm0, %xmm5 + pxor %xmm0, %xmm5 + movdqa %xmm0, %xmm6 + pclmulqdq $0x11, %xmm7, %xmm6 + aesenc 80(%r15), %xmm9 + aesenc 80(%r15), %xmm10 + pclmulqdq $0x00, %xmm0, %xmm7 + aesenc 80(%r15), %xmm11 + aesenc 80(%r15), %xmm12 + pclmulqdq $0x00, %xmm5, %xmm4 + aesenc 80(%r15), %xmm13 + aesenc 80(%r15), %xmm14 + aesenc 80(%r15), %xmm15 + pxor %xmm7, %xmm1 + pxor %xmm7, %xmm2 + pxor %xmm6, %xmm1 + pxor %xmm6, %xmm3 + pxor %xmm4, %xmm1 + movdqa 32(%rsp), %xmm7 + movdqu 80(%rcx), %xmm0 + pshufd $0x4e, %xmm7, %xmm4 + pshufb L_aes_gcm_bswap_mask(%rip), %xmm0 + aesenc 96(%r15), %xmm8 + pxor %xmm7, %xmm4 + pshufd $0x4e, %xmm0, %xmm5 + pxor %xmm0, %xmm5 + movdqa %xmm0, %xmm6 + pclmulqdq $0x11, %xmm7, %xmm6 + aesenc 96(%r15), %xmm9 + aesenc 96(%r15), %xmm10 + pclmulqdq $0x00, %xmm0, %xmm7 + aesenc 96(%r15), %xmm11 + aesenc 96(%r15), %xmm12 + pclmulqdq $0x00, %xmm5, %xmm4 + aesenc 96(%r15), %xmm13 + aesenc 96(%r15), %xmm14 + aesenc 96(%r15), %xmm15 + pxor %xmm7, %xmm1 + pxor %xmm7, %xmm2 + pxor %xmm6, %xmm1 + pxor %xmm6, %xmm3 + pxor %xmm4, %xmm1 + movdqa 16(%rsp), %xmm7 + movdqu 96(%rcx), %xmm0 + pshufd $0x4e, %xmm7, %xmm4 + pshufb L_aes_gcm_bswap_mask(%rip), %xmm0 + aesenc 112(%r15), %xmm8 + pxor %xmm7, %xmm4 + pshufd $0x4e, %xmm0, %xmm5 + pxor %xmm0, %xmm5 + movdqa %xmm0, %xmm6 + pclmulqdq $0x11, %xmm7, %xmm6 + aesenc 112(%r15), %xmm9 + aesenc 112(%r15), %xmm10 + pclmulqdq $0x00, %xmm0, %xmm7 + aesenc 112(%r15), %xmm11 + aesenc 112(%r15), %xmm12 + pclmulqdq $0x00, %xmm5, %xmm4 + aesenc 112(%r15), %xmm13 + aesenc 112(%r15), %xmm14 + aesenc 112(%r15), %xmm15 + pxor %xmm7, %xmm1 + pxor %xmm7, %xmm2 + pxor %xmm6, %xmm1 + pxor %xmm6, %xmm3 + pxor %xmm4, %xmm1 + movdqa (%rsp), %xmm7 + movdqu 112(%rcx), %xmm0 + pshufd $0x4e, %xmm7, %xmm4 + pshufb L_aes_gcm_bswap_mask(%rip), %xmm0 + aesenc 128(%r15), %xmm8 + pxor %xmm7, %xmm4 + pshufd $0x4e, %xmm0, %xmm5 + pxor %xmm0, %xmm5 + movdqa %xmm0, %xmm6 + pclmulqdq $0x11, %xmm7, %xmm6 + aesenc 128(%r15), %xmm9 + aesenc 128(%r15), %xmm10 + pclmulqdq $0x00, %xmm0, %xmm7 + aesenc 128(%r15), %xmm11 + aesenc 128(%r15), %xmm12 + pclmulqdq $0x00, %xmm5, %xmm4 + aesenc 128(%r15), %xmm13 + aesenc 128(%r15), %xmm14 + aesenc 128(%r15), %xmm15 + pxor %xmm7, %xmm1 + pxor %xmm7, %xmm2 + pxor %xmm6, %xmm1 + pxor %xmm6, %xmm3 + pxor %xmm4, %xmm1 + movdqa %xmm1, %xmm5 + psrldq $8, %xmm1 + pslldq $8, %xmm5 + aesenc 144(%r15), %xmm8 + pxor %xmm5, %xmm2 + pxor %xmm1, %xmm3 + movdqa %xmm2, %xmm7 + movdqa %xmm2, %xmm4 + movdqa %xmm2, %xmm5 + aesenc 144(%r15), %xmm9 + pslld $31, %xmm7 + pslld $30, %xmm4 + pslld $25, %xmm5 + aesenc 144(%r15), %xmm10 + pxor %xmm4, %xmm7 + pxor %xmm5, %xmm7 + aesenc 144(%r15), %xmm11 + movdqa %xmm7, %xmm4 + pslldq $12, %xmm7 + psrldq $4, %xmm4 + aesenc 144(%r15), %xmm12 + pxor %xmm7, %xmm2 + movdqa %xmm2, %xmm5 + movdqa %xmm2, %xmm1 + movdqa %xmm2, %xmm0 + aesenc 144(%r15), %xmm13 + psrld $0x01, %xmm5 + psrld $2, %xmm1 + psrld $7, %xmm0 + aesenc 144(%r15), %xmm14 + pxor %xmm1, %xmm5 + pxor %xmm0, %xmm5 + aesenc 144(%r15), %xmm15 + pxor %xmm4, %xmm5 + pxor %xmm5, %xmm2 + pxor %xmm3, %xmm2 + cmpl $11, %r10d + movdqa 160(%r15), %xmm7 + jl L_AES_GCM_decrypt_aesenc_128_ghash_avx_done + aesenc %xmm7, %xmm8 + aesenc %xmm7, %xmm9 + aesenc %xmm7, %xmm10 + aesenc %xmm7, %xmm11 + aesenc %xmm7, %xmm12 + aesenc %xmm7, %xmm13 + aesenc %xmm7, %xmm14 + aesenc %xmm7, %xmm15 + movdqa 176(%r15), %xmm7 + aesenc %xmm7, %xmm8 + aesenc %xmm7, %xmm9 + aesenc %xmm7, %xmm10 + aesenc %xmm7, %xmm11 + aesenc %xmm7, %xmm12 + aesenc %xmm7, %xmm13 + aesenc %xmm7, %xmm14 + aesenc %xmm7, %xmm15 + cmpl $13, %r10d + movdqa 192(%r15), %xmm7 + jl L_AES_GCM_decrypt_aesenc_128_ghash_avx_done + aesenc %xmm7, %xmm8 + aesenc %xmm7, %xmm9 + aesenc %xmm7, %xmm10 + aesenc %xmm7, %xmm11 + aesenc %xmm7, %xmm12 + aesenc %xmm7, %xmm13 + aesenc %xmm7, %xmm14 + aesenc %xmm7, %xmm15 + movdqa 208(%r15), %xmm7 + aesenc %xmm7, %xmm8 + aesenc %xmm7, %xmm9 + aesenc %xmm7, %xmm10 + aesenc %xmm7, %xmm11 + aesenc %xmm7, %xmm12 + aesenc %xmm7, %xmm13 + aesenc %xmm7, %xmm14 + aesenc %xmm7, %xmm15 + movdqa 224(%r15), %xmm7 +L_AES_GCM_decrypt_aesenc_128_ghash_avx_done: + aesenclast %xmm7, %xmm8 + aesenclast %xmm7, %xmm9 + movdqu (%rcx), %xmm0 + movdqu 16(%rcx), %xmm1 + pxor %xmm0, %xmm8 + pxor %xmm1, %xmm9 + movdqu %xmm8, (%rdx) + movdqu %xmm9, 16(%rdx) + aesenclast %xmm7, %xmm10 + aesenclast %xmm7, %xmm11 + movdqu 32(%rcx), %xmm0 + movdqu 48(%rcx), %xmm1 + pxor %xmm0, %xmm10 + pxor %xmm1, %xmm11 + movdqu %xmm10, 32(%rdx) + movdqu %xmm11, 48(%rdx) + aesenclast %xmm7, %xmm12 + aesenclast %xmm7, %xmm13 + movdqu 64(%rcx), %xmm0 + movdqu 80(%rcx), %xmm1 + pxor %xmm0, %xmm12 + pxor %xmm1, %xmm13 + movdqu %xmm12, 64(%rdx) + movdqu %xmm13, 80(%rdx) + aesenclast %xmm7, %xmm14 + aesenclast %xmm7, %xmm15 + movdqu 96(%rcx), %xmm0 + movdqu 112(%rcx), %xmm1 + pxor %xmm0, %xmm14 + pxor %xmm1, %xmm15 + movdqu %xmm14, 96(%rdx) + movdqu %xmm15, 112(%rdx) + addl $0x80, %ebx + cmpl %r13d, %ebx + jl L_AES_GCM_decrypt_ghash_128 + movdqa %xmm2, %xmm6 + movdqa (%rsp), %xmm5 +L_AES_GCM_decrypt_done_128: + movl %r9d, %edx + cmpl %edx, %ebx + jge L_AES_GCM_decrypt_done_dec + movl %r9d, %r13d + andl $0xfffffff0, %r13d + cmpl %r13d, %ebx + jge L_AES_GCM_decrypt_last_block_done +L_AES_GCM_decrypt_last_block_start: + leaq (%rdi,%rbx,1), %rcx + leaq (%rsi,%rbx,1), %rdx + movdqu (%rcx), %xmm1 + movdqa %xmm5, %xmm0 + pshufb L_aes_gcm_bswap_mask(%rip), %xmm1 + pxor %xmm6, %xmm1 + movdqa 128(%rsp), %xmm8 + movdqa %xmm8, %xmm9 + pshufb L_aes_gcm_bswap_epi64(%rip), %xmm8 + paddd L_aes_gcm_one(%rip), %xmm9 + pxor (%r15), %xmm8 + movdqa %xmm9, 128(%rsp) + movdqa %xmm1, %xmm10 + pclmulqdq $16, %xmm0, %xmm10 + aesenc 16(%r15), %xmm8 + aesenc 32(%r15), %xmm8 + movdqa %xmm1, %xmm11 + pclmulqdq $0x01, %xmm0, %xmm11 + aesenc 48(%r15), %xmm8 + aesenc 64(%r15), %xmm8 + movdqa %xmm1, %xmm12 + pclmulqdq $0x00, %xmm0, %xmm12 + aesenc 80(%r15), %xmm8 + movdqa %xmm1, %xmm1 + pclmulqdq $0x11, %xmm0, %xmm1 + aesenc 96(%r15), %xmm8 + pxor %xmm11, %xmm10 + movdqa %xmm10, %xmm2 + psrldq $8, %xmm10 + pslldq $8, %xmm2 + aesenc 112(%r15), %xmm8 + movdqa %xmm1, %xmm3 + pxor %xmm12, %xmm2 + pxor %xmm10, %xmm3 + movdqa L_aes_gcm_mod2_128(%rip), %xmm0 + movdqa %xmm2, %xmm11 + pclmulqdq $16, %xmm0, %xmm11 + aesenc 128(%r15), %xmm8 + pshufd $0x4e, %xmm2, %xmm10 + pxor %xmm11, %xmm10 + movdqa %xmm10, %xmm11 + pclmulqdq $16, %xmm0, %xmm11 + aesenc 144(%r15), %xmm8 + pshufd $0x4e, %xmm10, %xmm6 + pxor %xmm11, %xmm6 + pxor %xmm3, %xmm6 + cmpl $11, %r10d + movdqa 160(%r15), %xmm9 + jl L_AES_GCM_decrypt_aesenc_gfmul_last + aesenc %xmm9, %xmm8 + aesenc 176(%r15), %xmm8 + cmpl $13, %r10d + movdqa 192(%r15), %xmm9 + jl L_AES_GCM_decrypt_aesenc_gfmul_last + aesenc %xmm9, %xmm8 + aesenc 208(%r15), %xmm8 + movdqa 224(%r15), %xmm9 +L_AES_GCM_decrypt_aesenc_gfmul_last: + aesenclast %xmm9, %xmm8 + movdqu (%rcx), %xmm9 + pxor %xmm9, %xmm8 + movdqu %xmm8, (%rdx) + addl $16, %ebx + cmpl %r13d, %ebx + jl L_AES_GCM_decrypt_last_block_start +L_AES_GCM_decrypt_last_block_done: + movl %r9d, %ecx + movl %ecx, %edx + andl $15, %ecx + jz L_AES_GCM_decrypt_aesenc_last15_dec_avx_done + movdqa 128(%rsp), %xmm4 + pshufb L_aes_gcm_bswap_epi64(%rip), %xmm4 + pxor (%r15), %xmm4 + aesenc 16(%r15), %xmm4 + aesenc 32(%r15), %xmm4 + aesenc 48(%r15), %xmm4 + aesenc 64(%r15), %xmm4 + aesenc 80(%r15), %xmm4 + aesenc 96(%r15), %xmm4 + aesenc 112(%r15), %xmm4 + aesenc 128(%r15), %xmm4 + aesenc 144(%r15), %xmm4 + cmpl $11, %r10d + movdqa 160(%r15), %xmm9 + jl L_AES_GCM_decrypt_aesenc_last15_dec_avx_aesenc_avx_last + aesenc %xmm9, %xmm4 + aesenc 176(%r15), %xmm4 + cmpl $13, %r10d + movdqa 192(%r15), %xmm9 + jl L_AES_GCM_decrypt_aesenc_last15_dec_avx_aesenc_avx_last + aesenc %xmm9, %xmm4 + aesenc 208(%r15), %xmm4 + movdqa 224(%r15), %xmm9 +L_AES_GCM_decrypt_aesenc_last15_dec_avx_aesenc_avx_last: + aesenclast %xmm9, %xmm4 + subq $32, %rsp + xorl %ecx, %ecx + movdqa %xmm4, (%rsp) + pxor %xmm0, %xmm0 + movdqa %xmm0, 16(%rsp) +L_AES_GCM_decrypt_aesenc_last15_dec_avx_loop: + movzbl (%rdi,%rbx,1), %r13d + movb %r13b, 16(%rsp,%rcx,1) + xorb (%rsp,%rcx,1), %r13b + movb %r13b, (%rsi,%rbx,1) + incl %ebx + incl %ecx + cmpl %edx, %ebx + jl L_AES_GCM_decrypt_aesenc_last15_dec_avx_loop + movdqa 16(%rsp), %xmm4 + addq $32, %rsp + pshufb L_aes_gcm_bswap_mask(%rip), %xmm4 + pxor %xmm4, %xmm6 + pshufd $0x4e, %xmm5, %xmm9 + pshufd $0x4e, %xmm6, %xmm10 + movdqa %xmm6, %xmm11 + movdqa %xmm6, %xmm8 + pclmulqdq $0x11, %xmm5, %xmm11 + pclmulqdq $0x00, %xmm5, %xmm8 + pxor %xmm5, %xmm9 + pxor %xmm6, %xmm10 + pclmulqdq $0x00, %xmm10, %xmm9 + pxor %xmm8, %xmm9 + pxor %xmm11, %xmm9 + movdqa %xmm9, %xmm10 + movdqa %xmm11, %xmm6 + pslldq $8, %xmm10 + psrldq $8, %xmm9 + pxor %xmm10, %xmm8 + pxor %xmm9, %xmm6 + movdqa %xmm8, %xmm12 + movdqa %xmm8, %xmm13 + movdqa %xmm8, %xmm14 + pslld $31, %xmm12 + pslld $30, %xmm13 + pslld $25, %xmm14 + pxor %xmm13, %xmm12 + pxor %xmm14, %xmm12 + movdqa %xmm12, %xmm13 + psrldq $4, %xmm13 + pslldq $12, %xmm12 + pxor %xmm12, %xmm8 + movdqa %xmm8, %xmm14 + movdqa %xmm8, %xmm10 + movdqa %xmm8, %xmm9 + psrld $0x01, %xmm14 + psrld $2, %xmm10 + psrld $7, %xmm9 + pxor %xmm10, %xmm14 + pxor %xmm9, %xmm14 + pxor %xmm13, %xmm14 + pxor %xmm8, %xmm14 + pxor %xmm14, %xmm6 +L_AES_GCM_decrypt_aesenc_last15_dec_avx_done: +L_AES_GCM_decrypt_done_dec: + movl %r9d, %edx + movl %r11d, %ecx + shlq $3, %rdx + shlq $3, %rcx + pinsrq $0x00, %rdx, %xmm0 + pinsrq $0x01, %rcx, %xmm0 + pxor %xmm0, %xmm6 + pshufd $0x4e, %xmm5, %xmm9 + pshufd $0x4e, %xmm6, %xmm10 + movdqa %xmm6, %xmm11 + movdqa %xmm6, %xmm8 + pclmulqdq $0x11, %xmm5, %xmm11 + pclmulqdq $0x00, %xmm5, %xmm8 + pxor %xmm5, %xmm9 + pxor %xmm6, %xmm10 + pclmulqdq $0x00, %xmm10, %xmm9 + pxor %xmm8, %xmm9 + pxor %xmm11, %xmm9 + movdqa %xmm9, %xmm10 + movdqa %xmm11, %xmm6 + pslldq $8, %xmm10 + psrldq $8, %xmm9 + pxor %xmm10, %xmm8 + pxor %xmm9, %xmm6 + movdqa %xmm8, %xmm12 + movdqa %xmm8, %xmm13 + movdqa %xmm8, %xmm14 + pslld $31, %xmm12 + pslld $30, %xmm13 + pslld $25, %xmm14 + pxor %xmm13, %xmm12 + pxor %xmm14, %xmm12 + movdqa %xmm12, %xmm13 + psrldq $4, %xmm13 + pslldq $12, %xmm12 + pxor %xmm12, %xmm8 + movdqa %xmm8, %xmm14 + movdqa %xmm8, %xmm10 + movdqa %xmm8, %xmm9 + psrld $0x01, %xmm14 + psrld $2, %xmm10 + psrld $7, %xmm9 + pxor %xmm10, %xmm14 + pxor %xmm9, %xmm14 + pxor %xmm13, %xmm14 + pxor %xmm8, %xmm14 + pxor %xmm14, %xmm6 + pshufb L_aes_gcm_bswap_mask(%rip), %xmm6 + movdqa 144(%rsp), %xmm0 + pxor %xmm6, %xmm0 + cmpl $16, %r14d + je L_AES_GCM_decrypt_cmp_tag_16 + subq $16, %rsp + xorq %rcx, %rcx + xorq %rbx, %rbx + movdqa %xmm0, (%rsp) +L_AES_GCM_decrypt_cmp_tag_loop: + movzbl (%rsp,%rcx,1), %r13d + xorb (%r8,%rcx,1), %r13b + orb %r13b, %bl + incl %ecx + cmpl %r14d, %ecx + jne L_AES_GCM_decrypt_cmp_tag_loop + cmpb $0x00, %bl + sete %bl + addq $16, %rsp + xorq %rcx, %rcx + jmp L_AES_GCM_decrypt_cmp_tag_done +L_AES_GCM_decrypt_cmp_tag_16: + movdqu (%r8), %xmm1 + pcmpeqb %xmm1, %xmm0 + pmovmskb %xmm0, %rdx + # %%edx == 0xFFFF then return 1 else => return 0 + xorl %ebx, %ebx + cmpl $0xffff, %edx + sete %bl +L_AES_GCM_decrypt_cmp_tag_done: + movl %ebx, (%rbp) + addq $0xa8, %rsp + popq %rbp + popq %r15 + popq %r14 + popq %rbx + popq %r12 + popq %r13 + repz retq +.size AES_GCM_decrypt,.-AES_GCM_decrypt +#ifdef HAVE_INTEL_AVX1 +.align 16 +L_avx1_aes_gcm_one: +.quad 0x0, 0x1 +.align 16 +L_avx1_aes_gcm_two: +.quad 0x0, 0x2 +.align 16 +L_avx1_aes_gcm_three: +.quad 0x0, 0x3 +.align 16 +L_avx1_aes_gcm_four: +.quad 0x0, 0x4 +.align 16 +L_avx1_aes_gcm_five: +.quad 0x0, 0x5 +.align 16 +L_avx1_aes_gcm_six: +.quad 0x0, 0x6 +.align 16 +L_avx1_aes_gcm_seven: +.quad 0x0, 0x7 +.align 16 +L_avx1_aes_gcm_eight: +.quad 0x0, 0x8 +.align 16 +L_avx1_aes_gcm_bswap_epi64: +.quad 0x1020304050607, 0x8090a0b0c0d0e0f +.align 16 +L_avx1_aes_gcm_bswap_mask: +.quad 0x8090a0b0c0d0e0f, 0x1020304050607 +.align 16 +L_avx1_aes_gcm_mod2_128: +.quad 0x1, 0xc200000000000000 +.globl AES_GCM_encrypt_avx1 +.type AES_GCM_encrypt_avx1,@function +.align 4 +AES_GCM_encrypt_avx1: + pushq %r13 + pushq %r12 + pushq %rbx + pushq %r14 + pushq %r15 + movq %rdx, %r12 + movq %rcx, %rax + movq 48(%rsp), %r11 + movq 56(%rsp), %rbx + movq 64(%rsp), %r14 + movq 72(%rsp), %r15 + movq 80(%rsp), %r10 + subq $0xa0, %rsp + vpxor %xmm4, %xmm4, %xmm4 + vpxor %xmm6, %xmm6, %xmm6 + movl %ebx, %edx + cmpl $12, %edx + jne L_AES_GCM_encrypt_avx1_iv_not_12 + # # Calculate values when IV is 12 bytes + # Set counter based on IV + movl $0x1000000, %ecx + vpinsrq $0x00, (%rax), %xmm4, %xmm4 + vpinsrd $2, 8(%rax), %xmm4, %xmm4 + vpinsrd $3, %ecx, %xmm4, %xmm4 + # H = Encrypt X(=0) and T = Encrypt counter + vmovdqa (%r15), %xmm5 + vpxor %xmm5, %xmm4, %xmm1 + vmovdqa 16(%r15), %xmm7 + vaesenc %xmm7, %xmm5, %xmm5 + vaesenc %xmm7, %xmm1, %xmm1 + vmovdqa 32(%r15), %xmm7 + vaesenc %xmm7, %xmm5, %xmm5 + vaesenc %xmm7, %xmm1, %xmm1 + vmovdqa 48(%r15), %xmm7 + vaesenc %xmm7, %xmm5, %xmm5 + vaesenc %xmm7, %xmm1, %xmm1 + vmovdqa 64(%r15), %xmm7 + vaesenc %xmm7, %xmm5, %xmm5 + vaesenc %xmm7, %xmm1, %xmm1 + vmovdqa 80(%r15), %xmm7 + vaesenc %xmm7, %xmm5, %xmm5 + vaesenc %xmm7, %xmm1, %xmm1 + vmovdqa 96(%r15), %xmm7 + vaesenc %xmm7, %xmm5, %xmm5 + vaesenc %xmm7, %xmm1, %xmm1 + vmovdqa 112(%r15), %xmm7 + vaesenc %xmm7, %xmm5, %xmm5 + vaesenc %xmm7, %xmm1, %xmm1 + vmovdqa 128(%r15), %xmm7 + vaesenc %xmm7, %xmm5, %xmm5 + vaesenc %xmm7, %xmm1, %xmm1 + vmovdqa 144(%r15), %xmm7 + vaesenc %xmm7, %xmm5, %xmm5 + vaesenc %xmm7, %xmm1, %xmm1 + cmpl $11, %r10d + vmovdqa 160(%r15), %xmm7 + jl L_AES_GCM_encrypt_avx1_calc_iv_12_last + vaesenc %xmm7, %xmm5, %xmm5 + vaesenc %xmm7, %xmm1, %xmm1 + vmovdqa 176(%r15), %xmm7 + vaesenc %xmm7, %xmm5, %xmm5 + vaesenc %xmm7, %xmm1, %xmm1 + cmpl $13, %r10d + vmovdqa 192(%r15), %xmm7 + jl L_AES_GCM_encrypt_avx1_calc_iv_12_last + vaesenc %xmm7, %xmm5, %xmm5 + vaesenc %xmm7, %xmm1, %xmm1 + vmovdqa 208(%r15), %xmm7 + vaesenc %xmm7, %xmm5, %xmm5 + vaesenc %xmm7, %xmm1, %xmm1 + vmovdqa 224(%r15), %xmm7 +L_AES_GCM_encrypt_avx1_calc_iv_12_last: + vaesenclast %xmm7, %xmm5, %xmm5 + vaesenclast %xmm7, %xmm1, %xmm1 + vpshufb L_avx1_aes_gcm_bswap_mask(%rip), %xmm5, %xmm5 + vmovdqa %xmm1, 144(%rsp) + jmp L_AES_GCM_encrypt_avx1_iv_done +L_AES_GCM_encrypt_avx1_iv_not_12: + # Calculate values when IV is not 12 bytes + # H = Encrypt X(=0) + vmovdqa (%r15), %xmm5 + vaesenc 16(%r15), %xmm5, %xmm5 + vaesenc 32(%r15), %xmm5, %xmm5 + vaesenc 48(%r15), %xmm5, %xmm5 + vaesenc 64(%r15), %xmm5, %xmm5 + vaesenc 80(%r15), %xmm5, %xmm5 + vaesenc 96(%r15), %xmm5, %xmm5 + vaesenc 112(%r15), %xmm5, %xmm5 + vaesenc 128(%r15), %xmm5, %xmm5 + vaesenc 144(%r15), %xmm5, %xmm5 + cmpl $11, %r10d + vmovdqa 160(%r15), %xmm9 + jl L_AES_GCM_encrypt_avx1_calc_iv_1_aesenc_avx_last + vaesenc %xmm9, %xmm5, %xmm5 + vaesenc 176(%r15), %xmm5, %xmm5 + cmpl $13, %r10d + vmovdqa 192(%r15), %xmm9 + jl L_AES_GCM_encrypt_avx1_calc_iv_1_aesenc_avx_last + vaesenc %xmm9, %xmm5, %xmm5 + vaesenc 208(%r15), %xmm5, %xmm5 + vmovdqa 224(%r15), %xmm9 +L_AES_GCM_encrypt_avx1_calc_iv_1_aesenc_avx_last: + vaesenclast %xmm9, %xmm5, %xmm5 + vpshufb L_avx1_aes_gcm_bswap_mask(%rip), %xmm5, %xmm5 + # Calc counter + # Initialization vector + cmpl $0x00, %edx + movq $0x00, %rcx + je L_AES_GCM_encrypt_avx1_calc_iv_done + cmpl $16, %edx + jl L_AES_GCM_encrypt_avx1_calc_iv_lt16 + andl $0xfffffff0, %edx +L_AES_GCM_encrypt_avx1_calc_iv_16_loop: + vmovdqu (%rax,%rcx,1), %xmm8 + vpshufb L_avx1_aes_gcm_bswap_mask(%rip), %xmm8, %xmm8 + vpxor %xmm8, %xmm4, %xmm4 + # ghash_gfmul_avx + vpshufd $0x4e, %xmm4, %xmm1 + vpshufd $0x4e, %xmm5, %xmm2 + vpclmulqdq $0x11, %xmm4, %xmm5, %xmm3 + vpclmulqdq $0x00, %xmm4, %xmm5, %xmm0 + vpxor %xmm4, %xmm1, %xmm1 + vpxor %xmm5, %xmm2, %xmm2 + vpclmulqdq $0x00, %xmm2, %xmm1, %xmm1 + vpxor %xmm0, %xmm1, %xmm1 + vpxor %xmm3, %xmm1, %xmm1 + vmovdqa %xmm0, %xmm7 + vmovdqa %xmm3, %xmm4 + vpslldq $8, %xmm1, %xmm2 + vpsrldq $8, %xmm1, %xmm1 + vpxor %xmm2, %xmm7, %xmm7 + vpxor %xmm1, %xmm4, %xmm4 + vpsrld $31, %xmm7, %xmm0 + vpsrld $31, %xmm4, %xmm1 + vpslld $0x01, %xmm7, %xmm7 + vpslld $0x01, %xmm4, %xmm4 + vpsrldq $12, %xmm0, %xmm2 + vpslldq $4, %xmm0, %xmm0 + vpslldq $4, %xmm1, %xmm1 + vpor %xmm2, %xmm4, %xmm4 + vpor %xmm0, %xmm7, %xmm7 + vpor %xmm1, %xmm4, %xmm4 + vpslld $31, %xmm7, %xmm0 + vpslld $30, %xmm7, %xmm1 + vpslld $25, %xmm7, %xmm2 + vpxor %xmm1, %xmm0, %xmm0 + vpxor %xmm2, %xmm0, %xmm0 + vmovdqa %xmm0, %xmm1 + vpsrldq $4, %xmm1, %xmm1 + vpslldq $12, %xmm0, %xmm0 + vpxor %xmm0, %xmm7, %xmm7 + vpsrld $0x01, %xmm7, %xmm2 + vpsrld $2, %xmm7, %xmm3 + vpsrld $7, %xmm7, %xmm0 + vpxor %xmm3, %xmm2, %xmm2 + vpxor %xmm0, %xmm2, %xmm2 + vpxor %xmm1, %xmm2, %xmm2 + vpxor %xmm7, %xmm2, %xmm2 + vpxor %xmm2, %xmm4, %xmm4 + addl $16, %ecx + cmpl %edx, %ecx + jl L_AES_GCM_encrypt_avx1_calc_iv_16_loop + movl %ebx, %edx + cmpl %edx, %ecx + je L_AES_GCM_encrypt_avx1_calc_iv_done +L_AES_GCM_encrypt_avx1_calc_iv_lt16: + subq $16, %rsp + vpxor %xmm8, %xmm8, %xmm8 + xorl %ebx, %ebx + vmovdqa %xmm8, (%rsp) +L_AES_GCM_encrypt_avx1_calc_iv_loop: + movzbl (%rax,%rcx,1), %r13d + movb %r13b, (%rsp,%rbx,1) + incl %ecx + incl %ebx + cmpl %edx, %ecx + jl L_AES_GCM_encrypt_avx1_calc_iv_loop + vmovdqa (%rsp), %xmm8 + addq $16, %rsp + vpshufb L_avx1_aes_gcm_bswap_mask(%rip), %xmm8, %xmm8 + vpxor %xmm8, %xmm4, %xmm4 + # ghash_gfmul_avx + vpshufd $0x4e, %xmm4, %xmm1 + vpshufd $0x4e, %xmm5, %xmm2 + vpclmulqdq $0x11, %xmm4, %xmm5, %xmm3 + vpclmulqdq $0x00, %xmm4, %xmm5, %xmm0 + vpxor %xmm4, %xmm1, %xmm1 + vpxor %xmm5, %xmm2, %xmm2 + vpclmulqdq $0x00, %xmm2, %xmm1, %xmm1 + vpxor %xmm0, %xmm1, %xmm1 + vpxor %xmm3, %xmm1, %xmm1 + vmovdqa %xmm0, %xmm7 + vmovdqa %xmm3, %xmm4 + vpslldq $8, %xmm1, %xmm2 + vpsrldq $8, %xmm1, %xmm1 + vpxor %xmm2, %xmm7, %xmm7 + vpxor %xmm1, %xmm4, %xmm4 + vpsrld $31, %xmm7, %xmm0 + vpsrld $31, %xmm4, %xmm1 + vpslld $0x01, %xmm7, %xmm7 + vpslld $0x01, %xmm4, %xmm4 + vpsrldq $12, %xmm0, %xmm2 + vpslldq $4, %xmm0, %xmm0 + vpslldq $4, %xmm1, %xmm1 + vpor %xmm2, %xmm4, %xmm4 + vpor %xmm0, %xmm7, %xmm7 + vpor %xmm1, %xmm4, %xmm4 + vpslld $31, %xmm7, %xmm0 + vpslld $30, %xmm7, %xmm1 + vpslld $25, %xmm7, %xmm2 + vpxor %xmm1, %xmm0, %xmm0 + vpxor %xmm2, %xmm0, %xmm0 + vmovdqa %xmm0, %xmm1 + vpsrldq $4, %xmm1, %xmm1 + vpslldq $12, %xmm0, %xmm0 + vpxor %xmm0, %xmm7, %xmm7 + vpsrld $0x01, %xmm7, %xmm2 + vpsrld $2, %xmm7, %xmm3 + vpsrld $7, %xmm7, %xmm0 + vpxor %xmm3, %xmm2, %xmm2 + vpxor %xmm0, %xmm2, %xmm2 + vpxor %xmm1, %xmm2, %xmm2 + vpxor %xmm7, %xmm2, %xmm2 + vpxor %xmm2, %xmm4, %xmm4 +L_AES_GCM_encrypt_avx1_calc_iv_done: + # T = Encrypt counter + vpxor %xmm0, %xmm0, %xmm0 + shll $3, %edx + vpinsrq $0x00, %rdx, %xmm0, %xmm0 + vpxor %xmm0, %xmm4, %xmm4 + # ghash_gfmul_avx + vpshufd $0x4e, %xmm4, %xmm1 + vpshufd $0x4e, %xmm5, %xmm2 + vpclmulqdq $0x11, %xmm4, %xmm5, %xmm3 + vpclmulqdq $0x00, %xmm4, %xmm5, %xmm0 + vpxor %xmm4, %xmm1, %xmm1 + vpxor %xmm5, %xmm2, %xmm2 + vpclmulqdq $0x00, %xmm2, %xmm1, %xmm1 + vpxor %xmm0, %xmm1, %xmm1 + vpxor %xmm3, %xmm1, %xmm1 + vmovdqa %xmm0, %xmm7 + vmovdqa %xmm3, %xmm4 + vpslldq $8, %xmm1, %xmm2 + vpsrldq $8, %xmm1, %xmm1 + vpxor %xmm2, %xmm7, %xmm7 + vpxor %xmm1, %xmm4, %xmm4 + vpsrld $31, %xmm7, %xmm0 + vpsrld $31, %xmm4, %xmm1 + vpslld $0x01, %xmm7, %xmm7 + vpslld $0x01, %xmm4, %xmm4 + vpsrldq $12, %xmm0, %xmm2 + vpslldq $4, %xmm0, %xmm0 + vpslldq $4, %xmm1, %xmm1 + vpor %xmm2, %xmm4, %xmm4 + vpor %xmm0, %xmm7, %xmm7 + vpor %xmm1, %xmm4, %xmm4 + vpslld $31, %xmm7, %xmm0 + vpslld $30, %xmm7, %xmm1 + vpslld $25, %xmm7, %xmm2 + vpxor %xmm1, %xmm0, %xmm0 + vpxor %xmm2, %xmm0, %xmm0 + vmovdqa %xmm0, %xmm1 + vpsrldq $4, %xmm1, %xmm1 + vpslldq $12, %xmm0, %xmm0 + vpxor %xmm0, %xmm7, %xmm7 + vpsrld $0x01, %xmm7, %xmm2 + vpsrld $2, %xmm7, %xmm3 + vpsrld $7, %xmm7, %xmm0 + vpxor %xmm3, %xmm2, %xmm2 + vpxor %xmm0, %xmm2, %xmm2 + vpxor %xmm1, %xmm2, %xmm2 + vpxor %xmm7, %xmm2, %xmm2 + vpxor %xmm2, %xmm4, %xmm4 + vpshufb L_avx1_aes_gcm_bswap_mask(%rip), %xmm4, %xmm4 + # Encrypt counter + vmovdqa (%r15), %xmm8 + vpxor %xmm4, %xmm8, %xmm8 + vaesenc 16(%r15), %xmm8, %xmm8 + vaesenc 32(%r15), %xmm8, %xmm8 + vaesenc 48(%r15), %xmm8, %xmm8 + vaesenc 64(%r15), %xmm8, %xmm8 + vaesenc 80(%r15), %xmm8, %xmm8 + vaesenc 96(%r15), %xmm8, %xmm8 + vaesenc 112(%r15), %xmm8, %xmm8 + vaesenc 128(%r15), %xmm8, %xmm8 + vaesenc 144(%r15), %xmm8, %xmm8 + cmpl $11, %r10d + vmovdqa 160(%r15), %xmm9 + jl L_AES_GCM_encrypt_avx1_calc_iv_2_aesenc_avx_last + vaesenc %xmm9, %xmm8, %xmm8 + vaesenc 176(%r15), %xmm8, %xmm8 + cmpl $13, %r10d + vmovdqa 192(%r15), %xmm9 + jl L_AES_GCM_encrypt_avx1_calc_iv_2_aesenc_avx_last + vaesenc %xmm9, %xmm8, %xmm8 + vaesenc 208(%r15), %xmm8, %xmm8 + vmovdqa 224(%r15), %xmm9 +L_AES_GCM_encrypt_avx1_calc_iv_2_aesenc_avx_last: + vaesenclast %xmm9, %xmm8, %xmm8 + vmovdqa %xmm8, 144(%rsp) +L_AES_GCM_encrypt_avx1_iv_done: + # Additional authentication data + movl %r11d, %edx + cmpl $0x00, %edx + je L_AES_GCM_encrypt_avx1_calc_aad_done + xorl %ecx, %ecx + cmpl $16, %edx + jl L_AES_GCM_encrypt_avx1_calc_aad_lt16 + andl $0xfffffff0, %edx +L_AES_GCM_encrypt_avx1_calc_aad_16_loop: + vmovdqu (%r12,%rcx,1), %xmm8 + vpshufb L_avx1_aes_gcm_bswap_mask(%rip), %xmm8, %xmm8 + vpxor %xmm8, %xmm6, %xmm6 + # ghash_gfmul_avx + vpshufd $0x4e, %xmm6, %xmm1 + vpshufd $0x4e, %xmm5, %xmm2 + vpclmulqdq $0x11, %xmm6, %xmm5, %xmm3 + vpclmulqdq $0x00, %xmm6, %xmm5, %xmm0 + vpxor %xmm6, %xmm1, %xmm1 + vpxor %xmm5, %xmm2, %xmm2 + vpclmulqdq $0x00, %xmm2, %xmm1, %xmm1 + vpxor %xmm0, %xmm1, %xmm1 + vpxor %xmm3, %xmm1, %xmm1 + vmovdqa %xmm0, %xmm7 + vmovdqa %xmm3, %xmm6 + vpslldq $8, %xmm1, %xmm2 + vpsrldq $8, %xmm1, %xmm1 + vpxor %xmm2, %xmm7, %xmm7 + vpxor %xmm1, %xmm6, %xmm6 + vpsrld $31, %xmm7, %xmm0 + vpsrld $31, %xmm6, %xmm1 + vpslld $0x01, %xmm7, %xmm7 + vpslld $0x01, %xmm6, %xmm6 + vpsrldq $12, %xmm0, %xmm2 + vpslldq $4, %xmm0, %xmm0 + vpslldq $4, %xmm1, %xmm1 + vpor %xmm2, %xmm6, %xmm6 + vpor %xmm0, %xmm7, %xmm7 + vpor %xmm1, %xmm6, %xmm6 + vpslld $31, %xmm7, %xmm0 + vpslld $30, %xmm7, %xmm1 + vpslld $25, %xmm7, %xmm2 + vpxor %xmm1, %xmm0, %xmm0 + vpxor %xmm2, %xmm0, %xmm0 + vmovdqa %xmm0, %xmm1 + vpsrldq $4, %xmm1, %xmm1 + vpslldq $12, %xmm0, %xmm0 + vpxor %xmm0, %xmm7, %xmm7 + vpsrld $0x01, %xmm7, %xmm2 + vpsrld $2, %xmm7, %xmm3 + vpsrld $7, %xmm7, %xmm0 + vpxor %xmm3, %xmm2, %xmm2 + vpxor %xmm0, %xmm2, %xmm2 + vpxor %xmm1, %xmm2, %xmm2 + vpxor %xmm7, %xmm2, %xmm2 + vpxor %xmm2, %xmm6, %xmm6 + addl $16, %ecx + cmpl %edx, %ecx + jl L_AES_GCM_encrypt_avx1_calc_aad_16_loop + movl %r11d, %edx + cmpl %edx, %ecx + je L_AES_GCM_encrypt_avx1_calc_aad_done +L_AES_GCM_encrypt_avx1_calc_aad_lt16: + subq $16, %rsp + vpxor %xmm8, %xmm8, %xmm8 + xorl %ebx, %ebx + vmovdqa %xmm8, (%rsp) +L_AES_GCM_encrypt_avx1_calc_aad_loop: + movzbl (%r12,%rcx,1), %r13d + movb %r13b, (%rsp,%rbx,1) + incl %ecx + incl %ebx + cmpl %edx, %ecx + jl L_AES_GCM_encrypt_avx1_calc_aad_loop + vmovdqa (%rsp), %xmm8 + addq $16, %rsp + vpshufb L_avx1_aes_gcm_bswap_mask(%rip), %xmm8, %xmm8 + vpxor %xmm8, %xmm6, %xmm6 + # ghash_gfmul_avx + vpshufd $0x4e, %xmm6, %xmm1 + vpshufd $0x4e, %xmm5, %xmm2 + vpclmulqdq $0x11, %xmm6, %xmm5, %xmm3 + vpclmulqdq $0x00, %xmm6, %xmm5, %xmm0 + vpxor %xmm6, %xmm1, %xmm1 + vpxor %xmm5, %xmm2, %xmm2 + vpclmulqdq $0x00, %xmm2, %xmm1, %xmm1 + vpxor %xmm0, %xmm1, %xmm1 + vpxor %xmm3, %xmm1, %xmm1 + vmovdqa %xmm0, %xmm7 + vmovdqa %xmm3, %xmm6 + vpslldq $8, %xmm1, %xmm2 + vpsrldq $8, %xmm1, %xmm1 + vpxor %xmm2, %xmm7, %xmm7 + vpxor %xmm1, %xmm6, %xmm6 + vpsrld $31, %xmm7, %xmm0 + vpsrld $31, %xmm6, %xmm1 + vpslld $0x01, %xmm7, %xmm7 + vpslld $0x01, %xmm6, %xmm6 + vpsrldq $12, %xmm0, %xmm2 + vpslldq $4, %xmm0, %xmm0 + vpslldq $4, %xmm1, %xmm1 + vpor %xmm2, %xmm6, %xmm6 + vpor %xmm0, %xmm7, %xmm7 + vpor %xmm1, %xmm6, %xmm6 + vpslld $31, %xmm7, %xmm0 + vpslld $30, %xmm7, %xmm1 + vpslld $25, %xmm7, %xmm2 + vpxor %xmm1, %xmm0, %xmm0 + vpxor %xmm2, %xmm0, %xmm0 + vmovdqa %xmm0, %xmm1 + vpsrldq $4, %xmm1, %xmm1 + vpslldq $12, %xmm0, %xmm0 + vpxor %xmm0, %xmm7, %xmm7 + vpsrld $0x01, %xmm7, %xmm2 + vpsrld $2, %xmm7, %xmm3 + vpsrld $7, %xmm7, %xmm0 + vpxor %xmm3, %xmm2, %xmm2 + vpxor %xmm0, %xmm2, %xmm2 + vpxor %xmm1, %xmm2, %xmm2 + vpxor %xmm7, %xmm2, %xmm2 + vpxor %xmm2, %xmm6, %xmm6 +L_AES_GCM_encrypt_avx1_calc_aad_done: + # Calculate counter and H + vpsrlq $63, %xmm5, %xmm9 + vpsllq $0x01, %xmm5, %xmm8 + vpslldq $8, %xmm9, %xmm9 + vpor %xmm9, %xmm8, %xmm8 + vpshufd $0xff, %xmm5, %xmm5 + vpsrad $31, %xmm5, %xmm5 + vpshufb L_avx1_aes_gcm_bswap_epi64(%rip), %xmm4, %xmm4 + vpand L_avx1_aes_gcm_mod2_128(%rip), %xmm5, %xmm5 + vpaddd L_avx1_aes_gcm_one(%rip), %xmm4, %xmm4 + vpxor %xmm8, %xmm5, %xmm5 + vmovdqa %xmm4, 128(%rsp) + xorl %ebx, %ebx + cmpl $0x80, %r9d + movl %r9d, %r13d + jl L_AES_GCM_encrypt_avx1_done_128 + andl $0xffffff80, %r13d + vmovdqa %xmm6, %xmm2 + # H ^ 1 + vmovdqa %xmm5, (%rsp) + # H ^ 2 + vpclmulqdq $0x00, %xmm5, %xmm5, %xmm8 + vpclmulqdq $0x11, %xmm5, %xmm5, %xmm0 + vpslld $31, %xmm8, %xmm12 + vpslld $30, %xmm8, %xmm13 + vpslld $25, %xmm8, %xmm14 + vpxor %xmm13, %xmm12, %xmm12 + vpxor %xmm14, %xmm12, %xmm12 + vpsrldq $4, %xmm12, %xmm13 + vpslldq $12, %xmm12, %xmm12 + vpxor %xmm12, %xmm8, %xmm8 + vpsrld $0x01, %xmm8, %xmm14 + vpsrld $2, %xmm8, %xmm10 + vpsrld $7, %xmm8, %xmm9 + vpxor %xmm10, %xmm14, %xmm14 + vpxor %xmm9, %xmm14, %xmm14 + vpxor %xmm13, %xmm14, %xmm14 + vpxor %xmm8, %xmm14, %xmm14 + vpxor %xmm14, %xmm0, %xmm0 + vmovdqa %xmm0, 16(%rsp) + # H ^ 3 + # ghash_gfmul_red_avx + vpshufd $0x4e, %xmm5, %xmm9 + vpshufd $0x4e, %xmm0, %xmm10 + vpclmulqdq $0x11, %xmm5, %xmm0, %xmm11 + vpclmulqdq $0x00, %xmm5, %xmm0, %xmm8 + vpxor %xmm5, %xmm9, %xmm9 + vpxor %xmm0, %xmm10, %xmm10 + vpclmulqdq $0x00, %xmm10, %xmm9, %xmm9 + vpxor %xmm8, %xmm9, %xmm9 + vpxor %xmm11, %xmm9, %xmm9 + vpslldq $8, %xmm9, %xmm10 + vpsrldq $8, %xmm9, %xmm9 + vpxor %xmm10, %xmm8, %xmm8 + vpxor %xmm9, %xmm11, %xmm1 + vpslld $31, %xmm8, %xmm12 + vpslld $30, %xmm8, %xmm13 + vpslld $25, %xmm8, %xmm14 + vpxor %xmm13, %xmm12, %xmm12 + vpxor %xmm14, %xmm12, %xmm12 + vpsrldq $4, %xmm12, %xmm13 + vpslldq $12, %xmm12, %xmm12 + vpxor %xmm12, %xmm8, %xmm8 + vpsrld $0x01, %xmm8, %xmm14 + vpsrld $2, %xmm8, %xmm10 + vpsrld $7, %xmm8, %xmm9 + vpxor %xmm10, %xmm14, %xmm14 + vpxor %xmm9, %xmm14, %xmm14 + vpxor %xmm13, %xmm14, %xmm14 + vpxor %xmm8, %xmm14, %xmm14 + vpxor %xmm14, %xmm1, %xmm1 + vmovdqa %xmm1, 32(%rsp) + # H ^ 4 + vpclmulqdq $0x00, %xmm0, %xmm0, %xmm8 + vpclmulqdq $0x11, %xmm0, %xmm0, %xmm3 + vpslld $31, %xmm8, %xmm12 + vpslld $30, %xmm8, %xmm13 + vpslld $25, %xmm8, %xmm14 + vpxor %xmm13, %xmm12, %xmm12 + vpxor %xmm14, %xmm12, %xmm12 + vpsrldq $4, %xmm12, %xmm13 + vpslldq $12, %xmm12, %xmm12 + vpxor %xmm12, %xmm8, %xmm8 + vpsrld $0x01, %xmm8, %xmm14 + vpsrld $2, %xmm8, %xmm10 + vpsrld $7, %xmm8, %xmm9 + vpxor %xmm10, %xmm14, %xmm14 + vpxor %xmm9, %xmm14, %xmm14 + vpxor %xmm13, %xmm14, %xmm14 + vpxor %xmm8, %xmm14, %xmm14 + vpxor %xmm14, %xmm3, %xmm3 + vmovdqa %xmm3, 48(%rsp) + # H ^ 5 + # ghash_gfmul_red_avx + vpshufd $0x4e, %xmm0, %xmm9 + vpshufd $0x4e, %xmm1, %xmm10 + vpclmulqdq $0x11, %xmm0, %xmm1, %xmm11 + vpclmulqdq $0x00, %xmm0, %xmm1, %xmm8 + vpxor %xmm0, %xmm9, %xmm9 + vpxor %xmm1, %xmm10, %xmm10 + vpclmulqdq $0x00, %xmm10, %xmm9, %xmm9 + vpxor %xmm8, %xmm9, %xmm9 + vpxor %xmm11, %xmm9, %xmm9 + vpslldq $8, %xmm9, %xmm10 + vpsrldq $8, %xmm9, %xmm9 + vpxor %xmm10, %xmm8, %xmm8 + vpxor %xmm9, %xmm11, %xmm7 + vpslld $31, %xmm8, %xmm12 + vpslld $30, %xmm8, %xmm13 + vpslld $25, %xmm8, %xmm14 + vpxor %xmm13, %xmm12, %xmm12 + vpxor %xmm14, %xmm12, %xmm12 + vpsrldq $4, %xmm12, %xmm13 + vpslldq $12, %xmm12, %xmm12 + vpxor %xmm12, %xmm8, %xmm8 + vpsrld $0x01, %xmm8, %xmm14 + vpsrld $2, %xmm8, %xmm10 + vpsrld $7, %xmm8, %xmm9 + vpxor %xmm10, %xmm14, %xmm14 + vpxor %xmm9, %xmm14, %xmm14 + vpxor %xmm13, %xmm14, %xmm14 + vpxor %xmm8, %xmm14, %xmm14 + vpxor %xmm14, %xmm7, %xmm7 + vmovdqa %xmm7, 64(%rsp) + # H ^ 6 + vpclmulqdq $0x00, %xmm1, %xmm1, %xmm8 + vpclmulqdq $0x11, %xmm1, %xmm1, %xmm7 + vpslld $31, %xmm8, %xmm12 + vpslld $30, %xmm8, %xmm13 + vpslld $25, %xmm8, %xmm14 + vpxor %xmm13, %xmm12, %xmm12 + vpxor %xmm14, %xmm12, %xmm12 + vpsrldq $4, %xmm12, %xmm13 + vpslldq $12, %xmm12, %xmm12 + vpxor %xmm12, %xmm8, %xmm8 + vpsrld $0x01, %xmm8, %xmm14 + vpsrld $2, %xmm8, %xmm10 + vpsrld $7, %xmm8, %xmm9 + vpxor %xmm10, %xmm14, %xmm14 + vpxor %xmm9, %xmm14, %xmm14 + vpxor %xmm13, %xmm14, %xmm14 + vpxor %xmm8, %xmm14, %xmm14 + vpxor %xmm14, %xmm7, %xmm7 + vmovdqa %xmm7, 80(%rsp) + # H ^ 7 + # ghash_gfmul_red_avx + vpshufd $0x4e, %xmm1, %xmm9 + vpshufd $0x4e, %xmm3, %xmm10 + vpclmulqdq $0x11, %xmm1, %xmm3, %xmm11 + vpclmulqdq $0x00, %xmm1, %xmm3, %xmm8 + vpxor %xmm1, %xmm9, %xmm9 + vpxor %xmm3, %xmm10, %xmm10 + vpclmulqdq $0x00, %xmm10, %xmm9, %xmm9 + vpxor %xmm8, %xmm9, %xmm9 + vpxor %xmm11, %xmm9, %xmm9 + vpslldq $8, %xmm9, %xmm10 + vpsrldq $8, %xmm9, %xmm9 + vpxor %xmm10, %xmm8, %xmm8 + vpxor %xmm9, %xmm11, %xmm7 + vpslld $31, %xmm8, %xmm12 + vpslld $30, %xmm8, %xmm13 + vpslld $25, %xmm8, %xmm14 + vpxor %xmm13, %xmm12, %xmm12 + vpxor %xmm14, %xmm12, %xmm12 + vpsrldq $4, %xmm12, %xmm13 + vpslldq $12, %xmm12, %xmm12 + vpxor %xmm12, %xmm8, %xmm8 + vpsrld $0x01, %xmm8, %xmm14 + vpsrld $2, %xmm8, %xmm10 + vpsrld $7, %xmm8, %xmm9 + vpxor %xmm10, %xmm14, %xmm14 + vpxor %xmm9, %xmm14, %xmm14 + vpxor %xmm13, %xmm14, %xmm14 + vpxor %xmm8, %xmm14, %xmm14 + vpxor %xmm14, %xmm7, %xmm7 + vmovdqa %xmm7, 96(%rsp) + # H ^ 8 + vpclmulqdq $0x00, %xmm3, %xmm3, %xmm8 + vpclmulqdq $0x11, %xmm3, %xmm3, %xmm7 + vpslld $31, %xmm8, %xmm12 + vpslld $30, %xmm8, %xmm13 + vpslld $25, %xmm8, %xmm14 + vpxor %xmm13, %xmm12, %xmm12 + vpxor %xmm14, %xmm12, %xmm12 + vpsrldq $4, %xmm12, %xmm13 + vpslldq $12, %xmm12, %xmm12 + vpxor %xmm12, %xmm8, %xmm8 + vpsrld $0x01, %xmm8, %xmm14 + vpsrld $2, %xmm8, %xmm10 + vpsrld $7, %xmm8, %xmm9 + vpxor %xmm10, %xmm14, %xmm14 + vpxor %xmm9, %xmm14, %xmm14 + vpxor %xmm13, %xmm14, %xmm14 + vpxor %xmm8, %xmm14, %xmm14 + vpxor %xmm14, %xmm7, %xmm7 + vmovdqa %xmm7, 112(%rsp) + # First 128 bytes of input + vmovdqa 128(%rsp), %xmm0 + vmovdqa L_avx1_aes_gcm_bswap_epi64(%rip), %xmm1 + vpshufb %xmm1, %xmm0, %xmm8 + vpaddd L_avx1_aes_gcm_one(%rip), %xmm0, %xmm9 + vpshufb %xmm1, %xmm9, %xmm9 + vpaddd L_avx1_aes_gcm_two(%rip), %xmm0, %xmm10 + vpshufb %xmm1, %xmm10, %xmm10 + vpaddd L_avx1_aes_gcm_three(%rip), %xmm0, %xmm11 + vpshufb %xmm1, %xmm11, %xmm11 + vpaddd L_avx1_aes_gcm_four(%rip), %xmm0, %xmm12 + vpshufb %xmm1, %xmm12, %xmm12 + vpaddd L_avx1_aes_gcm_five(%rip), %xmm0, %xmm13 + vpshufb %xmm1, %xmm13, %xmm13 + vpaddd L_avx1_aes_gcm_six(%rip), %xmm0, %xmm14 + vpshufb %xmm1, %xmm14, %xmm14 + vpaddd L_avx1_aes_gcm_seven(%rip), %xmm0, %xmm15 + vpshufb %xmm1, %xmm15, %xmm15 + vpaddd L_avx1_aes_gcm_eight(%rip), %xmm0, %xmm0 + vmovdqa (%r15), %xmm7 + vmovdqa %xmm0, 128(%rsp) + vpxor %xmm7, %xmm8, %xmm8 + vpxor %xmm7, %xmm9, %xmm9 + vpxor %xmm7, %xmm10, %xmm10 + vpxor %xmm7, %xmm11, %xmm11 + vpxor %xmm7, %xmm12, %xmm12 + vpxor %xmm7, %xmm13, %xmm13 + vpxor %xmm7, %xmm14, %xmm14 + vpxor %xmm7, %xmm15, %xmm15 + vmovdqa 16(%r15), %xmm7 + vaesenc %xmm7, %xmm8, %xmm8 + vaesenc %xmm7, %xmm9, %xmm9 + vaesenc %xmm7, %xmm10, %xmm10 + vaesenc %xmm7, %xmm11, %xmm11 + vaesenc %xmm7, %xmm12, %xmm12 + vaesenc %xmm7, %xmm13, %xmm13 + vaesenc %xmm7, %xmm14, %xmm14 + vaesenc %xmm7, %xmm15, %xmm15 + vmovdqa 32(%r15), %xmm7 + vaesenc %xmm7, %xmm8, %xmm8 + vaesenc %xmm7, %xmm9, %xmm9 + vaesenc %xmm7, %xmm10, %xmm10 + vaesenc %xmm7, %xmm11, %xmm11 + vaesenc %xmm7, %xmm12, %xmm12 + vaesenc %xmm7, %xmm13, %xmm13 + vaesenc %xmm7, %xmm14, %xmm14 + vaesenc %xmm7, %xmm15, %xmm15 + vmovdqa 48(%r15), %xmm7 + vaesenc %xmm7, %xmm8, %xmm8 + vaesenc %xmm7, %xmm9, %xmm9 + vaesenc %xmm7, %xmm10, %xmm10 + vaesenc %xmm7, %xmm11, %xmm11 + vaesenc %xmm7, %xmm12, %xmm12 + vaesenc %xmm7, %xmm13, %xmm13 + vaesenc %xmm7, %xmm14, %xmm14 + vaesenc %xmm7, %xmm15, %xmm15 + vmovdqa 64(%r15), %xmm7 + vaesenc %xmm7, %xmm8, %xmm8 + vaesenc %xmm7, %xmm9, %xmm9 + vaesenc %xmm7, %xmm10, %xmm10 + vaesenc %xmm7, %xmm11, %xmm11 + vaesenc %xmm7, %xmm12, %xmm12 + vaesenc %xmm7, %xmm13, %xmm13 + vaesenc %xmm7, %xmm14, %xmm14 + vaesenc %xmm7, %xmm15, %xmm15 + vmovdqa 80(%r15), %xmm7 + vaesenc %xmm7, %xmm8, %xmm8 + vaesenc %xmm7, %xmm9, %xmm9 + vaesenc %xmm7, %xmm10, %xmm10 + vaesenc %xmm7, %xmm11, %xmm11 + vaesenc %xmm7, %xmm12, %xmm12 + vaesenc %xmm7, %xmm13, %xmm13 + vaesenc %xmm7, %xmm14, %xmm14 + vaesenc %xmm7, %xmm15, %xmm15 + vmovdqa 96(%r15), %xmm7 + vaesenc %xmm7, %xmm8, %xmm8 + vaesenc %xmm7, %xmm9, %xmm9 + vaesenc %xmm7, %xmm10, %xmm10 + vaesenc %xmm7, %xmm11, %xmm11 + vaesenc %xmm7, %xmm12, %xmm12 + vaesenc %xmm7, %xmm13, %xmm13 + vaesenc %xmm7, %xmm14, %xmm14 + vaesenc %xmm7, %xmm15, %xmm15 + vmovdqa 112(%r15), %xmm7 + vaesenc %xmm7, %xmm8, %xmm8 + vaesenc %xmm7, %xmm9, %xmm9 + vaesenc %xmm7, %xmm10, %xmm10 + vaesenc %xmm7, %xmm11, %xmm11 + vaesenc %xmm7, %xmm12, %xmm12 + vaesenc %xmm7, %xmm13, %xmm13 + vaesenc %xmm7, %xmm14, %xmm14 + vaesenc %xmm7, %xmm15, %xmm15 + vmovdqa 128(%r15), %xmm7 + vaesenc %xmm7, %xmm8, %xmm8 + vaesenc %xmm7, %xmm9, %xmm9 + vaesenc %xmm7, %xmm10, %xmm10 + vaesenc %xmm7, %xmm11, %xmm11 + vaesenc %xmm7, %xmm12, %xmm12 + vaesenc %xmm7, %xmm13, %xmm13 + vaesenc %xmm7, %xmm14, %xmm14 + vaesenc %xmm7, %xmm15, %xmm15 + vmovdqa 144(%r15), %xmm7 + vaesenc %xmm7, %xmm8, %xmm8 + vaesenc %xmm7, %xmm9, %xmm9 + vaesenc %xmm7, %xmm10, %xmm10 + vaesenc %xmm7, %xmm11, %xmm11 + vaesenc %xmm7, %xmm12, %xmm12 + vaesenc %xmm7, %xmm13, %xmm13 + vaesenc %xmm7, %xmm14, %xmm14 + vaesenc %xmm7, %xmm15, %xmm15 + cmpl $11, %r10d + vmovdqa 160(%r15), %xmm7 + jl L_AES_GCM_encrypt_avx1_aesenc_128_enc_done + vaesenc %xmm7, %xmm8, %xmm8 + vaesenc %xmm7, %xmm9, %xmm9 + vaesenc %xmm7, %xmm10, %xmm10 + vaesenc %xmm7, %xmm11, %xmm11 + vaesenc %xmm7, %xmm12, %xmm12 + vaesenc %xmm7, %xmm13, %xmm13 + vaesenc %xmm7, %xmm14, %xmm14 + vaesenc %xmm7, %xmm15, %xmm15 + vmovdqa 176(%r15), %xmm7 + vaesenc %xmm7, %xmm8, %xmm8 + vaesenc %xmm7, %xmm9, %xmm9 + vaesenc %xmm7, %xmm10, %xmm10 + vaesenc %xmm7, %xmm11, %xmm11 + vaesenc %xmm7, %xmm12, %xmm12 + vaesenc %xmm7, %xmm13, %xmm13 + vaesenc %xmm7, %xmm14, %xmm14 + vaesenc %xmm7, %xmm15, %xmm15 + cmpl $13, %r10d + vmovdqa 192(%r15), %xmm7 + jl L_AES_GCM_encrypt_avx1_aesenc_128_enc_done + vaesenc %xmm7, %xmm8, %xmm8 + vaesenc %xmm7, %xmm9, %xmm9 + vaesenc %xmm7, %xmm10, %xmm10 + vaesenc %xmm7, %xmm11, %xmm11 + vaesenc %xmm7, %xmm12, %xmm12 + vaesenc %xmm7, %xmm13, %xmm13 + vaesenc %xmm7, %xmm14, %xmm14 + vaesenc %xmm7, %xmm15, %xmm15 + vmovdqa 208(%r15), %xmm7 + vaesenc %xmm7, %xmm8, %xmm8 + vaesenc %xmm7, %xmm9, %xmm9 + vaesenc %xmm7, %xmm10, %xmm10 + vaesenc %xmm7, %xmm11, %xmm11 + vaesenc %xmm7, %xmm12, %xmm12 + vaesenc %xmm7, %xmm13, %xmm13 + vaesenc %xmm7, %xmm14, %xmm14 + vaesenc %xmm7, %xmm15, %xmm15 + vmovdqa 224(%r15), %xmm7 +L_AES_GCM_encrypt_avx1_aesenc_128_enc_done: + vaesenclast %xmm7, %xmm8, %xmm8 + vaesenclast %xmm7, %xmm9, %xmm9 + vmovdqu (%rdi), %xmm0 + vmovdqu 16(%rdi), %xmm1 + vpxor %xmm0, %xmm8, %xmm8 + vpxor %xmm1, %xmm9, %xmm9 + vmovdqu %xmm8, (%rsi) + vmovdqu %xmm9, 16(%rsi) + vaesenclast %xmm7, %xmm10, %xmm10 + vaesenclast %xmm7, %xmm11, %xmm11 + vmovdqu 32(%rdi), %xmm0 + vmovdqu 48(%rdi), %xmm1 + vpxor %xmm0, %xmm10, %xmm10 + vpxor %xmm1, %xmm11, %xmm11 + vmovdqu %xmm10, 32(%rsi) + vmovdqu %xmm11, 48(%rsi) + vaesenclast %xmm7, %xmm12, %xmm12 + vaesenclast %xmm7, %xmm13, %xmm13 + vmovdqu 64(%rdi), %xmm0 + vmovdqu 80(%rdi), %xmm1 + vpxor %xmm0, %xmm12, %xmm12 + vpxor %xmm1, %xmm13, %xmm13 + vmovdqu %xmm12, 64(%rsi) + vmovdqu %xmm13, 80(%rsi) + vaesenclast %xmm7, %xmm14, %xmm14 + vaesenclast %xmm7, %xmm15, %xmm15 + vmovdqu 96(%rdi), %xmm0 + vmovdqu 112(%rdi), %xmm1 + vpxor %xmm0, %xmm14, %xmm14 + vpxor %xmm1, %xmm15, %xmm15 + vmovdqu %xmm14, 96(%rsi) + vmovdqu %xmm15, 112(%rsi) + cmpl $0x80, %r13d + movl $0x80, %ebx + jle L_AES_GCM_encrypt_avx1_end_128 + # More 128 bytes of input +L_AES_GCM_encrypt_avx1_ghash_128: + leaq (%rdi,%rbx,1), %rcx + leaq (%rsi,%rbx,1), %rdx + vmovdqa 128(%rsp), %xmm0 + vmovdqa L_avx1_aes_gcm_bswap_epi64(%rip), %xmm1 + vpshufb %xmm1, %xmm0, %xmm8 + vpaddd L_avx1_aes_gcm_one(%rip), %xmm0, %xmm9 + vpshufb %xmm1, %xmm9, %xmm9 + vpaddd L_avx1_aes_gcm_two(%rip), %xmm0, %xmm10 + vpshufb %xmm1, %xmm10, %xmm10 + vpaddd L_avx1_aes_gcm_three(%rip), %xmm0, %xmm11 + vpshufb %xmm1, %xmm11, %xmm11 + vpaddd L_avx1_aes_gcm_four(%rip), %xmm0, %xmm12 + vpshufb %xmm1, %xmm12, %xmm12 + vpaddd L_avx1_aes_gcm_five(%rip), %xmm0, %xmm13 + vpshufb %xmm1, %xmm13, %xmm13 + vpaddd L_avx1_aes_gcm_six(%rip), %xmm0, %xmm14 + vpshufb %xmm1, %xmm14, %xmm14 + vpaddd L_avx1_aes_gcm_seven(%rip), %xmm0, %xmm15 + vpshufb %xmm1, %xmm15, %xmm15 + vpaddd L_avx1_aes_gcm_eight(%rip), %xmm0, %xmm0 + vmovdqa (%r15), %xmm7 + vmovdqa %xmm0, 128(%rsp) + vpxor %xmm7, %xmm8, %xmm8 + vpxor %xmm7, %xmm9, %xmm9 + vpxor %xmm7, %xmm10, %xmm10 + vpxor %xmm7, %xmm11, %xmm11 + vpxor %xmm7, %xmm12, %xmm12 + vpxor %xmm7, %xmm13, %xmm13 + vpxor %xmm7, %xmm14, %xmm14 + vpxor %xmm7, %xmm15, %xmm15 + vmovdqa 112(%rsp), %xmm7 + vmovdqu -128(%rdx), %xmm0 + vaesenc 16(%r15), %xmm8, %xmm8 + vpshufb L_avx1_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0 + vpxor %xmm2, %xmm0, %xmm0 + vpshufd $0x4e, %xmm7, %xmm1 + vpshufd $0x4e, %xmm0, %xmm5 + vpxor %xmm7, %xmm1, %xmm1 + vpxor %xmm0, %xmm5, %xmm5 + vpclmulqdq $0x11, %xmm7, %xmm0, %xmm3 + vaesenc 16(%r15), %xmm9, %xmm9 + vaesenc 16(%r15), %xmm10, %xmm10 + vpclmulqdq $0x00, %xmm7, %xmm0, %xmm2 + vaesenc 16(%r15), %xmm11, %xmm11 + vaesenc 16(%r15), %xmm12, %xmm12 + vpclmulqdq $0x00, %xmm5, %xmm1, %xmm1 + vaesenc 16(%r15), %xmm13, %xmm13 + vaesenc 16(%r15), %xmm14, %xmm14 + vaesenc 16(%r15), %xmm15, %xmm15 + vpxor %xmm2, %xmm1, %xmm1 + vpxor %xmm3, %xmm1, %xmm1 + vmovdqa 96(%rsp), %xmm7 + vmovdqu -112(%rdx), %xmm0 + vpshufd $0x4e, %xmm7, %xmm4 + vpshufb L_avx1_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0 + vaesenc 32(%r15), %xmm8, %xmm8 + vpxor %xmm7, %xmm4, %xmm4 + vpshufd $0x4e, %xmm0, %xmm5 + vpxor %xmm0, %xmm5, %xmm5 + vpclmulqdq $0x11, %xmm7, %xmm0, %xmm6 + vaesenc 32(%r15), %xmm9, %xmm9 + vaesenc 32(%r15), %xmm10, %xmm10 + vpclmulqdq $0x00, %xmm7, %xmm0, %xmm7 + vaesenc 32(%r15), %xmm11, %xmm11 + vaesenc 32(%r15), %xmm12, %xmm12 + vpclmulqdq $0x00, %xmm5, %xmm4, %xmm4 + vaesenc 32(%r15), %xmm13, %xmm13 + vaesenc 32(%r15), %xmm14, %xmm14 + vaesenc 32(%r15), %xmm15, %xmm15 + vpxor %xmm7, %xmm1, %xmm1 + vpxor %xmm7, %xmm2, %xmm2 + vpxor %xmm6, %xmm1, %xmm1 + vpxor %xmm6, %xmm3, %xmm3 + vpxor %xmm4, %xmm1, %xmm1 + vmovdqa 80(%rsp), %xmm7 + vmovdqu -96(%rdx), %xmm0 + vpshufd $0x4e, %xmm7, %xmm4 + vpshufb L_avx1_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0 + vaesenc 48(%r15), %xmm8, %xmm8 + vpxor %xmm7, %xmm4, %xmm4 + vpshufd $0x4e, %xmm0, %xmm5 + vpxor %xmm0, %xmm5, %xmm5 + vpclmulqdq $0x11, %xmm7, %xmm0, %xmm6 + vaesenc 48(%r15), %xmm9, %xmm9 + vaesenc 48(%r15), %xmm10, %xmm10 + vpclmulqdq $0x00, %xmm7, %xmm0, %xmm7 + vaesenc 48(%r15), %xmm11, %xmm11 + vaesenc 48(%r15), %xmm12, %xmm12 + vpclmulqdq $0x00, %xmm5, %xmm4, %xmm4 + vaesenc 48(%r15), %xmm13, %xmm13 + vaesenc 48(%r15), %xmm14, %xmm14 + vaesenc 48(%r15), %xmm15, %xmm15 + vpxor %xmm7, %xmm1, %xmm1 + vpxor %xmm7, %xmm2, %xmm2 + vpxor %xmm6, %xmm1, %xmm1 + vpxor %xmm6, %xmm3, %xmm3 + vpxor %xmm4, %xmm1, %xmm1 + vmovdqa 64(%rsp), %xmm7 + vmovdqu -80(%rdx), %xmm0 + vpshufd $0x4e, %xmm7, %xmm4 + vpshufb L_avx1_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0 + vaesenc 64(%r15), %xmm8, %xmm8 + vpxor %xmm7, %xmm4, %xmm4 + vpshufd $0x4e, %xmm0, %xmm5 + vpxor %xmm0, %xmm5, %xmm5 + vpclmulqdq $0x11, %xmm7, %xmm0, %xmm6 + vaesenc 64(%r15), %xmm9, %xmm9 + vaesenc 64(%r15), %xmm10, %xmm10 + vpclmulqdq $0x00, %xmm7, %xmm0, %xmm7 + vaesenc 64(%r15), %xmm11, %xmm11 + vaesenc 64(%r15), %xmm12, %xmm12 + vpclmulqdq $0x00, %xmm5, %xmm4, %xmm4 + vaesenc 64(%r15), %xmm13, %xmm13 + vaesenc 64(%r15), %xmm14, %xmm14 + vaesenc 64(%r15), %xmm15, %xmm15 + vpxor %xmm7, %xmm1, %xmm1 + vpxor %xmm7, %xmm2, %xmm2 + vpxor %xmm6, %xmm1, %xmm1 + vpxor %xmm6, %xmm3, %xmm3 + vpxor %xmm4, %xmm1, %xmm1 + vmovdqa 48(%rsp), %xmm7 + vmovdqu -64(%rdx), %xmm0 + vpshufd $0x4e, %xmm7, %xmm4 + vpshufb L_avx1_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0 + vaesenc 80(%r15), %xmm8, %xmm8 + vpxor %xmm7, %xmm4, %xmm4 + vpshufd $0x4e, %xmm0, %xmm5 + vpxor %xmm0, %xmm5, %xmm5 + vpclmulqdq $0x11, %xmm7, %xmm0, %xmm6 + vaesenc 80(%r15), %xmm9, %xmm9 + vaesenc 80(%r15), %xmm10, %xmm10 + vpclmulqdq $0x00, %xmm7, %xmm0, %xmm7 + vaesenc 80(%r15), %xmm11, %xmm11 + vaesenc 80(%r15), %xmm12, %xmm12 + vpclmulqdq $0x00, %xmm5, %xmm4, %xmm4 + vaesenc 80(%r15), %xmm13, %xmm13 + vaesenc 80(%r15), %xmm14, %xmm14 + vaesenc 80(%r15), %xmm15, %xmm15 + vpxor %xmm7, %xmm1, %xmm1 + vpxor %xmm7, %xmm2, %xmm2 + vpxor %xmm6, %xmm1, %xmm1 + vpxor %xmm6, %xmm3, %xmm3 + vpxor %xmm4, %xmm1, %xmm1 + vmovdqa 32(%rsp), %xmm7 + vmovdqu -48(%rdx), %xmm0 + vpshufd $0x4e, %xmm7, %xmm4 + vpshufb L_avx1_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0 + vaesenc 96(%r15), %xmm8, %xmm8 + vpxor %xmm7, %xmm4, %xmm4 + vpshufd $0x4e, %xmm0, %xmm5 + vpxor %xmm0, %xmm5, %xmm5 + vpclmulqdq $0x11, %xmm7, %xmm0, %xmm6 + vaesenc 96(%r15), %xmm9, %xmm9 + vaesenc 96(%r15), %xmm10, %xmm10 + vpclmulqdq $0x00, %xmm7, %xmm0, %xmm7 + vaesenc 96(%r15), %xmm11, %xmm11 + vaesenc 96(%r15), %xmm12, %xmm12 + vpclmulqdq $0x00, %xmm5, %xmm4, %xmm4 + vaesenc 96(%r15), %xmm13, %xmm13 + vaesenc 96(%r15), %xmm14, %xmm14 + vaesenc 96(%r15), %xmm15, %xmm15 + vpxor %xmm7, %xmm1, %xmm1 + vpxor %xmm7, %xmm2, %xmm2 + vpxor %xmm6, %xmm1, %xmm1 + vpxor %xmm6, %xmm3, %xmm3 + vpxor %xmm4, %xmm1, %xmm1 + vmovdqa 16(%rsp), %xmm7 + vmovdqu -32(%rdx), %xmm0 + vpshufd $0x4e, %xmm7, %xmm4 + vpshufb L_avx1_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0 + vaesenc 112(%r15), %xmm8, %xmm8 + vpxor %xmm7, %xmm4, %xmm4 + vpshufd $0x4e, %xmm0, %xmm5 + vpxor %xmm0, %xmm5, %xmm5 + vpclmulqdq $0x11, %xmm7, %xmm0, %xmm6 + vaesenc 112(%r15), %xmm9, %xmm9 + vaesenc 112(%r15), %xmm10, %xmm10 + vpclmulqdq $0x00, %xmm7, %xmm0, %xmm7 + vaesenc 112(%r15), %xmm11, %xmm11 + vaesenc 112(%r15), %xmm12, %xmm12 + vpclmulqdq $0x00, %xmm5, %xmm4, %xmm4 + vaesenc 112(%r15), %xmm13, %xmm13 + vaesenc 112(%r15), %xmm14, %xmm14 + vaesenc 112(%r15), %xmm15, %xmm15 + vpxor %xmm7, %xmm1, %xmm1 + vpxor %xmm7, %xmm2, %xmm2 + vpxor %xmm6, %xmm1, %xmm1 + vpxor %xmm6, %xmm3, %xmm3 + vpxor %xmm4, %xmm1, %xmm1 + vmovdqa (%rsp), %xmm7 + vmovdqu -16(%rdx), %xmm0 + vpshufd $0x4e, %xmm7, %xmm4 + vpshufb L_avx1_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0 + vaesenc 128(%r15), %xmm8, %xmm8 + vpxor %xmm7, %xmm4, %xmm4 + vpshufd $0x4e, %xmm0, %xmm5 + vpxor %xmm0, %xmm5, %xmm5 + vpclmulqdq $0x11, %xmm7, %xmm0, %xmm6 + vaesenc 128(%r15), %xmm9, %xmm9 + vaesenc 128(%r15), %xmm10, %xmm10 + vpclmulqdq $0x00, %xmm7, %xmm0, %xmm7 + vaesenc 128(%r15), %xmm11, %xmm11 + vaesenc 128(%r15), %xmm12, %xmm12 + vpclmulqdq $0x00, %xmm5, %xmm4, %xmm4 + vaesenc 128(%r15), %xmm13, %xmm13 + vaesenc 128(%r15), %xmm14, %xmm14 + vaesenc 128(%r15), %xmm15, %xmm15 + vpxor %xmm7, %xmm1, %xmm1 + vpxor %xmm7, %xmm2, %xmm2 + vpxor %xmm6, %xmm1, %xmm1 + vpxor %xmm6, %xmm3, %xmm3 + vpxor %xmm4, %xmm1, %xmm1 + vpslldq $8, %xmm1, %xmm5 + vpsrldq $8, %xmm1, %xmm1 + vaesenc 144(%r15), %xmm8, %xmm8 + vpxor %xmm5, %xmm2, %xmm2 + vpxor %xmm1, %xmm3, %xmm3 + vaesenc 144(%r15), %xmm9, %xmm9 + vpslld $31, %xmm2, %xmm7 + vpslld $30, %xmm2, %xmm4 + vpslld $25, %xmm2, %xmm5 + vaesenc 144(%r15), %xmm10, %xmm10 + vpxor %xmm4, %xmm7, %xmm7 + vpxor %xmm5, %xmm7, %xmm7 + vaesenc 144(%r15), %xmm11, %xmm11 + vpsrldq $4, %xmm7, %xmm4 + vpslldq $12, %xmm7, %xmm7 + vaesenc 144(%r15), %xmm12, %xmm12 + vpxor %xmm7, %xmm2, %xmm2 + vpsrld $0x01, %xmm2, %xmm5 + vaesenc 144(%r15), %xmm13, %xmm13 + vpsrld $2, %xmm2, %xmm1 + vpsrld $7, %xmm2, %xmm0 + vaesenc 144(%r15), %xmm14, %xmm14 + vpxor %xmm1, %xmm5, %xmm5 + vpxor %xmm0, %xmm5, %xmm5 + vaesenc 144(%r15), %xmm15, %xmm15 + vpxor %xmm4, %xmm5, %xmm5 + vpxor %xmm5, %xmm2, %xmm2 + vpxor %xmm3, %xmm2, %xmm2 + cmpl $11, %r10d + vmovdqa 160(%r15), %xmm7 + jl L_AES_GCM_encrypt_avx1_aesenc_128_ghash_avx_done + vaesenc %xmm7, %xmm8, %xmm8 + vaesenc %xmm7, %xmm9, %xmm9 + vaesenc %xmm7, %xmm10, %xmm10 + vaesenc %xmm7, %xmm11, %xmm11 + vaesenc %xmm7, %xmm12, %xmm12 + vaesenc %xmm7, %xmm13, %xmm13 + vaesenc %xmm7, %xmm14, %xmm14 + vaesenc %xmm7, %xmm15, %xmm15 + vmovdqa 176(%r15), %xmm7 + vaesenc %xmm7, %xmm8, %xmm8 + vaesenc %xmm7, %xmm9, %xmm9 + vaesenc %xmm7, %xmm10, %xmm10 + vaesenc %xmm7, %xmm11, %xmm11 + vaesenc %xmm7, %xmm12, %xmm12 + vaesenc %xmm7, %xmm13, %xmm13 + vaesenc %xmm7, %xmm14, %xmm14 + vaesenc %xmm7, %xmm15, %xmm15 + cmpl $13, %r10d + vmovdqa 192(%r15), %xmm7 + jl L_AES_GCM_encrypt_avx1_aesenc_128_ghash_avx_done + vaesenc %xmm7, %xmm8, %xmm8 + vaesenc %xmm7, %xmm9, %xmm9 + vaesenc %xmm7, %xmm10, %xmm10 + vaesenc %xmm7, %xmm11, %xmm11 + vaesenc %xmm7, %xmm12, %xmm12 + vaesenc %xmm7, %xmm13, %xmm13 + vaesenc %xmm7, %xmm14, %xmm14 + vaesenc %xmm7, %xmm15, %xmm15 + vmovdqa 208(%r15), %xmm7 + vaesenc %xmm7, %xmm8, %xmm8 + vaesenc %xmm7, %xmm9, %xmm9 + vaesenc %xmm7, %xmm10, %xmm10 + vaesenc %xmm7, %xmm11, %xmm11 + vaesenc %xmm7, %xmm12, %xmm12 + vaesenc %xmm7, %xmm13, %xmm13 + vaesenc %xmm7, %xmm14, %xmm14 + vaesenc %xmm7, %xmm15, %xmm15 + vmovdqa 224(%r15), %xmm7 +L_AES_GCM_encrypt_avx1_aesenc_128_ghash_avx_done: + vaesenclast %xmm7, %xmm8, %xmm8 + vaesenclast %xmm7, %xmm9, %xmm9 + vmovdqu (%rcx), %xmm0 + vmovdqu 16(%rcx), %xmm1 + vpxor %xmm0, %xmm8, %xmm8 + vpxor %xmm1, %xmm9, %xmm9 + vmovdqu %xmm8, (%rdx) + vmovdqu %xmm9, 16(%rdx) + vaesenclast %xmm7, %xmm10, %xmm10 + vaesenclast %xmm7, %xmm11, %xmm11 + vmovdqu 32(%rcx), %xmm0 + vmovdqu 48(%rcx), %xmm1 + vpxor %xmm0, %xmm10, %xmm10 + vpxor %xmm1, %xmm11, %xmm11 + vmovdqu %xmm10, 32(%rdx) + vmovdqu %xmm11, 48(%rdx) + vaesenclast %xmm7, %xmm12, %xmm12 + vaesenclast %xmm7, %xmm13, %xmm13 + vmovdqu 64(%rcx), %xmm0 + vmovdqu 80(%rcx), %xmm1 + vpxor %xmm0, %xmm12, %xmm12 + vpxor %xmm1, %xmm13, %xmm13 + vmovdqu %xmm12, 64(%rdx) + vmovdqu %xmm13, 80(%rdx) + vaesenclast %xmm7, %xmm14, %xmm14 + vaesenclast %xmm7, %xmm15, %xmm15 + vmovdqu 96(%rcx), %xmm0 + vmovdqu 112(%rcx), %xmm1 + vpxor %xmm0, %xmm14, %xmm14 + vpxor %xmm1, %xmm15, %xmm15 + vmovdqu %xmm14, 96(%rdx) + vmovdqu %xmm15, 112(%rdx) + addl $0x80, %ebx + cmpl %r13d, %ebx + jl L_AES_GCM_encrypt_avx1_ghash_128 +L_AES_GCM_encrypt_avx1_end_128: + vmovdqa L_avx1_aes_gcm_bswap_mask(%rip), %xmm4 + vpshufb %xmm4, %xmm8, %xmm8 + vpshufb %xmm4, %xmm9, %xmm9 + vpshufb %xmm4, %xmm10, %xmm10 + vpshufb %xmm4, %xmm11, %xmm11 + vpxor %xmm2, %xmm8, %xmm8 + vpshufb %xmm4, %xmm12, %xmm12 + vpshufb %xmm4, %xmm13, %xmm13 + vpshufb %xmm4, %xmm14, %xmm14 + vpshufb %xmm4, %xmm15, %xmm15 + vmovdqa (%rsp), %xmm7 + vmovdqa 16(%rsp), %xmm5 + # ghash_gfmul_avx + vpshufd $0x4e, %xmm15, %xmm1 + vpshufd $0x4e, %xmm7, %xmm2 + vpclmulqdq $0x11, %xmm15, %xmm7, %xmm3 + vpclmulqdq $0x00, %xmm15, %xmm7, %xmm0 + vpxor %xmm15, %xmm1, %xmm1 + vpxor %xmm7, %xmm2, %xmm2 + vpclmulqdq $0x00, %xmm2, %xmm1, %xmm1 + vpxor %xmm0, %xmm1, %xmm1 + vpxor %xmm3, %xmm1, %xmm1 + vmovdqa %xmm0, %xmm4 + vmovdqa %xmm3, %xmm6 + vpslldq $8, %xmm1, %xmm2 + vpsrldq $8, %xmm1, %xmm1 + vpxor %xmm2, %xmm4, %xmm4 + vpxor %xmm1, %xmm6, %xmm6 + # ghash_gfmul_xor_avx + vpshufd $0x4e, %xmm14, %xmm1 + vpshufd $0x4e, %xmm5, %xmm2 + vpclmulqdq $0x11, %xmm14, %xmm5, %xmm3 + vpclmulqdq $0x00, %xmm14, %xmm5, %xmm0 + vpxor %xmm14, %xmm1, %xmm1 + vpxor %xmm5, %xmm2, %xmm2 + vpclmulqdq $0x00, %xmm2, %xmm1, %xmm1 + vpxor %xmm0, %xmm1, %xmm1 + vpxor %xmm3, %xmm1, %xmm1 + vpxor %xmm0, %xmm4, %xmm4 + vpxor %xmm3, %xmm6, %xmm6 + vpslldq $8, %xmm1, %xmm2 + vpsrldq $8, %xmm1, %xmm1 + vpxor %xmm2, %xmm4, %xmm4 + vpxor %xmm1, %xmm6, %xmm6 + vmovdqa 32(%rsp), %xmm7 + vmovdqa 48(%rsp), %xmm5 + # ghash_gfmul_xor_avx + vpshufd $0x4e, %xmm13, %xmm1 + vpshufd $0x4e, %xmm7, %xmm2 + vpclmulqdq $0x11, %xmm13, %xmm7, %xmm3 + vpclmulqdq $0x00, %xmm13, %xmm7, %xmm0 + vpxor %xmm13, %xmm1, %xmm1 + vpxor %xmm7, %xmm2, %xmm2 + vpclmulqdq $0x00, %xmm2, %xmm1, %xmm1 + vpxor %xmm0, %xmm1, %xmm1 + vpxor %xmm3, %xmm1, %xmm1 + vpxor %xmm0, %xmm4, %xmm4 + vpxor %xmm3, %xmm6, %xmm6 + vpslldq $8, %xmm1, %xmm2 + vpsrldq $8, %xmm1, %xmm1 + vpxor %xmm2, %xmm4, %xmm4 + vpxor %xmm1, %xmm6, %xmm6 + # ghash_gfmul_xor_avx + vpshufd $0x4e, %xmm12, %xmm1 + vpshufd $0x4e, %xmm5, %xmm2 + vpclmulqdq $0x11, %xmm12, %xmm5, %xmm3 + vpclmulqdq $0x00, %xmm12, %xmm5, %xmm0 + vpxor %xmm12, %xmm1, %xmm1 + vpxor %xmm5, %xmm2, %xmm2 + vpclmulqdq $0x00, %xmm2, %xmm1, %xmm1 + vpxor %xmm0, %xmm1, %xmm1 + vpxor %xmm3, %xmm1, %xmm1 + vpxor %xmm0, %xmm4, %xmm4 + vpxor %xmm3, %xmm6, %xmm6 + vpslldq $8, %xmm1, %xmm2 + vpsrldq $8, %xmm1, %xmm1 + vpxor %xmm2, %xmm4, %xmm4 + vpxor %xmm1, %xmm6, %xmm6 + vmovdqa 64(%rsp), %xmm7 + vmovdqa 80(%rsp), %xmm5 + # ghash_gfmul_xor_avx + vpshufd $0x4e, %xmm11, %xmm1 + vpshufd $0x4e, %xmm7, %xmm2 + vpclmulqdq $0x11, %xmm11, %xmm7, %xmm3 + vpclmulqdq $0x00, %xmm11, %xmm7, %xmm0 + vpxor %xmm11, %xmm1, %xmm1 + vpxor %xmm7, %xmm2, %xmm2 + vpclmulqdq $0x00, %xmm2, %xmm1, %xmm1 + vpxor %xmm0, %xmm1, %xmm1 + vpxor %xmm3, %xmm1, %xmm1 + vpxor %xmm0, %xmm4, %xmm4 + vpxor %xmm3, %xmm6, %xmm6 + vpslldq $8, %xmm1, %xmm2 + vpsrldq $8, %xmm1, %xmm1 + vpxor %xmm2, %xmm4, %xmm4 + vpxor %xmm1, %xmm6, %xmm6 + # ghash_gfmul_xor_avx + vpshufd $0x4e, %xmm10, %xmm1 + vpshufd $0x4e, %xmm5, %xmm2 + vpclmulqdq $0x11, %xmm10, %xmm5, %xmm3 + vpclmulqdq $0x00, %xmm10, %xmm5, %xmm0 + vpxor %xmm10, %xmm1, %xmm1 + vpxor %xmm5, %xmm2, %xmm2 + vpclmulqdq $0x00, %xmm2, %xmm1, %xmm1 + vpxor %xmm0, %xmm1, %xmm1 + vpxor %xmm3, %xmm1, %xmm1 + vpxor %xmm0, %xmm4, %xmm4 + vpxor %xmm3, %xmm6, %xmm6 + vpslldq $8, %xmm1, %xmm2 + vpsrldq $8, %xmm1, %xmm1 + vpxor %xmm2, %xmm4, %xmm4 + vpxor %xmm1, %xmm6, %xmm6 + vmovdqa 96(%rsp), %xmm7 + vmovdqa 112(%rsp), %xmm5 + # ghash_gfmul_xor_avx + vpshufd $0x4e, %xmm9, %xmm1 + vpshufd $0x4e, %xmm7, %xmm2 + vpclmulqdq $0x11, %xmm9, %xmm7, %xmm3 + vpclmulqdq $0x00, %xmm9, %xmm7, %xmm0 + vpxor %xmm9, %xmm1, %xmm1 + vpxor %xmm7, %xmm2, %xmm2 + vpclmulqdq $0x00, %xmm2, %xmm1, %xmm1 + vpxor %xmm0, %xmm1, %xmm1 + vpxor %xmm3, %xmm1, %xmm1 + vpxor %xmm0, %xmm4, %xmm4 + vpxor %xmm3, %xmm6, %xmm6 + vpslldq $8, %xmm1, %xmm2 + vpsrldq $8, %xmm1, %xmm1 + vpxor %xmm2, %xmm4, %xmm4 + vpxor %xmm1, %xmm6, %xmm6 + # ghash_gfmul_xor_avx + vpshufd $0x4e, %xmm8, %xmm1 + vpshufd $0x4e, %xmm5, %xmm2 + vpclmulqdq $0x11, %xmm8, %xmm5, %xmm3 + vpclmulqdq $0x00, %xmm8, %xmm5, %xmm0 + vpxor %xmm8, %xmm1, %xmm1 + vpxor %xmm5, %xmm2, %xmm2 + vpclmulqdq $0x00, %xmm2, %xmm1, %xmm1 + vpxor %xmm0, %xmm1, %xmm1 + vpxor %xmm3, %xmm1, %xmm1 + vpxor %xmm0, %xmm4, %xmm4 + vpxor %xmm3, %xmm6, %xmm6 + vpslldq $8, %xmm1, %xmm2 + vpsrldq $8, %xmm1, %xmm1 + vpxor %xmm2, %xmm4, %xmm4 + vpxor %xmm1, %xmm6, %xmm6 + vpslld $31, %xmm4, %xmm0 + vpslld $30, %xmm4, %xmm1 + vpslld $25, %xmm4, %xmm2 + vpxor %xmm1, %xmm0, %xmm0 + vpxor %xmm2, %xmm0, %xmm0 + vmovdqa %xmm0, %xmm1 + vpsrldq $4, %xmm1, %xmm1 + vpslldq $12, %xmm0, %xmm0 + vpxor %xmm0, %xmm4, %xmm4 + vpsrld $0x01, %xmm4, %xmm2 + vpsrld $2, %xmm4, %xmm3 + vpsrld $7, %xmm4, %xmm0 + vpxor %xmm3, %xmm2, %xmm2 + vpxor %xmm0, %xmm2, %xmm2 + vpxor %xmm1, %xmm2, %xmm2 + vpxor %xmm4, %xmm2, %xmm2 + vpxor %xmm2, %xmm6, %xmm6 + vmovdqa (%rsp), %xmm5 +L_AES_GCM_encrypt_avx1_done_128: + movl %r9d, %edx + cmpl %edx, %ebx + jge L_AES_GCM_encrypt_avx1_done_enc + movl %r9d, %r13d + andl $0xfffffff0, %r13d + cmpl %r13d, %ebx + jge L_AES_GCM_encrypt_avx1_last_block_done + vmovdqa 128(%rsp), %xmm9 + vpshufb L_avx1_aes_gcm_bswap_epi64(%rip), %xmm9, %xmm8 + vpaddd L_avx1_aes_gcm_one(%rip), %xmm9, %xmm9 + vmovdqa %xmm9, 128(%rsp) + vpxor (%r15), %xmm8, %xmm8 + vaesenc 16(%r15), %xmm8, %xmm8 + vaesenc 32(%r15), %xmm8, %xmm8 + vaesenc 48(%r15), %xmm8, %xmm8 + vaesenc 64(%r15), %xmm8, %xmm8 + vaesenc 80(%r15), %xmm8, %xmm8 + vaesenc 96(%r15), %xmm8, %xmm8 + vaesenc 112(%r15), %xmm8, %xmm8 + vaesenc 128(%r15), %xmm8, %xmm8 + vaesenc 144(%r15), %xmm8, %xmm8 + cmpl $11, %r10d + vmovdqa 160(%r15), %xmm9 + jl L_AES_GCM_encrypt_avx1_aesenc_block_last + vaesenc %xmm9, %xmm8, %xmm8 + vaesenc 176(%r15), %xmm8, %xmm8 + cmpl $13, %r10d + vmovdqa 192(%r15), %xmm9 + jl L_AES_GCM_encrypt_avx1_aesenc_block_last + vaesenc %xmm9, %xmm8, %xmm8 + vaesenc 208(%r15), %xmm8, %xmm8 + vmovdqa 224(%r15), %xmm9 +L_AES_GCM_encrypt_avx1_aesenc_block_last: + vaesenclast %xmm9, %xmm8, %xmm8 + vmovdqu (%rdi,%rbx,1), %xmm9 + vpxor %xmm9, %xmm8, %xmm8 + vmovdqu %xmm8, (%rsi,%rbx,1) + vpshufb L_avx1_aes_gcm_bswap_mask(%rip), %xmm8, %xmm8 + vpxor %xmm8, %xmm6, %xmm6 + addl $16, %ebx + cmpl %r13d, %ebx + jge L_AES_GCM_encrypt_avx1_last_block_ghash +L_AES_GCM_encrypt_avx1_last_block_start: + vmovdqu (%rdi,%rbx,1), %xmm13 + vmovdqa 128(%rsp), %xmm9 + vpshufb L_avx1_aes_gcm_bswap_epi64(%rip), %xmm9, %xmm8 + vpaddd L_avx1_aes_gcm_one(%rip), %xmm9, %xmm9 + vmovdqa %xmm9, 128(%rsp) + vpxor (%r15), %xmm8, %xmm8 + vpclmulqdq $16, %xmm5, %xmm6, %xmm10 + vaesenc 16(%r15), %xmm8, %xmm8 + vaesenc 32(%r15), %xmm8, %xmm8 + vpclmulqdq $0x01, %xmm5, %xmm6, %xmm11 + vaesenc 48(%r15), %xmm8, %xmm8 + vaesenc 64(%r15), %xmm8, %xmm8 + vpclmulqdq $0x00, %xmm5, %xmm6, %xmm12 + vaesenc 80(%r15), %xmm8, %xmm8 + vpclmulqdq $0x11, %xmm5, %xmm6, %xmm1 + vaesenc 96(%r15), %xmm8, %xmm8 + vpxor %xmm11, %xmm10, %xmm10 + vpslldq $8, %xmm10, %xmm2 + vpsrldq $8, %xmm10, %xmm10 + vaesenc 112(%r15), %xmm8, %xmm8 + vpxor %xmm12, %xmm2, %xmm2 + vpxor %xmm10, %xmm1, %xmm3 + vmovdqa L_avx1_aes_gcm_mod2_128(%rip), %xmm0 + vpclmulqdq $16, %xmm0, %xmm2, %xmm11 + vaesenc 128(%r15), %xmm8, %xmm8 + vpshufd $0x4e, %xmm2, %xmm10 + vpxor %xmm11, %xmm10, %xmm10 + vpclmulqdq $16, %xmm0, %xmm10, %xmm11 + vaesenc 144(%r15), %xmm8, %xmm8 + vpshufd $0x4e, %xmm10, %xmm10 + vpxor %xmm11, %xmm10, %xmm10 + vpxor %xmm3, %xmm10, %xmm6 + cmpl $11, %r10d + vmovdqa 160(%r15), %xmm9 + jl L_AES_GCM_encrypt_avx1_aesenc_gfmul_last + vaesenc %xmm9, %xmm8, %xmm8 + vaesenc 176(%r15), %xmm8, %xmm8 + cmpl $13, %r10d + vmovdqa 192(%r15), %xmm9 + jl L_AES_GCM_encrypt_avx1_aesenc_gfmul_last + vaesenc %xmm9, %xmm8, %xmm8 + vaesenc 208(%r15), %xmm8, %xmm8 + vmovdqa 224(%r15), %xmm9 +L_AES_GCM_encrypt_avx1_aesenc_gfmul_last: + vaesenclast %xmm9, %xmm8, %xmm8 + vmovdqa %xmm13, %xmm0 + vpxor %xmm0, %xmm8, %xmm8 + vmovdqu %xmm8, (%rsi,%rbx,1) + vpshufb L_avx1_aes_gcm_bswap_mask(%rip), %xmm8, %xmm8 + addl $16, %ebx + vpxor %xmm8, %xmm6, %xmm6 + cmpl %r13d, %ebx + jl L_AES_GCM_encrypt_avx1_last_block_start +L_AES_GCM_encrypt_avx1_last_block_ghash: + # ghash_gfmul_red_avx + vpshufd $0x4e, %xmm5, %xmm9 + vpshufd $0x4e, %xmm6, %xmm10 + vpclmulqdq $0x11, %xmm5, %xmm6, %xmm11 + vpclmulqdq $0x00, %xmm5, %xmm6, %xmm8 + vpxor %xmm5, %xmm9, %xmm9 + vpxor %xmm6, %xmm10, %xmm10 + vpclmulqdq $0x00, %xmm10, %xmm9, %xmm9 + vpxor %xmm8, %xmm9, %xmm9 + vpxor %xmm11, %xmm9, %xmm9 + vpslldq $8, %xmm9, %xmm10 + vpsrldq $8, %xmm9, %xmm9 + vpxor %xmm10, %xmm8, %xmm8 + vpxor %xmm9, %xmm11, %xmm6 + vpslld $31, %xmm8, %xmm12 + vpslld $30, %xmm8, %xmm13 + vpslld $25, %xmm8, %xmm14 + vpxor %xmm13, %xmm12, %xmm12 + vpxor %xmm14, %xmm12, %xmm12 + vpsrldq $4, %xmm12, %xmm13 + vpslldq $12, %xmm12, %xmm12 + vpxor %xmm12, %xmm8, %xmm8 + vpsrld $0x01, %xmm8, %xmm14 + vpsrld $2, %xmm8, %xmm10 + vpsrld $7, %xmm8, %xmm9 + vpxor %xmm10, %xmm14, %xmm14 + vpxor %xmm9, %xmm14, %xmm14 + vpxor %xmm13, %xmm14, %xmm14 + vpxor %xmm8, %xmm14, %xmm14 + vpxor %xmm14, %xmm6, %xmm6 +L_AES_GCM_encrypt_avx1_last_block_done: + movl %r9d, %ecx + movl %ecx, %edx + andl $15, %ecx + jz L_AES_GCM_encrypt_avx1_aesenc_last15_enc_avx_done + vmovdqa 128(%rsp), %xmm4 + vpshufb L_avx1_aes_gcm_bswap_epi64(%rip), %xmm4, %xmm4 + vpxor (%r15), %xmm4, %xmm4 + vaesenc 16(%r15), %xmm4, %xmm4 + vaesenc 32(%r15), %xmm4, %xmm4 + vaesenc 48(%r15), %xmm4, %xmm4 + vaesenc 64(%r15), %xmm4, %xmm4 + vaesenc 80(%r15), %xmm4, %xmm4 + vaesenc 96(%r15), %xmm4, %xmm4 + vaesenc 112(%r15), %xmm4, %xmm4 + vaesenc 128(%r15), %xmm4, %xmm4 + vaesenc 144(%r15), %xmm4, %xmm4 + cmpl $11, %r10d + vmovdqa 160(%r15), %xmm9 + jl L_AES_GCM_encrypt_avx1_aesenc_last15_enc_avx_aesenc_avx_last + vaesenc %xmm9, %xmm4, %xmm4 + vaesenc 176(%r15), %xmm4, %xmm4 + cmpl $13, %r10d + vmovdqa 192(%r15), %xmm9 + jl L_AES_GCM_encrypt_avx1_aesenc_last15_enc_avx_aesenc_avx_last + vaesenc %xmm9, %xmm4, %xmm4 + vaesenc 208(%r15), %xmm4, %xmm4 + vmovdqa 224(%r15), %xmm9 +L_AES_GCM_encrypt_avx1_aesenc_last15_enc_avx_aesenc_avx_last: + vaesenclast %xmm9, %xmm4, %xmm4 + subq $16, %rsp + xorl %ecx, %ecx + vmovdqa %xmm4, (%rsp) +L_AES_GCM_encrypt_avx1_aesenc_last15_enc_avx_loop: + movzbl (%rdi,%rbx,1), %r13d + xorb (%rsp,%rcx,1), %r13b + movb %r13b, (%rsi,%rbx,1) + movb %r13b, (%rsp,%rcx,1) + incl %ebx + incl %ecx + cmpl %edx, %ebx + jl L_AES_GCM_encrypt_avx1_aesenc_last15_enc_avx_loop + xorq %r13, %r13 + cmpl $16, %ecx + je L_AES_GCM_encrypt_avx1_aesenc_last15_enc_avx_finish_enc +L_AES_GCM_encrypt_avx1_aesenc_last15_enc_avx_byte_loop: + movb %r13b, (%rsp,%rcx,1) + incl %ecx + cmpl $16, %ecx + jl L_AES_GCM_encrypt_avx1_aesenc_last15_enc_avx_byte_loop +L_AES_GCM_encrypt_avx1_aesenc_last15_enc_avx_finish_enc: + vmovdqa (%rsp), %xmm4 + addq $16, %rsp + vpshufb L_avx1_aes_gcm_bswap_mask(%rip), %xmm4, %xmm4 + vpxor %xmm4, %xmm6, %xmm6 + # ghash_gfmul_red_avx + vpshufd $0x4e, %xmm5, %xmm9 + vpshufd $0x4e, %xmm6, %xmm10 + vpclmulqdq $0x11, %xmm5, %xmm6, %xmm11 + vpclmulqdq $0x00, %xmm5, %xmm6, %xmm8 + vpxor %xmm5, %xmm9, %xmm9 + vpxor %xmm6, %xmm10, %xmm10 + vpclmulqdq $0x00, %xmm10, %xmm9, %xmm9 + vpxor %xmm8, %xmm9, %xmm9 + vpxor %xmm11, %xmm9, %xmm9 + vpslldq $8, %xmm9, %xmm10 + vpsrldq $8, %xmm9, %xmm9 + vpxor %xmm10, %xmm8, %xmm8 + vpxor %xmm9, %xmm11, %xmm6 + vpslld $31, %xmm8, %xmm12 + vpslld $30, %xmm8, %xmm13 + vpslld $25, %xmm8, %xmm14 + vpxor %xmm13, %xmm12, %xmm12 + vpxor %xmm14, %xmm12, %xmm12 + vpsrldq $4, %xmm12, %xmm13 + vpslldq $12, %xmm12, %xmm12 + vpxor %xmm12, %xmm8, %xmm8 + vpsrld $0x01, %xmm8, %xmm14 + vpsrld $2, %xmm8, %xmm10 + vpsrld $7, %xmm8, %xmm9 + vpxor %xmm10, %xmm14, %xmm14 + vpxor %xmm9, %xmm14, %xmm14 + vpxor %xmm13, %xmm14, %xmm14 + vpxor %xmm8, %xmm14, %xmm14 + vpxor %xmm14, %xmm6, %xmm6 +L_AES_GCM_encrypt_avx1_aesenc_last15_enc_avx_done: +L_AES_GCM_encrypt_avx1_done_enc: + movl %r9d, %edx + movl %r11d, %ecx + shlq $3, %rdx + shlq $3, %rcx + vpinsrq $0x00, %rdx, %xmm0, %xmm0 + vpinsrq $0x01, %rcx, %xmm0, %xmm0 + vpxor %xmm0, %xmm6, %xmm6 + # ghash_gfmul_red_avx + vpshufd $0x4e, %xmm5, %xmm9 + vpshufd $0x4e, %xmm6, %xmm10 + vpclmulqdq $0x11, %xmm5, %xmm6, %xmm11 + vpclmulqdq $0x00, %xmm5, %xmm6, %xmm8 + vpxor %xmm5, %xmm9, %xmm9 + vpxor %xmm6, %xmm10, %xmm10 + vpclmulqdq $0x00, %xmm10, %xmm9, %xmm9 + vpxor %xmm8, %xmm9, %xmm9 + vpxor %xmm11, %xmm9, %xmm9 + vpslldq $8, %xmm9, %xmm10 + vpsrldq $8, %xmm9, %xmm9 + vpxor %xmm10, %xmm8, %xmm8 + vpxor %xmm9, %xmm11, %xmm6 + vpslld $31, %xmm8, %xmm12 + vpslld $30, %xmm8, %xmm13 + vpslld $25, %xmm8, %xmm14 + vpxor %xmm13, %xmm12, %xmm12 + vpxor %xmm14, %xmm12, %xmm12 + vpsrldq $4, %xmm12, %xmm13 + vpslldq $12, %xmm12, %xmm12 + vpxor %xmm12, %xmm8, %xmm8 + vpsrld $0x01, %xmm8, %xmm14 + vpsrld $2, %xmm8, %xmm10 + vpsrld $7, %xmm8, %xmm9 + vpxor %xmm10, %xmm14, %xmm14 + vpxor %xmm9, %xmm14, %xmm14 + vpxor %xmm13, %xmm14, %xmm14 + vpxor %xmm8, %xmm14, %xmm14 + vpxor %xmm14, %xmm6, %xmm6 + vpshufb L_avx1_aes_gcm_bswap_mask(%rip), %xmm6, %xmm6 + vpxor 144(%rsp), %xmm6, %xmm0 + cmpl $16, %r14d + je L_AES_GCM_encrypt_avx1_store_tag_16 + xorq %rcx, %rcx + vmovdqa %xmm0, (%rsp) +L_AES_GCM_encrypt_avx1_store_tag_loop: + movzbl (%rsp,%rcx,1), %r13d + movb %r13b, (%r8,%rcx,1) + incl %ecx + cmpl %r14d, %ecx + jne L_AES_GCM_encrypt_avx1_store_tag_loop + jmp L_AES_GCM_encrypt_avx1_store_tag_done +L_AES_GCM_encrypt_avx1_store_tag_16: + vmovdqu %xmm0, (%r8) +L_AES_GCM_encrypt_avx1_store_tag_done: + vzeroupper + addq $0xa0, %rsp + popq %r15 + popq %r14 + popq %rbx + popq %r12 + popq %r13 + repz retq +.size AES_GCM_encrypt_avx1,.-AES_GCM_encrypt_avx1 +.globl AES_GCM_decrypt_avx1 +.type AES_GCM_decrypt_avx1,@function +.align 4 +AES_GCM_decrypt_avx1: + pushq %r13 + pushq %r12 + pushq %rbx + pushq %r14 + pushq %r15 + pushq %rbp + movq %rdx, %r12 + movq %rcx, %rax + movq 56(%rsp), %r11 + movq 64(%rsp), %rbx + movq 72(%rsp), %r14 + movq 80(%rsp), %r15 + movq 88(%rsp), %r10 + movq 96(%rsp), %rbp + subq $0xa8, %rsp + vpxor %xmm4, %xmm4, %xmm4 + vpxor %xmm6, %xmm6, %xmm6 + cmpl $12, %ebx + movl %ebx, %edx + jne L_AES_GCM_decrypt_avx1_iv_not_12 + # # Calculate values when IV is 12 bytes + # Set counter based on IV + movl $0x1000000, %ecx + vpinsrq $0x00, (%rax), %xmm4, %xmm4 + vpinsrd $2, 8(%rax), %xmm4, %xmm4 + vpinsrd $3, %ecx, %xmm4, %xmm4 + # H = Encrypt X(=0) and T = Encrypt counter + vmovdqa (%r15), %xmm5 + vpxor %xmm5, %xmm4, %xmm1 + vmovdqa 16(%r15), %xmm7 + vaesenc %xmm7, %xmm5, %xmm5 + vaesenc %xmm7, %xmm1, %xmm1 + vmovdqa 32(%r15), %xmm7 + vaesenc %xmm7, %xmm5, %xmm5 + vaesenc %xmm7, %xmm1, %xmm1 + vmovdqa 48(%r15), %xmm7 + vaesenc %xmm7, %xmm5, %xmm5 + vaesenc %xmm7, %xmm1, %xmm1 + vmovdqa 64(%r15), %xmm7 + vaesenc %xmm7, %xmm5, %xmm5 + vaesenc %xmm7, %xmm1, %xmm1 + vmovdqa 80(%r15), %xmm7 + vaesenc %xmm7, %xmm5, %xmm5 + vaesenc %xmm7, %xmm1, %xmm1 + vmovdqa 96(%r15), %xmm7 + vaesenc %xmm7, %xmm5, %xmm5 + vaesenc %xmm7, %xmm1, %xmm1 + vmovdqa 112(%r15), %xmm7 + vaesenc %xmm7, %xmm5, %xmm5 + vaesenc %xmm7, %xmm1, %xmm1 + vmovdqa 128(%r15), %xmm7 + vaesenc %xmm7, %xmm5, %xmm5 + vaesenc %xmm7, %xmm1, %xmm1 + vmovdqa 144(%r15), %xmm7 + vaesenc %xmm7, %xmm5, %xmm5 + vaesenc %xmm7, %xmm1, %xmm1 + cmpl $11, %r10d + vmovdqa 160(%r15), %xmm7 + jl L_AES_GCM_decrypt_avx1_calc_iv_12_last + vaesenc %xmm7, %xmm5, %xmm5 + vaesenc %xmm7, %xmm1, %xmm1 + vmovdqa 176(%r15), %xmm7 + vaesenc %xmm7, %xmm5, %xmm5 + vaesenc %xmm7, %xmm1, %xmm1 + cmpl $13, %r10d + vmovdqa 192(%r15), %xmm7 + jl L_AES_GCM_decrypt_avx1_calc_iv_12_last + vaesenc %xmm7, %xmm5, %xmm5 + vaesenc %xmm7, %xmm1, %xmm1 + vmovdqa 208(%r15), %xmm7 + vaesenc %xmm7, %xmm5, %xmm5 + vaesenc %xmm7, %xmm1, %xmm1 + vmovdqa 224(%r15), %xmm7 +L_AES_GCM_decrypt_avx1_calc_iv_12_last: + vaesenclast %xmm7, %xmm5, %xmm5 + vaesenclast %xmm7, %xmm1, %xmm1 + vpshufb L_avx1_aes_gcm_bswap_mask(%rip), %xmm5, %xmm5 + vmovdqa %xmm1, 144(%rsp) + jmp L_AES_GCM_decrypt_avx1_iv_done +L_AES_GCM_decrypt_avx1_iv_not_12: + # Calculate values when IV is not 12 bytes + # H = Encrypt X(=0) + vmovdqa (%r15), %xmm5 + vaesenc 16(%r15), %xmm5, %xmm5 + vaesenc 32(%r15), %xmm5, %xmm5 + vaesenc 48(%r15), %xmm5, %xmm5 + vaesenc 64(%r15), %xmm5, %xmm5 + vaesenc 80(%r15), %xmm5, %xmm5 + vaesenc 96(%r15), %xmm5, %xmm5 + vaesenc 112(%r15), %xmm5, %xmm5 + vaesenc 128(%r15), %xmm5, %xmm5 + vaesenc 144(%r15), %xmm5, %xmm5 + cmpl $11, %r10d + vmovdqa 160(%r15), %xmm9 + jl L_AES_GCM_decrypt_avx1_calc_iv_1_aesenc_avx_last + vaesenc %xmm9, %xmm5, %xmm5 + vaesenc 176(%r15), %xmm5, %xmm5 + cmpl $13, %r10d + vmovdqa 192(%r15), %xmm9 + jl L_AES_GCM_decrypt_avx1_calc_iv_1_aesenc_avx_last + vaesenc %xmm9, %xmm5, %xmm5 + vaesenc 208(%r15), %xmm5, %xmm5 + vmovdqa 224(%r15), %xmm9 +L_AES_GCM_decrypt_avx1_calc_iv_1_aesenc_avx_last: + vaesenclast %xmm9, %xmm5, %xmm5 + vpshufb L_avx1_aes_gcm_bswap_mask(%rip), %xmm5, %xmm5 + # Calc counter + # Initialization vector + cmpl $0x00, %edx + movq $0x00, %rcx + je L_AES_GCM_decrypt_avx1_calc_iv_done + cmpl $16, %edx + jl L_AES_GCM_decrypt_avx1_calc_iv_lt16 + andl $0xfffffff0, %edx +L_AES_GCM_decrypt_avx1_calc_iv_16_loop: + vmovdqu (%rax,%rcx,1), %xmm8 + vpshufb L_avx1_aes_gcm_bswap_mask(%rip), %xmm8, %xmm8 + vpxor %xmm8, %xmm4, %xmm4 + # ghash_gfmul_avx + vpshufd $0x4e, %xmm4, %xmm1 + vpshufd $0x4e, %xmm5, %xmm2 + vpclmulqdq $0x11, %xmm4, %xmm5, %xmm3 + vpclmulqdq $0x00, %xmm4, %xmm5, %xmm0 + vpxor %xmm4, %xmm1, %xmm1 + vpxor %xmm5, %xmm2, %xmm2 + vpclmulqdq $0x00, %xmm2, %xmm1, %xmm1 + vpxor %xmm0, %xmm1, %xmm1 + vpxor %xmm3, %xmm1, %xmm1 + vmovdqa %xmm0, %xmm7 + vmovdqa %xmm3, %xmm4 + vpslldq $8, %xmm1, %xmm2 + vpsrldq $8, %xmm1, %xmm1 + vpxor %xmm2, %xmm7, %xmm7 + vpxor %xmm1, %xmm4, %xmm4 + vpsrld $31, %xmm7, %xmm0 + vpsrld $31, %xmm4, %xmm1 + vpslld $0x01, %xmm7, %xmm7 + vpslld $0x01, %xmm4, %xmm4 + vpsrldq $12, %xmm0, %xmm2 + vpslldq $4, %xmm0, %xmm0 + vpslldq $4, %xmm1, %xmm1 + vpor %xmm2, %xmm4, %xmm4 + vpor %xmm0, %xmm7, %xmm7 + vpor %xmm1, %xmm4, %xmm4 + vpslld $31, %xmm7, %xmm0 + vpslld $30, %xmm7, %xmm1 + vpslld $25, %xmm7, %xmm2 + vpxor %xmm1, %xmm0, %xmm0 + vpxor %xmm2, %xmm0, %xmm0 + vmovdqa %xmm0, %xmm1 + vpsrldq $4, %xmm1, %xmm1 + vpslldq $12, %xmm0, %xmm0 + vpxor %xmm0, %xmm7, %xmm7 + vpsrld $0x01, %xmm7, %xmm2 + vpsrld $2, %xmm7, %xmm3 + vpsrld $7, %xmm7, %xmm0 + vpxor %xmm3, %xmm2, %xmm2 + vpxor %xmm0, %xmm2, %xmm2 + vpxor %xmm1, %xmm2, %xmm2 + vpxor %xmm7, %xmm2, %xmm2 + vpxor %xmm2, %xmm4, %xmm4 + addl $16, %ecx + cmpl %edx, %ecx + jl L_AES_GCM_decrypt_avx1_calc_iv_16_loop + movl %ebx, %edx + cmpl %edx, %ecx + je L_AES_GCM_decrypt_avx1_calc_iv_done +L_AES_GCM_decrypt_avx1_calc_iv_lt16: + subq $16, %rsp + vpxor %xmm8, %xmm8, %xmm8 + xorl %ebx, %ebx + vmovdqa %xmm8, (%rsp) +L_AES_GCM_decrypt_avx1_calc_iv_loop: + movzbl (%rax,%rcx,1), %r13d + movb %r13b, (%rsp,%rbx,1) + incl %ecx + incl %ebx + cmpl %edx, %ecx + jl L_AES_GCM_decrypt_avx1_calc_iv_loop + vmovdqa (%rsp), %xmm8 + addq $16, %rsp + vpshufb L_avx1_aes_gcm_bswap_mask(%rip), %xmm8, %xmm8 + vpxor %xmm8, %xmm4, %xmm4 + # ghash_gfmul_avx + vpshufd $0x4e, %xmm4, %xmm1 + vpshufd $0x4e, %xmm5, %xmm2 + vpclmulqdq $0x11, %xmm4, %xmm5, %xmm3 + vpclmulqdq $0x00, %xmm4, %xmm5, %xmm0 + vpxor %xmm4, %xmm1, %xmm1 + vpxor %xmm5, %xmm2, %xmm2 + vpclmulqdq $0x00, %xmm2, %xmm1, %xmm1 + vpxor %xmm0, %xmm1, %xmm1 + vpxor %xmm3, %xmm1, %xmm1 + vmovdqa %xmm0, %xmm7 + vmovdqa %xmm3, %xmm4 + vpslldq $8, %xmm1, %xmm2 + vpsrldq $8, %xmm1, %xmm1 + vpxor %xmm2, %xmm7, %xmm7 + vpxor %xmm1, %xmm4, %xmm4 + vpsrld $31, %xmm7, %xmm0 + vpsrld $31, %xmm4, %xmm1 + vpslld $0x01, %xmm7, %xmm7 + vpslld $0x01, %xmm4, %xmm4 + vpsrldq $12, %xmm0, %xmm2 + vpslldq $4, %xmm0, %xmm0 + vpslldq $4, %xmm1, %xmm1 + vpor %xmm2, %xmm4, %xmm4 + vpor %xmm0, %xmm7, %xmm7 + vpor %xmm1, %xmm4, %xmm4 + vpslld $31, %xmm7, %xmm0 + vpslld $30, %xmm7, %xmm1 + vpslld $25, %xmm7, %xmm2 + vpxor %xmm1, %xmm0, %xmm0 + vpxor %xmm2, %xmm0, %xmm0 + vmovdqa %xmm0, %xmm1 + vpsrldq $4, %xmm1, %xmm1 + vpslldq $12, %xmm0, %xmm0 + vpxor %xmm0, %xmm7, %xmm7 + vpsrld $0x01, %xmm7, %xmm2 + vpsrld $2, %xmm7, %xmm3 + vpsrld $7, %xmm7, %xmm0 + vpxor %xmm3, %xmm2, %xmm2 + vpxor %xmm0, %xmm2, %xmm2 + vpxor %xmm1, %xmm2, %xmm2 + vpxor %xmm7, %xmm2, %xmm2 + vpxor %xmm2, %xmm4, %xmm4 +L_AES_GCM_decrypt_avx1_calc_iv_done: + # T = Encrypt counter + vpxor %xmm0, %xmm0, %xmm0 + shll $3, %edx + vpinsrq $0x00, %rdx, %xmm0, %xmm0 + vpxor %xmm0, %xmm4, %xmm4 + # ghash_gfmul_avx + vpshufd $0x4e, %xmm4, %xmm1 + vpshufd $0x4e, %xmm5, %xmm2 + vpclmulqdq $0x11, %xmm4, %xmm5, %xmm3 + vpclmulqdq $0x00, %xmm4, %xmm5, %xmm0 + vpxor %xmm4, %xmm1, %xmm1 + vpxor %xmm5, %xmm2, %xmm2 + vpclmulqdq $0x00, %xmm2, %xmm1, %xmm1 + vpxor %xmm0, %xmm1, %xmm1 + vpxor %xmm3, %xmm1, %xmm1 + vmovdqa %xmm0, %xmm7 + vmovdqa %xmm3, %xmm4 + vpslldq $8, %xmm1, %xmm2 + vpsrldq $8, %xmm1, %xmm1 + vpxor %xmm2, %xmm7, %xmm7 + vpxor %xmm1, %xmm4, %xmm4 + vpsrld $31, %xmm7, %xmm0 + vpsrld $31, %xmm4, %xmm1 + vpslld $0x01, %xmm7, %xmm7 + vpslld $0x01, %xmm4, %xmm4 + vpsrldq $12, %xmm0, %xmm2 + vpslldq $4, %xmm0, %xmm0 + vpslldq $4, %xmm1, %xmm1 + vpor %xmm2, %xmm4, %xmm4 + vpor %xmm0, %xmm7, %xmm7 + vpor %xmm1, %xmm4, %xmm4 + vpslld $31, %xmm7, %xmm0 + vpslld $30, %xmm7, %xmm1 + vpslld $25, %xmm7, %xmm2 + vpxor %xmm1, %xmm0, %xmm0 + vpxor %xmm2, %xmm0, %xmm0 + vmovdqa %xmm0, %xmm1 + vpsrldq $4, %xmm1, %xmm1 + vpslldq $12, %xmm0, %xmm0 + vpxor %xmm0, %xmm7, %xmm7 + vpsrld $0x01, %xmm7, %xmm2 + vpsrld $2, %xmm7, %xmm3 + vpsrld $7, %xmm7, %xmm0 + vpxor %xmm3, %xmm2, %xmm2 + vpxor %xmm0, %xmm2, %xmm2 + vpxor %xmm1, %xmm2, %xmm2 + vpxor %xmm7, %xmm2, %xmm2 + vpxor %xmm2, %xmm4, %xmm4 + vpshufb L_avx1_aes_gcm_bswap_mask(%rip), %xmm4, %xmm4 + # Encrypt counter + vmovdqa (%r15), %xmm8 + vpxor %xmm4, %xmm8, %xmm8 + vaesenc 16(%r15), %xmm8, %xmm8 + vaesenc 32(%r15), %xmm8, %xmm8 + vaesenc 48(%r15), %xmm8, %xmm8 + vaesenc 64(%r15), %xmm8, %xmm8 + vaesenc 80(%r15), %xmm8, %xmm8 + vaesenc 96(%r15), %xmm8, %xmm8 + vaesenc 112(%r15), %xmm8, %xmm8 + vaesenc 128(%r15), %xmm8, %xmm8 + vaesenc 144(%r15), %xmm8, %xmm8 + cmpl $11, %r10d + vmovdqa 160(%r15), %xmm9 + jl L_AES_GCM_decrypt_avx1_calc_iv_2_aesenc_avx_last + vaesenc %xmm9, %xmm8, %xmm8 + vaesenc 176(%r15), %xmm8, %xmm8 + cmpl $13, %r10d + vmovdqa 192(%r15), %xmm9 + jl L_AES_GCM_decrypt_avx1_calc_iv_2_aesenc_avx_last + vaesenc %xmm9, %xmm8, %xmm8 + vaesenc 208(%r15), %xmm8, %xmm8 + vmovdqa 224(%r15), %xmm9 +L_AES_GCM_decrypt_avx1_calc_iv_2_aesenc_avx_last: + vaesenclast %xmm9, %xmm8, %xmm8 + vmovdqa %xmm8, 144(%rsp) +L_AES_GCM_decrypt_avx1_iv_done: + # Additional authentication data + movl %r11d, %edx + cmpl $0x00, %edx + je L_AES_GCM_decrypt_avx1_calc_aad_done + xorl %ecx, %ecx + cmpl $16, %edx + jl L_AES_GCM_decrypt_avx1_calc_aad_lt16 + andl $0xfffffff0, %edx +L_AES_GCM_decrypt_avx1_calc_aad_16_loop: + vmovdqu (%r12,%rcx,1), %xmm8 + vpshufb L_avx1_aes_gcm_bswap_mask(%rip), %xmm8, %xmm8 + vpxor %xmm8, %xmm6, %xmm6 + # ghash_gfmul_avx + vpshufd $0x4e, %xmm6, %xmm1 + vpshufd $0x4e, %xmm5, %xmm2 + vpclmulqdq $0x11, %xmm6, %xmm5, %xmm3 + vpclmulqdq $0x00, %xmm6, %xmm5, %xmm0 + vpxor %xmm6, %xmm1, %xmm1 + vpxor %xmm5, %xmm2, %xmm2 + vpclmulqdq $0x00, %xmm2, %xmm1, %xmm1 + vpxor %xmm0, %xmm1, %xmm1 + vpxor %xmm3, %xmm1, %xmm1 + vmovdqa %xmm0, %xmm7 + vmovdqa %xmm3, %xmm6 + vpslldq $8, %xmm1, %xmm2 + vpsrldq $8, %xmm1, %xmm1 + vpxor %xmm2, %xmm7, %xmm7 + vpxor %xmm1, %xmm6, %xmm6 + vpsrld $31, %xmm7, %xmm0 + vpsrld $31, %xmm6, %xmm1 + vpslld $0x01, %xmm7, %xmm7 + vpslld $0x01, %xmm6, %xmm6 + vpsrldq $12, %xmm0, %xmm2 + vpslldq $4, %xmm0, %xmm0 + vpslldq $4, %xmm1, %xmm1 + vpor %xmm2, %xmm6, %xmm6 + vpor %xmm0, %xmm7, %xmm7 + vpor %xmm1, %xmm6, %xmm6 + vpslld $31, %xmm7, %xmm0 + vpslld $30, %xmm7, %xmm1 + vpslld $25, %xmm7, %xmm2 + vpxor %xmm1, %xmm0, %xmm0 + vpxor %xmm2, %xmm0, %xmm0 + vmovdqa %xmm0, %xmm1 + vpsrldq $4, %xmm1, %xmm1 + vpslldq $12, %xmm0, %xmm0 + vpxor %xmm0, %xmm7, %xmm7 + vpsrld $0x01, %xmm7, %xmm2 + vpsrld $2, %xmm7, %xmm3 + vpsrld $7, %xmm7, %xmm0 + vpxor %xmm3, %xmm2, %xmm2 + vpxor %xmm0, %xmm2, %xmm2 + vpxor %xmm1, %xmm2, %xmm2 + vpxor %xmm7, %xmm2, %xmm2 + vpxor %xmm2, %xmm6, %xmm6 + addl $16, %ecx + cmpl %edx, %ecx + jl L_AES_GCM_decrypt_avx1_calc_aad_16_loop + movl %r11d, %edx + cmpl %edx, %ecx + je L_AES_GCM_decrypt_avx1_calc_aad_done +L_AES_GCM_decrypt_avx1_calc_aad_lt16: + subq $16, %rsp + vpxor %xmm8, %xmm8, %xmm8 + xorl %ebx, %ebx + vmovdqa %xmm8, (%rsp) +L_AES_GCM_decrypt_avx1_calc_aad_loop: + movzbl (%r12,%rcx,1), %r13d + movb %r13b, (%rsp,%rbx,1) + incl %ecx + incl %ebx + cmpl %edx, %ecx + jl L_AES_GCM_decrypt_avx1_calc_aad_loop + vmovdqa (%rsp), %xmm8 + addq $16, %rsp + vpshufb L_avx1_aes_gcm_bswap_mask(%rip), %xmm8, %xmm8 + vpxor %xmm8, %xmm6, %xmm6 + # ghash_gfmul_avx + vpshufd $0x4e, %xmm6, %xmm1 + vpshufd $0x4e, %xmm5, %xmm2 + vpclmulqdq $0x11, %xmm6, %xmm5, %xmm3 + vpclmulqdq $0x00, %xmm6, %xmm5, %xmm0 + vpxor %xmm6, %xmm1, %xmm1 + vpxor %xmm5, %xmm2, %xmm2 + vpclmulqdq $0x00, %xmm2, %xmm1, %xmm1 + vpxor %xmm0, %xmm1, %xmm1 + vpxor %xmm3, %xmm1, %xmm1 + vmovdqa %xmm0, %xmm7 + vmovdqa %xmm3, %xmm6 + vpslldq $8, %xmm1, %xmm2 + vpsrldq $8, %xmm1, %xmm1 + vpxor %xmm2, %xmm7, %xmm7 + vpxor %xmm1, %xmm6, %xmm6 + vpsrld $31, %xmm7, %xmm0 + vpsrld $31, %xmm6, %xmm1 + vpslld $0x01, %xmm7, %xmm7 + vpslld $0x01, %xmm6, %xmm6 + vpsrldq $12, %xmm0, %xmm2 + vpslldq $4, %xmm0, %xmm0 + vpslldq $4, %xmm1, %xmm1 + vpor %xmm2, %xmm6, %xmm6 + vpor %xmm0, %xmm7, %xmm7 + vpor %xmm1, %xmm6, %xmm6 + vpslld $31, %xmm7, %xmm0 + vpslld $30, %xmm7, %xmm1 + vpslld $25, %xmm7, %xmm2 + vpxor %xmm1, %xmm0, %xmm0 + vpxor %xmm2, %xmm0, %xmm0 + vmovdqa %xmm0, %xmm1 + vpsrldq $4, %xmm1, %xmm1 + vpslldq $12, %xmm0, %xmm0 + vpxor %xmm0, %xmm7, %xmm7 + vpsrld $0x01, %xmm7, %xmm2 + vpsrld $2, %xmm7, %xmm3 + vpsrld $7, %xmm7, %xmm0 + vpxor %xmm3, %xmm2, %xmm2 + vpxor %xmm0, %xmm2, %xmm2 + vpxor %xmm1, %xmm2, %xmm2 + vpxor %xmm7, %xmm2, %xmm2 + vpxor %xmm2, %xmm6, %xmm6 +L_AES_GCM_decrypt_avx1_calc_aad_done: + # Calculate counter and H + vpsrlq $63, %xmm5, %xmm9 + vpsllq $0x01, %xmm5, %xmm8 + vpslldq $8, %xmm9, %xmm9 + vpor %xmm9, %xmm8, %xmm8 + vpshufd $0xff, %xmm5, %xmm5 + vpsrad $31, %xmm5, %xmm5 + vpshufb L_avx1_aes_gcm_bswap_epi64(%rip), %xmm4, %xmm4 + vpand L_avx1_aes_gcm_mod2_128(%rip), %xmm5, %xmm5 + vpaddd L_avx1_aes_gcm_one(%rip), %xmm4, %xmm4 + vpxor %xmm8, %xmm5, %xmm5 + vmovdqa %xmm4, 128(%rsp) + xorl %ebx, %ebx + cmpl $0x80, %r9d + movl %r9d, %r13d + jl L_AES_GCM_decrypt_avx1_done_128 + andl $0xffffff80, %r13d + vmovdqa %xmm6, %xmm2 + # H ^ 1 + vmovdqa %xmm5, (%rsp) + # H ^ 2 + vpclmulqdq $0x00, %xmm5, %xmm5, %xmm8 + vpclmulqdq $0x11, %xmm5, %xmm5, %xmm0 + vpslld $31, %xmm8, %xmm12 + vpslld $30, %xmm8, %xmm13 + vpslld $25, %xmm8, %xmm14 + vpxor %xmm13, %xmm12, %xmm12 + vpxor %xmm14, %xmm12, %xmm12 + vpsrldq $4, %xmm12, %xmm13 + vpslldq $12, %xmm12, %xmm12 + vpxor %xmm12, %xmm8, %xmm8 + vpsrld $0x01, %xmm8, %xmm14 + vpsrld $2, %xmm8, %xmm10 + vpsrld $7, %xmm8, %xmm9 + vpxor %xmm10, %xmm14, %xmm14 + vpxor %xmm9, %xmm14, %xmm14 + vpxor %xmm13, %xmm14, %xmm14 + vpxor %xmm8, %xmm14, %xmm14 + vpxor %xmm14, %xmm0, %xmm0 + vmovdqa %xmm0, 16(%rsp) + # H ^ 3 + # ghash_gfmul_red_avx + vpshufd $0x4e, %xmm5, %xmm9 + vpshufd $0x4e, %xmm0, %xmm10 + vpclmulqdq $0x11, %xmm5, %xmm0, %xmm11 + vpclmulqdq $0x00, %xmm5, %xmm0, %xmm8 + vpxor %xmm5, %xmm9, %xmm9 + vpxor %xmm0, %xmm10, %xmm10 + vpclmulqdq $0x00, %xmm10, %xmm9, %xmm9 + vpxor %xmm8, %xmm9, %xmm9 + vpxor %xmm11, %xmm9, %xmm9 + vpslldq $8, %xmm9, %xmm10 + vpsrldq $8, %xmm9, %xmm9 + vpxor %xmm10, %xmm8, %xmm8 + vpxor %xmm9, %xmm11, %xmm1 + vpslld $31, %xmm8, %xmm12 + vpslld $30, %xmm8, %xmm13 + vpslld $25, %xmm8, %xmm14 + vpxor %xmm13, %xmm12, %xmm12 + vpxor %xmm14, %xmm12, %xmm12 + vpsrldq $4, %xmm12, %xmm13 + vpslldq $12, %xmm12, %xmm12 + vpxor %xmm12, %xmm8, %xmm8 + vpsrld $0x01, %xmm8, %xmm14 + vpsrld $2, %xmm8, %xmm10 + vpsrld $7, %xmm8, %xmm9 + vpxor %xmm10, %xmm14, %xmm14 + vpxor %xmm9, %xmm14, %xmm14 + vpxor %xmm13, %xmm14, %xmm14 + vpxor %xmm8, %xmm14, %xmm14 + vpxor %xmm14, %xmm1, %xmm1 + vmovdqa %xmm1, 32(%rsp) + # H ^ 4 + vpclmulqdq $0x00, %xmm0, %xmm0, %xmm8 + vpclmulqdq $0x11, %xmm0, %xmm0, %xmm3 + vpslld $31, %xmm8, %xmm12 + vpslld $30, %xmm8, %xmm13 + vpslld $25, %xmm8, %xmm14 + vpxor %xmm13, %xmm12, %xmm12 + vpxor %xmm14, %xmm12, %xmm12 + vpsrldq $4, %xmm12, %xmm13 + vpslldq $12, %xmm12, %xmm12 + vpxor %xmm12, %xmm8, %xmm8 + vpsrld $0x01, %xmm8, %xmm14 + vpsrld $2, %xmm8, %xmm10 + vpsrld $7, %xmm8, %xmm9 + vpxor %xmm10, %xmm14, %xmm14 + vpxor %xmm9, %xmm14, %xmm14 + vpxor %xmm13, %xmm14, %xmm14 + vpxor %xmm8, %xmm14, %xmm14 + vpxor %xmm14, %xmm3, %xmm3 + vmovdqa %xmm3, 48(%rsp) + # H ^ 5 + # ghash_gfmul_red_avx + vpshufd $0x4e, %xmm0, %xmm9 + vpshufd $0x4e, %xmm1, %xmm10 + vpclmulqdq $0x11, %xmm0, %xmm1, %xmm11 + vpclmulqdq $0x00, %xmm0, %xmm1, %xmm8 + vpxor %xmm0, %xmm9, %xmm9 + vpxor %xmm1, %xmm10, %xmm10 + vpclmulqdq $0x00, %xmm10, %xmm9, %xmm9 + vpxor %xmm8, %xmm9, %xmm9 + vpxor %xmm11, %xmm9, %xmm9 + vpslldq $8, %xmm9, %xmm10 + vpsrldq $8, %xmm9, %xmm9 + vpxor %xmm10, %xmm8, %xmm8 + vpxor %xmm9, %xmm11, %xmm7 + vpslld $31, %xmm8, %xmm12 + vpslld $30, %xmm8, %xmm13 + vpslld $25, %xmm8, %xmm14 + vpxor %xmm13, %xmm12, %xmm12 + vpxor %xmm14, %xmm12, %xmm12 + vpsrldq $4, %xmm12, %xmm13 + vpslldq $12, %xmm12, %xmm12 + vpxor %xmm12, %xmm8, %xmm8 + vpsrld $0x01, %xmm8, %xmm14 + vpsrld $2, %xmm8, %xmm10 + vpsrld $7, %xmm8, %xmm9 + vpxor %xmm10, %xmm14, %xmm14 + vpxor %xmm9, %xmm14, %xmm14 + vpxor %xmm13, %xmm14, %xmm14 + vpxor %xmm8, %xmm14, %xmm14 + vpxor %xmm14, %xmm7, %xmm7 + vmovdqa %xmm7, 64(%rsp) + # H ^ 6 + vpclmulqdq $0x00, %xmm1, %xmm1, %xmm8 + vpclmulqdq $0x11, %xmm1, %xmm1, %xmm7 + vpslld $31, %xmm8, %xmm12 + vpslld $30, %xmm8, %xmm13 + vpslld $25, %xmm8, %xmm14 + vpxor %xmm13, %xmm12, %xmm12 + vpxor %xmm14, %xmm12, %xmm12 + vpsrldq $4, %xmm12, %xmm13 + vpslldq $12, %xmm12, %xmm12 + vpxor %xmm12, %xmm8, %xmm8 + vpsrld $0x01, %xmm8, %xmm14 + vpsrld $2, %xmm8, %xmm10 + vpsrld $7, %xmm8, %xmm9 + vpxor %xmm10, %xmm14, %xmm14 + vpxor %xmm9, %xmm14, %xmm14 + vpxor %xmm13, %xmm14, %xmm14 + vpxor %xmm8, %xmm14, %xmm14 + vpxor %xmm14, %xmm7, %xmm7 + vmovdqa %xmm7, 80(%rsp) + # H ^ 7 + # ghash_gfmul_red_avx + vpshufd $0x4e, %xmm1, %xmm9 + vpshufd $0x4e, %xmm3, %xmm10 + vpclmulqdq $0x11, %xmm1, %xmm3, %xmm11 + vpclmulqdq $0x00, %xmm1, %xmm3, %xmm8 + vpxor %xmm1, %xmm9, %xmm9 + vpxor %xmm3, %xmm10, %xmm10 + vpclmulqdq $0x00, %xmm10, %xmm9, %xmm9 + vpxor %xmm8, %xmm9, %xmm9 + vpxor %xmm11, %xmm9, %xmm9 + vpslldq $8, %xmm9, %xmm10 + vpsrldq $8, %xmm9, %xmm9 + vpxor %xmm10, %xmm8, %xmm8 + vpxor %xmm9, %xmm11, %xmm7 + vpslld $31, %xmm8, %xmm12 + vpslld $30, %xmm8, %xmm13 + vpslld $25, %xmm8, %xmm14 + vpxor %xmm13, %xmm12, %xmm12 + vpxor %xmm14, %xmm12, %xmm12 + vpsrldq $4, %xmm12, %xmm13 + vpslldq $12, %xmm12, %xmm12 + vpxor %xmm12, %xmm8, %xmm8 + vpsrld $0x01, %xmm8, %xmm14 + vpsrld $2, %xmm8, %xmm10 + vpsrld $7, %xmm8, %xmm9 + vpxor %xmm10, %xmm14, %xmm14 + vpxor %xmm9, %xmm14, %xmm14 + vpxor %xmm13, %xmm14, %xmm14 + vpxor %xmm8, %xmm14, %xmm14 + vpxor %xmm14, %xmm7, %xmm7 + vmovdqa %xmm7, 96(%rsp) + # H ^ 8 + vpclmulqdq $0x00, %xmm3, %xmm3, %xmm8 + vpclmulqdq $0x11, %xmm3, %xmm3, %xmm7 + vpslld $31, %xmm8, %xmm12 + vpslld $30, %xmm8, %xmm13 + vpslld $25, %xmm8, %xmm14 + vpxor %xmm13, %xmm12, %xmm12 + vpxor %xmm14, %xmm12, %xmm12 + vpsrldq $4, %xmm12, %xmm13 + vpslldq $12, %xmm12, %xmm12 + vpxor %xmm12, %xmm8, %xmm8 + vpsrld $0x01, %xmm8, %xmm14 + vpsrld $2, %xmm8, %xmm10 + vpsrld $7, %xmm8, %xmm9 + vpxor %xmm10, %xmm14, %xmm14 + vpxor %xmm9, %xmm14, %xmm14 + vpxor %xmm13, %xmm14, %xmm14 + vpxor %xmm8, %xmm14, %xmm14 + vpxor %xmm14, %xmm7, %xmm7 + vmovdqa %xmm7, 112(%rsp) +L_AES_GCM_decrypt_avx1_ghash_128: + leaq (%rdi,%rbx,1), %rcx + leaq (%rsi,%rbx,1), %rdx + vmovdqa 128(%rsp), %xmm0 + vmovdqa L_avx1_aes_gcm_bswap_epi64(%rip), %xmm1 + vpshufb %xmm1, %xmm0, %xmm8 + vpaddd L_avx1_aes_gcm_one(%rip), %xmm0, %xmm9 + vpshufb %xmm1, %xmm9, %xmm9 + vpaddd L_avx1_aes_gcm_two(%rip), %xmm0, %xmm10 + vpshufb %xmm1, %xmm10, %xmm10 + vpaddd L_avx1_aes_gcm_three(%rip), %xmm0, %xmm11 + vpshufb %xmm1, %xmm11, %xmm11 + vpaddd L_avx1_aes_gcm_four(%rip), %xmm0, %xmm12 + vpshufb %xmm1, %xmm12, %xmm12 + vpaddd L_avx1_aes_gcm_five(%rip), %xmm0, %xmm13 + vpshufb %xmm1, %xmm13, %xmm13 + vpaddd L_avx1_aes_gcm_six(%rip), %xmm0, %xmm14 + vpshufb %xmm1, %xmm14, %xmm14 + vpaddd L_avx1_aes_gcm_seven(%rip), %xmm0, %xmm15 + vpshufb %xmm1, %xmm15, %xmm15 + vpaddd L_avx1_aes_gcm_eight(%rip), %xmm0, %xmm0 + vmovdqa (%r15), %xmm7 + vmovdqa %xmm0, 128(%rsp) + vpxor %xmm7, %xmm8, %xmm8 + vpxor %xmm7, %xmm9, %xmm9 + vpxor %xmm7, %xmm10, %xmm10 + vpxor %xmm7, %xmm11, %xmm11 + vpxor %xmm7, %xmm12, %xmm12 + vpxor %xmm7, %xmm13, %xmm13 + vpxor %xmm7, %xmm14, %xmm14 + vpxor %xmm7, %xmm15, %xmm15 + vmovdqa 112(%rsp), %xmm7 + vmovdqu (%rcx), %xmm0 + vaesenc 16(%r15), %xmm8, %xmm8 + vpshufb L_avx1_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0 + vpxor %xmm2, %xmm0, %xmm0 + vpshufd $0x4e, %xmm7, %xmm1 + vpshufd $0x4e, %xmm0, %xmm5 + vpxor %xmm7, %xmm1, %xmm1 + vpxor %xmm0, %xmm5, %xmm5 + vpclmulqdq $0x11, %xmm7, %xmm0, %xmm3 + vaesenc 16(%r15), %xmm9, %xmm9 + vaesenc 16(%r15), %xmm10, %xmm10 + vpclmulqdq $0x00, %xmm7, %xmm0, %xmm2 + vaesenc 16(%r15), %xmm11, %xmm11 + vaesenc 16(%r15), %xmm12, %xmm12 + vpclmulqdq $0x00, %xmm5, %xmm1, %xmm1 + vaesenc 16(%r15), %xmm13, %xmm13 + vaesenc 16(%r15), %xmm14, %xmm14 + vaesenc 16(%r15), %xmm15, %xmm15 + vpxor %xmm2, %xmm1, %xmm1 + vpxor %xmm3, %xmm1, %xmm1 + vmovdqa 96(%rsp), %xmm7 + vmovdqu 16(%rcx), %xmm0 + vpshufd $0x4e, %xmm7, %xmm4 + vpshufb L_avx1_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0 + vaesenc 32(%r15), %xmm8, %xmm8 + vpxor %xmm7, %xmm4, %xmm4 + vpshufd $0x4e, %xmm0, %xmm5 + vpxor %xmm0, %xmm5, %xmm5 + vpclmulqdq $0x11, %xmm7, %xmm0, %xmm6 + vaesenc 32(%r15), %xmm9, %xmm9 + vaesenc 32(%r15), %xmm10, %xmm10 + vpclmulqdq $0x00, %xmm7, %xmm0, %xmm7 + vaesenc 32(%r15), %xmm11, %xmm11 + vaesenc 32(%r15), %xmm12, %xmm12 + vpclmulqdq $0x00, %xmm5, %xmm4, %xmm4 + vaesenc 32(%r15), %xmm13, %xmm13 + vaesenc 32(%r15), %xmm14, %xmm14 + vaesenc 32(%r15), %xmm15, %xmm15 + vpxor %xmm7, %xmm1, %xmm1 + vpxor %xmm7, %xmm2, %xmm2 + vpxor %xmm6, %xmm1, %xmm1 + vpxor %xmm6, %xmm3, %xmm3 + vpxor %xmm4, %xmm1, %xmm1 + vmovdqa 80(%rsp), %xmm7 + vmovdqu 32(%rcx), %xmm0 + vpshufd $0x4e, %xmm7, %xmm4 + vpshufb L_avx1_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0 + vaesenc 48(%r15), %xmm8, %xmm8 + vpxor %xmm7, %xmm4, %xmm4 + vpshufd $0x4e, %xmm0, %xmm5 + vpxor %xmm0, %xmm5, %xmm5 + vpclmulqdq $0x11, %xmm7, %xmm0, %xmm6 + vaesenc 48(%r15), %xmm9, %xmm9 + vaesenc 48(%r15), %xmm10, %xmm10 + vpclmulqdq $0x00, %xmm7, %xmm0, %xmm7 + vaesenc 48(%r15), %xmm11, %xmm11 + vaesenc 48(%r15), %xmm12, %xmm12 + vpclmulqdq $0x00, %xmm5, %xmm4, %xmm4 + vaesenc 48(%r15), %xmm13, %xmm13 + vaesenc 48(%r15), %xmm14, %xmm14 + vaesenc 48(%r15), %xmm15, %xmm15 + vpxor %xmm7, %xmm1, %xmm1 + vpxor %xmm7, %xmm2, %xmm2 + vpxor %xmm6, %xmm1, %xmm1 + vpxor %xmm6, %xmm3, %xmm3 + vpxor %xmm4, %xmm1, %xmm1 + vmovdqa 64(%rsp), %xmm7 + vmovdqu 48(%rcx), %xmm0 + vpshufd $0x4e, %xmm7, %xmm4 + vpshufb L_avx1_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0 + vaesenc 64(%r15), %xmm8, %xmm8 + vpxor %xmm7, %xmm4, %xmm4 + vpshufd $0x4e, %xmm0, %xmm5 + vpxor %xmm0, %xmm5, %xmm5 + vpclmulqdq $0x11, %xmm7, %xmm0, %xmm6 + vaesenc 64(%r15), %xmm9, %xmm9 + vaesenc 64(%r15), %xmm10, %xmm10 + vpclmulqdq $0x00, %xmm7, %xmm0, %xmm7 + vaesenc 64(%r15), %xmm11, %xmm11 + vaesenc 64(%r15), %xmm12, %xmm12 + vpclmulqdq $0x00, %xmm5, %xmm4, %xmm4 + vaesenc 64(%r15), %xmm13, %xmm13 + vaesenc 64(%r15), %xmm14, %xmm14 + vaesenc 64(%r15), %xmm15, %xmm15 + vpxor %xmm7, %xmm1, %xmm1 + vpxor %xmm7, %xmm2, %xmm2 + vpxor %xmm6, %xmm1, %xmm1 + vpxor %xmm6, %xmm3, %xmm3 + vpxor %xmm4, %xmm1, %xmm1 + vmovdqa 48(%rsp), %xmm7 + vmovdqu 64(%rcx), %xmm0 + vpshufd $0x4e, %xmm7, %xmm4 + vpshufb L_avx1_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0 + vaesenc 80(%r15), %xmm8, %xmm8 + vpxor %xmm7, %xmm4, %xmm4 + vpshufd $0x4e, %xmm0, %xmm5 + vpxor %xmm0, %xmm5, %xmm5 + vpclmulqdq $0x11, %xmm7, %xmm0, %xmm6 + vaesenc 80(%r15), %xmm9, %xmm9 + vaesenc 80(%r15), %xmm10, %xmm10 + vpclmulqdq $0x00, %xmm7, %xmm0, %xmm7 + vaesenc 80(%r15), %xmm11, %xmm11 + vaesenc 80(%r15), %xmm12, %xmm12 + vpclmulqdq $0x00, %xmm5, %xmm4, %xmm4 + vaesenc 80(%r15), %xmm13, %xmm13 + vaesenc 80(%r15), %xmm14, %xmm14 + vaesenc 80(%r15), %xmm15, %xmm15 + vpxor %xmm7, %xmm1, %xmm1 + vpxor %xmm7, %xmm2, %xmm2 + vpxor %xmm6, %xmm1, %xmm1 + vpxor %xmm6, %xmm3, %xmm3 + vpxor %xmm4, %xmm1, %xmm1 + vmovdqa 32(%rsp), %xmm7 + vmovdqu 80(%rcx), %xmm0 + vpshufd $0x4e, %xmm7, %xmm4 + vpshufb L_avx1_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0 + vaesenc 96(%r15), %xmm8, %xmm8 + vpxor %xmm7, %xmm4, %xmm4 + vpshufd $0x4e, %xmm0, %xmm5 + vpxor %xmm0, %xmm5, %xmm5 + vpclmulqdq $0x11, %xmm7, %xmm0, %xmm6 + vaesenc 96(%r15), %xmm9, %xmm9 + vaesenc 96(%r15), %xmm10, %xmm10 + vpclmulqdq $0x00, %xmm7, %xmm0, %xmm7 + vaesenc 96(%r15), %xmm11, %xmm11 + vaesenc 96(%r15), %xmm12, %xmm12 + vpclmulqdq $0x00, %xmm5, %xmm4, %xmm4 + vaesenc 96(%r15), %xmm13, %xmm13 + vaesenc 96(%r15), %xmm14, %xmm14 + vaesenc 96(%r15), %xmm15, %xmm15 + vpxor %xmm7, %xmm1, %xmm1 + vpxor %xmm7, %xmm2, %xmm2 + vpxor %xmm6, %xmm1, %xmm1 + vpxor %xmm6, %xmm3, %xmm3 + vpxor %xmm4, %xmm1, %xmm1 + vmovdqa 16(%rsp), %xmm7 + vmovdqu 96(%rcx), %xmm0 + vpshufd $0x4e, %xmm7, %xmm4 + vpshufb L_avx1_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0 + vaesenc 112(%r15), %xmm8, %xmm8 + vpxor %xmm7, %xmm4, %xmm4 + vpshufd $0x4e, %xmm0, %xmm5 + vpxor %xmm0, %xmm5, %xmm5 + vpclmulqdq $0x11, %xmm7, %xmm0, %xmm6 + vaesenc 112(%r15), %xmm9, %xmm9 + vaesenc 112(%r15), %xmm10, %xmm10 + vpclmulqdq $0x00, %xmm7, %xmm0, %xmm7 + vaesenc 112(%r15), %xmm11, %xmm11 + vaesenc 112(%r15), %xmm12, %xmm12 + vpclmulqdq $0x00, %xmm5, %xmm4, %xmm4 + vaesenc 112(%r15), %xmm13, %xmm13 + vaesenc 112(%r15), %xmm14, %xmm14 + vaesenc 112(%r15), %xmm15, %xmm15 + vpxor %xmm7, %xmm1, %xmm1 + vpxor %xmm7, %xmm2, %xmm2 + vpxor %xmm6, %xmm1, %xmm1 + vpxor %xmm6, %xmm3, %xmm3 + vpxor %xmm4, %xmm1, %xmm1 + vmovdqa (%rsp), %xmm7 + vmovdqu 112(%rcx), %xmm0 + vpshufd $0x4e, %xmm7, %xmm4 + vpshufb L_avx1_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0 + vaesenc 128(%r15), %xmm8, %xmm8 + vpxor %xmm7, %xmm4, %xmm4 + vpshufd $0x4e, %xmm0, %xmm5 + vpxor %xmm0, %xmm5, %xmm5 + vpclmulqdq $0x11, %xmm7, %xmm0, %xmm6 + vaesenc 128(%r15), %xmm9, %xmm9 + vaesenc 128(%r15), %xmm10, %xmm10 + vpclmulqdq $0x00, %xmm7, %xmm0, %xmm7 + vaesenc 128(%r15), %xmm11, %xmm11 + vaesenc 128(%r15), %xmm12, %xmm12 + vpclmulqdq $0x00, %xmm5, %xmm4, %xmm4 + vaesenc 128(%r15), %xmm13, %xmm13 + vaesenc 128(%r15), %xmm14, %xmm14 + vaesenc 128(%r15), %xmm15, %xmm15 + vpxor %xmm7, %xmm1, %xmm1 + vpxor %xmm7, %xmm2, %xmm2 + vpxor %xmm6, %xmm1, %xmm1 + vpxor %xmm6, %xmm3, %xmm3 + vpxor %xmm4, %xmm1, %xmm1 + vpslldq $8, %xmm1, %xmm5 + vpsrldq $8, %xmm1, %xmm1 + vaesenc 144(%r15), %xmm8, %xmm8 + vpxor %xmm5, %xmm2, %xmm2 + vpxor %xmm1, %xmm3, %xmm3 + vaesenc 144(%r15), %xmm9, %xmm9 + vpslld $31, %xmm2, %xmm7 + vpslld $30, %xmm2, %xmm4 + vpslld $25, %xmm2, %xmm5 + vaesenc 144(%r15), %xmm10, %xmm10 + vpxor %xmm4, %xmm7, %xmm7 + vpxor %xmm5, %xmm7, %xmm7 + vaesenc 144(%r15), %xmm11, %xmm11 + vpsrldq $4, %xmm7, %xmm4 + vpslldq $12, %xmm7, %xmm7 + vaesenc 144(%r15), %xmm12, %xmm12 + vpxor %xmm7, %xmm2, %xmm2 + vpsrld $0x01, %xmm2, %xmm5 + vaesenc 144(%r15), %xmm13, %xmm13 + vpsrld $2, %xmm2, %xmm1 + vpsrld $7, %xmm2, %xmm0 + vaesenc 144(%r15), %xmm14, %xmm14 + vpxor %xmm1, %xmm5, %xmm5 + vpxor %xmm0, %xmm5, %xmm5 + vaesenc 144(%r15), %xmm15, %xmm15 + vpxor %xmm4, %xmm5, %xmm5 + vpxor %xmm5, %xmm2, %xmm2 + vpxor %xmm3, %xmm2, %xmm2 + cmpl $11, %r10d + vmovdqa 160(%r15), %xmm7 + jl L_AES_GCM_decrypt_avx1_aesenc_128_ghash_avx_done + vaesenc %xmm7, %xmm8, %xmm8 + vaesenc %xmm7, %xmm9, %xmm9 + vaesenc %xmm7, %xmm10, %xmm10 + vaesenc %xmm7, %xmm11, %xmm11 + vaesenc %xmm7, %xmm12, %xmm12 + vaesenc %xmm7, %xmm13, %xmm13 + vaesenc %xmm7, %xmm14, %xmm14 + vaesenc %xmm7, %xmm15, %xmm15 + vmovdqa 176(%r15), %xmm7 + vaesenc %xmm7, %xmm8, %xmm8 + vaesenc %xmm7, %xmm9, %xmm9 + vaesenc %xmm7, %xmm10, %xmm10 + vaesenc %xmm7, %xmm11, %xmm11 + vaesenc %xmm7, %xmm12, %xmm12 + vaesenc %xmm7, %xmm13, %xmm13 + vaesenc %xmm7, %xmm14, %xmm14 + vaesenc %xmm7, %xmm15, %xmm15 + cmpl $13, %r10d + vmovdqa 192(%r15), %xmm7 + jl L_AES_GCM_decrypt_avx1_aesenc_128_ghash_avx_done + vaesenc %xmm7, %xmm8, %xmm8 + vaesenc %xmm7, %xmm9, %xmm9 + vaesenc %xmm7, %xmm10, %xmm10 + vaesenc %xmm7, %xmm11, %xmm11 + vaesenc %xmm7, %xmm12, %xmm12 + vaesenc %xmm7, %xmm13, %xmm13 + vaesenc %xmm7, %xmm14, %xmm14 + vaesenc %xmm7, %xmm15, %xmm15 + vmovdqa 208(%r15), %xmm7 + vaesenc %xmm7, %xmm8, %xmm8 + vaesenc %xmm7, %xmm9, %xmm9 + vaesenc %xmm7, %xmm10, %xmm10 + vaesenc %xmm7, %xmm11, %xmm11 + vaesenc %xmm7, %xmm12, %xmm12 + vaesenc %xmm7, %xmm13, %xmm13 + vaesenc %xmm7, %xmm14, %xmm14 + vaesenc %xmm7, %xmm15, %xmm15 + vmovdqa 224(%r15), %xmm7 +L_AES_GCM_decrypt_avx1_aesenc_128_ghash_avx_done: + vaesenclast %xmm7, %xmm8, %xmm8 + vaesenclast %xmm7, %xmm9, %xmm9 + vmovdqu (%rcx), %xmm0 + vmovdqu 16(%rcx), %xmm1 + vpxor %xmm0, %xmm8, %xmm8 + vpxor %xmm1, %xmm9, %xmm9 + vmovdqu %xmm8, (%rdx) + vmovdqu %xmm9, 16(%rdx) + vaesenclast %xmm7, %xmm10, %xmm10 + vaesenclast %xmm7, %xmm11, %xmm11 + vmovdqu 32(%rcx), %xmm0 + vmovdqu 48(%rcx), %xmm1 + vpxor %xmm0, %xmm10, %xmm10 + vpxor %xmm1, %xmm11, %xmm11 + vmovdqu %xmm10, 32(%rdx) + vmovdqu %xmm11, 48(%rdx) + vaesenclast %xmm7, %xmm12, %xmm12 + vaesenclast %xmm7, %xmm13, %xmm13 + vmovdqu 64(%rcx), %xmm0 + vmovdqu 80(%rcx), %xmm1 + vpxor %xmm0, %xmm12, %xmm12 + vpxor %xmm1, %xmm13, %xmm13 + vmovdqu %xmm12, 64(%rdx) + vmovdqu %xmm13, 80(%rdx) + vaesenclast %xmm7, %xmm14, %xmm14 + vaesenclast %xmm7, %xmm15, %xmm15 + vmovdqu 96(%rcx), %xmm0 + vmovdqu 112(%rcx), %xmm1 + vpxor %xmm0, %xmm14, %xmm14 + vpxor %xmm1, %xmm15, %xmm15 + vmovdqu %xmm14, 96(%rdx) + vmovdqu %xmm15, 112(%rdx) + addl $0x80, %ebx + cmpl %r13d, %ebx + jl L_AES_GCM_decrypt_avx1_ghash_128 + vmovdqa %xmm2, %xmm6 + vmovdqa (%rsp), %xmm5 +L_AES_GCM_decrypt_avx1_done_128: + movl %r9d, %edx + cmpl %edx, %ebx + jge L_AES_GCM_decrypt_avx1_done_dec + movl %r9d, %r13d + andl $0xfffffff0, %r13d + cmpl %r13d, %ebx + jge L_AES_GCM_decrypt_avx1_last_block_done +L_AES_GCM_decrypt_avx1_last_block_start: + vmovdqu (%rdi,%rbx,1), %xmm13 + vmovdqa %xmm5, %xmm0 + vpshufb L_avx1_aes_gcm_bswap_mask(%rip), %xmm13, %xmm1 + vpxor %xmm6, %xmm1, %xmm1 + vmovdqa 128(%rsp), %xmm9 + vpshufb L_avx1_aes_gcm_bswap_epi64(%rip), %xmm9, %xmm8 + vpaddd L_avx1_aes_gcm_one(%rip), %xmm9, %xmm9 + vmovdqa %xmm9, 128(%rsp) + vpxor (%r15), %xmm8, %xmm8 + vpclmulqdq $16, %xmm0, %xmm1, %xmm10 + vaesenc 16(%r15), %xmm8, %xmm8 + vaesenc 32(%r15), %xmm8, %xmm8 + vpclmulqdq $0x01, %xmm0, %xmm1, %xmm11 + vaesenc 48(%r15), %xmm8, %xmm8 + vaesenc 64(%r15), %xmm8, %xmm8 + vpclmulqdq $0x00, %xmm0, %xmm1, %xmm12 + vaesenc 80(%r15), %xmm8, %xmm8 + vpclmulqdq $0x11, %xmm0, %xmm1, %xmm1 + vaesenc 96(%r15), %xmm8, %xmm8 + vpxor %xmm11, %xmm10, %xmm10 + vpslldq $8, %xmm10, %xmm2 + vpsrldq $8, %xmm10, %xmm10 + vaesenc 112(%r15), %xmm8, %xmm8 + vpxor %xmm12, %xmm2, %xmm2 + vpxor %xmm10, %xmm1, %xmm3 + vmovdqa L_avx1_aes_gcm_mod2_128(%rip), %xmm0 + vpclmulqdq $16, %xmm0, %xmm2, %xmm11 + vaesenc 128(%r15), %xmm8, %xmm8 + vpshufd $0x4e, %xmm2, %xmm10 + vpxor %xmm11, %xmm10, %xmm10 + vpclmulqdq $16, %xmm0, %xmm10, %xmm11 + vaesenc 144(%r15), %xmm8, %xmm8 + vpshufd $0x4e, %xmm10, %xmm10 + vpxor %xmm11, %xmm10, %xmm10 + vpxor %xmm3, %xmm10, %xmm6 + cmpl $11, %r10d + vmovdqa 160(%r15), %xmm9 + jl L_AES_GCM_decrypt_avx1_aesenc_gfmul_last + vaesenc %xmm9, %xmm8, %xmm8 + vaesenc 176(%r15), %xmm8, %xmm8 + cmpl $13, %r10d + vmovdqa 192(%r15), %xmm9 + jl L_AES_GCM_decrypt_avx1_aesenc_gfmul_last + vaesenc %xmm9, %xmm8, %xmm8 + vaesenc 208(%r15), %xmm8, %xmm8 + vmovdqa 224(%r15), %xmm9 +L_AES_GCM_decrypt_avx1_aesenc_gfmul_last: + vaesenclast %xmm9, %xmm8, %xmm8 + vmovdqa %xmm13, %xmm0 + vpxor %xmm0, %xmm8, %xmm8 + vmovdqu %xmm8, (%rsi,%rbx,1) + addl $16, %ebx + cmpl %r13d, %ebx + jl L_AES_GCM_decrypt_avx1_last_block_start +L_AES_GCM_decrypt_avx1_last_block_done: + movl %r9d, %ecx + movl %ecx, %edx + andl $15, %ecx + jz L_AES_GCM_decrypt_avx1_aesenc_last15_dec_avx_done + vmovdqa 128(%rsp), %xmm4 + vpshufb L_avx1_aes_gcm_bswap_epi64(%rip), %xmm4, %xmm4 + vpxor (%r15), %xmm4, %xmm4 + vaesenc 16(%r15), %xmm4, %xmm4 + vaesenc 32(%r15), %xmm4, %xmm4 + vaesenc 48(%r15), %xmm4, %xmm4 + vaesenc 64(%r15), %xmm4, %xmm4 + vaesenc 80(%r15), %xmm4, %xmm4 + vaesenc 96(%r15), %xmm4, %xmm4 + vaesenc 112(%r15), %xmm4, %xmm4 + vaesenc 128(%r15), %xmm4, %xmm4 + vaesenc 144(%r15), %xmm4, %xmm4 + cmpl $11, %r10d + vmovdqa 160(%r15), %xmm9 + jl L_AES_GCM_decrypt_avx1_aesenc_last15_dec_avx_aesenc_avx_last + vaesenc %xmm9, %xmm4, %xmm4 + vaesenc 176(%r15), %xmm4, %xmm4 + cmpl $13, %r10d + vmovdqa 192(%r15), %xmm9 + jl L_AES_GCM_decrypt_avx1_aesenc_last15_dec_avx_aesenc_avx_last + vaesenc %xmm9, %xmm4, %xmm4 + vaesenc 208(%r15), %xmm4, %xmm4 + vmovdqa 224(%r15), %xmm9 +L_AES_GCM_decrypt_avx1_aesenc_last15_dec_avx_aesenc_avx_last: + vaesenclast %xmm9, %xmm4, %xmm4 + subq $32, %rsp + xorl %ecx, %ecx + vmovdqa %xmm4, (%rsp) + vpxor %xmm0, %xmm0, %xmm0 + vmovdqa %xmm0, 16(%rsp) +L_AES_GCM_decrypt_avx1_aesenc_last15_dec_avx_loop: + movzbl (%rdi,%rbx,1), %r13d + movb %r13b, 16(%rsp,%rcx,1) + xorb (%rsp,%rcx,1), %r13b + movb %r13b, (%rsi,%rbx,1) + incl %ebx + incl %ecx + cmpl %edx, %ebx + jl L_AES_GCM_decrypt_avx1_aesenc_last15_dec_avx_loop + vmovdqa 16(%rsp), %xmm4 + addq $32, %rsp + vpshufb L_avx1_aes_gcm_bswap_mask(%rip), %xmm4, %xmm4 + vpxor %xmm4, %xmm6, %xmm6 + # ghash_gfmul_red_avx + vpshufd $0x4e, %xmm5, %xmm9 + vpshufd $0x4e, %xmm6, %xmm10 + vpclmulqdq $0x11, %xmm5, %xmm6, %xmm11 + vpclmulqdq $0x00, %xmm5, %xmm6, %xmm8 + vpxor %xmm5, %xmm9, %xmm9 + vpxor %xmm6, %xmm10, %xmm10 + vpclmulqdq $0x00, %xmm10, %xmm9, %xmm9 + vpxor %xmm8, %xmm9, %xmm9 + vpxor %xmm11, %xmm9, %xmm9 + vpslldq $8, %xmm9, %xmm10 + vpsrldq $8, %xmm9, %xmm9 + vpxor %xmm10, %xmm8, %xmm8 + vpxor %xmm9, %xmm11, %xmm6 + vpslld $31, %xmm8, %xmm12 + vpslld $30, %xmm8, %xmm13 + vpslld $25, %xmm8, %xmm14 + vpxor %xmm13, %xmm12, %xmm12 + vpxor %xmm14, %xmm12, %xmm12 + vpsrldq $4, %xmm12, %xmm13 + vpslldq $12, %xmm12, %xmm12 + vpxor %xmm12, %xmm8, %xmm8 + vpsrld $0x01, %xmm8, %xmm14 + vpsrld $2, %xmm8, %xmm10 + vpsrld $7, %xmm8, %xmm9 + vpxor %xmm10, %xmm14, %xmm14 + vpxor %xmm9, %xmm14, %xmm14 + vpxor %xmm13, %xmm14, %xmm14 + vpxor %xmm8, %xmm14, %xmm14 + vpxor %xmm14, %xmm6, %xmm6 +L_AES_GCM_decrypt_avx1_aesenc_last15_dec_avx_done: +L_AES_GCM_decrypt_avx1_done_dec: + movl %r9d, %edx + movl %r11d, %ecx + shlq $3, %rdx + shlq $3, %rcx + vpinsrq $0x00, %rdx, %xmm0, %xmm0 + vpinsrq $0x01, %rcx, %xmm0, %xmm0 + vpxor %xmm0, %xmm6, %xmm6 + # ghash_gfmul_red_avx + vpshufd $0x4e, %xmm5, %xmm9 + vpshufd $0x4e, %xmm6, %xmm10 + vpclmulqdq $0x11, %xmm5, %xmm6, %xmm11 + vpclmulqdq $0x00, %xmm5, %xmm6, %xmm8 + vpxor %xmm5, %xmm9, %xmm9 + vpxor %xmm6, %xmm10, %xmm10 + vpclmulqdq $0x00, %xmm10, %xmm9, %xmm9 + vpxor %xmm8, %xmm9, %xmm9 + vpxor %xmm11, %xmm9, %xmm9 + vpslldq $8, %xmm9, %xmm10 + vpsrldq $8, %xmm9, %xmm9 + vpxor %xmm10, %xmm8, %xmm8 + vpxor %xmm9, %xmm11, %xmm6 + vpslld $31, %xmm8, %xmm12 + vpslld $30, %xmm8, %xmm13 + vpslld $25, %xmm8, %xmm14 + vpxor %xmm13, %xmm12, %xmm12 + vpxor %xmm14, %xmm12, %xmm12 + vpsrldq $4, %xmm12, %xmm13 + vpslldq $12, %xmm12, %xmm12 + vpxor %xmm12, %xmm8, %xmm8 + vpsrld $0x01, %xmm8, %xmm14 + vpsrld $2, %xmm8, %xmm10 + vpsrld $7, %xmm8, %xmm9 + vpxor %xmm10, %xmm14, %xmm14 + vpxor %xmm9, %xmm14, %xmm14 + vpxor %xmm13, %xmm14, %xmm14 + vpxor %xmm8, %xmm14, %xmm14 + vpxor %xmm14, %xmm6, %xmm6 + vpshufb L_avx1_aes_gcm_bswap_mask(%rip), %xmm6, %xmm6 + vpxor 144(%rsp), %xmm6, %xmm0 + cmpl $16, %r14d + je L_AES_GCM_decrypt_avx1_cmp_tag_16 + subq $16, %rsp + xorq %rcx, %rcx + xorq %rbx, %rbx + vmovdqa %xmm0, (%rsp) +L_AES_GCM_decrypt_avx1_cmp_tag_loop: + movzbl (%rsp,%rcx,1), %r13d + xorb (%r8,%rcx,1), %r13b + orb %r13b, %bl + incl %ecx + cmpl %r14d, %ecx + jne L_AES_GCM_decrypt_avx1_cmp_tag_loop + cmpb $0x00, %bl + sete %bl + addq $16, %rsp + xorq %rcx, %rcx + jmp L_AES_GCM_decrypt_avx1_cmp_tag_done +L_AES_GCM_decrypt_avx1_cmp_tag_16: + vmovdqu (%r8), %xmm1 + vpcmpeqb %xmm1, %xmm0, %xmm0 + vpmovmskb %xmm0, %rdx + # %%edx == 0xFFFF then return 1 else => return 0 + xorl %ebx, %ebx + cmpl $0xffff, %edx + sete %bl +L_AES_GCM_decrypt_avx1_cmp_tag_done: + movl %ebx, (%rbp) + vzeroupper + addq $0xa8, %rsp + popq %rbp + popq %r15 + popq %r14 + popq %rbx + popq %r12 + popq %r13 + repz retq +.size AES_GCM_decrypt_avx1,.-AES_GCM_decrypt_avx1 +#endif /* HAVE_INTEL_AVX1 */ +#ifdef HAVE_INTEL_AVX2 +.align 16 +L_avx2_aes_gcm_one: +.quad 0x0, 0x1 +.align 16 +L_avx2_aes_gcm_two: +.quad 0x0, 0x2 +.align 16 +L_avx2_aes_gcm_three: +.quad 0x0, 0x3 +.align 16 +L_avx2_aes_gcm_four: +.quad 0x0, 0x4 +.align 16 +L_avx2_aes_gcm_five: +.quad 0x0, 0x5 +.align 16 +L_avx2_aes_gcm_six: +.quad 0x0, 0x6 +.align 16 +L_avx2_aes_gcm_seven: +.quad 0x0, 0x7 +.align 16 +L_avx2_aes_gcm_eight: +.quad 0x0, 0x8 +.align 16 +L_avx2_aes_gcm_bswap_one: +.quad 0x0, 0x100000000000000 +.align 16 +L_avx2_aes_gcm_bswap_epi64: +.quad 0x1020304050607, 0x8090a0b0c0d0e0f +.align 16 +L_avx2_aes_gcm_bswap_mask: +.quad 0x8090a0b0c0d0e0f, 0x1020304050607 +.align 16 +L_avx2_aes_gcm_mod2_128: +.quad 0x1, 0xc200000000000000 +.globl AES_GCM_encrypt_avx2 +.type AES_GCM_encrypt_avx2,@function +.align 4 +AES_GCM_encrypt_avx2: + pushq %r13 + pushq %r12 + pushq %r15 + pushq %rbx + pushq %r14 + movq %rdx, %r12 + movq %rcx, %rax + movq %r8, %r15 + movq %rsi, %r8 + movq %r9, %r10 + movq 48(%rsp), %r11 + movq 56(%rsp), %rbx + movq 64(%rsp), %r14 + movq 72(%rsp), %rsi + movq 80(%rsp), %r9 + subq $0xa0, %rsp + vpxor %xmm4, %xmm4, %xmm4 + vpxor %xmm6, %xmm6, %xmm6 + movl %ebx, %edx + cmpl $12, %edx + je L_AES_GCM_encrypt_avx2_iv_12 + # Calculate values when IV is not 12 bytes + # H = Encrypt X(=0) + vmovdqa (%rsi), %xmm5 + vaesenc 16(%rsi), %xmm5, %xmm5 + vaesenc 32(%rsi), %xmm5, %xmm5 + vaesenc 48(%rsi), %xmm5, %xmm5 + vaesenc 64(%rsi), %xmm5, %xmm5 + vaesenc 80(%rsi), %xmm5, %xmm5 + vaesenc 96(%rsi), %xmm5, %xmm5 + vaesenc 112(%rsi), %xmm5, %xmm5 + vaesenc 128(%rsi), %xmm5, %xmm5 + vaesenc 144(%rsi), %xmm5, %xmm5 + cmpl $11, %r9d + vmovdqa 160(%rsi), %xmm0 + jl L_AES_GCM_encrypt_avx2_calc_iv_1_aesenc_avx_last + vaesenc %xmm0, %xmm5, %xmm5 + vaesenc 176(%rsi), %xmm5, %xmm5 + cmpl $13, %r9d + vmovdqa 192(%rsi), %xmm0 + jl L_AES_GCM_encrypt_avx2_calc_iv_1_aesenc_avx_last + vaesenc %xmm0, %xmm5, %xmm5 + vaesenc 208(%rsi), %xmm5, %xmm5 + vmovdqa 224(%rsi), %xmm0 +L_AES_GCM_encrypt_avx2_calc_iv_1_aesenc_avx_last: + vaesenclast %xmm0, %xmm5, %xmm5 + vpshufb L_avx2_aes_gcm_bswap_mask(%rip), %xmm5, %xmm5 + # Calc counter + # Initialization vector + cmpl $0x00, %edx + movq $0x00, %rcx + je L_AES_GCM_encrypt_avx2_calc_iv_done + cmpl $16, %edx + jl L_AES_GCM_encrypt_avx2_calc_iv_lt16 + andl $0xfffffff0, %edx +L_AES_GCM_encrypt_avx2_calc_iv_16_loop: + vmovdqu (%rax,%rcx,1), %xmm0 + vpshufb L_avx2_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0 + vpxor %xmm0, %xmm4, %xmm4 + # ghash_gfmul_avx + vpclmulqdq $16, %xmm4, %xmm5, %xmm2 + vpclmulqdq $0x01, %xmm4, %xmm5, %xmm1 + vpclmulqdq $0x00, %xmm4, %xmm5, %xmm0 + vpclmulqdq $0x11, %xmm4, %xmm5, %xmm3 + vpxor %xmm1, %xmm2, %xmm2 + vpslldq $8, %xmm2, %xmm1 + vpsrldq $8, %xmm2, %xmm2 + vpxor %xmm1, %xmm0, %xmm7 + vpxor %xmm2, %xmm3, %xmm4 + # ghash_mid + vpsrld $31, %xmm7, %xmm0 + vpsrld $31, %xmm4, %xmm1 + vpslld $0x01, %xmm7, %xmm7 + vpslld $0x01, %xmm4, %xmm4 + vpsrldq $12, %xmm0, %xmm2 + vpslldq $4, %xmm0, %xmm0 + vpslldq $4, %xmm1, %xmm1 + vpor %xmm2, %xmm4, %xmm4 + vpor %xmm0, %xmm7, %xmm7 + vpor %xmm1, %xmm4, %xmm4 + # ghash_red + vmovdqa L_avx2_aes_gcm_mod2_128(%rip), %xmm2 + vpclmulqdq $16, %xmm2, %xmm7, %xmm0 + vpshufd $0x4e, %xmm7, %xmm1 + vpxor %xmm0, %xmm1, %xmm1 + vpclmulqdq $16, %xmm2, %xmm1, %xmm0 + vpshufd $0x4e, %xmm1, %xmm1 + vpxor %xmm0, %xmm1, %xmm1 + vpxor %xmm1, %xmm4, %xmm4 + addl $16, %ecx + cmpl %edx, %ecx + jl L_AES_GCM_encrypt_avx2_calc_iv_16_loop + movl %ebx, %edx + cmpl %edx, %ecx + je L_AES_GCM_encrypt_avx2_calc_iv_done +L_AES_GCM_encrypt_avx2_calc_iv_lt16: + vpxor %xmm0, %xmm0, %xmm0 + xorl %ebx, %ebx + vmovdqa %xmm0, (%rsp) +L_AES_GCM_encrypt_avx2_calc_iv_loop: + movzbl (%rax,%rcx,1), %r13d + movb %r13b, (%rsp,%rbx,1) + incl %ecx + incl %ebx + cmpl %edx, %ecx + jl L_AES_GCM_encrypt_avx2_calc_iv_loop + vmovdqa (%rsp), %xmm0 + vpshufb L_avx2_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0 + vpxor %xmm0, %xmm4, %xmm4 + # ghash_gfmul_avx + vpclmulqdq $16, %xmm4, %xmm5, %xmm2 + vpclmulqdq $0x01, %xmm4, %xmm5, %xmm1 + vpclmulqdq $0x00, %xmm4, %xmm5, %xmm0 + vpclmulqdq $0x11, %xmm4, %xmm5, %xmm3 + vpxor %xmm1, %xmm2, %xmm2 + vpslldq $8, %xmm2, %xmm1 + vpsrldq $8, %xmm2, %xmm2 + vpxor %xmm1, %xmm0, %xmm7 + vpxor %xmm2, %xmm3, %xmm4 + # ghash_mid + vpsrld $31, %xmm7, %xmm0 + vpsrld $31, %xmm4, %xmm1 + vpslld $0x01, %xmm7, %xmm7 + vpslld $0x01, %xmm4, %xmm4 + vpsrldq $12, %xmm0, %xmm2 + vpslldq $4, %xmm0, %xmm0 + vpslldq $4, %xmm1, %xmm1 + vpor %xmm2, %xmm4, %xmm4 + vpor %xmm0, %xmm7, %xmm7 + vpor %xmm1, %xmm4, %xmm4 + # ghash_red + vmovdqa L_avx2_aes_gcm_mod2_128(%rip), %xmm2 + vpclmulqdq $16, %xmm2, %xmm7, %xmm0 + vpshufd $0x4e, %xmm7, %xmm1 + vpxor %xmm0, %xmm1, %xmm1 + vpclmulqdq $16, %xmm2, %xmm1, %xmm0 + vpshufd $0x4e, %xmm1, %xmm1 + vpxor %xmm0, %xmm1, %xmm1 + vpxor %xmm1, %xmm4, %xmm4 +L_AES_GCM_encrypt_avx2_calc_iv_done: + # T = Encrypt counter + vpxor %xmm0, %xmm0, %xmm0 + shll $3, %edx + vpinsrq $0x00, %rdx, %xmm0, %xmm0 + vpxor %xmm0, %xmm4, %xmm4 + # ghash_gfmul_avx + vpclmulqdq $16, %xmm4, %xmm5, %xmm2 + vpclmulqdq $0x01, %xmm4, %xmm5, %xmm1 + vpclmulqdq $0x00, %xmm4, %xmm5, %xmm0 + vpclmulqdq $0x11, %xmm4, %xmm5, %xmm3 + vpxor %xmm1, %xmm2, %xmm2 + vpslldq $8, %xmm2, %xmm1 + vpsrldq $8, %xmm2, %xmm2 + vpxor %xmm1, %xmm0, %xmm7 + vpxor %xmm2, %xmm3, %xmm4 + # ghash_mid + vpsrld $31, %xmm7, %xmm0 + vpsrld $31, %xmm4, %xmm1 + vpslld $0x01, %xmm7, %xmm7 + vpslld $0x01, %xmm4, %xmm4 + vpsrldq $12, %xmm0, %xmm2 + vpslldq $4, %xmm0, %xmm0 + vpslldq $4, %xmm1, %xmm1 + vpor %xmm2, %xmm4, %xmm4 + vpor %xmm0, %xmm7, %xmm7 + vpor %xmm1, %xmm4, %xmm4 + # ghash_red + vmovdqa L_avx2_aes_gcm_mod2_128(%rip), %xmm2 + vpclmulqdq $16, %xmm2, %xmm7, %xmm0 + vpshufd $0x4e, %xmm7, %xmm1 + vpxor %xmm0, %xmm1, %xmm1 + vpclmulqdq $16, %xmm2, %xmm1, %xmm0 + vpshufd $0x4e, %xmm1, %xmm1 + vpxor %xmm0, %xmm1, %xmm1 + vpxor %xmm1, %xmm4, %xmm4 + vpshufb L_avx2_aes_gcm_bswap_mask(%rip), %xmm4, %xmm4 + # Encrypt counter + vmovdqa (%rsi), %xmm15 + vpxor %xmm4, %xmm15, %xmm15 + vaesenc 16(%rsi), %xmm15, %xmm15 + vaesenc 32(%rsi), %xmm15, %xmm15 + vaesenc 48(%rsi), %xmm15, %xmm15 + vaesenc 64(%rsi), %xmm15, %xmm15 + vaesenc 80(%rsi), %xmm15, %xmm15 + vaesenc 96(%rsi), %xmm15, %xmm15 + vaesenc 112(%rsi), %xmm15, %xmm15 + vaesenc 128(%rsi), %xmm15, %xmm15 + vaesenc 144(%rsi), %xmm15, %xmm15 + cmpl $11, %r9d + vmovdqa 160(%rsi), %xmm0 + jl L_AES_GCM_encrypt_avx2_calc_iv_2_aesenc_avx_last + vaesenc %xmm0, %xmm15, %xmm15 + vaesenc 176(%rsi), %xmm15, %xmm15 + cmpl $13, %r9d + vmovdqa 192(%rsi), %xmm0 + jl L_AES_GCM_encrypt_avx2_calc_iv_2_aesenc_avx_last + vaesenc %xmm0, %xmm15, %xmm15 + vaesenc 208(%rsi), %xmm15, %xmm15 + vmovdqa 224(%rsi), %xmm0 +L_AES_GCM_encrypt_avx2_calc_iv_2_aesenc_avx_last: + vaesenclast %xmm0, %xmm15, %xmm15 + jmp L_AES_GCM_encrypt_avx2_iv_done +L_AES_GCM_encrypt_avx2_iv_12: + # # Calculate values when IV is 12 bytes + # Set counter based on IV + vmovdqa L_avx2_aes_gcm_bswap_one(%rip), %xmm4 + vmovdqa (%rsi), %xmm5 + vpblendd $7, (%rax), %xmm4, %xmm4 + # H = Encrypt X(=0) and T = Encrypt counter + vmovdqa 16(%rsi), %xmm7 + vpxor %xmm5, %xmm4, %xmm15 + vaesenc %xmm7, %xmm5, %xmm5 + vaesenc %xmm7, %xmm15, %xmm15 + vmovdqa 32(%rsi), %xmm0 + vaesenc %xmm0, %xmm5, %xmm5 + vaesenc %xmm0, %xmm15, %xmm15 + vmovdqa 48(%rsi), %xmm0 + vaesenc %xmm0, %xmm5, %xmm5 + vaesenc %xmm0, %xmm15, %xmm15 + vmovdqa 64(%rsi), %xmm0 + vaesenc %xmm0, %xmm5, %xmm5 + vaesenc %xmm0, %xmm15, %xmm15 + vmovdqa 80(%rsi), %xmm0 + vaesenc %xmm0, %xmm5, %xmm5 + vaesenc %xmm0, %xmm15, %xmm15 + vmovdqa 96(%rsi), %xmm0 + vaesenc %xmm0, %xmm5, %xmm5 + vaesenc %xmm0, %xmm15, %xmm15 + vmovdqa 112(%rsi), %xmm0 + vaesenc %xmm0, %xmm5, %xmm5 + vaesenc %xmm0, %xmm15, %xmm15 + vmovdqa 128(%rsi), %xmm0 + vaesenc %xmm0, %xmm5, %xmm5 + vaesenc %xmm0, %xmm15, %xmm15 + vmovdqa 144(%rsi), %xmm0 + vaesenc %xmm0, %xmm5, %xmm5 + vaesenc %xmm0, %xmm15, %xmm15 + cmpl $11, %r9d + vmovdqa 160(%rsi), %xmm0 + jl L_AES_GCM_encrypt_avx2_calc_iv_12_last + vaesenc %xmm0, %xmm5, %xmm5 + vaesenc %xmm0, %xmm15, %xmm15 + vmovdqa 176(%rsi), %xmm0 + vaesenc %xmm0, %xmm5, %xmm5 + vaesenc %xmm0, %xmm15, %xmm15 + cmpl $13, %r9d + vmovdqa 192(%rsi), %xmm0 + jl L_AES_GCM_encrypt_avx2_calc_iv_12_last + vaesenc %xmm0, %xmm5, %xmm5 + vaesenc %xmm0, %xmm15, %xmm15 + vmovdqa 208(%rsi), %xmm0 + vaesenc %xmm0, %xmm5, %xmm5 + vaesenc %xmm0, %xmm15, %xmm15 + vmovdqa 224(%rsi), %xmm0 +L_AES_GCM_encrypt_avx2_calc_iv_12_last: + vaesenclast %xmm0, %xmm5, %xmm5 + vaesenclast %xmm0, %xmm15, %xmm15 + vpshufb L_avx2_aes_gcm_bswap_mask(%rip), %xmm5, %xmm5 +L_AES_GCM_encrypt_avx2_iv_done: + # Additional authentication data + movl %r11d, %edx + cmpl $0x00, %edx + je L_AES_GCM_encrypt_avx2_calc_aad_done + xorl %ecx, %ecx + cmpl $16, %edx + jl L_AES_GCM_encrypt_avx2_calc_aad_lt16 + andl $0xfffffff0, %edx +L_AES_GCM_encrypt_avx2_calc_aad_16_loop: + vmovdqu (%r12,%rcx,1), %xmm0 + vpshufb L_avx2_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0 + vpxor %xmm0, %xmm6, %xmm6 + # ghash_gfmul_avx + vpclmulqdq $16, %xmm6, %xmm5, %xmm2 + vpclmulqdq $0x01, %xmm6, %xmm5, %xmm1 + vpclmulqdq $0x00, %xmm6, %xmm5, %xmm0 + vpclmulqdq $0x11, %xmm6, %xmm5, %xmm3 + vpxor %xmm1, %xmm2, %xmm2 + vpslldq $8, %xmm2, %xmm1 + vpsrldq $8, %xmm2, %xmm2 + vpxor %xmm1, %xmm0, %xmm7 + vpxor %xmm2, %xmm3, %xmm6 + # ghash_mid + vpsrld $31, %xmm7, %xmm0 + vpsrld $31, %xmm6, %xmm1 + vpslld $0x01, %xmm7, %xmm7 + vpslld $0x01, %xmm6, %xmm6 + vpsrldq $12, %xmm0, %xmm2 + vpslldq $4, %xmm0, %xmm0 + vpslldq $4, %xmm1, %xmm1 + vpor %xmm2, %xmm6, %xmm6 + vpor %xmm0, %xmm7, %xmm7 + vpor %xmm1, %xmm6, %xmm6 + # ghash_red + vmovdqa L_avx2_aes_gcm_mod2_128(%rip), %xmm2 + vpclmulqdq $16, %xmm2, %xmm7, %xmm0 + vpshufd $0x4e, %xmm7, %xmm1 + vpxor %xmm0, %xmm1, %xmm1 + vpclmulqdq $16, %xmm2, %xmm1, %xmm0 + vpshufd $0x4e, %xmm1, %xmm1 + vpxor %xmm0, %xmm1, %xmm1 + vpxor %xmm1, %xmm6, %xmm6 + addl $16, %ecx + cmpl %edx, %ecx + jl L_AES_GCM_encrypt_avx2_calc_aad_16_loop + movl %r11d, %edx + cmpl %edx, %ecx + je L_AES_GCM_encrypt_avx2_calc_aad_done +L_AES_GCM_encrypt_avx2_calc_aad_lt16: + vpxor %xmm0, %xmm0, %xmm0 + xorl %ebx, %ebx + vmovdqa %xmm0, (%rsp) +L_AES_GCM_encrypt_avx2_calc_aad_loop: + movzbl (%r12,%rcx,1), %r13d + movb %r13b, (%rsp,%rbx,1) + incl %ecx + incl %ebx + cmpl %edx, %ecx + jl L_AES_GCM_encrypt_avx2_calc_aad_loop + vmovdqa (%rsp), %xmm0 + vpshufb L_avx2_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0 + vpxor %xmm0, %xmm6, %xmm6 + # ghash_gfmul_avx + vpclmulqdq $16, %xmm6, %xmm5, %xmm2 + vpclmulqdq $0x01, %xmm6, %xmm5, %xmm1 + vpclmulqdq $0x00, %xmm6, %xmm5, %xmm0 + vpclmulqdq $0x11, %xmm6, %xmm5, %xmm3 + vpxor %xmm1, %xmm2, %xmm2 + vpslldq $8, %xmm2, %xmm1 + vpsrldq $8, %xmm2, %xmm2 + vpxor %xmm1, %xmm0, %xmm7 + vpxor %xmm2, %xmm3, %xmm6 + # ghash_mid + vpsrld $31, %xmm7, %xmm0 + vpsrld $31, %xmm6, %xmm1 + vpslld $0x01, %xmm7, %xmm7 + vpslld $0x01, %xmm6, %xmm6 + vpsrldq $12, %xmm0, %xmm2 + vpslldq $4, %xmm0, %xmm0 + vpslldq $4, %xmm1, %xmm1 + vpor %xmm2, %xmm6, %xmm6 + vpor %xmm0, %xmm7, %xmm7 + vpor %xmm1, %xmm6, %xmm6 + # ghash_red + vmovdqa L_avx2_aes_gcm_mod2_128(%rip), %xmm2 + vpclmulqdq $16, %xmm2, %xmm7, %xmm0 + vpshufd $0x4e, %xmm7, %xmm1 + vpxor %xmm0, %xmm1, %xmm1 + vpclmulqdq $16, %xmm2, %xmm1, %xmm0 + vpshufd $0x4e, %xmm1, %xmm1 + vpxor %xmm0, %xmm1, %xmm1 + vpxor %xmm1, %xmm6, %xmm6 +L_AES_GCM_encrypt_avx2_calc_aad_done: + # Calculate counter and H + vpsrlq $63, %xmm5, %xmm1 + vpsllq $0x01, %xmm5, %xmm0 + vpslldq $8, %xmm1, %xmm1 + vpor %xmm1, %xmm0, %xmm0 + vpshufd $0xff, %xmm5, %xmm5 + vpsrad $31, %xmm5, %xmm5 + vpshufb L_avx2_aes_gcm_bswap_epi64(%rip), %xmm4, %xmm4 + vpand L_avx2_aes_gcm_mod2_128(%rip), %xmm5, %xmm5 + vpaddd L_avx2_aes_gcm_one(%rip), %xmm4, %xmm4 + vpxor %xmm0, %xmm5, %xmm5 + xorl %ebx, %ebx + cmpl $0x80, %r10d + movl %r10d, %r13d + jl L_AES_GCM_encrypt_avx2_done_128 + andl $0xffffff80, %r13d + vmovdqa %xmm4, 128(%rsp) + vmovdqa %xmm15, 144(%rsp) + vmovdqa L_avx2_aes_gcm_mod2_128(%rip), %xmm3 + # H ^ 1 and H ^ 2 + vpclmulqdq $0x00, %xmm5, %xmm5, %xmm9 + vpclmulqdq $0x11, %xmm5, %xmm5, %xmm10 + vpclmulqdq $16, %xmm3, %xmm9, %xmm8 + vpshufd $0x4e, %xmm9, %xmm9 + vpxor %xmm8, %xmm9, %xmm9 + vpclmulqdq $16, %xmm3, %xmm9, %xmm8 + vpshufd $0x4e, %xmm9, %xmm9 + vpxor %xmm8, %xmm9, %xmm9 + vpxor %xmm9, %xmm10, %xmm0 + vmovdqa %xmm5, (%rsp) + vmovdqa %xmm0, 16(%rsp) + # H ^ 3 and H ^ 4 + vpclmulqdq $16, %xmm5, %xmm0, %xmm11 + vpclmulqdq $0x01, %xmm5, %xmm0, %xmm10 + vpclmulqdq $0x00, %xmm5, %xmm0, %xmm9 + vpclmulqdq $0x11, %xmm5, %xmm0, %xmm12 + vpclmulqdq $0x00, %xmm0, %xmm0, %xmm13 + vpclmulqdq $0x11, %xmm0, %xmm0, %xmm14 + vpxor %xmm10, %xmm11, %xmm11 + vpslldq $8, %xmm11, %xmm10 + vpsrldq $8, %xmm11, %xmm11 + vpxor %xmm9, %xmm10, %xmm10 + vpclmulqdq $16, %xmm3, %xmm13, %xmm8 + vpclmulqdq $16, %xmm3, %xmm10, %xmm9 + vpshufd $0x4e, %xmm10, %xmm10 + vpshufd $0x4e, %xmm13, %xmm13 + vpxor %xmm9, %xmm10, %xmm10 + vpxor %xmm8, %xmm13, %xmm13 + vpclmulqdq $16, %xmm3, %xmm10, %xmm9 + vpclmulqdq $16, %xmm3, %xmm13, %xmm8 + vpshufd $0x4e, %xmm10, %xmm10 + vpshufd $0x4e, %xmm13, %xmm13 + vpxor %xmm11, %xmm12, %xmm12 + vpxor %xmm8, %xmm13, %xmm13 + vpxor %xmm12, %xmm10, %xmm10 + vpxor %xmm14, %xmm13, %xmm2 + vpxor %xmm9, %xmm10, %xmm1 + vmovdqa %xmm1, 32(%rsp) + vmovdqa %xmm2, 48(%rsp) + # H ^ 5 and H ^ 6 + vpclmulqdq $16, %xmm0, %xmm1, %xmm11 + vpclmulqdq $0x01, %xmm0, %xmm1, %xmm10 + vpclmulqdq $0x00, %xmm0, %xmm1, %xmm9 + vpclmulqdq $0x11, %xmm0, %xmm1, %xmm12 + vpclmulqdq $0x00, %xmm1, %xmm1, %xmm13 + vpclmulqdq $0x11, %xmm1, %xmm1, %xmm14 + vpxor %xmm10, %xmm11, %xmm11 + vpslldq $8, %xmm11, %xmm10 + vpsrldq $8, %xmm11, %xmm11 + vpxor %xmm9, %xmm10, %xmm10 + vpclmulqdq $16, %xmm3, %xmm13, %xmm8 + vpclmulqdq $16, %xmm3, %xmm10, %xmm9 + vpshufd $0x4e, %xmm10, %xmm10 + vpshufd $0x4e, %xmm13, %xmm13 + vpxor %xmm9, %xmm10, %xmm10 + vpxor %xmm8, %xmm13, %xmm13 + vpclmulqdq $16, %xmm3, %xmm10, %xmm9 + vpclmulqdq $16, %xmm3, %xmm13, %xmm8 + vpshufd $0x4e, %xmm10, %xmm10 + vpshufd $0x4e, %xmm13, %xmm13 + vpxor %xmm11, %xmm12, %xmm12 + vpxor %xmm8, %xmm13, %xmm13 + vpxor %xmm12, %xmm10, %xmm10 + vpxor %xmm14, %xmm13, %xmm0 + vpxor %xmm9, %xmm10, %xmm7 + vmovdqa %xmm7, 64(%rsp) + vmovdqa %xmm0, 80(%rsp) + # H ^ 7 and H ^ 8 + vpclmulqdq $16, %xmm1, %xmm2, %xmm11 + vpclmulqdq $0x01, %xmm1, %xmm2, %xmm10 + vpclmulqdq $0x00, %xmm1, %xmm2, %xmm9 + vpclmulqdq $0x11, %xmm1, %xmm2, %xmm12 + vpclmulqdq $0x00, %xmm2, %xmm2, %xmm13 + vpclmulqdq $0x11, %xmm2, %xmm2, %xmm14 + vpxor %xmm10, %xmm11, %xmm11 + vpslldq $8, %xmm11, %xmm10 + vpsrldq $8, %xmm11, %xmm11 + vpxor %xmm9, %xmm10, %xmm10 + vpclmulqdq $16, %xmm3, %xmm13, %xmm8 + vpclmulqdq $16, %xmm3, %xmm10, %xmm9 + vpshufd $0x4e, %xmm10, %xmm10 + vpshufd $0x4e, %xmm13, %xmm13 + vpxor %xmm9, %xmm10, %xmm10 + vpxor %xmm8, %xmm13, %xmm13 + vpclmulqdq $16, %xmm3, %xmm10, %xmm9 + vpclmulqdq $16, %xmm3, %xmm13, %xmm8 + vpshufd $0x4e, %xmm10, %xmm10 + vpshufd $0x4e, %xmm13, %xmm13 + vpxor %xmm11, %xmm12, %xmm12 + vpxor %xmm8, %xmm13, %xmm13 + vpxor %xmm12, %xmm10, %xmm10 + vpxor %xmm14, %xmm13, %xmm0 + vpxor %xmm9, %xmm10, %xmm7 + vmovdqa %xmm7, 96(%rsp) + vmovdqa %xmm0, 112(%rsp) + # First 128 bytes of input + # aesenc_128 + # aesenc_ctr + vmovdqa 128(%rsp), %xmm0 + vmovdqa L_avx2_aes_gcm_bswap_epi64(%rip), %xmm1 + vpaddd L_avx2_aes_gcm_one(%rip), %xmm0, %xmm9 + vpshufb %xmm1, %xmm0, %xmm8 + vpaddd L_avx2_aes_gcm_two(%rip), %xmm0, %xmm10 + vpshufb %xmm1, %xmm9, %xmm9 + vpaddd L_avx2_aes_gcm_three(%rip), %xmm0, %xmm11 + vpshufb %xmm1, %xmm10, %xmm10 + vpaddd L_avx2_aes_gcm_four(%rip), %xmm0, %xmm12 + vpshufb %xmm1, %xmm11, %xmm11 + vpaddd L_avx2_aes_gcm_five(%rip), %xmm0, %xmm13 + vpshufb %xmm1, %xmm12, %xmm12 + vpaddd L_avx2_aes_gcm_six(%rip), %xmm0, %xmm14 + vpshufb %xmm1, %xmm13, %xmm13 + vpaddd L_avx2_aes_gcm_seven(%rip), %xmm0, %xmm15 + vpshufb %xmm1, %xmm14, %xmm14 + vpaddd L_avx2_aes_gcm_eight(%rip), %xmm0, %xmm0 + vpshufb %xmm1, %xmm15, %xmm15 + # aesenc_xor + vmovdqa (%rsi), %xmm7 + vmovdqa %xmm0, 128(%rsp) + vpxor %xmm7, %xmm8, %xmm8 + vpxor %xmm7, %xmm9, %xmm9 + vpxor %xmm7, %xmm10, %xmm10 + vpxor %xmm7, %xmm11, %xmm11 + vpxor %xmm7, %xmm12, %xmm12 + vpxor %xmm7, %xmm13, %xmm13 + vpxor %xmm7, %xmm14, %xmm14 + vpxor %xmm7, %xmm15, %xmm15 + vmovdqa 16(%rsi), %xmm7 + vaesenc %xmm7, %xmm8, %xmm8 + vaesenc %xmm7, %xmm9, %xmm9 + vaesenc %xmm7, %xmm10, %xmm10 + vaesenc %xmm7, %xmm11, %xmm11 + vaesenc %xmm7, %xmm12, %xmm12 + vaesenc %xmm7, %xmm13, %xmm13 + vaesenc %xmm7, %xmm14, %xmm14 + vaesenc %xmm7, %xmm15, %xmm15 + vmovdqa 32(%rsi), %xmm7 + vaesenc %xmm7, %xmm8, %xmm8 + vaesenc %xmm7, %xmm9, %xmm9 + vaesenc %xmm7, %xmm10, %xmm10 + vaesenc %xmm7, %xmm11, %xmm11 + vaesenc %xmm7, %xmm12, %xmm12 + vaesenc %xmm7, %xmm13, %xmm13 + vaesenc %xmm7, %xmm14, %xmm14 + vaesenc %xmm7, %xmm15, %xmm15 + vmovdqa 48(%rsi), %xmm7 + vaesenc %xmm7, %xmm8, %xmm8 + vaesenc %xmm7, %xmm9, %xmm9 + vaesenc %xmm7, %xmm10, %xmm10 + vaesenc %xmm7, %xmm11, %xmm11 + vaesenc %xmm7, %xmm12, %xmm12 + vaesenc %xmm7, %xmm13, %xmm13 + vaesenc %xmm7, %xmm14, %xmm14 + vaesenc %xmm7, %xmm15, %xmm15 + vmovdqa 64(%rsi), %xmm7 + vaesenc %xmm7, %xmm8, %xmm8 + vaesenc %xmm7, %xmm9, %xmm9 + vaesenc %xmm7, %xmm10, %xmm10 + vaesenc %xmm7, %xmm11, %xmm11 + vaesenc %xmm7, %xmm12, %xmm12 + vaesenc %xmm7, %xmm13, %xmm13 + vaesenc %xmm7, %xmm14, %xmm14 + vaesenc %xmm7, %xmm15, %xmm15 + vmovdqa 80(%rsi), %xmm7 + vaesenc %xmm7, %xmm8, %xmm8 + vaesenc %xmm7, %xmm9, %xmm9 + vaesenc %xmm7, %xmm10, %xmm10 + vaesenc %xmm7, %xmm11, %xmm11 + vaesenc %xmm7, %xmm12, %xmm12 + vaesenc %xmm7, %xmm13, %xmm13 + vaesenc %xmm7, %xmm14, %xmm14 + vaesenc %xmm7, %xmm15, %xmm15 + vmovdqa 96(%rsi), %xmm7 + vaesenc %xmm7, %xmm8, %xmm8 + vaesenc %xmm7, %xmm9, %xmm9 + vaesenc %xmm7, %xmm10, %xmm10 + vaesenc %xmm7, %xmm11, %xmm11 + vaesenc %xmm7, %xmm12, %xmm12 + vaesenc %xmm7, %xmm13, %xmm13 + vaesenc %xmm7, %xmm14, %xmm14 + vaesenc %xmm7, %xmm15, %xmm15 + vmovdqa 112(%rsi), %xmm7 + vaesenc %xmm7, %xmm8, %xmm8 + vaesenc %xmm7, %xmm9, %xmm9 + vaesenc %xmm7, %xmm10, %xmm10 + vaesenc %xmm7, %xmm11, %xmm11 + vaesenc %xmm7, %xmm12, %xmm12 + vaesenc %xmm7, %xmm13, %xmm13 + vaesenc %xmm7, %xmm14, %xmm14 + vaesenc %xmm7, %xmm15, %xmm15 + vmovdqa 128(%rsi), %xmm7 + vaesenc %xmm7, %xmm8, %xmm8 + vaesenc %xmm7, %xmm9, %xmm9 + vaesenc %xmm7, %xmm10, %xmm10 + vaesenc %xmm7, %xmm11, %xmm11 + vaesenc %xmm7, %xmm12, %xmm12 + vaesenc %xmm7, %xmm13, %xmm13 + vaesenc %xmm7, %xmm14, %xmm14 + vaesenc %xmm7, %xmm15, %xmm15 + vmovdqa 144(%rsi), %xmm7 + vaesenc %xmm7, %xmm8, %xmm8 + vaesenc %xmm7, %xmm9, %xmm9 + vaesenc %xmm7, %xmm10, %xmm10 + vaesenc %xmm7, %xmm11, %xmm11 + vaesenc %xmm7, %xmm12, %xmm12 + vaesenc %xmm7, %xmm13, %xmm13 + vaesenc %xmm7, %xmm14, %xmm14 + vaesenc %xmm7, %xmm15, %xmm15 + cmpl $11, %r9d + vmovdqa 160(%rsi), %xmm7 + jl L_AES_GCM_encrypt_avx2_aesenc_128_enc_done + vaesenc %xmm7, %xmm8, %xmm8 + vaesenc %xmm7, %xmm9, %xmm9 + vaesenc %xmm7, %xmm10, %xmm10 + vaesenc %xmm7, %xmm11, %xmm11 + vaesenc %xmm7, %xmm12, %xmm12 + vaesenc %xmm7, %xmm13, %xmm13 + vaesenc %xmm7, %xmm14, %xmm14 + vaesenc %xmm7, %xmm15, %xmm15 + vmovdqa 176(%rsi), %xmm7 + vaesenc %xmm7, %xmm8, %xmm8 + vaesenc %xmm7, %xmm9, %xmm9 + vaesenc %xmm7, %xmm10, %xmm10 + vaesenc %xmm7, %xmm11, %xmm11 + vaesenc %xmm7, %xmm12, %xmm12 + vaesenc %xmm7, %xmm13, %xmm13 + vaesenc %xmm7, %xmm14, %xmm14 + vaesenc %xmm7, %xmm15, %xmm15 + cmpl $13, %r9d + vmovdqa 192(%rsi), %xmm7 + jl L_AES_GCM_encrypt_avx2_aesenc_128_enc_done + vaesenc %xmm7, %xmm8, %xmm8 + vaesenc %xmm7, %xmm9, %xmm9 + vaesenc %xmm7, %xmm10, %xmm10 + vaesenc %xmm7, %xmm11, %xmm11 + vaesenc %xmm7, %xmm12, %xmm12 + vaesenc %xmm7, %xmm13, %xmm13 + vaesenc %xmm7, %xmm14, %xmm14 + vaesenc %xmm7, %xmm15, %xmm15 + vmovdqa 208(%rsi), %xmm7 + vaesenc %xmm7, %xmm8, %xmm8 + vaesenc %xmm7, %xmm9, %xmm9 + vaesenc %xmm7, %xmm10, %xmm10 + vaesenc %xmm7, %xmm11, %xmm11 + vaesenc %xmm7, %xmm12, %xmm12 + vaesenc %xmm7, %xmm13, %xmm13 + vaesenc %xmm7, %xmm14, %xmm14 + vaesenc %xmm7, %xmm15, %xmm15 + vmovdqa 224(%rsi), %xmm7 +L_AES_GCM_encrypt_avx2_aesenc_128_enc_done: + # aesenc_last + vaesenclast %xmm7, %xmm8, %xmm8 + vaesenclast %xmm7, %xmm9, %xmm9 + vaesenclast %xmm7, %xmm10, %xmm10 + vaesenclast %xmm7, %xmm11, %xmm11 + vmovdqu (%rdi), %xmm0 + vmovdqu 16(%rdi), %xmm1 + vmovdqu 32(%rdi), %xmm2 + vmovdqu 48(%rdi), %xmm3 + vpxor %xmm0, %xmm8, %xmm8 + vpxor %xmm1, %xmm9, %xmm9 + vpxor %xmm2, %xmm10, %xmm10 + vpxor %xmm3, %xmm11, %xmm11 + vmovdqu %xmm8, (%r8) + vmovdqu %xmm9, 16(%r8) + vmovdqu %xmm10, 32(%r8) + vmovdqu %xmm11, 48(%r8) + vaesenclast %xmm7, %xmm12, %xmm12 + vaesenclast %xmm7, %xmm13, %xmm13 + vaesenclast %xmm7, %xmm14, %xmm14 + vaesenclast %xmm7, %xmm15, %xmm15 + vmovdqu 64(%rdi), %xmm0 + vmovdqu 80(%rdi), %xmm1 + vmovdqu 96(%rdi), %xmm2 + vmovdqu 112(%rdi), %xmm3 + vpxor %xmm0, %xmm12, %xmm12 + vpxor %xmm1, %xmm13, %xmm13 + vpxor %xmm2, %xmm14, %xmm14 + vpxor %xmm3, %xmm15, %xmm15 + vmovdqu %xmm12, 64(%r8) + vmovdqu %xmm13, 80(%r8) + vmovdqu %xmm14, 96(%r8) + vmovdqu %xmm15, 112(%r8) + cmpl $0x80, %r13d + movl $0x80, %ebx + jle L_AES_GCM_encrypt_avx2_end_128 + # More 128 bytes of input +L_AES_GCM_encrypt_avx2_ghash_128: + # aesenc_128_ghash + leaq (%rdi,%rbx,1), %rcx + leaq (%r8,%rbx,1), %rdx + # aesenc_ctr + vmovdqa 128(%rsp), %xmm0 + vmovdqa L_avx2_aes_gcm_bswap_epi64(%rip), %xmm1 + vpaddd L_avx2_aes_gcm_one(%rip), %xmm0, %xmm9 + vpshufb %xmm1, %xmm0, %xmm8 + vpaddd L_avx2_aes_gcm_two(%rip), %xmm0, %xmm10 + vpshufb %xmm1, %xmm9, %xmm9 + vpaddd L_avx2_aes_gcm_three(%rip), %xmm0, %xmm11 + vpshufb %xmm1, %xmm10, %xmm10 + vpaddd L_avx2_aes_gcm_four(%rip), %xmm0, %xmm12 + vpshufb %xmm1, %xmm11, %xmm11 + vpaddd L_avx2_aes_gcm_five(%rip), %xmm0, %xmm13 + vpshufb %xmm1, %xmm12, %xmm12 + vpaddd L_avx2_aes_gcm_six(%rip), %xmm0, %xmm14 + vpshufb %xmm1, %xmm13, %xmm13 + vpaddd L_avx2_aes_gcm_seven(%rip), %xmm0, %xmm15 + vpshufb %xmm1, %xmm14, %xmm14 + vpaddd L_avx2_aes_gcm_eight(%rip), %xmm0, %xmm0 + vpshufb %xmm1, %xmm15, %xmm15 + # aesenc_xor + vmovdqa (%rsi), %xmm7 + vmovdqa %xmm0, 128(%rsp) + vpxor %xmm7, %xmm8, %xmm8 + vpxor %xmm7, %xmm9, %xmm9 + vpxor %xmm7, %xmm10, %xmm10 + vpxor %xmm7, %xmm11, %xmm11 + vpxor %xmm7, %xmm12, %xmm12 + vpxor %xmm7, %xmm13, %xmm13 + vpxor %xmm7, %xmm14, %xmm14 + vpxor %xmm7, %xmm15, %xmm15 + # aesenc_pclmul_1 + vmovdqu -128(%rdx), %xmm1 + vmovdqu 16(%rsi), %xmm0 + vpshufb L_avx2_aes_gcm_bswap_mask(%rip), %xmm1, %xmm1 + vmovdqa 112(%rsp), %xmm2 + vpxor %xmm6, %xmm1, %xmm1 + vpclmulqdq $16, %xmm2, %xmm1, %xmm5 + vpclmulqdq $0x01, %xmm2, %xmm1, %xmm3 + vpclmulqdq $0x00, %xmm2, %xmm1, %xmm6 + vpclmulqdq $0x11, %xmm2, %xmm1, %xmm7 + vaesenc %xmm0, %xmm8, %xmm8 + vaesenc %xmm0, %xmm9, %xmm9 + vaesenc %xmm0, %xmm10, %xmm10 + vaesenc %xmm0, %xmm11, %xmm11 + vaesenc %xmm0, %xmm12, %xmm12 + vaesenc %xmm0, %xmm13, %xmm13 + vaesenc %xmm0, %xmm14, %xmm14 + vaesenc %xmm0, %xmm15, %xmm15 + # aesenc_pclmul_2 + vmovdqu -112(%rdx), %xmm1 + vmovdqa 96(%rsp), %xmm0 + vpshufb L_avx2_aes_gcm_bswap_mask(%rip), %xmm1, %xmm1 + vpxor %xmm3, %xmm5, %xmm5 + vpclmulqdq $16, %xmm0, %xmm1, %xmm2 + vpclmulqdq $0x01, %xmm0, %xmm1, %xmm3 + vpclmulqdq $0x00, %xmm0, %xmm1, %xmm4 + vpclmulqdq $0x11, %xmm0, %xmm1, %xmm1 + vmovdqu 32(%rsi), %xmm0 + vpxor %xmm1, %xmm7, %xmm7 + vaesenc %xmm0, %xmm8, %xmm8 + vaesenc %xmm0, %xmm9, %xmm9 + vaesenc %xmm0, %xmm10, %xmm10 + vaesenc %xmm0, %xmm11, %xmm11 + vaesenc %xmm0, %xmm12, %xmm12 + vaesenc %xmm0, %xmm13, %xmm13 + vaesenc %xmm0, %xmm14, %xmm14 + vaesenc %xmm0, %xmm15, %xmm15 + # aesenc_pclmul_n + vmovdqu -96(%rdx), %xmm1 + vmovdqa 80(%rsp), %xmm0 + vpshufb L_avx2_aes_gcm_bswap_mask(%rip), %xmm1, %xmm1 + vpxor %xmm2, %xmm5, %xmm5 + vpclmulqdq $16, %xmm0, %xmm1, %xmm2 + vpxor %xmm3, %xmm5, %xmm5 + vpclmulqdq $0x01, %xmm0, %xmm1, %xmm3 + vpxor %xmm4, %xmm6, %xmm6 + vpclmulqdq $0x00, %xmm0, %xmm1, %xmm4 + vpclmulqdq $0x11, %xmm0, %xmm1, %xmm1 + vmovdqu 48(%rsi), %xmm0 + vpxor %xmm1, %xmm7, %xmm7 + vaesenc %xmm0, %xmm8, %xmm8 + vaesenc %xmm0, %xmm9, %xmm9 + vaesenc %xmm0, %xmm10, %xmm10 + vaesenc %xmm0, %xmm11, %xmm11 + vaesenc %xmm0, %xmm12, %xmm12 + vaesenc %xmm0, %xmm13, %xmm13 + vaesenc %xmm0, %xmm14, %xmm14 + vaesenc %xmm0, %xmm15, %xmm15 + # aesenc_pclmul_n + vmovdqu -80(%rdx), %xmm1 + vmovdqa 64(%rsp), %xmm0 + vpshufb L_avx2_aes_gcm_bswap_mask(%rip), %xmm1, %xmm1 + vpxor %xmm2, %xmm5, %xmm5 + vpclmulqdq $16, %xmm0, %xmm1, %xmm2 + vpxor %xmm3, %xmm5, %xmm5 + vpclmulqdq $0x01, %xmm0, %xmm1, %xmm3 + vpxor %xmm4, %xmm6, %xmm6 + vpclmulqdq $0x00, %xmm0, %xmm1, %xmm4 + vpclmulqdq $0x11, %xmm0, %xmm1, %xmm1 + vmovdqu 64(%rsi), %xmm0 + vpxor %xmm1, %xmm7, %xmm7 + vaesenc %xmm0, %xmm8, %xmm8 + vaesenc %xmm0, %xmm9, %xmm9 + vaesenc %xmm0, %xmm10, %xmm10 + vaesenc %xmm0, %xmm11, %xmm11 + vaesenc %xmm0, %xmm12, %xmm12 + vaesenc %xmm0, %xmm13, %xmm13 + vaesenc %xmm0, %xmm14, %xmm14 + vaesenc %xmm0, %xmm15, %xmm15 + # aesenc_pclmul_n + vmovdqu -64(%rdx), %xmm1 + vmovdqa 48(%rsp), %xmm0 + vpshufb L_avx2_aes_gcm_bswap_mask(%rip), %xmm1, %xmm1 + vpxor %xmm2, %xmm5, %xmm5 + vpclmulqdq $16, %xmm0, %xmm1, %xmm2 + vpxor %xmm3, %xmm5, %xmm5 + vpclmulqdq $0x01, %xmm0, %xmm1, %xmm3 + vpxor %xmm4, %xmm6, %xmm6 + vpclmulqdq $0x00, %xmm0, %xmm1, %xmm4 + vpclmulqdq $0x11, %xmm0, %xmm1, %xmm1 + vmovdqu 80(%rsi), %xmm0 + vpxor %xmm1, %xmm7, %xmm7 + vaesenc %xmm0, %xmm8, %xmm8 + vaesenc %xmm0, %xmm9, %xmm9 + vaesenc %xmm0, %xmm10, %xmm10 + vaesenc %xmm0, %xmm11, %xmm11 + vaesenc %xmm0, %xmm12, %xmm12 + vaesenc %xmm0, %xmm13, %xmm13 + vaesenc %xmm0, %xmm14, %xmm14 + vaesenc %xmm0, %xmm15, %xmm15 + # aesenc_pclmul_n + vmovdqu -48(%rdx), %xmm1 + vmovdqa 32(%rsp), %xmm0 + vpshufb L_avx2_aes_gcm_bswap_mask(%rip), %xmm1, %xmm1 + vpxor %xmm2, %xmm5, %xmm5 + vpclmulqdq $16, %xmm0, %xmm1, %xmm2 + vpxor %xmm3, %xmm5, %xmm5 + vpclmulqdq $0x01, %xmm0, %xmm1, %xmm3 + vpxor %xmm4, %xmm6, %xmm6 + vpclmulqdq $0x00, %xmm0, %xmm1, %xmm4 + vpclmulqdq $0x11, %xmm0, %xmm1, %xmm1 + vmovdqu 96(%rsi), %xmm0 + vpxor %xmm1, %xmm7, %xmm7 + vaesenc %xmm0, %xmm8, %xmm8 + vaesenc %xmm0, %xmm9, %xmm9 + vaesenc %xmm0, %xmm10, %xmm10 + vaesenc %xmm0, %xmm11, %xmm11 + vaesenc %xmm0, %xmm12, %xmm12 + vaesenc %xmm0, %xmm13, %xmm13 + vaesenc %xmm0, %xmm14, %xmm14 + vaesenc %xmm0, %xmm15, %xmm15 + # aesenc_pclmul_n + vmovdqu -32(%rdx), %xmm1 + vmovdqa 16(%rsp), %xmm0 + vpshufb L_avx2_aes_gcm_bswap_mask(%rip), %xmm1, %xmm1 + vpxor %xmm2, %xmm5, %xmm5 + vpclmulqdq $16, %xmm0, %xmm1, %xmm2 + vpxor %xmm3, %xmm5, %xmm5 + vpclmulqdq $0x01, %xmm0, %xmm1, %xmm3 + vpxor %xmm4, %xmm6, %xmm6 + vpclmulqdq $0x00, %xmm0, %xmm1, %xmm4 + vpclmulqdq $0x11, %xmm0, %xmm1, %xmm1 + vmovdqu 112(%rsi), %xmm0 + vpxor %xmm1, %xmm7, %xmm7 + vaesenc %xmm0, %xmm8, %xmm8 + vaesenc %xmm0, %xmm9, %xmm9 + vaesenc %xmm0, %xmm10, %xmm10 + vaesenc %xmm0, %xmm11, %xmm11 + vaesenc %xmm0, %xmm12, %xmm12 + vaesenc %xmm0, %xmm13, %xmm13 + vaesenc %xmm0, %xmm14, %xmm14 + vaesenc %xmm0, %xmm15, %xmm15 + # aesenc_pclmul_n + vmovdqu -16(%rdx), %xmm1 + vmovdqa (%rsp), %xmm0 + vpshufb L_avx2_aes_gcm_bswap_mask(%rip), %xmm1, %xmm1 + vpxor %xmm2, %xmm5, %xmm5 + vpclmulqdq $16, %xmm0, %xmm1, %xmm2 + vpxor %xmm3, %xmm5, %xmm5 + vpclmulqdq $0x01, %xmm0, %xmm1, %xmm3 + vpxor %xmm4, %xmm6, %xmm6 + vpclmulqdq $0x00, %xmm0, %xmm1, %xmm4 + vpclmulqdq $0x11, %xmm0, %xmm1, %xmm1 + vmovdqu 128(%rsi), %xmm0 + vpxor %xmm1, %xmm7, %xmm7 + vaesenc %xmm0, %xmm8, %xmm8 + vaesenc %xmm0, %xmm9, %xmm9 + vaesenc %xmm0, %xmm10, %xmm10 + vaesenc %xmm0, %xmm11, %xmm11 + vaesenc %xmm0, %xmm12, %xmm12 + vaesenc %xmm0, %xmm13, %xmm13 + vaesenc %xmm0, %xmm14, %xmm14 + vaesenc %xmm0, %xmm15, %xmm15 + # aesenc_pclmul_l + vpxor %xmm2, %xmm5, %xmm5 + vpxor %xmm4, %xmm6, %xmm6 + vpxor %xmm3, %xmm5, %xmm5 + vpslldq $8, %xmm5, %xmm1 + vpsrldq $8, %xmm5, %xmm5 + vmovdqa 144(%rsi), %xmm4 + vmovdqa L_avx2_aes_gcm_mod2_128(%rip), %xmm0 + vaesenc %xmm4, %xmm8, %xmm8 + vpxor %xmm1, %xmm6, %xmm6 + vpxor %xmm5, %xmm7, %xmm7 + vpclmulqdq $16, %xmm0, %xmm6, %xmm3 + vaesenc %xmm4, %xmm9, %xmm9 + vaesenc %xmm4, %xmm10, %xmm10 + vaesenc %xmm4, %xmm11, %xmm11 + vpshufd $0x4e, %xmm6, %xmm6 + vpxor %xmm3, %xmm6, %xmm6 + vpclmulqdq $16, %xmm0, %xmm6, %xmm3 + vaesenc %xmm4, %xmm12, %xmm12 + vaesenc %xmm4, %xmm13, %xmm13 + vaesenc %xmm4, %xmm14, %xmm14 + vpshufd $0x4e, %xmm6, %xmm6 + vpxor %xmm3, %xmm6, %xmm6 + vpxor %xmm7, %xmm6, %xmm6 + vaesenc %xmm4, %xmm15, %xmm15 + cmpl $11, %r9d + vmovdqa 160(%rsi), %xmm7 + jl L_AES_GCM_encrypt_avx2_aesenc_128_ghash_avx_done + vaesenc %xmm7, %xmm8, %xmm8 + vaesenc %xmm7, %xmm9, %xmm9 + vaesenc %xmm7, %xmm10, %xmm10 + vaesenc %xmm7, %xmm11, %xmm11 + vaesenc %xmm7, %xmm12, %xmm12 + vaesenc %xmm7, %xmm13, %xmm13 + vaesenc %xmm7, %xmm14, %xmm14 + vaesenc %xmm7, %xmm15, %xmm15 + vmovdqa 176(%rsi), %xmm7 + vaesenc %xmm7, %xmm8, %xmm8 + vaesenc %xmm7, %xmm9, %xmm9 + vaesenc %xmm7, %xmm10, %xmm10 + vaesenc %xmm7, %xmm11, %xmm11 + vaesenc %xmm7, %xmm12, %xmm12 + vaesenc %xmm7, %xmm13, %xmm13 + vaesenc %xmm7, %xmm14, %xmm14 + vaesenc %xmm7, %xmm15, %xmm15 + cmpl $13, %r9d + vmovdqa 192(%rsi), %xmm7 + jl L_AES_GCM_encrypt_avx2_aesenc_128_ghash_avx_done + vaesenc %xmm7, %xmm8, %xmm8 + vaesenc %xmm7, %xmm9, %xmm9 + vaesenc %xmm7, %xmm10, %xmm10 + vaesenc %xmm7, %xmm11, %xmm11 + vaesenc %xmm7, %xmm12, %xmm12 + vaesenc %xmm7, %xmm13, %xmm13 + vaesenc %xmm7, %xmm14, %xmm14 + vaesenc %xmm7, %xmm15, %xmm15 + vmovdqa 208(%rsi), %xmm7 + vaesenc %xmm7, %xmm8, %xmm8 + vaesenc %xmm7, %xmm9, %xmm9 + vaesenc %xmm7, %xmm10, %xmm10 + vaesenc %xmm7, %xmm11, %xmm11 + vaesenc %xmm7, %xmm12, %xmm12 + vaesenc %xmm7, %xmm13, %xmm13 + vaesenc %xmm7, %xmm14, %xmm14 + vaesenc %xmm7, %xmm15, %xmm15 + vmovdqa 224(%rsi), %xmm7 +L_AES_GCM_encrypt_avx2_aesenc_128_ghash_avx_done: + # aesenc_last + vaesenclast %xmm7, %xmm8, %xmm8 + vaesenclast %xmm7, %xmm9, %xmm9 + vaesenclast %xmm7, %xmm10, %xmm10 + vaesenclast %xmm7, %xmm11, %xmm11 + vmovdqu (%rcx), %xmm0 + vmovdqu 16(%rcx), %xmm1 + vmovdqu 32(%rcx), %xmm2 + vmovdqu 48(%rcx), %xmm3 + vpxor %xmm0, %xmm8, %xmm8 + vpxor %xmm1, %xmm9, %xmm9 + vpxor %xmm2, %xmm10, %xmm10 + vpxor %xmm3, %xmm11, %xmm11 + vmovdqu %xmm8, (%rdx) + vmovdqu %xmm9, 16(%rdx) + vmovdqu %xmm10, 32(%rdx) + vmovdqu %xmm11, 48(%rdx) + vaesenclast %xmm7, %xmm12, %xmm12 + vaesenclast %xmm7, %xmm13, %xmm13 + vaesenclast %xmm7, %xmm14, %xmm14 + vaesenclast %xmm7, %xmm15, %xmm15 + vmovdqu 64(%rcx), %xmm0 + vmovdqu 80(%rcx), %xmm1 + vmovdqu 96(%rcx), %xmm2 + vmovdqu 112(%rcx), %xmm3 + vpxor %xmm0, %xmm12, %xmm12 + vpxor %xmm1, %xmm13, %xmm13 + vpxor %xmm2, %xmm14, %xmm14 + vpxor %xmm3, %xmm15, %xmm15 + vmovdqu %xmm12, 64(%rdx) + vmovdqu %xmm13, 80(%rdx) + vmovdqu %xmm14, 96(%rdx) + vmovdqu %xmm15, 112(%rdx) + # aesenc_128_ghash - end + addl $0x80, %ebx + cmpl %r13d, %ebx + jl L_AES_GCM_encrypt_avx2_ghash_128 +L_AES_GCM_encrypt_avx2_end_128: + vmovdqa L_avx2_aes_gcm_bswap_mask(%rip), %xmm4 + vpshufb %xmm4, %xmm8, %xmm8 + vpshufb %xmm4, %xmm9, %xmm9 + vpshufb %xmm4, %xmm10, %xmm10 + vpshufb %xmm4, %xmm11, %xmm11 + vpshufb %xmm4, %xmm12, %xmm12 + vpshufb %xmm4, %xmm13, %xmm13 + vpshufb %xmm4, %xmm14, %xmm14 + vpshufb %xmm4, %xmm15, %xmm15 + vpxor %xmm6, %xmm8, %xmm8 + vmovdqu (%rsp), %xmm7 + vpclmulqdq $16, %xmm15, %xmm7, %xmm5 + vpclmulqdq $0x01, %xmm15, %xmm7, %xmm1 + vpclmulqdq $0x00, %xmm15, %xmm7, %xmm4 + vpclmulqdq $0x11, %xmm15, %xmm7, %xmm6 + vpxor %xmm1, %xmm5, %xmm5 + vmovdqu 16(%rsp), %xmm7 + vpclmulqdq $16, %xmm14, %xmm7, %xmm2 + vpclmulqdq $0x01, %xmm14, %xmm7, %xmm1 + vpclmulqdq $0x00, %xmm14, %xmm7, %xmm0 + vpclmulqdq $0x11, %xmm14, %xmm7, %xmm3 + vpxor %xmm1, %xmm2, %xmm2 + vpxor %xmm3, %xmm6, %xmm6 + vpxor %xmm2, %xmm5, %xmm5 + vpxor %xmm0, %xmm4, %xmm4 + vmovdqu 32(%rsp), %xmm15 + vmovdqu 48(%rsp), %xmm7 + vpclmulqdq $16, %xmm13, %xmm15, %xmm2 + vpclmulqdq $0x01, %xmm13, %xmm15, %xmm1 + vpclmulqdq $0x00, %xmm13, %xmm15, %xmm0 + vpclmulqdq $0x11, %xmm13, %xmm15, %xmm3 + vpxor %xmm1, %xmm2, %xmm2 + vpxor %xmm3, %xmm6, %xmm6 + vpxor %xmm2, %xmm5, %xmm5 + vpxor %xmm0, %xmm4, %xmm4 + vpclmulqdq $16, %xmm12, %xmm7, %xmm2 + vpclmulqdq $0x01, %xmm12, %xmm7, %xmm1 + vpclmulqdq $0x00, %xmm12, %xmm7, %xmm0 + vpclmulqdq $0x11, %xmm12, %xmm7, %xmm3 + vpxor %xmm1, %xmm2, %xmm2 + vpxor %xmm3, %xmm6, %xmm6 + vpxor %xmm2, %xmm5, %xmm5 + vpxor %xmm0, %xmm4, %xmm4 + vmovdqu 64(%rsp), %xmm15 + vmovdqu 80(%rsp), %xmm7 + vpclmulqdq $16, %xmm11, %xmm15, %xmm2 + vpclmulqdq $0x01, %xmm11, %xmm15, %xmm1 + vpclmulqdq $0x00, %xmm11, %xmm15, %xmm0 + vpclmulqdq $0x11, %xmm11, %xmm15, %xmm3 + vpxor %xmm1, %xmm2, %xmm2 + vpxor %xmm3, %xmm6, %xmm6 + vpxor %xmm2, %xmm5, %xmm5 + vpxor %xmm0, %xmm4, %xmm4 + vpclmulqdq $16, %xmm10, %xmm7, %xmm2 + vpclmulqdq $0x01, %xmm10, %xmm7, %xmm1 + vpclmulqdq $0x00, %xmm10, %xmm7, %xmm0 + vpclmulqdq $0x11, %xmm10, %xmm7, %xmm3 + vpxor %xmm1, %xmm2, %xmm2 + vpxor %xmm3, %xmm6, %xmm6 + vpxor %xmm2, %xmm5, %xmm5 + vpxor %xmm0, %xmm4, %xmm4 + vmovdqu 96(%rsp), %xmm15 + vmovdqu 112(%rsp), %xmm7 + vpclmulqdq $16, %xmm9, %xmm15, %xmm2 + vpclmulqdq $0x01, %xmm9, %xmm15, %xmm1 + vpclmulqdq $0x00, %xmm9, %xmm15, %xmm0 + vpclmulqdq $0x11, %xmm9, %xmm15, %xmm3 + vpxor %xmm1, %xmm2, %xmm2 + vpxor %xmm3, %xmm6, %xmm6 + vpxor %xmm2, %xmm5, %xmm5 + vpxor %xmm0, %xmm4, %xmm4 + vpclmulqdq $16, %xmm8, %xmm7, %xmm2 + vpclmulqdq $0x01, %xmm8, %xmm7, %xmm1 + vpclmulqdq $0x00, %xmm8, %xmm7, %xmm0 + vpclmulqdq $0x11, %xmm8, %xmm7, %xmm3 + vpxor %xmm1, %xmm2, %xmm2 + vpxor %xmm3, %xmm6, %xmm6 + vpxor %xmm2, %xmm5, %xmm5 + vpxor %xmm0, %xmm4, %xmm4 + vpslldq $8, %xmm5, %xmm7 + vpsrldq $8, %xmm5, %xmm5 + vpxor %xmm7, %xmm4, %xmm4 + vpxor %xmm5, %xmm6, %xmm6 + # ghash_red + vmovdqa L_avx2_aes_gcm_mod2_128(%rip), %xmm2 + vpclmulqdq $16, %xmm2, %xmm4, %xmm0 + vpshufd $0x4e, %xmm4, %xmm1 + vpxor %xmm0, %xmm1, %xmm1 + vpclmulqdq $16, %xmm2, %xmm1, %xmm0 + vpshufd $0x4e, %xmm1, %xmm1 + vpxor %xmm0, %xmm1, %xmm1 + vpxor %xmm1, %xmm6, %xmm6 + vmovdqa (%rsp), %xmm5 + vmovdqu 128(%rsp), %xmm4 + vmovdqu 144(%rsp), %xmm15 +L_AES_GCM_encrypt_avx2_done_128: + cmpl %r10d, %ebx + je L_AES_GCM_encrypt_avx2_done_enc + movl %r10d, %r13d + andl $0xfffffff0, %r13d + cmpl %r13d, %ebx + jge L_AES_GCM_encrypt_avx2_last_block_done + # aesenc_block + vmovdqa %xmm4, %xmm1 + vpshufb L_avx2_aes_gcm_bswap_epi64(%rip), %xmm1, %xmm0 + vpaddd L_avx2_aes_gcm_one(%rip), %xmm1, %xmm1 + vpxor (%rsi), %xmm0, %xmm0 + vmovdqa 16(%rsi), %xmm2 + vaesenc %xmm2, %xmm0, %xmm0 + vmovdqa 32(%rsi), %xmm2 + vaesenc %xmm2, %xmm0, %xmm0 + vmovdqa 48(%rsi), %xmm2 + vaesenc %xmm2, %xmm0, %xmm0 + vmovdqa 64(%rsi), %xmm2 + vaesenc %xmm2, %xmm0, %xmm0 + vmovdqa 80(%rsi), %xmm2 + vaesenc %xmm2, %xmm0, %xmm0 + vmovdqa 96(%rsi), %xmm2 + vaesenc %xmm2, %xmm0, %xmm0 + vmovdqa 112(%rsi), %xmm2 + vaesenc %xmm2, %xmm0, %xmm0 + vmovdqa 128(%rsi), %xmm2 + vaesenc %xmm2, %xmm0, %xmm0 + vmovdqa 144(%rsi), %xmm2 + vaesenc %xmm2, %xmm0, %xmm0 + vmovdqa %xmm1, %xmm4 + cmpl $11, %r9d + vmovdqa 160(%rsi), %xmm1 + jl L_AES_GCM_encrypt_avx2_aesenc_block_last + vaesenc %xmm1, %xmm0, %xmm0 + vmovdqa 176(%rsi), %xmm2 + vaesenc %xmm2, %xmm0, %xmm0 + cmpl $13, %r9d + vmovdqa 192(%rsi), %xmm1 + jl L_AES_GCM_encrypt_avx2_aesenc_block_last + vaesenc %xmm1, %xmm0, %xmm0 + vmovdqa 208(%rsi), %xmm2 + vaesenc %xmm2, %xmm0, %xmm0 + vmovdqa 224(%rsi), %xmm1 +L_AES_GCM_encrypt_avx2_aesenc_block_last: + vaesenclast %xmm1, %xmm0, %xmm0 + vmovdqu (%rdi,%rbx,1), %xmm1 + vpxor %xmm1, %xmm0, %xmm0 + vmovdqu %xmm0, (%r8,%rbx,1) + vpshufb L_avx2_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0 + vpxor %xmm0, %xmm6, %xmm6 + addl $16, %ebx + cmpl %r13d, %ebx + jge L_AES_GCM_encrypt_avx2_last_block_ghash +L_AES_GCM_encrypt_avx2_last_block_start: + vmovdqu (%rdi,%rbx,1), %xmm12 + vpshufb L_avx2_aes_gcm_bswap_epi64(%rip), %xmm4, %xmm11 + vpaddd L_avx2_aes_gcm_one(%rip), %xmm4, %xmm4 + # aesenc_gfmul_sb + vpclmulqdq $0x01, %xmm5, %xmm6, %xmm2 + vpclmulqdq $16, %xmm5, %xmm6, %xmm3 + vpclmulqdq $0x00, %xmm5, %xmm6, %xmm1 + vpclmulqdq $0x11, %xmm5, %xmm6, %xmm8 + vpxor (%rsi), %xmm11, %xmm11 + vaesenc 16(%rsi), %xmm11, %xmm11 + vpxor %xmm2, %xmm3, %xmm3 + vpslldq $8, %xmm3, %xmm2 + vpsrldq $8, %xmm3, %xmm3 + vaesenc 32(%rsi), %xmm11, %xmm11 + vpxor %xmm1, %xmm2, %xmm2 + vpclmulqdq $16, L_avx2_aes_gcm_mod2_128(%rip), %xmm2, %xmm1 + vaesenc 48(%rsi), %xmm11, %xmm11 + vaesenc 64(%rsi), %xmm11, %xmm11 + vaesenc 80(%rsi), %xmm11, %xmm11 + vpshufd $0x4e, %xmm2, %xmm2 + vpxor %xmm1, %xmm2, %xmm2 + vpclmulqdq $16, L_avx2_aes_gcm_mod2_128(%rip), %xmm2, %xmm1 + vaesenc 96(%rsi), %xmm11, %xmm11 + vaesenc 112(%rsi), %xmm11, %xmm11 + vaesenc 128(%rsi), %xmm11, %xmm11 + vpshufd $0x4e, %xmm2, %xmm2 + vaesenc 144(%rsi), %xmm11, %xmm11 + vpxor %xmm3, %xmm8, %xmm8 + vpxor %xmm8, %xmm2, %xmm2 + vmovdqa 160(%rsi), %xmm0 + cmpl $11, %r9d + jl L_AES_GCM_encrypt_avx2_aesenc_gfmul_sb_last + vaesenc %xmm0, %xmm11, %xmm11 + vaesenc 176(%rsi), %xmm11, %xmm11 + vmovdqa 192(%rsi), %xmm0 + cmpl $13, %r9d + jl L_AES_GCM_encrypt_avx2_aesenc_gfmul_sb_last + vaesenc %xmm0, %xmm11, %xmm11 + vaesenc 208(%rsi), %xmm11, %xmm11 + vmovdqa 224(%rsi), %xmm0 +L_AES_GCM_encrypt_avx2_aesenc_gfmul_sb_last: + vaesenclast %xmm0, %xmm11, %xmm11 + vpxor %xmm1, %xmm2, %xmm6 + vpxor %xmm12, %xmm11, %xmm11 + vmovdqu %xmm11, (%r8,%rbx,1) + vpshufb L_avx2_aes_gcm_bswap_mask(%rip), %xmm11, %xmm11 + vpxor %xmm11, %xmm6, %xmm6 + addl $16, %ebx + cmpl %r13d, %ebx + jl L_AES_GCM_encrypt_avx2_last_block_start +L_AES_GCM_encrypt_avx2_last_block_ghash: + # ghash_gfmul_red + vpclmulqdq $16, %xmm5, %xmm6, %xmm10 + vpclmulqdq $0x01, %xmm5, %xmm6, %xmm9 + vpclmulqdq $0x00, %xmm5, %xmm6, %xmm8 + vpxor %xmm9, %xmm10, %xmm10 + vpslldq $8, %xmm10, %xmm9 + vpsrldq $8, %xmm10, %xmm10 + vpxor %xmm8, %xmm9, %xmm9 + vpclmulqdq $0x11, %xmm5, %xmm6, %xmm6 + vpclmulqdq $16, L_avx2_aes_gcm_mod2_128(%rip), %xmm9, %xmm8 + vpshufd $0x4e, %xmm9, %xmm9 + vpxor %xmm8, %xmm9, %xmm9 + vpclmulqdq $16, L_avx2_aes_gcm_mod2_128(%rip), %xmm9, %xmm8 + vpshufd $0x4e, %xmm9, %xmm9 + vpxor %xmm10, %xmm6, %xmm6 + vpxor %xmm9, %xmm6, %xmm6 + vpxor %xmm8, %xmm6, %xmm6 +L_AES_GCM_encrypt_avx2_last_block_done: + movl %r10d, %ecx + movl %r10d, %edx + andl $15, %ecx + jz L_AES_GCM_encrypt_avx2_done_enc + # aesenc_last15_enc + vpshufb L_avx2_aes_gcm_bswap_epi64(%rip), %xmm4, %xmm4 + vpxor (%rsi), %xmm4, %xmm4 + vaesenc 16(%rsi), %xmm4, %xmm4 + vaesenc 32(%rsi), %xmm4, %xmm4 + vaesenc 48(%rsi), %xmm4, %xmm4 + vaesenc 64(%rsi), %xmm4, %xmm4 + vaesenc 80(%rsi), %xmm4, %xmm4 + vaesenc 96(%rsi), %xmm4, %xmm4 + vaesenc 112(%rsi), %xmm4, %xmm4 + vaesenc 128(%rsi), %xmm4, %xmm4 + vaesenc 144(%rsi), %xmm4, %xmm4 + cmpl $11, %r9d + vmovdqa 160(%rsi), %xmm0 + jl L_AES_GCM_encrypt_avx2_aesenc_last15_enc_avx_aesenc_avx_last + vaesenc %xmm0, %xmm4, %xmm4 + vaesenc 176(%rsi), %xmm4, %xmm4 + cmpl $13, %r9d + vmovdqa 192(%rsi), %xmm0 + jl L_AES_GCM_encrypt_avx2_aesenc_last15_enc_avx_aesenc_avx_last + vaesenc %xmm0, %xmm4, %xmm4 + vaesenc 208(%rsi), %xmm4, %xmm4 + vmovdqa 224(%rsi), %xmm0 +L_AES_GCM_encrypt_avx2_aesenc_last15_enc_avx_aesenc_avx_last: + vaesenclast %xmm0, %xmm4, %xmm4 + xorl %ecx, %ecx + vpxor %xmm0, %xmm0, %xmm0 + vmovdqa %xmm4, (%rsp) + vmovdqa %xmm0, 16(%rsp) +L_AES_GCM_encrypt_avx2_aesenc_last15_enc_avx_loop: + movzbl (%rdi,%rbx,1), %r13d + xorb (%rsp,%rcx,1), %r13b + movb %r13b, 16(%rsp,%rcx,1) + movb %r13b, (%r8,%rbx,1) + incl %ebx + incl %ecx + cmpl %edx, %ebx + jl L_AES_GCM_encrypt_avx2_aesenc_last15_enc_avx_loop +L_AES_GCM_encrypt_avx2_aesenc_last15_enc_avx_finish_enc: + vmovdqa 16(%rsp), %xmm4 + vpshufb L_avx2_aes_gcm_bswap_mask(%rip), %xmm4, %xmm4 + vpxor %xmm4, %xmm6, %xmm6 + # ghash_gfmul_red + vpclmulqdq $16, %xmm5, %xmm6, %xmm2 + vpclmulqdq $0x01, %xmm5, %xmm6, %xmm1 + vpclmulqdq $0x00, %xmm5, %xmm6, %xmm0 + vpxor %xmm1, %xmm2, %xmm2 + vpslldq $8, %xmm2, %xmm1 + vpsrldq $8, %xmm2, %xmm2 + vpxor %xmm0, %xmm1, %xmm1 + vpclmulqdq $0x11, %xmm5, %xmm6, %xmm6 + vpclmulqdq $16, L_avx2_aes_gcm_mod2_128(%rip), %xmm1, %xmm0 + vpshufd $0x4e, %xmm1, %xmm1 + vpxor %xmm0, %xmm1, %xmm1 + vpclmulqdq $16, L_avx2_aes_gcm_mod2_128(%rip), %xmm1, %xmm0 + vpshufd $0x4e, %xmm1, %xmm1 + vpxor %xmm2, %xmm6, %xmm6 + vpxor %xmm1, %xmm6, %xmm6 + vpxor %xmm0, %xmm6, %xmm6 +L_AES_GCM_encrypt_avx2_done_enc: + # calc_tag + shlq $3, %r10 + vpinsrq $0x00, %r10, %xmm0, %xmm0 + shlq $3, %r11 + vpinsrq $0x01, %r11, %xmm1, %xmm1 + vpblendd $12, %xmm1, %xmm0, %xmm0 + vpxor %xmm6, %xmm0, %xmm0 + # ghash_gfmul_red + vpclmulqdq $16, %xmm5, %xmm0, %xmm4 + vpclmulqdq $0x01, %xmm5, %xmm0, %xmm3 + vpclmulqdq $0x00, %xmm5, %xmm0, %xmm2 + vpxor %xmm3, %xmm4, %xmm4 + vpslldq $8, %xmm4, %xmm3 + vpsrldq $8, %xmm4, %xmm4 + vpxor %xmm2, %xmm3, %xmm3 + vpclmulqdq $0x11, %xmm5, %xmm0, %xmm0 + vpclmulqdq $16, L_avx2_aes_gcm_mod2_128(%rip), %xmm3, %xmm2 + vpshufd $0x4e, %xmm3, %xmm3 + vpxor %xmm2, %xmm3, %xmm3 + vpclmulqdq $16, L_avx2_aes_gcm_mod2_128(%rip), %xmm3, %xmm2 + vpshufd $0x4e, %xmm3, %xmm3 + vpxor %xmm4, %xmm0, %xmm0 + vpxor %xmm3, %xmm0, %xmm0 + vpxor %xmm2, %xmm0, %xmm0 + vpshufb L_avx2_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0 + vpxor %xmm15, %xmm0, %xmm0 + # store_tag + cmpl $16, %r14d + je L_AES_GCM_encrypt_avx2_store_tag_16 + xorq %rcx, %rcx + vmovdqa %xmm0, (%rsp) +L_AES_GCM_encrypt_avx2_store_tag_loop: + movzbl (%rsp,%rcx,1), %r13d + movb %r13b, (%r15,%rcx,1) + incl %ecx + cmpl %r14d, %ecx + jne L_AES_GCM_encrypt_avx2_store_tag_loop + jmp L_AES_GCM_encrypt_avx2_store_tag_done +L_AES_GCM_encrypt_avx2_store_tag_16: + vmovdqu %xmm0, (%r15) +L_AES_GCM_encrypt_avx2_store_tag_done: + vzeroupper + addq $0xa0, %rsp + popq %r14 + popq %rbx + popq %r15 + popq %r12 + popq %r13 + repz retq +.size AES_GCM_encrypt_avx2,.-AES_GCM_encrypt_avx2 +.globl AES_GCM_decrypt_avx2 +.type AES_GCM_decrypt_avx2,@function +.align 4 +AES_GCM_decrypt_avx2: + pushq %r13 + pushq %r12 + pushq %r14 + pushq %rbx + pushq %r15 + pushq %rbp + movq %rdx, %r12 + movq %rcx, %rax + movq %r8, %r14 + movq %rsi, %r8 + movq %r9, %r10 + movq 56(%rsp), %r11 + movq 64(%rsp), %rbx + movq 72(%rsp), %r15 + movq 80(%rsp), %rsi + movq 88(%rsp), %r9 + movq 96(%rsp), %rbp + subq $0xa8, %rsp + vpxor %xmm4, %xmm4, %xmm4 + vpxor %xmm6, %xmm6, %xmm6 + movl %ebx, %edx + cmpl $12, %edx + je L_AES_GCM_decrypt_avx2_iv_12 + # Calculate values when IV is not 12 bytes + # H = Encrypt X(=0) + vmovdqa (%rsi), %xmm5 + vaesenc 16(%rsi), %xmm5, %xmm5 + vaesenc 32(%rsi), %xmm5, %xmm5 + vaesenc 48(%rsi), %xmm5, %xmm5 + vaesenc 64(%rsi), %xmm5, %xmm5 + vaesenc 80(%rsi), %xmm5, %xmm5 + vaesenc 96(%rsi), %xmm5, %xmm5 + vaesenc 112(%rsi), %xmm5, %xmm5 + vaesenc 128(%rsi), %xmm5, %xmm5 + vaesenc 144(%rsi), %xmm5, %xmm5 + cmpl $11, %r9d + vmovdqa 160(%rsi), %xmm0 + jl L_AES_GCM_decrypt_avx2_calc_iv_1_aesenc_avx_last + vaesenc %xmm0, %xmm5, %xmm5 + vaesenc 176(%rsi), %xmm5, %xmm5 + cmpl $13, %r9d + vmovdqa 192(%rsi), %xmm0 + jl L_AES_GCM_decrypt_avx2_calc_iv_1_aesenc_avx_last + vaesenc %xmm0, %xmm5, %xmm5 + vaesenc 208(%rsi), %xmm5, %xmm5 + vmovdqa 224(%rsi), %xmm0 +L_AES_GCM_decrypt_avx2_calc_iv_1_aesenc_avx_last: + vaesenclast %xmm0, %xmm5, %xmm5 + vpshufb L_avx2_aes_gcm_bswap_mask(%rip), %xmm5, %xmm5 + # Calc counter + # Initialization vector + cmpl $0x00, %edx + movq $0x00, %rcx + je L_AES_GCM_decrypt_avx2_calc_iv_done + cmpl $16, %edx + jl L_AES_GCM_decrypt_avx2_calc_iv_lt16 + andl $0xfffffff0, %edx +L_AES_GCM_decrypt_avx2_calc_iv_16_loop: + vmovdqu (%rax,%rcx,1), %xmm0 + vpshufb L_avx2_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0 + vpxor %xmm0, %xmm4, %xmm4 + # ghash_gfmul_avx + vpclmulqdq $16, %xmm4, %xmm5, %xmm2 + vpclmulqdq $0x01, %xmm4, %xmm5, %xmm1 + vpclmulqdq $0x00, %xmm4, %xmm5, %xmm0 + vpclmulqdq $0x11, %xmm4, %xmm5, %xmm3 + vpxor %xmm1, %xmm2, %xmm2 + vpslldq $8, %xmm2, %xmm1 + vpsrldq $8, %xmm2, %xmm2 + vpxor %xmm1, %xmm0, %xmm7 + vpxor %xmm2, %xmm3, %xmm4 + # ghash_mid + vpsrld $31, %xmm7, %xmm0 + vpsrld $31, %xmm4, %xmm1 + vpslld $0x01, %xmm7, %xmm7 + vpslld $0x01, %xmm4, %xmm4 + vpsrldq $12, %xmm0, %xmm2 + vpslldq $4, %xmm0, %xmm0 + vpslldq $4, %xmm1, %xmm1 + vpor %xmm2, %xmm4, %xmm4 + vpor %xmm0, %xmm7, %xmm7 + vpor %xmm1, %xmm4, %xmm4 + # ghash_red + vmovdqa L_avx2_aes_gcm_mod2_128(%rip), %xmm2 + vpclmulqdq $16, %xmm2, %xmm7, %xmm0 + vpshufd $0x4e, %xmm7, %xmm1 + vpxor %xmm0, %xmm1, %xmm1 + vpclmulqdq $16, %xmm2, %xmm1, %xmm0 + vpshufd $0x4e, %xmm1, %xmm1 + vpxor %xmm0, %xmm1, %xmm1 + vpxor %xmm1, %xmm4, %xmm4 + addl $16, %ecx + cmpl %edx, %ecx + jl L_AES_GCM_decrypt_avx2_calc_iv_16_loop + movl %ebx, %edx + cmpl %edx, %ecx + je L_AES_GCM_decrypt_avx2_calc_iv_done +L_AES_GCM_decrypt_avx2_calc_iv_lt16: + vpxor %xmm0, %xmm0, %xmm0 + xorl %ebx, %ebx + vmovdqa %xmm0, (%rsp) +L_AES_GCM_decrypt_avx2_calc_iv_loop: + movzbl (%rax,%rcx,1), %r13d + movb %r13b, (%rsp,%rbx,1) + incl %ecx + incl %ebx + cmpl %edx, %ecx + jl L_AES_GCM_decrypt_avx2_calc_iv_loop + vmovdqa (%rsp), %xmm0 + vpshufb L_avx2_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0 + vpxor %xmm0, %xmm4, %xmm4 + # ghash_gfmul_avx + vpclmulqdq $16, %xmm4, %xmm5, %xmm2 + vpclmulqdq $0x01, %xmm4, %xmm5, %xmm1 + vpclmulqdq $0x00, %xmm4, %xmm5, %xmm0 + vpclmulqdq $0x11, %xmm4, %xmm5, %xmm3 + vpxor %xmm1, %xmm2, %xmm2 + vpslldq $8, %xmm2, %xmm1 + vpsrldq $8, %xmm2, %xmm2 + vpxor %xmm1, %xmm0, %xmm7 + vpxor %xmm2, %xmm3, %xmm4 + # ghash_mid + vpsrld $31, %xmm7, %xmm0 + vpsrld $31, %xmm4, %xmm1 + vpslld $0x01, %xmm7, %xmm7 + vpslld $0x01, %xmm4, %xmm4 + vpsrldq $12, %xmm0, %xmm2 + vpslldq $4, %xmm0, %xmm0 + vpslldq $4, %xmm1, %xmm1 + vpor %xmm2, %xmm4, %xmm4 + vpor %xmm0, %xmm7, %xmm7 + vpor %xmm1, %xmm4, %xmm4 + # ghash_red + vmovdqa L_avx2_aes_gcm_mod2_128(%rip), %xmm2 + vpclmulqdq $16, %xmm2, %xmm7, %xmm0 + vpshufd $0x4e, %xmm7, %xmm1 + vpxor %xmm0, %xmm1, %xmm1 + vpclmulqdq $16, %xmm2, %xmm1, %xmm0 + vpshufd $0x4e, %xmm1, %xmm1 + vpxor %xmm0, %xmm1, %xmm1 + vpxor %xmm1, %xmm4, %xmm4 +L_AES_GCM_decrypt_avx2_calc_iv_done: + # T = Encrypt counter + vpxor %xmm0, %xmm0, %xmm0 + shll $3, %edx + vpinsrq $0x00, %rdx, %xmm0, %xmm0 + vpxor %xmm0, %xmm4, %xmm4 + # ghash_gfmul_avx + vpclmulqdq $16, %xmm4, %xmm5, %xmm2 + vpclmulqdq $0x01, %xmm4, %xmm5, %xmm1 + vpclmulqdq $0x00, %xmm4, %xmm5, %xmm0 + vpclmulqdq $0x11, %xmm4, %xmm5, %xmm3 + vpxor %xmm1, %xmm2, %xmm2 + vpslldq $8, %xmm2, %xmm1 + vpsrldq $8, %xmm2, %xmm2 + vpxor %xmm1, %xmm0, %xmm7 + vpxor %xmm2, %xmm3, %xmm4 + # ghash_mid + vpsrld $31, %xmm7, %xmm0 + vpsrld $31, %xmm4, %xmm1 + vpslld $0x01, %xmm7, %xmm7 + vpslld $0x01, %xmm4, %xmm4 + vpsrldq $12, %xmm0, %xmm2 + vpslldq $4, %xmm0, %xmm0 + vpslldq $4, %xmm1, %xmm1 + vpor %xmm2, %xmm4, %xmm4 + vpor %xmm0, %xmm7, %xmm7 + vpor %xmm1, %xmm4, %xmm4 + # ghash_red + vmovdqa L_avx2_aes_gcm_mod2_128(%rip), %xmm2 + vpclmulqdq $16, %xmm2, %xmm7, %xmm0 + vpshufd $0x4e, %xmm7, %xmm1 + vpxor %xmm0, %xmm1, %xmm1 + vpclmulqdq $16, %xmm2, %xmm1, %xmm0 + vpshufd $0x4e, %xmm1, %xmm1 + vpxor %xmm0, %xmm1, %xmm1 + vpxor %xmm1, %xmm4, %xmm4 + vpshufb L_avx2_aes_gcm_bswap_mask(%rip), %xmm4, %xmm4 + # Encrypt counter + vmovdqa (%rsi), %xmm15 + vpxor %xmm4, %xmm15, %xmm15 + vaesenc 16(%rsi), %xmm15, %xmm15 + vaesenc 32(%rsi), %xmm15, %xmm15 + vaesenc 48(%rsi), %xmm15, %xmm15 + vaesenc 64(%rsi), %xmm15, %xmm15 + vaesenc 80(%rsi), %xmm15, %xmm15 + vaesenc 96(%rsi), %xmm15, %xmm15 + vaesenc 112(%rsi), %xmm15, %xmm15 + vaesenc 128(%rsi), %xmm15, %xmm15 + vaesenc 144(%rsi), %xmm15, %xmm15 + cmpl $11, %r9d + vmovdqa 160(%rsi), %xmm0 + jl L_AES_GCM_decrypt_avx2_calc_iv_2_aesenc_avx_last + vaesenc %xmm0, %xmm15, %xmm15 + vaesenc 176(%rsi), %xmm15, %xmm15 + cmpl $13, %r9d + vmovdqa 192(%rsi), %xmm0 + jl L_AES_GCM_decrypt_avx2_calc_iv_2_aesenc_avx_last + vaesenc %xmm0, %xmm15, %xmm15 + vaesenc 208(%rsi), %xmm15, %xmm15 + vmovdqa 224(%rsi), %xmm0 +L_AES_GCM_decrypt_avx2_calc_iv_2_aesenc_avx_last: + vaesenclast %xmm0, %xmm15, %xmm15 + jmp L_AES_GCM_decrypt_avx2_iv_done +L_AES_GCM_decrypt_avx2_iv_12: + # # Calculate values when IV is 12 bytes + # Set counter based on IV + vmovdqa L_avx2_aes_gcm_bswap_one(%rip), %xmm4 + vmovdqa (%rsi), %xmm5 + vpblendd $7, (%rax), %xmm4, %xmm4 + # H = Encrypt X(=0) and T = Encrypt counter + vmovdqa 16(%rsi), %xmm7 + vpxor %xmm5, %xmm4, %xmm15 + vaesenc %xmm7, %xmm5, %xmm5 + vaesenc %xmm7, %xmm15, %xmm15 + vmovdqa 32(%rsi), %xmm0 + vaesenc %xmm0, %xmm5, %xmm5 + vaesenc %xmm0, %xmm15, %xmm15 + vmovdqa 48(%rsi), %xmm0 + vaesenc %xmm0, %xmm5, %xmm5 + vaesenc %xmm0, %xmm15, %xmm15 + vmovdqa 64(%rsi), %xmm0 + vaesenc %xmm0, %xmm5, %xmm5 + vaesenc %xmm0, %xmm15, %xmm15 + vmovdqa 80(%rsi), %xmm0 + vaesenc %xmm0, %xmm5, %xmm5 + vaesenc %xmm0, %xmm15, %xmm15 + vmovdqa 96(%rsi), %xmm0 + vaesenc %xmm0, %xmm5, %xmm5 + vaesenc %xmm0, %xmm15, %xmm15 + vmovdqa 112(%rsi), %xmm0 + vaesenc %xmm0, %xmm5, %xmm5 + vaesenc %xmm0, %xmm15, %xmm15 + vmovdqa 128(%rsi), %xmm0 + vaesenc %xmm0, %xmm5, %xmm5 + vaesenc %xmm0, %xmm15, %xmm15 + vmovdqa 144(%rsi), %xmm0 + vaesenc %xmm0, %xmm5, %xmm5 + vaesenc %xmm0, %xmm15, %xmm15 + cmpl $11, %r9d + vmovdqa 160(%rsi), %xmm0 + jl L_AES_GCM_decrypt_avx2_calc_iv_12_last + vaesenc %xmm0, %xmm5, %xmm5 + vaesenc %xmm0, %xmm15, %xmm15 + vmovdqa 176(%rsi), %xmm0 + vaesenc %xmm0, %xmm5, %xmm5 + vaesenc %xmm0, %xmm15, %xmm15 + cmpl $13, %r9d + vmovdqa 192(%rsi), %xmm0 + jl L_AES_GCM_decrypt_avx2_calc_iv_12_last + vaesenc %xmm0, %xmm5, %xmm5 + vaesenc %xmm0, %xmm15, %xmm15 + vmovdqa 208(%rsi), %xmm0 + vaesenc %xmm0, %xmm5, %xmm5 + vaesenc %xmm0, %xmm15, %xmm15 + vmovdqa 224(%rsi), %xmm0 +L_AES_GCM_decrypt_avx2_calc_iv_12_last: + vaesenclast %xmm0, %xmm5, %xmm5 + vaesenclast %xmm0, %xmm15, %xmm15 + vpshufb L_avx2_aes_gcm_bswap_mask(%rip), %xmm5, %xmm5 +L_AES_GCM_decrypt_avx2_iv_done: + # Additional authentication data + movl %r11d, %edx + cmpl $0x00, %edx + je L_AES_GCM_decrypt_avx2_calc_aad_done + xorl %ecx, %ecx + cmpl $16, %edx + jl L_AES_GCM_decrypt_avx2_calc_aad_lt16 + andl $0xfffffff0, %edx +L_AES_GCM_decrypt_avx2_calc_aad_16_loop: + vmovdqu (%r12,%rcx,1), %xmm0 + vpshufb L_avx2_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0 + vpxor %xmm0, %xmm6, %xmm6 + # ghash_gfmul_avx + vpclmulqdq $16, %xmm6, %xmm5, %xmm2 + vpclmulqdq $0x01, %xmm6, %xmm5, %xmm1 + vpclmulqdq $0x00, %xmm6, %xmm5, %xmm0 + vpclmulqdq $0x11, %xmm6, %xmm5, %xmm3 + vpxor %xmm1, %xmm2, %xmm2 + vpslldq $8, %xmm2, %xmm1 + vpsrldq $8, %xmm2, %xmm2 + vpxor %xmm1, %xmm0, %xmm7 + vpxor %xmm2, %xmm3, %xmm6 + # ghash_mid + vpsrld $31, %xmm7, %xmm0 + vpsrld $31, %xmm6, %xmm1 + vpslld $0x01, %xmm7, %xmm7 + vpslld $0x01, %xmm6, %xmm6 + vpsrldq $12, %xmm0, %xmm2 + vpslldq $4, %xmm0, %xmm0 + vpslldq $4, %xmm1, %xmm1 + vpor %xmm2, %xmm6, %xmm6 + vpor %xmm0, %xmm7, %xmm7 + vpor %xmm1, %xmm6, %xmm6 + # ghash_red + vmovdqa L_avx2_aes_gcm_mod2_128(%rip), %xmm2 + vpclmulqdq $16, %xmm2, %xmm7, %xmm0 + vpshufd $0x4e, %xmm7, %xmm1 + vpxor %xmm0, %xmm1, %xmm1 + vpclmulqdq $16, %xmm2, %xmm1, %xmm0 + vpshufd $0x4e, %xmm1, %xmm1 + vpxor %xmm0, %xmm1, %xmm1 + vpxor %xmm1, %xmm6, %xmm6 + addl $16, %ecx + cmpl %edx, %ecx + jl L_AES_GCM_decrypt_avx2_calc_aad_16_loop + movl %r11d, %edx + cmpl %edx, %ecx + je L_AES_GCM_decrypt_avx2_calc_aad_done +L_AES_GCM_decrypt_avx2_calc_aad_lt16: + vpxor %xmm0, %xmm0, %xmm0 + xorl %ebx, %ebx + vmovdqa %xmm0, (%rsp) +L_AES_GCM_decrypt_avx2_calc_aad_loop: + movzbl (%r12,%rcx,1), %r13d + movb %r13b, (%rsp,%rbx,1) + incl %ecx + incl %ebx + cmpl %edx, %ecx + jl L_AES_GCM_decrypt_avx2_calc_aad_loop + vmovdqa (%rsp), %xmm0 + vpshufb L_avx2_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0 + vpxor %xmm0, %xmm6, %xmm6 + # ghash_gfmul_avx + vpclmulqdq $16, %xmm6, %xmm5, %xmm2 + vpclmulqdq $0x01, %xmm6, %xmm5, %xmm1 + vpclmulqdq $0x00, %xmm6, %xmm5, %xmm0 + vpclmulqdq $0x11, %xmm6, %xmm5, %xmm3 + vpxor %xmm1, %xmm2, %xmm2 + vpslldq $8, %xmm2, %xmm1 + vpsrldq $8, %xmm2, %xmm2 + vpxor %xmm1, %xmm0, %xmm7 + vpxor %xmm2, %xmm3, %xmm6 + # ghash_mid + vpsrld $31, %xmm7, %xmm0 + vpsrld $31, %xmm6, %xmm1 + vpslld $0x01, %xmm7, %xmm7 + vpslld $0x01, %xmm6, %xmm6 + vpsrldq $12, %xmm0, %xmm2 + vpslldq $4, %xmm0, %xmm0 + vpslldq $4, %xmm1, %xmm1 + vpor %xmm2, %xmm6, %xmm6 + vpor %xmm0, %xmm7, %xmm7 + vpor %xmm1, %xmm6, %xmm6 + # ghash_red + vmovdqa L_avx2_aes_gcm_mod2_128(%rip), %xmm2 + vpclmulqdq $16, %xmm2, %xmm7, %xmm0 + vpshufd $0x4e, %xmm7, %xmm1 + vpxor %xmm0, %xmm1, %xmm1 + vpclmulqdq $16, %xmm2, %xmm1, %xmm0 + vpshufd $0x4e, %xmm1, %xmm1 + vpxor %xmm0, %xmm1, %xmm1 + vpxor %xmm1, %xmm6, %xmm6 +L_AES_GCM_decrypt_avx2_calc_aad_done: + # Calculate counter and H + vpsrlq $63, %xmm5, %xmm1 + vpsllq $0x01, %xmm5, %xmm0 + vpslldq $8, %xmm1, %xmm1 + vpor %xmm1, %xmm0, %xmm0 + vpshufd $0xff, %xmm5, %xmm5 + vpsrad $31, %xmm5, %xmm5 + vpshufb L_avx2_aes_gcm_bswap_epi64(%rip), %xmm4, %xmm4 + vpand L_avx2_aes_gcm_mod2_128(%rip), %xmm5, %xmm5 + vpaddd L_avx2_aes_gcm_one(%rip), %xmm4, %xmm4 + vpxor %xmm0, %xmm5, %xmm5 + xorl %ebx, %ebx + cmpl $0x80, %r10d + movl %r10d, %r13d + jl L_AES_GCM_decrypt_avx2_done_128 + andl $0xffffff80, %r13d + vmovdqa %xmm4, 128(%rsp) + vmovdqa %xmm15, 144(%rsp) + vmovdqa L_avx2_aes_gcm_mod2_128(%rip), %xmm3 + # H ^ 1 and H ^ 2 + vpclmulqdq $0x00, %xmm5, %xmm5, %xmm9 + vpclmulqdq $0x11, %xmm5, %xmm5, %xmm10 + vpclmulqdq $16, %xmm3, %xmm9, %xmm8 + vpshufd $0x4e, %xmm9, %xmm9 + vpxor %xmm8, %xmm9, %xmm9 + vpclmulqdq $16, %xmm3, %xmm9, %xmm8 + vpshufd $0x4e, %xmm9, %xmm9 + vpxor %xmm8, %xmm9, %xmm9 + vpxor %xmm9, %xmm10, %xmm0 + vmovdqa %xmm5, (%rsp) + vmovdqa %xmm0, 16(%rsp) + # H ^ 3 and H ^ 4 + vpclmulqdq $16, %xmm5, %xmm0, %xmm11 + vpclmulqdq $0x01, %xmm5, %xmm0, %xmm10 + vpclmulqdq $0x00, %xmm5, %xmm0, %xmm9 + vpclmulqdq $0x11, %xmm5, %xmm0, %xmm12 + vpclmulqdq $0x00, %xmm0, %xmm0, %xmm13 + vpclmulqdq $0x11, %xmm0, %xmm0, %xmm14 + vpxor %xmm10, %xmm11, %xmm11 + vpslldq $8, %xmm11, %xmm10 + vpsrldq $8, %xmm11, %xmm11 + vpxor %xmm9, %xmm10, %xmm10 + vpclmulqdq $16, %xmm3, %xmm13, %xmm8 + vpclmulqdq $16, %xmm3, %xmm10, %xmm9 + vpshufd $0x4e, %xmm10, %xmm10 + vpshufd $0x4e, %xmm13, %xmm13 + vpxor %xmm9, %xmm10, %xmm10 + vpxor %xmm8, %xmm13, %xmm13 + vpclmulqdq $16, %xmm3, %xmm10, %xmm9 + vpclmulqdq $16, %xmm3, %xmm13, %xmm8 + vpshufd $0x4e, %xmm10, %xmm10 + vpshufd $0x4e, %xmm13, %xmm13 + vpxor %xmm11, %xmm12, %xmm12 + vpxor %xmm8, %xmm13, %xmm13 + vpxor %xmm12, %xmm10, %xmm10 + vpxor %xmm14, %xmm13, %xmm2 + vpxor %xmm9, %xmm10, %xmm1 + vmovdqa %xmm1, 32(%rsp) + vmovdqa %xmm2, 48(%rsp) + # H ^ 5 and H ^ 6 + vpclmulqdq $16, %xmm0, %xmm1, %xmm11 + vpclmulqdq $0x01, %xmm0, %xmm1, %xmm10 + vpclmulqdq $0x00, %xmm0, %xmm1, %xmm9 + vpclmulqdq $0x11, %xmm0, %xmm1, %xmm12 + vpclmulqdq $0x00, %xmm1, %xmm1, %xmm13 + vpclmulqdq $0x11, %xmm1, %xmm1, %xmm14 + vpxor %xmm10, %xmm11, %xmm11 + vpslldq $8, %xmm11, %xmm10 + vpsrldq $8, %xmm11, %xmm11 + vpxor %xmm9, %xmm10, %xmm10 + vpclmulqdq $16, %xmm3, %xmm13, %xmm8 + vpclmulqdq $16, %xmm3, %xmm10, %xmm9 + vpshufd $0x4e, %xmm10, %xmm10 + vpshufd $0x4e, %xmm13, %xmm13 + vpxor %xmm9, %xmm10, %xmm10 + vpxor %xmm8, %xmm13, %xmm13 + vpclmulqdq $16, %xmm3, %xmm10, %xmm9 + vpclmulqdq $16, %xmm3, %xmm13, %xmm8 + vpshufd $0x4e, %xmm10, %xmm10 + vpshufd $0x4e, %xmm13, %xmm13 + vpxor %xmm11, %xmm12, %xmm12 + vpxor %xmm8, %xmm13, %xmm13 + vpxor %xmm12, %xmm10, %xmm10 + vpxor %xmm14, %xmm13, %xmm0 + vpxor %xmm9, %xmm10, %xmm7 + vmovdqa %xmm7, 64(%rsp) + vmovdqa %xmm0, 80(%rsp) + # H ^ 7 and H ^ 8 + vpclmulqdq $16, %xmm1, %xmm2, %xmm11 + vpclmulqdq $0x01, %xmm1, %xmm2, %xmm10 + vpclmulqdq $0x00, %xmm1, %xmm2, %xmm9 + vpclmulqdq $0x11, %xmm1, %xmm2, %xmm12 + vpclmulqdq $0x00, %xmm2, %xmm2, %xmm13 + vpclmulqdq $0x11, %xmm2, %xmm2, %xmm14 + vpxor %xmm10, %xmm11, %xmm11 + vpslldq $8, %xmm11, %xmm10 + vpsrldq $8, %xmm11, %xmm11 + vpxor %xmm9, %xmm10, %xmm10 + vpclmulqdq $16, %xmm3, %xmm13, %xmm8 + vpclmulqdq $16, %xmm3, %xmm10, %xmm9 + vpshufd $0x4e, %xmm10, %xmm10 + vpshufd $0x4e, %xmm13, %xmm13 + vpxor %xmm9, %xmm10, %xmm10 + vpxor %xmm8, %xmm13, %xmm13 + vpclmulqdq $16, %xmm3, %xmm10, %xmm9 + vpclmulqdq $16, %xmm3, %xmm13, %xmm8 + vpshufd $0x4e, %xmm10, %xmm10 + vpshufd $0x4e, %xmm13, %xmm13 + vpxor %xmm11, %xmm12, %xmm12 + vpxor %xmm8, %xmm13, %xmm13 + vpxor %xmm12, %xmm10, %xmm10 + vpxor %xmm14, %xmm13, %xmm0 + vpxor %xmm9, %xmm10, %xmm7 + vmovdqa %xmm7, 96(%rsp) + vmovdqa %xmm0, 112(%rsp) +L_AES_GCM_decrypt_avx2_ghash_128: + # aesenc_128_ghash + leaq (%rdi,%rbx,1), %rcx + leaq (%r8,%rbx,1), %rdx + # aesenc_ctr + vmovdqa 128(%rsp), %xmm0 + vmovdqa L_avx2_aes_gcm_bswap_epi64(%rip), %xmm1 + vpaddd L_avx2_aes_gcm_one(%rip), %xmm0, %xmm9 + vpshufb %xmm1, %xmm0, %xmm8 + vpaddd L_avx2_aes_gcm_two(%rip), %xmm0, %xmm10 + vpshufb %xmm1, %xmm9, %xmm9 + vpaddd L_avx2_aes_gcm_three(%rip), %xmm0, %xmm11 + vpshufb %xmm1, %xmm10, %xmm10 + vpaddd L_avx2_aes_gcm_four(%rip), %xmm0, %xmm12 + vpshufb %xmm1, %xmm11, %xmm11 + vpaddd L_avx2_aes_gcm_five(%rip), %xmm0, %xmm13 + vpshufb %xmm1, %xmm12, %xmm12 + vpaddd L_avx2_aes_gcm_six(%rip), %xmm0, %xmm14 + vpshufb %xmm1, %xmm13, %xmm13 + vpaddd L_avx2_aes_gcm_seven(%rip), %xmm0, %xmm15 + vpshufb %xmm1, %xmm14, %xmm14 + vpaddd L_avx2_aes_gcm_eight(%rip), %xmm0, %xmm0 + vpshufb %xmm1, %xmm15, %xmm15 + # aesenc_xor + vmovdqa (%rsi), %xmm7 + vmovdqa %xmm0, 128(%rsp) + vpxor %xmm7, %xmm8, %xmm8 + vpxor %xmm7, %xmm9, %xmm9 + vpxor %xmm7, %xmm10, %xmm10 + vpxor %xmm7, %xmm11, %xmm11 + vpxor %xmm7, %xmm12, %xmm12 + vpxor %xmm7, %xmm13, %xmm13 + vpxor %xmm7, %xmm14, %xmm14 + vpxor %xmm7, %xmm15, %xmm15 + # aesenc_pclmul_1 + vmovdqu (%rcx), %xmm1 + vmovdqu 16(%rsi), %xmm0 + vpshufb L_avx2_aes_gcm_bswap_mask(%rip), %xmm1, %xmm1 + vmovdqa 112(%rsp), %xmm2 + vpxor %xmm6, %xmm1, %xmm1 + vpclmulqdq $16, %xmm2, %xmm1, %xmm5 + vpclmulqdq $0x01, %xmm2, %xmm1, %xmm3 + vpclmulqdq $0x00, %xmm2, %xmm1, %xmm6 + vpclmulqdq $0x11, %xmm2, %xmm1, %xmm7 + vaesenc %xmm0, %xmm8, %xmm8 + vaesenc %xmm0, %xmm9, %xmm9 + vaesenc %xmm0, %xmm10, %xmm10 + vaesenc %xmm0, %xmm11, %xmm11 + vaesenc %xmm0, %xmm12, %xmm12 + vaesenc %xmm0, %xmm13, %xmm13 + vaesenc %xmm0, %xmm14, %xmm14 + vaesenc %xmm0, %xmm15, %xmm15 + # aesenc_pclmul_2 + vmovdqu 16(%rcx), %xmm1 + vmovdqa 96(%rsp), %xmm0 + vpshufb L_avx2_aes_gcm_bswap_mask(%rip), %xmm1, %xmm1 + vpxor %xmm3, %xmm5, %xmm5 + vpclmulqdq $16, %xmm0, %xmm1, %xmm2 + vpclmulqdq $0x01, %xmm0, %xmm1, %xmm3 + vpclmulqdq $0x00, %xmm0, %xmm1, %xmm4 + vpclmulqdq $0x11, %xmm0, %xmm1, %xmm1 + vmovdqu 32(%rsi), %xmm0 + vpxor %xmm1, %xmm7, %xmm7 + vaesenc %xmm0, %xmm8, %xmm8 + vaesenc %xmm0, %xmm9, %xmm9 + vaesenc %xmm0, %xmm10, %xmm10 + vaesenc %xmm0, %xmm11, %xmm11 + vaesenc %xmm0, %xmm12, %xmm12 + vaesenc %xmm0, %xmm13, %xmm13 + vaesenc %xmm0, %xmm14, %xmm14 + vaesenc %xmm0, %xmm15, %xmm15 + # aesenc_pclmul_n + vmovdqu 32(%rcx), %xmm1 + vmovdqa 80(%rsp), %xmm0 + vpshufb L_avx2_aes_gcm_bswap_mask(%rip), %xmm1, %xmm1 + vpxor %xmm2, %xmm5, %xmm5 + vpclmulqdq $16, %xmm0, %xmm1, %xmm2 + vpxor %xmm3, %xmm5, %xmm5 + vpclmulqdq $0x01, %xmm0, %xmm1, %xmm3 + vpxor %xmm4, %xmm6, %xmm6 + vpclmulqdq $0x00, %xmm0, %xmm1, %xmm4 + vpclmulqdq $0x11, %xmm0, %xmm1, %xmm1 + vmovdqu 48(%rsi), %xmm0 + vpxor %xmm1, %xmm7, %xmm7 + vaesenc %xmm0, %xmm8, %xmm8 + vaesenc %xmm0, %xmm9, %xmm9 + vaesenc %xmm0, %xmm10, %xmm10 + vaesenc %xmm0, %xmm11, %xmm11 + vaesenc %xmm0, %xmm12, %xmm12 + vaesenc %xmm0, %xmm13, %xmm13 + vaesenc %xmm0, %xmm14, %xmm14 + vaesenc %xmm0, %xmm15, %xmm15 + # aesenc_pclmul_n + vmovdqu 48(%rcx), %xmm1 + vmovdqa 64(%rsp), %xmm0 + vpshufb L_avx2_aes_gcm_bswap_mask(%rip), %xmm1, %xmm1 + vpxor %xmm2, %xmm5, %xmm5 + vpclmulqdq $16, %xmm0, %xmm1, %xmm2 + vpxor %xmm3, %xmm5, %xmm5 + vpclmulqdq $0x01, %xmm0, %xmm1, %xmm3 + vpxor %xmm4, %xmm6, %xmm6 + vpclmulqdq $0x00, %xmm0, %xmm1, %xmm4 + vpclmulqdq $0x11, %xmm0, %xmm1, %xmm1 + vmovdqu 64(%rsi), %xmm0 + vpxor %xmm1, %xmm7, %xmm7 + vaesenc %xmm0, %xmm8, %xmm8 + vaesenc %xmm0, %xmm9, %xmm9 + vaesenc %xmm0, %xmm10, %xmm10 + vaesenc %xmm0, %xmm11, %xmm11 + vaesenc %xmm0, %xmm12, %xmm12 + vaesenc %xmm0, %xmm13, %xmm13 + vaesenc %xmm0, %xmm14, %xmm14 + vaesenc %xmm0, %xmm15, %xmm15 + # aesenc_pclmul_n + vmovdqu 64(%rcx), %xmm1 + vmovdqa 48(%rsp), %xmm0 + vpshufb L_avx2_aes_gcm_bswap_mask(%rip), %xmm1, %xmm1 + vpxor %xmm2, %xmm5, %xmm5 + vpclmulqdq $16, %xmm0, %xmm1, %xmm2 + vpxor %xmm3, %xmm5, %xmm5 + vpclmulqdq $0x01, %xmm0, %xmm1, %xmm3 + vpxor %xmm4, %xmm6, %xmm6 + vpclmulqdq $0x00, %xmm0, %xmm1, %xmm4 + vpclmulqdq $0x11, %xmm0, %xmm1, %xmm1 + vmovdqu 80(%rsi), %xmm0 + vpxor %xmm1, %xmm7, %xmm7 + vaesenc %xmm0, %xmm8, %xmm8 + vaesenc %xmm0, %xmm9, %xmm9 + vaesenc %xmm0, %xmm10, %xmm10 + vaesenc %xmm0, %xmm11, %xmm11 + vaesenc %xmm0, %xmm12, %xmm12 + vaesenc %xmm0, %xmm13, %xmm13 + vaesenc %xmm0, %xmm14, %xmm14 + vaesenc %xmm0, %xmm15, %xmm15 + # aesenc_pclmul_n + vmovdqu 80(%rcx), %xmm1 + vmovdqa 32(%rsp), %xmm0 + vpshufb L_avx2_aes_gcm_bswap_mask(%rip), %xmm1, %xmm1 + vpxor %xmm2, %xmm5, %xmm5 + vpclmulqdq $16, %xmm0, %xmm1, %xmm2 + vpxor %xmm3, %xmm5, %xmm5 + vpclmulqdq $0x01, %xmm0, %xmm1, %xmm3 + vpxor %xmm4, %xmm6, %xmm6 + vpclmulqdq $0x00, %xmm0, %xmm1, %xmm4 + vpclmulqdq $0x11, %xmm0, %xmm1, %xmm1 + vmovdqu 96(%rsi), %xmm0 + vpxor %xmm1, %xmm7, %xmm7 + vaesenc %xmm0, %xmm8, %xmm8 + vaesenc %xmm0, %xmm9, %xmm9 + vaesenc %xmm0, %xmm10, %xmm10 + vaesenc %xmm0, %xmm11, %xmm11 + vaesenc %xmm0, %xmm12, %xmm12 + vaesenc %xmm0, %xmm13, %xmm13 + vaesenc %xmm0, %xmm14, %xmm14 + vaesenc %xmm0, %xmm15, %xmm15 + # aesenc_pclmul_n + vmovdqu 96(%rcx), %xmm1 + vmovdqa 16(%rsp), %xmm0 + vpshufb L_avx2_aes_gcm_bswap_mask(%rip), %xmm1, %xmm1 + vpxor %xmm2, %xmm5, %xmm5 + vpclmulqdq $16, %xmm0, %xmm1, %xmm2 + vpxor %xmm3, %xmm5, %xmm5 + vpclmulqdq $0x01, %xmm0, %xmm1, %xmm3 + vpxor %xmm4, %xmm6, %xmm6 + vpclmulqdq $0x00, %xmm0, %xmm1, %xmm4 + vpclmulqdq $0x11, %xmm0, %xmm1, %xmm1 + vmovdqu 112(%rsi), %xmm0 + vpxor %xmm1, %xmm7, %xmm7 + vaesenc %xmm0, %xmm8, %xmm8 + vaesenc %xmm0, %xmm9, %xmm9 + vaesenc %xmm0, %xmm10, %xmm10 + vaesenc %xmm0, %xmm11, %xmm11 + vaesenc %xmm0, %xmm12, %xmm12 + vaesenc %xmm0, %xmm13, %xmm13 + vaesenc %xmm0, %xmm14, %xmm14 + vaesenc %xmm0, %xmm15, %xmm15 + # aesenc_pclmul_n + vmovdqu 112(%rcx), %xmm1 + vmovdqa (%rsp), %xmm0 + vpshufb L_avx2_aes_gcm_bswap_mask(%rip), %xmm1, %xmm1 + vpxor %xmm2, %xmm5, %xmm5 + vpclmulqdq $16, %xmm0, %xmm1, %xmm2 + vpxor %xmm3, %xmm5, %xmm5 + vpclmulqdq $0x01, %xmm0, %xmm1, %xmm3 + vpxor %xmm4, %xmm6, %xmm6 + vpclmulqdq $0x00, %xmm0, %xmm1, %xmm4 + vpclmulqdq $0x11, %xmm0, %xmm1, %xmm1 + vmovdqu 128(%rsi), %xmm0 + vpxor %xmm1, %xmm7, %xmm7 + vaesenc %xmm0, %xmm8, %xmm8 + vaesenc %xmm0, %xmm9, %xmm9 + vaesenc %xmm0, %xmm10, %xmm10 + vaesenc %xmm0, %xmm11, %xmm11 + vaesenc %xmm0, %xmm12, %xmm12 + vaesenc %xmm0, %xmm13, %xmm13 + vaesenc %xmm0, %xmm14, %xmm14 + vaesenc %xmm0, %xmm15, %xmm15 + # aesenc_pclmul_l + vpxor %xmm2, %xmm5, %xmm5 + vpxor %xmm4, %xmm6, %xmm6 + vpxor %xmm3, %xmm5, %xmm5 + vpslldq $8, %xmm5, %xmm1 + vpsrldq $8, %xmm5, %xmm5 + vmovdqa 144(%rsi), %xmm4 + vmovdqa L_avx2_aes_gcm_mod2_128(%rip), %xmm0 + vaesenc %xmm4, %xmm8, %xmm8 + vpxor %xmm1, %xmm6, %xmm6 + vpxor %xmm5, %xmm7, %xmm7 + vpclmulqdq $16, %xmm0, %xmm6, %xmm3 + vaesenc %xmm4, %xmm9, %xmm9 + vaesenc %xmm4, %xmm10, %xmm10 + vaesenc %xmm4, %xmm11, %xmm11 + vpshufd $0x4e, %xmm6, %xmm6 + vpxor %xmm3, %xmm6, %xmm6 + vpclmulqdq $16, %xmm0, %xmm6, %xmm3 + vaesenc %xmm4, %xmm12, %xmm12 + vaesenc %xmm4, %xmm13, %xmm13 + vaesenc %xmm4, %xmm14, %xmm14 + vpshufd $0x4e, %xmm6, %xmm6 + vpxor %xmm3, %xmm6, %xmm6 + vpxor %xmm7, %xmm6, %xmm6 + vaesenc %xmm4, %xmm15, %xmm15 + cmpl $11, %r9d + vmovdqa 160(%rsi), %xmm7 + jl L_AES_GCM_decrypt_avx2_aesenc_128_ghash_avx_done + vaesenc %xmm7, %xmm8, %xmm8 + vaesenc %xmm7, %xmm9, %xmm9 + vaesenc %xmm7, %xmm10, %xmm10 + vaesenc %xmm7, %xmm11, %xmm11 + vaesenc %xmm7, %xmm12, %xmm12 + vaesenc %xmm7, %xmm13, %xmm13 + vaesenc %xmm7, %xmm14, %xmm14 + vaesenc %xmm7, %xmm15, %xmm15 + vmovdqa 176(%rsi), %xmm7 + vaesenc %xmm7, %xmm8, %xmm8 + vaesenc %xmm7, %xmm9, %xmm9 + vaesenc %xmm7, %xmm10, %xmm10 + vaesenc %xmm7, %xmm11, %xmm11 + vaesenc %xmm7, %xmm12, %xmm12 + vaesenc %xmm7, %xmm13, %xmm13 + vaesenc %xmm7, %xmm14, %xmm14 + vaesenc %xmm7, %xmm15, %xmm15 + cmpl $13, %r9d + vmovdqa 192(%rsi), %xmm7 + jl L_AES_GCM_decrypt_avx2_aesenc_128_ghash_avx_done + vaesenc %xmm7, %xmm8, %xmm8 + vaesenc %xmm7, %xmm9, %xmm9 + vaesenc %xmm7, %xmm10, %xmm10 + vaesenc %xmm7, %xmm11, %xmm11 + vaesenc %xmm7, %xmm12, %xmm12 + vaesenc %xmm7, %xmm13, %xmm13 + vaesenc %xmm7, %xmm14, %xmm14 + vaesenc %xmm7, %xmm15, %xmm15 + vmovdqa 208(%rsi), %xmm7 + vaesenc %xmm7, %xmm8, %xmm8 + vaesenc %xmm7, %xmm9, %xmm9 + vaesenc %xmm7, %xmm10, %xmm10 + vaesenc %xmm7, %xmm11, %xmm11 + vaesenc %xmm7, %xmm12, %xmm12 + vaesenc %xmm7, %xmm13, %xmm13 + vaesenc %xmm7, %xmm14, %xmm14 + vaesenc %xmm7, %xmm15, %xmm15 + vmovdqa 224(%rsi), %xmm7 +L_AES_GCM_decrypt_avx2_aesenc_128_ghash_avx_done: + # aesenc_last + vaesenclast %xmm7, %xmm8, %xmm8 + vaesenclast %xmm7, %xmm9, %xmm9 + vaesenclast %xmm7, %xmm10, %xmm10 + vaesenclast %xmm7, %xmm11, %xmm11 + vmovdqu (%rcx), %xmm0 + vmovdqu 16(%rcx), %xmm1 + vmovdqu 32(%rcx), %xmm2 + vmovdqu 48(%rcx), %xmm3 + vpxor %xmm0, %xmm8, %xmm8 + vpxor %xmm1, %xmm9, %xmm9 + vpxor %xmm2, %xmm10, %xmm10 + vpxor %xmm3, %xmm11, %xmm11 + vmovdqu %xmm8, (%rdx) + vmovdqu %xmm9, 16(%rdx) + vmovdqu %xmm10, 32(%rdx) + vmovdqu %xmm11, 48(%rdx) + vaesenclast %xmm7, %xmm12, %xmm12 + vaesenclast %xmm7, %xmm13, %xmm13 + vaesenclast %xmm7, %xmm14, %xmm14 + vaesenclast %xmm7, %xmm15, %xmm15 + vmovdqu 64(%rcx), %xmm0 + vmovdqu 80(%rcx), %xmm1 + vmovdqu 96(%rcx), %xmm2 + vmovdqu 112(%rcx), %xmm3 + vpxor %xmm0, %xmm12, %xmm12 + vpxor %xmm1, %xmm13, %xmm13 + vpxor %xmm2, %xmm14, %xmm14 + vpxor %xmm3, %xmm15, %xmm15 + vmovdqu %xmm12, 64(%rdx) + vmovdqu %xmm13, 80(%rdx) + vmovdqu %xmm14, 96(%rdx) + vmovdqu %xmm15, 112(%rdx) + # aesenc_128_ghash - end + addl $0x80, %ebx + cmpl %r13d, %ebx + jl L_AES_GCM_decrypt_avx2_ghash_128 + vmovdqa (%rsp), %xmm5 + vmovdqa 128(%rsp), %xmm4 + vmovdqa 144(%rsp), %xmm15 +L_AES_GCM_decrypt_avx2_done_128: + cmpl %r10d, %ebx + jge L_AES_GCM_decrypt_avx2_done_dec + movl %r10d, %r13d + andl $0xfffffff0, %r13d + cmpl %r13d, %ebx + jge L_AES_GCM_decrypt_avx2_last_block_done +L_AES_GCM_decrypt_avx2_last_block_start: + vmovdqu (%rdi,%rbx,1), %xmm11 + vpshufb L_avx2_aes_gcm_bswap_epi64(%rip), %xmm4, %xmm10 + vpshufb L_avx2_aes_gcm_bswap_mask(%rip), %xmm11, %xmm12 + vpaddd L_avx2_aes_gcm_one(%rip), %xmm4, %xmm4 + vpxor %xmm6, %xmm12, %xmm12 + # aesenc_gfmul_sb + vpclmulqdq $0x01, %xmm5, %xmm12, %xmm2 + vpclmulqdq $16, %xmm5, %xmm12, %xmm3 + vpclmulqdq $0x00, %xmm5, %xmm12, %xmm1 + vpclmulqdq $0x11, %xmm5, %xmm12, %xmm8 + vpxor (%rsi), %xmm10, %xmm10 + vaesenc 16(%rsi), %xmm10, %xmm10 + vpxor %xmm2, %xmm3, %xmm3 + vpslldq $8, %xmm3, %xmm2 + vpsrldq $8, %xmm3, %xmm3 + vaesenc 32(%rsi), %xmm10, %xmm10 + vpxor %xmm1, %xmm2, %xmm2 + vpclmulqdq $16, L_avx2_aes_gcm_mod2_128(%rip), %xmm2, %xmm1 + vaesenc 48(%rsi), %xmm10, %xmm10 + vaesenc 64(%rsi), %xmm10, %xmm10 + vaesenc 80(%rsi), %xmm10, %xmm10 + vpshufd $0x4e, %xmm2, %xmm2 + vpxor %xmm1, %xmm2, %xmm2 + vpclmulqdq $16, L_avx2_aes_gcm_mod2_128(%rip), %xmm2, %xmm1 + vaesenc 96(%rsi), %xmm10, %xmm10 + vaesenc 112(%rsi), %xmm10, %xmm10 + vaesenc 128(%rsi), %xmm10, %xmm10 + vpshufd $0x4e, %xmm2, %xmm2 + vaesenc 144(%rsi), %xmm10, %xmm10 + vpxor %xmm3, %xmm8, %xmm8 + vpxor %xmm8, %xmm2, %xmm2 + vmovdqa 160(%rsi), %xmm0 + cmpl $11, %r9d + jl L_AES_GCM_decrypt_avx2_aesenc_gfmul_sb_last + vaesenc %xmm0, %xmm10, %xmm10 + vaesenc 176(%rsi), %xmm10, %xmm10 + vmovdqa 192(%rsi), %xmm0 + cmpl $13, %r9d + jl L_AES_GCM_decrypt_avx2_aesenc_gfmul_sb_last + vaesenc %xmm0, %xmm10, %xmm10 + vaesenc 208(%rsi), %xmm10, %xmm10 + vmovdqa 224(%rsi), %xmm0 +L_AES_GCM_decrypt_avx2_aesenc_gfmul_sb_last: + vaesenclast %xmm0, %xmm10, %xmm10 + vpxor %xmm1, %xmm2, %xmm6 + vpxor %xmm11, %xmm10, %xmm10 + vmovdqu %xmm10, (%r8,%rbx,1) + addl $16, %ebx + cmpl %r13d, %ebx + jl L_AES_GCM_decrypt_avx2_last_block_start +L_AES_GCM_decrypt_avx2_last_block_done: + movl %r10d, %ecx + movl %r10d, %edx + andl $15, %ecx + jz L_AES_GCM_decrypt_avx2_done_dec + # aesenc_last15_dec + vpshufb L_avx2_aes_gcm_bswap_epi64(%rip), %xmm4, %xmm4 + vpxor (%rsi), %xmm4, %xmm4 + vaesenc 16(%rsi), %xmm4, %xmm4 + vaesenc 32(%rsi), %xmm4, %xmm4 + vaesenc 48(%rsi), %xmm4, %xmm4 + vaesenc 64(%rsi), %xmm4, %xmm4 + vaesenc 80(%rsi), %xmm4, %xmm4 + vaesenc 96(%rsi), %xmm4, %xmm4 + vaesenc 112(%rsi), %xmm4, %xmm4 + vaesenc 128(%rsi), %xmm4, %xmm4 + vaesenc 144(%rsi), %xmm4, %xmm4 + cmpl $11, %r9d + vmovdqa 160(%rsi), %xmm1 + jl L_AES_GCM_decrypt_avx2_aesenc_last15_dec_avx_aesenc_avx_last + vaesenc %xmm1, %xmm4, %xmm4 + vaesenc 176(%rsi), %xmm4, %xmm4 + cmpl $13, %r9d + vmovdqa 192(%rsi), %xmm1 + jl L_AES_GCM_decrypt_avx2_aesenc_last15_dec_avx_aesenc_avx_last + vaesenc %xmm1, %xmm4, %xmm4 + vaesenc 208(%rsi), %xmm4, %xmm4 + vmovdqa 224(%rsi), %xmm1 +L_AES_GCM_decrypt_avx2_aesenc_last15_dec_avx_aesenc_avx_last: + vaesenclast %xmm1, %xmm4, %xmm4 + xorl %ecx, %ecx + vpxor %xmm0, %xmm0, %xmm0 + vmovdqa %xmm4, (%rsp) + vmovdqa %xmm0, 16(%rsp) +L_AES_GCM_decrypt_avx2_aesenc_last15_dec_avx_loop: + movzbl (%rdi,%rbx,1), %r13d + movb %r13b, 16(%rsp,%rcx,1) + xorb (%rsp,%rcx,1), %r13b + movb %r13b, (%r8,%rbx,1) + incl %ebx + incl %ecx + cmpl %edx, %ebx + jl L_AES_GCM_decrypt_avx2_aesenc_last15_dec_avx_loop + vmovdqa 16(%rsp), %xmm4 + vpshufb L_avx2_aes_gcm_bswap_mask(%rip), %xmm4, %xmm4 + vpxor %xmm4, %xmm6, %xmm6 + # ghash_gfmul_red + vpclmulqdq $16, %xmm5, %xmm6, %xmm2 + vpclmulqdq $0x01, %xmm5, %xmm6, %xmm1 + vpclmulqdq $0x00, %xmm5, %xmm6, %xmm0 + vpxor %xmm1, %xmm2, %xmm2 + vpslldq $8, %xmm2, %xmm1 + vpsrldq $8, %xmm2, %xmm2 + vpxor %xmm0, %xmm1, %xmm1 + vpclmulqdq $0x11, %xmm5, %xmm6, %xmm6 + vpclmulqdq $16, L_avx2_aes_gcm_mod2_128(%rip), %xmm1, %xmm0 + vpshufd $0x4e, %xmm1, %xmm1 + vpxor %xmm0, %xmm1, %xmm1 + vpclmulqdq $16, L_avx2_aes_gcm_mod2_128(%rip), %xmm1, %xmm0 + vpshufd $0x4e, %xmm1, %xmm1 + vpxor %xmm2, %xmm6, %xmm6 + vpxor %xmm1, %xmm6, %xmm6 + vpxor %xmm0, %xmm6, %xmm6 +L_AES_GCM_decrypt_avx2_done_dec: + # calc_tag + shlq $3, %r10 + vpinsrq $0x00, %r10, %xmm0, %xmm0 + shlq $3, %r11 + vpinsrq $0x01, %r11, %xmm1, %xmm1 + vpblendd $12, %xmm1, %xmm0, %xmm0 + vpxor %xmm6, %xmm0, %xmm0 + # ghash_gfmul_red + vpclmulqdq $16, %xmm5, %xmm0, %xmm4 + vpclmulqdq $0x01, %xmm5, %xmm0, %xmm3 + vpclmulqdq $0x00, %xmm5, %xmm0, %xmm2 + vpxor %xmm3, %xmm4, %xmm4 + vpslldq $8, %xmm4, %xmm3 + vpsrldq $8, %xmm4, %xmm4 + vpxor %xmm2, %xmm3, %xmm3 + vpclmulqdq $0x11, %xmm5, %xmm0, %xmm0 + vpclmulqdq $16, L_avx2_aes_gcm_mod2_128(%rip), %xmm3, %xmm2 + vpshufd $0x4e, %xmm3, %xmm3 + vpxor %xmm2, %xmm3, %xmm3 + vpclmulqdq $16, L_avx2_aes_gcm_mod2_128(%rip), %xmm3, %xmm2 + vpshufd $0x4e, %xmm3, %xmm3 + vpxor %xmm4, %xmm0, %xmm0 + vpxor %xmm3, %xmm0, %xmm0 + vpxor %xmm2, %xmm0, %xmm0 + vpshufb L_avx2_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0 + vpxor %xmm15, %xmm0, %xmm0 + # cmp_tag + cmpl $16, %r15d + je L_AES_GCM_decrypt_avx2_cmp_tag_16 + xorq %rdx, %rdx + xorq %rax, %rax + vmovdqa %xmm0, (%rsp) +L_AES_GCM_decrypt_avx2_cmp_tag_loop: + movzbl (%rsp,%rdx,1), %r13d + xorb (%r14,%rdx,1), %r13b + orb %r13b, %al + incl %edx + cmpl %r15d, %edx + jne L_AES_GCM_decrypt_avx2_cmp_tag_loop + cmpb $0x00, %al + sete %al + jmp L_AES_GCM_decrypt_avx2_cmp_tag_done +L_AES_GCM_decrypt_avx2_cmp_tag_16: + vmovdqu (%r14), %xmm1 + vpcmpeqb %xmm1, %xmm0, %xmm0 + vpmovmskb %xmm0, %rdx + # %%edx == 0xFFFF then return 1 else => return 0 + xorl %eax, %eax + cmpl $0xffff, %edx + sete %al +L_AES_GCM_decrypt_avx2_cmp_tag_done: + movl %eax, (%rbp) + vzeroupper + addq $0xa8, %rsp + popq %rbp + popq %r15 + popq %rbx + popq %r14 + popq %r12 + popq %r13 + repz retq +.size AES_GCM_decrypt_avx2,.-AES_GCM_decrypt_avx2 +#endif /* HAVE_INTEL_AVX2 */ diff --git a/wolfcrypt/src/chacha.c b/wolfcrypt/src/chacha.c index b02b900806..da549701c5 100644 --- a/wolfcrypt/src/chacha.c +++ b/wolfcrypt/src/chacha.c @@ -70,14 +70,6 @@ #define HAVE_INTEL_AVX2 #endif - #if defined(_MSC_VER) - #define CHACHA20_NOINLINE __declspec(noinline) - #elif defined(__GNUC__) - #define CHACHA20_NOINLINE __attribute__((noinline)) - #else - #define CHACHA20_NOINLINE - #endif - static int cpuidFlagsSet = 0; static int cpuidFlags = 0; #endif @@ -107,182 +99,6 @@ x[c] = PLUS(x[c],x[d]); x[b] = ROTATE(XOR(x[b],x[c]), 7); - -#define QUARTERROUND_INTEL_ASM(a0,b0,c0,d0, \ - a1,b1,c1,d1, \ - a2,b2,c2,d2, \ - a3,b3,c3,d3, \ - t1,o1) \ - "vpaddd "#b0", "#a0", "#a0"\n\t" \ - "vpxor "#a0", "#d0", "#d0"\n\t" \ - "vmovdqa "#o1"(%[x]), "#c3"\n\t" \ - "vpshufb %[rotl16], "#d0", "#d0"\n\t" \ - "vpaddd "#d0", "#c0", "#c0"\n\t" \ - "vpxor "#c0", "#b0", "#b0"\n\t" \ - "vpaddd "#b1", "#a1", "#a1"\n\t" \ - "vpxor "#a1", "#d1", "#d1"\n\t" \ - "vpshufb %[rotl16], "#d1", "#d1"\n\t" \ - "vpaddd "#d1", "#c1", "#c1"\n\t" \ - "vpxor "#c1", "#b1", "#b1"\n\t" \ - "vpaddd "#b2", "#a2", "#a2"\n\t" \ - "vpxor "#a2", "#d2", "#d2"\n\t" \ - "vpshufb %[rotl16], "#d2", "#d2"\n\t" \ - "vpaddd "#d2", "#c2", "#c2"\n\t" \ - "vpxor "#c2", "#b2", "#b2"\n\t" \ - "vpaddd "#b3", "#a3", "#a3"\n\t" \ - "vpxor "#a3", "#d3", "#d3"\n\t" \ - "vpshufb %[rotl16], "#d3", "#d3"\n\t" \ - "vpaddd "#d3", "#c3", "#c3"\n\t" \ - "vpxor "#c3", "#b3", "#b3"\n\t" \ - "vmovdqa "#c3", "#o1"(%[x])\n\t" \ - "vpsrld $20, "#b0", "#t1"\n\t" \ - "vpslld $12, "#b0", "#b0"\n\t" \ - "vpxor "#t1", "#b0", "#b0"\n\t" \ - "vpsrld $20, "#b1", "#t1"\n\t" \ - "vpslld $12, "#b1", "#b1"\n\t" \ - "vpxor "#t1", "#b1", "#b1"\n\t" \ - "vpsrld $20, "#b2", "#t1"\n\t" \ - "vpslld $12, "#b2", "#b2"\n\t" \ - "vpxor "#t1", "#b2", "#b2"\n\t" \ - "vpsrld $20, "#b3", "#t1"\n\t" \ - "vpslld $12, "#b3", "#b3"\n\t" \ - "vpxor "#t1", "#b3", "#b3"\n\t" \ - "vpaddd "#b0", "#a0", "#a0"\n\t" \ - "vpxor "#a0", "#d0", "#d0"\n\t" \ - "vmovdqa "#o1"(%[x]), "#c3"\n\t" \ - "vpshufb %[rotl8], "#d0", "#d0"\n\t" \ - "vpaddd "#d0", "#c0", "#c0"\n\t" \ - "vpxor "#c0", "#b0", "#b0"\n\t" \ - "vpaddd "#b1", "#a1", "#a1"\n\t" \ - "vpxor "#a1", "#d1", "#d1"\n\t" \ - "vpshufb %[rotl8], "#d1", "#d1"\n\t" \ - "vpaddd "#d1", "#c1", "#c1"\n\t" \ - "vpxor "#c1", "#b1", "#b1"\n\t" \ - "vpaddd "#b2", "#a2", "#a2"\n\t" \ - "vpxor "#a2", "#d2", "#d2"\n\t" \ - "vpshufb %[rotl8], "#d2", "#d2"\n\t" \ - "vpaddd "#d2", "#c2", "#c2"\n\t" \ - "vpxor "#c2", "#b2", "#b2"\n\t" \ - "vpaddd "#b3", "#a3", "#a3"\n\t" \ - "vpxor "#a3", "#d3", "#d3"\n\t" \ - "vpshufb %[rotl8], "#d3", "#d3"\n\t" \ - "vpaddd "#d3", "#c3", "#c3"\n\t" \ - "vpxor "#c3", "#b3", "#b3"\n\t" \ - "vmovdqa "#c3", "#o1"(%[x])\n\t" \ - "vpsrld $25, "#b0", "#t1"\n\t" \ - "vpslld $7, "#b0", "#b0"\n\t" \ - "vpxor "#t1", "#b0", "#b0"\n\t" \ - "vpsrld $25, "#b1", "#t1"\n\t" \ - "vpslld $7, "#b1", "#b1"\n\t" \ - "vpxor "#t1", "#b1", "#b1"\n\t" \ - "vpsrld $25, "#b2", "#t1"\n\t" \ - "vpslld $7, "#b2", "#b2"\n\t" \ - "vpxor "#t1", "#b2", "#b2"\n\t" \ - "vpsrld $25, "#b3", "#t1"\n\t" \ - "vpslld $7, "#b3", "#b3"\n\t" \ - "vpxor "#t1", "#b3", "#b3"\n\t" - -#define QUARTERROUND_INTEL_ASM_2(a0,b0,c0,d0, \ - a1,b1,c1,d1, \ - a2,b2,c2,d2, \ - a3,b3,c3,d3, \ - t1,o1) \ - "vpaddd "#b0", "#a0", "#a0"\n\t" \ - "vpxor "#a0", "#d0", "#d0"\n\t" \ - "vmovdqa "#o1"(%[x]), "#c1"\n\t" \ - "vpshufb %[rotl16], "#d0", "#d0"\n\t" \ - "vpaddd "#d0", "#c0", "#c0"\n\t" \ - "vpxor "#c0", "#b0", "#b0"\n\t" \ - "vpaddd "#b1", "#a1", "#a1"\n\t" \ - "vpxor "#a1", "#d1", "#d1"\n\t" \ - "vpshufb %[rotl16], "#d1", "#d1"\n\t" \ - "vpaddd "#d1", "#c1", "#c1"\n\t" \ - "vpxor "#c1", "#b1", "#b1"\n\t" \ - "vpaddd "#b2", "#a2", "#a2"\n\t" \ - "vpxor "#a2", "#d2", "#d2"\n\t" \ - "vpshufb %[rotl16], "#d2", "#d2"\n\t" \ - "vpaddd "#d2", "#c2", "#c2"\n\t" \ - "vpxor "#c2", "#b2", "#b2"\n\t" \ - "vpaddd "#b3", "#a3", "#a3"\n\t" \ - "vpxor "#a3", "#d3", "#d3"\n\t" \ - "vpshufb %[rotl16], "#d3", "#d3"\n\t" \ - "vpaddd "#d3", "#c3", "#c3"\n\t" \ - "vpxor "#c3", "#b3", "#b3"\n\t" \ - "vmovdqa "#c1", "#o1"(%[x])\n\t" \ - "vpsrld $20, "#b0", "#t1"\n\t" \ - "vpslld $12, "#b0", "#b0"\n\t" \ - "vpxor "#t1", "#b0", "#b0"\n\t" \ - "vpsrld $20, "#b1", "#t1"\n\t" \ - "vpslld $12, "#b1", "#b1"\n\t" \ - "vpxor "#t1", "#b1", "#b1"\n\t" \ - "vpsrld $20, "#b2", "#t1"\n\t" \ - "vpslld $12, "#b2", "#b2"\n\t" \ - "vpxor "#t1", "#b2", "#b2"\n\t" \ - "vpsrld $20, "#b3", "#t1"\n\t" \ - "vpslld $12, "#b3", "#b3"\n\t" \ - "vpxor "#t1", "#b3", "#b3"\n\t" \ - "vpaddd "#b0", "#a0", "#a0"\n\t" \ - "vpxor "#a0", "#d0", "#d0"\n\t" \ - "vmovdqa "#o1"(%[x]), "#c1"\n\t" \ - "vpshufb %[rotl8], "#d0", "#d0"\n\t" \ - "vpaddd "#d0", "#c0", "#c0"\n\t" \ - "vpxor "#c0", "#b0", "#b0"\n\t" \ - "vpaddd "#b1", "#a1", "#a1"\n\t" \ - "vpxor "#a1", "#d1", "#d1"\n\t" \ - "vpshufb %[rotl8], "#d1", "#d1"\n\t" \ - "vpaddd "#d1", "#c1", "#c1"\n\t" \ - "vpxor "#c1", "#b1", "#b1"\n\t" \ - "vpaddd "#b2", "#a2", "#a2"\n\t" \ - "vpxor "#a2", "#d2", "#d2"\n\t" \ - "vpshufb %[rotl8], "#d2", "#d2"\n\t" \ - "vpaddd "#d2", "#c2", "#c2"\n\t" \ - "vpxor "#c2", "#b2", "#b2"\n\t" \ - "vpaddd "#b3", "#a3", "#a3"\n\t" \ - "vpxor "#a3", "#d3", "#d3"\n\t" \ - "vpshufb %[rotl8], "#d3", "#d3"\n\t" \ - "vpaddd "#d3", "#c3", "#c3"\n\t" \ - "vpxor "#c3", "#b3", "#b3"\n\t" \ - "vmovdqa "#c1", "#o1"(%[x])\n\t" \ - "vpsrld $25, "#b0", "#t1"\n\t" \ - "vpslld $7, "#b0", "#b0"\n\t" \ - "vpxor "#t1", "#b0", "#b0"\n\t" \ - "vpsrld $25, "#b1", "#t1"\n\t" \ - "vpslld $7, "#b1", "#b1"\n\t" \ - "vpxor "#t1", "#b1", "#b1"\n\t" \ - "vpsrld $25, "#b2", "#t1"\n\t" \ - "vpslld $7, "#b2", "#b2"\n\t" \ - "vpxor "#t1", "#b2", "#b2"\n\t" \ - "vpsrld $25, "#b3", "#t1"\n\t" \ - "vpslld $7, "#b3", "#b3"\n\t" \ - "vpxor "#t1", "#b3", "#b3"\n\t" - - -#define QUARTERROUND_XMM() \ - QUARTERROUND_INTEL_ASM(%%xmm0,%%xmm4,%%xmm8,%%xmm12, \ - %%xmm1,%%xmm5,%%xmm9,%%xmm13, \ - %%xmm2,%%xmm6,%%xmm10,%%xmm14, \ - %%xmm3,%%xmm7,%%xmm11,%%xmm15, \ - %%xmm11,48) -#define QUARTERROUND_XMM_2() \ - QUARTERROUND_INTEL_ASM_2(%%xmm0,%%xmm5,%%xmm10,%%xmm15, \ - %%xmm1,%%xmm6,%%xmm11,%%xmm12, \ - %%xmm2,%%xmm7,%%xmm8,%%xmm13, \ - %%xmm3,%%xmm4,%%xmm9,%%xmm14, \ - %%xmm11,48) - -#define QUARTERROUND_YMM() \ - QUARTERROUND_INTEL_ASM(%%ymm0,%%ymm4,%%ymm8,%%ymm12, \ - %%ymm1,%%ymm5,%%ymm9,%%ymm13, \ - %%ymm2,%%ymm6,%%ymm10,%%ymm14, \ - %%ymm3,%%ymm7,%%ymm11,%%ymm15, \ - %%ymm11,96) -#define QUARTERROUND_YMM_2() \ - QUARTERROUND_INTEL_ASM_2(%%ymm0,%%ymm5,%%ymm10,%%ymm15, \ - %%ymm1,%%ymm6,%%ymm11,%%ymm12, \ - %%ymm2,%%ymm7,%%ymm8,%%ymm13, \ - %%ymm3,%%ymm4,%%ymm9,%%ymm14, \ - %%ymm11,96) - /** * Set up iv(nonce). Earlier versions used 64 bits instead of 96, this version * uses the typical AEAD 96 bit nonce and can do record sizes of 256 GB. @@ -416,1037 +232,21 @@ static WC_INLINE void wc_Chacha_wordtobyte(word32 output[CHACHA_CHUNK_WORDS], } } +#ifdef __cplusplus + extern "C" { +#endif -#ifdef USE_INTEL_CHACHA_SPEEDUP +extern void chacha_encrypt_x64(ChaCha* ctx, const byte* m, byte* c, + word32 bytes); +extern void chacha_encrypt_avx1(ChaCha* ctx, const byte* m, byte* c, + word32 bytes); +extern void chacha_encrypt_avx2(ChaCha* ctx, const byte* m, byte* c, + word32 bytes); -#define QUARTERROUND_2_X64(r11, r12, r13, r14, r21, r22, r23, r24) \ - "addl "#r12", "#r11"\n\t" \ - "addl "#r22", "#r21"\n\t" \ - "xorl "#r11", "#r14"\n\t" \ - "xorl "#r21", "#r24"\n\t" \ - "roll $16, "#r14"\n\t" \ - "roll $16, "#r24"\n\t" \ - "addl "#r14", "#r13"\n\t" \ - "addl "#r24", "#r23"\n\t" \ - "xorl "#r13", "#r12"\n\t" \ - "xorl "#r23", "#r22"\n\t" \ - "roll $12, "#r12"\n\t" \ - "roll $12, "#r22"\n\t" \ - "addl "#r12", "#r11"\n\t" \ - "addl "#r22", "#r21"\n\t" \ - "xorl "#r11", "#r14"\n\t" \ - "xorl "#r21", "#r24"\n\t" \ - "roll $8, "#r14"\n\t" \ - "roll $8, "#r24"\n\t" \ - "addl "#r14", "#r13"\n\t" \ - "addl "#r24", "#r23"\n\t" \ - "xorl "#r13", "#r12"\n\t" \ - "xorl "#r23", "#r22"\n\t" \ - "roll $7, "#r12"\n\t" \ - "roll $7, "#r22"\n\t" \ +#ifdef __cplusplus + } /* extern "C" */ +#endif -#define CHACHA_CRYPT_X64() \ - "subq $40, %%rsp\n\t" \ - "movq 32(%[input]), %%rax\n\t" \ - "movq 40(%[input]), %%rdx\n\t" \ - "movq %%rax, 8(%%rsp)\n\t" \ - "movq %%rdx, 16(%%rsp)\n\t" \ - "movl 0(%[input]), %%eax\n\t" \ - "movl 4(%[input]), %%ebx\n\t" \ - "movl 8(%[input]), %%ecx\n\t" \ - "movl 12(%[input]), %%edx\n\t" \ - "movl 16(%[input]), %%r8d\n\t" \ - "movl 20(%[input]), %%r9d\n\t" \ - "movl 24(%[input]), %%r10d\n\t" \ - "movl 28(%[input]), %%r11d\n\t" \ - "movl 48(%[input]), %%r12d\n\t" \ - "movl 52(%[input]), %%r13d\n\t" \ - "movl 56(%[input]), %%r14d\n\t" \ - "movl 60(%[input]), %%r15d\n\t" \ - "movb $10, (%%rsp)\n\t" \ - "movq %%rsi, 32(%%rsp)\n\t" \ - "movq %%rdi, 24(%%rsp)\n\t" \ - "movl 8(%%rsp), %%esi\n\t" \ - "movl 12(%%rsp), %%edi\n\t" \ - "\n" \ - "1:\n\t" \ - QUARTERROUND_2_X64(%%eax, %%r8d, %%esi, %%r12d, \ - %%ebx, %%r9d, %%edi, %%r13d) \ - "movl %%esi, 8(%%rsp)\n\t" \ - "movl %%edi, 12(%%rsp)\n\t" \ - "movl 16(%%rsp), %%esi\n\t" \ - "movl 20(%%rsp), %%edi\n\t" \ - QUARTERROUND_2_X64(%%ecx, %%r10d, %%esi, %%r14d, \ - %%edx, %%r11d, %%edi, %%r15d) \ - QUARTERROUND_2_X64(%%eax, %%r9d, %%esi, %%r15d, \ - %%ebx, %%r10d, %%edi, %%r12d) \ - "movl %%esi, 16(%%rsp)\n\t" \ - "movl %%edi, 20(%%rsp)\n\t" \ - "movl 8(%%rsp), %%esi\n\t" \ - "movl 12(%%rsp), %%edi\n\t" \ - QUARTERROUND_2_X64(%%ecx, %%r11d, %%esi, %%r13d, \ - %%edx, %%r8d, %%edi, %%r14d) \ - "decb (%%rsp)\n\t" \ - "jnz 1b\n\t" \ - "movl %%esi, 8(%%rsp)\n\t" \ - "movl %%edi, 12(%%rsp)\n\t" \ - "movq 32(%%rsp), %%rsi\n\t" \ - "movq 24(%%rsp), %%rdi\n\t" \ - "addl 0(%[input]), %%eax\n\t" \ - "addl 4(%[input]), %%ebx\n\t" \ - "addl 8(%[input]), %%ecx\n\t" \ - "addl 12(%[input]), %%edx\n\t" \ - "addl 16(%[input]), %%r8d\n\t" \ - "addl 20(%[input]), %%r9d\n\t" \ - "addl 24(%[input]), %%r10d\n\t" \ - "addl 28(%[input]), %%r11d\n\t" \ - "addl 48(%[input]), %%r12d\n\t" \ - "addl 52(%[input]), %%r13d\n\t" \ - "addl 56(%[input]), %%r14d\n\t" \ - "addl 60(%[input]), %%r15d\n\t" \ - -#define CHACHA_PARTIAL_CHUNK_X64() \ - __asm__ __volatile__ ( \ - CHACHA_CRYPT_X64() \ - "movl %%eax , 0(%[c])\n\t" \ - "movl %%ebx , 4(%[c])\n\t" \ - "movl %%ecx , 8(%[c])\n\t" \ - "movl %%edx , 12(%[c])\n\t" \ - "movl %%r8d , 16(%[c])\n\t" \ - "movl %%r9d , 20(%[c])\n\t" \ - "movl %%r10d, 24(%[c])\n\t" \ - "movl %%r11d, 28(%[c])\n\t" \ - "movl %%r12d, 48(%[c])\n\t" \ - "movl %%r13d, 52(%[c])\n\t" \ - "movl %%r14d, 56(%[c])\n\t" \ - "movl %%r15d, 60(%[c])\n\t" \ - "movl 8(%%rsp), %%eax\n\t" \ - "movl 12(%%rsp), %%ebx\n\t" \ - "movl 16(%%rsp), %%ecx\n\t" \ - "movl 20(%%rsp), %%edx\n\t" \ - "addl 32(%[input]), %%eax\n\t" \ - "addl 36(%[input]), %%ebx\n\t" \ - "addl 40(%[input]), %%ecx\n\t" \ - "addl 44(%[input]), %%edx\n\t" \ - "movl %%eax , 32(%[c])\n\t" \ - "movl %%ebx , 36(%[c])\n\t" \ - "movl %%ecx , 40(%[c])\n\t" \ - "movl %%edx , 44(%[c])\n\t" \ - "addl $1, 48(%[input])\n\t" \ - "addq $40, %%rsp\n\t" \ - "movq %[output], %%rax\n\t" \ - "movq %[m], %%rbx\n\t" \ - "movl %[bytes], %%r8d\n\t" \ - "xorq %%rdx, %%rdx\n\t" \ - "movl %%r8d, %%r9d\n\t" \ - "andl $7, %%r9d\n\t" \ - "jz 4f\n\t" \ - "\n" \ - "2:\n\t" \ - "movzbl (%[c],%%rdx,1), %%ecx\n\t" \ - "xorb (%%rbx,%%rdx,1), %%cl\n\t" \ - "movb %%cl, (%%rax,%%rdx,1)\n\t" \ - "incl %%edx\n\t" \ - "cmpl %%r9d, %%edx\n\t" \ - "jne 2b\n\t" \ - "je 3f\n\t" \ - "\n" \ - "4:\n\t" \ - "movq (%[c],%%rdx,1), %%rcx\n\t" \ - "xorq (%%rbx,%%rdx,1), %%rcx\n\t" \ - "movq %%rcx, (%%rax,%%rdx,1)\n\t" \ - "addl $8, %%edx\n\t" \ - "\n" \ - "3:\n\t" \ - "cmpl %%r8d, %%edx\n\t" \ - "jne 4b\n\t" \ - : \ - : [input] "r" (ctx->X), [c] "r" (x), \ - [output] "m" (c), [bytes] "m" (bytes), [m] "m" (m) \ - : "eax", "ebx", "ecx", "edx", "r8", "r9", "r10", "r11", "r12", "r13", \ - "r14", "r15", "memory" \ - ) - - -#define CHACHA_CHUNK_X64() \ - __asm__ __volatile__ ( \ - CHACHA_CRYPT_X64() \ - "movq %%rsi, 32(%%rsp)\n\t" \ - "addq $40, %%rsp\n\t" \ - "movq %[m], %%rsi\n\t" \ - "subq $40, %%rsp\n\t" \ - "xorl 0(%%rsi), %%eax\n\t" \ - "xorl 4(%%rsi), %%ebx\n\t" \ - "xorl 8(%%rsi), %%ecx\n\t" \ - "xorl 12(%%rsi), %%edx\n\t" \ - "xorl 16(%%rsi), %%r8d\n\t" \ - "xorl 20(%%rsi), %%r9d\n\t" \ - "xorl 24(%%rsi), %%r10d\n\t" \ - "xorl 28(%%rsi), %%r11d\n\t" \ - "xorl 48(%%rsi), %%r12d\n\t" \ - "xorl 52(%%rsi), %%r13d\n\t" \ - "xorl 56(%%rsi), %%r14d\n\t" \ - "xorl 60(%%rsi), %%r15d\n\t" \ - "movq 32(%%rsp), %%rsi\n\t" \ - "movl %%eax , 0(%[c])\n\t" \ - "movl %%ebx , 4(%[c])\n\t" \ - "movl %%ecx , 8(%[c])\n\t" \ - "movl %%edx , 12(%[c])\n\t" \ - "movl %%r8d , 16(%[c])\n\t" \ - "movl %%r9d , 20(%[c])\n\t" \ - "movl %%r10d, 24(%[c])\n\t" \ - "movl %%r11d, 28(%[c])\n\t" \ - "movl %%r12d, 48(%[c])\n\t" \ - "movl %%r13d, 52(%[c])\n\t" \ - "movl %%r14d, 56(%[c])\n\t" \ - "movl %%r15d, 60(%[c])\n\t" \ - "addq $40, %%rsp\n\t" \ - "movq %[m], %%r8\n\t" \ - "subq $40, %%rsp\n\t" \ - "movl 8(%%rsp), %%eax\n\t" \ - "movl 12(%%rsp), %%ebx\n\t" \ - "movl 16(%%rsp), %%ecx\n\t" \ - "movl 20(%%rsp), %%edx\n\t" \ - "addl 32(%[input]), %%eax\n\t" \ - "addl 36(%[input]), %%ebx\n\t" \ - "addl 40(%[input]), %%ecx\n\t" \ - "addl 44(%[input]), %%edx\n\t" \ - "xorl 32(%%r8), %%eax\n\t" \ - "xorl 36(%%r8), %%ebx\n\t" \ - "xorl 40(%%r8), %%ecx\n\t" \ - "xorl 44(%%r8), %%edx\n\t" \ - "movl %%eax , 32(%[c])\n\t" \ - "movl %%ebx , 36(%[c])\n\t" \ - "movl %%ecx , 40(%[c])\n\t" \ - "movl %%edx , 44(%[c])\n\t" \ - "addl $1, 48(%[input])\n\t" \ - "addq $40, %%rsp\n\t" \ - : \ - : [input] "r" (ctx->X), [c] "r" (c), [m] "m" (m) \ - : "eax", "ebx", "ecx", "edx", "r8", "r9", "r10", "r11", "r12", "r13", \ - "r14", "r15", "memory" \ - ) - - -static void chacha_encrypt_x64(ChaCha* ctx, const byte* m, byte* c, - word32 bytes) -{ - word32 x[CHACHA_CHUNK_WORDS]; - - if (bytes == 0) - return; - - for (; bytes >= CHACHA_CHUNK_BYTES;) { - CHACHA_CHUNK_X64(); - bytes -= CHACHA_CHUNK_BYTES; - c += CHACHA_CHUNK_BYTES; - m += CHACHA_CHUNK_BYTES; - } - if (bytes > 0) { - CHACHA_PARTIAL_CHUNK_X64(); - } -} - -#if defined(HAVE_INTEL_AVX1) || defined(HAVE_INTEL_AVX2) -static const __m128i rotl8 = { 0x0605040702010003UL,0x0e0d0c0f0a09080bUL }; -static const __m128i rotl16 = { 0x0504070601000302UL,0x0d0c0f0e09080b0aUL }; -#endif /* HAVE_INTEL_AVX1 || HAVE_INTEL_AVX2 */ - -#ifdef HAVE_INTEL_AVX1 -#define QUARTERROUND_2_AVX() \ - "paddd %%xmm1, %%xmm0\n\t" \ - "pxor %%xmm0, %%xmm3\n\t" \ - "pshufb %[rotl16], %%xmm3\n\t" \ - "paddd %%xmm3, %%xmm2\n\t" \ - "pxor %%xmm2, %%xmm1\n\t" \ - "movdqa %%xmm1, %%xmm4\n\t" \ - "pslld $12, %%xmm1\n\t" \ - "psrld $20, %%xmm4\n\t" \ - "pxor %%xmm4, %%xmm1\n\t" \ - "paddd %%xmm1, %%xmm0\n\t" \ - "pxor %%xmm0, %%xmm3\n\t" \ - "pshufb %[rotl8], %%xmm3\n\t" \ - "paddd %%xmm3, %%xmm2\n\t" \ - "pxor %%xmm2, %%xmm1\n\t" \ - "movdqa %%xmm1, %%xmm4\n\t" \ - "pslld $7, %%xmm1\n\t" \ - "psrld $25, %%xmm4\n\t" \ - "pxor %%xmm4, %%xmm1\n\t" \ - "# Swap words for next round\n\t" \ - "pshufd $0x39, %%xmm1, %%xmm1\n\t" \ - "pshufd $0x4e, %%xmm2, %%xmm2\n\t" \ - "pshufd $0x93, %%xmm3, %%xmm3\n\t" \ - "paddd %%xmm1, %%xmm0\n\t" \ - "pxor %%xmm0, %%xmm3\n\t" \ - "pshufb %[rotl16], %%xmm3\n\t" \ - "paddd %%xmm3, %%xmm2\n\t" \ - "pxor %%xmm2, %%xmm1\n\t" \ - "movdqa %%xmm1, %%xmm4\n\t" \ - "pslld $12, %%xmm1\n\t" \ - "psrld $20, %%xmm4\n\t" \ - "pxor %%xmm4, %%xmm1\n\t" \ - "paddd %%xmm1, %%xmm0\n\t" \ - "pxor %%xmm0, %%xmm3\n\t" \ - "pshufb %[rotl8], %%xmm3\n\t" \ - "paddd %%xmm3, %%xmm2\n\t" \ - "pxor %%xmm2, %%xmm1\n\t" \ - "movdqa %%xmm1, %%xmm4\n\t" \ - "pslld $7, %%xmm1\n\t" \ - "psrld $25, %%xmm4\n\t" \ - "pxor %%xmm4, %%xmm1\n\t" \ - "# Swap words back\n\t" \ - "pshufd $0x93, %%xmm1, %%xmm1\n\t" \ - "pshufd $0x4e, %%xmm2, %%xmm2\n\t" \ - "pshufd $0x39, %%xmm3, %%xmm3\n\t" \ - -#define CHACHA_CRYPT_AVX() \ - "movdqu 0(%[input]), %%xmm0\n\t" \ - "movdqu 16(%[input]), %%xmm1\n\t" \ - "movdqu 32(%[input]), %%xmm2\n\t" \ - "movdqu 48(%[input]), %%xmm3\n\t" \ - "movb $10, %%al\n\t" \ - "\n" \ - "1:\n\t" \ - QUARTERROUND_2_AVX() \ - "decb %%al\n\t" \ - "jnz 1b\n\t" \ - "movdqu 0(%[input]), %%xmm4\n\t" \ - "movdqu 16(%[input]), %%xmm5\n\t" \ - "movdqu 32(%[input]), %%xmm6\n\t" \ - "movdqu 48(%[input]), %%xmm7\n\t" \ - "paddd %%xmm4, %%xmm0\n\t" \ - "paddd %%xmm5, %%xmm1\n\t" \ - "paddd %%xmm6, %%xmm2\n\t" \ - "paddd %%xmm7, %%xmm3\n\t" \ - -#define CHACHA_PARTIAL_CHUNK_AVX() \ - __asm__ __volatile__ ( \ - CHACHA_CRYPT_AVX() \ - "movdqu %%xmm0, 0(%[c])\n\t" \ - "movdqu %%xmm1, 16(%[c])\n\t" \ - "movdqu %%xmm2, 32(%[c])\n\t" \ - "movdqu %%xmm3, 48(%[c])\n\t" \ - "addl $1, 48(%[input])\n\t" \ - "movl %[bytes], %%r8d\n\t" \ - "xorq %%rdx, %%rdx\n\t" \ - "movl %%r8d, %%r9d\n\t" \ - "andl $7, %%r9d\n\t" \ - "jz 4f\n\t" \ - "\n" \ - "2:\n\t" \ - "movzbl (%[c],%%rdx,1), %%ecx\n\t" \ - "xorb (%[m],%%rdx,1), %%cl\n\t" \ - "movb %%cl, (%[output],%%rdx,1)\n\t" \ - "incl %%edx\n\t" \ - "cmpl %%r9d, %%edx\n\t" \ - "jne 2b\n\t" \ - "je 3f\n\t" \ - "\n" \ - "4:\n\t" \ - "movq (%[c],%%rdx,1), %%rcx\n\t" \ - "xorq (%[m],%%rdx,1), %%rcx\n\t" \ - "movq %%rcx, (%[output],%%rdx,1)\n\t" \ - "addl $8, %%edx\n\t" \ - "\n" \ - "3:\n\t" \ - "cmpl %%r8d, %%edx\n\t" \ - "jne 4b\n\t" \ - : \ - : [input] "r" (ctx->X), [c] "r" (x), \ - [output] "r" (c), [bytes] "r" (bytes), [m] "r" (m), \ - [rotl8] "xrm" (rotl8), [rotl16] "xrm" (rotl16) \ - : "eax", "ecx", "edx", "r8", "r9", "memory", \ - "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7" \ - ) - - -#define CHACHA_CHUNK_AVX() \ - __asm__ __volatile__ ( \ - CHACHA_CRYPT_AVX() \ - "movdqu 0(%[m]), %%xmm4\n\t" \ - "movdqu 16(%[m]), %%xmm5\n\t" \ - "movdqu 32(%[m]), %%xmm6\n\t" \ - "movdqu 48(%[m]), %%xmm7\n\t" \ - "pxor %%xmm4, %%xmm0\n\t" \ - "pxor %%xmm5, %%xmm1\n\t" \ - "pxor %%xmm6, %%xmm2\n\t" \ - "pxor %%xmm7, %%xmm3\n\t" \ - "movdqu %%xmm0, 0(%[c])\n\t" \ - "movdqu %%xmm1, 16(%[c])\n\t" \ - "movdqu %%xmm2, 32(%[c])\n\t" \ - "movdqu %%xmm3, 48(%[c])\n\t" \ - "addl $1, 48(%[input])\n\t" \ - : \ - : [input] "r" (ctx->X), [c] "r" (c), [m] "r" (m), \ - [rotl8] "xrm" (rotl8), [rotl16] "xrm" (rotl16) \ - : "rax", "memory", \ - "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7" \ - ) - -CHACHA20_NOINLINE static void chacha_encrypt_avx(ChaCha* ctx, const byte* m, - byte* c, word32 bytes) -{ - ALIGN128 word32 X[4*CHACHA_CHUNK_WORDS]; /* used to make sure aligned */ - ALIGN128 word32 x[2*CHACHA_CHUNK_WORDS]; /* used to make sure aligned */ - word32 cnt = 0; - static const __m128i add = { 0x0000000100000000UL,0x0000000300000002UL }; - static const __m128i four = { 0x0000000400000004UL,0x0000000400000004UL }; - - if (bytes == 0) - return; - - __asm__ __volatile__ ( - "movl %[bytes], %[cnt]\n\t" - "shrl $8, %[cnt]\n\t" - "jz L_end128\n\t" - - "vpshufd $0, (%[key]), %%xmm0\n\t" - "vpshufd $0, 4(%[key]), %%xmm1\n\t" - "vpshufd $0, 8(%[key]), %%xmm2\n\t" - "vpshufd $0, 12(%[key]), %%xmm3\n\t" - "vpshufd $0, 16(%[key]), %%xmm4\n\t" - "vpshufd $0, 20(%[key]), %%xmm5\n\t" - "vpshufd $0, 24(%[key]), %%xmm6\n\t" - "vpshufd $0, 28(%[key]), %%xmm7\n\t" - "vpshufd $0, 32(%[key]), %%xmm8\n\t" - "vpshufd $0, 36(%[key]), %%xmm9\n\t" - "vpshufd $0, 40(%[key]), %%xmm10\n\t" - "vpshufd $0, 44(%[key]), %%xmm11\n\t" - "vpshufd $0, 48(%[key]), %%xmm12\n\t" - "vpshufd $0, 52(%[key]), %%xmm13\n\t" - "vpshufd $0, 56(%[key]), %%xmm14\n\t" - "vpshufd $0, 60(%[key]), %%xmm15\n\t" - - "vpaddd %[add], %%xmm12, %%xmm12\n\t" - - "vmovdqa %%xmm0, (%[X])\n\t" - "vmovdqa %%xmm1, 16(%[X])\n\t" - "vmovdqa %%xmm2, 32(%[X])\n\t" - "vmovdqa %%xmm3, 48(%[X])\n\t" - "vmovdqa %%xmm4, 64(%[X])\n\t" - "vmovdqa %%xmm5, 80(%[X])\n\t" - "vmovdqa %%xmm6, 96(%[X])\n\t" - "vmovdqa %%xmm7, 112(%[X])\n\t" - "vmovdqa %%xmm8, 128(%[X])\n\t" - "vmovdqa %%xmm9, 144(%[X])\n\t" - "vmovdqa %%xmm10, 160(%[X])\n\t" - "vmovdqa %%xmm11, 176(%[X])\n\t" - "vmovdqa %%xmm12, 192(%[X])\n\t" - "vmovdqa %%xmm13, 208(%[X])\n\t" - "vmovdqa %%xmm14, 224(%[X])\n\t" - "vmovdqa %%xmm15, 240(%[X])\n\t" - "\n" - "L_enc128_loop:\n\t" - "vmovdqa %%xmm11, 48(%[x])\n\t" - QUARTERROUND_XMM() - QUARTERROUND_XMM_2() - QUARTERROUND_XMM() - QUARTERROUND_XMM_2() - QUARTERROUND_XMM() - QUARTERROUND_XMM_2() - QUARTERROUND_XMM() - QUARTERROUND_XMM_2() - QUARTERROUND_XMM() - QUARTERROUND_XMM_2() - QUARTERROUND_XMM() - QUARTERROUND_XMM_2() - QUARTERROUND_XMM() - QUARTERROUND_XMM_2() - QUARTERROUND_XMM() - QUARTERROUND_XMM_2() - QUARTERROUND_XMM() - QUARTERROUND_XMM_2() - QUARTERROUND_XMM() - QUARTERROUND_XMM_2() - "vmovdqa 48(%[x]), %%xmm11\n\t" - - "vpaddd (%[X]), %%xmm0, %%xmm0\n\t" - "vpaddd 16(%[X]), %%xmm1, %%xmm1\n\t" - "vpaddd 32(%[X]), %%xmm2, %%xmm2\n\t" - "vpaddd 48(%[X]), %%xmm3, %%xmm3\n\t" - "vpaddd 64(%[X]), %%xmm4, %%xmm4\n\t" - "vpaddd 80(%[X]), %%xmm5, %%xmm5\n\t" - "vpaddd 96(%[X]), %%xmm6, %%xmm6\n\t" - "vpaddd 112(%[X]), %%xmm7, %%xmm7\n\t" - "vpaddd 128(%[X]), %%xmm8, %%xmm8\n\t" - "vpaddd 144(%[X]), %%xmm9, %%xmm9\n\t" - "vpaddd 160(%[X]), %%xmm10, %%xmm10\n\t" - "vpaddd 176(%[X]), %%xmm11, %%xmm11\n\t" - "vpaddd 192(%[X]), %%xmm12, %%xmm12\n\t" - "vpaddd 208(%[X]), %%xmm13, %%xmm13\n\t" - "vpaddd 224(%[X]), %%xmm14, %%xmm14\n\t" - "vpaddd 240(%[X]), %%xmm15, %%xmm15\n\t" - - "vmovdqa %%xmm8, (%[x])\n\t" - "vmovdqa %%xmm9, 16(%[x])\n\t" - "vmovdqa %%xmm10, 32(%[x])\n\t" - "vmovdqa %%xmm11, 48(%[x])\n\t" - "vmovdqa %%xmm12, 64(%[x])\n\t" - "vmovdqa %%xmm13, 80(%[x])\n\t" - "vmovdqa %%xmm14, 96(%[x])\n\t" - "vmovdqa %%xmm15, 112(%[x])\n\t" - - "vpunpckldq %%xmm1, %%xmm0, %%xmm8\n\t" - "vpunpckldq %%xmm3, %%xmm2, %%xmm9\n\t" - "vpunpckhdq %%xmm1, %%xmm0, %%xmm12\n\t" - "vpunpckhdq %%xmm3, %%xmm2, %%xmm13\n\t" - "vpunpckldq %%xmm5, %%xmm4, %%xmm10\n\t" - "vpunpckldq %%xmm7, %%xmm6, %%xmm11\n\t" - "vpunpckhdq %%xmm5, %%xmm4, %%xmm14\n\t" - "vpunpckhdq %%xmm7, %%xmm6, %%xmm15\n\t" - "vpunpcklqdq %%xmm9, %%xmm8, %%xmm0\n\t" - "vpunpcklqdq %%xmm11, %%xmm10, %%xmm1\n\t" - "vpunpckhqdq %%xmm9, %%xmm8, %%xmm2\n\t" - "vpunpckhqdq %%xmm11, %%xmm10, %%xmm3\n\t" - "vpunpcklqdq %%xmm13, %%xmm12, %%xmm4\n\t" - "vpunpcklqdq %%xmm15, %%xmm14, %%xmm5\n\t" - "vpunpckhqdq %%xmm13, %%xmm12, %%xmm6\n\t" - "vpunpckhqdq %%xmm15, %%xmm14, %%xmm7\n\t" - "vmovdqu (%[in]), %%xmm8\n\t" - "vmovdqu 16(%[in]), %%xmm9\n\t" - "vmovdqu 64(%[in]), %%xmm10\n\t" - "vmovdqu 80(%[in]), %%xmm11\n\t" - "vmovdqu 128(%[in]), %%xmm12\n\t" - "vmovdqu 144(%[in]), %%xmm13\n\t" - "vmovdqu 192(%[in]), %%xmm14\n\t" - "vmovdqu 208(%[in]), %%xmm15\n\t" - "vpxor %%xmm8, %%xmm0, %%xmm0\n\t" - "vpxor %%xmm9, %%xmm1, %%xmm1\n\t" - "vpxor %%xmm10, %%xmm2, %%xmm2\n\t" - "vpxor %%xmm11, %%xmm3, %%xmm3\n\t" - "vpxor %%xmm12, %%xmm4, %%xmm4\n\t" - "vpxor %%xmm13, %%xmm5, %%xmm5\n\t" - "vpxor %%xmm14, %%xmm6, %%xmm6\n\t" - "vpxor %%xmm15, %%xmm7, %%xmm7\n\t" - "vmovdqu %%xmm0, (%[out])\n\t" - "vmovdqu %%xmm1, 16(%[out])\n\t" - "vmovdqu %%xmm2, 64(%[out])\n\t" - "vmovdqu %%xmm3, 80(%[out])\n\t" - "vmovdqu %%xmm4, 128(%[out])\n\t" - "vmovdqu %%xmm5, 144(%[out])\n\t" - "vmovdqu %%xmm6, 192(%[out])\n\t" - "vmovdqu %%xmm7, 208(%[out])\n\t" - - "vmovdqa (%[x]), %%xmm0\n\t" - "vmovdqa 16(%[x]), %%xmm1\n\t" - "vmovdqa 32(%[x]), %%xmm2\n\t" - "vmovdqa 48(%[x]), %%xmm3\n\t" - "vmovdqa 64(%[x]), %%xmm4\n\t" - "vmovdqa 80(%[x]), %%xmm5\n\t" - "vmovdqa 96(%[x]), %%xmm6\n\t" - "vmovdqa 112(%[x]), %%xmm7\n\t" - - "vpunpckldq %%xmm1, %%xmm0, %%xmm8\n\t" - "vpunpckldq %%xmm3, %%xmm2, %%xmm9\n\t" - "vpunpckhdq %%xmm1, %%xmm0, %%xmm12\n\t" - "vpunpckhdq %%xmm3, %%xmm2, %%xmm13\n\t" - "vpunpckldq %%xmm5, %%xmm4, %%xmm10\n\t" - "vpunpckldq %%xmm7, %%xmm6, %%xmm11\n\t" - "vpunpckhdq %%xmm5, %%xmm4, %%xmm14\n\t" - "vpunpckhdq %%xmm7, %%xmm6, %%xmm15\n\t" - "vpunpcklqdq %%xmm9, %%xmm8, %%xmm0\n\t" - "vpunpcklqdq %%xmm11, %%xmm10, %%xmm1\n\t" - "vpunpckhqdq %%xmm9, %%xmm8, %%xmm2\n\t" - "vpunpckhqdq %%xmm11, %%xmm10, %%xmm3\n\t" - "vpunpcklqdq %%xmm13, %%xmm12, %%xmm4\n\t" - "vpunpcklqdq %%xmm15, %%xmm14, %%xmm5\n\t" - "vpunpckhqdq %%xmm13, %%xmm12, %%xmm6\n\t" - "vpunpckhqdq %%xmm15, %%xmm14, %%xmm7\n\t" - "vmovdqu 32(%[in]), %%xmm8\n\t" - "vmovdqu 48(%[in]), %%xmm9\n\t" - "vmovdqu 96(%[in]), %%xmm10\n\t" - "vmovdqu 112(%[in]), %%xmm11\n\t" - "vmovdqu 160(%[in]), %%xmm12\n\t" - "vmovdqu 176(%[in]), %%xmm13\n\t" - "vmovdqu 224(%[in]), %%xmm14\n\t" - "vmovdqu 240(%[in]), %%xmm15\n\t" - "vpxor %%xmm8, %%xmm0, %%xmm0\n\t" - "vpxor %%xmm9, %%xmm1, %%xmm1\n\t" - "vpxor %%xmm10, %%xmm2, %%xmm2\n\t" - "vpxor %%xmm11, %%xmm3, %%xmm3\n\t" - "vpxor %%xmm12, %%xmm4, %%xmm4\n\t" - "vpxor %%xmm13, %%xmm5, %%xmm5\n\t" - "vpxor %%xmm14, %%xmm6, %%xmm6\n\t" - "vpxor %%xmm15, %%xmm7, %%xmm7\n\t" - "vmovdqu %%xmm0, 32(%[out])\n\t" - "vmovdqu %%xmm1, 48(%[out])\n\t" - "vmovdqu %%xmm2, 96(%[out])\n\t" - "vmovdqu %%xmm3, 112(%[out])\n\t" - "vmovdqu %%xmm4, 160(%[out])\n\t" - "vmovdqu %%xmm5, 176(%[out])\n\t" - "vmovdqu %%xmm6, 224(%[out])\n\t" - "vmovdqu %%xmm7, 240(%[out])\n\t" - - "vmovdqa 192(%[X]), %%xmm12\n\t" - "add $256, %[in]\n\t" - "add $256, %[out]\n\t" - "vpaddd %[four], %%xmm12, %%xmm12\n\t" - "sub $256, %[bytes]\n\t" - "vmovdqa %%xmm12, 192(%[X])\n\t" - "cmp $256, %[bytes]\n\t" - "jl L_done\n\t" - - "vmovdqa (%[X]), %%xmm0\n\t" - "vmovdqa 16(%[X]), %%xmm1\n\t" - "vmovdqa 32(%[X]), %%xmm2\n\t" - "vmovdqa 48(%[X]), %%xmm3\n\t" - "vmovdqa 64(%[X]), %%xmm4\n\t" - "vmovdqa 80(%[X]), %%xmm5\n\t" - "vmovdqa 96(%[X]), %%xmm6\n\t" - "vmovdqa 112(%[X]), %%xmm7\n\t" - "vmovdqa 128(%[X]), %%xmm8\n\t" - "vmovdqa 144(%[X]), %%xmm9\n\t" - "vmovdqa 160(%[X]), %%xmm10\n\t" - "vmovdqa 176(%[X]), %%xmm11\n\t" - "vmovdqa 192(%[X]), %%xmm12\n\t" - "vmovdqa 208(%[X]), %%xmm13\n\t" - "vmovdqa 224(%[X]), %%xmm14\n\t" - "vmovdqa 240(%[X]), %%xmm15\n\t" - "jmp L_enc128_loop\n\t" - - "\n" - "L_done:\n\t" - - "shl $2, %[cnt]\n\t" - "add 48(%[key]), %[cnt]\n\t" - "movl %[cnt], 48(%[key])\n\t" - "\n" - "L_end128:\n\t" - : [bytes] "+r" (bytes), [cnt] "+r" (cnt), - [in] "+r" (m), [out] "+r" (c) - : [X] "r" (X), [x] "r" (x), [key] "r" (ctx->X), - [add] "xrm" (add), [four] "xrm" (four), - [rotl8] "xrm" (rotl8), [rotl16] "xrm" (rotl16) - : "xmm0", "xmm1", "xmm2", "xmm3", - "xmm4", "xmm5", "xmm6", "xmm7", - "xmm8", "xmm9", "xmm10", "xmm11", - "xmm12", "xmm13", "xmm14", "xmm15", "memory" - ); - - for (; bytes >= CHACHA_CHUNK_BYTES;) { - CHACHA_CHUNK_AVX(); - bytes -= CHACHA_CHUNK_BYTES; - c += CHACHA_CHUNK_BYTES; - m += CHACHA_CHUNK_BYTES; - } - if (bytes > 0) { - CHACHA_PARTIAL_CHUNK_AVX(); - } -} -#endif /* HAVE_INTEL_AVX1 */ - -#ifdef HAVE_INTEL_AVX2 -#define QUARTERROUND_2_AVX2() \ - "vpaddd %%xmm1, %%xmm0, %%xmm0\n\t" \ - "vpxor %%xmm0, %%xmm3, %%xmm3\n\t" \ - "vpshufb %[rotl16], %%xmm3, %%xmm3\n\t" \ - "vpaddd %%xmm3, %%xmm2, %%xmm2\n\t" \ - "vpxor %%xmm2, %%xmm1, %%xmm1\n\t" \ - "vpsrld $20, %%xmm1, %%xmm4\n\t" \ - "vpslld $12, %%xmm1, %%xmm1\n\t" \ - "vpxor %%xmm4, %%xmm1, %%xmm1\n\t" \ - "vpaddd %%xmm1, %%xmm0, %%xmm0\n\t" \ - "vpxor %%xmm0, %%xmm3, %%xmm3\n\t" \ - "vpshufb %[rotl8], %%xmm3, %%xmm3\n\t" \ - "vpaddd %%xmm3, %%xmm2, %%xmm2\n\t" \ - "vpxor %%xmm2, %%xmm1, %%xmm1\n\t" \ - "vpsrld $25, %%xmm1, %%xmm4\n\t" \ - "vpslld $7, %%xmm1, %%xmm1\n\t" \ - "vpxor %%xmm4, %%xmm1, %%xmm1\n\t" \ - "# Swap words for next round\n\t" \ - "vpshufd $0x39, %%xmm1, %%xmm1\n\t" \ - "vpshufd $0x4e, %%xmm2, %%xmm2\n\t" \ - "vpshufd $0x93, %%xmm3, %%xmm3\n\t" \ - "vpaddd %%xmm1, %%xmm0, %%xmm0\n\t" \ - "vpxor %%xmm0, %%xmm3, %%xmm3\n\t" \ - "vpshufb %[rotl16], %%xmm3, %%xmm3\n\t" \ - "vpaddd %%xmm3, %%xmm2, %%xmm2\n\t" \ - "vpxor %%xmm2, %%xmm1, %%xmm1\n\t" \ - "vpsrld $20, %%xmm1, %%xmm4\n\t" \ - "vpslld $12, %%xmm1, %%xmm1\n\t" \ - "vpxor %%xmm4, %%xmm1, %%xmm1\n\t" \ - "vpaddd %%xmm1, %%xmm0, %%xmm0\n\t" \ - "vpxor %%xmm0, %%xmm3, %%xmm3\n\t" \ - "vpshufb %[rotl8], %%xmm3, %%xmm3\n\t" \ - "vpaddd %%xmm3, %%xmm2, %%xmm2\n\t" \ - "vpxor %%xmm2, %%xmm1, %%xmm1\n\t" \ - "vpsrld $25, %%Xmm1, %%xmm4\n\t" \ - "vpslld $7, %%xmm1, %%xmm1\n\t" \ - "vpxor %%xmm4, %%xmm1, %%xmm1\n\t" \ - "# Swap words back\n\t" \ - "vpshufd $0x93, %%xmm1, %%xmm1\n\t" \ - "vpshufd $0x4e, %%xmm2, %%xmm2\n\t" \ - "vpshufd $0x39, %%xmm3, %%xmm3\n\t" \ - -#define CHACHA_CRYPT_AVX2() \ - "vmovdqu 0(%[input]), %%xmm8\n\t" \ - "vmovdqu 16(%[input]), %%xmm9\n\t" \ - "vmovdqu 32(%[input]), %%xmm10\n\t" \ - "vmovdqu 48(%[input]), %%xmm11\n\t" \ - "vmovdqu %%xmm8, %%xmm0\n\t" \ - "vmovdqu %%xmm9, %%xmm1\n\t" \ - "vmovdqu %%xmm10, %%xmm2\n\t" \ - "vmovdqu %%xmm11, %%xmm3\n\t" \ - "movb $10, %%al\n\t" \ - "\n" \ - "1:\n\t" \ - QUARTERROUND_2_AVX2() \ - "decb %%al\n\t" \ - "jnz 1b\n\t" \ - "vpaddd %%xmm8, %%xmm0, %%xmm0\n\t" \ - "vpaddd %%xmm9, %%xmm1, %%xmm1\n\t" \ - "vpaddd %%xmm10, %%xmm2, %%xmm2\n\t" \ - "vpaddd %%xmm11, %%xmm3, %%xmm3\n\t" \ - -#define CHACHA_PARTIAL_CHUNK_AVX2() \ - __asm__ __volatile__ ( \ - CHACHA_CRYPT_AVX2() \ - "vmovdqu %%xmm0, 0(%[c])\n\t" \ - "vmovdqu %%xmm1, 16(%[c])\n\t" \ - "vmovdqu %%xmm2, 32(%[c])\n\t" \ - "vmovdqu %%xmm3, 48(%[c])\n\t" \ - "addl $1, 48(%[input])\n\t" \ - "movl %[bytes], %%r8d\n\t" \ - "xorq %%rdx, %%rdx\n\t" \ - "movl %%r8d, %%r9d\n\t" \ - "andl $7, %%r9d\n\t" \ - "jz 4f\n\t" \ - "\n" \ - "2:\n\t" \ - "movzbl (%[c],%%rdx,1), %%ecx\n\t" \ - "xorb (%[m],%%rdx,1), %%cl\n\t" \ - "movb %%cl, (%[output],%%rdx,1)\n\t" \ - "incl %%edx\n\t" \ - "cmpl %%r9d, %%edx\n\t" \ - "jne 2b\n\t" \ - "je 3f\n\t" \ - "\n" \ - "4:\n\t" \ - "movq (%[c],%%rdx,1), %%rcx\n\t" \ - "xorq (%[m],%%rdx,1), %%rcx\n\t" \ - "movq %%rcx, (%[output],%%rdx,1)\n\t" \ - "addl $8, %%edx\n\t" \ - "\n" \ - "3:\n\t" \ - "cmpl %%r8d, %%edx\n\t" \ - "jne 4b\n\t" \ - : \ - : [input] "r" (ctx->X), [c] "r" (x), \ - [output] "r" (c), [bytes] "r" (bytes), [m] "r" (m), \ - [rotl8] "xrm" (rotl8), [rotl16] "xrm" (rotl16) \ - : "eax", "ecx", "edx", "r8", "r9", "memory", \ - "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", \ - "xmm8", "xmm9", "xmm10", "xmm11" \ - ) - - -#define CHACHA_CHUNK_AVX2() \ - __asm__ __volatile__ ( \ - CHACHA_CRYPT_AVX2() \ - "vmovdqu 0(%[m]), %%xmm4\n\t" \ - "vmovdqu 16(%[m]), %%xmm5\n\t" \ - "vmovdqu 32(%[m]), %%xmm6\n\t" \ - "vmovdqu 48(%[m]), %%xmm7\n\t" \ - "vpxor %%xmm4, %%xmm0, %%xmm0\n\t" \ - "vpxor %%xmm5, %%xmm1, %%xmm1\n\t" \ - "vpxor %%xmm6, %%xmm2, %%xmm2\n\t" \ - "vpxor %%xmm7, %%xmm3, %%xmm3\n\t" \ - "vmovdqu %%xmm0, 0(%[c])\n\t" \ - "vmovdqu %%xmm1, 16(%[c])\n\t" \ - "vmovdqu %%xmm2, 32(%[c])\n\t" \ - "vmovdqu %%xmm3, 48(%[c])\n\t" \ - "addl $1, 48(%[input])\n\t" \ - : \ - : [input] "r" (ctx->X), [c] "r" (c), [m] "r" (m), \ - [rotl8] "xrm" (rotl8), [rotl16] "xrm" (rotl16) \ - : "rax", "memory", \ - "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", \ - "xmm8", "xmm9", "xmm10", "xmm11" \ - ) - - -static void chacha_encrypt_avx2(ChaCha* ctx, const byte* m, byte* c, - word32 bytes) -{ - ALIGN256 word32 X[8*CHACHA_CHUNK_WORDS]; /* used to make sure aligned */ - ALIGN256 word32 x[4*CHACHA_CHUNK_WORDS]; /* used to make sure aligned */ - word32 cnt = 0; - static const __m256i add = { 0x0000000100000000UL,0x0000000300000002UL, - 0x0000000500000004UL,0x0000000700000006UL }; - static const __m256i eight = { 0x0000000800000008UL,0x0000000800000008UL, - 0x0000000800000008UL,0x0000000800000008UL }; - static const __m256i rotl8_256 = - { 0x0605040702010003UL,0x0e0d0c0f0a09080bUL, - 0x0605040702010003UL,0x0e0d0c0f0a09080bUL }; - static const __m256i rotl16_256 = - { 0x0504070601000302UL,0x0d0c0f0e09080b0aUL, - 0x0504070601000302UL,0x0d0c0f0e09080b0aUL }; - - if (bytes == 0) - return; - - __asm__ __volatile__ ( - "movl %[bytes], %[cnt]\n\t" - "shrl $9, %[cnt]\n\t" - "jz L_end256\n\t" - - "vpbroadcastd (%[key]), %%ymm0\n\t" - "vpbroadcastd 4(%[key]), %%ymm1\n\t" - "vpbroadcastd 8(%[key]), %%ymm2\n\t" - "vpbroadcastd 12(%[key]), %%ymm3\n\t" - "vpbroadcastd 16(%[key]), %%ymm4\n\t" - "vpbroadcastd 20(%[key]), %%ymm5\n\t" - "vpbroadcastd 24(%[key]), %%ymm6\n\t" - "vpbroadcastd 28(%[key]), %%ymm7\n\t" - "vpbroadcastd 32(%[key]), %%ymm8\n\t" - "vpbroadcastd 36(%[key]), %%ymm9\n\t" - "vpbroadcastd 40(%[key]), %%ymm10\n\t" - "vpbroadcastd 44(%[key]), %%ymm11\n\t" - "vpbroadcastd 48(%[key]), %%ymm12\n\t" - "vpbroadcastd 52(%[key]), %%ymm13\n\t" - "vpbroadcastd 56(%[key]), %%ymm14\n\t" - "vpbroadcastd 60(%[key]), %%ymm15\n\t" - - "vpaddd %[add], %%ymm12, %%ymm12\n\t" - - "vmovdqa %%ymm0, (%[X])\n\t" - "vmovdqa %%ymm1, 32(%[X])\n\t" - "vmovdqa %%ymm2, 64(%[X])\n\t" - "vmovdqa %%ymm3, 96(%[X])\n\t" - "vmovdqa %%ymm4, 128(%[X])\n\t" - "vmovdqa %%ymm5, 160(%[X])\n\t" - "vmovdqa %%ymm6, 192(%[X])\n\t" - "vmovdqa %%ymm7, 224(%[X])\n\t" - "vmovdqa %%ymm8, 256(%[X])\n\t" - "vmovdqa %%ymm9, 288(%[X])\n\t" - "vmovdqa %%ymm10, 320(%[X])\n\t" - "vmovdqa %%ymm11, 352(%[X])\n\t" - "vmovdqa %%ymm12, 384(%[X])\n\t" - "vmovdqa %%ymm13, 416(%[X])\n\t" - "vmovdqa %%ymm14, 448(%[X])\n\t" - "vmovdqa %%ymm15, 480(%[X])\n\t" - "\n" - "L_enc256_loop:\n\t" - "vmovdqa %%ymm11, 96(%[x])\n\t" - QUARTERROUND_YMM() - QUARTERROUND_YMM_2() - QUARTERROUND_YMM() - QUARTERROUND_YMM_2() - QUARTERROUND_YMM() - QUARTERROUND_YMM_2() - QUARTERROUND_YMM() - QUARTERROUND_YMM_2() - QUARTERROUND_YMM() - QUARTERROUND_YMM_2() - QUARTERROUND_YMM() - QUARTERROUND_YMM_2() - QUARTERROUND_YMM() - QUARTERROUND_YMM_2() - QUARTERROUND_YMM() - QUARTERROUND_YMM_2() - QUARTERROUND_YMM() - QUARTERROUND_YMM_2() - QUARTERROUND_YMM() - QUARTERROUND_YMM_2() - "vmovdqa 96(%[x]), %%ymm11\n\t" - - "vpaddd (%[X]), %%ymm0, %%ymm0\n\t" - "vpaddd 32(%[X]), %%ymm1, %%ymm1\n\t" - "vpaddd 64(%[X]), %%ymm2, %%ymm2\n\t" - "vpaddd 96(%[X]), %%ymm3, %%ymm3\n\t" - "vpaddd 128(%[X]), %%ymm4, %%ymm4\n\t" - "vpaddd 160(%[X]), %%ymm5, %%ymm5\n\t" - "vpaddd 192(%[X]), %%ymm6, %%ymm6\n\t" - "vpaddd 224(%[X]), %%ymm7, %%ymm7\n\t" - "vpaddd 256(%[X]), %%ymm8, %%ymm8\n\t" - "vpaddd 288(%[X]), %%ymm9, %%ymm9\n\t" - "vpaddd 320(%[X]), %%ymm10, %%ymm10\n\t" - "vpaddd 352(%[X]), %%ymm11, %%ymm11\n\t" - "vpaddd 384(%[X]), %%ymm12, %%ymm12\n\t" - "vpaddd 416(%[X]), %%ymm13, %%ymm13\n\t" - "vpaddd 448(%[X]), %%ymm14, %%ymm14\n\t" - "vpaddd 480(%[X]), %%ymm15, %%ymm15\n\t" - - "vmovdqa %%ymm8, (%[x])\n\t" - "vmovdqa %%ymm9, 32(%[x])\n\t" - "vmovdqa %%ymm10, 64(%[x])\n\t" - "vmovdqa %%ymm11, 96(%[x])\n\t" - "vmovdqa %%ymm12, 128(%[x])\n\t" - "vmovdqa %%ymm13, 160(%[x])\n\t" - "vmovdqa %%ymm14, 192(%[x])\n\t" - "vmovdqa %%ymm15, 224(%[x])\n\t" - - "vpunpckldq %%ymm1, %%ymm0, %%ymm8\n\t" - "vpunpckldq %%ymm3, %%ymm2, %%ymm9\n\t" - "vpunpckhdq %%ymm1, %%ymm0, %%ymm12\n\t" - "vpunpckhdq %%ymm3, %%ymm2, %%ymm13\n\t" - "vpunpckldq %%ymm5, %%ymm4, %%ymm10\n\t" - "vpunpckldq %%ymm7, %%ymm6, %%ymm11\n\t" - "vpunpckhdq %%ymm5, %%ymm4, %%ymm14\n\t" - "vpunpckhdq %%ymm7, %%ymm6, %%ymm15\n\t" - "vpunpcklqdq %%ymm9, %%ymm8, %%ymm0\n\t" - "vpunpcklqdq %%ymm11, %%ymm10, %%ymm1\n\t" - "vpunpckhqdq %%ymm9, %%ymm8, %%ymm2\n\t" - "vpunpckhqdq %%ymm11, %%ymm10, %%ymm3\n\t" - "vpunpcklqdq %%ymm13, %%ymm12, %%ymm4\n\t" - "vpunpcklqdq %%ymm15, %%ymm14, %%ymm5\n\t" - "vpunpckhqdq %%ymm13, %%ymm12, %%ymm6\n\t" - "vpunpckhqdq %%ymm15, %%ymm14, %%ymm7\n\t" - "vperm2i128 $0x20, %%ymm1, %%ymm0, %%ymm8\n\t" - "vperm2i128 $0x20, %%ymm3, %%ymm2, %%ymm9\n\t" - "vperm2i128 $0x31, %%ymm1, %%ymm0, %%ymm12\n\t" - "vperm2i128 $0x31, %%ymm3, %%ymm2, %%ymm13\n\t" - "vperm2i128 $0x20, %%ymm5, %%ymm4, %%ymm10\n\t" - "vperm2i128 $0x20, %%ymm7, %%ymm6, %%ymm11\n\t" - "vperm2i128 $0x31, %%ymm5, %%ymm4, %%ymm14\n\t" - "vperm2i128 $0x31, %%ymm7, %%ymm6, %%ymm15\n\t" - - "vmovdqu (%[in]), %%ymm0\n\t" - "vmovdqu 64(%[in]), %%ymm1\n\t" - "vmovdqu 128(%[in]), %%ymm2\n\t" - "vmovdqu 192(%[in]), %%ymm3\n\t" - "vmovdqu 256(%[in]), %%ymm4\n\t" - "vmovdqu 320(%[in]), %%ymm5\n\t" - "vmovdqu 384(%[in]), %%ymm6\n\t" - "vmovdqu 448(%[in]), %%ymm7\n\t" - "vpxor %%ymm0, %%ymm8, %%ymm8\n\t" - "vpxor %%ymm1, %%ymm9, %%ymm9\n\t" - "vpxor %%ymm2, %%ymm10, %%ymm10\n\t" - "vpxor %%ymm3, %%ymm11, %%ymm11\n\t" - "vpxor %%ymm4, %%ymm12, %%ymm12\n\t" - "vpxor %%ymm5, %%ymm13, %%ymm13\n\t" - "vpxor %%ymm6, %%ymm14, %%ymm14\n\t" - "vpxor %%ymm7, %%ymm15, %%ymm15\n\t" - "vmovdqu %%ymm8, (%[out])\n\t" - "vmovdqu %%ymm9, 64(%[out])\n\t" - "vmovdqu %%ymm10, 128(%[out])\n\t" - "vmovdqu %%ymm11, 192(%[out])\n\t" - "vmovdqu %%ymm12, 256(%[out])\n\t" - "vmovdqu %%ymm13, 320(%[out])\n\t" - "vmovdqu %%ymm14, 384(%[out])\n\t" - "vmovdqu %%ymm15, 448(%[out])\n\t" - - "vmovdqa (%[x]), %%ymm0\n\t" - "vmovdqa 32(%[x]), %%ymm1\n\t" - "vmovdqa 64(%[x]), %%ymm2\n\t" - "vmovdqa 96(%[x]), %%ymm3\n\t" - "vmovdqa 128(%[x]), %%ymm4\n\t" - "vmovdqa 160(%[x]), %%ymm5\n\t" - "vmovdqa 192(%[x]), %%ymm6\n\t" - "vmovdqa 224(%[x]), %%ymm7\n\t" - - "vpunpckldq %%ymm1, %%ymm0, %%ymm8\n\t" - "vpunpckldq %%ymm3, %%ymm2, %%ymm9\n\t" - "vpunpckhdq %%ymm1, %%ymm0, %%ymm12\n\t" - "vpunpckhdq %%ymm3, %%ymm2, %%ymm13\n\t" - "vpunpckldq %%ymm5, %%ymm4, %%ymm10\n\t" - "vpunpckldq %%ymm7, %%ymm6, %%ymm11\n\t" - "vpunpckhdq %%ymm5, %%ymm4, %%ymm14\n\t" - "vpunpckhdq %%ymm7, %%ymm6, %%ymm15\n\t" - "vpunpcklqdq %%ymm9, %%ymm8, %%ymm0\n\t" - "vpunpcklqdq %%ymm11, %%ymm10, %%ymm1\n\t" - "vpunpckhqdq %%ymm9 , %%ymm8, %%ymm2\n\t" - "vpunpckhqdq %%ymm11, %%ymm10, %%ymm3\n\t" - "vpunpcklqdq %%ymm13, %%ymm12, %%ymm4\n\t" - "vpunpcklqdq %%ymm15, %%ymm14, %%ymm5\n\t" - "vpunpckhqdq %%ymm13, %%ymm12, %%ymm6\n\t" - "vpunpckhqdq %%ymm15, %%ymm14, %%ymm7\n\t" - "vperm2i128 $0x20, %%ymm1, %%ymm0, %%ymm8\n\t" - "vperm2i128 $0x20, %%ymm3, %%ymm2, %%ymm9\n\t" - "vperm2i128 $0x31, %%ymm1, %%ymm0, %%ymm12\n\t" - "vperm2i128 $0x31, %%ymm3, %%ymm2, %%ymm13\n\t" - "vperm2i128 $0x20, %%ymm5, %%ymm4, %%ymm10\n\t" - "vperm2i128 $0x20, %%ymm7, %%ymm6, %%ymm11\n\t" - "vperm2i128 $0x31, %%ymm5, %%ymm4, %%ymm14\n\t" - "vperm2i128 $0x31, %%ymm7, %%ymm6, %%ymm15\n\t" - - "vmovdqu 32(%[in]), %%ymm0\n\t" - "vmovdqu 96(%[in]), %%ymm1\n\t" - "vmovdqu 160(%[in]), %%ymm2\n\t" - "vmovdqu 224(%[in]), %%ymm3\n\t" - "vmovdqu 288(%[in]), %%ymm4\n\t" - "vmovdqu 352(%[in]), %%ymm5\n\t" - "vmovdqu 416(%[in]), %%ymm6\n\t" - "vmovdqu 480(%[in]), %%ymm7\n\t" - "vpxor %%ymm0, %%ymm8, %%ymm8\n\t" - "vpxor %%ymm1, %%ymm9, %%ymm9\n\t" - "vpxor %%ymm2, %%ymm10, %%ymm10\n\t" - "vpxor %%ymm3, %%ymm11, %%ymm11\n\t" - "vpxor %%ymm4, %%ymm12, %%ymm12\n\t" - "vpxor %%ymm5, %%ymm13, %%ymm13\n\t" - "vpxor %%ymm6, %%ymm14, %%ymm14\n\t" - "vpxor %%ymm7, %%ymm15, %%ymm15\n\t" - "vmovdqu %%ymm8, 32(%[out])\n\t" - "vmovdqu %%ymm9, 96(%[out])\n\t" - "vmovdqu %%ymm10, 160(%[out])\n\t" - "vmovdqu %%ymm11, 224(%[out])\n\t" - "vmovdqu %%ymm12, 288(%[out])\n\t" - "vmovdqu %%ymm13, 352(%[out])\n\t" - "vmovdqu %%ymm14, 416(%[out])\n\t" - "vmovdqu %%ymm15, 480(%[out])\n\t" - - "vmovdqa 384(%[X]), %%ymm12\n\t" - "add $512, %[in]\n\t" - "add $512, %[out]\n\t" - "vpaddd %[eight], %%ymm12, %%ymm12\n\t" - "sub $512, %[bytes]\n\t" - "vmovdqa %%ymm12, 384(%[X])\n\t" - "cmp $512, %[bytes]\n\t" - "jl L_done256\n\t" - - "vmovdqa (%[X]), %%ymm0\n\t" - "vmovdqa 32(%[X]), %%ymm1\n\t" - "vmovdqa 64(%[X]), %%ymm2\n\t" - "vmovdqa 96(%[X]), %%ymm3\n\t" - "vmovdqa 128(%[X]), %%ymm4\n\t" - "vmovdqa 160(%[X]), %%ymm5\n\t" - "vmovdqa 192(%[X]), %%ymm6\n\t" - "vmovdqa 224(%[X]), %%ymm7\n\t" - "vmovdqa 256(%[X]), %%ymm8\n\t" - "vmovdqa 288(%[X]), %%ymm9\n\t" - "vmovdqa 320(%[X]), %%ymm10\n\t" - "vmovdqa 352(%[X]), %%ymm11\n\t" - "vmovdqa 384(%[X]), %%ymm12\n\t" - "vmovdqa 416(%[X]), %%ymm13\n\t" - "vmovdqa 448(%[X]), %%ymm14\n\t" - "vmovdqa 480(%[X]), %%ymm15\n\t" - "jmp L_enc256_loop\n\t" - "\n" - "L_done256:\n\t" - "shl $3, %[cnt]\n\t" - "add 48(%[key]), %[cnt]\n\t" - "movl %[cnt], 48(%[key])\n\t" - "\n" - "L_end256:\n\t" - : [bytes] "+r" (bytes), [cnt] "+r" (cnt), - [in] "+r" (m), [out] "+r" (c) - : [X] "r" (X), [x] "r" (x), [key] "r" (ctx->X), - [add] "m" (add), [eight] "m" (eight), - [rotl8] "m" (rotl8_256), [rotl16] "m" (rotl16_256) - : "ymm0", "ymm1", "ymm2", "ymm3", - "ymm4", "ymm5", "ymm6", "ymm7", - "ymm8", "ymm9", "ymm10", "ymm11", - "ymm12", "ymm13", "ymm14", "ymm15", "memory" - ); - - /* AVX code optimised for multiples of 256 bytes. */ - if (bytes == 256) { - chacha_encrypt_avx(ctx, m, c, bytes); - bytes -= 256; - } - - for (; bytes >= CHACHA_CHUNK_BYTES;) { - CHACHA_CHUNK_AVX2(); - bytes -= CHACHA_CHUNK_BYTES; - c += CHACHA_CHUNK_BYTES; - m += CHACHA_CHUNK_BYTES; - } - if (bytes > 0) { - CHACHA_PARTIAL_CHUNK_AVX2(); - } -} -#endif /* HAVE_INTEL_AVX2 */ -#endif /* USE_INTEL_CHACHA_SPEEDUP */ /** * Encrypt a stream of bytes @@ -1500,7 +300,7 @@ int wc_Chacha_Process(ChaCha* ctx, byte* output, const byte* input, } #endif if (IS_INTEL_AVX1(cpuidFlags)) { - chacha_encrypt_avx(ctx, input, output, msglen); + chacha_encrypt_avx1(ctx, input, output, msglen); return 0; } else { diff --git a/wolfcrypt/src/chacha_asm.S b/wolfcrypt/src/chacha_asm.S new file mode 100644 index 0000000000..4be9cdb0f7 --- /dev/null +++ b/wolfcrypt/src/chacha_asm.S @@ -0,0 +1,1314 @@ +/* chacha_asm + * + * Copyright (C) 2006-2018 wolfSSL Inc. + * + * This file is part of wolfSSL. + * + * wolfSSL is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * wolfSSL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA + */ + +#ifndef HAVE_INTEL_AVX1 +#define HAVE_INTEL_AVX1 +#endif /* HAVE_INTEL_AVX1 */ +#ifndef HAVE_INTEL_AVX2 +#define HAVE_INTEL_AVX2 +#endif /* HAVE_INTEL_AVX2 */ + +.globl chacha_encrypt_x64 +.type chacha_encrypt_x64,@function +.align 4 +chacha_encrypt_x64: + pushq %rbx + pushq %rbp + pushq %r12 + pushq %r13 + pushq %r14 + pushq %r15 + subq $0x40, %rsp + cmpl $0x40, %ecx + jl L_chacha_x64_small +L_chacha_x64_start: + subq $48, %rsp + movq %rdx, 24(%rsp) + movq %rsi, 32(%rsp) + movq %rcx, 40(%rsp) + movq 32(%rdi), %rax + movq 40(%rdi), %rbx + movq %rax, 8(%rsp) + movq %rbx, 16(%rsp) + movl (%rdi), %eax + movl 4(%rdi), %ebx + movl 8(%rdi), %ecx + movl 12(%rdi), %edx + movl 16(%rdi), %r8d + movl 20(%rdi), %r9d + movl 24(%rdi), %r10d + movl 28(%rdi), %r11d + movl 48(%rdi), %r12d + movl 52(%rdi), %r13d + movl 56(%rdi), %r14d + movl 60(%rdi), %r15d + movb $10, (%rsp) + movl 8(%rsp), %esi + movl 12(%rsp), %ebp +L_chacha_x64_block_crypt_start: + addl %r8d, %eax + addl %r9d, %ebx + xorl %eax, %r12d + xorl %ebx, %r13d + roll $16, %r12d + roll $16, %r13d + addl %r12d, %esi + addl %r13d, %ebp + xorl %esi, %r8d + xorl %ebp, %r9d + roll $12, %r8d + roll $12, %r9d + addl %r8d, %eax + addl %r9d, %ebx + xorl %eax, %r12d + xorl %ebx, %r13d + roll $8, %r12d + roll $8, %r13d + addl %r12d, %esi + addl %r13d, %ebp + xorl %esi, %r8d + xorl %ebp, %r9d + roll $7, %r8d + roll $7, %r9d + movl %esi, 8(%rsp) + movl %ebp, 12(%rsp) + movl 16(%rsp), %esi + movl 20(%rsp), %ebp + addl %r10d, %ecx + addl %r11d, %edx + xorl %ecx, %r14d + xorl %edx, %r15d + roll $16, %r14d + roll $16, %r15d + addl %r14d, %esi + addl %r15d, %ebp + xorl %esi, %r10d + xorl %ebp, %r11d + roll $12, %r10d + roll $12, %r11d + addl %r10d, %ecx + addl %r11d, %edx + xorl %ecx, %r14d + xorl %edx, %r15d + roll $8, %r14d + roll $8, %r15d + addl %r14d, %esi + addl %r15d, %ebp + xorl %esi, %r10d + xorl %ebp, %r11d + roll $7, %r10d + roll $7, %r11d + addl %r9d, %eax + addl %r10d, %ebx + xorl %eax, %r15d + xorl %ebx, %r12d + roll $16, %r15d + roll $16, %r12d + addl %r15d, %esi + addl %r12d, %ebp + xorl %esi, %r9d + xorl %ebp, %r10d + roll $12, %r9d + roll $12, %r10d + addl %r9d, %eax + addl %r10d, %ebx + xorl %eax, %r15d + xorl %ebx, %r12d + roll $8, %r15d + roll $8, %r12d + addl %r15d, %esi + addl %r12d, %ebp + xorl %esi, %r9d + xorl %ebp, %r10d + roll $7, %r9d + roll $7, %r10d + movl %esi, 16(%rsp) + movl %ebp, 20(%rsp) + movl 8(%rsp), %esi + movl 12(%rsp), %ebp + addl %r11d, %ecx + addl %r8d, %edx + xorl %ecx, %r13d + xorl %edx, %r14d + roll $16, %r13d + roll $16, %r14d + addl %r13d, %esi + addl %r14d, %ebp + xorl %esi, %r11d + xorl %ebp, %r8d + roll $12, %r11d + roll $12, %r8d + addl %r11d, %ecx + addl %r8d, %edx + xorl %ecx, %r13d + xorl %edx, %r14d + roll $8, %r13d + roll $8, %r14d + addl %r13d, %esi + addl %r14d, %ebp + xorl %esi, %r11d + xorl %ebp, %r8d + roll $7, %r11d + roll $7, %r8d + decb (%rsp) + jnz L_chacha_x64_block_crypt_start + movl %esi, 8(%rsp) + movl %ebp, 12(%rsp) + movq 32(%rsp), %rsi + movq 24(%rsp), %rbp + addl (%rdi), %eax + addl 4(%rdi), %ebx + addl 8(%rdi), %ecx + addl 12(%rdi), %edx + addl 16(%rdi), %r8d + addl 20(%rdi), %r9d + addl 24(%rdi), %r10d + addl 28(%rdi), %r11d + addl 48(%rdi), %r12d + addl 52(%rdi), %r13d + addl 56(%rdi), %r14d + addl 60(%rdi), %r15d + xorl (%rsi), %eax + xorl 4(%rsi), %ebx + xorl 8(%rsi), %ecx + xorl 12(%rsi), %edx + xorl 16(%rsi), %r8d + xorl 20(%rsi), %r9d + xorl 24(%rsi), %r10d + xorl 28(%rsi), %r11d + xorl 48(%rsi), %r12d + xorl 52(%rsi), %r13d + xorl 56(%rsi), %r14d + xorl 60(%rsi), %r15d + movl %eax, (%rbp) + movl %ebx, 4(%rbp) + movl %ecx, 8(%rbp) + movl %edx, 12(%rbp) + movl %r8d, 16(%rbp) + movl %r9d, 20(%rbp) + movl %r10d, 24(%rbp) + movl %r11d, 28(%rbp) + movl %r12d, 48(%rbp) + movl %r13d, 52(%rbp) + movl %r14d, 56(%rbp) + movl %r15d, 60(%rbp) + movl 8(%rsp), %eax + movl 12(%rsp), %ebx + movl 16(%rsp), %ecx + movl 20(%rsp), %edx + addl 32(%rdi), %eax + addl 36(%rdi), %ebx + addl 40(%rdi), %ecx + addl 44(%rdi), %edx + xorl 32(%rsi), %eax + xorl 36(%rsi), %ebx + xorl 40(%rsi), %ecx + xorl 44(%rsi), %edx + movl %eax, 32(%rbp) + movl %ebx, 36(%rbp) + movl %ecx, 40(%rbp) + movl %edx, 44(%rbp) + movq 24(%rsp), %rdx + movq 40(%rsp), %rcx + addl $0x01, 48(%rdi) + addq $48, %rsp + subl $0x40, %ecx + addq $0x40, %rsi + addq $0x40, %rdx + cmpl $0x40, %ecx + jge L_chacha_x64_start +L_chacha_x64_small: + cmpl $0x00, %ecx + je L_chacha_x64_done + subq $48, %rsp + movq %rdx, 24(%rsp) + movq %rsi, 32(%rsp) + movq %rcx, 40(%rsp) + movq 32(%rdi), %rax + movq 40(%rdi), %rbx + movq %rax, 8(%rsp) + movq %rbx, 16(%rsp) + movl (%rdi), %eax + movl 4(%rdi), %ebx + movl 8(%rdi), %ecx + movl 12(%rdi), %edx + movl 16(%rdi), %r8d + movl 20(%rdi), %r9d + movl 24(%rdi), %r10d + movl 28(%rdi), %r11d + movl 48(%rdi), %r12d + movl 52(%rdi), %r13d + movl 56(%rdi), %r14d + movl 60(%rdi), %r15d + movb $10, (%rsp) + movl 8(%rsp), %esi + movl 12(%rsp), %ebp +L_chacha_x64_partial_crypt_start: + addl %r8d, %eax + addl %r9d, %ebx + xorl %eax, %r12d + xorl %ebx, %r13d + roll $16, %r12d + roll $16, %r13d + addl %r12d, %esi + addl %r13d, %ebp + xorl %esi, %r8d + xorl %ebp, %r9d + roll $12, %r8d + roll $12, %r9d + addl %r8d, %eax + addl %r9d, %ebx + xorl %eax, %r12d + xorl %ebx, %r13d + roll $8, %r12d + roll $8, %r13d + addl %r12d, %esi + addl %r13d, %ebp + xorl %esi, %r8d + xorl %ebp, %r9d + roll $7, %r8d + roll $7, %r9d + movl %esi, 8(%rsp) + movl %ebp, 12(%rsp) + movl 16(%rsp), %esi + movl 20(%rsp), %ebp + addl %r10d, %ecx + addl %r11d, %edx + xorl %ecx, %r14d + xorl %edx, %r15d + roll $16, %r14d + roll $16, %r15d + addl %r14d, %esi + addl %r15d, %ebp + xorl %esi, %r10d + xorl %ebp, %r11d + roll $12, %r10d + roll $12, %r11d + addl %r10d, %ecx + addl %r11d, %edx + xorl %ecx, %r14d + xorl %edx, %r15d + roll $8, %r14d + roll $8, %r15d + addl %r14d, %esi + addl %r15d, %ebp + xorl %esi, %r10d + xorl %ebp, %r11d + roll $7, %r10d + roll $7, %r11d + addl %r9d, %eax + addl %r10d, %ebx + xorl %eax, %r15d + xorl %ebx, %r12d + roll $16, %r15d + roll $16, %r12d + addl %r15d, %esi + addl %r12d, %ebp + xorl %esi, %r9d + xorl %ebp, %r10d + roll $12, %r9d + roll $12, %r10d + addl %r9d, %eax + addl %r10d, %ebx + xorl %eax, %r15d + xorl %ebx, %r12d + roll $8, %r15d + roll $8, %r12d + addl %r15d, %esi + addl %r12d, %ebp + xorl %esi, %r9d + xorl %ebp, %r10d + roll $7, %r9d + roll $7, %r10d + movl %esi, 16(%rsp) + movl %ebp, 20(%rsp) + movl 8(%rsp), %esi + movl 12(%rsp), %ebp + addl %r11d, %ecx + addl %r8d, %edx + xorl %ecx, %r13d + xorl %edx, %r14d + roll $16, %r13d + roll $16, %r14d + addl %r13d, %esi + addl %r14d, %ebp + xorl %esi, %r11d + xorl %ebp, %r8d + roll $12, %r11d + roll $12, %r8d + addl %r11d, %ecx + addl %r8d, %edx + xorl %ecx, %r13d + xorl %edx, %r14d + roll $8, %r13d + roll $8, %r14d + addl %r13d, %esi + addl %r14d, %ebp + xorl %esi, %r11d + xorl %ebp, %r8d + roll $7, %r11d + roll $7, %r8d + decb (%rsp) + jnz L_chacha_x64_partial_crypt_start + movl %esi, 8(%rsp) + movl %ebp, 12(%rsp) + movq 32(%rsp), %rsi + addl (%rdi), %eax + addl 4(%rdi), %ebx + addl 8(%rdi), %ecx + addl 12(%rdi), %edx + addl 16(%rdi), %r8d + addl 20(%rdi), %r9d + addl 24(%rdi), %r10d + addl 28(%rdi), %r11d + addl 48(%rdi), %r12d + addl 52(%rdi), %r13d + addl 56(%rdi), %r14d + addl 60(%rdi), %r15d + movl %eax, 48(%rsp) + movl %ebx, 52(%rsp) + movl %ecx, 56(%rsp) + movl %edx, 60(%rsp) + movl %r8d, 64(%rsp) + movl %r9d, 68(%rsp) + movl %r10d, 72(%rsp) + movl %r11d, 76(%rsp) + movl %r12d, 96(%rsp) + movl %r13d, 100(%rsp) + movl %r14d, 104(%rsp) + movl %r15d, 108(%rsp) + movl 8(%rsp), %eax + movl 12(%rsp), %ebx + movl 16(%rsp), %ecx + movl 20(%rsp), %edx + addl 32(%rdi), %eax + addl 36(%rdi), %ebx + addl 40(%rdi), %ecx + addl 44(%rdi), %edx + movl %eax, 80(%rsp) + movl %ebx, 84(%rsp) + movl %ecx, 88(%rsp) + movl %edx, 92(%rsp) + movq 24(%rsp), %rdx + movq 40(%rsp), %rcx + addl $0x01, 48(%rdi) + addq $48, %rsp + movl %ecx, %r8d + xorq %rbx, %rbx + andl $7, %r8d + jz L_chacha_x64_partial_start64 +L_chacha_x64_partial_start8: + movzbl (%rsp,%rbx,1), %eax + xorb (%rsi,%rbx,1), %al + movb %al, (%rdx,%rbx,1) + incl %ebx + cmpl %r8d, %ebx + jne L_chacha_x64_partial_start8 + je L_chacha_x64_partial_end64 +L_chacha_x64_partial_start64: + movq (%rsp,%rbx,1), %rax + xorq (%rsi,%rbx,1), %rax + movq %rax, (%rdx,%rbx,1) + addl $8, %ebx +L_chacha_x64_partial_end64: + cmpl %ecx, %ebx + jne L_chacha_x64_partial_start64 +L_chacha_x64_done: + addq $0x40, %rsp + popq %r15 + popq %r14 + popq %r13 + popq %r12 + popq %rbp + popq %rbx + repz retq +.size chacha_encrypt_x64,.-chacha_encrypt_x64 +#ifdef HAVE_INTEL_AVX1 +.align 16 +L_chacha20_avx1_rotl8: +.quad 0x605040702010003, 0xe0d0c0f0a09080b +.align 16 +L_chacha20_avx1_rotl16: +.quad 0x504070601000302, 0xd0c0f0e09080b0a +.align 16 +L_chacha20_avx1_add: +.quad 0x100000000, 0x300000002 +.align 16 +L_chacha20_avx1_four: +.quad 0x400000004, 0x400000004 +.globl chacha_encrypt_avx1 +.type chacha_encrypt_avx1,@function +.align 4 +chacha_encrypt_avx1: + subq $0x190, %rsp + movq %rsp, %r9 + leaq 256(%rsp), %r10 + andq $-16, %r9 + andq $-16, %r10 + movl %ecx, %eax + shrl $8, %eax + jz L_chacha20_avx1_end128 + vpshufd $0x00, (%rdi), %xmm0 + vpshufd $0x00, 4(%rdi), %xmm1 + vpshufd $0x00, 8(%rdi), %xmm2 + vpshufd $0x00, 12(%rdi), %xmm3 + vpshufd $0x00, 16(%rdi), %xmm4 + vpshufd $0x00, 20(%rdi), %xmm5 + vpshufd $0x00, 24(%rdi), %xmm6 + vpshufd $0x00, 28(%rdi), %xmm7 + vpshufd $0x00, 32(%rdi), %xmm8 + vpshufd $0x00, 36(%rdi), %xmm9 + vpshufd $0x00, 40(%rdi), %xmm10 + vpshufd $0x00, 44(%rdi), %xmm11 + vpshufd $0x00, 48(%rdi), %xmm12 + vpshufd $0x00, 52(%rdi), %xmm13 + vpshufd $0x00, 56(%rdi), %xmm14 + vpshufd $0x00, 60(%rdi), %xmm15 + vpaddd L_chacha20_avx1_add(%rip), %xmm12, %xmm12 + vmovdqa %xmm0, (%r9) + vmovdqa %xmm1, 16(%r9) + vmovdqa %xmm2, 32(%r9) + vmovdqa %xmm3, 48(%r9) + vmovdqa %xmm4, 64(%r9) + vmovdqa %xmm5, 80(%r9) + vmovdqa %xmm6, 96(%r9) + vmovdqa %xmm7, 112(%r9) + vmovdqa %xmm8, 128(%r9) + vmovdqa %xmm9, 144(%r9) + vmovdqa %xmm10, 160(%r9) + vmovdqa %xmm11, 176(%r9) + vmovdqa %xmm12, 192(%r9) + vmovdqa %xmm13, 208(%r9) + vmovdqa %xmm14, 224(%r9) + vmovdqa %xmm15, 240(%r9) +L_chacha20_avx1_start128: + vmovdqa %xmm11, 48(%r10) + movb $10, %r8b +L_chacha20_avx1_loop128: + vpaddd %xmm4, %xmm0, %xmm0 + vpxor %xmm0, %xmm12, %xmm12 + vmovdqa 48(%r10), %xmm11 + vpshufb L_chacha20_avx1_rotl16(%rip), %xmm12, %xmm12 + vpaddd %xmm12, %xmm8, %xmm8 + vpxor %xmm8, %xmm4, %xmm4 + vpaddd %xmm5, %xmm1, %xmm1 + vpxor %xmm1, %xmm13, %xmm13 + vpshufb L_chacha20_avx1_rotl16(%rip), %xmm13, %xmm13 + vpaddd %xmm13, %xmm9, %xmm9 + vpxor %xmm9, %xmm5, %xmm5 + vpaddd %xmm6, %xmm2, %xmm2 + vpxor %xmm2, %xmm14, %xmm14 + vpshufb L_chacha20_avx1_rotl16(%rip), %xmm14, %xmm14 + vpaddd %xmm14, %xmm10, %xmm10 + vpxor %xmm10, %xmm6, %xmm6 + vpaddd %xmm7, %xmm3, %xmm3 + vpxor %xmm3, %xmm15, %xmm15 + vpshufb L_chacha20_avx1_rotl16(%rip), %xmm15, %xmm15 + vpaddd %xmm15, %xmm11, %xmm11 + vpxor %xmm11, %xmm7, %xmm7 + vmovdqa %xmm11, 48(%r10) + vpsrld $20, %xmm4, %xmm11 + vpslld $12, %xmm4, %xmm4 + vpxor %xmm11, %xmm4, %xmm4 + vpsrld $20, %xmm5, %xmm11 + vpslld $12, %xmm5, %xmm5 + vpxor %xmm11, %xmm5, %xmm5 + vpsrld $20, %xmm6, %xmm11 + vpslld $12, %xmm6, %xmm6 + vpxor %xmm11, %xmm6, %xmm6 + vpsrld $20, %xmm7, %xmm11 + vpslld $12, %xmm7, %xmm7 + vpxor %xmm11, %xmm7, %xmm7 + vpaddd %xmm4, %xmm0, %xmm0 + vpxor %xmm0, %xmm12, %xmm12 + vmovdqa 48(%r10), %xmm11 + vpshufb L_chacha20_avx1_rotl8(%rip), %xmm12, %xmm12 + vpaddd %xmm12, %xmm8, %xmm8 + vpxor %xmm8, %xmm4, %xmm4 + vpaddd %xmm5, %xmm1, %xmm1 + vpxor %xmm1, %xmm13, %xmm13 + vpshufb L_chacha20_avx1_rotl8(%rip), %xmm13, %xmm13 + vpaddd %xmm13, %xmm9, %xmm9 + vpxor %xmm9, %xmm5, %xmm5 + vpaddd %xmm6, %xmm2, %xmm2 + vpxor %xmm2, %xmm14, %xmm14 + vpshufb L_chacha20_avx1_rotl8(%rip), %xmm14, %xmm14 + vpaddd %xmm14, %xmm10, %xmm10 + vpxor %xmm10, %xmm6, %xmm6 + vpaddd %xmm7, %xmm3, %xmm3 + vpxor %xmm3, %xmm15, %xmm15 + vpshufb L_chacha20_avx1_rotl8(%rip), %xmm15, %xmm15 + vpaddd %xmm15, %xmm11, %xmm11 + vpxor %xmm11, %xmm7, %xmm7 + vmovdqa %xmm11, 48(%r10) + vpsrld $25, %xmm4, %xmm11 + vpslld $7, %xmm4, %xmm4 + vpxor %xmm11, %xmm4, %xmm4 + vpsrld $25, %xmm5, %xmm11 + vpslld $7, %xmm5, %xmm5 + vpxor %xmm11, %xmm5, %xmm5 + vpsrld $25, %xmm6, %xmm11 + vpslld $7, %xmm6, %xmm6 + vpxor %xmm11, %xmm6, %xmm6 + vpsrld $25, %xmm7, %xmm11 + vpslld $7, %xmm7, %xmm7 + vpxor %xmm11, %xmm7, %xmm7 + vpaddd %xmm5, %xmm0, %xmm0 + vpxor %xmm0, %xmm15, %xmm15 + vmovdqa 48(%r10), %xmm11 + vpshufb L_chacha20_avx1_rotl16(%rip), %xmm15, %xmm15 + vpaddd %xmm15, %xmm10, %xmm10 + vpxor %xmm10, %xmm5, %xmm5 + vpaddd %xmm6, %xmm1, %xmm1 + vpxor %xmm1, %xmm12, %xmm12 + vpshufb L_chacha20_avx1_rotl16(%rip), %xmm12, %xmm12 + vpaddd %xmm12, %xmm11, %xmm11 + vpxor %xmm11, %xmm6, %xmm6 + vpaddd %xmm7, %xmm2, %xmm2 + vpxor %xmm2, %xmm13, %xmm13 + vpshufb L_chacha20_avx1_rotl16(%rip), %xmm13, %xmm13 + vpaddd %xmm13, %xmm8, %xmm8 + vpxor %xmm8, %xmm7, %xmm7 + vpaddd %xmm4, %xmm3, %xmm3 + vpxor %xmm3, %xmm14, %xmm14 + vpshufb L_chacha20_avx1_rotl16(%rip), %xmm14, %xmm14 + vpaddd %xmm14, %xmm9, %xmm9 + vpxor %xmm9, %xmm4, %xmm4 + vmovdqa %xmm11, 48(%r10) + vpsrld $20, %xmm5, %xmm11 + vpslld $12, %xmm5, %xmm5 + vpxor %xmm11, %xmm5, %xmm5 + vpsrld $20, %xmm6, %xmm11 + vpslld $12, %xmm6, %xmm6 + vpxor %xmm11, %xmm6, %xmm6 + vpsrld $20, %xmm7, %xmm11 + vpslld $12, %xmm7, %xmm7 + vpxor %xmm11, %xmm7, %xmm7 + vpsrld $20, %xmm4, %xmm11 + vpslld $12, %xmm4, %xmm4 + vpxor %xmm11, %xmm4, %xmm4 + vpaddd %xmm5, %xmm0, %xmm0 + vpxor %xmm0, %xmm15, %xmm15 + vmovdqa 48(%r10), %xmm11 + vpshufb L_chacha20_avx1_rotl8(%rip), %xmm15, %xmm15 + vpaddd %xmm15, %xmm10, %xmm10 + vpxor %xmm10, %xmm5, %xmm5 + vpaddd %xmm6, %xmm1, %xmm1 + vpxor %xmm1, %xmm12, %xmm12 + vpshufb L_chacha20_avx1_rotl8(%rip), %xmm12, %xmm12 + vpaddd %xmm12, %xmm11, %xmm11 + vpxor %xmm11, %xmm6, %xmm6 + vpaddd %xmm7, %xmm2, %xmm2 + vpxor %xmm2, %xmm13, %xmm13 + vpshufb L_chacha20_avx1_rotl8(%rip), %xmm13, %xmm13 + vpaddd %xmm13, %xmm8, %xmm8 + vpxor %xmm8, %xmm7, %xmm7 + vpaddd %xmm4, %xmm3, %xmm3 + vpxor %xmm3, %xmm14, %xmm14 + vpshufb L_chacha20_avx1_rotl8(%rip), %xmm14, %xmm14 + vpaddd %xmm14, %xmm9, %xmm9 + vpxor %xmm9, %xmm4, %xmm4 + vmovdqa %xmm11, 48(%r10) + vpsrld $25, %xmm5, %xmm11 + vpslld $7, %xmm5, %xmm5 + vpxor %xmm11, %xmm5, %xmm5 + vpsrld $25, %xmm6, %xmm11 + vpslld $7, %xmm6, %xmm6 + vpxor %xmm11, %xmm6, %xmm6 + vpsrld $25, %xmm7, %xmm11 + vpslld $7, %xmm7, %xmm7 + vpxor %xmm11, %xmm7, %xmm7 + vpsrld $25, %xmm4, %xmm11 + vpslld $7, %xmm4, %xmm4 + vpxor %xmm11, %xmm4, %xmm4 + decb %r8b + jnz L_chacha20_avx1_loop128 + vmovdqa 48(%r10), %xmm11 + vpaddd (%r9), %xmm0, %xmm0 + vpaddd 16(%r9), %xmm1, %xmm1 + vpaddd 32(%r9), %xmm2, %xmm2 + vpaddd 48(%r9), %xmm3, %xmm3 + vpaddd 64(%r9), %xmm4, %xmm4 + vpaddd 80(%r9), %xmm5, %xmm5 + vpaddd 96(%r9), %xmm6, %xmm6 + vpaddd 112(%r9), %xmm7, %xmm7 + vpaddd 128(%r9), %xmm8, %xmm8 + vpaddd 144(%r9), %xmm9, %xmm9 + vpaddd 160(%r9), %xmm10, %xmm10 + vpaddd 176(%r9), %xmm11, %xmm11 + vpaddd 192(%r9), %xmm12, %xmm12 + vpaddd 208(%r9), %xmm13, %xmm13 + vpaddd 224(%r9), %xmm14, %xmm14 + vpaddd 240(%r9), %xmm15, %xmm15 + vmovdqa %xmm8, (%r10) + vmovdqa %xmm9, 16(%r10) + vmovdqa %xmm10, 32(%r10) + vmovdqa %xmm11, 48(%r10) + vmovdqa %xmm12, 64(%r10) + vmovdqa %xmm13, 80(%r10) + vmovdqa %xmm14, 96(%r10) + vmovdqa %xmm15, 112(%r10) + vpunpckldq %xmm1, %xmm0, %xmm8 + vpunpckldq %xmm3, %xmm2, %xmm9 + vpunpckhdq %xmm1, %xmm0, %xmm12 + vpunpckhdq %xmm3, %xmm2, %xmm13 + vpunpckldq %xmm5, %xmm4, %xmm10 + vpunpckldq %xmm7, %xmm6, %xmm11 + vpunpckhdq %xmm5, %xmm4, %xmm14 + vpunpckhdq %xmm7, %xmm6, %xmm15 + vpunpcklqdq %xmm9, %xmm8, %xmm0 + vpunpcklqdq %xmm11, %xmm10, %xmm1 + vpunpckhqdq %xmm9, %xmm8, %xmm2 + vpunpckhqdq %xmm11, %xmm10, %xmm3 + vpunpcklqdq %xmm13, %xmm12, %xmm4 + vpunpcklqdq %xmm15, %xmm14, %xmm5 + vpunpckhqdq %xmm13, %xmm12, %xmm6 + vpunpckhqdq %xmm15, %xmm14, %xmm7 + vmovdqu (%rsi), %xmm8 + vmovdqu 16(%rsi), %xmm9 + vmovdqu 64(%rsi), %xmm10 + vmovdqu 80(%rsi), %xmm11 + vmovdqu 128(%rsi), %xmm12 + vmovdqu 144(%rsi), %xmm13 + vmovdqu 192(%rsi), %xmm14 + vmovdqu 208(%rsi), %xmm15 + vpxor %xmm8, %xmm0, %xmm0 + vpxor %xmm9, %xmm1, %xmm1 + vpxor %xmm10, %xmm2, %xmm2 + vpxor %xmm11, %xmm3, %xmm3 + vpxor %xmm12, %xmm4, %xmm4 + vpxor %xmm13, %xmm5, %xmm5 + vpxor %xmm14, %xmm6, %xmm6 + vpxor %xmm15, %xmm7, %xmm7 + vmovdqu %xmm0, (%rdx) + vmovdqu %xmm1, 16(%rdx) + vmovdqu %xmm2, 64(%rdx) + vmovdqu %xmm3, 80(%rdx) + vmovdqu %xmm4, 128(%rdx) + vmovdqu %xmm5, 144(%rdx) + vmovdqu %xmm6, 192(%rdx) + vmovdqu %xmm7, 208(%rdx) + vmovdqa (%r10), %xmm0 + vmovdqa 16(%r10), %xmm1 + vmovdqa 32(%r10), %xmm2 + vmovdqa 48(%r10), %xmm3 + vmovdqa 64(%r10), %xmm4 + vmovdqa 80(%r10), %xmm5 + vmovdqa 96(%r10), %xmm6 + vmovdqa 112(%r10), %xmm7 + vpunpckldq %xmm1, %xmm0, %xmm8 + vpunpckldq %xmm3, %xmm2, %xmm9 + vpunpckhdq %xmm1, %xmm0, %xmm12 + vpunpckhdq %xmm3, %xmm2, %xmm13 + vpunpckldq %xmm5, %xmm4, %xmm10 + vpunpckldq %xmm7, %xmm6, %xmm11 + vpunpckhdq %xmm5, %xmm4, %xmm14 + vpunpckhdq %xmm7, %xmm6, %xmm15 + vpunpcklqdq %xmm9, %xmm8, %xmm0 + vpunpcklqdq %xmm11, %xmm10, %xmm1 + vpunpckhqdq %xmm9, %xmm8, %xmm2 + vpunpckhqdq %xmm11, %xmm10, %xmm3 + vpunpcklqdq %xmm13, %xmm12, %xmm4 + vpunpcklqdq %xmm15, %xmm14, %xmm5 + vpunpckhqdq %xmm13, %xmm12, %xmm6 + vpunpckhqdq %xmm15, %xmm14, %xmm7 + vmovdqu 32(%rsi), %xmm8 + vmovdqu 48(%rsi), %xmm9 + vmovdqu 96(%rsi), %xmm10 + vmovdqu 112(%rsi), %xmm11 + vmovdqu 160(%rsi), %xmm12 + vmovdqu 176(%rsi), %xmm13 + vmovdqu 224(%rsi), %xmm14 + vmovdqu 240(%rsi), %xmm15 + vpxor %xmm8, %xmm0, %xmm0 + vpxor %xmm9, %xmm1, %xmm1 + vpxor %xmm10, %xmm2, %xmm2 + vpxor %xmm11, %xmm3, %xmm3 + vpxor %xmm12, %xmm4, %xmm4 + vpxor %xmm13, %xmm5, %xmm5 + vpxor %xmm14, %xmm6, %xmm6 + vpxor %xmm15, %xmm7, %xmm7 + vmovdqu %xmm0, 32(%rdx) + vmovdqu %xmm1, 48(%rdx) + vmovdqu %xmm2, 96(%rdx) + vmovdqu %xmm3, 112(%rdx) + vmovdqu %xmm4, 160(%rdx) + vmovdqu %xmm5, 176(%rdx) + vmovdqu %xmm6, 224(%rdx) + vmovdqu %xmm7, 240(%rdx) + vmovdqa 192(%r9), %xmm12 + addq $0x100, %rsi + addq $0x100, %rdx + vpaddd L_chacha20_avx1_four(%rip), %xmm12, %xmm12 + subl $0x100, %ecx + vmovdqa %xmm12, 192(%r9) + cmpl $0x100, %ecx + jl L_chacha20_avx1_done128 + vmovdqa (%r9), %xmm0 + vmovdqa 16(%r9), %xmm1 + vmovdqa 32(%r9), %xmm2 + vmovdqa 48(%r9), %xmm3 + vmovdqa 64(%r9), %xmm4 + vmovdqa 80(%r9), %xmm5 + vmovdqa 96(%r9), %xmm6 + vmovdqa 112(%r9), %xmm7 + vmovdqa 128(%r9), %xmm8 + vmovdqa 144(%r9), %xmm9 + vmovdqa 160(%r9), %xmm10 + vmovdqa 176(%r9), %xmm11 + vmovdqa 192(%r9), %xmm12 + vmovdqa 208(%r9), %xmm13 + vmovdqa 224(%r9), %xmm14 + vmovdqa 240(%r9), %xmm15 + jmp L_chacha20_avx1_start128 +L_chacha20_avx1_done128: + shl $2, %eax + addl %eax, 48(%rdi) +L_chacha20_avx1_end128: + cmpl $0x40, %ecx + jl L_chacha20_avx1_block_done +L_chacha20_avx1_block_start: + vmovdqu (%rdi), %xmm0 + vmovdqu 16(%rdi), %xmm1 + vmovdqu 32(%rdi), %xmm2 + vmovdqu 48(%rdi), %xmm3 + vmovdqa %xmm0, %xmm5 + vmovdqa %xmm1, %xmm6 + vmovdqa %xmm2, %xmm7 + vmovdqa %xmm3, %xmm8 + movb $10, %al +L_chacha20_avx1_block_crypt_start: + vpaddd %xmm1, %xmm0, %xmm0 + vpxor %xmm0, %xmm3, %xmm3 + vpshufb L_chacha20_avx1_rotl16(%rip), %xmm3, %xmm3 + vpaddd %xmm3, %xmm2, %xmm2 + vpxor %xmm2, %xmm1, %xmm1 + vpsrld $20, %xmm1, %xmm4 + vpslld $12, %xmm1, %xmm1 + vpxor %xmm4, %xmm1, %xmm1 + vpaddd %xmm1, %xmm0, %xmm0 + vpxor %xmm0, %xmm3, %xmm3 + vpshufb L_chacha20_avx1_rotl8(%rip), %xmm3, %xmm3 + vpaddd %xmm3, %xmm2, %xmm2 + vpxor %xmm2, %xmm1, %xmm1 + vpsrld $25, %xmm1, %xmm4 + vpslld $7, %xmm1, %xmm1 + vpxor %xmm4, %xmm1, %xmm1 + vpshufd $57, %xmm1, %xmm1 + vpshufd $0x4e, %xmm2, %xmm2 + vpshufd $0x93, %xmm3, %xmm3 + vpaddd %xmm1, %xmm0, %xmm0 + vpxor %xmm0, %xmm3, %xmm3 + vpshufb L_chacha20_avx1_rotl16(%rip), %xmm3, %xmm3 + vpaddd %xmm3, %xmm2, %xmm2 + vpxor %xmm2, %xmm1, %xmm1 + vpsrld $20, %xmm1, %xmm4 + vpslld $12, %xmm1, %xmm1 + vpxor %xmm4, %xmm1, %xmm1 + vpaddd %xmm1, %xmm0, %xmm0 + vpxor %xmm0, %xmm3, %xmm3 + vpshufb L_chacha20_avx1_rotl8(%rip), %xmm3, %xmm3 + vpaddd %xmm3, %xmm2, %xmm2 + vpxor %xmm2, %xmm1, %xmm1 + vpsrld $25, %xmm1, %xmm4 + vpslld $7, %xmm1, %xmm1 + vpxor %xmm4, %xmm1, %xmm1 + vpshufd $0x93, %xmm1, %xmm1 + vpshufd $0x4e, %xmm2, %xmm2 + vpshufd $57, %xmm3, %xmm3 + decb %al + jnz L_chacha20_avx1_block_crypt_start + vpaddd %xmm5, %xmm0, %xmm0 + vpaddd %xmm6, %xmm1, %xmm1 + vpaddd %xmm7, %xmm2, %xmm2 + vpaddd %xmm8, %xmm3, %xmm3 + vmovdqu (%rsi), %xmm5 + vmovdqu 16(%rsi), %xmm6 + vmovdqu 32(%rsi), %xmm7 + vmovdqu 48(%rsi), %xmm8 + vpxor %xmm5, %xmm0, %xmm0 + vpxor %xmm6, %xmm1, %xmm1 + vpxor %xmm7, %xmm2, %xmm2 + vpxor %xmm8, %xmm3, %xmm3 + vmovdqu %xmm0, (%rdx) + vmovdqu %xmm1, 16(%rdx) + vmovdqu %xmm2, 32(%rdx) + vmovdqu %xmm3, 48(%rdx) + addl $0x01, 48(%rdi) + subl $0x40, %ecx + addq $0x40, %rsi + addq $0x40, %rdx + cmpl $0x40, %ecx + jge L_chacha20_avx1_block_start +L_chacha20_avx1_block_done: + cmpl $0x00, %ecx + je L_chacha20_avx1_partial_done + vmovdqu (%rdi), %xmm0 + vmovdqu 16(%rdi), %xmm1 + vmovdqu 32(%rdi), %xmm2 + vmovdqu 48(%rdi), %xmm3 + vmovdqa %xmm0, %xmm5 + vmovdqa %xmm1, %xmm6 + vmovdqa %xmm2, %xmm7 + vmovdqa %xmm3, %xmm8 + movb $10, %al +L_chacha20_avx1_partial_crypt_start: + vpaddd %xmm1, %xmm0, %xmm0 + vpxor %xmm0, %xmm3, %xmm3 + vpshufb L_chacha20_avx1_rotl16(%rip), %xmm3, %xmm3 + vpaddd %xmm3, %xmm2, %xmm2 + vpxor %xmm2, %xmm1, %xmm1 + vpsrld $20, %xmm1, %xmm4 + vpslld $12, %xmm1, %xmm1 + vpxor %xmm4, %xmm1, %xmm1 + vpaddd %xmm1, %xmm0, %xmm0 + vpxor %xmm0, %xmm3, %xmm3 + vpshufb L_chacha20_avx1_rotl8(%rip), %xmm3, %xmm3 + vpaddd %xmm3, %xmm2, %xmm2 + vpxor %xmm2, %xmm1, %xmm1 + vpsrld $25, %xmm1, %xmm4 + vpslld $7, %xmm1, %xmm1 + vpxor %xmm4, %xmm1, %xmm1 + vpshufd $57, %xmm1, %xmm1 + vpshufd $0x4e, %xmm2, %xmm2 + vpshufd $0x93, %xmm3, %xmm3 + vpaddd %xmm1, %xmm0, %xmm0 + vpxor %xmm0, %xmm3, %xmm3 + vpshufb L_chacha20_avx1_rotl16(%rip), %xmm3, %xmm3 + vpaddd %xmm3, %xmm2, %xmm2 + vpxor %xmm2, %xmm1, %xmm1 + vpsrld $20, %xmm1, %xmm4 + vpslld $12, %xmm1, %xmm1 + vpxor %xmm4, %xmm1, %xmm1 + vpaddd %xmm1, %xmm0, %xmm0 + vpxor %xmm0, %xmm3, %xmm3 + vpshufb L_chacha20_avx1_rotl8(%rip), %xmm3, %xmm3 + vpaddd %xmm3, %xmm2, %xmm2 + vpxor %xmm2, %xmm1, %xmm1 + vpsrld $25, %xmm1, %xmm4 + vpslld $7, %xmm1, %xmm1 + vpxor %xmm4, %xmm1, %xmm1 + vpshufd $0x93, %xmm1, %xmm1 + vpshufd $0x4e, %xmm2, %xmm2 + vpshufd $57, %xmm3, %xmm3 + decb %al + jnz L_chacha20_avx1_partial_crypt_start + vpaddd %xmm5, %xmm0, %xmm0 + vpaddd %xmm6, %xmm1, %xmm1 + vpaddd %xmm7, %xmm2, %xmm2 + vpaddd %xmm8, %xmm3, %xmm3 + vmovdqu %xmm0, (%r10) + vmovdqu %xmm1, 16(%r10) + vmovdqu %xmm2, 32(%r10) + vmovdqu %xmm3, 48(%r10) + addl $0x01, 48(%rdi) + movl %ecx, %r8d + xorq %rbx, %rbx + andl $7, %r8d + jz L_chacha20_avx1_partial_start64 +L_chacha20_avx1_partial_start8: + movzbl (%r10,%rbx,1), %eax + xorb (%rsi,%rbx,1), %al + movb %al, (%rdx,%rbx,1) + incl %ebx + cmpl %r8d, %ebx + jne L_chacha20_avx1_partial_start8 + je L_chacha20_avx1_partial_end64 +L_chacha20_avx1_partial_start64: + movq (%r10,%rbx,1), %rax + xorq (%rsi,%rbx,1), %rax + movq %rax, (%rdx,%rbx,1) + addl $8, %ebx +L_chacha20_avx1_partial_end64: + cmpl %ecx, %ebx + jne L_chacha20_avx1_partial_start64 +L_chacha20_avx1_partial_done: + addq $0x190, %rsp + repz retq +.size chacha_encrypt_avx1,.-chacha_encrypt_avx1 +#endif /* HAVE_INTEL_AVX1 */ +#ifdef HAVE_INTEL_AVX2 +.align 32 +L_chacha20_avx2_rotl8: +.quad 0x605040702010003, 0xe0d0c0f0a09080b +.quad 0x605040702010003, 0xe0d0c0f0a09080b +.align 32 +L_chacha20_avx2_rotl16: +.quad 0x504070601000302, 0xd0c0f0e09080b0a +.quad 0x504070601000302, 0xd0c0f0e09080b0a +.align 32 +L_chacha20_avx2_add: +.quad 0x100000000, 0x300000002 +.quad 0x500000004, 0x700000006 +.align 32 +L_chacha20_avx2_eight: +.quad 0x800000008, 0x800000008 +.quad 0x800000008, 0x800000008 +.globl chacha_encrypt_avx2 +.type chacha_encrypt_avx2,@function +.align 4 +chacha_encrypt_avx2: + subq $0x310, %rsp + movq %rsp, %r9 + leaq 512(%rsp), %r10 + andq $-32, %r9 + andq $-32, %r10 + movl %ecx, %eax + shrl $9, %eax + jz L_chacha20_avx2_end256 + vpbroadcastd (%rdi), %ymm0 + vpbroadcastd 4(%rdi), %ymm1 + vpbroadcastd 8(%rdi), %ymm2 + vpbroadcastd 12(%rdi), %ymm3 + vpbroadcastd 16(%rdi), %ymm4 + vpbroadcastd 20(%rdi), %ymm5 + vpbroadcastd 24(%rdi), %ymm6 + vpbroadcastd 28(%rdi), %ymm7 + vpbroadcastd 32(%rdi), %ymm8 + vpbroadcastd 36(%rdi), %ymm9 + vpbroadcastd 40(%rdi), %ymm10 + vpbroadcastd 44(%rdi), %ymm11 + vpbroadcastd 48(%rdi), %ymm12 + vpbroadcastd 52(%rdi), %ymm13 + vpbroadcastd 56(%rdi), %ymm14 + vpbroadcastd 60(%rdi), %ymm15 + vpaddd L_chacha20_avx2_add(%rip), %ymm12, %ymm12 + vmovdqa %ymm0, (%r9) + vmovdqa %ymm1, 32(%r9) + vmovdqa %ymm2, 64(%r9) + vmovdqa %ymm3, 96(%r9) + vmovdqa %ymm4, 128(%r9) + vmovdqa %ymm5, 160(%r9) + vmovdqa %ymm6, 192(%r9) + vmovdqa %ymm7, 224(%r9) + vmovdqa %ymm8, 256(%r9) + vmovdqa %ymm9, 288(%r9) + vmovdqa %ymm10, 320(%r9) + vmovdqa %ymm11, 352(%r9) + vmovdqa %ymm12, 384(%r9) + vmovdqa %ymm13, 416(%r9) + vmovdqa %ymm14, 448(%r9) + vmovdqa %ymm15, 480(%r9) +L_chacha20_avx2_start256: + movb $10, %r8b + vmovdqa %ymm11, 96(%r10) +L_chacha20_avx2_loop256: + vpaddd %ymm4, %ymm0, %ymm0 + vpxor %ymm0, %ymm12, %ymm12 + vmovdqa 96(%r10), %ymm11 + vpshufb L_chacha20_avx2_rotl16(%rip), %ymm12, %ymm12 + vpaddd %ymm12, %ymm8, %ymm8 + vpxor %ymm8, %ymm4, %ymm4 + vpaddd %ymm5, %ymm1, %ymm1 + vpxor %ymm1, %ymm13, %ymm13 + vpshufb L_chacha20_avx2_rotl16(%rip), %ymm13, %ymm13 + vpaddd %ymm13, %ymm9, %ymm9 + vpxor %ymm9, %ymm5, %ymm5 + vpaddd %ymm6, %ymm2, %ymm2 + vpxor %ymm2, %ymm14, %ymm14 + vpshufb L_chacha20_avx2_rotl16(%rip), %ymm14, %ymm14 + vpaddd %ymm14, %ymm10, %ymm10 + vpxor %ymm10, %ymm6, %ymm6 + vpaddd %ymm7, %ymm3, %ymm3 + vpxor %ymm3, %ymm15, %ymm15 + vpshufb L_chacha20_avx2_rotl16(%rip), %ymm15, %ymm15 + vpaddd %ymm15, %ymm11, %ymm11 + vpxor %ymm11, %ymm7, %ymm7 + vmovdqa %ymm11, 96(%r10) + vpsrld $20, %ymm4, %ymm11 + vpslld $12, %ymm4, %ymm4 + vpxor %ymm11, %ymm4, %ymm4 + vpsrld $20, %ymm5, %ymm11 + vpslld $12, %ymm5, %ymm5 + vpxor %ymm11, %ymm5, %ymm5 + vpsrld $20, %ymm6, %ymm11 + vpslld $12, %ymm6, %ymm6 + vpxor %ymm11, %ymm6, %ymm6 + vpsrld $20, %ymm7, %ymm11 + vpslld $12, %ymm7, %ymm7 + vpxor %ymm11, %ymm7, %ymm7 + vpaddd %ymm4, %ymm0, %ymm0 + vpxor %ymm0, %ymm12, %ymm12 + vmovdqa 96(%r10), %ymm11 + vpshufb L_chacha20_avx2_rotl8(%rip), %ymm12, %ymm12 + vpaddd %ymm12, %ymm8, %ymm8 + vpxor %ymm8, %ymm4, %ymm4 + vpaddd %ymm5, %ymm1, %ymm1 + vpxor %ymm1, %ymm13, %ymm13 + vpshufb L_chacha20_avx2_rotl8(%rip), %ymm13, %ymm13 + vpaddd %ymm13, %ymm9, %ymm9 + vpxor %ymm9, %ymm5, %ymm5 + vpaddd %ymm6, %ymm2, %ymm2 + vpxor %ymm2, %ymm14, %ymm14 + vpshufb L_chacha20_avx2_rotl8(%rip), %ymm14, %ymm14 + vpaddd %ymm14, %ymm10, %ymm10 + vpxor %ymm10, %ymm6, %ymm6 + vpaddd %ymm7, %ymm3, %ymm3 + vpxor %ymm3, %ymm15, %ymm15 + vpshufb L_chacha20_avx2_rotl8(%rip), %ymm15, %ymm15 + vpaddd %ymm15, %ymm11, %ymm11 + vpxor %ymm11, %ymm7, %ymm7 + vmovdqa %ymm11, 96(%r10) + vpsrld $25, %ymm4, %ymm11 + vpslld $7, %ymm4, %ymm4 + vpxor %ymm11, %ymm4, %ymm4 + vpsrld $25, %ymm5, %ymm11 + vpslld $7, %ymm5, %ymm5 + vpxor %ymm11, %ymm5, %ymm5 + vpsrld $25, %ymm6, %ymm11 + vpslld $7, %ymm6, %ymm6 + vpxor %ymm11, %ymm6, %ymm6 + vpsrld $25, %ymm7, %ymm11 + vpslld $7, %ymm7, %ymm7 + vpxor %ymm11, %ymm7, %ymm7 + vpaddd %ymm5, %ymm0, %ymm0 + vpxor %ymm0, %ymm15, %ymm15 + vmovdqa 96(%r10), %ymm11 + vpshufb L_chacha20_avx2_rotl16(%rip), %ymm15, %ymm15 + vpaddd %ymm15, %ymm10, %ymm10 + vpxor %ymm10, %ymm5, %ymm5 + vpaddd %ymm6, %ymm1, %ymm1 + vpxor %ymm1, %ymm12, %ymm12 + vpshufb L_chacha20_avx2_rotl16(%rip), %ymm12, %ymm12 + vpaddd %ymm12, %ymm11, %ymm11 + vpxor %ymm11, %ymm6, %ymm6 + vpaddd %ymm7, %ymm2, %ymm2 + vpxor %ymm2, %ymm13, %ymm13 + vpshufb L_chacha20_avx2_rotl16(%rip), %ymm13, %ymm13 + vpaddd %ymm13, %ymm8, %ymm8 + vpxor %ymm8, %ymm7, %ymm7 + vpaddd %ymm4, %ymm3, %ymm3 + vpxor %ymm3, %ymm14, %ymm14 + vpshufb L_chacha20_avx2_rotl16(%rip), %ymm14, %ymm14 + vpaddd %ymm14, %ymm9, %ymm9 + vpxor %ymm9, %ymm4, %ymm4 + vmovdqa %ymm11, 96(%r10) + vpsrld $20, %ymm5, %ymm11 + vpslld $12, %ymm5, %ymm5 + vpxor %ymm11, %ymm5, %ymm5 + vpsrld $20, %ymm6, %ymm11 + vpslld $12, %ymm6, %ymm6 + vpxor %ymm11, %ymm6, %ymm6 + vpsrld $20, %ymm7, %ymm11 + vpslld $12, %ymm7, %ymm7 + vpxor %ymm11, %ymm7, %ymm7 + vpsrld $20, %ymm4, %ymm11 + vpslld $12, %ymm4, %ymm4 + vpxor %ymm11, %ymm4, %ymm4 + vpaddd %ymm5, %ymm0, %ymm0 + vpxor %ymm0, %ymm15, %ymm15 + vmovdqa 96(%r10), %ymm11 + vpshufb L_chacha20_avx2_rotl8(%rip), %ymm15, %ymm15 + vpaddd %ymm15, %ymm10, %ymm10 + vpxor %ymm10, %ymm5, %ymm5 + vpaddd %ymm6, %ymm1, %ymm1 + vpxor %ymm1, %ymm12, %ymm12 + vpshufb L_chacha20_avx2_rotl8(%rip), %ymm12, %ymm12 + vpaddd %ymm12, %ymm11, %ymm11 + vpxor %ymm11, %ymm6, %ymm6 + vpaddd %ymm7, %ymm2, %ymm2 + vpxor %ymm2, %ymm13, %ymm13 + vpshufb L_chacha20_avx2_rotl8(%rip), %ymm13, %ymm13 + vpaddd %ymm13, %ymm8, %ymm8 + vpxor %ymm8, %ymm7, %ymm7 + vpaddd %ymm4, %ymm3, %ymm3 + vpxor %ymm3, %ymm14, %ymm14 + vpshufb L_chacha20_avx2_rotl8(%rip), %ymm14, %ymm14 + vpaddd %ymm14, %ymm9, %ymm9 + vpxor %ymm9, %ymm4, %ymm4 + vmovdqa %ymm11, 96(%r10) + vpsrld $25, %ymm5, %ymm11 + vpslld $7, %ymm5, %ymm5 + vpxor %ymm11, %ymm5, %ymm5 + vpsrld $25, %ymm6, %ymm11 + vpslld $7, %ymm6, %ymm6 + vpxor %ymm11, %ymm6, %ymm6 + vpsrld $25, %ymm7, %ymm11 + vpslld $7, %ymm7, %ymm7 + vpxor %ymm11, %ymm7, %ymm7 + vpsrld $25, %ymm4, %ymm11 + vpslld $7, %ymm4, %ymm4 + vpxor %ymm11, %ymm4, %ymm4 + decb %r8b + jnz L_chacha20_avx2_loop256 + vmovdqa 96(%r10), %ymm11 + vpaddd (%r9), %ymm0, %ymm0 + vpaddd 32(%r9), %ymm1, %ymm1 + vpaddd 64(%r9), %ymm2, %ymm2 + vpaddd 96(%r9), %ymm3, %ymm3 + vpaddd 128(%r9), %ymm4, %ymm4 + vpaddd 160(%r9), %ymm5, %ymm5 + vpaddd 192(%r9), %ymm6, %ymm6 + vpaddd 224(%r9), %ymm7, %ymm7 + vpaddd 256(%r9), %ymm8, %ymm8 + vpaddd 288(%r9), %ymm9, %ymm9 + vpaddd 320(%r9), %ymm10, %ymm10 + vpaddd 352(%r9), %ymm11, %ymm11 + vpaddd 384(%r9), %ymm12, %ymm12 + vpaddd 416(%r9), %ymm13, %ymm13 + vpaddd 448(%r9), %ymm14, %ymm14 + vpaddd 480(%r9), %ymm15, %ymm15 + vmovdqa %ymm8, (%r10) + vmovdqa %ymm9, 32(%r10) + vmovdqa %ymm10, 64(%r10) + vmovdqa %ymm11, 96(%r10) + vmovdqa %ymm12, 128(%r10) + vmovdqa %ymm13, 160(%r10) + vmovdqa %ymm14, 192(%r10) + vmovdqa %ymm15, 224(%r10) + vpunpckldq %ymm1, %ymm0, %ymm8 + vpunpckldq %ymm3, %ymm2, %ymm9 + vpunpckhdq %ymm1, %ymm0, %ymm12 + vpunpckhdq %ymm3, %ymm2, %ymm13 + vpunpckldq %ymm5, %ymm4, %ymm10 + vpunpckldq %ymm7, %ymm6, %ymm11 + vpunpckhdq %ymm5, %ymm4, %ymm14 + vpunpckhdq %ymm7, %ymm6, %ymm15 + vpunpcklqdq %ymm9, %ymm8, %ymm0 + vpunpcklqdq %ymm11, %ymm10, %ymm1 + vpunpckhqdq %ymm9, %ymm8, %ymm2 + vpunpckhqdq %ymm11, %ymm10, %ymm3 + vpunpcklqdq %ymm13, %ymm12, %ymm4 + vpunpcklqdq %ymm15, %ymm14, %ymm5 + vpunpckhqdq %ymm13, %ymm12, %ymm6 + vpunpckhqdq %ymm15, %ymm14, %ymm7 + vperm2i128 $32, %ymm1, %ymm0, %ymm8 + vperm2i128 $32, %ymm3, %ymm2, %ymm9 + vperm2i128 $49, %ymm1, %ymm0, %ymm12 + vperm2i128 $49, %ymm3, %ymm2, %ymm14 + vperm2i128 $32, %ymm5, %ymm4, %ymm10 + vperm2i128 $32, %ymm7, %ymm6, %ymm11 + vperm2i128 $49, %ymm5, %ymm4, %ymm14 + vperm2i128 $49, %ymm7, %ymm6, %ymm15 + vmovdqu (%rsi), %ymm0 + vmovdqu 64(%rsi), %ymm1 + vmovdqu 128(%rsi), %ymm2 + vmovdqu 192(%rsi), %ymm3 + vmovdqu 256(%rsi), %ymm4 + vmovdqu 320(%rsi), %ymm5 + vmovdqu 384(%rsi), %ymm6 + vmovdqu 448(%rsi), %ymm7 + vpxor %ymm0, %ymm8, %ymm8 + vpxor %ymm1, %ymm9, %ymm9 + vpxor %ymm2, %ymm10, %ymm10 + vpxor %ymm3, %ymm11, %ymm11 + vpxor %ymm4, %ymm12, %ymm12 + vpxor %ymm5, %ymm13, %ymm13 + vpxor %ymm6, %ymm14, %ymm14 + vpxor %ymm7, %ymm15, %ymm15 + vmovdqu %ymm8, (%rdx) + vmovdqu %ymm9, 64(%rdx) + vmovdqu %ymm10, 128(%rdx) + vmovdqu %ymm11, 192(%rdx) + vmovdqu %ymm12, 256(%rdx) + vmovdqu %ymm13, 320(%rdx) + vmovdqu %ymm14, 384(%rdx) + vmovdqu %ymm15, 448(%rdx) + vmovdqa (%r10), %ymm0 + vmovdqa 32(%r10), %ymm1 + vmovdqa 64(%r10), %ymm2 + vmovdqa 96(%r10), %ymm3 + vmovdqa 128(%r10), %ymm4 + vmovdqa 160(%r10), %ymm5 + vmovdqa 192(%r10), %ymm6 + vmovdqa 224(%r10), %ymm7 + vpunpckldq %ymm1, %ymm0, %ymm8 + vpunpckldq %ymm3, %ymm2, %ymm9 + vpunpckhdq %ymm1, %ymm0, %ymm12 + vpunpckhdq %ymm3, %ymm2, %ymm13 + vpunpckldq %ymm5, %ymm4, %ymm10 + vpunpckldq %ymm7, %ymm6, %ymm11 + vpunpckhdq %ymm5, %ymm4, %ymm14 + vpunpckhdq %ymm7, %ymm6, %ymm15 + vpunpcklqdq %ymm9, %ymm8, %ymm0 + vpunpcklqdq %ymm11, %ymm10, %ymm1 + vpunpckhqdq %ymm9, %ymm8, %ymm2 + vpunpckhqdq %ymm11, %ymm10, %ymm3 + vpunpcklqdq %ymm13, %ymm12, %ymm4 + vpunpcklqdq %ymm15, %ymm14, %ymm5 + vpunpckhqdq %ymm13, %ymm12, %ymm6 + vpunpckhqdq %ymm15, %ymm14, %ymm7 + vperm2i128 $32, %ymm1, %ymm0, %ymm8 + vperm2i128 $32, %ymm3, %ymm2, %ymm9 + vperm2i128 $49, %ymm1, %ymm0, %ymm12 + vperm2i128 $49, %ymm3, %ymm2, %ymm14 + vperm2i128 $32, %ymm5, %ymm4, %ymm10 + vperm2i128 $32, %ymm7, %ymm6, %ymm11 + vperm2i128 $49, %ymm5, %ymm4, %ymm14 + vperm2i128 $49, %ymm7, %ymm6, %ymm15 + vmovdqu 32(%rsi), %ymm0 + vmovdqu 96(%rsi), %ymm1 + vmovdqu 160(%rsi), %ymm2 + vmovdqu 224(%rsi), %ymm3 + vmovdqu 288(%rsi), %ymm4 + vmovdqu 352(%rsi), %ymm5 + vmovdqu 416(%rsi), %ymm6 + vmovdqu 480(%rsi), %ymm7 + vpxor %ymm0, %ymm8, %ymm8 + vpxor %ymm1, %ymm9, %ymm9 + vpxor %ymm2, %ymm10, %ymm10 + vpxor %ymm3, %ymm11, %ymm11 + vpxor %ymm4, %ymm12, %ymm12 + vpxor %ymm5, %ymm13, %ymm13 + vpxor %ymm6, %ymm14, %ymm14 + vpxor %ymm7, %ymm15, %ymm15 + vmovdqu %ymm8, 32(%rdx) + vmovdqu %ymm9, 96(%rdx) + vmovdqu %ymm10, 160(%rdx) + vmovdqu %ymm11, 224(%rdx) + vmovdqu %ymm12, 288(%rdx) + vmovdqu %ymm13, 352(%rdx) + vmovdqu %ymm14, 416(%rdx) + vmovdqu %ymm15, 480(%rdx) + vmovdqa 384(%r9), %ymm12 + addq $0x200, %rsi + addq $0x200, %rdx + vpaddd L_chacha20_avx2_eight(%rip), %ymm12, %ymm12 + subl $0x200, %ecx + vmovdqa %ymm12, 384(%r9) + cmpl $0x200, %ecx + jl L_chacha20_avx2_done256 + vmovdqa (%r9), %ymm0 + vmovdqa 32(%r9), %ymm1 + vmovdqa 64(%r9), %ymm2 + vmovdqa 96(%r9), %ymm3 + vmovdqa 128(%r9), %ymm4 + vmovdqa 160(%r9), %ymm5 + vmovdqa 192(%r9), %ymm6 + vmovdqa 224(%r9), %ymm7 + vmovdqa 256(%r9), %ymm8 + vmovdqa 288(%r9), %ymm9 + vmovdqa 320(%r9), %ymm10 + vmovdqa 352(%r9), %ymm11 + vmovdqa 384(%r9), %ymm12 + vmovdqa 416(%r9), %ymm13 + vmovdqa 448(%r9), %ymm14 + vmovdqa 480(%r9), %ymm15 + jmp L_chacha20_avx2_start256 +L_chacha20_avx2_done256: + shl $3, %eax + addl %eax, 48(%rdi) +L_chacha20_avx2_end256: + callq chacha_encrypt_avx1@plt + addq $0x310, %rsp + repz retq +.size chacha_encrypt_avx2,.-chacha_encrypt_avx2 +#endif /* HAVE_INTEL_AVX2 */ diff --git a/wolfcrypt/src/poly1305.c b/wolfcrypt/src/poly1305.c index 9feb5caea9..f803367644 100644 --- a/wolfcrypt/src/poly1305.c +++ b/wolfcrypt/src/poly1305.c @@ -118,827 +118,55 @@ static word32 cpu_flags_set = 0; #endif #ifdef USE_INTEL_SPEEDUP +#ifdef __cplusplus + extern "C" { +#endif + #ifdef HAVE_INTEL_AVX1 /* Process one block (16 bytes) of data. * * ctx Poly1305 context. * m One block of message data. */ -static void poly1305_block_avx(Poly1305* ctx, const unsigned char *m) -{ - __asm__ __volatile__ ( - "movq (%[ctx]), %%r15\n\t" - "movq 24(%[ctx]), %%r8\n\t" - "movq 32(%[ctx]), %%r9\n\t" - "movq 40(%[ctx]), %%r10\n\t" - "xorq %%rbx, %%rbx\n\t" - "movb %[nfin], %%bl\n\t" - "# h += m\n\t" - "movq (%[m]), %%r11\n\t" - "movq 8(%[m]), %%r12\n\t" - "addq %%r11, %%r8\n\t" - "adcq %%r12, %%r9\n\t" - "movq 8(%[ctx]), %%rax\n\t" - "adcq %%rbx, %%r10\n\t" - "# r[1] * h[0] => rdx, rax ==> t2, t1\n\t" - "mulq %%r8\n\t" - "movq %%rax, %%r12\n\t" - "movq %%rdx, %%r13\n\t" - "# r[0] * h[1] => rdx, rax ++> t2, t1\n\t" - "movq %%r15, %%rax\n\t" - "mulq %%r9\n\t" - "addq %%rax, %%r12\n\t" - "movq %%r15, %%rax\n\t" - "adcq %%rdx, %%r13\n\t" - "# r[0] * h[0] => rdx, rax ==> t4, t0\n\t" - "mulq %%r8\n\t" - "movq %%rax, %%r11\n\t" - "movq %%rdx, %%r8\n\t" - "# r[1] * h[1] => rdx, rax =+> t3, t2\n\t" - "movq 8(%[ctx]), %%rax\n\t" - "mulq %%r9\n\t" - "# r[0] * h[2] +> t2\n\t" - "addq 352(%[ctx],%%r10,8), %%r13\n\t" - "movq %%rdx, %%r14\n\t" - "addq %%r8, %%r12\n\t" - "adcq %%rax, %%r13\n\t" - "# r[1] * h[2] +> t3\n\t" - "adcq 408(%[ctx],%%r10,8), %%r14\n\t" - "# r * h in r14, r13, r12, r11 \n\t" - "# h = (r * h) mod 2^130 - 5\n\t" - "movq %%r13, %%r10\n\t" - "andq $-4, %%r13\n\t" - "andq $3, %%r10\n\t" - "addq %%r13, %%r11\n\t" - "movq %%r13, %%r8\n\t" - "adcq %%r14, %%r12\n\t" - "adcq $0, %%r10\n\t" - "shrdq $2, %%r14, %%r8\n\t" - "shrq $2, %%r14\n\t" - "addq %%r11, %%r8\n\t" - "adcq %%r14, %%r12\n\t" - "movq %%r12, %%r9\n\t" - "adcq $0, %%r10\n\t" - "# h in r10, r9, r8 \n\t" - "# Store h to ctx\n\t" - "movq %%r8, 24(%[ctx])\n\t" - "movq %%r9, 32(%[ctx])\n\t" - "movq %%r10, 40(%[ctx])\n\t" - : - : [m] "r" (m), [ctx] "r" (ctx), [nfin] "m" (ctx->finished) - : "rax", "rdx", "r11", "r12", "r13", "r14", "r15", "rbx", - "r8", "r9", "r10", "memory" - ); -} - +extern void poly1305_block_avx(Poly1305* ctx, const unsigned char *m); /* Process multiple blocks (n * 16 bytes) of data. * * ctx Poly1305 context. * m Blocks of message data. * bytes The number of bytes to process. */ -POLY1305_NOINLINE static void poly1305_blocks_avx(Poly1305* ctx, - const unsigned char* m, size_t bytes) -{ - __asm__ __volatile__ ( - "movq (%[ctx]), %%r15\n\t" - "movq 24(%[ctx]), %%r8\n\t" - "movq 32(%[ctx]), %%r9\n\t" - "movq 40(%[ctx]), %%r10\n" - "L_avx_start:\n\t" - "# h += m\n\t" - "movq (%[m]), %%r11\n\t" - "movq 8(%[m]), %%r12\n\t" - "addq %%r11, %%r8\n\t" - "adcq %%r12, %%r9\n\t" - "movq 8(%[ctx]), %%rax\n\t" - "adcq $0, %%r10\n\t" - "# r[1] * h[0] => rdx, rax ==> t2, t1\n\t" - "mulq %%r8\n\t" - "movq %%rax, %%r12\n\t" - "movq %%rdx, %%r13\n\t" - "# r[0] * h[1] => rdx, rax ++> t2, t1\n\t" - "movq %%r15, %%rax\n\t" - "mulq %%r9\n\t" - "addq %%rax, %%r12\n\t" - "movq %%r15, %%rax\n\t" - "adcq %%rdx, %%r13\n\t" - "# r[0] * h[0] => rdx, rax ==> t4, t0\n\t" - "mulq %%r8\n\t" - "movq %%rax, %%r11\n\t" - "movq %%rdx, %%r8\n\t" - "# r[1] * h[1] => rdx, rax =+> t3, t2\n\t" - "movq 8(%[ctx]), %%rax\n\t" - "mulq %%r9\n\t" - "# r[0] * h[2] +> t2\n\t" - "addq 360(%[ctx],%%r10,8), %%r13\n\t" - "movq %%rdx, %%r14\n\t" - "addq %%r8, %%r12\n\t" - "adcq %%rax, %%r13\n\t" - "# r[1] * h[2] +> t3\n\t" - "adcq 416(%[ctx],%%r10,8), %%r14\n\t" - "# r * h in r14, r13, r12, r11 \n\t" - "# h = (r * h) mod 2^130 - 5\n\t" - "movq %%r13, %%r10\n\t" - "andq $-4, %%r13\n\t" - "andq $3, %%r10\n\t" - "addq %%r13, %%r11\n\t" - "movq %%r13, %%r8\n\t" - "adcq %%r14, %%r12\n\t" - "adcq $0, %%r10\n\t" - "shrdq $2, %%r14, %%r8\n\t" - "shrq $2, %%r14\n\t" - "addq %%r11, %%r8\n\t" - "adcq %%r14, %%r12\n\t" - "movq %%r12, %%r9\n\t" - "adcq $0, %%r10\n\t" - "# h in r10, r9, r8 \n\t" - "# Next block from message\n\t" - "addq $16, %[m]\n\t" - "subq $16, %[bytes]\n\t" - "cmp $16, %[bytes]\n\t" - "jge L_avx_start\n\t" - "# Store h to ctx\n\t" - "movq %%r8, 24(%[ctx])\n\t" - "movq %%r9, 32(%[ctx])\n\t" - "movq %%r10, 40(%[ctx])\n\t" - : [m] "+r" (m), [bytes] "+r" (bytes) - : [ctx] "r" (ctx) - : "rax", "rdx", "r11", "r12", "r13", "r14", "r15", - "r8", "r9", "r10", "memory" - ); -} - +extern void poly1305_blocks_avx(Poly1305* ctx, const unsigned char* m, + size_t bytes); /* Set the key to use when processing data. * Initialize the context. * * ctx Poly1305 context. * key The key data (16 bytes). */ -static void poly1305_setkey_avx(Poly1305* ctx, const byte* key) -{ - int i; - - ctx->r[0] = *(word64*)(key + 0) & 0x0ffffffc0fffffffL; - ctx->r[1] = *(word64*)(key + 8) & 0x0ffffffc0ffffffcL; - - for (i=0; i<7; i++) { - ctx->hm[i + 0] = ctx->r[0] * i; - ctx->hm[i + 7] = ctx->r[1] * i; - } - - /* h (accumulator) = 0 */ - ctx->h[0] = 0; - ctx->h[1] = 0; - ctx->h[2] = 0; - - /* save pad for later */ - ctx->pad[0] = *(word64*)(key + 16); - ctx->pad[1] = *(word64*)(key + 24); - - ctx->leftover = 0; - ctx->finished = 1; -} - +extern void poly1305_setkey_avx(Poly1305* ctx, const byte* key); /* Calculate the final result - authentication data. * Zeros out the private data in the context. * * ctx Poly1305 context. * mac Buffer to hold 16 bytes. */ -static void poly1305_final_avx(Poly1305* ctx, byte* mac) -{ - word64 h0, h1, h2; - - /* process the remaining block */ - if (ctx->leftover) { - size_t i = ctx->leftover; - ctx->buffer[i] = 1; - for (i = i + 1; i < POLY1305_BLOCK_SIZE; i++) - ctx->buffer[i] = 0; - ctx->finished = 0; - poly1305_block_avx(ctx, ctx->buffer); - } - - h0 = ctx->h[0]; - h1 = ctx->h[1]; - h2 = ctx->h[2]; - - /* h %= p */ - /* h = (h + pad) */ - __asm__ __volatile__ ( - "# mod 2^130 - 5\n\t" - "movq %[h2], %%r13\n\t" - "andq $0x3, %[h2]\n\t" - "shrq $0x2, %%r13\n\t" - "leaq (%%r13, %%r13, 4), %%r13\n\t" - "add %%r13, %[h0]\n\t" - "adc $0, %[h1]\n\t" - "adc $0, %[h2]\n\t" - "# Fixup when between (1 << 130) - 1 and (1 << 130) - 5\n\t" - "movq %[h0], %%r13\n\t" - "movq %[h1], %%r14\n\t" - "movq %[h2], %%r15\n\t" - "addq $5, %%r13\n\t" - "adcq $0, %%r14\n\t" - "adcq $0, %%r15\n\t" - "movq %%r15, %%r12\n\t" - "andq $3, %%r15\n\t" - "cmpq $4, %%r12\n\t" - "cmove %%r13, %[h0]\n\t" - "cmove %%r14, %[h1]\n\t" - "cmove %%r15, %[h2]\n\t" - "# h += pad\n\t" - "add %[p0], %[h0]\n\t" - "adc %[p1], %[h1]\n\t" - "movq %[h0], (%[m])\n\t" - "movq %[h1], 8(%[m])\n\t" - : [h0] "+r" (h0), [h1] "+r" (h1), [h2] "+r" (h2), - [p0] "+r" (ctx->pad[0]), [p1] "+r" (ctx->pad[1]) - : [m] "r" (mac) - : "memory", "r15", "r14", "r13", "r12" - ); - - /* zero out the state */ - ctx->h[0] = 0; - ctx->h[1] = 0; - ctx->h[2] = 0; - ctx->r[0] = 0; - ctx->r[1] = 0; - ctx->pad[0] = 0; - ctx->pad[1] = 0; -} +extern void poly1305_final_avx(Poly1305* ctx, byte* mac); #endif #ifdef HAVE_INTEL_AVX2 -#if defined(_MSC_VER) - #define POLY1305_NOINLINE __declspec(noinline) -#elif defined(__GNUC__) - #define POLY1305_NOINLINE __attribute__((noinline)) -#else - #define POLY1305_NOINLINE -#endif - -/* Load H into five 256-bit registers. - * - * h is the memory location of the data - 26 of 32 bits. - * h0-h4 the 4 H values with 26 bits stored in 64 for multiply. - */ -#define LOAD_H(h, h0, h1, h2, h3, h4) \ - "vmovdqu ("#h"), "#h0"\n\t" \ - "vmovdqu 32("#h"), "#h1"\n\t" \ - "vmovdqu 64("#h"), "#h2"\n\t" \ - "vmovdqu 96("#h"), "#h3"\n\t" \ - "vmovdqu 128("#h"), "#h4"\n\t" - -/* Store H, five 256-bit registers, packed. - * - * h is the memory location of the data - 26 bits in 32. - * h0-h4 the 4 H values with 26 bits stored in 64. - * x4 is the xmm register of h4. - */ -#define STORE_H(h, h0, h1, h2, h3, h4, x4) \ - "vmovdqu "#h0", ("#h")\n\t" \ - "vmovdqu "#h1", 32("#h")\n\t" \ - "vmovdqu "#h2", 64("#h")\n\t" \ - "vmovdqu "#h3", 96("#h")\n\t" \ - "vmovdqu "#h4", 128("#h")\n\t" - -/* Load four powers of r into position to be multiplied by the 4 H values. - * - * r0-r4 holds the loaded values with 26 bits stored in 64 for multiply. - * t0-t3 are temporary registers. - */ -#define LOAD_Rx4(r0, r1, r2, r3, r4, \ - t0, t1, t2, t3) \ - "vmovdqu 224(%[ctx]), "#r3"\n\t" \ - "vmovdqu 256(%[ctx]), "#r2"\n\t" \ - "vmovdqu 288(%[ctx]), "#r1"\n\t" \ - "vmovdqu 320(%[ctx]), "#r0"\n\t" \ - "vpermq $0xd8, "#r0", "#r0"\n\t" \ - "vpermq $0xd8, "#r1", "#r1"\n\t" \ - "vpermq $0xd8, "#r2", "#r2"\n\t" \ - "vpermq $0xd8, "#r3", "#r3"\n\t" \ - "vpunpcklqdq "#r1", "#r0", "#t0"\n\t" \ - "vpunpckhqdq "#r1", "#r0", "#t1"\n\t" \ - "vpunpcklqdq "#r3", "#r2", "#t2"\n\t" \ - "vpunpckhqdq "#r3", "#r2", "#t3"\n\t" \ - "vperm2i128 $0x20, "#t2", "#t0", "#r0"\n\t" \ - "vperm2i128 $0x31, "#t2", "#t0", "#r2"\n\t" \ - "vperm2i128 $0x20, "#t3", "#t1", "#r4"\n\t" \ - "vpsrlq $32, "#r0", "#r1"\n\t" \ - "vpsrlq $32, "#r2", "#r3"\n\t" - -/* Load the r^4 value into position to be multiplied by all 4 H values. - * - * r4 holds r^4 as five 26 bits each in 32. - * r0-r4 holds the loaded values with 26 bits stored in 64 for multiply. - * t0-t1 are temporary registers. - */ -#define LOAD_R4(r4, r40, r41, r42, r43, r44, \ - t0, t1) \ - "vmovdqu "#r4", "#t0"\n\t" \ - "vpermq $0x0, "#t0", "#r40"\n\t" \ - "vpsrlq $32, "#t0", "#t1"\n\t" \ - "vpermq $0x55, "#t0", "#r42"\n\t" \ - "vpermq $0xaa, "#t0", "#r44"\n\t" \ - "vpermq $0x0, "#t1", "#r41"\n\t" \ - "vpermq $0x55, "#t1", "#r43"\n\t" - -/* Multiply the top 4 26-bit values in 64 bits of each H by 5 for reduction in - * multiply. - * - * s1-s4 are each 64 bit value in r1-r4 multiplied by 5. - * r1-r4 are the top 4 - */ -#define MUL5(s1, s2, s3, s4, r1, r2, r3, r4) \ - "vpslld $2, "#r1", "#s1"\n\t" \ - "vpslld $2, "#r2", "#s2"\n\t" \ - "vpslld $2, "#r3", "#s3"\n\t" \ - "vpslld $2, "#r4", "#s4"\n\t" \ - "vpaddq "#s1", "#r1", "#s1"\n\t" \ - "vpaddq "#s2", "#r2", "#s2"\n\t" \ - "vpaddq "#s3", "#r3", "#s3"\n\t" \ - "vpaddq "#s4", "#r4", "#s4"\n\t" - -/* Add the 4 H values together. - * Each 64 bits in a register is 26 bits of one of the H values. - * - * h0-h4 contains the 4 H values. - * t1-t4 are temporary registers. - */ -#define FINALIZE_H(h0, h1, h2, h3, h4, \ - t0, t1, t2, t3, t4) \ - "vpsrldq $8, "#h0", "#t0"\n\t" \ - "vpsrldq $8, "#h1", "#t1"\n\t" \ - "vpsrldq $8, "#h2", "#t2"\n\t" \ - "vpsrldq $8, "#h3", "#t3"\n\t" \ - "vpsrldq $8, "#h4", "#t4"\n\t" \ - "vpaddq "#h0", "#t0", "#h0"\n\t" \ - "vpaddq "#h1", "#t1", "#h1"\n\t" \ - "vpaddq "#h2", "#t2", "#h2"\n\t" \ - "vpaddq "#h3", "#t3", "#h3"\n\t" \ - "vpaddq "#h4", "#t4", "#h4"\n\t" \ - "vpermq $0x02, "#h0", "#t0"\n\t" \ - "vpermq $0x02, "#h1", "#t1"\n\t" \ - "vpermq $0x02, "#h2", "#t2"\n\t" \ - "vpermq $0x02, "#h3", "#t3"\n\t" \ - "vpermq $0x02, "#h4", "#t4"\n\t" \ - "vpaddq "#h0", "#t0", "#h0"\n\t" \ - "vpaddq "#h1", "#t1", "#h1"\n\t" \ - "vpaddq "#h2", "#t2", "#h2"\n\t" \ - "vpaddq "#h3", "#t3", "#h3"\n\t" \ - "vpaddq "#h4", "#t4", "#h4"\n\t" - -/* Move 32 bits from each xmm register to a 32 bit register. - * - * x0-x4 are the xmm version of the ymm registers used. - * t0-t4 are the 32-bit registers to store data in. - */ -#define MOVE_TO_32(x0, x1, x2, x3, x4, \ - t0, t1, t2, t3, t4) \ - "vmovd "#x0", "#t0"\n\t" \ - "vmovd "#x1", "#t1"\n\t" \ - "vmovd "#x2", "#t2"\n\t" \ - "vmovd "#x3", "#t3"\n\t" \ - "vmovd "#x4", "#t4"\n\t" - -/* Multiply using AVX2 instructions. - * Each register contains up to 32 bits of data in 64 bits. - * This is a 4 way parallel multiply. - * - * h0-h4 contain 4 H values with the 32 bits of each per register. - * r0-r4 contain the 4 powers of r. - * s1-s4 contain r1-r4 times 5. - * t0-t4 and v0-v3 are temporary registers. - */ -#define MUL_AVX2(h0, h1, h2, h3, h4, \ - r0, r1, r2, r3, r4, \ - s1, s2, s3, s4, \ - t0, t1, t2, t3, t4, \ - v0, v1, v2, v3) \ - "vpmuludq "#s1", "#h4", "#t0"\n\t" \ - "vpmuludq "#s2", "#h3", "#v0"\n\t" \ - "vpmuludq "#s2", "#h4", "#t1"\n\t" \ - "vpmuludq "#s3", "#h3", "#v1"\n\t" \ - "vpmuludq "#s3", "#h4", "#t2"\n\t" \ - "vpaddq "#t0", "#v0", "#t0"\n\t" \ - "vpmuludq "#s3", "#h2", "#v2"\n\t" \ - "vpmuludq "#s4", "#h4", "#t3"\n\t" \ - "vpaddq "#t1", "#v1", "#t1"\n\t" \ - "vpmuludq "#s4", "#h1", "#v3"\n\t" \ - "vpmuludq "#s4", "#h2", "#v0"\n\t" \ - "vpaddq "#t0", "#v2", "#t0"\n\t" \ - "vpmuludq "#s4", "#h3", "#v1"\n\t" \ - "vpmuludq "#r0", "#h3", "#v2"\n\t" \ - "vpaddq "#t0", "#v3", "#t0"\n\t" \ - "vpmuludq "#r0", "#h4", "#t4"\n\t" \ - "vpaddq "#t1", "#v0", "#t1"\n\t" \ - "vpmuludq "#r0", "#h0", "#v3"\n\t" \ - "vpaddq "#t2", "#v1", "#t2"\n\t" \ - "vpmuludq "#r0", "#h1", "#v0"\n\t" \ - "vpaddq "#t3", "#v2", "#t3"\n\t" \ - "vpmuludq "#r0", "#h2", "#v1"\n\t" \ - "vpmuludq "#r1", "#h2", "#v2"\n\t" \ - "vpaddq "#t0", "#v3", "#t0"\n\t" \ - "vpmuludq "#r1", "#h3", "#v3"\n\t" \ - "vpaddq "#t1", "#v0", "#t1"\n\t" \ - "vpmuludq "#r1", "#h0", "#v0"\n\t" \ - "vpaddq "#t2", "#v1", "#t2"\n\t" \ - "vpmuludq "#r1", "#h1", "#v1"\n\t" \ - "vpaddq "#t3", "#v2", "#t3"\n\t" \ - "vpmuludq "#r2", "#h1", "#v2"\n\t" \ - "vpaddq "#t4", "#v3", "#t4"\n\t" \ - "vpmuludq "#r2", "#h2", "#v3"\n\t" \ - "vpaddq "#t1", "#v0", "#t1"\n\t" \ - "vpmuludq "#r2", "#h0", "#v0"\n\t" \ - "vpaddq "#t2", "#v1", "#t2"\n\t" \ - "vpmuludq "#r3", "#h0", "#v1"\n\t" \ - "vpaddq "#t3", "#v2", "#t3"\n\t" \ - "vpmuludq "#r3", "#h1", "#v2"\n\t" \ - "vpaddq "#t4", "#v3", "#t4"\n\t" \ - "vpmuludq "#r4", "#h0", "#v3"\n\t" \ - "vpaddq "#t2", "#v0", "#t2"\n\t" \ - "vpaddq "#t3", "#v1", "#t3"\n\t" \ - "vpaddq "#t4", "#v2", "#t4"\n\t" \ - "vpaddq "#t4", "#v3", "#t4"\n\t" - -/* Load the 4 blocks of the message. - * - * m the address of the message to load. - * m0-m4 is the loaded message with 32 bits in 64. Loaded so data is parallel. - * hi is the high bits of the 4 m (1 << 128 as not final block). - * z is zero. - */ -#define LOAD_M(m, m0, m1, m2, m3, m4, hi, z) \ - "vmovdqu (%[m]), "#m0"\n\t" \ - "vmovdqu 32(%[m]), "#m1"\n\t" \ - "vperm2i128 $0x20, "#m1", "#m0", "#m2"\n\t" \ - "vperm2i128 $0x31, "#m1", "#m0", "#m0"\n\t" \ - "vpunpckldq "#m0", "#m2", "#m1"\n\t" \ - "vpunpckhdq "#m0", "#m2", "#m3"\n\t" \ - "vpunpckldq "#z", "#m1", "#m0"\n\t" \ - "vpunpckhdq "#z", "#m1", "#m1"\n\t" \ - "vpunpckldq "#z", "#m3", "#m2"\n\t" \ - "vpunpckhdq "#z", "#m3", "#m3"\n\t" \ - "vmovdqu "#hi", "#m4"\n\t" \ - "vpsllq $6, "#m1", "#m1"\n\t" \ - "vpsllq $12, "#m2", "#m2"\n\t" \ - "vpsllq $18, "#m3", "#m3"\n\t" - - -/* Multiply using AVX2 instructions - adding with message. - * Each register contains up to 32 bits of data in 64 bits. - * This is a 4 way parallel multiply. - * The message data is loaded first and the multiplication adds into it. - * - * h0-h4 contain 4 H values with the 32 bits of each per register. - * r0-r4 contain the 4 powers of r. - * s1-s4 contain r1-r4 times 5. - * t0-t4 and v0-v3 are temporary registers. - * hi is the high bits of the 4 m (1 << 128 as not final block). - * z is zero. - */ -#define MUL_ADD_AVX2(h0, h1, h2, h3, h4, \ - r0, r1, r2, r3, r4, \ - s1, s2, s3, s4, \ - t0, t1, t2, t3, t4, \ - v0, v1, v2, v3, \ - hi, z) \ - "vmovdqu (%[m]), "#t0"\n\t" \ - "vmovdqu 32(%[m]), "#t1"\n\t" \ - "vperm2i128 $0x20, "#t1", "#t0", "#t2"\n\t" \ - "vperm2i128 $0x31, "#t1", "#t0", "#t0"\n\t" \ - "vpunpckldq "#t0", "#t2", "#t1"\n\t" \ - "vpunpckhdq "#t0", "#t2", "#t3"\n\t" \ - "vpunpckldq "#z", "#t1", "#t0"\n\t" \ - "vpunpckhdq "#z", "#t1", "#t1"\n\t" \ - "vpunpckldq "#z", "#t3", "#t2"\n\t" \ - "vpunpckhdq "#z", "#t3", "#t3"\n\t" \ - "vmovdqu "#hi", "#t4"\n\t" \ - "vpsllq $6, "#t1", "#t1"\n\t" \ - "vpsllq $12, "#t2", "#t2"\n\t" \ - "vpsllq $18, "#t3", "#t3"\n\t" \ - "vpmuludq "#s1", "#h4", "#v0"\n\t" \ - "vpaddq "#t0", "#v0", "#t0"\n\t" \ - "vpmuludq "#s2", "#h3", "#v0"\n\t" \ - "vpmuludq "#s2", "#h4", "#v1"\n\t" \ - "vpaddq "#t1", "#v1", "#t1"\n\t" \ - "vpmuludq "#s3", "#h3", "#v1"\n\t" \ - "vpmuludq "#s3", "#h4", "#v2"\n\t" \ - "vpaddq "#t2", "#v2", "#t2"\n\t" \ - "vpaddq "#t0", "#v0", "#t0"\n\t" \ - "vpmuludq "#s3", "#h2", "#v2"\n\t" \ - "vpmuludq "#s4", "#h4", "#v3"\n\t" \ - "vpaddq "#t3", "#v3", "#t3"\n\t" \ - "vpaddq "#t1", "#v1", "#t1"\n\t" \ - "vpmuludq "#s4", "#h1", "#v3"\n\t" \ - "vpmuludq "#s4", "#h2", "#v0"\n\t" \ - "vpaddq "#t0", "#v2", "#t0"\n\t" \ - "vpmuludq "#s4", "#h3", "#v1"\n\t" \ - "vpmuludq "#r0", "#h3", "#v2"\n\t" \ - "vpaddq "#t0", "#v3", "#t0"\n\t" \ - "vpmuludq "#r0", "#h4", "#v3"\n\t" \ - "vpaddq "#t4", "#v3", "#t4"\n\t" \ - "vpaddq "#t1", "#v0", "#t1"\n\t" \ - "vpmuludq "#r0", "#h0", "#v3"\n\t" \ - "vpaddq "#t2", "#v1", "#t2"\n\t" \ - "vpmuludq "#r0", "#h1", "#v0"\n\t" \ - "vpaddq "#t3", "#v2", "#t3"\n\t" \ - "vpmuludq "#r0", "#h2", "#v1"\n\t" \ - "vpmuludq "#r1", "#h2", "#v2"\n\t" \ - "vpaddq "#t0", "#v3", "#t0"\n\t" \ - "vpmuludq "#r1", "#h3", "#v3"\n\t" \ - "vpaddq "#t1", "#v0", "#t1"\n\t" \ - "vpmuludq "#r1", "#h0", "#v0"\n\t" \ - "vpaddq "#t2", "#v1", "#t2"\n\t" \ - "vpmuludq "#r1", "#h1", "#v1"\n\t" \ - "vpaddq "#t3", "#v2", "#t3"\n\t" \ - "vpmuludq "#r2", "#h1", "#v2"\n\t" \ - "vpaddq "#t4", "#v3", "#t4"\n\t" \ - "vpmuludq "#r2", "#h2", "#v3"\n\t" \ - "vpaddq "#t1", "#v0", "#t1"\n\t" \ - "vpmuludq "#r2", "#h0", "#v0"\n\t" \ - "vpaddq "#t2", "#v1", "#t2"\n\t" \ - "vpmuludq "#r3", "#h0", "#v1"\n\t" \ - "vpaddq "#t3", "#v2", "#t3"\n\t" \ - "vpmuludq "#r3", "#h1", "#v2"\n\t" \ - "vpaddq "#t4", "#v3", "#t4"\n\t" \ - "vpmuludq "#r4", "#h0", "#v3"\n\t" \ - "vpaddq "#t2", "#v0", "#t2"\n\t" \ - "vpaddq "#t3", "#v1", "#t3"\n\t" \ - "vpaddq "#t4", "#v2", "#t4"\n\t" \ - "vpaddq "#t4", "#v3", "#t4"\n\t" - -/* Reduce the 64 bits of data to 26 bits. - * - * h0-h4 contain the reduced H values. - * m0-m4 contain the 4 H values to reduce. - * t0-t2 are temporaries. - * mask contains the 26-bit mask for each 64 bit value in the 256 bit register. - */ -#define REDUCE(h0, h1, h2, h3, h4, \ - m0, m1, m2, m3, m4, \ - t0, t1, t2, mask) \ - "vpsrlq $26, "#m0", "#t0"\n\t" \ - "vpsrlq $26, "#m3", "#t1"\n\t" \ - "vpand "#mask", "#m0", "#m0"\n\t" \ - "vpand "#mask", "#m3", "#m3"\n\t" \ - "vpaddq "#m1", "#t0", "#m1"\n\t" \ - "vpaddq "#m4", "#t1", "#m4"\n\t" \ - \ - "vpsrlq $26, "#m1", "#t0"\n\t" \ - "vpsrlq $26, "#m4", "#t1"\n\t" \ - "vpand "#mask", "#m1", "#h1"\n\t" \ - "vpand "#mask", "#m4", "#h4"\n\t" \ - "vpaddq "#m2", "#t0", "#m2"\n\t" \ - "vpslld $2, "#t1", "#t2"\n\t" \ - "vpaddd "#t2", "#t1", "#t2"\n\t" \ - \ - "vpsrlq $26, "#m2", "#t0"\n\t" \ - "vpaddq "#m0", "#t2", "#m0"\n\t" \ - "vpsrlq $26, "#m0", "#t1"\n\t" \ - "vpand "#mask", "#m2", "#h2"\n\t" \ - "vpand "#mask", "#m0", "#h0"\n\t" \ - "vpaddq "#m3", "#t0", "#m3"\n\t" \ - "vpaddq "#h1", "#t1", "#h1"\n\t" \ - \ - "vpsrlq $26, "#m3", "#t0"\n\t" \ - "vpand "#mask", "#m3", "#h3"\n\t" \ - "vpaddq "#h4", "#t0", "#h4"\n\t" \ - - /* Process multiple blocks (n * 16 bytes) of data. * * ctx Poly1305 context. * m Blocks of message data. * bytes The number of bytes to process. */ -POLY1305_NOINLINE static void poly1305_blocks_avx2(Poly1305* ctx, - const unsigned char* m, size_t bytes) -{ - ALIGN256 word64 r4[5][4]; - ALIGN256 word64 s[4][4]; - register word32 t0 asm("r8") = 0; - register word32 t1 asm("r9") = 0; - register word32 t2 asm("r10") = 0; - register word32 t3 asm("r11") = 0; - register word32 t4 asm("r12") = 0; - static const word64 mask[4] = { 0x0000000003ffffff, 0x0000000003ffffff, - 0x0000000003ffffff, 0x0000000003ffffff }; - static const word64 hibit[4] = { 0x1000000, 0x1000000, - 0x1000000, 0x1000000 }; - - __asm__ __volatile__ ( - "vpxor %%ymm15, %%ymm15, %%ymm15\n\t" - "cmpb $1, %[started]\n\t" - "je L_begin\n\t" - "cmpb $1, %[fin]\n\t" - "je L_begin\n\t" - "# Load the message data\n\t" - LOAD_M(m, %%ymm0, %%ymm1, %%ymm2, %%ymm3, %%ymm4, %[hibit], %%ymm15) - "vmovdqu %[mask], %%ymm14\n\t" - "# Reduce, in place, the message data\n\t" - REDUCE(%%ymm0, %%ymm1, %%ymm2, %%ymm3, %%ymm4, - %%ymm0, %%ymm1, %%ymm2, %%ymm3, %%ymm4, - %%ymm10, %%ymm11, %%ymm12, %%ymm14) - "addq $64, %[m]\n\t" - "subq $64, %[bytes]\n\t" - "jz L_store\n\t" - "jmp L_load_r4\n\t" - "\n" - "L_begin:\n\t" - "# Load the H values.\n\t" - LOAD_H(%[h], %%ymm0, %%ymm1, %%ymm2, %%ymm3, %%ymm4) - "# Check if there is a power of r to load - otherwise use r^4.\n\t" - "cmpb $0, %[fin]\n\t" - "je L_load_r4\n\t" - "\n\t" - "# Load the 4 powers of r - r^4, r^3, r^2, r^1.\n\t" - LOAD_Rx4(%%ymm5, %%ymm6, %%ymm7, %%ymm8, %%ymm9, - %%ymm10, %%ymm11, %%ymm12, %%ymm13) - "jmp L_mul_5\n\t" - "\n" - "L_load_r4:\n\t" - "# Load r^4 into all four positions.\n\t" - LOAD_R4(320(%[ctx]), %%ymm5, %%ymm6, %%ymm7, %%ymm8, %%ymm9, - %%ymm13, %%ymm14) - "\n" - "L_mul_5:\n\t" - "# Multiply top 4 26-bit values of all four H by 5\n\t" - MUL5(%%ymm10, %%ymm11, %%ymm12, %%ymm13, %%ymm6, %%ymm7, %%ymm8, %%ymm9) - "# Store powers of r and multiple of 5 for use in multiply.\n\t" - "vmovdqa %%ymm10, (%[s])\n\t" - "vmovdqa %%ymm11, 32(%[s])\n\t" - "vmovdqa %%ymm12, 64(%[s])\n\t" - "vmovdqa %%ymm13, 96(%[s])\n\t" - "vmovdqa %%ymm5 , (%[r4])\n\t" - "vmovdqa %%ymm6 , 32(%[r4])\n\t" - "vmovdqa %%ymm7 , 64(%[r4])\n\t" - "vmovdqa %%ymm8 , 96(%[r4])\n\t" - "vmovdqa %%ymm9 , 128(%[r4])\n\t" - "vmovdqu %[mask], %%ymm14\n\t" - "\n" - "# If not finished then loop over data\n\t" - "cmpb $0x1, %[fin]\n\t" - "jne L_start\n\t" - "# Do last multiply, reduce, add the four H together and move to\n\t" - "# 32-bit registers\n\t" - MUL_AVX2(%%ymm0, %%ymm1, %%ymm2, %%ymm3, %%ymm4, - (%[r4]), 32(%[r4]), 64(%[r4]), 96(%[r4]), 128(%[r4]), - (%[s]), 32(%[s]), 64(%[s]), 96(%[s]), - %%ymm5, %%ymm6, %%ymm7, %%ymm8, %%ymm9, - %%ymm10, %%ymm11, %%ymm12, %%ymm13) - REDUCE(%%ymm0, %%ymm1, %%ymm2, %%ymm3, %%ymm4, - %%ymm5, %%ymm6, %%ymm7, %%ymm8, %%ymm9, - %%ymm10, %%ymm11, %%ymm12, %%ymm14) - FINALIZE_H(%%ymm0, %%ymm1, %%ymm2, %%ymm3, %%ymm4, - %%ymm5, %%ymm6, %%ymm7, %%ymm8, %%ymm9) - MOVE_TO_32(%%xmm0, %%xmm1, %%xmm2, %%xmm3, %%xmm4, - %[t0], %[t1], %[t2], %[t3], %[t4]) - "jmp L_end\n\t" - "\n" - "L_start:\n\t" - MUL_ADD_AVX2(%%ymm0, %%ymm1, %%ymm2, %%ymm3, %%ymm4, - (%[r4]), 32(%[r4]), 64(%[r4]), 96(%[r4]), 128(%[r4]), - (%[s]), 32(%[s]), 64(%[s]), 96(%[s]), - %%ymm5, %%ymm6, %%ymm7, %%ymm8, %%ymm9, - %%ymm10, %%ymm11, %%ymm12, %%ymm13, - %[hibit], %%ymm15) - REDUCE(%%ymm0, %%ymm1, %%ymm2, %%ymm3, %%ymm4, - %%ymm5, %%ymm6, %%ymm7, %%ymm8, %%ymm9, - %%ymm10, %%ymm11, %%ymm12, %%ymm14) - "addq $64, %[m]\n\t" - "subq $64, %[bytes]\n\t" - "jnz L_start\n\t" - "\n" - "L_store:\n\t" - "# Store four H values - state\n\t" - STORE_H(%[h], %%ymm0, %%ymm1, %%ymm2, %%ymm3, %%ymm4, %%xmm4) - "\n" - "L_end:\n\t" - : [m] "+r" (m), [bytes] "+r" (bytes), - [t0] "+r" (t0), [t1] "+r" (t1), [t2] "+r" (t2), - [t3] "+r" (t3), [t4] "+r" (t4) - : [ctx] "r" (ctx), [h] "r" (ctx->hh), - [r4] "r" (r4), [s] "r" (s), - [fin] "m" (ctx->finished), [started] "m" (ctx->started), - [mask] "m" (mask), [hibit] "m" (hibit) - : "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", "ymm7", - "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", "ymm14", "ymm15", - "memory" - ); - - if (ctx->finished) - { - word64 h0, h1, h2, c; - - /* Convert to 64-bit form. */ - h0 = (((word64)(t1 & 0x3FFFF)) << 26) + t0; - h1 = (((word64)(t3 & 0x3FF)) << 34) + - (((word64) t2 ) << 8) + (t1 >> 18); - h2 = (((word64) t4 ) << 16) + (t3 >> 10); - - /* Perform modulur reduction. */ - c = (h1 >> 44); h1 &= 0xfffffffffff; - h2 += c; c = (h2 >> 42); h2 &= 0x3ffffffffff; - h0 += c * 5; c = (h0 >> 44); h0 &= 0xfffffffffff; - h1 += c; c = (h1 >> 44); h1 &= 0xfffffffffff; - h2 += c; c = (h2 >> 42); h2 &= 0x3ffffffffff; - h0 += c * 5; c = (h0 >> 44); h0 &= 0xfffffffffff; - h1 += c; - - /* Convert from 42/44/44 to 2/64/64 bits used and store result. */ - ctx->h[0] = h0 | (h1 << 44); - ctx->h[1] = (h1 >> 20) | (h2 << 24); - ctx->h[2] = h2 >> 40; - } - - ctx->started = 1; -} - -/* Multiply two 130-bit numbers in 64-bit registers and reduce. - * 44 + 44 + 42 = 130 bits - * - * r0-r2 are the first operand and the result. - * a0-a2 are the second operand. - */ -#define MUL_64(r0, r1, r2, a0, a1, a2) \ - s1 = a1 * (5 << 2); \ - s2 = a2 * (5 << 2); \ - MUL(d0, r0, a0); MUL(d, r1, s2); ADD(d0, d); MUL(d, r2, s1); ADD(d0, d); \ - MUL(d1, r0, a1); MUL(d, r1, a0); ADD(d1, d); MUL(d, r2, s2); ADD(d1, d); \ - MUL(d2, r0, a2); MUL(d, r1, a1); ADD(d2, d); MUL(d, r2, a0); ADD(d2, d); \ - \ - c = SHR(d0, 44); r0 = LO(d0) & 0xfffffffffff; \ - ADDLO(d1, c); c = SHR(d1, 44); r1 = LO(d1) & 0xfffffffffff; \ - ADDLO(d2, c); c = SHR(d2, 42); r2 = LO(d2) & 0x3ffffffffff; \ - r0 += c * 5; c = (r0 >> 44); r0 = r0 & 0xfffffffffff; \ - r1 += c - -#define SQR_64(r0, r1, r2) \ - s2 = r2 * (5 << 2); \ - MUL(d0, r1, s2); ADD(d0, d0); MUL(d, r0, r0); ADD(d0, d); \ - MUL(d1, r0, r1); ADD(d1, d1); MUL(d, r2, s2); ADD(d1, d); \ - MUL(d2, r0, r2); ADD(d2, d2); MUL(d, r1, r1); ADD(d2, d); \ - \ - c = SHR(d0, 44); r0 = LO(d0) & 0xfffffffffff; \ - ADDLO(d1, c); c = SHR(d1, 44); r1 = LO(d1) & 0xfffffffffff; \ - ADDLO(d2, c); c = SHR(d2, 42); r2 = LO(d2) & 0x3ffffffffff; \ - r0 += c * 5; c = (r0 >> 44); r0 = r0 & 0xfffffffffff; \ - r1 += c - -/* Store the 130-bit number in 64-bit registers as 26-bit values in 32 bits. - * - * r0-r2 contains the 130-bit number in 64-bit registers. - * r is the address of where to store the 26 of 32 bits result. - */ -#define CONV_64_TO_32(r0, r1, r2, r) \ - r[0] = (word32)( r0 ) & 0x3ffffff; \ - r[1] = (word32)((r0 >> 26) | (r1 << 18)) & 0x3ffffff; \ - r[2] = (word32)( r1 >> 8 ) & 0x3ffffff; \ - r[3] = (word32)((r1 >> 34) | (r2 << 10)) & 0x3ffffff; \ - r[4] = (word32)( r2 >> 16 ) - +extern void poly1305_blocks_avx2(Poly1305* ctx, const unsigned char* m, + size_t bytes); /* Calculate R^1, R^2, R^3 and R^4 and store them in the context. * * ctx Poly1305 context. */ -static void poly1305_calc_powers(Poly1305* ctx) -{ - word64 r0, r1, r2, t0, t1, c; - word64 r20, r21, r22; - word64 r30, r31, r32; - word64 r40, r41, r42; - word64 s1, s2; - word128 d0, d1, d2, d; - - t0 = ctx->r[0]; - t1 = ctx->r[1]; - r0 = ( t0 ) & 0xfffffffffff; - r1 = ((t0 >> 44) | (t1 << 20)) & 0xfffffffffff; - r2 = ((t1 >> 24) ) & 0x00fffffffff; - - /* Store r^1 */ - CONV_64_TO_32(r0, r1, r2, ctx->r1); - - /* Calc and store r^2 */ - r20 = r0; r21 = r1; r22 = r2; - SQR_64(r20, r21, r22); - CONV_64_TO_32(r20, r21, r22, ctx->r2); - - /* Calc and store r^3 */ - r30 = r20; r31 = r21; r32 = r22; - MUL_64(r30, r31, r32, r0, r1, r2); - CONV_64_TO_32(r30, r31, r32, ctx->r3); - - /* Calc and store r^4 */ - r40 = r20; r41 = r21; r42 = r22; - SQR_64(r40, r41, r42); - CONV_64_TO_32(r40, r41, r42, ctx->r4); - -} - +extern void poly1305_calc_powers_avx2(Poly1305* ctx); /* Set the key to use when processing data. * Initialize the context. * Calls AVX set key function as final function calls AVX code. @@ -946,27 +174,7 @@ static void poly1305_calc_powers(Poly1305* ctx) * ctx Poly1305 context. * key The key data (16 bytes). */ -static void poly1305_setkey_avx2(Poly1305* ctx, const byte* key) -{ - poly1305_setkey_avx(ctx, key); - - __asm__ __volatile__ ( - "vpxor %%ymm0, %%ymm0, %%ymm0\n\t" - "vmovdqu %%ymm0, (%[hh])\n\t" - "vmovdqu %%ymm0, 32(%[hh])\n\t" - "vmovdqu %%ymm0, 64(%[hh])\n\t" - "vmovdqu %%ymm0, 96(%[hh])\n\t" - "vmovdqu %%ymm0, 128(%[hh])\n\t" - : - : [hh] "r" (ctx->hh) - : "memory", "ymm0" - ); - - ctx->leftover = 0; - ctx->finished = 0; - ctx->started = 0; -} - +extern void poly1305_setkey_avx2(Poly1305* ctx, const byte* key); /* Calculate the final result - authentication data. * Zeros out the private data in the context. * Calls AVX final function to quickly process last blocks. @@ -974,46 +182,11 @@ static void poly1305_setkey_avx2(Poly1305* ctx, const byte* key) * ctx Poly1305 context. * mac Buffer to hold 16 bytes - authentication data. */ -static void poly1305_final_avx2(Poly1305* ctx, byte* mac) -{ - int i, j; - int l = (int)ctx->leftover; +extern void poly1305_final_avx2(Poly1305* ctx, byte* mac); +#endif - ctx->finished = 1; - if (ctx->started) - poly1305_blocks_avx2(ctx, ctx->buffer, POLY1305_BLOCK_SIZE * 4); - - i = l & ~(POLY1305_BLOCK_SIZE - 1); - if (i > 0) - poly1305_blocks_avx(ctx, ctx->buffer, i); - ctx->leftover -= i; - for (j = 0; i < l; i++, j++) - ctx->buffer[j] = ctx->buffer[i]; - - poly1305_final_avx(ctx, mac); - - /* zero out the state */ - __asm__ __volatile__ ( - "vpxor %%ymm0, %%ymm0, %%ymm0\n\t" - "vmovdqu %%ymm0, (%[hh])\n\t" - "vmovdqu %%ymm0, 32(%[hh])\n\t" - "vmovdqu %%ymm0, 64(%[hh])\n\t" - "vmovdqu %%ymm0, 96(%[hh])\n\t" - "vmovdqu %%ymm0, 128(%[hh])\n\t" - "vmovdqu %%ymm0, (%[r1])\n\t" - "vmovdqu %%ymm0, (%[r2])\n\t" - "vmovdqu %%ymm0, (%[r3])\n\t" - "vmovdqu %%ymm0, (%[r4])\n\t" - : - : [hh] "r" (ctx->hh), [r1] "r" (ctx->r1), [r2] "r" (ctx->r2), - [r3] "r" (ctx->r3), [r4] "r" (ctx->r4) - : "memory", "ymm0" - ); - - ctx->leftover = 0; - ctx->finished = 0; - ctx->started = 0; -} +#ifdef __cplusplus + } /* extern "C" */ #endif #elif defined(POLY130564) @@ -1511,7 +684,7 @@ int wc_Poly1305Update(Poly1305* ctx, const byte* m, word32 bytes) return 0; if (!ctx->started) - poly1305_calc_powers(ctx); + poly1305_calc_powers_avx2(ctx); poly1305_blocks_avx2(ctx, ctx->buffer, sizeof(ctx->buffer)); ctx->leftover = 0; } @@ -1521,7 +694,7 @@ int wc_Poly1305Update(Poly1305* ctx, const byte* m, word32 bytes) size_t want = bytes & ~(sizeof(ctx->buffer) - 1); if (!ctx->started) - poly1305_calc_powers(ctx); + poly1305_calc_powers_avx2(ctx); poly1305_blocks_avx2(ctx, m, want); m += want; bytes -= (word32)want; diff --git a/wolfcrypt/src/poly1305_asm.S b/wolfcrypt/src/poly1305_asm.S new file mode 100644 index 0000000000..ef7fab433f --- /dev/null +++ b/wolfcrypt/src/poly1305_asm.S @@ -0,0 +1,986 @@ +/* poly1305_asm + * + * Copyright (C) 2006-2018 wolfSSL Inc. + * + * This file is part of wolfSSL. + * + * wolfSSL is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * wolfSSL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA + */ + +#ifndef HAVE_INTEL_AVX1 +#define HAVE_INTEL_AVX1 +#endif /* HAVE_INTEL_AVX1 */ +#ifndef HAVE_INTEL_AVX2 +#define HAVE_INTEL_AVX2 +#endif /* HAVE_INTEL_AVX2 */ + +#ifdef HAVE_INTEL_AVX1 +.globl poly1305_setkey_avx +.type poly1305_setkey_avx,@function +.align 4 +poly1305_setkey_avx: + movabsq $0xffffffc0fffffff, %r10 + movabsq $0xffffffc0ffffffc, %r11 + movq (%rsi), %rdx + movq 8(%rsi), %rax + movq 16(%rsi), %rcx + movq 24(%rsi), %r8 + andq %r10, %rdx + andq %r11, %rax + movq %rdx, %r10 + movq %rax, %r11 + xorq %r9, %r9 + movq %rdx, (%rdi) + movq %rax, 8(%rdi) + movq %r9, 24(%rdi) + movq %r9, 32(%rdi) + movq %r9, 40(%rdi) + movq %rcx, 48(%rdi) + movq %r8, 56(%rdi) + movq %r9, 352(%rdi) + movq %r9, 408(%rdi) + movq %rdx, 360(%rdi) + movq %rax, 416(%rdi) + addq %rdx, %r10 + addq %rax, %r11 + movq %r10, 368(%rdi) + movq %r11, 424(%rdi) + addq %rdx, %r10 + addq %rax, %r11 + movq %r10, 376(%rdi) + movq %r11, 432(%rdi) + addq %rdx, %r10 + addq %rax, %r11 + movq %r10, 384(%rdi) + movq %r11, 440(%rdi) + addq %rdx, %r10 + addq %rax, %r11 + movq %r10, 392(%rdi) + movq %r11, 448(%rdi) + addq %rdx, %r10 + addq %rax, %r11 + movq %r10, 400(%rdi) + movq %r11, 456(%rdi) + movq %r9, 608(%rdi) + movb $0x01, 616(%rdi) + repz retq +.size poly1305_setkey_avx,.-poly1305_setkey_avx +.globl poly1305_block_avx +.type poly1305_block_avx,@function +.align 4 +poly1305_block_avx: + pushq %r15 + pushq %rbx + pushq %r12 + pushq %r13 + pushq %r14 + movq (%rdi), %r15 + movq 8(%rdi), %rbx + movq 24(%rdi), %r8 + movq 32(%rdi), %r9 + movq 40(%rdi), %r10 + xorq %r14, %r14 + movb 616(%rdi), %r14b + # h += m + movq (%rsi), %r11 + movq 8(%rsi), %r12 + addq %r11, %r8 + adcq %r12, %r9 + movq %rbx, %rax + adcq %r14, %r10 + # r[1] * h[0] => rdx, rax ==> t2, t1 + mulq %r8 + movq %rax, %r12 + movq %rdx, %r13 + # r[0] * h[1] => rdx, rax ++> t2, t1 + movq %r15, %rax + mulq %r9 + addq %rax, %r12 + movq %r15, %rax + adcq %rdx, %r13 + # r[0] * h[0] => rdx, rax ==> t4, t0 + mulq %r8 + movq %rax, %r11 + movq %rdx, %r8 + # r[1] * h[1] => rdx, rax =+> t3, t2 + movq %rbx, %rax + mulq %r9 + # r[0] * h[2] +> t2 + addq 352(%rdi,%r10,8), %r13 + movq %rdx, %r14 + addq %r8, %r12 + adcq %rax, %r13 + # r[1] * h[2] +> t3 + adcq 408(%rdi,%r10,8), %r14 + # r * h in r14, r13, r12, r11 + # h = (r * h) mod 2^130 - 5 + movq %r13, %r10 + andq $-4, %r13 + andq $3, %r10 + addq %r13, %r11 + movq %r13, %r8 + adcq %r14, %r12 + adcq $0x00, %r10 + shrdq $2, %r14, %r8 + shrq $2, %r14 + addq %r11, %r8 + adcq %r14, %r12 + movq %r12, %r9 + adcq $0x00, %r10 + # h in r10, r9, r8 + # Store h to ctx + movq %r8, 24(%rdi) + movq %r9, 32(%rdi) + movq %r10, 40(%rdi) + popq %r14 + popq %r13 + popq %r12 + popq %rbx + popq %r15 + repz retq +.size poly1305_block_avx,.-poly1305_block_avx +.globl poly1305_blocks_avx +.type poly1305_blocks_avx,@function +.align 4 +poly1305_blocks_avx: + pushq %r15 + pushq %rbx + pushq %r12 + pushq %r13 + pushq %r14 + movq %rdx, %rcx + movq (%rdi), %r15 + movq 8(%rdi), %rbx + movq 24(%rdi), %r8 + movq 32(%rdi), %r9 + movq 40(%rdi), %r10 +L_poly1305_avx_blocks_start: + # h += m + movq (%rsi), %r11 + movq 8(%rsi), %r12 + addq %r11, %r8 + adcq %r12, %r9 + movq %rbx, %rax + adcq $0x00, %r10 + # r[1] * h[0] => rdx, rax ==> t2, t1 + mulq %r8 + movq %rax, %r12 + movq %rdx, %r13 + # r[0] * h[1] => rdx, rax ++> t2, t1 + movq %r15, %rax + mulq %r9 + addq %rax, %r12 + movq %r15, %rax + adcq %rdx, %r13 + # r[0] * h[0] => rdx, rax ==> t4, t0 + mulq %r8 + movq %rax, %r11 + movq %rdx, %r8 + # r[1] * h[1] => rdx, rax =+> t3, t2 + movq %rbx, %rax + mulq %r9 + # r[0] * h[2] +> t2 + addq 360(%rdi,%r10,8), %r13 + movq %rdx, %r14 + addq %r8, %r12 + adcq %rax, %r13 + # r[1] * h[2] +> t3 + adcq 416(%rdi,%r10,8), %r14 + # r * h in r14, r13, r12, r11 + # h = (r * h) mod 2^130 - 5 + movq %r13, %r10 + andq $-4, %r13 + andq $3, %r10 + addq %r13, %r11 + movq %r13, %r8 + adcq %r14, %r12 + adcq $0x00, %r10 + shrdq $2, %r14, %r8 + shrq $2, %r14 + addq %r11, %r8 + adcq %r14, %r12 + movq %r12, %r9 + adcq $0x00, %r10 + # h in r10, r9, r8 + # Next block from message + addq $16, %rsi + subq $16, %rcx + jg L_poly1305_avx_blocks_start + # Store h to ctx + movq %r8, 24(%rdi) + movq %r9, 32(%rdi) + movq %r10, 40(%rdi) + popq %r14 + popq %r13 + popq %r12 + popq %rbx + popq %r15 + repz retq +.size poly1305_blocks_avx,.-poly1305_blocks_avx +.globl poly1305_final_avx +.type poly1305_final_avx,@function +.align 4 +poly1305_final_avx: + pushq %rbx + pushq %r12 + movq %rsi, %rbx + movq 608(%rdi), %rax + testq %rax, %rax + je L_poly1305_avx_final_no_more + movb $0x01, 480(%rdi,%rax,1) + jmp L_poly1305_avx_final_cmp_rem +L_poly1305_avx_final_zero_rem: + movb $0x00, 480(%rdi,%rax,1) +L_poly1305_avx_final_cmp_rem: + incb %al + cmpq $16, %rax + jl L_poly1305_avx_final_zero_rem + movb $0x00, 616(%rdi) + leaq 480(%rdi), %rsi + callq poly1305_block_avx@plt +L_poly1305_avx_final_no_more: + movq 24(%rdi), %rax + movq 32(%rdi), %rdx + movq 40(%rdi), %rcx + movq 48(%rdi), %r11 + movq 56(%rdi), %r12 + # h %= p + # h = (h + pad) + # mod 2^130 - 5 + movq %rcx, %r8 + andq $3, %rcx + shrq $2, %r8 + # Multily by 5 + leaq 0(%r8,%r8,4), %r8 + addq %r8, %rax + adcq $0x00, %rdx + adcq $0x00, %rcx + # Fixup when between (1 << 130) - 1 and (1 << 130) - 5 + movq %rax, %r8 + movq %rdx, %r9 + movq %rcx, %r10 + addq $5, %r8 + adcq $0x00, %r9 + adcq $0x00, %r10 + cmpq $4, %r10 + cmoveq %r8, %rax + cmoveq %r9, %rdx + # h += pad + addq %r11, %rax + adcq %r12, %rdx + movq %rax, (%rbx) + movq %rdx, 8(%rbx) + # Zero out r + movq $0x00, (%rdi) + movq $0x00, 8(%rdi) + # Zero out h + movq $0x00, 24(%rdi) + movq $0x00, 32(%rdi) + movq $0x00, 40(%rdi) + # Zero out pad + movq $0x00, 48(%rdi) + movq $0x00, 56(%rdi) + popq %r12 + popq %rbx + repz retq +.size poly1305_final_avx,.-poly1305_final_avx +#endif /* HAVE_INTEL_AVX1 */ +#ifdef HAVE_INTEL_AVX2 +.globl poly1305_calc_powers_avx2 +.type poly1305_calc_powers_avx2,@function +.align 4 +poly1305_calc_powers_avx2: + pushq %r12 + pushq %r13 + pushq %r14 + pushq %r15 + pushq %rbx + pushq %rbp + movq (%rdi), %rcx + movq 8(%rdi), %r8 + xorq %r9, %r9 + # Convert to 26 bits in 32 + movq %rcx, %rax + movq %rcx, %rdx + movq %rcx, %rsi + movq %r8, %rbx + movq %r8, %rbp + shrq $26, %rdx + shrdq $52, %r8, %rsi + shrq $14, %rbx + shrdq $40, %r9, %rbp + andq $0x3ffffff, %rax + andq $0x3ffffff, %rdx + andq $0x3ffffff, %rsi + andq $0x3ffffff, %rbx + andq $0x3ffffff, %rbp + movl %eax, 224(%rdi) + movl %edx, 228(%rdi) + movl %esi, 232(%rdi) + movl %ebx, 236(%rdi) + movl %ebp, 240(%rdi) + movl $0x00, 244(%rdi) + # Square 128-bit + movq %r8, %rax + mulq %rcx + xorq %r13, %r13 + movq %rax, %r11 + movq %rdx, %r12 + addq %rax, %r11 + adcq %rdx, %r12 + adcq $0x00, %r13 + movq %rcx, %rax + mulq %rax + movq %rax, %r10 + movq %rdx, %r15 + movq %r8, %rax + mulq %rax + addq %r15, %r11 + adcq %rax, %r12 + adcq %rdx, %r13 + # Reduce 256-bit to 130-bit + movq %r12, %rax + movq %r13, %rdx + andq $-4, %rax + andq $3, %r12 + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0x00, %r12 + shrdq $2, %rdx, %rax + shrq $2, %rdx + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0x00, %r12 + movq %r12, %rax + shrq $2, %rax + leaq 0(%rax,%rax,4), %rax + addq %rax, %r10 + adcq $0x00, %r11 + adcq $0x00, %r12 + # Convert to 26 bits in 32 + movq %r10, %rax + movq %r10, %rdx + movq %r10, %rsi + movq %r11, %rbx + movq %r11, %rbp + shrq $26, %rdx + shrdq $52, %r11, %rsi + shrq $14, %rbx + shrdq $40, %r12, %rbp + andq $0x3ffffff, %rax + andq $0x3ffffff, %rdx + andq $0x3ffffff, %rsi + andq $0x3ffffff, %rbx + andq $0x3ffffff, %rbp + movl %eax, 256(%rdi) + movl %edx, 260(%rdi) + movl %esi, 264(%rdi) + movl %ebx, 268(%rdi) + movl %ebp, 272(%rdi) + movl $0x00, 276(%rdi) + # Multiply 128-bit by 130-bit + # r1[0] * r2[0] + movq %rcx, %rax + mulq %r10 + movq %rax, %r13 + movq %rdx, %r14 + # r1[0] * r2[1] + movq %rcx, %rax + mulq %r11 + movq $0x00, %r15 + addq %rax, %r14 + adcq %rdx, %r15 + # r1[1] * r2[0] + movq %r8, %rax + mulq %r10 + movq $0x00, %rsi + addq %rax, %r14 + adcq %rdx, %r15 + adcq $0x00, %rsi + # r1[0] * r2[2] + movq %rcx, %rax + mulq %r12 + addq %rax, %r15 + adcq %rdx, %rsi + # r1[1] * r2[1] + movq %r8, %rax + mulq %r11 + movq $0x00, %rbx + addq %rax, %r15 + adcq %rdx, %rsi + adcq $0x00, %rbx + # r1[1] * r2[2] + movq %r8, %rax + mulq %r12 + addq %rax, %rsi + adcq %rdx, %rbx + # Reduce 260-bit to 130-bit + movq %r15, %rax + movq %rsi, %rdx + movq %rbx, %rbx + andq $-4, %rax + andq $3, %r15 + addq %rax, %r13 + adcq %rdx, %r14 + adcq %rbx, %r15 + shrdq $2, %rdx, %rax + shrdq $2, %rbx, %rdx + shrq $2, %rbx + addq %rax, %r13 + adcq %rdx, %r14 + adcq %rbx, %r15 + movq %r15, %rax + andq $3, %r15 + shrq $2, %rax + leaq 0(%rax,%rax,4), %rax + addq %rax, %r13 + adcq $0x00, %r14 + adcq $0x00, %r15 + # Convert to 26 bits in 32 + movq %r13, %rax + movq %r13, %rdx + movq %r13, %rsi + movq %r14, %rbx + movq %r14, %rbp + shrq $26, %rdx + shrdq $52, %r14, %rsi + shrq $14, %rbx + shrdq $40, %r15, %rbp + andq $0x3ffffff, %rax + andq $0x3ffffff, %rdx + andq $0x3ffffff, %rsi + andq $0x3ffffff, %rbx + andq $0x3ffffff, %rbp + movl %eax, 288(%rdi) + movl %edx, 292(%rdi) + movl %esi, 296(%rdi) + movl %ebx, 300(%rdi) + movl %ebp, 304(%rdi) + movl $0x00, 308(%rdi) + # Square 130-bit + movq %r11, %rax + mulq %r10 + xorq %r13, %r13 + movq %rax, %r8 + movq %rdx, %r9 + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0x00, %r13 + movq %r10, %rax + mulq %rax + movq %rax, %rcx + movq %rdx, %r15 + movq %r11, %rax + mulq %rax + addq %r15, %r8 + adcq %rax, %r9 + adcq %rdx, %r13 + movq %r12, %rax + mulq %rax + movq %rax, %r14 + movq %r12, %rax + mulq %r10 + addq %rax, %r9 + adcq %rdx, %r13 + adcq $0x00, %r14 + addq %rax, %r9 + adcq %rdx, %r13 + adcq $0x00, %r14 + movq %r12, %rax + mulq %r11 + addq %rax, %r13 + adcq %rdx, %r14 + addq %rax, %r13 + adcq %rdx, %r14 + # Reduce 260-bit to 130-bit + movq %r9, %rax + movq %r13, %rdx + movq %r14, %r15 + andq $-4, %rax + andq $3, %r9 + addq %rax, %rcx + adcq %rdx, %r8 + adcq %r15, %r9 + shrdq $2, %rdx, %rax + shrdq $2, %r15, %rdx + shrq $2, %r15 + addq %rax, %rcx + adcq %rdx, %r8 + adcq %r15, %r9 + movq %r9, %rax + andq $3, %r9 + shrq $2, %rax + leaq 0(%rax,%rax,4), %rax + addq %rax, %rcx + adcq $0x00, %r8 + adcq $0x00, %r9 + # Convert to 26 bits in 32 + movq %rcx, %rax + movq %rcx, %rdx + movq %rcx, %rsi + movq %r8, %rbx + movq %r8, %rbp + shrq $26, %rdx + shrdq $52, %r8, %rsi + shrq $14, %rbx + shrdq $40, %r9, %rbp + andq $0x3ffffff, %rax + andq $0x3ffffff, %rdx + andq $0x3ffffff, %rsi + andq $0x3ffffff, %rbx + andq $0x3ffffff, %rbp + movl %eax, 320(%rdi) + movl %edx, 324(%rdi) + movl %esi, 328(%rdi) + movl %ebx, 332(%rdi) + movl %ebp, 336(%rdi) + movl $0x00, 340(%rdi) + popq %rbp + popq %rbx + popq %r15 + popq %r14 + popq %r13 + popq %r12 + repz retq +.size poly1305_calc_powers_avx2,.-poly1305_calc_powers_avx2 +.globl poly1305_setkey_avx2 +.type poly1305_setkey_avx2,@function +.align 4 +poly1305_setkey_avx2: + callq poly1305_setkey_avx@plt + vpxor %ymm0, %ymm0, %ymm0 + vmovdqu %ymm0, 64(%rdi) + vmovdqu %ymm0, 96(%rdi) + vmovdqu %ymm0, 128(%rdi) + vmovdqu %ymm0, 160(%rdi) + vmovdqu %ymm0, 192(%rdi) + movq $0x00, 608(%rdi) + movw $0x00, 616(%rdi) + repz retq +.size poly1305_setkey_avx2,.-poly1305_setkey_avx2 +.align 32 +L_poly1305_avx2_blocks_mask: +.quad 0x3ffffff, 0x3ffffff +.quad 0x3ffffff, 0x3ffffff +.align 32 +L_poly1305_avx2_blocks_hibit: +.quad 0x1000000, 0x1000000 +.quad 0x1000000, 0x1000000 +.globl poly1305_blocks_avx2 +.type poly1305_blocks_avx2,@function +.align 4 +poly1305_blocks_avx2: + pushq %r12 + pushq %rbx + subq $0x140, %rsp + movq %rsp, %rcx + andq $-32, %rcx + addq $32, %rcx + vpxor %ymm15, %ymm15, %ymm15 + movq %rcx, %rbx + leaq 64(%rdi), %rax + addq $0xa0, %rbx + cmpw $0x00, 616(%rdi) + jne L_poly1305_avx2_blocks_begin_h + # Load the message data + vmovdqu (%rsi), %ymm0 + vmovdqu 32(%rsi), %ymm1 + vperm2i128 $32, %ymm1, %ymm0, %ymm2 + vperm2i128 $49, %ymm1, %ymm0, %ymm0 + vpunpckldq %ymm0, %ymm2, %ymm1 + vpunpckhdq %ymm0, %ymm2, %ymm3 + vpunpckldq %ymm15, %ymm1, %ymm0 + vpunpckhdq %ymm15, %ymm1, %ymm1 + vpunpckldq %ymm15, %ymm3, %ymm2 + vpunpckhdq %ymm15, %ymm3, %ymm3 + vmovdqu L_poly1305_avx2_blocks_hibit(%rip), %ymm4 + vpsllq $6, %ymm1, %ymm1 + vpsllq $12, %ymm2, %ymm2 + vpsllq $18, %ymm3, %ymm3 + vmovdqu L_poly1305_avx2_blocks_mask(%rip), %ymm14 + # Reduce, in place, the message data + vpsrlq $26, %ymm0, %ymm10 + vpsrlq $26, %ymm3, %ymm11 + vpand %ymm14, %ymm0, %ymm0 + vpand %ymm14, %ymm3, %ymm3 + vpaddq %ymm1, %ymm10, %ymm1 + vpaddq %ymm4, %ymm11, %ymm4 + vpsrlq $26, %ymm1, %ymm10 + vpsrlq $26, %ymm4, %ymm11 + vpand %ymm14, %ymm1, %ymm1 + vpand %ymm14, %ymm4, %ymm4 + vpaddq %ymm2, %ymm10, %ymm2 + vpslld $2, %ymm11, %ymm12 + vpaddd %ymm12, %ymm11, %ymm12 + vpsrlq $26, %ymm2, %ymm10 + vpaddq %ymm0, %ymm12, %ymm0 + vpsrlq $26, %ymm0, %ymm11 + vpand %ymm14, %ymm2, %ymm2 + vpand %ymm14, %ymm0, %ymm0 + vpaddq %ymm3, %ymm10, %ymm3 + vpaddq %ymm1, %ymm11, %ymm1 + vpsrlq $26, %ymm3, %ymm10 + vpand %ymm14, %ymm3, %ymm3 + vpaddq %ymm4, %ymm10, %ymm4 + addq $0x40, %rsi + subq $0x40, %rdx + jz L_poly1305_avx2_blocks_store + jmp L_poly1305_avx2_blocks_load_r4 +L_poly1305_avx2_blocks_begin_h: + # Load the H values. + vmovdqu (%rax), %ymm0 + vmovdqu 32(%rax), %ymm1 + vmovdqu 64(%rax), %ymm2 + vmovdqu 96(%rax), %ymm3 + vmovdqu 128(%rax), %ymm4 + # Check if there is a power of r to load - otherwise use r^4. + cmpb $0x00, 616(%rdi) + je L_poly1305_avx2_blocks_load_r4 + # Load the 4 powers of r - r^4, r^3, r^2, r^1. + vmovdqu 224(%rdi), %ymm8 + vmovdqu 256(%rdi), %ymm7 + vmovdqu 288(%rdi), %ymm6 + vmovdqu 320(%rdi), %ymm5 + vpermq $0xd8, %ymm5, %ymm5 + vpermq $0xd8, %ymm6, %ymm6 + vpermq $0xd8, %ymm7, %ymm7 + vpermq $0xd8, %ymm8, %ymm8 + vpunpcklqdq %ymm6, %ymm5, %ymm10 + vpunpckhqdq %ymm6, %ymm5, %ymm11 + vpunpcklqdq %ymm8, %ymm7, %ymm12 + vpunpckhqdq %ymm8, %ymm7, %ymm13 + vperm2i128 $32, %ymm12, %ymm10, %ymm5 + vperm2i128 $49, %ymm12, %ymm10, %ymm7 + vperm2i128 $32, %ymm13, %ymm11, %ymm9 + vpsrlq $32, %ymm5, %ymm6 + vpsrlq $32, %ymm7, %ymm8 + jmp L_poly1305_avx2_blocks_mul_5 +L_poly1305_avx2_blocks_load_r4: + # Load r^4 into all four positions. + vmovdqu 320(%rdi), %ymm13 + vpermq $0x00, %ymm13, %ymm5 + vpsrlq $32, %ymm13, %ymm14 + vpermq $0x55, %ymm13, %ymm7 + vpermq $0xaa, %ymm13, %ymm9 + vpermq $0x00, %ymm14, %ymm6 + vpermq $0x55, %ymm14, %ymm8 +L_poly1305_avx2_blocks_mul_5: + # Multiply top 4 26-bit values of all four H by 5 + vpslld $2, %ymm6, %ymm10 + vpslld $2, %ymm7, %ymm11 + vpslld $2, %ymm8, %ymm12 + vpslld $2, %ymm9, %ymm13 + vpaddq %ymm10, %ymm6, %ymm10 + vpaddq %ymm11, %ymm7, %ymm11 + vpaddq %ymm12, %ymm8, %ymm12 + vpaddq %ymm13, %ymm9, %ymm13 + # Store powers of r and multiple of 5 for use in multiply. + vmovdqa %ymm10, (%rbx) + vmovdqa %ymm11, 32(%rbx) + vmovdqa %ymm12, 64(%rbx) + vmovdqa %ymm13, 96(%rbx) + vmovdqa %ymm5, (%rcx) + vmovdqa %ymm6, 32(%rcx) + vmovdqa %ymm7, 64(%rcx) + vmovdqa %ymm8, 96(%rcx) + vmovdqa %ymm9, 128(%rcx) + vmovdqu L_poly1305_avx2_blocks_mask(%rip), %ymm14 + # If not finished then loop over data + cmpb $0x01, 616(%rdi) + jne L_poly1305_avx2_blocks_start + # Do last multiply, reduce, add the four H together and move to + # 32-bit registers + vpmuludq (%rbx), %ymm4, %ymm5 + vpmuludq 32(%rbx), %ymm3, %ymm10 + vpmuludq 32(%rbx), %ymm4, %ymm6 + vpmuludq 64(%rbx), %ymm3, %ymm11 + vpmuludq 64(%rbx), %ymm4, %ymm7 + vpaddq %ymm5, %ymm10, %ymm5 + vpmuludq 64(%rbx), %ymm2, %ymm12 + vpmuludq 96(%rbx), %ymm4, %ymm8 + vpaddq %ymm6, %ymm11, %ymm6 + vpmuludq 96(%rbx), %ymm1, %ymm13 + vpmuludq 96(%rbx), %ymm2, %ymm10 + vpaddq %ymm5, %ymm12, %ymm5 + vpmuludq 96(%rbx), %ymm3, %ymm11 + vpmuludq (%rcx), %ymm3, %ymm12 + vpaddq %ymm5, %ymm13, %ymm5 + vpmuludq (%rcx), %ymm4, %ymm9 + vpaddq %ymm6, %ymm10, %ymm6 + vpmuludq (%rcx), %ymm0, %ymm13 + vpaddq %ymm7, %ymm11, %ymm7 + vpmuludq (%rcx), %ymm1, %ymm10 + vpaddq %ymm8, %ymm12, %ymm8 + vpmuludq (%rcx), %ymm2, %ymm11 + vpmuludq 32(%rcx), %ymm2, %ymm12 + vpaddq %ymm5, %ymm13, %ymm5 + vpmuludq 32(%rcx), %ymm3, %ymm13 + vpaddq %ymm6, %ymm10, %ymm6 + vpmuludq 32(%rcx), %ymm0, %ymm10 + vpaddq %ymm7, %ymm11, %ymm7 + vpmuludq 32(%rcx), %ymm1, %ymm11 + vpaddq %ymm8, %ymm12, %ymm8 + vpmuludq 64(%rcx), %ymm1, %ymm12 + vpaddq %ymm9, %ymm13, %ymm9 + vpmuludq 64(%rcx), %ymm2, %ymm13 + vpaddq %ymm6, %ymm10, %ymm6 + vpmuludq 64(%rcx), %ymm0, %ymm10 + vpaddq %ymm7, %ymm11, %ymm7 + vpmuludq 96(%rcx), %ymm0, %ymm11 + vpaddq %ymm8, %ymm12, %ymm8 + vpmuludq 96(%rcx), %ymm1, %ymm12 + vpaddq %ymm9, %ymm13, %ymm9 + vpaddq %ymm7, %ymm10, %ymm7 + vpmuludq 128(%rcx), %ymm0, %ymm13 + vpaddq %ymm8, %ymm11, %ymm8 + vpaddq %ymm9, %ymm12, %ymm9 + vpaddq %ymm9, %ymm13, %ymm9 + vpsrlq $26, %ymm5, %ymm10 + vpsrlq $26, %ymm8, %ymm11 + vpand %ymm14, %ymm5, %ymm5 + vpand %ymm14, %ymm8, %ymm8 + vpaddq %ymm6, %ymm10, %ymm6 + vpaddq %ymm9, %ymm11, %ymm9 + vpsrlq $26, %ymm6, %ymm10 + vpsrlq $26, %ymm9, %ymm11 + vpand %ymm14, %ymm6, %ymm1 + vpand %ymm14, %ymm9, %ymm4 + vpaddq %ymm7, %ymm10, %ymm7 + vpslld $2, %ymm11, %ymm12 + vpaddd %ymm12, %ymm11, %ymm12 + vpsrlq $26, %ymm7, %ymm10 + vpaddq %ymm5, %ymm12, %ymm5 + vpsrlq $26, %ymm5, %ymm11 + vpand %ymm14, %ymm7, %ymm2 + vpand %ymm14, %ymm5, %ymm0 + vpaddq %ymm8, %ymm10, %ymm8 + vpaddq %ymm1, %ymm11, %ymm1 + vpsrlq $26, %ymm8, %ymm10 + vpand %ymm14, %ymm8, %ymm3 + vpaddq %ymm4, %ymm10, %ymm4 + vpsrldq $8, %ymm0, %ymm5 + vpsrldq $8, %ymm1, %ymm6 + vpsrldq $8, %ymm2, %ymm7 + vpsrldq $8, %ymm3, %ymm8 + vpsrldq $8, %ymm4, %ymm9 + vpaddq %ymm0, %ymm5, %ymm0 + vpaddq %ymm1, %ymm6, %ymm1 + vpaddq %ymm2, %ymm7, %ymm2 + vpaddq %ymm3, %ymm8, %ymm3 + vpaddq %ymm4, %ymm9, %ymm4 + vpermq $2, %ymm0, %ymm5 + vpermq $2, %ymm1, %ymm6 + vpermq $2, %ymm2, %ymm7 + vpermq $2, %ymm3, %ymm8 + vpermq $2, %ymm4, %ymm9 + vpaddq %ymm0, %ymm5, %ymm0 + vpaddq %ymm1, %ymm6, %ymm1 + vpaddq %ymm2, %ymm7, %ymm2 + vpaddq %ymm3, %ymm8, %ymm3 + vpaddq %ymm4, %ymm9, %ymm4 + vmovd %xmm0, %r8d + vmovd %xmm1, %r9d + vmovd %xmm2, %r10d + vmovd %xmm3, %r11d + vmovd %xmm4, %r12d + jmp L_poly1305_avx2_blocks_end_calc +L_poly1305_avx2_blocks_start: + vmovdqu (%rsi), %ymm5 + vmovdqu 32(%rsi), %ymm6 + vperm2i128 $32, %ymm6, %ymm5, %ymm7 + vperm2i128 $49, %ymm6, %ymm5, %ymm5 + vpunpckldq %ymm5, %ymm7, %ymm6 + vpunpckhdq %ymm5, %ymm7, %ymm8 + vpunpckldq %ymm15, %ymm6, %ymm5 + vpunpckhdq %ymm15, %ymm6, %ymm6 + vpunpckldq %ymm15, %ymm8, %ymm7 + vpunpckhdq %ymm15, %ymm8, %ymm8 + vmovdqu L_poly1305_avx2_blocks_hibit(%rip), %ymm9 + vpsllq $6, %ymm6, %ymm6 + vpsllq $12, %ymm7, %ymm7 + vpsllq $18, %ymm8, %ymm8 + vpmuludq (%rbx), %ymm4, %ymm10 + vpaddq %ymm5, %ymm10, %ymm5 + vpmuludq 32(%rbx), %ymm3, %ymm10 + vpmuludq 32(%rbx), %ymm4, %ymm11 + vpaddq %ymm6, %ymm11, %ymm6 + vpmuludq 64(%rbx), %ymm3, %ymm11 + vpmuludq 64(%rbx), %ymm4, %ymm12 + vpaddq %ymm7, %ymm12, %ymm7 + vpaddq %ymm5, %ymm10, %ymm5 + vpmuludq 64(%rbx), %ymm2, %ymm12 + vpmuludq 96(%rbx), %ymm4, %ymm13 + vpaddq %ymm8, %ymm13, %ymm8 + vpaddq %ymm6, %ymm11, %ymm6 + vpmuludq 96(%rbx), %ymm1, %ymm13 + vpmuludq 96(%rbx), %ymm2, %ymm10 + vpaddq %ymm5, %ymm12, %ymm5 + vpmuludq 96(%rbx), %ymm3, %ymm11 + vpmuludq (%rcx), %ymm3, %ymm12 + vpaddq %ymm5, %ymm13, %ymm5 + vpmuludq (%rcx), %ymm4, %ymm13 + vpaddq %ymm9, %ymm13, %ymm9 + vpaddq %ymm6, %ymm10, %ymm6 + vpmuludq (%rcx), %ymm0, %ymm13 + vpaddq %ymm7, %ymm11, %ymm7 + vpmuludq (%rcx), %ymm1, %ymm10 + vpaddq %ymm8, %ymm12, %ymm8 + vpmuludq (%rcx), %ymm2, %ymm11 + vpmuludq 32(%rcx), %ymm2, %ymm12 + vpaddq %ymm5, %ymm13, %ymm5 + vpmuludq 32(%rcx), %ymm3, %ymm13 + vpaddq %ymm6, %ymm10, %ymm6 + vpmuludq 32(%rcx), %ymm0, %ymm10 + vpaddq %ymm7, %ymm11, %ymm7 + vpmuludq 32(%rcx), %ymm1, %ymm11 + vpaddq %ymm8, %ymm12, %ymm8 + vpmuludq 64(%rcx), %ymm1, %ymm12 + vpaddq %ymm9, %ymm13, %ymm9 + vpmuludq 64(%rcx), %ymm2, %ymm13 + vpaddq %ymm6, %ymm10, %ymm6 + vpmuludq 64(%rcx), %ymm0, %ymm10 + vpaddq %ymm7, %ymm11, %ymm7 + vpmuludq 96(%rcx), %ymm0, %ymm11 + vpaddq %ymm8, %ymm12, %ymm8 + vpmuludq 96(%rcx), %ymm1, %ymm12 + vpaddq %ymm9, %ymm13, %ymm9 + vpaddq %ymm7, %ymm10, %ymm7 + vpmuludq 128(%rcx), %ymm0, %ymm13 + vpaddq %ymm8, %ymm11, %ymm8 + vpaddq %ymm9, %ymm12, %ymm9 + vpaddq %ymm9, %ymm13, %ymm9 + vpsrlq $26, %ymm5, %ymm10 + vpsrlq $26, %ymm8, %ymm11 + vpand %ymm14, %ymm5, %ymm5 + vpand %ymm14, %ymm8, %ymm8 + vpaddq %ymm6, %ymm10, %ymm6 + vpaddq %ymm9, %ymm11, %ymm9 + vpsrlq $26, %ymm6, %ymm10 + vpsrlq $26, %ymm9, %ymm11 + vpand %ymm14, %ymm6, %ymm1 + vpand %ymm14, %ymm9, %ymm4 + vpaddq %ymm7, %ymm10, %ymm7 + vpslld $2, %ymm11, %ymm12 + vpaddd %ymm12, %ymm11, %ymm12 + vpsrlq $26, %ymm7, %ymm10 + vpaddq %ymm5, %ymm12, %ymm5 + vpsrlq $26, %ymm5, %ymm11 + vpand %ymm14, %ymm7, %ymm2 + vpand %ymm14, %ymm5, %ymm0 + vpaddq %ymm8, %ymm10, %ymm8 + vpaddq %ymm1, %ymm11, %ymm1 + vpsrlq $26, %ymm8, %ymm10 + vpand %ymm14, %ymm8, %ymm3 + vpaddq %ymm4, %ymm10, %ymm4 + addq $0x40, %rsi + subq $0x40, %rdx + jnz L_poly1305_avx2_blocks_start +L_poly1305_avx2_blocks_store: + # Store four H values - state + vmovdqu %ymm0, (%rax) + vmovdqu %ymm1, 32(%rax) + vmovdqu %ymm2, 64(%rax) + vmovdqu %ymm3, 96(%rax) + vmovdqu %ymm4, 128(%rax) +L_poly1305_avx2_blocks_end_calc: + cmpb $0x00, 616(%rdi) + je L_poly1305_avx2_blocks_complete + movq %r8, %rax + movq %r10, %rdx + movq %r12, %rcx + shrq $12, %rdx + shrq $24, %rcx + shlq $26, %r9 + shlq $52, %r10 + shlq $14, %r11 + shlq $40, %r12 + addq %r9, %rax + adcq %r10, %rax + addq %r11, %rdx + adcq %r12, %rdx + adcq $0x00, %rcx + movq %rcx, %r8 + andq $3, %rcx + shrq $2, %r8 + leaq 0(%r8,%r8,4), %r8 + addq %r8, %rax + adcq $0x00, %rdx + adcq $0x00, %rcx + movq %rax, 24(%rdi) + movq %rdx, 32(%rdi) + movq %rcx, 40(%rdi) +L_poly1305_avx2_blocks_complete: + movb $0x01, 617(%rdi) + addq $0x140, %rsp + popq %rbx + popq %r12 + repz retq +.size poly1305_blocks_avx2,.-poly1305_blocks_avx2 +.globl poly1305_final_avx2 +.type poly1305_final_avx2,@function +.align 4 +poly1305_final_avx2: + movb $0x01, 616(%rdi) + movb 617(%rdi), %cl + cmpb $0x00, %cl + je L_poly1305_avx2_final_done_blocks_X4 + pushq %rsi + movq $0x40, %rdx + xorq %rsi, %rsi + callq poly1305_blocks_avx2@plt + popq %rsi +L_poly1305_avx2_final_done_blocks_X4: + movq 608(%rdi), %rax + movq %rax, %rcx + andq $-16, %rcx + cmpb $0x00, %cl + je L_poly1305_avx2_final_done_blocks + pushq %rcx + pushq %rax + pushq %rsi + movq %rcx, %rdx + leaq 480(%rdi), %rsi + callq poly1305_blocks_avx@plt + popq %rsi + popq %rax + popq %rcx +L_poly1305_avx2_final_done_blocks: + subq %rcx, 608(%rdi) + xorq %rdx, %rdx + jmp L_poly1305_avx2_final_cmp_copy +L_poly1305_avx2_final_start_copy: + movb 480(%rdi,%rcx,1), %r8b + movb %r8b, 480(%rdi,%rdx,1) + incb %cl + incb %dl +L_poly1305_avx2_final_cmp_copy: + cmp %rcx, %rax + jne L_poly1305_avx2_final_start_copy + callq poly1305_final_avx@plt + vpxor %ymm0, %ymm0, %ymm0 + vmovdqu %ymm0, 64(%rdi) + vmovdqu %ymm0, 96(%rdi) + vmovdqu %ymm0, 128(%rdi) + vmovdqu %ymm0, 160(%rdi) + vmovdqu %ymm0, 192(%rdi) + vmovdqu %ymm0, 224(%rdi) + vmovdqu %ymm0, 256(%rdi) + vmovdqu %ymm0, 288(%rdi) + vmovdqu %ymm0, 320(%rdi) + movq $0x00, 608(%rdi) + movw $0x00, 616(%rdi) + repz retq +.size poly1305_final_avx2,.-poly1305_final_avx2 +#endif /* HAVE_INTEL_AVX2 */ diff --git a/wolfcrypt/src/sha256.c b/wolfcrypt/src/sha256.c index cf1320d08d..d8c278c2e8 100644 --- a/wolfcrypt/src/sha256.c +++ b/wolfcrypt/src/sha256.c @@ -235,28 +235,43 @@ static int InitSha256(wc_Sha256* sha256) /* #if defined(HAVE_INTEL_AVX1/2) at the tail of sha256 */ static int Transform_Sha256(wc_Sha256* sha256); + +#ifdef __cplusplus + extern "C" { +#endif + #if defined(HAVE_INTEL_AVX1) - static int Transform_Sha256_AVX1(wc_Sha256 *sha256); - static int Transform_Sha256_AVX1_Len(wc_Sha256* sha256, word32 len); + extern int Transform_Sha256_AVX1(wc_Sha256 *sha256); + extern int Transform_Sha256_AVX1_Len(wc_Sha256* sha256, + const byte* data, word32 len); #endif #if defined(HAVE_INTEL_AVX2) - static int Transform_Sha256_AVX2(wc_Sha256 *sha256); - static int Transform_Sha256_AVX2_Len(wc_Sha256* sha256, word32 len); + extern int Transform_Sha256_AVX2(wc_Sha256 *sha256); + extern int Transform_Sha256_AVX2_Len(wc_Sha256* sha256, + const byte* data, word32 len); #ifdef HAVE_INTEL_RORX - static int Transform_Sha256_AVX1_RORX(wc_Sha256 *sha256); - static int Transform_Sha256_AVX1_RORX_Len(wc_Sha256* sha256, word32 len); - static int Transform_Sha256_AVX2_RORX(wc_Sha256 *sha256); - static int Transform_Sha256_AVX2_RORX_Len(wc_Sha256* sha256, word32 len); + extern int Transform_Sha256_AVX1_RORX(wc_Sha256 *sha256); + extern int Transform_Sha256_AVX1_RORX_Len(wc_Sha256* sha256, + const byte* data, word32 len); + extern int Transform_Sha256_AVX2_RORX(wc_Sha256 *sha256); + extern int Transform_Sha256_AVX2_RORX_Len(wc_Sha256* sha256, + const byte* data, word32 len); #endif #endif + +#ifdef __cplusplus + } /* extern "C" */ +#endif + static int (*Transform_Sha256_p)(wc_Sha256* sha256); /* = _Transform_Sha256 */ - static int (*Transform_Sha256_Len_p)(wc_Sha256* sha256, word32 len); + static int (*Transform_Sha256_Len_p)(wc_Sha256* sha256, const byte* data, + word32 len); /* = NULL */ static int transform_check = 0; static word32 intel_flags; - #define XTRANSFORM(S) (*Transform_Sha256_p)((S)) - #define XTRANSFORM_LEN(S, L) (*Transform_Sha256_Len_p)((S),(L)) + #define XTRANSFORM(S) (*Transform_Sha256_p)((S)) + #define XTRANSFORM_LEN(S, D, L) (*Transform_Sha256_Len_p)((S),(D),(L)) static void Sha256_SetTransform(void) { @@ -267,7 +282,7 @@ static int InitSha256(wc_Sha256* sha256) intel_flags = cpuid_get_flags(); #ifdef HAVE_INTEL_AVX2 - if (IS_INTEL_AVX2(intel_flags)) { + if (1 && IS_INTEL_AVX2(intel_flags)) { #ifdef HAVE_INTEL_RORX if (IS_INTEL_BMI2(intel_flags)) { Transform_Sha256_p = Transform_Sha256_AVX2_RORX; @@ -351,8 +366,8 @@ static int InitSha256(wc_Sha256* sha256) #include "fsl_mmcau.h" #endif - #define XTRANSFORM(S) Transform_Sha256((S)) - #define XTRANSFORM_LEN(S,L) Transform_Sha256_Len((S),(L)) + #define XTRANSFORM(S) Transform_Sha256((S)) + #define XTRANSFORM_LEN(S, D, L) Transform_Sha256_Len((S),(D),(L)) int wc_InitSha256_ex(wc_Sha256* sha256, void* heap, int devId) { @@ -593,8 +608,8 @@ static int InitSha256(wc_Sha256* sha256) h(j) = t0 + t1 #ifndef XTRANSFORM - #define XTRANSFORM(S) Transform_Sha256((S)) - #define XTRANSFORM_LEN(S,L) Transform_Sha256_Len((S),(L)) + #define XTRANSFORM(S) Transform_Sha256((S)) + #define XTRANSFORM_LEN(S, D, L) Transform_Sha256_Len((S),(D),(L)) #endif static int Transform_Sha256(wc_Sha256* sha256) @@ -684,6 +699,8 @@ static int InitSha256(wc_Sha256* sha256) return 0; } + AddLength(sha256, len); + /* do block size increments */ local = (byte*)sha256->buffer; @@ -723,10 +740,8 @@ static int InitSha256(wc_Sha256* sha256) esp_sha256_process(sha256); } #endif - if (ret == 0) { - AddLength(sha256, WC_SHA256_BLOCK_SIZE); + if (ret == 0) sha256->buffLen = 0; - } else len = 0; } @@ -735,12 +750,9 @@ static int InitSha256(wc_Sha256* sha256) #if defined(HAVE_INTEL_AVX1) || defined(HAVE_INTEL_AVX2) if (Transform_Sha256_Len_p != NULL) { word32 blocksLen = len & ~(WC_SHA256_BLOCK_SIZE-1); - if (blocksLen > 0) { - AddLength(sha256, blocksLen); - sha256->data = data; /* Byte reversal performed in function if required. */ - XTRANSFORM_LEN(sha256, blocksLen); + XTRANSFORM_LEN(sha256, data, blocksLen); data += blocksLen; len -= blocksLen; } @@ -750,16 +762,19 @@ static int InitSha256(wc_Sha256* sha256) #if !defined(LITTLE_ENDIAN_ORDER) || defined(FREESCALE_MMCAU_SHA) || \ defined(HAVE_INTEL_AVX1) || defined(HAVE_INTEL_AVX2) { - word32 blocksLen = len & ~(WC_SHA256_BLOCK_SIZE-1); - - AddLength(sha256, blocksLen); while (len >= WC_SHA256_BLOCK_SIZE) { XMEMCPY(local, data, WC_SHA256_BLOCK_SIZE); data += WC_SHA256_BLOCK_SIZE; len -= WC_SHA256_BLOCK_SIZE; - /* Byte reversal performed in function if required. */ + #if defined(HAVE_INTEL_AVX1) || defined(HAVE_INTEL_AVX2) + if (!IS_INTEL_AVX1(intel_flags) && !IS_INTEL_AVX2(intel_flags)) + { + ByteReverseWords(sha256->buffer, sha256->buffer, + WC_SHA256_BLOCK_SIZE); + } + #endif ret = XTRANSFORM(sha256); if (ret != 0) break; @@ -767,9 +782,6 @@ static int InitSha256(wc_Sha256* sha256) } #else { - word32 blocksLen = len & ~(WC_SHA256_BLOCK_SIZE-1); - - AddLength(sha256, blocksLen); while (len >= WC_SHA256_BLOCK_SIZE) { XMEMCPY(local, data, WC_SHA256_BLOCK_SIZE); @@ -846,7 +858,6 @@ static int InitSha256(wc_Sha256* sha256) return BAD_FUNC_ARG; } - AddLength(sha256, sha256->buffLen); /* before adding pads */ local[sha256->buffLen++] = 0x80; /* add 1 */ /* pad with zeros */ @@ -997,1680 +1008,6 @@ static int InitSha256(wc_Sha256* sha256) #endif /* XTRANSFORM */ - -#if defined(HAVE_INTEL_AVX1) || defined(HAVE_INTEL_AVX2) - -#define _LOAD_DIGEST() \ - "movl (%[sha256]), %%r8d \n\t" \ - "movl 4(%[sha256]), %%r9d \n\t" \ - "movl 8(%[sha256]), %%r10d\n\t" \ - "movl 12(%[sha256]), %%r11d\n\t" \ - "movl 16(%[sha256]), %%r12d\n\t" \ - "movl 20(%[sha256]), %%r13d\n\t" \ - "movl 24(%[sha256]), %%r14d\n\t" \ - "movl 28(%[sha256]), %%r15d\n\t" - -#define _STORE_ADD_DIGEST() \ - "addl %%r8d , (%[sha256])\n\t" \ - "addl %%r9d , 4(%[sha256])\n\t" \ - "addl %%r10d, 8(%[sha256])\n\t" \ - "addl %%r11d, 12(%[sha256])\n\t" \ - "addl %%r12d, 16(%[sha256])\n\t" \ - "addl %%r13d, 20(%[sha256])\n\t" \ - "addl %%r14d, 24(%[sha256])\n\t" \ - "addl %%r15d, 28(%[sha256])\n\t" - -#define _ADD_DIGEST() \ - "addl (%[sha256]), %%r8d \n\t" \ - "addl 4(%[sha256]), %%r9d \n\t" \ - "addl 8(%[sha256]), %%r10d\n\t" \ - "addl 12(%[sha256]), %%r11d\n\t" \ - "addl 16(%[sha256]), %%r12d\n\t" \ - "addl 20(%[sha256]), %%r13d\n\t" \ - "addl 24(%[sha256]), %%r14d\n\t" \ - "addl 28(%[sha256]), %%r15d\n\t" - -#define _STORE_DIGEST() \ - "movl %%r8d , (%[sha256])\n\t" \ - "movl %%r9d , 4(%[sha256])\n\t" \ - "movl %%r10d, 8(%[sha256])\n\t" \ - "movl %%r11d, 12(%[sha256])\n\t" \ - "movl %%r12d, 16(%[sha256])\n\t" \ - "movl %%r13d, 20(%[sha256])\n\t" \ - "movl %%r14d, 24(%[sha256])\n\t" \ - "movl %%r15d, 28(%[sha256])\n\t" - -#define LOAD_DIGEST() \ - _LOAD_DIGEST() - -#define STORE_ADD_DIGEST() \ - _STORE_ADD_DIGEST() - -#define ADD_DIGEST() \ - _ADD_DIGEST() - -#define STORE_DIGEST() \ - _STORE_DIGEST() - - -#define S_0 %r8d -#define S_1 %r9d -#define S_2 %r10d -#define S_3 %r11d -#define S_4 %r12d -#define S_5 %r13d -#define S_6 %r14d -#define S_7 %r15d - -#define L1 "%%edx" -#define L2 "%%ecx" -#define L3 "%%eax" -#define L4 "%%ebx" -#define WK "%%rsp" - -#define WORK_REGS "eax", "ebx", "ecx", "edx" -#define STATE_REGS "r8","r9","r10","r11","r12","r13","r14","r15" -#define XMM_REGS "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", \ - "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13" - -#if defined(HAVE_INTEL_RORX) -#define RND_STEP_RORX_0_1(a, b, c, d, e, f, g, h, i) \ - /* L3 = f */ \ - "movl %" #f ", " L3 "\n\t" \ - /* L2 = e>>>11 */ \ - "rorx $11, %" #e ", " L2 "\n\t" \ - /* h += w_k */ \ - "addl (" #i ")*4(" WK "), %" #h "\n\t" \ - -#define RND_STEP_RORX_0_2(a, b, c, d, e, f, g, h, i) \ - /* L2 = (e>>>6) ^ (e>>>11) */ \ - "xorl " L1 ", " L2 "\n\t" \ - /* L3 = f ^ g */ \ - "xorl %" #g ", " L3 "\n\t" \ - /* L1 = e>>>25 */ \ - "rorx $25, %" #e ", " L1 "\n\t" \ - -#define RND_STEP_RORX_0_3(a, b, c, d, e, f, g, h, i) \ - /* L3 = (f ^ g) & e */ \ - "andl %" #e ", " L3 "\n\t" \ - /* L1 = Sigma1(e) */ \ - "xorl " L2 ", " L1 "\n\t" \ - /* L2 = a>>>13 */ \ - "rorx $13, %" #a ", " L2 "\n\t" \ - -#define RND_STEP_RORX_0_4(a, b, c, d, e, f, g, h, i) \ - /* h += Sigma1(e) */ \ - "addl " L1 ", %" #h "\n\t" \ - /* L1 = a>>>2 */ \ - "rorx $2, %" #a ", " L1 "\n\t" \ - /* L3 = Ch(e,f,g) */ \ - "xorl %" #g ", " L3 "\n\t" \ - -#define RND_STEP_RORX_0_5(a, b, c, d, e, f, g, h, i) \ - /* L2 = (a>>>2) ^ (a>>>13) */ \ - "xorl " L1 ", " L2 "\n\t" \ - /* L1 = a>>>22 */ \ - "rorx $22, %" #a ", " L1 "\n\t" \ - /* h += Ch(e,f,g) */ \ - "addl " L3 ", %" #h "\n\t" \ - -#define RND_STEP_RORX_0_6(a, b, c, d, e, f, g, h, i) \ - /* L1 = Sigma0(a) */ \ - "xorl " L2 ", " L1 "\n\t" \ - /* L3 = b */ \ - "movl %" #b ", " L3 "\n\t" \ - /* d += h + w_k + Sigma1(e) + Ch(e,f,g) */ \ - "addl %" #h ", %" #d "\n\t" \ - -#define RND_STEP_RORX_0_7(a, b, c, d, e, f, g, h, i) \ - /* L3 = a ^ b */ \ - "xorl %" #a ", " L3 "\n\t" \ - /* h += Sigma0(a) */ \ - "addl " L1 ", %" #h "\n\t" \ - /* L4 = (a ^ b) & (b ^ c) */ \ - "andl " L3 ", " L4 "\n\t" \ - -#define RND_STEP_RORX_0_8(a, b, c, d, e, f, g, h, i) \ - /* L4 = Maj(a,b,c) */ \ - "xorl %" #b ", " L4 "\n\t" \ - /* L1 = d>>>6 (= e>>>6 next RND) */ \ - "rorx $6, %" #d ", " L1 "\n\t" \ - /* h += Maj(a,b,c) */ \ - "addl " L4 ", %" #h "\n\t" \ - -#define RND_STEP_RORX_1_1(a, b, c, d, e, f, g, h, i) \ - /* L4 = f */ \ - "movl %" #f ", " L4 "\n\t" \ - /* L2 = e>>>11 */ \ - "rorx $11, %" #e ", " L2 "\n\t" \ - /* h += w_k */ \ - "addl (" #i ")*4(" WK "), %" #h "\n\t" \ - -#define RND_STEP_RORX_1_2(a, b, c, d, e, f, g, h, i) \ - /* L2 = (e>>>6) ^ (e>>>11) */ \ - "xorl " L1 ", " L2 "\n\t" \ - /* L4 = f ^ g */ \ - "xorl %" #g ", " L4 "\n\t" \ - /* L1 = e>>>25 */ \ - "rorx $25, %" #e ", " L1 "\n\t" \ - -#define RND_STEP_RORX_1_3(a, b, c, d, e, f, g, h, i) \ - /* L4 = (f ^ g) & e */ \ - "andl %" #e ", " L4 "\n\t" \ - /* L1 = Sigma1(e) */ \ - "xorl " L2 ", " L1 "\n\t" \ - /* L2 = a>>>13 */ \ - "rorx $13, %" #a ", " L2 "\n\t" \ - -#define RND_STEP_RORX_1_4(a, b, c, d, e, f, g, h, i) \ - /* h += Sigma1(e) */ \ - "addl " L1 ", %" #h "\n\t" \ - /* L1 = a>>>2 */ \ - "rorx $2, %" #a ", " L1 "\n\t" \ - /* L4 = Ch(e,f,g) */ \ - "xorl %" #g ", " L4 "\n\t" \ - -#define RND_STEP_RORX_1_5(a, b, c, d, e, f, g, h, i) \ - /* L2 = (a>>>2) ^ (a>>>13) */ \ - "xorl " L1 ", " L2 "\n\t" \ - /* L1 = a>>>22 */ \ - "rorx $22, %" #a ", " L1 "\n\t" \ - /* h += Ch(e,f,g) */ \ - "addl " L4 ", %" #h "\n\t" \ - -#define RND_STEP_RORX_1_6(a, b, c, d, e, f, g, h, i) \ - /* L1 = Sigma0(a) */ \ - "xorl " L2 ", " L1 "\n\t" \ - /* L4 = b */ \ - "movl %" #b ", " L4 "\n\t" \ - /* d += h + w_k + Sigma1(e) + Ch(e,f,g) */ \ - "addl %" #h ", %" #d "\n\t" \ - -#define RND_STEP_RORX_1_7(a, b, c, d, e, f, g, h, i) \ - /* L4 = a ^ b */ \ - "xorl %" #a ", " L4 "\n\t" \ - /* h += Sigma0(a) */ \ - "addl " L1 ", %" #h "\n\t" \ - /* L3 = (a ^ b) & (b ^ c) */ \ - "andl " L4 ", " L3 "\n\t" \ - -#define RND_STEP_RORX_1_8(a, b, c, d, e, f, g, h, i) \ - /* L3 = Maj(a,b,c) */ \ - "xorl %" #b ", " L3 "\n\t" \ - /* L1 = d>>>6 (= e>>>6 next RND) */ \ - "rorx $6, %" #d ", " L1 "\n\t" \ - /* h += Maj(a,b,c) */ \ - "addl " L3 ", %" #h "\n\t" \ - -#define _RND_RORX_X_0(a, b, c, d, e, f, g, h, i) \ - /* L1 = e>>>6 */ \ - "rorx $6, %" #e ", " L1 "\n\t" \ - /* L2 = e>>>11 */ \ - "rorx $11, %" #e ", " L2 "\n\t" \ - /* Prev RND: h += Maj(a,b,c) */ \ - "addl " L3 ", %" #a "\n\t" \ - /* h += w_k */ \ - "addl (" #i ")*4(" WK "), %" #h "\n\t" \ - /* L3 = f */ \ - "movl %" #f ", " L3 "\n\t" \ - /* L2 = (e>>>6) ^ (e>>>11) */ \ - "xorl " L1 ", " L2 "\n\t" \ - /* L3 = f ^ g */ \ - "xorl %" #g ", " L3 "\n\t" \ - /* L1 = e>>>25 */ \ - "rorx $25, %" #e ", " L1 "\n\t" \ - /* L1 = Sigma1(e) */ \ - "xorl " L2 ", " L1 "\n\t" \ - /* L3 = (f ^ g) & e */ \ - "andl %" #e ", " L3 "\n\t" \ - /* h += Sigma1(e) */ \ - "addl " L1 ", %" #h "\n\t" \ - /* L1 = a>>>2 */ \ - "rorx $2, %" #a ", " L1 "\n\t" \ - /* L2 = a>>>13 */ \ - "rorx $13, %" #a ", " L2 "\n\t" \ - /* L3 = Ch(e,f,g) */ \ - "xorl %" #g ", " L3 "\n\t" \ - /* L2 = (a>>>2) ^ (a>>>13) */ \ - "xorl " L1 ", " L2 "\n\t" \ - /* L1 = a>>>22 */ \ - "rorx $22, %" #a ", " L1 "\n\t" \ - /* h += Ch(e,f,g) */ \ - "addl " L3 ", %" #h "\n\t" \ - /* L1 = Sigma0(a) */ \ - "xorl " L2 ", " L1 "\n\t" \ - /* L3 = b */ \ - "movl %" #b ", " L3 "\n\t" \ - /* d += h + w_k + Sigma1(e) + Ch(e,f,g) */ \ - "addl %" #h ", %" #d "\n\t" \ - /* L3 = a ^ b */ \ - "xorl %" #a ", " L3 "\n\t" \ - /* L4 = (a ^ b) & (b ^ c) */ \ - "andl " L3 ", " L4 "\n\t" \ - /* h += Sigma0(a) */ \ - "addl " L1 ", %" #h "\n\t" \ - /* L4 = Maj(a,b,c) */ \ - "xorl %" #b ", " L4 "\n\t" \ - -#define _RND_RORX_X_1(a, b, c, d, e, f, g, h, i) \ - /* L1 = e>>>6 */ \ - "rorx $6, %" #e ", " L1 "\n\t" \ - /* L2 = e>>>11 */ \ - "rorx $11, %" #e ", " L2 "\n\t" \ - /* Prev RND: h += Maj(a,b,c) */ \ - "addl " L4 ", %" #a "\n\t" \ - /* h += w_k */ \ - "addl (" #i ")*4(" WK "), %" #h "\n\t" \ - /* L4 = f */ \ - "movl %" #f ", " L4 "\n\t" \ - /* L2 = (e>>>6) ^ (e>>>11) */ \ - "xorl " L1 ", " L2 "\n\t" \ - /* L4 = f ^ g */ \ - "xorl %" #g ", " L4 "\n\t" \ - /* L1 = e>>>25 */ \ - "rorx $25, %" #e ", " L1 "\n\t" \ - /* L1 = Sigma1(e) */ \ - "xorl " L2 ", " L1 "\n\t" \ - /* L4 = (f ^ g) & e */ \ - "andl %" #e ", " L4 "\n\t" \ - /* h += Sigma1(e) */ \ - "addl " L1 ", %" #h "\n\t" \ - /* L1 = a>>>2 */ \ - "rorx $2, %" #a ", " L1 "\n\t" \ - /* L2 = a>>>13 */ \ - "rorx $13, %" #a ", " L2 "\n\t" \ - /* L4 = Ch(e,f,g) */ \ - "xorl %" #g ", " L4 "\n\t" \ - /* L2 = (a>>>2) ^ (a>>>13) */ \ - "xorl " L1 ", " L2 "\n\t" \ - /* L1 = a>>>22 */ \ - "rorx $22, %" #a ", " L1 "\n\t" \ - /* h += Ch(e,f,g) */ \ - "addl " L4 ", %" #h "\n\t" \ - /* L1 = Sigma0(a) */ \ - "xorl " L2 ", " L1 "\n\t" \ - /* L4 = b */ \ - "movl %" #b ", " L4 "\n\t" \ - /* d += h + w_k + Sigma1(e) + Ch(e,f,g) */ \ - "addl %" #h ", %" #d "\n\t" \ - /* L4 = a ^ b */ \ - "xorl %" #a ", " L4 "\n\t" \ - /* L2 = (a ^ b) & (b ^ c) */ \ - "andl " L4 ", " L3 "\n\t" \ - /* h += Sigma0(a) */ \ - "addl " L1 ", %" #h "\n\t" \ - /* L3 = Maj(a,b,c) */ \ - "xorl %" #b ", " L3 "\n\t" \ - - -#define RND_RORX_X_0(a,b,c,d,e,f,g,h,i) \ - _RND_RORX_X_0(a,b,c,d,e,f,g,h,i) -#define RND_RORX_X_1(a,b,c,d,e,f,g,h,i) \ - _RND_RORX_X_1(a,b,c,d,e,f,g,h,i) - -#define RND_RORX_X4(a,b,c,d,e,f,g,h,i) \ - RND_RORX_X_0(a,b,c,d,e,f,g,h,i+0) \ - RND_RORX_X_1(h,a,b,c,d,e,f,g,i+1) \ - RND_RORX_X_0(g,h,a,b,c,d,e,f,i+2) \ - RND_RORX_X_1(f,g,h,a,b,c,d,e,i+3) - -#endif /* HAVE_INTEL_RORX */ - -#define RND_STEP_0_1(a,b,c,d,e,f,g,h,i) \ - /* L1 = e>>>14 */ \ - "rorl $14, " L1 "\n\t" \ - -#define RND_STEP_0_2(a,b,c,d,e,f,g,h,i) \ - /* L3 = b */ \ - "movl %" #b ", " L3 "\n\t" \ - /* L2 = f */ \ - "movl %" #f ", " L2 "\n\t" \ - /* h += w_k */ \ - "addl (" #i ")*4(" WK "), %" #h "\n\t" \ - /* L2 = f ^ g */ \ - "xorl %" #g ", " L2 "\n\t" \ - -#define RND_STEP_0_3(a,b,c,d,e,f,g,h,i) \ - /* L1 = (e>>>14) ^ e */ \ - "xorl %" #e ", " L1 "\n\t" \ - /* L2 = (f ^ g) & e */ \ - "andl %" #e ", " L2 "\n\t" \ - -#define RND_STEP_0_4(a,b,c,d,e,f,g,h,i) \ - /* L1 = ((e>>>14) ^ e) >>> 5 */ \ - "rorl $5, " L1 "\n\t" \ - /* L2 = Ch(e,f,g) */ \ - "xorl %" #g ", " L2 "\n\t" \ - /* L1 = (((e>>>14) ^ e) >>> 5) ^ e */ \ - "xorl %" #e ", " L1 "\n\t" \ - /* h += Ch(e,f,g) */ \ - "addl " L2 ", %" #h "\n\t" \ - -#define RND_STEP_0_5(a,b,c,d,e,f,g,h,i) \ - /* L1 = ((((e>>>14) ^ e) >>> 5) ^ e) >>> 6 */ \ - "rorl $6, " L1 "\n\t" \ - /* L3 = a ^ b (= b ^ c of next RND) */ \ - "xorl %" #a ", " L3 "\n\t" \ - /* h = h + w_k + Sigma1(e) */ \ - "addl " L1 ", %" #h "\n\t" \ - /* L2 = a */ \ - "movl %" #a ", " L2 "\n\t" \ - -#define RND_STEP_0_6(a,b,c,d,e,f,g,h,i) \ - /* L3 = (a ^ b) & (b ^ c) */ \ - "andl " L3 ", " L4 "\n\t" \ - /* L2 = a>>>9 */ \ - "rorl $9, " L2 "\n\t" \ - /* L2 = (a>>>9) ^ a */ \ - "xorl %" #a ", " L2 "\n\t" \ - /* L1 = Maj(a,b,c) */ \ - "xorl %" #b ", " L4 "\n\t" \ - -#define RND_STEP_0_7(a,b,c,d,e,f,g,h,i) \ - /* L2 = ((a>>>9) ^ a) >>> 11 */ \ - "rorl $11, " L2 "\n\t" \ - /* d += h + w_k + Sigma1(e) + Ch(e,f,g) */ \ - "addl %" #h ", %" #d "\n\t" \ - /* L2 = (((a>>>9) ^ a) >>> 11) ^ a */ \ - "xorl %" #a ", " L2 "\n\t" \ - /* h = h + w_k + Sigma1(e) + Ch(e,f,g) + Maj(a,b,c) */ \ - "addl " L4 ", %" #h "\n\t" \ - -#define RND_STEP_0_8(a,b,c,d,e,f,g,h,i) \ - /* L2 = ((((a>>>9) ^ a) >>> 11) ^ a) >>> 2 */ \ - "rorl $2, " L2 "\n\t" \ - /* L1 = d (e of next RND) */ \ - "movl %" #d ", " L1 "\n\t" \ - /* h = h + w_k + Sigma1(e) Sigma0(a) + Ch(e,f,g) + Maj(a,b,c) */ \ - "addl " L2 ", %" #h "\n\t" \ - -#define RND_STEP_1_1(a,b,c,d,e,f,g,h,i) \ - /* L1 = e>>>14 */ \ - "rorl $14, " L1 "\n\t" \ - -#define RND_STEP_1_2(a,b,c,d,e,f,g,h,i) \ - /* L3 = b */ \ - "movl %" #b ", " L4 "\n\t" \ - /* L2 = f */ \ - "movl %" #f ", " L2 "\n\t" \ - /* h += w_k */ \ - "addl (" #i ")*4(" WK "), %" #h "\n\t" \ - /* L2 = f ^ g */ \ - "xorl %" #g ", " L2 "\n\t" \ - -#define RND_STEP_1_3(a,b,c,d,e,f,g,h,i) \ - /* L1 = (e>>>14) ^ e */ \ - "xorl %" #e ", " L1 "\n\t" \ - /* L2 = (f ^ g) & e */ \ - "andl %" #e ", " L2 "\n\t" \ - -#define RND_STEP_1_4(a,b,c,d,e,f,g,h,i) \ - /* L1 = ((e>>>14) ^ e) >>> 5 */ \ - "rorl $5, " L1 "\n\t" \ - /* L2 = Ch(e,f,g) */ \ - "xorl %" #g ", " L2 "\n\t" \ - /* L1 = (((e>>>14) ^ e) >>> 5) ^ e */ \ - "xorl %" #e ", " L1 "\n\t" \ - /* h += Ch(e,f,g) */ \ - "addl " L2 ", %" #h "\n\t" \ - -#define RND_STEP_1_5(a,b,c,d,e,f,g,h,i) \ - /* L1 = ((((e>>>14) ^ e) >>> 5) ^ e) >>> 6 */ \ - "rorl $6, " L1 "\n\t" \ - /* L4 = a ^ b (= b ^ c of next RND) */ \ - "xorl %" #a ", " L4 "\n\t" \ - /* h = h + w_k + Sigma1(e) */ \ - "addl " L1 ", %" #h "\n\t" \ - /* L2 = a */ \ - "movl %" #a ", " L2 "\n\t" \ - -#define RND_STEP_1_6(a,b,c,d,e,f,g,h,i) \ - /* L3 = (a ^ b) & (b ^ c) */ \ - "andl " L4 ", " L3 "\n\t" \ - /* L2 = a>>>9 */ \ - "rorl $9, " L2 "\n\t" \ - /* L2 = (a>>>9) ^ a */ \ - "xorl %" #a ", " L2 "\n\t" \ - /* L1 = Maj(a,b,c) */ \ - "xorl %" #b ", " L3 "\n\t" \ - -#define RND_STEP_1_7(a,b,c,d,e,f,g,h,i) \ - /* L2 = ((a>>>9) ^ a) >>> 11 */ \ - "rorl $11, " L2 "\n\t" \ - /* d += h + w_k + Sigma1(e) + Ch(e,f,g) */ \ - "addl %" #h ", %" #d "\n\t" \ - /* L2 = (((a>>>9) ^ a) >>> 11) ^ a */ \ - "xorl %" #a ", " L2 "\n\t" \ - /* h = h + w_k + Sigma1(e) + Ch(e,f,g) + Maj(a,b,c) */ \ - "addl " L3 ", %" #h "\n\t" \ - -#define RND_STEP_1_8(a,b,c,d,e,f,g,h,i) \ - /* L2 = ((((a>>>9) ^ a) >>> 11) ^ a) >>> 2 */ \ - "rorl $2, " L2 "\n\t" \ - /* L1 = d (e of next RND) */ \ - "movl %" #d ", " L1 "\n\t" \ - /* h = h + w_k + Sigma1(e) Sigma0(a) + Ch(e,f,g) + Maj(a,b,c) */ \ - "addl " L2 ", %" #h "\n\t" \ - -#define _RND_ALL_0(a,b,c,d,e,f,g,h,i) \ - /* h += w_k */ \ - "addl (" #i ")*4(" WK "), %" #h "\n\t" \ - /* L2 = f */ \ - "movl %" #f ", " L2 "\n\t" \ - /* L3 = b */ \ - "movl %" #b ", " L3 "\n\t" \ - /* L2 = f ^ g */ \ - "xorl %" #g ", " L2 "\n\t" \ - /* L1 = e>>>14 */ \ - "rorl $14, " L1 "\n\t" \ - /* L2 = (f ^ g) & e */ \ - "andl %" #e ", " L2 "\n\t" \ - /* L1 = (e>>>14) ^ e */ \ - "xorl %" #e ", " L1 "\n\t" \ - /* L2 = Ch(e,f,g) */ \ - "xorl %" #g ", " L2 "\n\t" \ - /* L1 = ((e>>>14) ^ e) >>> 5 */ \ - "rorl $5, " L1 "\n\t" \ - /* h += Ch(e,f,g) */ \ - "addl " L2 ", %" #h "\n\t" \ - /* L1 = (((e>>>14) ^ e) >>> 5) ^ e */ \ - "xorl %" #e ", " L1 "\n\t" \ - /* L3 = a ^ b */ \ - "xorl %" #a ", " L3 "\n\t" \ - /* L1 = ((((e>>>14) ^ e) >>> 5) ^ e) >>> 6 */ \ - "rorl $6, " L1 "\n\t" \ - /* L2 = a */ \ - "movl %" #a ", " L2 "\n\t" \ - /* h = h + w_k + Sigma1(e) */ \ - "addl " L1 ", %" #h "\n\t" \ - /* L2 = a>>>9 */ \ - "rorl $9, " L2 "\n\t" \ - /* L3 = (a ^ b) & (b ^ c) */ \ - "andl " L3 ", " L4 "\n\t" \ - /* L2 = (a>>>9) ^ a */ \ - "xorl %" #a ", " L2 "\n\t" \ - /* L1 = Maj(a,b,c) */ \ - "xorl %" #b ", " L4 "\n\t" \ - /* L2 = ((a>>>9) ^ a) >>> 11 */ \ - "rorl $11, " L2 "\n\t" \ - /* d += h + w_k + Sigma1(e) + Ch(e,f,g) */ \ - "addl %" #h ", %" #d "\n\t" \ - /* L2 = (((a>>>9) ^ a) >>> 11) ^ a */ \ - "xorl %" #a ", " L2 "\n\t" \ - /* h = h + w_k + Sigma1(e) + Ch(e,f,g) + Maj(a,b,c) */ \ - "addl " L4 ", %" #h "\n\t" \ - /* L2 = ((((a>>>9) ^ a) >>> 11) ^ a) >>> 2 */ \ - "rorl $2, " L2 "\n\t" \ - /* L1 = d (e of next RND) */ \ - "movl %" #d ", " L1 "\n\t" \ - /* h = h + w_k + Sigma1(e) Sigma0(a) + Ch(e,f,g) + Maj(a,b,c) */ \ - "addl " L2 ", %" #h "\n\t" \ - -#define _RND_ALL_1(a,b,c,d,e,f,g,h,i) \ - /* h += w_k */ \ - "addl (" #i ")*4(" WK "), %" #h "\n\t" \ - /* L2 = f */ \ - "movl %" #f ", " L2 "\n\t" \ - /* L3 = b */ \ - "movl %" #b ", " L4 "\n\t" \ - /* L2 = f ^ g */ \ - "xorl %" #g ", " L2 "\n\t" \ - /* L1 = e>>>14 */ \ - "rorl $14, " L1 "\n\t" \ - /* L2 = (f ^ g) & e */ \ - "andl %" #e ", " L2 "\n\t" \ - /* L1 = (e>>>14) ^ e */ \ - "xorl %" #e ", " L1 "\n\t" \ - /* L2 = Ch(e,f,g) */ \ - "xorl %" #g ", " L2 "\n\t" \ - /* L1 = ((e>>>14) ^ e) >>> 5 */ \ - "rorl $5, " L1 "\n\t" \ - /* h += Ch(e,f,g) */ \ - "addl " L2 ", %" #h "\n\t" \ - /* L1 = (((e>>>14) ^ e) >>> 5) ^ e */ \ - "xorl %" #e ", " L1 "\n\t" \ - /* L3 = a ^ b */ \ - "xorl %" #a ", " L4 "\n\t" \ - /* L1 = ((((e>>>14) ^ e) >>> 5) ^ e) >>> 6 */ \ - "rorl $6, " L1 "\n\t" \ - /* L2 = a */ \ - "movl %" #a ", " L2 "\n\t" \ - /* h = h + w_k + Sigma1(e) */ \ - "addl " L1 ", %" #h "\n\t" \ - /* L2 = a>>>9 */ \ - "rorl $9, " L2 "\n\t" \ - /* L3 = (a ^ b) & (b ^ c) */ \ - "andl " L4 ", " L3 "\n\t" \ - /* L2 = (a>>>9) ^ a */ \ - "xorl %" #a", " L2 "\n\t" \ - /* L1 = Maj(a,b,c) */ \ - "xorl %" #b ", " L3 "\n\t" \ - /* L2 = ((a>>>9) ^ a) >>> 11 */ \ - "rorl $11, " L2 "\n\t" \ - /* d += h + w_k + Sigma1(e) + Ch(e,f,g) */ \ - "addl %" #h ", %" #d "\n\t" \ - /* L2 = (((a>>>9) ^ a) >>> 11) ^ a */ \ - "xorl %" #a ", " L2 "\n\t" \ - /* h = h + w_k + Sigma1(e) + Ch(e,f,g) + Maj(a,b,c) */ \ - "addl " L3 ", %" #h "\n\t" \ - /* L2 = ((((a>>>9) ^ a) >>> 11) ^ a) >>> 2 */ \ - "rorl $2, " L2 "\n\t" \ - /* L1 = d (e of next RND) */ \ - "movl %" #d ", " L1 "\n\t" \ - /* h = h + w_k + Sigma1(e) Sigma0(a) + Ch(e,f,g) + Maj(a,b,c) */ \ - "addl " L2 ", %" #h "\n\t" \ - - -#define RND_ALL_0(a, b, c, d, e, f, g, h, i) \ - _RND_ALL_0(a, b, c, d, e, f, g, h, i) -#define RND_ALL_1(a, b, c, d, e, f, g, h, i) \ - _RND_ALL_1(a, b, c, d, e, f, g, h, i) - -#define RND_ALL_4(a, b, c, d, e, f, g, h, i) \ - RND_ALL_0(a, b, c, d, e, f, g, h, i+0) \ - RND_ALL_1(h, a, b, c, d, e, f, g, i+1) \ - RND_ALL_0(g, h, a, b, c, d, e, f, i+2) \ - RND_ALL_1(f, g, h, a, b, c, d, e, i+3) - -#endif /* defined(HAVE_INTEL_AVX1) || defined(HAVE_INTEL_AVX2) */ - -#if defined(HAVE_INTEL_AVX1) /* inline Assember for Intel AVX1 instructions */ - -#define _VPALIGNR(op1, op2, op3, op4) \ - "vpalignr $" #op4", %" #op3", %" #op2", %" #op1"\n\t" -#define VPALIGNR(op1, op2, op3, op4) \ - _VPALIGNR(op1, op2, op3, op4) -#define _VPADDD(op1, op2, op3) \ - "vpaddd %" #op3", %" #op2", %" #op1"\n\t" -#define VPADDD(op1, op2, op3) \ - _VPADDD(op1, op2, op3) -#define _VPSRLD(op1, op2, op3) \ - "vpsrld $" #op3", %" #op2", %" #op1"\n\t" -#define VPSRLD(op1, op2, op3) \ - _VPSRLD(op1, op2, op3) -#define _VPSRLQ(op1, op2, op3) \ - "vpsrlq $" #op3", %" #op2", %" #op1"\n\t" -#define VPSRLQ(op1,op2,op3) \ - _VPSRLQ(op1,op2,op3) -#define _VPSLLD(op1,op2,op3) \ - "vpslld $" #op3", %" #op2", %" #op1"\n\t" -#define VPSLLD(op1,op2,op3) \ - _VPSLLD(op1,op2,op3) -#define _VPOR(op1,op2,op3) \ - "vpor %" #op3", %" #op2", %" #op1"\n\t" -#define VPOR(op1,op2,op3) \ - _VPOR(op1,op2,op3) -#define _VPXOR(op1,op2,op3) \ - "vpxor %" #op3", %" #op2", %" #op1"\n\t" -#define VPXOR(op1,op2,op3) \ - _VPXOR(op1,op2,op3) -#define _VPSHUFD(op1,op2,op3) \ - "vpshufd $" #op3", %" #op2", %" #op1"\n\t" -#define VPSHUFD(op1,op2,op3) \ - _VPSHUFD(op1,op2,op3) -#define _VPSHUFB(op1,op2,op3) \ - "vpshufb %" #op3", %" #op2", %" #op1"\n\t" -#define VPSHUFB(op1,op2,op3) \ - _VPSHUFB(op1,op2,op3) -#define _VPSLLDQ(op1,op2,op3) \ - "vpslldq $" #op3", %" #op2", %" #op1"\n\t" -#define VPSLLDQ(op1,op2,op3) \ - _VPSLLDQ(op1,op2,op3) - -#define MsgSched(X0,X1,X2,X3,a,b,c,d,e,f,g,h,_i) \ - RND_STEP_0_1(a,b,c,d,e,f,g,h,_i) \ - VPALIGNR (XTMP1, X1, X0, 4) /* XTMP1 = W[-15] */ \ - VPALIGNR (XTMP0, X3, X2, 4) /* XTMP0 = W[-7] */ \ - RND_STEP_0_2(a,b,c,d,e,f,g,h,_i) \ - RND_STEP_0_3(a,b,c,d,e,f,g,h,_i) \ - VPSRLD (XTMP2, XTMP1, 7) /* XTMP2 = W[-15] >> 7 */ \ - VPSLLD (XTMP3, XTMP1, 25) /* XTEMP3 = W[-15] << (32-7) */ \ - RND_STEP_0_4(a,b,c,d,e,f,g,h,_i) \ - RND_STEP_0_5(a,b,c,d,e,f,g,h,_i) \ - VPSRLD (XTMP4, XTMP1, 18) /* XTEMP4 = W[-15] >> 18 */ \ - VPSLLD (XTMP5, XTMP1, 14) /* XTEMP5 = W[-15] << (32-18) */ \ - RND_STEP_0_6(a,b,c,d,e,f,g,h,_i) \ - RND_STEP_0_7(a,b,c,d,e,f,g,h,_i) \ - VPOR (XTMP2, XTMP3, XTMP2) /* XTMP2 = W[-15] >>> 7 */ \ - VPOR (XTMP4, XTMP5, XTMP4) /* XTMP4 = W[-15] >>> 18 */ \ - RND_STEP_0_8(a,b,c,d,e,f,g,h,_i) \ - RND_STEP_1_1(h,a,b,c,d,e,f,g,_i+1) \ - RND_STEP_1_2(h,a,b,c,d,e,f,g,_i+1) \ - VPSRLD (XTMP5, XTMP1, 3) /* XTMP4 = W[-15] >> 3 */ \ - VPXOR (XTMP2, XTMP4, XTMP2) \ - /* XTMP2 = W[-15] MY_ROR 7 ^ W[-15] MY_ROR 18 */ \ - RND_STEP_1_3(h,a,b,c,d,e,f,g,_i+1) \ - RND_STEP_1_4(h,a,b,c,d,e,f,g,_i+1) \ - VPXOR (XTMP1, XTMP5, XTMP2) /* XTMP1 = s0 */ \ - VPSHUFD (XTMP2, X3, 0b11111010) /* XTMP2 = W[-2] {BBAA}*/ \ - RND_STEP_1_5(h,a,b,c,d,e,f,g,_i+1) \ - RND_STEP_1_6(h,a,b,c,d,e,f,g,_i+1) \ - VPSRLD (XTMP4, XTMP2, 10) /* XTMP4 = W[-2] >> 10 {BBAA} */ \ - VPSRLQ (XTMP3, XTMP2, 19) /* XTMP3 = W[-2] MY_ROR 19 {xBxA} */ \ - RND_STEP_1_7(h,a,b,c,d,e,f,g,_i+1) \ - RND_STEP_1_8(h,a,b,c,d,e,f,g,_i+1) \ - RND_STEP_0_1(g,h,a,b,c,d,e,f,_i+2) \ - VPSRLQ (XTMP2, XTMP2, 17) /* XTMP2 = W[-2] MY_ROR 17 {xBxA} */ \ - VPADDD (XTMP0, XTMP0, X0) \ - RND_STEP_0_2(g,h,a,b,c,d,e,f,_i+2) \ - RND_STEP_0_3(g,h,a,b,c,d,e,f,_i+2) \ - RND_STEP_0_4(g,h,a,b,c,d,e,f,_i+2) \ - VPXOR (XTMP2, XTMP3, XTMP2) \ - VPADDD (XTMP0, XTMP0, XTMP1) /* XTMP0 = W[-16] + W[-7] + s0 */ \ - RND_STEP_0_5(g,h,a,b,c,d,e,f,_i+2) \ - VPXOR (XTMP4, XTMP4, XTMP2) /* XTMP4 = s1 {xBxA} */ \ - RND_STEP_0_6(g,h,a,b,c,d,e,f,_i+2) \ - VPSHUFB (XTMP4, XTMP4, SHUF_00BA) /* XTMP4 = s1 {00BA} */ \ - RND_STEP_0_7(g,h,a,b,c,d,e,f,_i+2) \ - VPADDD (XTMP0, XTMP0, XTMP4) /* XTMP0 = {..., ..., W[1], W[0]} */ \ - RND_STEP_0_8(g,h,a,b,c,d,e,f,_i+2) \ - RND_STEP_1_1(f,g,h,a,b,c,d,e,_i+3) \ - VPSHUFD (XTMP2, XTMP0, 0b01010000) /* XTMP2 = W[-2] {DDCC} */ \ - RND_STEP_1_2(f,g,h,a,b,c,d,e,_i+3) \ - VPSRLQ (XTMP4, XTMP2, 17) /* XTMP4 = W[-2] MY_ROR 17 {xDxC} */ \ - VPSRLQ (XTMP3, XTMP2, 19) /* XTMP3 = W[-2] MY_ROR 19 {xDxC} */ \ - RND_STEP_1_3(f,g,h,a,b,c,d,e,_i+3) \ - RND_STEP_1_4(f,g,h,a,b,c,d,e,_i+3) \ - VPSRLD (XTMP5, XTMP2, 10) /* XTMP5 = W[-2] >> 10 {DDCC} */ \ - VPXOR (XTMP4, XTMP3, XTMP4) \ - RND_STEP_1_5(f,g,h,a,b,c,d,e,_i+3) \ - RND_STEP_1_6(f,g,h,a,b,c,d,e,_i+3) \ - VPXOR (XTMP5, XTMP4, XTMP5) /* XTMP5 = s1 {xDxC} */ \ - RND_STEP_1_7(f,g,h,a,b,c,d,e,_i+3) \ - VPSHUFB (XTMP5, XTMP5, SHUF_DC00) /* XTMP5 = s1 {DC00} */ \ - RND_STEP_1_8(f,g,h,a,b,c,d,e,_i+3) \ - VPADDD (X0, XTMP5, XTMP0) /* X0 = {W[3], W[2], W[1], W[0]} */ - -#if defined(HAVE_INTEL_RORX) - -#define MsgSched_RORX(X0,X1,X2,X3,a,b,c,d,e,f,g,h,_i) \ - RND_STEP_RORX_0_1(a,b,c,d,e,f,g,h,_i) \ - VPALIGNR (XTMP0, X3, X2, 4) \ - VPALIGNR (XTMP1, X1, X0, 4) /* XTMP1 = W[-15] */ \ - RND_STEP_RORX_0_2(a,b,c,d,e,f,g,h,_i) \ - RND_STEP_RORX_0_3(a,b,c,d,e,f,g,h,_i) \ - VPSRLD (XTMP2, XTMP1, 7) \ - VPSLLD (XTMP3, XTMP1, 25) /* VPSLLD (XTMP3, XTMP1, (32-7)) */ \ - RND_STEP_RORX_0_4(a,b,c,d,e,f,g,h,_i) \ - RND_STEP_RORX_0_5(a,b,c,d,e,f,g,h,_i) \ - VPSRLD (XTMP4, XTMP1, 3) /* XTMP4 = W[-15] >> 3 */ \ - VPOR (XTMP3, XTMP3, XTMP2) /* XTMP1 = W[-15] MY_ROR 7 */ \ - RND_STEP_RORX_0_6(a,b,c,d,e,f,g,h,_i) \ - RND_STEP_RORX_0_7(a,b,c,d,e,f,g,h,_i) \ - RND_STEP_RORX_0_8(a,b,c,d,e,f,g,h,_i) \ - \ - RND_STEP_RORX_1_1(h,a,b,c,d,e,f,g,_i+1) \ - VPSRLD (XTMP2, XTMP1,18) \ - RND_STEP_RORX_1_2(h,a,b,c,d,e,f,g,_i+1) \ - VPSLLD (XTMP1, XTMP1, 14) /* VPSLLD (XTMP1, XTMP1, (32-18)) */ \ - RND_STEP_RORX_1_3(h,a,b,c,d,e,f,g,_i+1) \ - VPXOR (XTMP3, XTMP3, XTMP1) \ - RND_STEP_RORX_1_4(h,a,b,c,d,e,f,g,_i+1) \ - VPXOR (XTMP3, XTMP3, XTMP2) \ - /* XTMP1 = W[-15] MY_ROR 7 ^ W[-15] MY_ROR 18 */ \ - RND_STEP_RORX_1_5(h,a,b,c,d,e,f,g,_i+1) \ - VPSHUFD (XTMP2, X3, 0b11111010) /* XTMP2 = W[-2] {BBAA}*/ \ - RND_STEP_RORX_1_6(h,a,b,c,d,e,f,g,_i+1) \ - VPXOR (XTMP1, XTMP3, XTMP4) /* XTMP1 = s0 */ \ - RND_STEP_RORX_1_7(h,a,b,c,d,e,f,g,_i+1) \ - VPSRLD (XTMP4, XTMP2, 10) /* XTMP4 = W[-2] >> 10 {BBAA} */ \ - RND_STEP_RORX_1_8(h,a,b,c,d,e,f,g,_i+1) \ - \ - RND_STEP_RORX_0_1(g,h,a,b,c,d,e,f,_i+2) \ - VPSRLQ (XTMP3, XTMP2, 19) /* XTMP3 = W[-2] MY_ROR 19 {xBxA} */ \ - RND_STEP_RORX_0_2(g,h,a,b,c,d,e,f,_i+2) \ - VPSRLQ (XTMP2, XTMP2, 17) /* XTMP2 = W[-2] MY_ROR 17 {xBxA} */ \ - VPADDD (XTMP0, XTMP0, X0) \ - RND_STEP_RORX_0_3(g,h,a,b,c,d,e,f,_i+2) \ - VPADDD (XTMP0, XTMP0, XTMP1) /* XTMP0 = W[-16] + W[-7] + s0 */ \ - RND_STEP_RORX_0_4(g,h,a,b,c,d,e,f,_i+2) \ - VPXOR (XTMP2, XTMP2, XTMP3) \ - RND_STEP_RORX_0_5(g,h,a,b,c,d,e,f,_i+2) \ - VPXOR (XTMP4, XTMP4, XTMP2) /* XTMP4 = s1 {xBxA} */ \ - RND_STEP_RORX_0_6(g,h,a,b,c,d,e,f,_i+2) \ - VPSHUFB (XTMP4, XTMP4, SHUF_00BA) /* XTMP4 = s1 {00BA} */ \ - RND_STEP_RORX_0_7(g,h,a,b,c,d,e,f,_i+2) \ - VPADDD (XTMP0, XTMP0, XTMP4) /* XTMP0 = {..., ..., W[1], W[0]} */ \ - RND_STEP_RORX_0_8(g,h,a,b,c,d,e,f,_i+2) \ - \ - RND_STEP_RORX_1_1(f,g,h,a,b,c,d,e,_i+3) \ - VPSHUFD (XTMP2, XTMP0, 0b01010000) /* XTMP2 = W[-2] {DDCC} */ \ - RND_STEP_RORX_1_2(f,g,h,a,b,c,d,e,_i+3) \ - VPSRLD (XTMP5, XTMP2, 10) /* XTMP5 = W[-2] >> 10 {DDCC} */ \ - RND_STEP_RORX_1_3(f,g,h,a,b,c,d,e,_i+3) \ - VPSRLQ (XTMP3, XTMP2, 19) /* XTMP3 = W[-2] MY_ROR 19 {xDxC} */ \ - RND_STEP_RORX_1_4(f,g,h,a,b,c,d,e,_i+3) \ - VPSRLQ (XTMP2, XTMP2, 17) /* XTMP2 = W[-2] MY_ROR 17 {xDxC} */ \ - RND_STEP_RORX_1_5(f,g,h,a,b,c,d,e,_i+3) \ - VPXOR (XTMP2, XTMP2, XTMP3) \ - RND_STEP_RORX_1_6(f,g,h,a,b,c,d,e,_i+3) \ - VPXOR (XTMP5, XTMP5, XTMP2) /* XTMP5 = s1 {xDxC} */ \ - RND_STEP_RORX_1_7(f,g,h,a,b,c,d,e,_i+3) \ - VPSHUFB (XTMP5, XTMP5, SHUF_DC00) /* XTMP5 = s1 {DC00} */ \ - RND_STEP_RORX_1_8(f,g,h,a,b,c,d,e,_i+3) \ - VPADDD (X0, XTMP5, XTMP0) /* X0 = {W[3], W[2], W[1], W[0]} */ - -#endif /* HAVE_INTEL_RORX */ - - -#define _W_K_from_buff(X0, X1, X2, X3, BYTE_FLIP_MASK) \ - "# X0, X1, X2, X3 = W[0..15]\n\t" \ - "vmovdqu (%%rax), %" #X0 "\n\t" \ - "vmovdqu 16(%%rax), %" #X1 "\n\t" \ - VPSHUFB(X0, X0, BYTE_FLIP_MASK) \ - VPSHUFB(X1, X1, BYTE_FLIP_MASK) \ - "vmovdqu 32(%%rax), %" #X2 "\n\t" \ - "vmovdqu 48(%%rax), %" #X3 "\n\t" \ - VPSHUFB(X2, X2, BYTE_FLIP_MASK) \ - VPSHUFB(X3, X3, BYTE_FLIP_MASK) - -#define W_K_from_buff(X0, X1, X2, X3, BYTE_FLIP_MASK) \ - _W_K_from_buff(X0, X1, X2, X3, BYTE_FLIP_MASK) - - -#define _SET_W_K_XFER_4(i) \ - "vpaddd (" #i "*4)+ 0+%[K], %%xmm0, %%xmm4\n\t" \ - "vpaddd (" #i "*4)+16+%[K], %%xmm1, %%xmm5\n\t" \ - "vmovdqu %%xmm4, (" WK ")\n\t" \ - "vmovdqu %%xmm5, 16(" WK ")\n\t" \ - "vpaddd (" #i "*4)+32+%[K], %%xmm2, %%xmm6\n\t" \ - "vpaddd (" #i "*4)+48+%[K], %%xmm3, %%xmm7\n\t" \ - "vmovdqu %%xmm6, 32(" WK ")\n\t" \ - "vmovdqu %%xmm7, 48(" WK ")\n\t" - -#define SET_W_K_XFER_4(i) \ - _SET_W_K_XFER_4(i) - - -static const ALIGN32 word64 mSHUF_00BA[] = - { 0x0b0a090803020100, 0xFFFFFFFFFFFFFFFF }; /* shuffle xBxA -> 00BA */ -static const ALIGN32 word64 mSHUF_DC00[] = - { 0xFFFFFFFFFFFFFFFF, 0x0b0a090803020100 }; /* shuffle xDxC -> DC00 */ -static const ALIGN32 word64 mBYTE_FLIP_MASK[] = - { 0x0405060700010203, 0x0c0d0e0f08090a0b }; - -#define _Init_Masks(mask1, mask2, mask3) \ - "vmovdqa %[FLIP], %" #mask1 "\n\t" \ - "vmovdqa %[SHUF00BA], %" #mask2 "\n\t" \ - "vmovdqa %[SHUFDC00], %" #mask3 "\n\t" - -#define Init_Masks(BYTE_FLIP_MASK, SHUF_00BA, SHUF_DC00) \ - _Init_Masks(BYTE_FLIP_MASK, SHUF_00BA, SHUF_DC00) - -#define X0 %xmm0 -#define X1 %xmm1 -#define X2 %xmm2 -#define X3 %xmm3 - -#define XTMP0 %xmm4 -#define XTMP1 %xmm5 -#define XTMP2 %xmm6 -#define XTMP3 %xmm7 -#define XTMP4 %xmm8 -#define XTMP5 %xmm9 -#define XFER %xmm10 - -#define SHUF_00BA %xmm11 /* shuffle xBxA -> 00BA */ -#define SHUF_DC00 %xmm12 /* shuffle xDxC -> DC00 */ -#define BYTE_FLIP_MASK %xmm13 - - -SHA256_NOINLINE static int Transform_Sha256_AVX1(wc_Sha256* sha256) -{ - __asm__ __volatile__ ( - - "subq $64, %%rsp\n\t" - - "leaq 32(%[sha256]), %%rax\n\t" - Init_Masks(BYTE_FLIP_MASK, SHUF_00BA, SHUF_DC00) - LOAD_DIGEST() - - W_K_from_buff(X0, X1, X2, X3, BYTE_FLIP_MASK) - - "movl %%r9d, " L4 "\n\t" - "movl %%r12d, " L1 "\n\t" - "xorl %%r10d, " L4 "\n\t" - - SET_W_K_XFER_4(0) - MsgSched(X0, X1, X2, X3, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 0) - MsgSched(X1, X2, X3, X0, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 4) - MsgSched(X2, X3, X0, X1, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 8) - MsgSched(X3, X0, X1, X2, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 12) - - SET_W_K_XFER_4(16) - MsgSched(X0, X1, X2, X3, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 0) - MsgSched(X1, X2, X3, X0, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 4) - MsgSched(X2, X3, X0, X1, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 8) - MsgSched(X3, X0, X1, X2, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 12) - - SET_W_K_XFER_4(32) - MsgSched(X0, X1, X2, X3, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 0) - MsgSched(X1, X2, X3, X0, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 4) - MsgSched(X2, X3, X0, X1, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 8) - MsgSched(X3, X0, X1, X2, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 12) - - SET_W_K_XFER_4(48) - RND_ALL_4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 0) - RND_ALL_4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 4) - RND_ALL_4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 8) - RND_ALL_4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 12) - - STORE_ADD_DIGEST() - - "addq $64, %%rsp\n\t" - - : - : [FLIP] "m" (mBYTE_FLIP_MASK[0]), - [SHUF00BA] "m" (mSHUF_00BA[0]), - [SHUFDC00] "m" (mSHUF_DC00[0]), - [sha256] "r" (sha256), - [K] "m" (K) - : WORK_REGS, STATE_REGS, XMM_REGS, "memory" - ); - - return 0; -} - -SHA256_NOINLINE static int Transform_Sha256_AVX1_Len(wc_Sha256* sha256, - word32 len) -{ - __asm__ __volatile__ ( - - "subq $64, %%rsp\n\t" - "movq 120(%[sha256]), %%rax\n\t" - - Init_Masks(BYTE_FLIP_MASK, SHUF_00BA, SHUF_DC00) - LOAD_DIGEST() - - "# Start of loop processing a block\n" - "1:\n\t" - - W_K_from_buff(X0, X1, X2, X3, BYTE_FLIP_MASK) - - "movl %%r9d, " L4 "\n\t" - "movl %%r12d, " L1 "\n\t" - "xorl %%r10d, " L4 "\n\t" - - SET_W_K_XFER_4(0) - MsgSched(X0, X1, X2, X3, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 0) - MsgSched(X1, X2, X3, X0, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 4) - MsgSched(X2, X3, X0, X1, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 8) - MsgSched(X3, X0, X1, X2, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 12) - - SET_W_K_XFER_4(16) - MsgSched(X0, X1, X2, X3, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 0) - MsgSched(X1, X2, X3, X0, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 4) - MsgSched(X2, X3, X0, X1, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 8) - MsgSched(X3, X0, X1, X2, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 12) - - SET_W_K_XFER_4(32) - MsgSched(X0, X1, X2, X3, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 0) - MsgSched(X1, X2, X3, X0, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 4) - MsgSched(X2, X3, X0, X1, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 8) - MsgSched(X3, X0, X1, X2, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 12) - - SET_W_K_XFER_4(48) - RND_ALL_4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 0) - RND_ALL_4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 4) - RND_ALL_4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 8) - RND_ALL_4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 12) - "movq 120(%[sha256]), %%rax\n\t" - - ADD_DIGEST() - - "addq $64, %%rax\n\t" - "subl $64, %[len]\n\t" - - STORE_DIGEST() - - "movq %%rax, 120(%[sha256])\n\t" - "jnz 1b\n\t" - - "addq $64, %%rsp\n\t" - - : - : [FLIP] "m" (mBYTE_FLIP_MASK[0]), - [SHUF00BA] "m" (mSHUF_00BA[0]), - [SHUFDC00] "m" (mSHUF_DC00[0]), - [sha256] "r" (sha256), - [len] "r" (len), - [K] "m" (K) - : WORK_REGS, STATE_REGS, XMM_REGS, "memory" - ); - - return 0; -} -#endif /* HAVE_INTEL_AVX1 */ - -#if defined(HAVE_INTEL_AVX2) && defined(HAVE_INTEL_RORX) -SHA256_NOINLINE static int Transform_Sha256_AVX1_RORX(wc_Sha256* sha256) -{ - __asm__ __volatile__ ( - - "subq $64, %%rsp\n\t" - - Init_Masks(BYTE_FLIP_MASK, SHUF_00BA, SHUF_DC00) - "leaq 32(%[sha256]), %%rax\n\t" - W_K_from_buff(X0, X1, X2, X3, BYTE_FLIP_MASK) - - LOAD_DIGEST() - - SET_W_K_XFER_4(0) - "movl %%r9d, " L4 "\n\t" - "rorx $6, %%r12d, " L1 "\n\t" - "xorl %%r10d, " L4 "\n\t" - MsgSched_RORX(X0, X1, X2, X3, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 0) - MsgSched_RORX(X1, X2, X3, X0, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 4) - MsgSched_RORX(X2, X3, X0, X1, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 8) - MsgSched_RORX(X3, X0, X1, X2, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 12) - - SET_W_K_XFER_4(16) - MsgSched_RORX(X0, X1, X2, X3, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 0) - MsgSched_RORX(X1, X2, X3, X0, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 4) - MsgSched_RORX(X2, X3, X0, X1, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 8) - MsgSched_RORX(X3, X0, X1, X2, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 12) - - SET_W_K_XFER_4(32) - MsgSched_RORX(X0, X1, X2, X3, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 0) - MsgSched_RORX(X1, X2, X3, X0, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 4) - MsgSched_RORX(X2, X3, X0, X1, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 8) - MsgSched_RORX(X3, X0, X1, X2, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 12) - - SET_W_K_XFER_4(48) - "xorl " L3 ", " L3 "\n\t" - RND_RORX_X4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 0) - RND_RORX_X4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 4) - RND_RORX_X4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 8) - RND_RORX_X4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 12) - /* Prev RND: h += Maj(a,b,c) */ - "addl " L3 ", %%r8d\n\t" - - STORE_ADD_DIGEST() - - "addq $64, %%rsp\n\t" - - : - : [FLIP] "m" (mBYTE_FLIP_MASK[0]), - [SHUF00BA] "m" (mSHUF_00BA[0]), - [SHUFDC00] "m" (mSHUF_DC00[0]), - [sha256] "r" (sha256), - [K] "m" (K) - : WORK_REGS, STATE_REGS, XMM_REGS, "memory" - ); - - return 0; -} - -SHA256_NOINLINE static int Transform_Sha256_AVX1_RORX_Len(wc_Sha256* sha256, - word32 len) -{ - __asm__ __volatile__ ( - - "subq $64, %%rsp\n\t" - "movq 120(%[sha256]), %%rax\n\t" - - Init_Masks(BYTE_FLIP_MASK, SHUF_00BA, SHUF_DC00) - LOAD_DIGEST() - - "# Start of loop processing a block\n" - "1:\n\t" - - W_K_from_buff(X0, X1, X2, X3, BYTE_FLIP_MASK) - - SET_W_K_XFER_4(0) - "movl %%r9d, " L4 "\n\t" - "rorx $6, %%r12d, " L1 "\n\t" - "xorl %%r10d, " L4 "\n\t" - MsgSched_RORX(X0, X1, X2, X3, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 0) - MsgSched_RORX(X1, X2, X3, X0, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 4) - MsgSched_RORX(X2, X3, X0, X1, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 8) - MsgSched_RORX(X3, X0, X1, X2, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 12) - - SET_W_K_XFER_4(16) - MsgSched_RORX(X0, X1, X2, X3, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 0) - MsgSched_RORX(X1, X2, X3, X0, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 4) - MsgSched_RORX(X2, X3, X0, X1, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 8) - MsgSched_RORX(X3, X0, X1, X2, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 12) - - SET_W_K_XFER_4(32) - MsgSched_RORX(X0, X1, X2, X3, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 0) - MsgSched_RORX(X1, X2, X3, X0, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 4) - MsgSched_RORX(X2, X3, X0, X1, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 8) - MsgSched_RORX(X3, X0, X1, X2, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 12) - - SET_W_K_XFER_4(48) - "xorl " L3 ", " L3 "\n\t" - "xorl " L2 ", " L2 "\n\t" - RND_RORX_X4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 0) - RND_RORX_X4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 4) - RND_RORX_X4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 8) - RND_RORX_X4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 12) - /* Prev RND: h += Maj(a,b,c) */ - "addl " L3 ", %%r8d\n\t" - "movq 120(%[sha256]), %%rax\n\t" - - ADD_DIGEST() - - "addq $64, %%rax\n\t" - "subl $64, %[len]\n\t" - - STORE_DIGEST() - - "movq %%rax, 120(%[sha256])\n\t" - "jnz 1b\n\t" - - "addq $64, %%rsp\n\t" - - : - : [FLIP] "m" (mBYTE_FLIP_MASK[0]), - [SHUF00BA] "m" (mSHUF_00BA[0]), - [SHUFDC00] "m" (mSHUF_DC00[0]), - [sha256] "r" (sha256), - [len] "r" (len), - [K] "m" (K) - : WORK_REGS, STATE_REGS, XMM_REGS, "memory" - ); - - return 0; -} -#endif /* HAVE_INTEL_AVX2 && HAVE_INTEL_RORX */ - - -#if defined(HAVE_INTEL_AVX2) -#define Y0 %ymm0 -#define Y1 %ymm1 -#define Y2 %ymm2 -#define Y3 %ymm3 - -#define YTMP0 %ymm4 -#define YTMP1 %ymm5 -#define YTMP2 %ymm6 -#define YTMP3 %ymm7 -#define YTMP4 %ymm8 -#define YTMP5 %ymm9 -#define YXFER %ymm10 - -#define SHUF_Y_00BA %ymm11 /* shuffle xBxA -> 00BA */ -#define SHUF_Y_DC00 %ymm12 /* shuffle xDxC -> DC00 */ -#define BYTE_FLIP_Y_MASK %ymm13 - -#define YMM_REGS "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", \ - "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13" - -#define MsgSched_Y(Y0,Y1,Y2,Y3,a,b,c,d,e,f,g,h,_i) \ - RND_STEP_0_1(a,b,c,d,e,f,g,h,_i) \ - VPALIGNR (YTMP1, Y1, Y0, 4) /* YTMP1 = W[-15] */ \ - VPALIGNR (YTMP0, Y3, Y2, 4) /* YTMP0 = W[-7] */ \ - RND_STEP_0_2(a,b,c,d,e,f,g,h,_i) \ - RND_STEP_0_3(a,b,c,d,e,f,g,h,_i) \ - VPSRLD (YTMP2, YTMP1, 7) /* YTMP2 = W[-15] >> 7 */ \ - VPSLLD (YTMP3, YTMP1, 25) /* YTEMP3 = W[-15] << (32-7) */ \ - RND_STEP_0_4(a,b,c,d,e,f,g,h,_i) \ - RND_STEP_0_5(a,b,c,d,e,f,g,h,_i) \ - VPSRLD (YTMP4, YTMP1, 18) /* YTEMP4 = W[-15] >> 18 */ \ - VPSLLD (YTMP5, YTMP1, 14) /* YTEMP5 = W[-15] << (32-18) */ \ - RND_STEP_0_6(a,b,c,d,e,f,g,h,_i) \ - RND_STEP_0_7(a,b,c,d,e,f,g,h,_i) \ - VPOR (YTMP2, YTMP3, YTMP2) /* YTMP2 = W[-15] >>> 7 */ \ - VPOR (YTMP4, YTMP5, YTMP4) /* YTMP4 = W[-15] >>> 18 */ \ - RND_STEP_0_8(a,b,c,d,e,f,g,h,_i) \ - RND_STEP_1_1(h,a,b,c,d,e,f,g,_i+1) \ - RND_STEP_1_2(h,a,b,c,d,e,f,g,_i+1) \ - VPSRLD (YTMP5, YTMP1, 3) /* YTMP4 = W[-15] >> 3 */ \ - VPXOR (YTMP2, YTMP4, YTMP2) /* YTMP2 = W[-15] >>> 7 ^ W[-15] >>> 18 */ \ - RND_STEP_1_3(h,a,b,c,d,e,f,g,_i+1) \ - RND_STEP_1_4(h,a,b,c,d,e,f,g,_i+1) \ - VPXOR (YTMP1, YTMP5, YTMP2) /* YTMP1 = s0 */ \ - VPSHUFD (YTMP2, Y3, 0b11111010) /* YTMP2 = W[-2] {BBAA}*/ \ - RND_STEP_1_5(h,a,b,c,d,e,f,g,_i+1) \ - RND_STEP_1_6(h,a,b,c,d,e,f,g,_i+1) \ - VPSRLD (YTMP4, YTMP2, 10) /* YTMP4 = W[-2] >> 10 {BBAA} */ \ - VPSRLQ (YTMP3, YTMP2, 19) /* YTMP3 = W[-2] MY_ROR 19 {xBxA} */ \ - RND_STEP_1_7(h,a,b,c,d,e,f,g,_i+1) \ - RND_STEP_1_8(h,a,b,c,d,e,f,g,_i+1) \ - RND_STEP_0_1(g,h,a,b,c,d,e,f,_i+2) \ - VPSRLQ (YTMP2, YTMP2, 17) /* YTMP2 = W[-2] MY_ROR 17 {xBxA} */ \ - VPADDD (YTMP0, YTMP0, Y0) \ - RND_STEP_0_2(g,h,a,b,c,d,e,f,_i+2) \ - RND_STEP_0_3(g,h,a,b,c,d,e,f,_i+2) \ - RND_STEP_0_4(g,h,a,b,c,d,e,f,_i+2) \ - VPXOR (YTMP2, YTMP3, YTMP2) \ - VPADDD (YTMP0, YTMP0, YTMP1) /* YTMP0 = W[-16] + W[-7] + s0 */ \ - RND_STEP_0_5(g,h,a,b,c,d,e,f,_i+2) \ - VPXOR (YTMP4, YTMP4, YTMP2) /* YTMP4 = s1 {xBxA} */ \ - RND_STEP_0_6(g,h,a,b,c,d,e,f,_i+2) \ - VPSHUFB (YTMP4, YTMP4, SHUF_Y_00BA) /* YTMP4 = s1 {00BA} */ \ - RND_STEP_0_7(g,h,a,b,c,d,e,f,_i+2) \ - VPADDD (YTMP0, YTMP0, YTMP4) /* YTMP0 = {..., ..., W[1], W[0]} */ \ - RND_STEP_0_8(g,h,a,b,c,d,e,f,_i+2) \ - RND_STEP_1_1(f,g,h,a,b,c,d,e,_i+3) \ - VPSHUFD (YTMP2, YTMP0, 0b01010000) /* YTMP2 = W[-2] {DDCC} */ \ - RND_STEP_1_2(f,g,h,a,b,c,d,e,_i+3) \ - VPSRLQ (YTMP4, YTMP2, 17) /* YTMP4 = W[-2] MY_ROR 17 {xDxC} */ \ - VPSRLQ (YTMP3, YTMP2, 19) /* YTMP3 = W[-2] MY_ROR 19 {xDxC} */ \ - RND_STEP_1_3(f,g,h,a,b,c,d,e,_i+3) \ - RND_STEP_1_4(f,g,h,a,b,c,d,e,_i+3) \ - VPSRLD (YTMP5, YTMP2, 10) /* YTMP5 = W[-2] >> 10 {DDCC} */ \ - VPXOR (YTMP4, YTMP3, YTMP4) \ - RND_STEP_1_5(f,g,h,a,b,c,d,e,_i+3) \ - RND_STEP_1_6(f,g,h,a,b,c,d,e,_i+3) \ - VPXOR (YTMP5, YTMP4, YTMP5) /* YTMP5 = s1 {xDxC} */ \ - RND_STEP_1_7(f,g,h,a,b,c,d,e,_i+3) \ - VPSHUFB (YTMP5, YTMP5, SHUF_Y_DC00) /* YTMP5 = s1 {DC00} */ \ - RND_STEP_1_8(f,g,h,a,b,c,d,e,_i+3) \ - VPADDD (Y0, YTMP5, YTMP0) /* Y0 = {W[3], W[2], W[1], W[0]} */ - -#if defined(HAVE_INTEL_RORX) - -#define MsgSched_Y_RORX(Y0,Y1,Y2,Y3,a,b,c,d,e,f,g,h,_i) \ - RND_STEP_RORX_0_1(a,b,c,d,e,f,g,h,_i) \ - VPALIGNR (YTMP1, Y1, Y0, 4) /* YTMP1 = W[-15] */ \ - RND_STEP_RORX_0_2(a,b,c,d,e,f,g,h,_i) \ - VPALIGNR (YTMP0, Y3, Y2, 4) /* YTMP0 = W[-7] */ \ - RND_STEP_RORX_0_3(a,b,c,d,e,f,g,h,_i) \ - VPSRLD (YTMP2, YTMP1, 7) /* YTMP2 = W[-15] >> 7 */ \ - RND_STEP_RORX_0_4(a,b,c,d,e,f,g,h,_i) \ - VPSLLD (YTMP3, YTMP1, 25) /* YTEMP3 = W[-15] << (32-7) */ \ - RND_STEP_RORX_0_5(a,b,c,d,e,f,g,h,_i) \ - VPSRLD (YTMP4, YTMP1, 18) /* YTEMP4 = W[-15] >> 18 */ \ - RND_STEP_RORX_0_6(a,b,c,d,e,f,g,h,_i) \ - VPSLLD (YTMP5, YTMP1, 14) /* YTEMP5 = W[-15] << (32-18) */ \ - RND_STEP_RORX_0_7(a,b,c,d,e,f,g,h,_i) \ - VPOR (YTMP2, YTMP2, YTMP3) /* YTMP2 = W[-15] >>> 7 */ \ - RND_STEP_RORX_0_8(a,b,c,d,e,f,g,h,_i) \ - VPOR (YTMP4, YTMP4, YTMP5) /* YTMP4 = W[-15] >>> 18 */ \ - RND_STEP_RORX_1_1(h,a,b,c,d,e,f,g,_i+1) \ - VPSRLD (YTMP5, YTMP1, 3) /* YTMP4 = W[-15] >> 3 */ \ - RND_STEP_RORX_1_2(h,a,b,c,d,e,f,g,_i+1) \ - VPXOR (YTMP2, YTMP2, YTMP4) /* YTMP2 = W[-15] >>> 7 ^ W[-15] >>> 18 */ \ - RND_STEP_RORX_1_3(h,a,b,c,d,e,f,g,_i+1) \ - VPSHUFD (YTMP3, Y3, 0b11111010) /* YTMP2 = W[-2] {BBAA}*/ \ - RND_STEP_RORX_1_4(h,a,b,c,d,e,f,g,_i+1) \ - VPXOR (YTMP1, YTMP5, YTMP2) /* YTMP1 = s0 */ \ - RND_STEP_RORX_1_5(h,a,b,c,d,e,f,g,_i+1) \ - VPSRLD (YTMP4, YTMP3, 10) /* YTMP4 = W[-2] >> 10 {BBAA} */ \ - RND_STEP_RORX_1_6(h,a,b,c,d,e,f,g,_i+1) \ - VPSRLQ (YTMP2, YTMP3, 19) /* YTMP3 = W[-2] MY_ROR 19 {xBxA} */ \ - RND_STEP_RORX_1_7(h,a,b,c,d,e,f,g,_i+1) \ - VPSRLQ (YTMP3, YTMP3, 17) /* YTMP2 = W[-2] MY_ROR 17 {xBxA} */ \ - RND_STEP_RORX_1_8(h,a,b,c,d,e,f,g,_i+1) \ - VPADDD (YTMP0, YTMP0, Y0) \ - RND_STEP_RORX_0_1(g,h,a,b,c,d,e,f,_i+2) \ - VPXOR (YTMP2, YTMP2, YTMP3) \ - RND_STEP_RORX_0_2(g,h,a,b,c,d,e,f,_i+2) \ - VPXOR (YTMP4, YTMP4, YTMP2) /* YTMP4 = s1 {xBxA} */ \ - RND_STEP_RORX_0_3(g,h,a,b,c,d,e,f,_i+2) \ - VPADDD (YTMP0, YTMP0, YTMP1) /* YTMP0 = W[-16] + W[-7] + s0 */ \ - RND_STEP_RORX_0_4(g,h,a,b,c,d,e,f,_i+2) \ - VPSHUFB (YTMP4, YTMP4, SHUF_Y_00BA) /* YTMP4 = s1 {00BA} */ \ - RND_STEP_RORX_0_5(g,h,a,b,c,d,e,f,_i+2) \ - VPADDD (YTMP0, YTMP0, YTMP4) /* YTMP0 = {..., ..., W[1], W[0]} */ \ - RND_STEP_RORX_0_6(g,h,a,b,c,d,e,f,_i+2) \ - VPSHUFD (YTMP2, YTMP0, 0b01010000) /* YTMP2 = W[-2] {DDCC} */ \ - RND_STEP_RORX_0_7(g,h,a,b,c,d,e,f,_i+2) \ - RND_STEP_RORX_0_8(g,h,a,b,c,d,e,f,_i+2) \ - VPSRLQ (YTMP4, YTMP2, 17) /* YTMP4 = W[-2] MY_ROR 17 {xDxC} */ \ - RND_STEP_RORX_1_1(f,g,h,a,b,c,d,e,_i+3) \ - VPSRLQ (YTMP3, YTMP2, 19) /* YTMP3 = W[-2] MY_ROR 19 {xDxC} */ \ - RND_STEP_RORX_1_2(f,g,h,a,b,c,d,e,_i+3) \ - VPSRLD (YTMP5, YTMP2, 10) /* YTMP5 = W[-2] >> 10 {DDCC} */ \ - RND_STEP_RORX_1_3(f,g,h,a,b,c,d,e,_i+3) \ - VPXOR (YTMP4, YTMP4, YTMP3) \ - RND_STEP_RORX_1_4(f,g,h,a,b,c,d,e,_i+3) \ - VPXOR (YTMP5, YTMP5, YTMP4) /* YTMP5 = s1 {xDxC} */ \ - RND_STEP_RORX_1_5(f,g,h,a,b,c,d,e,_i+3) \ - RND_STEP_RORX_1_6(f,g,h,a,b,c,d,e,_i+3) \ - VPSHUFB (YTMP5, YTMP5, SHUF_Y_DC00) /* YTMP5 = s1 {DC00} */ \ - RND_STEP_RORX_1_7(f,g,h,a,b,c,d,e,_i+3) \ - RND_STEP_RORX_1_8(f,g,h,a,b,c,d,e,_i+3) \ - VPADDD (Y0, YTMP5, YTMP0) /* Y0 = {W[3], W[2], W[1], W[0]} */ \ - -#endif /* HAVE_INTEL_RORX */ - -#define _VINSERTI128(op1,op2,op3,op4) \ - "vinserti128 $" #op4 ", %" #op3 ", %" #op2 ", %" #op1 "\n\t" -#define VINSERTI128(op1,op2,op3,op4) \ - _VINSERTI128(op1,op2,op3,op4) - - -#define _LOAD_W_K_LOW(BYTE_FLIP_MASK, reg) \ - "# X0, X1, X2, X3 = W[0..15]\n\t" \ - "vmovdqu (%%" #reg "), %%xmm0\n\t" \ - "vmovdqu 16(%%" #reg "), %%xmm1\n\t" \ - VPSHUFB(X0, X0, BYTE_FLIP_MASK) \ - VPSHUFB(X1, X1, BYTE_FLIP_MASK) \ - "vmovdqu 32(%%" #reg "), %%xmm2\n\t" \ - "vmovdqu 48(%%" #reg "), %%xmm3\n\t" \ - VPSHUFB(X2, X2, BYTE_FLIP_MASK) \ - VPSHUFB(X3, X3, BYTE_FLIP_MASK) - -#define LOAD_W_K_LOW(BYTE_FLIP_MASK, reg) \ - _LOAD_W_K_LOW(BYTE_FLIP_MASK, reg) - - -#define _LOAD_W_K(BYTE_FLIP_Y_MASK, reg) \ - "# X0, X1, X2, X3 = W[0..15]\n\t" \ - "vmovdqu (%%" #reg "), %%xmm0\n\t" \ - "vmovdqu 16(%%" #reg "), %%xmm1\n\t" \ - "vmovdqu 64(%%" #reg "), %%xmm4\n\t" \ - "vmovdqu 80(%%" #reg "), %%xmm5\n\t" \ - VINSERTI128(Y0, Y0, XTMP0, 1) \ - VINSERTI128(Y1, Y1, XTMP1, 1) \ - VPSHUFB(Y0, Y0, BYTE_FLIP_Y_MASK) \ - VPSHUFB(Y1, Y1, BYTE_FLIP_Y_MASK) \ - "vmovdqu 32(%%" #reg "), %%xmm2\n\t" \ - "vmovdqu 48(%%" #reg "), %%xmm3\n\t" \ - "vmovdqu 96(%%" #reg "), %%xmm6\n\t" \ - "vmovdqu 112(%%" #reg "), %%xmm7\n\t" \ - VINSERTI128(Y2, Y2, XTMP2, 1) \ - VINSERTI128(Y3, Y3, XTMP3, 1) \ - VPSHUFB(Y2, Y2, BYTE_FLIP_Y_MASK) \ - VPSHUFB(Y3, Y3, BYTE_FLIP_Y_MASK) - -#define LOAD_W_K(BYTE_FLIP_Y_MASK, reg) \ - _LOAD_W_K(BYTE_FLIP_Y_MASK, reg) - - -#define _SET_W_Y_4(i) \ - "vpaddd (" #i "*8)+ 0+%[K], %%ymm0, %%ymm4\n\t" \ - "vpaddd (" #i "*8)+32+%[K], %%ymm1, %%ymm5\n\t" \ - "vmovdqu %%ymm4, (" #i "*8)+ 0(" WK ")\n\t" \ - "vmovdqu %%ymm5, (" #i "*8)+32(" WK ")\n\t" \ - "vpaddd (" #i "*8)+64+%[K], %%ymm2, %%ymm4\n\t" \ - "vpaddd (" #i "*8)+96+%[K], %%ymm3, %%ymm5\n\t" \ - "vmovdqu %%ymm4, (" #i "*8)+64(" WK ")\n\t" \ - "vmovdqu %%ymm5, (" #i "*8)+96(" WK ")\n\t" - -#define SET_W_Y_4(i) \ - _SET_W_Y_4(i) - - -static const ALIGN32 word64 mSHUF_Y_00BA[] = - { 0x0b0a090803020100, 0xFFFFFFFFFFFFFFFF, - 0x0b0a090803020100, 0xFFFFFFFFFFFFFFFF }; /* shuffle xBxA -> 00BA */ -static const ALIGN32 word64 mSHUF_Y_DC00[] = - { 0xFFFFFFFFFFFFFFFF, 0x0b0a090803020100, - 0xFFFFFFFFFFFFFFFF, 0x0b0a090803020100 }; /* shuffle xDxC -> DC00 */ -static const ALIGN32 word64 mBYTE_FLIP_Y_MASK[] = - { 0x0405060700010203, 0x0c0d0e0f08090a0b, - 0x0405060700010203, 0x0c0d0e0f08090a0b }; - -#define _INIT_MASKS_Y(BYTE_FLIP_MASK, SHUF_00BA, SHUF_DC00) \ - "vmovdqa %[FLIP], %" #BYTE_FLIP_MASK "\n\t" \ - "vmovdqa %[SHUF00BA], %" #SHUF_00BA "\n\t" \ - "vmovdqa %[SHUFDC00], %" #SHUF_DC00 "\n\t" - -#define INIT_MASKS_Y(BYTE_FLIP_MASK, SHUF_00BA, SHUF_DC00) \ - _INIT_MASKS_Y(BYTE_FLIP_MASK, SHUF_00BA, SHUF_DC00) - -static const ALIGN32 word32 K256[128] = { - 0x428A2F98L, 0x71374491L, 0xB5C0FBCFL, 0xE9B5DBA5L, - 0x428A2F98L, 0x71374491L, 0xB5C0FBCFL, 0xE9B5DBA5L, - 0x3956C25BL, 0x59F111F1L, 0x923F82A4L, 0xAB1C5ED5L, - 0x3956C25BL, 0x59F111F1L, 0x923F82A4L, 0xAB1C5ED5L, - 0xD807AA98L, 0x12835B01L, 0x243185BEL, 0x550C7DC3L, - 0xD807AA98L, 0x12835B01L, 0x243185BEL, 0x550C7DC3L, - 0x72BE5D74L, 0x80DEB1FEL, 0x9BDC06A7L, 0xC19BF174L, - 0x72BE5D74L, 0x80DEB1FEL, 0x9BDC06A7L, 0xC19BF174L, - 0xE49B69C1L, 0xEFBE4786L, 0x0FC19DC6L, 0x240CA1CCL, - 0xE49B69C1L, 0xEFBE4786L, 0x0FC19DC6L, 0x240CA1CCL, - 0x2DE92C6FL, 0x4A7484AAL, 0x5CB0A9DCL, 0x76F988DAL, - 0x2DE92C6FL, 0x4A7484AAL, 0x5CB0A9DCL, 0x76F988DAL, - 0x983E5152L, 0xA831C66DL, 0xB00327C8L, 0xBF597FC7L, - 0x983E5152L, 0xA831C66DL, 0xB00327C8L, 0xBF597FC7L, - 0xC6E00BF3L, 0xD5A79147L, 0x06CA6351L, 0x14292967L, - 0xC6E00BF3L, 0xD5A79147L, 0x06CA6351L, 0x14292967L, - 0x27B70A85L, 0x2E1B2138L, 0x4D2C6DFCL, 0x53380D13L, - 0x27B70A85L, 0x2E1B2138L, 0x4D2C6DFCL, 0x53380D13L, - 0x650A7354L, 0x766A0ABBL, 0x81C2C92EL, 0x92722C85L, - 0x650A7354L, 0x766A0ABBL, 0x81C2C92EL, 0x92722C85L, - 0xA2BFE8A1L, 0xA81A664BL, 0xC24B8B70L, 0xC76C51A3L, - 0xA2BFE8A1L, 0xA81A664BL, 0xC24B8B70L, 0xC76C51A3L, - 0xD192E819L, 0xD6990624L, 0xF40E3585L, 0x106AA070L, - 0xD192E819L, 0xD6990624L, 0xF40E3585L, 0x106AA070L, - 0x19A4C116L, 0x1E376C08L, 0x2748774CL, 0x34B0BCB5L, - 0x19A4C116L, 0x1E376C08L, 0x2748774CL, 0x34B0BCB5L, - 0x391C0CB3L, 0x4ED8AA4AL, 0x5B9CCA4FL, 0x682E6FF3L, - 0x391C0CB3L, 0x4ED8AA4AL, 0x5B9CCA4FL, 0x682E6FF3L, - 0x748F82EEL, 0x78A5636FL, 0x84C87814L, 0x8CC70208L, - 0x748F82EEL, 0x78A5636FL, 0x84C87814L, 0x8CC70208L, - 0x90BEFFFAL, 0xA4506CEBL, 0xBEF9A3F7L, 0xC67178F2L, - 0x90BEFFFAL, 0xA4506CEBL, 0xBEF9A3F7L, 0xC67178F2L -}; - -SHA256_NOINLINE static int Transform_Sha256_AVX2(wc_Sha256* sha256) -{ - __asm__ __volatile__ ( - - "subq $512, %%rsp\n\t" - "leaq 32(%[sha256]), %%rax\n\t" - - INIT_MASKS_Y(BYTE_FLIP_MASK, SHUF_Y_00BA, SHUF_Y_DC00) - LOAD_DIGEST() - - LOAD_W_K_LOW(BYTE_FLIP_MASK, rax) - - "movl %%r9d, " L4 "\n\t" - "movl %%r12d, " L1 "\n\t" - "xorl %%r10d, " L4 "\n\t" - - SET_W_Y_4(0) - MsgSched_Y(Y0, Y1, Y2, Y3, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 0) - MsgSched_Y(Y1, Y2, Y3, Y0, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 8) - MsgSched_Y(Y2, Y3, Y0, Y1, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 16) - MsgSched_Y(Y3, Y0, Y1, Y2, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 24) - - SET_W_Y_4(16) - MsgSched_Y(Y0, Y1, Y2, Y3, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 32) - MsgSched_Y(Y1, Y2, Y3, Y0, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 40) - MsgSched_Y(Y2, Y3, Y0, Y1, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 48) - MsgSched_Y(Y3, Y0, Y1, Y2, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 56) - - SET_W_Y_4(32) - MsgSched_Y(Y0, Y1, Y2, Y3, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 64) - MsgSched_Y(Y1, Y2, Y3, Y0, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 72) - MsgSched_Y(Y2, Y3, Y0, Y1, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 80) - MsgSched_Y(Y3, Y0, Y1, Y2, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 88) - - SET_W_Y_4(48) - RND_ALL_4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 96) - RND_ALL_4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 104) - RND_ALL_4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 112) - RND_ALL_4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 120) - - STORE_ADD_DIGEST() - - "addq $512, %%rsp\n\t" - - : - : [FLIP] "m" (mBYTE_FLIP_MASK[0]), - [SHUF00BA] "m" (mSHUF_Y_00BA[0]), - [SHUFDC00] "m" (mSHUF_Y_DC00[0]), - [sha256] "r" (sha256), - [K] "m" (K256) - : WORK_REGS, STATE_REGS, YMM_REGS, "memory" - ); - - return 0; -} - -SHA256_NOINLINE static int Transform_Sha256_AVX2_Len(wc_Sha256* sha256, - word32 len) -{ - if ((len & WC_SHA256_BLOCK_SIZE) != 0) { - XMEMCPY(sha256->buffer, sha256->data, WC_SHA256_BLOCK_SIZE); - Transform_Sha256_AVX2(sha256); - sha256->data += WC_SHA256_BLOCK_SIZE; - len -= WC_SHA256_BLOCK_SIZE; - if (len == 0) - return 0; - } - - __asm__ __volatile__ ( - - "subq $512, %%rsp\n\t" - "movq 120(%[sha256]), %%rax\n\t" - - INIT_MASKS_Y(BYTE_FLIP_Y_MASK, SHUF_Y_00BA, SHUF_Y_DC00) - LOAD_DIGEST() - - "# Start of loop processing two blocks\n" - "1:\n\t" - - LOAD_W_K(BYTE_FLIP_Y_MASK, rax) - - "movl %%r9d, " L4 "\n\t" - "movl %%r12d, " L1 "\n\t" - "xorl %%r10d, " L4 "\n\t" - - SET_W_Y_4(0) - MsgSched_Y(Y0, Y1, Y2, Y3, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 0) - MsgSched_Y(Y1, Y2, Y3, Y0, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 8) - MsgSched_Y(Y2, Y3, Y0, Y1, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 16) - MsgSched_Y(Y3, Y0, Y1, Y2, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 24) - - SET_W_Y_4(16) - MsgSched_Y(Y0, Y1, Y2, Y3, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 32) - MsgSched_Y(Y1, Y2, Y3, Y0, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 40) - MsgSched_Y(Y2, Y3, Y0, Y1, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 48) - MsgSched_Y(Y3, Y0, Y1, Y2, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 56) - - SET_W_Y_4(32) - MsgSched_Y(Y0, Y1, Y2, Y3, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 64) - MsgSched_Y(Y1, Y2, Y3, Y0, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 72) - MsgSched_Y(Y2, Y3, Y0, Y1, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 80) - MsgSched_Y(Y3, Y0, Y1, Y2, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 88) - - SET_W_Y_4(48) - RND_ALL_4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 96) - RND_ALL_4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 104) - RND_ALL_4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 112) - RND_ALL_4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 120) - - ADD_DIGEST() - STORE_DIGEST() - - "movl %%r9d, " L4 "\n\t" - "movl %%r12d, " L1 "\n\t" - "xorl %%r10d, " L4 "\n\t" - - RND_ALL_4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 4) - RND_ALL_4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 12) - RND_ALL_4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 20) - RND_ALL_4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 28) - RND_ALL_4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 36) - RND_ALL_4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 44) - RND_ALL_4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 52) - RND_ALL_4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 60) - RND_ALL_4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 68) - RND_ALL_4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 76) - RND_ALL_4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 84) - RND_ALL_4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 92) - RND_ALL_4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 100) - RND_ALL_4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 108) - RND_ALL_4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 116) - RND_ALL_4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 124) - - ADD_DIGEST() - - "movq 120(%[sha256]), %%rax\n\t" - "addq $128, %%rax\n\t" - "subl $128, %[len]\n\t" - - STORE_DIGEST() - - "movq %%rax, 120(%[sha256])\n\t" - "jnz 1b\n\t" - - "addq $512, %%rsp\n\t" - - : - : [FLIP] "m" (mBYTE_FLIP_Y_MASK[0]), - [SHUF00BA] "m" (mSHUF_Y_00BA[0]), - [SHUFDC00] "m" (mSHUF_Y_DC00[0]), - [sha256] "r" (sha256), - [len] "r" (len), - [K] "m" (K256) - : WORK_REGS, STATE_REGS, YMM_REGS, "memory" - ); - - return 0; -} - -#if defined(HAVE_INTEL_RORX) -SHA256_NOINLINE static int Transform_Sha256_AVX2_RORX(wc_Sha256* sha256) -{ - __asm__ __volatile__ ( - - "subq $512, %%rsp\n\t" - "leaq 32(%[sha256]), %%rax\n\t" - - INIT_MASKS_Y(BYTE_FLIP_MASK, SHUF_Y_00BA, SHUF_Y_DC00) - LOAD_W_K_LOW(BYTE_FLIP_MASK, rax) - - LOAD_DIGEST() - - "movl %%r9d, " L4 "\n\t" - "rorx $6, %%r12d, " L1 "\n\t" - "xorl %%r10d, " L4 "\n\t" - - SET_W_Y_4(0) - MsgSched_Y_RORX(Y0, Y1, Y2, Y3, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 0) - MsgSched_Y_RORX(Y1, Y2, Y3, Y0, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 8) - MsgSched_Y_RORX(Y2, Y3, Y0, Y1, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 16) - MsgSched_Y_RORX(Y3, Y0, Y1, Y2, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 24) - - SET_W_Y_4(16) - MsgSched_Y_RORX(Y0, Y1, Y2, Y3, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 32) - MsgSched_Y_RORX(Y1, Y2, Y3, Y0, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 40) - MsgSched_Y_RORX(Y2, Y3, Y0, Y1, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 48) - MsgSched_Y_RORX(Y3, Y0, Y1, Y2, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 56) - - SET_W_Y_4(32) - MsgSched_Y_RORX(Y0, Y1, Y2, Y3, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 64) - MsgSched_Y_RORX(Y1, Y2, Y3, Y0, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 72) - MsgSched_Y_RORX(Y2, Y3, Y0, Y1, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 80) - MsgSched_Y_RORX(Y3, Y0, Y1, Y2, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 88) - - SET_W_Y_4(48) - "xorl " L3 ", " L3 "\n\t" - "xorl " L2 ", " L2 "\n\t" - RND_RORX_X4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 96) - RND_RORX_X4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 104) - RND_RORX_X4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 112) - RND_RORX_X4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 120) - /* Prev RND: h += Maj(a,b,c) */ - "addl " L3 ", %%r8d\n\t" - - STORE_ADD_DIGEST() - - "addq $512, %%rsp\n\t" - - : - : [FLIP] "m" (mBYTE_FLIP_MASK[0]), - [SHUF00BA] "m" (mSHUF_Y_00BA[0]), - [SHUFDC00] "m" (mSHUF_Y_DC00[0]), - [sha256] "r" (sha256), - [K] "m" (K256) - : WORK_REGS, STATE_REGS, YMM_REGS, "memory" - ); - - return 0; -} - -SHA256_NOINLINE static int Transform_Sha256_AVX2_RORX_Len(wc_Sha256* sha256, - word32 len) -{ - if ((len & WC_SHA256_BLOCK_SIZE) != 0) { - XMEMCPY(sha256->buffer, sha256->data, WC_SHA256_BLOCK_SIZE); - Transform_Sha256_AVX2_RORX(sha256); - sha256->data += WC_SHA256_BLOCK_SIZE; - len -= WC_SHA256_BLOCK_SIZE; - if (len == 0) - return 0; - } - - __asm__ __volatile__ ( - - "subq $512, %%rsp\n\t" - "movq 120(%[sha256]), %%rax\n\t" - - INIT_MASKS_Y(BYTE_FLIP_Y_MASK, SHUF_Y_00BA, SHUF_Y_DC00) - LOAD_DIGEST() - - "# Start of loop processing two blocks\n" - "1:\n\t" - - LOAD_W_K(BYTE_FLIP_Y_MASK, rax) - - "movl %%r9d, " L4 "\n\t" - "rorx $6, %%r12d, " L1 "\n\t" - "xorl %%r10d, " L4 "\n\t" - - SET_W_Y_4(0) - MsgSched_Y_RORX(Y0, Y1, Y2, Y3, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 0) - MsgSched_Y_RORX(Y1, Y2, Y3, Y0, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 8) - MsgSched_Y_RORX(Y2, Y3, Y0, Y1, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 16) - MsgSched_Y_RORX(Y3, Y0, Y1, Y2, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 24) - - SET_W_Y_4(16) - MsgSched_Y_RORX(Y0, Y1, Y2, Y3, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 32) - MsgSched_Y_RORX(Y1, Y2, Y3, Y0, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 40) - MsgSched_Y_RORX(Y2, Y3, Y0, Y1, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 48) - MsgSched_Y_RORX(Y3, Y0, Y1, Y2, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 56) - - SET_W_Y_4(32) - MsgSched_Y_RORX(Y0, Y1, Y2, Y3, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 64) - MsgSched_Y_RORX(Y1, Y2, Y3, Y0, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 72) - MsgSched_Y_RORX(Y2, Y3, Y0, Y1, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 80) - MsgSched_Y_RORX(Y3, Y0, Y1, Y2, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 88) - - SET_W_Y_4(48) - "xorl " L3 ", " L3 "\n\t" - "xorl " L2 ", " L2 "\n\t" - RND_RORX_X4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 96) - RND_RORX_X4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 104) - RND_RORX_X4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 112) - RND_RORX_X4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 120) - /* Prev RND: h += Maj(a,b,c) */ - "addl " L3 ", %%r8d\n\t" - "xorl " L2 ", " L2 "\n\t" - - ADD_DIGEST() - STORE_DIGEST() - - "movl %%r9d, " L4 "\n\t" - "xorl " L3 ", " L3 "\n\t" - "xorl %%r10d, " L4 "\n\t" - - RND_RORX_X4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 4) - RND_RORX_X4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 12) - RND_RORX_X4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 20) - RND_RORX_X4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 28) - RND_RORX_X4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 36) - RND_RORX_X4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 44) - RND_RORX_X4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 52) - RND_RORX_X4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 60) - RND_RORX_X4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 68) - RND_RORX_X4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 76) - RND_RORX_X4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 84) - RND_RORX_X4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 92) - RND_RORX_X4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 100) - RND_RORX_X4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 108) - RND_RORX_X4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 116) - RND_RORX_X4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 124) - /* Prev RND: h += Maj(a,b,c) */ - "addl " L3 ", %%r8d\n\t" - "movq 120(%[sha256]), %%rax\n\t" - - ADD_DIGEST() - - "addq $128, %%rax\n\t" - "subl $128, %[len]\n\t" - - STORE_DIGEST() - - "movq %%rax, 120(%[sha256])\n\t" - "jnz 1b\n\t" - - "addq $512, %%rsp\n\t" - - : - : [FLIP] "m" (mBYTE_FLIP_Y_MASK[0]), - [SHUF00BA] "m" (mSHUF_Y_00BA[0]), - [SHUFDC00] "m" (mSHUF_Y_DC00[0]), - [sha256] "r" (sha256), - [len] "r" (len), - [K] "m" (K256) - : WORK_REGS, STATE_REGS, YMM_REGS, "memory" - ); - - return 0; -} -#endif /* HAVE_INTEL_RORX */ -#endif /* HAVE_INTEL_AVX2 */ - - #ifdef WOLFSSL_SHA224 #ifdef STM32_HASH_SHA2 diff --git a/wolfcrypt/src/sha256_asm.S b/wolfcrypt/src/sha256_asm.S new file mode 100644 index 0000000000..09db6eb50a --- /dev/null +++ b/wolfcrypt/src/sha256_asm.S @@ -0,0 +1,22437 @@ +/* sha256_asm + * + * Copyright (C) 2006-2018 wolfSSL Inc. + * + * This file is part of wolfSSL. + * + * wolfSSL is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * wolfSSL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA + */ + +#ifndef HAVE_INTEL_AVX1 +#define HAVE_INTEL_AVX1 +#endif /* HAVE_INTEL_AVX1 */ +#ifndef HAVE_INTEL_AVX2 +#define HAVE_INTEL_AVX2 +#endif /* HAVE_INTEL_AVX2 */ + +#ifdef HAVE_INTEL_AVX1 +L_avx1_sha256_k: +.long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 +.long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 +.long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 +.long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 +.long 0xe49b69c1,0xefbe4786,0xfc19dc6,0x240ca1cc +.long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da +.long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 +.long 0xc6e00bf3,0xd5a79147,0x6ca6351,0x14292967 +.long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 +.long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 +.long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 +.long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 +.long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 +.long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 +.long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 +.long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 +.align 16 +L_avx1_sha256_shuf_00BA: +.quad 0xb0a090803020100, 0xffffffffffffffff +.align 16 +L_avx1_sha256_shuf_DC00: +.quad 0xffffffffffffffff, 0xb0a090803020100 +.align 16 +L_avx1_sha256_flip_mask: +.quad 0x405060700010203, 0xc0d0e0f08090a0b +.globl Transform_Sha256_AVX1 +.type Transform_Sha256_AVX1,@function +.align 4 +Transform_Sha256_AVX1: + pushq %rbx + pushq %r12 + pushq %r13 + pushq %r14 + pushq %r15 + subq $0x40, %rsp + leaq 32(%rdi), %rax + vmovdqa L_avx1_sha256_flip_mask(%rip), %xmm13 + vmovdqa L_avx1_sha256_shuf_00BA(%rip), %xmm11 + vmovdqa L_avx1_sha256_shuf_DC00(%rip), %xmm12 + movl (%rdi), %r8d + movl 4(%rdi), %r9d + movl 8(%rdi), %r10d + movl 12(%rdi), %r11d + movl 16(%rdi), %r12d + movl 20(%rdi), %r13d + movl 24(%rdi), %r14d + movl 28(%rdi), %r15d + # X0, X1, X2, X3 = W[0..15] + vmovdqu (%rax), %xmm0 + vmovdqu 16(%rax), %xmm1 + vpshufb %xmm13, %xmm0, %xmm0 + vpshufb %xmm13, %xmm1, %xmm1 + vmovdqu 32(%rax), %xmm2 + vmovdqu 48(%rax), %xmm3 + vpshufb %xmm13, %xmm2, %xmm2 + vpshufb %xmm13, %xmm3, %xmm3 + movl %r9d, %ebx + movl %r12d, %edx + xorl %r10d, %ebx + # set_w_k_xfer_4: 0 + vpaddd 0+L_avx1_sha256_k(%rip), %xmm0, %xmm4 + vpaddd 16+L_avx1_sha256_k(%rip), %xmm1, %xmm5 + vmovdqu %xmm4, (%rsp) + vmovdqu %xmm5, 16(%rsp) + vpaddd 32+L_avx1_sha256_k(%rip), %xmm2, %xmm6 + vpaddd 48+L_avx1_sha256_k(%rip), %xmm3, %xmm7 + vmovdqu %xmm6, 32(%rsp) + vmovdqu %xmm7, 48(%rsp) + # msg_sched: 0-3 + # rnd_0: 0 - 0 + rorl $14, %edx + vpalignr $4, %xmm0, %xmm1, %xmm5 + vpalignr $4, %xmm2, %xmm3, %xmm4 + # rnd_0: 1 - 2 + movl %r9d, %eax + movl %r13d, %ecx + addl (%rsp), %r15d + xorl %r14d, %ecx + xorl %r12d, %edx + andl %r12d, %ecx + vpsrld $7, %xmm5, %xmm6 + vpslld $25, %xmm5, %xmm7 + # rnd_0: 3 - 4 + rorl $5, %edx + xorl %r14d, %ecx + xorl %r12d, %edx + addl %ecx, %r15d + rorl $6, %edx + xorl %r8d, %eax + addl %edx, %r15d + movl %r8d, %ecx + vpsrld $18, %xmm5, %xmm8 + vpslld $14, %xmm5, %xmm9 + # rnd_0: 5 - 6 + andl %eax, %ebx + rorl $9, %ecx + xorl %r8d, %ecx + xorl %r9d, %ebx + rorl $11, %ecx + addl %r15d, %r11d + xorl %r8d, %ecx + addl %ebx, %r15d + vpor %xmm6, %xmm7, %xmm6 + vpor %xmm8, %xmm9, %xmm8 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r11d, %edx + addl %ecx, %r15d + # rnd_1: 0 - 1 + rorl $14, %edx + movl %r8d, %ebx + movl %r12d, %ecx + addl 4(%rsp), %r14d + xorl %r13d, %ecx + vpsrld $3, %xmm5, %xmm9 + vpxor %xmm6, %xmm8, %xmm6 + # rnd_1: 2 - 3 + xorl %r11d, %edx + andl %r11d, %ecx + rorl $5, %edx + xorl %r13d, %ecx + xorl %r11d, %edx + addl %ecx, %r14d + vpxor %xmm6, %xmm9, %xmm5 + vpshufd $0xfa, %xmm3, %xmm6 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r15d, %ebx + addl %edx, %r14d + movl %r15d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r15d, %ecx + xorl %r8d, %eax + vpsrld $10, %xmm6, %xmm8 + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 6 - 7 + rorl $11, %ecx + addl %r14d, %r10d + xorl %r15d, %ecx + addl %eax, %r14d + rorl $2, %ecx + movl %r10d, %edx + addl %ecx, %r14d + # rnd_0: 0 - 0 + rorl $14, %edx + vpsrlq $0x11, %xmm6, %xmm6 + vpaddd %xmm0, %xmm4, %xmm4 + # rnd_0: 1 - 3 + movl %r15d, %eax + movl %r11d, %ecx + addl 8(%rsp), %r13d + xorl %r12d, %ecx + xorl %r10d, %edx + andl %r10d, %ecx + rorl $5, %edx + xorl %r12d, %ecx + xorl %r10d, %edx + addl %ecx, %r13d + vpxor %xmm6, %xmm7, %xmm6 + vpaddd %xmm5, %xmm4, %xmm4 + # rnd_0: 4 - 4 + rorl $6, %edx + xorl %r14d, %eax + addl %edx, %r13d + movl %r14d, %ecx + vpxor %xmm6, %xmm8, %xmm8 + # rnd_0: 5 - 5 + andl %eax, %ebx + rorl $9, %ecx + xorl %r14d, %ecx + xorl %r15d, %ebx + vpshufb %xmm11, %xmm8, %xmm8 + # rnd_0: 6 - 6 + rorl $11, %ecx + addl %r13d, %r9d + xorl %r14d, %ecx + addl %ebx, %r13d + vpaddd %xmm8, %xmm4, %xmm4 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r9d, %edx + addl %ecx, %r13d + # rnd_1: 0 - 0 + rorl $14, %edx + vpshufd $0x50, %xmm4, %xmm6 + # rnd_1: 1 - 1 + movl %r14d, %ebx + movl %r10d, %ecx + addl 12(%rsp), %r12d + xorl %r11d, %ecx + vpsrlq $0x11, %xmm6, %xmm8 + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 2 - 3 + xorl %r9d, %edx + andl %r9d, %ecx + rorl $5, %edx + xorl %r11d, %ecx + xorl %r9d, %edx + addl %ecx, %r12d + vpsrld $10, %xmm6, %xmm9 + vpxor %xmm8, %xmm7, %xmm8 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r13d, %ebx + addl %edx, %r12d + movl %r13d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r13d, %ecx + xorl %r14d, %eax + vpxor %xmm9, %xmm8, %xmm9 + # rnd_1: 6 - 6 + rorl $11, %ecx + addl %r12d, %r8d + xorl %r13d, %ecx + addl %eax, %r12d + vpshufb %xmm12, %xmm9, %xmm9 + # rnd_1: 7 - 7 + rorl $2, %ecx + movl %r8d, %edx + addl %ecx, %r12d + vpaddd %xmm4, %xmm9, %xmm0 + # msg_sched done: 0-3 + # msg_sched: 4-7 + # rnd_0: 0 - 0 + rorl $14, %edx + vpalignr $4, %xmm1, %xmm2, %xmm5 + vpalignr $4, %xmm3, %xmm0, %xmm4 + # rnd_0: 1 - 2 + movl %r13d, %eax + movl %r9d, %ecx + addl 16(%rsp), %r11d + xorl %r10d, %ecx + xorl %r8d, %edx + andl %r8d, %ecx + vpsrld $7, %xmm5, %xmm6 + vpslld $25, %xmm5, %xmm7 + # rnd_0: 3 - 4 + rorl $5, %edx + xorl %r10d, %ecx + xorl %r8d, %edx + addl %ecx, %r11d + rorl $6, %edx + xorl %r12d, %eax + addl %edx, %r11d + movl %r12d, %ecx + vpsrld $18, %xmm5, %xmm8 + vpslld $14, %xmm5, %xmm9 + # rnd_0: 5 - 6 + andl %eax, %ebx + rorl $9, %ecx + xorl %r12d, %ecx + xorl %r13d, %ebx + rorl $11, %ecx + addl %r11d, %r15d + xorl %r12d, %ecx + addl %ebx, %r11d + vpor %xmm6, %xmm7, %xmm6 + vpor %xmm8, %xmm9, %xmm8 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r15d, %edx + addl %ecx, %r11d + # rnd_1: 0 - 1 + rorl $14, %edx + movl %r12d, %ebx + movl %r8d, %ecx + addl 20(%rsp), %r10d + xorl %r9d, %ecx + vpsrld $3, %xmm5, %xmm9 + vpxor %xmm6, %xmm8, %xmm6 + # rnd_1: 2 - 3 + xorl %r15d, %edx + andl %r15d, %ecx + rorl $5, %edx + xorl %r9d, %ecx + xorl %r15d, %edx + addl %ecx, %r10d + vpxor %xmm6, %xmm9, %xmm5 + vpshufd $0xfa, %xmm0, %xmm6 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r11d, %ebx + addl %edx, %r10d + movl %r11d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r11d, %ecx + xorl %r12d, %eax + vpsrld $10, %xmm6, %xmm8 + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 6 - 7 + rorl $11, %ecx + addl %r10d, %r14d + xorl %r11d, %ecx + addl %eax, %r10d + rorl $2, %ecx + movl %r14d, %edx + addl %ecx, %r10d + # rnd_0: 0 - 0 + rorl $14, %edx + vpsrlq $0x11, %xmm6, %xmm6 + vpaddd %xmm1, %xmm4, %xmm4 + # rnd_0: 1 - 3 + movl %r11d, %eax + movl %r15d, %ecx + addl 24(%rsp), %r9d + xorl %r8d, %ecx + xorl %r14d, %edx + andl %r14d, %ecx + rorl $5, %edx + xorl %r8d, %ecx + xorl %r14d, %edx + addl %ecx, %r9d + vpxor %xmm6, %xmm7, %xmm6 + vpaddd %xmm5, %xmm4, %xmm4 + # rnd_0: 4 - 4 + rorl $6, %edx + xorl %r10d, %eax + addl %edx, %r9d + movl %r10d, %ecx + vpxor %xmm6, %xmm8, %xmm8 + # rnd_0: 5 - 5 + andl %eax, %ebx + rorl $9, %ecx + xorl %r10d, %ecx + xorl %r11d, %ebx + vpshufb %xmm11, %xmm8, %xmm8 + # rnd_0: 6 - 6 + rorl $11, %ecx + addl %r9d, %r13d + xorl %r10d, %ecx + addl %ebx, %r9d + vpaddd %xmm8, %xmm4, %xmm4 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r13d, %edx + addl %ecx, %r9d + # rnd_1: 0 - 0 + rorl $14, %edx + vpshufd $0x50, %xmm4, %xmm6 + # rnd_1: 1 - 1 + movl %r10d, %ebx + movl %r14d, %ecx + addl 28(%rsp), %r8d + xorl %r15d, %ecx + vpsrlq $0x11, %xmm6, %xmm8 + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 2 - 3 + xorl %r13d, %edx + andl %r13d, %ecx + rorl $5, %edx + xorl %r15d, %ecx + xorl %r13d, %edx + addl %ecx, %r8d + vpsrld $10, %xmm6, %xmm9 + vpxor %xmm8, %xmm7, %xmm8 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r9d, %ebx + addl %edx, %r8d + movl %r9d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r9d, %ecx + xorl %r10d, %eax + vpxor %xmm9, %xmm8, %xmm9 + # rnd_1: 6 - 6 + rorl $11, %ecx + addl %r8d, %r12d + xorl %r9d, %ecx + addl %eax, %r8d + vpshufb %xmm12, %xmm9, %xmm9 + # rnd_1: 7 - 7 + rorl $2, %ecx + movl %r12d, %edx + addl %ecx, %r8d + vpaddd %xmm4, %xmm9, %xmm1 + # msg_sched done: 4-7 + # msg_sched: 8-11 + # rnd_0: 0 - 0 + rorl $14, %edx + vpalignr $4, %xmm2, %xmm3, %xmm5 + vpalignr $4, %xmm0, %xmm1, %xmm4 + # rnd_0: 1 - 2 + movl %r9d, %eax + movl %r13d, %ecx + addl 32(%rsp), %r15d + xorl %r14d, %ecx + xorl %r12d, %edx + andl %r12d, %ecx + vpsrld $7, %xmm5, %xmm6 + vpslld $25, %xmm5, %xmm7 + # rnd_0: 3 - 4 + rorl $5, %edx + xorl %r14d, %ecx + xorl %r12d, %edx + addl %ecx, %r15d + rorl $6, %edx + xorl %r8d, %eax + addl %edx, %r15d + movl %r8d, %ecx + vpsrld $18, %xmm5, %xmm8 + vpslld $14, %xmm5, %xmm9 + # rnd_0: 5 - 6 + andl %eax, %ebx + rorl $9, %ecx + xorl %r8d, %ecx + xorl %r9d, %ebx + rorl $11, %ecx + addl %r15d, %r11d + xorl %r8d, %ecx + addl %ebx, %r15d + vpor %xmm6, %xmm7, %xmm6 + vpor %xmm8, %xmm9, %xmm8 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r11d, %edx + addl %ecx, %r15d + # rnd_1: 0 - 1 + rorl $14, %edx + movl %r8d, %ebx + movl %r12d, %ecx + addl 36(%rsp), %r14d + xorl %r13d, %ecx + vpsrld $3, %xmm5, %xmm9 + vpxor %xmm6, %xmm8, %xmm6 + # rnd_1: 2 - 3 + xorl %r11d, %edx + andl %r11d, %ecx + rorl $5, %edx + xorl %r13d, %ecx + xorl %r11d, %edx + addl %ecx, %r14d + vpxor %xmm6, %xmm9, %xmm5 + vpshufd $0xfa, %xmm1, %xmm6 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r15d, %ebx + addl %edx, %r14d + movl %r15d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r15d, %ecx + xorl %r8d, %eax + vpsrld $10, %xmm6, %xmm8 + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 6 - 7 + rorl $11, %ecx + addl %r14d, %r10d + xorl %r15d, %ecx + addl %eax, %r14d + rorl $2, %ecx + movl %r10d, %edx + addl %ecx, %r14d + # rnd_0: 0 - 0 + rorl $14, %edx + vpsrlq $0x11, %xmm6, %xmm6 + vpaddd %xmm2, %xmm4, %xmm4 + # rnd_0: 1 - 3 + movl %r15d, %eax + movl %r11d, %ecx + addl 40(%rsp), %r13d + xorl %r12d, %ecx + xorl %r10d, %edx + andl %r10d, %ecx + rorl $5, %edx + xorl %r12d, %ecx + xorl %r10d, %edx + addl %ecx, %r13d + vpxor %xmm6, %xmm7, %xmm6 + vpaddd %xmm5, %xmm4, %xmm4 + # rnd_0: 4 - 4 + rorl $6, %edx + xorl %r14d, %eax + addl %edx, %r13d + movl %r14d, %ecx + vpxor %xmm6, %xmm8, %xmm8 + # rnd_0: 5 - 5 + andl %eax, %ebx + rorl $9, %ecx + xorl %r14d, %ecx + xorl %r15d, %ebx + vpshufb %xmm11, %xmm8, %xmm8 + # rnd_0: 6 - 6 + rorl $11, %ecx + addl %r13d, %r9d + xorl %r14d, %ecx + addl %ebx, %r13d + vpaddd %xmm8, %xmm4, %xmm4 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r9d, %edx + addl %ecx, %r13d + # rnd_1: 0 - 0 + rorl $14, %edx + vpshufd $0x50, %xmm4, %xmm6 + # rnd_1: 1 - 1 + movl %r14d, %ebx + movl %r10d, %ecx + addl 44(%rsp), %r12d + xorl %r11d, %ecx + vpsrlq $0x11, %xmm6, %xmm8 + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 2 - 3 + xorl %r9d, %edx + andl %r9d, %ecx + rorl $5, %edx + xorl %r11d, %ecx + xorl %r9d, %edx + addl %ecx, %r12d + vpsrld $10, %xmm6, %xmm9 + vpxor %xmm8, %xmm7, %xmm8 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r13d, %ebx + addl %edx, %r12d + movl %r13d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r13d, %ecx + xorl %r14d, %eax + vpxor %xmm9, %xmm8, %xmm9 + # rnd_1: 6 - 6 + rorl $11, %ecx + addl %r12d, %r8d + xorl %r13d, %ecx + addl %eax, %r12d + vpshufb %xmm12, %xmm9, %xmm9 + # rnd_1: 7 - 7 + rorl $2, %ecx + movl %r8d, %edx + addl %ecx, %r12d + vpaddd %xmm4, %xmm9, %xmm2 + # msg_sched done: 8-11 + # msg_sched: 12-15 + # rnd_0: 0 - 0 + rorl $14, %edx + vpalignr $4, %xmm3, %xmm0, %xmm5 + vpalignr $4, %xmm1, %xmm2, %xmm4 + # rnd_0: 1 - 2 + movl %r13d, %eax + movl %r9d, %ecx + addl 48(%rsp), %r11d + xorl %r10d, %ecx + xorl %r8d, %edx + andl %r8d, %ecx + vpsrld $7, %xmm5, %xmm6 + vpslld $25, %xmm5, %xmm7 + # rnd_0: 3 - 4 + rorl $5, %edx + xorl %r10d, %ecx + xorl %r8d, %edx + addl %ecx, %r11d + rorl $6, %edx + xorl %r12d, %eax + addl %edx, %r11d + movl %r12d, %ecx + vpsrld $18, %xmm5, %xmm8 + vpslld $14, %xmm5, %xmm9 + # rnd_0: 5 - 6 + andl %eax, %ebx + rorl $9, %ecx + xorl %r12d, %ecx + xorl %r13d, %ebx + rorl $11, %ecx + addl %r11d, %r15d + xorl %r12d, %ecx + addl %ebx, %r11d + vpor %xmm6, %xmm7, %xmm6 + vpor %xmm8, %xmm9, %xmm8 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r15d, %edx + addl %ecx, %r11d + # rnd_1: 0 - 1 + rorl $14, %edx + movl %r12d, %ebx + movl %r8d, %ecx + addl 52(%rsp), %r10d + xorl %r9d, %ecx + vpsrld $3, %xmm5, %xmm9 + vpxor %xmm6, %xmm8, %xmm6 + # rnd_1: 2 - 3 + xorl %r15d, %edx + andl %r15d, %ecx + rorl $5, %edx + xorl %r9d, %ecx + xorl %r15d, %edx + addl %ecx, %r10d + vpxor %xmm6, %xmm9, %xmm5 + vpshufd $0xfa, %xmm2, %xmm6 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r11d, %ebx + addl %edx, %r10d + movl %r11d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r11d, %ecx + xorl %r12d, %eax + vpsrld $10, %xmm6, %xmm8 + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 6 - 7 + rorl $11, %ecx + addl %r10d, %r14d + xorl %r11d, %ecx + addl %eax, %r10d + rorl $2, %ecx + movl %r14d, %edx + addl %ecx, %r10d + # rnd_0: 0 - 0 + rorl $14, %edx + vpsrlq $0x11, %xmm6, %xmm6 + vpaddd %xmm3, %xmm4, %xmm4 + # rnd_0: 1 - 3 + movl %r11d, %eax + movl %r15d, %ecx + addl 56(%rsp), %r9d + xorl %r8d, %ecx + xorl %r14d, %edx + andl %r14d, %ecx + rorl $5, %edx + xorl %r8d, %ecx + xorl %r14d, %edx + addl %ecx, %r9d + vpxor %xmm6, %xmm7, %xmm6 + vpaddd %xmm5, %xmm4, %xmm4 + # rnd_0: 4 - 4 + rorl $6, %edx + xorl %r10d, %eax + addl %edx, %r9d + movl %r10d, %ecx + vpxor %xmm6, %xmm8, %xmm8 + # rnd_0: 5 - 5 + andl %eax, %ebx + rorl $9, %ecx + xorl %r10d, %ecx + xorl %r11d, %ebx + vpshufb %xmm11, %xmm8, %xmm8 + # rnd_0: 6 - 6 + rorl $11, %ecx + addl %r9d, %r13d + xorl %r10d, %ecx + addl %ebx, %r9d + vpaddd %xmm8, %xmm4, %xmm4 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r13d, %edx + addl %ecx, %r9d + # rnd_1: 0 - 0 + rorl $14, %edx + vpshufd $0x50, %xmm4, %xmm6 + # rnd_1: 1 - 1 + movl %r10d, %ebx + movl %r14d, %ecx + addl 60(%rsp), %r8d + xorl %r15d, %ecx + vpsrlq $0x11, %xmm6, %xmm8 + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 2 - 3 + xorl %r13d, %edx + andl %r13d, %ecx + rorl $5, %edx + xorl %r15d, %ecx + xorl %r13d, %edx + addl %ecx, %r8d + vpsrld $10, %xmm6, %xmm9 + vpxor %xmm8, %xmm7, %xmm8 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r9d, %ebx + addl %edx, %r8d + movl %r9d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r9d, %ecx + xorl %r10d, %eax + vpxor %xmm9, %xmm8, %xmm9 + # rnd_1: 6 - 6 + rorl $11, %ecx + addl %r8d, %r12d + xorl %r9d, %ecx + addl %eax, %r8d + vpshufb %xmm12, %xmm9, %xmm9 + # rnd_1: 7 - 7 + rorl $2, %ecx + movl %r12d, %edx + addl %ecx, %r8d + vpaddd %xmm4, %xmm9, %xmm3 + # msg_sched done: 12-15 + # set_w_k_xfer_4: 4 + vpaddd 64+L_avx1_sha256_k(%rip), %xmm0, %xmm4 + vpaddd 80+L_avx1_sha256_k(%rip), %xmm1, %xmm5 + vmovdqu %xmm4, (%rsp) + vmovdqu %xmm5, 16(%rsp) + vpaddd 96+L_avx1_sha256_k(%rip), %xmm2, %xmm6 + vpaddd 112+L_avx1_sha256_k(%rip), %xmm3, %xmm7 + vmovdqu %xmm6, 32(%rsp) + vmovdqu %xmm7, 48(%rsp) + # msg_sched: 0-3 + # rnd_0: 0 - 0 + rorl $14, %edx + vpalignr $4, %xmm0, %xmm1, %xmm5 + vpalignr $4, %xmm2, %xmm3, %xmm4 + # rnd_0: 1 - 2 + movl %r9d, %eax + movl %r13d, %ecx + addl (%rsp), %r15d + xorl %r14d, %ecx + xorl %r12d, %edx + andl %r12d, %ecx + vpsrld $7, %xmm5, %xmm6 + vpslld $25, %xmm5, %xmm7 + # rnd_0: 3 - 4 + rorl $5, %edx + xorl %r14d, %ecx + xorl %r12d, %edx + addl %ecx, %r15d + rorl $6, %edx + xorl %r8d, %eax + addl %edx, %r15d + movl %r8d, %ecx + vpsrld $18, %xmm5, %xmm8 + vpslld $14, %xmm5, %xmm9 + # rnd_0: 5 - 6 + andl %eax, %ebx + rorl $9, %ecx + xorl %r8d, %ecx + xorl %r9d, %ebx + rorl $11, %ecx + addl %r15d, %r11d + xorl %r8d, %ecx + addl %ebx, %r15d + vpor %xmm6, %xmm7, %xmm6 + vpor %xmm8, %xmm9, %xmm8 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r11d, %edx + addl %ecx, %r15d + # rnd_1: 0 - 1 + rorl $14, %edx + movl %r8d, %ebx + movl %r12d, %ecx + addl 4(%rsp), %r14d + xorl %r13d, %ecx + vpsrld $3, %xmm5, %xmm9 + vpxor %xmm6, %xmm8, %xmm6 + # rnd_1: 2 - 3 + xorl %r11d, %edx + andl %r11d, %ecx + rorl $5, %edx + xorl %r13d, %ecx + xorl %r11d, %edx + addl %ecx, %r14d + vpxor %xmm6, %xmm9, %xmm5 + vpshufd $0xfa, %xmm3, %xmm6 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r15d, %ebx + addl %edx, %r14d + movl %r15d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r15d, %ecx + xorl %r8d, %eax + vpsrld $10, %xmm6, %xmm8 + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 6 - 7 + rorl $11, %ecx + addl %r14d, %r10d + xorl %r15d, %ecx + addl %eax, %r14d + rorl $2, %ecx + movl %r10d, %edx + addl %ecx, %r14d + # rnd_0: 0 - 0 + rorl $14, %edx + vpsrlq $0x11, %xmm6, %xmm6 + vpaddd %xmm0, %xmm4, %xmm4 + # rnd_0: 1 - 3 + movl %r15d, %eax + movl %r11d, %ecx + addl 8(%rsp), %r13d + xorl %r12d, %ecx + xorl %r10d, %edx + andl %r10d, %ecx + rorl $5, %edx + xorl %r12d, %ecx + xorl %r10d, %edx + addl %ecx, %r13d + vpxor %xmm6, %xmm7, %xmm6 + vpaddd %xmm5, %xmm4, %xmm4 + # rnd_0: 4 - 4 + rorl $6, %edx + xorl %r14d, %eax + addl %edx, %r13d + movl %r14d, %ecx + vpxor %xmm6, %xmm8, %xmm8 + # rnd_0: 5 - 5 + andl %eax, %ebx + rorl $9, %ecx + xorl %r14d, %ecx + xorl %r15d, %ebx + vpshufb %xmm11, %xmm8, %xmm8 + # rnd_0: 6 - 6 + rorl $11, %ecx + addl %r13d, %r9d + xorl %r14d, %ecx + addl %ebx, %r13d + vpaddd %xmm8, %xmm4, %xmm4 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r9d, %edx + addl %ecx, %r13d + # rnd_1: 0 - 0 + rorl $14, %edx + vpshufd $0x50, %xmm4, %xmm6 + # rnd_1: 1 - 1 + movl %r14d, %ebx + movl %r10d, %ecx + addl 12(%rsp), %r12d + xorl %r11d, %ecx + vpsrlq $0x11, %xmm6, %xmm8 + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 2 - 3 + xorl %r9d, %edx + andl %r9d, %ecx + rorl $5, %edx + xorl %r11d, %ecx + xorl %r9d, %edx + addl %ecx, %r12d + vpsrld $10, %xmm6, %xmm9 + vpxor %xmm8, %xmm7, %xmm8 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r13d, %ebx + addl %edx, %r12d + movl %r13d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r13d, %ecx + xorl %r14d, %eax + vpxor %xmm9, %xmm8, %xmm9 + # rnd_1: 6 - 6 + rorl $11, %ecx + addl %r12d, %r8d + xorl %r13d, %ecx + addl %eax, %r12d + vpshufb %xmm12, %xmm9, %xmm9 + # rnd_1: 7 - 7 + rorl $2, %ecx + movl %r8d, %edx + addl %ecx, %r12d + vpaddd %xmm4, %xmm9, %xmm0 + # msg_sched done: 0-3 + # msg_sched: 4-7 + # rnd_0: 0 - 0 + rorl $14, %edx + vpalignr $4, %xmm1, %xmm2, %xmm5 + vpalignr $4, %xmm3, %xmm0, %xmm4 + # rnd_0: 1 - 2 + movl %r13d, %eax + movl %r9d, %ecx + addl 16(%rsp), %r11d + xorl %r10d, %ecx + xorl %r8d, %edx + andl %r8d, %ecx + vpsrld $7, %xmm5, %xmm6 + vpslld $25, %xmm5, %xmm7 + # rnd_0: 3 - 4 + rorl $5, %edx + xorl %r10d, %ecx + xorl %r8d, %edx + addl %ecx, %r11d + rorl $6, %edx + xorl %r12d, %eax + addl %edx, %r11d + movl %r12d, %ecx + vpsrld $18, %xmm5, %xmm8 + vpslld $14, %xmm5, %xmm9 + # rnd_0: 5 - 6 + andl %eax, %ebx + rorl $9, %ecx + xorl %r12d, %ecx + xorl %r13d, %ebx + rorl $11, %ecx + addl %r11d, %r15d + xorl %r12d, %ecx + addl %ebx, %r11d + vpor %xmm6, %xmm7, %xmm6 + vpor %xmm8, %xmm9, %xmm8 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r15d, %edx + addl %ecx, %r11d + # rnd_1: 0 - 1 + rorl $14, %edx + movl %r12d, %ebx + movl %r8d, %ecx + addl 20(%rsp), %r10d + xorl %r9d, %ecx + vpsrld $3, %xmm5, %xmm9 + vpxor %xmm6, %xmm8, %xmm6 + # rnd_1: 2 - 3 + xorl %r15d, %edx + andl %r15d, %ecx + rorl $5, %edx + xorl %r9d, %ecx + xorl %r15d, %edx + addl %ecx, %r10d + vpxor %xmm6, %xmm9, %xmm5 + vpshufd $0xfa, %xmm0, %xmm6 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r11d, %ebx + addl %edx, %r10d + movl %r11d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r11d, %ecx + xorl %r12d, %eax + vpsrld $10, %xmm6, %xmm8 + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 6 - 7 + rorl $11, %ecx + addl %r10d, %r14d + xorl %r11d, %ecx + addl %eax, %r10d + rorl $2, %ecx + movl %r14d, %edx + addl %ecx, %r10d + # rnd_0: 0 - 0 + rorl $14, %edx + vpsrlq $0x11, %xmm6, %xmm6 + vpaddd %xmm1, %xmm4, %xmm4 + # rnd_0: 1 - 3 + movl %r11d, %eax + movl %r15d, %ecx + addl 24(%rsp), %r9d + xorl %r8d, %ecx + xorl %r14d, %edx + andl %r14d, %ecx + rorl $5, %edx + xorl %r8d, %ecx + xorl %r14d, %edx + addl %ecx, %r9d + vpxor %xmm6, %xmm7, %xmm6 + vpaddd %xmm5, %xmm4, %xmm4 + # rnd_0: 4 - 4 + rorl $6, %edx + xorl %r10d, %eax + addl %edx, %r9d + movl %r10d, %ecx + vpxor %xmm6, %xmm8, %xmm8 + # rnd_0: 5 - 5 + andl %eax, %ebx + rorl $9, %ecx + xorl %r10d, %ecx + xorl %r11d, %ebx + vpshufb %xmm11, %xmm8, %xmm8 + # rnd_0: 6 - 6 + rorl $11, %ecx + addl %r9d, %r13d + xorl %r10d, %ecx + addl %ebx, %r9d + vpaddd %xmm8, %xmm4, %xmm4 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r13d, %edx + addl %ecx, %r9d + # rnd_1: 0 - 0 + rorl $14, %edx + vpshufd $0x50, %xmm4, %xmm6 + # rnd_1: 1 - 1 + movl %r10d, %ebx + movl %r14d, %ecx + addl 28(%rsp), %r8d + xorl %r15d, %ecx + vpsrlq $0x11, %xmm6, %xmm8 + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 2 - 3 + xorl %r13d, %edx + andl %r13d, %ecx + rorl $5, %edx + xorl %r15d, %ecx + xorl %r13d, %edx + addl %ecx, %r8d + vpsrld $10, %xmm6, %xmm9 + vpxor %xmm8, %xmm7, %xmm8 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r9d, %ebx + addl %edx, %r8d + movl %r9d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r9d, %ecx + xorl %r10d, %eax + vpxor %xmm9, %xmm8, %xmm9 + # rnd_1: 6 - 6 + rorl $11, %ecx + addl %r8d, %r12d + xorl %r9d, %ecx + addl %eax, %r8d + vpshufb %xmm12, %xmm9, %xmm9 + # rnd_1: 7 - 7 + rorl $2, %ecx + movl %r12d, %edx + addl %ecx, %r8d + vpaddd %xmm4, %xmm9, %xmm1 + # msg_sched done: 4-7 + # msg_sched: 8-11 + # rnd_0: 0 - 0 + rorl $14, %edx + vpalignr $4, %xmm2, %xmm3, %xmm5 + vpalignr $4, %xmm0, %xmm1, %xmm4 + # rnd_0: 1 - 2 + movl %r9d, %eax + movl %r13d, %ecx + addl 32(%rsp), %r15d + xorl %r14d, %ecx + xorl %r12d, %edx + andl %r12d, %ecx + vpsrld $7, %xmm5, %xmm6 + vpslld $25, %xmm5, %xmm7 + # rnd_0: 3 - 4 + rorl $5, %edx + xorl %r14d, %ecx + xorl %r12d, %edx + addl %ecx, %r15d + rorl $6, %edx + xorl %r8d, %eax + addl %edx, %r15d + movl %r8d, %ecx + vpsrld $18, %xmm5, %xmm8 + vpslld $14, %xmm5, %xmm9 + # rnd_0: 5 - 6 + andl %eax, %ebx + rorl $9, %ecx + xorl %r8d, %ecx + xorl %r9d, %ebx + rorl $11, %ecx + addl %r15d, %r11d + xorl %r8d, %ecx + addl %ebx, %r15d + vpor %xmm6, %xmm7, %xmm6 + vpor %xmm8, %xmm9, %xmm8 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r11d, %edx + addl %ecx, %r15d + # rnd_1: 0 - 1 + rorl $14, %edx + movl %r8d, %ebx + movl %r12d, %ecx + addl 36(%rsp), %r14d + xorl %r13d, %ecx + vpsrld $3, %xmm5, %xmm9 + vpxor %xmm6, %xmm8, %xmm6 + # rnd_1: 2 - 3 + xorl %r11d, %edx + andl %r11d, %ecx + rorl $5, %edx + xorl %r13d, %ecx + xorl %r11d, %edx + addl %ecx, %r14d + vpxor %xmm6, %xmm9, %xmm5 + vpshufd $0xfa, %xmm1, %xmm6 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r15d, %ebx + addl %edx, %r14d + movl %r15d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r15d, %ecx + xorl %r8d, %eax + vpsrld $10, %xmm6, %xmm8 + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 6 - 7 + rorl $11, %ecx + addl %r14d, %r10d + xorl %r15d, %ecx + addl %eax, %r14d + rorl $2, %ecx + movl %r10d, %edx + addl %ecx, %r14d + # rnd_0: 0 - 0 + rorl $14, %edx + vpsrlq $0x11, %xmm6, %xmm6 + vpaddd %xmm2, %xmm4, %xmm4 + # rnd_0: 1 - 3 + movl %r15d, %eax + movl %r11d, %ecx + addl 40(%rsp), %r13d + xorl %r12d, %ecx + xorl %r10d, %edx + andl %r10d, %ecx + rorl $5, %edx + xorl %r12d, %ecx + xorl %r10d, %edx + addl %ecx, %r13d + vpxor %xmm6, %xmm7, %xmm6 + vpaddd %xmm5, %xmm4, %xmm4 + # rnd_0: 4 - 4 + rorl $6, %edx + xorl %r14d, %eax + addl %edx, %r13d + movl %r14d, %ecx + vpxor %xmm6, %xmm8, %xmm8 + # rnd_0: 5 - 5 + andl %eax, %ebx + rorl $9, %ecx + xorl %r14d, %ecx + xorl %r15d, %ebx + vpshufb %xmm11, %xmm8, %xmm8 + # rnd_0: 6 - 6 + rorl $11, %ecx + addl %r13d, %r9d + xorl %r14d, %ecx + addl %ebx, %r13d + vpaddd %xmm8, %xmm4, %xmm4 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r9d, %edx + addl %ecx, %r13d + # rnd_1: 0 - 0 + rorl $14, %edx + vpshufd $0x50, %xmm4, %xmm6 + # rnd_1: 1 - 1 + movl %r14d, %ebx + movl %r10d, %ecx + addl 44(%rsp), %r12d + xorl %r11d, %ecx + vpsrlq $0x11, %xmm6, %xmm8 + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 2 - 3 + xorl %r9d, %edx + andl %r9d, %ecx + rorl $5, %edx + xorl %r11d, %ecx + xorl %r9d, %edx + addl %ecx, %r12d + vpsrld $10, %xmm6, %xmm9 + vpxor %xmm8, %xmm7, %xmm8 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r13d, %ebx + addl %edx, %r12d + movl %r13d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r13d, %ecx + xorl %r14d, %eax + vpxor %xmm9, %xmm8, %xmm9 + # rnd_1: 6 - 6 + rorl $11, %ecx + addl %r12d, %r8d + xorl %r13d, %ecx + addl %eax, %r12d + vpshufb %xmm12, %xmm9, %xmm9 + # rnd_1: 7 - 7 + rorl $2, %ecx + movl %r8d, %edx + addl %ecx, %r12d + vpaddd %xmm4, %xmm9, %xmm2 + # msg_sched done: 8-11 + # msg_sched: 12-15 + # rnd_0: 0 - 0 + rorl $14, %edx + vpalignr $4, %xmm3, %xmm0, %xmm5 + vpalignr $4, %xmm1, %xmm2, %xmm4 + # rnd_0: 1 - 2 + movl %r13d, %eax + movl %r9d, %ecx + addl 48(%rsp), %r11d + xorl %r10d, %ecx + xorl %r8d, %edx + andl %r8d, %ecx + vpsrld $7, %xmm5, %xmm6 + vpslld $25, %xmm5, %xmm7 + # rnd_0: 3 - 4 + rorl $5, %edx + xorl %r10d, %ecx + xorl %r8d, %edx + addl %ecx, %r11d + rorl $6, %edx + xorl %r12d, %eax + addl %edx, %r11d + movl %r12d, %ecx + vpsrld $18, %xmm5, %xmm8 + vpslld $14, %xmm5, %xmm9 + # rnd_0: 5 - 6 + andl %eax, %ebx + rorl $9, %ecx + xorl %r12d, %ecx + xorl %r13d, %ebx + rorl $11, %ecx + addl %r11d, %r15d + xorl %r12d, %ecx + addl %ebx, %r11d + vpor %xmm6, %xmm7, %xmm6 + vpor %xmm8, %xmm9, %xmm8 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r15d, %edx + addl %ecx, %r11d + # rnd_1: 0 - 1 + rorl $14, %edx + movl %r12d, %ebx + movl %r8d, %ecx + addl 52(%rsp), %r10d + xorl %r9d, %ecx + vpsrld $3, %xmm5, %xmm9 + vpxor %xmm6, %xmm8, %xmm6 + # rnd_1: 2 - 3 + xorl %r15d, %edx + andl %r15d, %ecx + rorl $5, %edx + xorl %r9d, %ecx + xorl %r15d, %edx + addl %ecx, %r10d + vpxor %xmm6, %xmm9, %xmm5 + vpshufd $0xfa, %xmm2, %xmm6 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r11d, %ebx + addl %edx, %r10d + movl %r11d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r11d, %ecx + xorl %r12d, %eax + vpsrld $10, %xmm6, %xmm8 + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 6 - 7 + rorl $11, %ecx + addl %r10d, %r14d + xorl %r11d, %ecx + addl %eax, %r10d + rorl $2, %ecx + movl %r14d, %edx + addl %ecx, %r10d + # rnd_0: 0 - 0 + rorl $14, %edx + vpsrlq $0x11, %xmm6, %xmm6 + vpaddd %xmm3, %xmm4, %xmm4 + # rnd_0: 1 - 3 + movl %r11d, %eax + movl %r15d, %ecx + addl 56(%rsp), %r9d + xorl %r8d, %ecx + xorl %r14d, %edx + andl %r14d, %ecx + rorl $5, %edx + xorl %r8d, %ecx + xorl %r14d, %edx + addl %ecx, %r9d + vpxor %xmm6, %xmm7, %xmm6 + vpaddd %xmm5, %xmm4, %xmm4 + # rnd_0: 4 - 4 + rorl $6, %edx + xorl %r10d, %eax + addl %edx, %r9d + movl %r10d, %ecx + vpxor %xmm6, %xmm8, %xmm8 + # rnd_0: 5 - 5 + andl %eax, %ebx + rorl $9, %ecx + xorl %r10d, %ecx + xorl %r11d, %ebx + vpshufb %xmm11, %xmm8, %xmm8 + # rnd_0: 6 - 6 + rorl $11, %ecx + addl %r9d, %r13d + xorl %r10d, %ecx + addl %ebx, %r9d + vpaddd %xmm8, %xmm4, %xmm4 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r13d, %edx + addl %ecx, %r9d + # rnd_1: 0 - 0 + rorl $14, %edx + vpshufd $0x50, %xmm4, %xmm6 + # rnd_1: 1 - 1 + movl %r10d, %ebx + movl %r14d, %ecx + addl 60(%rsp), %r8d + xorl %r15d, %ecx + vpsrlq $0x11, %xmm6, %xmm8 + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 2 - 3 + xorl %r13d, %edx + andl %r13d, %ecx + rorl $5, %edx + xorl %r15d, %ecx + xorl %r13d, %edx + addl %ecx, %r8d + vpsrld $10, %xmm6, %xmm9 + vpxor %xmm8, %xmm7, %xmm8 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r9d, %ebx + addl %edx, %r8d + movl %r9d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r9d, %ecx + xorl %r10d, %eax + vpxor %xmm9, %xmm8, %xmm9 + # rnd_1: 6 - 6 + rorl $11, %ecx + addl %r8d, %r12d + xorl %r9d, %ecx + addl %eax, %r8d + vpshufb %xmm12, %xmm9, %xmm9 + # rnd_1: 7 - 7 + rorl $2, %ecx + movl %r12d, %edx + addl %ecx, %r8d + vpaddd %xmm4, %xmm9, %xmm3 + # msg_sched done: 12-15 + # set_w_k_xfer_4: 8 + vpaddd 128+L_avx1_sha256_k(%rip), %xmm0, %xmm4 + vpaddd 144+L_avx1_sha256_k(%rip), %xmm1, %xmm5 + vmovdqu %xmm4, (%rsp) + vmovdqu %xmm5, 16(%rsp) + vpaddd 160+L_avx1_sha256_k(%rip), %xmm2, %xmm6 + vpaddd 176+L_avx1_sha256_k(%rip), %xmm3, %xmm7 + vmovdqu %xmm6, 32(%rsp) + vmovdqu %xmm7, 48(%rsp) + # msg_sched: 0-3 + # rnd_0: 0 - 0 + rorl $14, %edx + vpalignr $4, %xmm0, %xmm1, %xmm5 + vpalignr $4, %xmm2, %xmm3, %xmm4 + # rnd_0: 1 - 2 + movl %r9d, %eax + movl %r13d, %ecx + addl (%rsp), %r15d + xorl %r14d, %ecx + xorl %r12d, %edx + andl %r12d, %ecx + vpsrld $7, %xmm5, %xmm6 + vpslld $25, %xmm5, %xmm7 + # rnd_0: 3 - 4 + rorl $5, %edx + xorl %r14d, %ecx + xorl %r12d, %edx + addl %ecx, %r15d + rorl $6, %edx + xorl %r8d, %eax + addl %edx, %r15d + movl %r8d, %ecx + vpsrld $18, %xmm5, %xmm8 + vpslld $14, %xmm5, %xmm9 + # rnd_0: 5 - 6 + andl %eax, %ebx + rorl $9, %ecx + xorl %r8d, %ecx + xorl %r9d, %ebx + rorl $11, %ecx + addl %r15d, %r11d + xorl %r8d, %ecx + addl %ebx, %r15d + vpor %xmm6, %xmm7, %xmm6 + vpor %xmm8, %xmm9, %xmm8 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r11d, %edx + addl %ecx, %r15d + # rnd_1: 0 - 1 + rorl $14, %edx + movl %r8d, %ebx + movl %r12d, %ecx + addl 4(%rsp), %r14d + xorl %r13d, %ecx + vpsrld $3, %xmm5, %xmm9 + vpxor %xmm6, %xmm8, %xmm6 + # rnd_1: 2 - 3 + xorl %r11d, %edx + andl %r11d, %ecx + rorl $5, %edx + xorl %r13d, %ecx + xorl %r11d, %edx + addl %ecx, %r14d + vpxor %xmm6, %xmm9, %xmm5 + vpshufd $0xfa, %xmm3, %xmm6 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r15d, %ebx + addl %edx, %r14d + movl %r15d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r15d, %ecx + xorl %r8d, %eax + vpsrld $10, %xmm6, %xmm8 + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 6 - 7 + rorl $11, %ecx + addl %r14d, %r10d + xorl %r15d, %ecx + addl %eax, %r14d + rorl $2, %ecx + movl %r10d, %edx + addl %ecx, %r14d + # rnd_0: 0 - 0 + rorl $14, %edx + vpsrlq $0x11, %xmm6, %xmm6 + vpaddd %xmm0, %xmm4, %xmm4 + # rnd_0: 1 - 3 + movl %r15d, %eax + movl %r11d, %ecx + addl 8(%rsp), %r13d + xorl %r12d, %ecx + xorl %r10d, %edx + andl %r10d, %ecx + rorl $5, %edx + xorl %r12d, %ecx + xorl %r10d, %edx + addl %ecx, %r13d + vpxor %xmm6, %xmm7, %xmm6 + vpaddd %xmm5, %xmm4, %xmm4 + # rnd_0: 4 - 4 + rorl $6, %edx + xorl %r14d, %eax + addl %edx, %r13d + movl %r14d, %ecx + vpxor %xmm6, %xmm8, %xmm8 + # rnd_0: 5 - 5 + andl %eax, %ebx + rorl $9, %ecx + xorl %r14d, %ecx + xorl %r15d, %ebx + vpshufb %xmm11, %xmm8, %xmm8 + # rnd_0: 6 - 6 + rorl $11, %ecx + addl %r13d, %r9d + xorl %r14d, %ecx + addl %ebx, %r13d + vpaddd %xmm8, %xmm4, %xmm4 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r9d, %edx + addl %ecx, %r13d + # rnd_1: 0 - 0 + rorl $14, %edx + vpshufd $0x50, %xmm4, %xmm6 + # rnd_1: 1 - 1 + movl %r14d, %ebx + movl %r10d, %ecx + addl 12(%rsp), %r12d + xorl %r11d, %ecx + vpsrlq $0x11, %xmm6, %xmm8 + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 2 - 3 + xorl %r9d, %edx + andl %r9d, %ecx + rorl $5, %edx + xorl %r11d, %ecx + xorl %r9d, %edx + addl %ecx, %r12d + vpsrld $10, %xmm6, %xmm9 + vpxor %xmm8, %xmm7, %xmm8 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r13d, %ebx + addl %edx, %r12d + movl %r13d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r13d, %ecx + xorl %r14d, %eax + vpxor %xmm9, %xmm8, %xmm9 + # rnd_1: 6 - 6 + rorl $11, %ecx + addl %r12d, %r8d + xorl %r13d, %ecx + addl %eax, %r12d + vpshufb %xmm12, %xmm9, %xmm9 + # rnd_1: 7 - 7 + rorl $2, %ecx + movl %r8d, %edx + addl %ecx, %r12d + vpaddd %xmm4, %xmm9, %xmm0 + # msg_sched done: 0-3 + # msg_sched: 4-7 + # rnd_0: 0 - 0 + rorl $14, %edx + vpalignr $4, %xmm1, %xmm2, %xmm5 + vpalignr $4, %xmm3, %xmm0, %xmm4 + # rnd_0: 1 - 2 + movl %r13d, %eax + movl %r9d, %ecx + addl 16(%rsp), %r11d + xorl %r10d, %ecx + xorl %r8d, %edx + andl %r8d, %ecx + vpsrld $7, %xmm5, %xmm6 + vpslld $25, %xmm5, %xmm7 + # rnd_0: 3 - 4 + rorl $5, %edx + xorl %r10d, %ecx + xorl %r8d, %edx + addl %ecx, %r11d + rorl $6, %edx + xorl %r12d, %eax + addl %edx, %r11d + movl %r12d, %ecx + vpsrld $18, %xmm5, %xmm8 + vpslld $14, %xmm5, %xmm9 + # rnd_0: 5 - 6 + andl %eax, %ebx + rorl $9, %ecx + xorl %r12d, %ecx + xorl %r13d, %ebx + rorl $11, %ecx + addl %r11d, %r15d + xorl %r12d, %ecx + addl %ebx, %r11d + vpor %xmm6, %xmm7, %xmm6 + vpor %xmm8, %xmm9, %xmm8 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r15d, %edx + addl %ecx, %r11d + # rnd_1: 0 - 1 + rorl $14, %edx + movl %r12d, %ebx + movl %r8d, %ecx + addl 20(%rsp), %r10d + xorl %r9d, %ecx + vpsrld $3, %xmm5, %xmm9 + vpxor %xmm6, %xmm8, %xmm6 + # rnd_1: 2 - 3 + xorl %r15d, %edx + andl %r15d, %ecx + rorl $5, %edx + xorl %r9d, %ecx + xorl %r15d, %edx + addl %ecx, %r10d + vpxor %xmm6, %xmm9, %xmm5 + vpshufd $0xfa, %xmm0, %xmm6 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r11d, %ebx + addl %edx, %r10d + movl %r11d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r11d, %ecx + xorl %r12d, %eax + vpsrld $10, %xmm6, %xmm8 + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 6 - 7 + rorl $11, %ecx + addl %r10d, %r14d + xorl %r11d, %ecx + addl %eax, %r10d + rorl $2, %ecx + movl %r14d, %edx + addl %ecx, %r10d + # rnd_0: 0 - 0 + rorl $14, %edx + vpsrlq $0x11, %xmm6, %xmm6 + vpaddd %xmm1, %xmm4, %xmm4 + # rnd_0: 1 - 3 + movl %r11d, %eax + movl %r15d, %ecx + addl 24(%rsp), %r9d + xorl %r8d, %ecx + xorl %r14d, %edx + andl %r14d, %ecx + rorl $5, %edx + xorl %r8d, %ecx + xorl %r14d, %edx + addl %ecx, %r9d + vpxor %xmm6, %xmm7, %xmm6 + vpaddd %xmm5, %xmm4, %xmm4 + # rnd_0: 4 - 4 + rorl $6, %edx + xorl %r10d, %eax + addl %edx, %r9d + movl %r10d, %ecx + vpxor %xmm6, %xmm8, %xmm8 + # rnd_0: 5 - 5 + andl %eax, %ebx + rorl $9, %ecx + xorl %r10d, %ecx + xorl %r11d, %ebx + vpshufb %xmm11, %xmm8, %xmm8 + # rnd_0: 6 - 6 + rorl $11, %ecx + addl %r9d, %r13d + xorl %r10d, %ecx + addl %ebx, %r9d + vpaddd %xmm8, %xmm4, %xmm4 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r13d, %edx + addl %ecx, %r9d + # rnd_1: 0 - 0 + rorl $14, %edx + vpshufd $0x50, %xmm4, %xmm6 + # rnd_1: 1 - 1 + movl %r10d, %ebx + movl %r14d, %ecx + addl 28(%rsp), %r8d + xorl %r15d, %ecx + vpsrlq $0x11, %xmm6, %xmm8 + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 2 - 3 + xorl %r13d, %edx + andl %r13d, %ecx + rorl $5, %edx + xorl %r15d, %ecx + xorl %r13d, %edx + addl %ecx, %r8d + vpsrld $10, %xmm6, %xmm9 + vpxor %xmm8, %xmm7, %xmm8 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r9d, %ebx + addl %edx, %r8d + movl %r9d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r9d, %ecx + xorl %r10d, %eax + vpxor %xmm9, %xmm8, %xmm9 + # rnd_1: 6 - 6 + rorl $11, %ecx + addl %r8d, %r12d + xorl %r9d, %ecx + addl %eax, %r8d + vpshufb %xmm12, %xmm9, %xmm9 + # rnd_1: 7 - 7 + rorl $2, %ecx + movl %r12d, %edx + addl %ecx, %r8d + vpaddd %xmm4, %xmm9, %xmm1 + # msg_sched done: 4-7 + # msg_sched: 8-11 + # rnd_0: 0 - 0 + rorl $14, %edx + vpalignr $4, %xmm2, %xmm3, %xmm5 + vpalignr $4, %xmm0, %xmm1, %xmm4 + # rnd_0: 1 - 2 + movl %r9d, %eax + movl %r13d, %ecx + addl 32(%rsp), %r15d + xorl %r14d, %ecx + xorl %r12d, %edx + andl %r12d, %ecx + vpsrld $7, %xmm5, %xmm6 + vpslld $25, %xmm5, %xmm7 + # rnd_0: 3 - 4 + rorl $5, %edx + xorl %r14d, %ecx + xorl %r12d, %edx + addl %ecx, %r15d + rorl $6, %edx + xorl %r8d, %eax + addl %edx, %r15d + movl %r8d, %ecx + vpsrld $18, %xmm5, %xmm8 + vpslld $14, %xmm5, %xmm9 + # rnd_0: 5 - 6 + andl %eax, %ebx + rorl $9, %ecx + xorl %r8d, %ecx + xorl %r9d, %ebx + rorl $11, %ecx + addl %r15d, %r11d + xorl %r8d, %ecx + addl %ebx, %r15d + vpor %xmm6, %xmm7, %xmm6 + vpor %xmm8, %xmm9, %xmm8 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r11d, %edx + addl %ecx, %r15d + # rnd_1: 0 - 1 + rorl $14, %edx + movl %r8d, %ebx + movl %r12d, %ecx + addl 36(%rsp), %r14d + xorl %r13d, %ecx + vpsrld $3, %xmm5, %xmm9 + vpxor %xmm6, %xmm8, %xmm6 + # rnd_1: 2 - 3 + xorl %r11d, %edx + andl %r11d, %ecx + rorl $5, %edx + xorl %r13d, %ecx + xorl %r11d, %edx + addl %ecx, %r14d + vpxor %xmm6, %xmm9, %xmm5 + vpshufd $0xfa, %xmm1, %xmm6 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r15d, %ebx + addl %edx, %r14d + movl %r15d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r15d, %ecx + xorl %r8d, %eax + vpsrld $10, %xmm6, %xmm8 + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 6 - 7 + rorl $11, %ecx + addl %r14d, %r10d + xorl %r15d, %ecx + addl %eax, %r14d + rorl $2, %ecx + movl %r10d, %edx + addl %ecx, %r14d + # rnd_0: 0 - 0 + rorl $14, %edx + vpsrlq $0x11, %xmm6, %xmm6 + vpaddd %xmm2, %xmm4, %xmm4 + # rnd_0: 1 - 3 + movl %r15d, %eax + movl %r11d, %ecx + addl 40(%rsp), %r13d + xorl %r12d, %ecx + xorl %r10d, %edx + andl %r10d, %ecx + rorl $5, %edx + xorl %r12d, %ecx + xorl %r10d, %edx + addl %ecx, %r13d + vpxor %xmm6, %xmm7, %xmm6 + vpaddd %xmm5, %xmm4, %xmm4 + # rnd_0: 4 - 4 + rorl $6, %edx + xorl %r14d, %eax + addl %edx, %r13d + movl %r14d, %ecx + vpxor %xmm6, %xmm8, %xmm8 + # rnd_0: 5 - 5 + andl %eax, %ebx + rorl $9, %ecx + xorl %r14d, %ecx + xorl %r15d, %ebx + vpshufb %xmm11, %xmm8, %xmm8 + # rnd_0: 6 - 6 + rorl $11, %ecx + addl %r13d, %r9d + xorl %r14d, %ecx + addl %ebx, %r13d + vpaddd %xmm8, %xmm4, %xmm4 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r9d, %edx + addl %ecx, %r13d + # rnd_1: 0 - 0 + rorl $14, %edx + vpshufd $0x50, %xmm4, %xmm6 + # rnd_1: 1 - 1 + movl %r14d, %ebx + movl %r10d, %ecx + addl 44(%rsp), %r12d + xorl %r11d, %ecx + vpsrlq $0x11, %xmm6, %xmm8 + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 2 - 3 + xorl %r9d, %edx + andl %r9d, %ecx + rorl $5, %edx + xorl %r11d, %ecx + xorl %r9d, %edx + addl %ecx, %r12d + vpsrld $10, %xmm6, %xmm9 + vpxor %xmm8, %xmm7, %xmm8 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r13d, %ebx + addl %edx, %r12d + movl %r13d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r13d, %ecx + xorl %r14d, %eax + vpxor %xmm9, %xmm8, %xmm9 + # rnd_1: 6 - 6 + rorl $11, %ecx + addl %r12d, %r8d + xorl %r13d, %ecx + addl %eax, %r12d + vpshufb %xmm12, %xmm9, %xmm9 + # rnd_1: 7 - 7 + rorl $2, %ecx + movl %r8d, %edx + addl %ecx, %r12d + vpaddd %xmm4, %xmm9, %xmm2 + # msg_sched done: 8-11 + # msg_sched: 12-15 + # rnd_0: 0 - 0 + rorl $14, %edx + vpalignr $4, %xmm3, %xmm0, %xmm5 + vpalignr $4, %xmm1, %xmm2, %xmm4 + # rnd_0: 1 - 2 + movl %r13d, %eax + movl %r9d, %ecx + addl 48(%rsp), %r11d + xorl %r10d, %ecx + xorl %r8d, %edx + andl %r8d, %ecx + vpsrld $7, %xmm5, %xmm6 + vpslld $25, %xmm5, %xmm7 + # rnd_0: 3 - 4 + rorl $5, %edx + xorl %r10d, %ecx + xorl %r8d, %edx + addl %ecx, %r11d + rorl $6, %edx + xorl %r12d, %eax + addl %edx, %r11d + movl %r12d, %ecx + vpsrld $18, %xmm5, %xmm8 + vpslld $14, %xmm5, %xmm9 + # rnd_0: 5 - 6 + andl %eax, %ebx + rorl $9, %ecx + xorl %r12d, %ecx + xorl %r13d, %ebx + rorl $11, %ecx + addl %r11d, %r15d + xorl %r12d, %ecx + addl %ebx, %r11d + vpor %xmm6, %xmm7, %xmm6 + vpor %xmm8, %xmm9, %xmm8 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r15d, %edx + addl %ecx, %r11d + # rnd_1: 0 - 1 + rorl $14, %edx + movl %r12d, %ebx + movl %r8d, %ecx + addl 52(%rsp), %r10d + xorl %r9d, %ecx + vpsrld $3, %xmm5, %xmm9 + vpxor %xmm6, %xmm8, %xmm6 + # rnd_1: 2 - 3 + xorl %r15d, %edx + andl %r15d, %ecx + rorl $5, %edx + xorl %r9d, %ecx + xorl %r15d, %edx + addl %ecx, %r10d + vpxor %xmm6, %xmm9, %xmm5 + vpshufd $0xfa, %xmm2, %xmm6 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r11d, %ebx + addl %edx, %r10d + movl %r11d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r11d, %ecx + xorl %r12d, %eax + vpsrld $10, %xmm6, %xmm8 + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 6 - 7 + rorl $11, %ecx + addl %r10d, %r14d + xorl %r11d, %ecx + addl %eax, %r10d + rorl $2, %ecx + movl %r14d, %edx + addl %ecx, %r10d + # rnd_0: 0 - 0 + rorl $14, %edx + vpsrlq $0x11, %xmm6, %xmm6 + vpaddd %xmm3, %xmm4, %xmm4 + # rnd_0: 1 - 3 + movl %r11d, %eax + movl %r15d, %ecx + addl 56(%rsp), %r9d + xorl %r8d, %ecx + xorl %r14d, %edx + andl %r14d, %ecx + rorl $5, %edx + xorl %r8d, %ecx + xorl %r14d, %edx + addl %ecx, %r9d + vpxor %xmm6, %xmm7, %xmm6 + vpaddd %xmm5, %xmm4, %xmm4 + # rnd_0: 4 - 4 + rorl $6, %edx + xorl %r10d, %eax + addl %edx, %r9d + movl %r10d, %ecx + vpxor %xmm6, %xmm8, %xmm8 + # rnd_0: 5 - 5 + andl %eax, %ebx + rorl $9, %ecx + xorl %r10d, %ecx + xorl %r11d, %ebx + vpshufb %xmm11, %xmm8, %xmm8 + # rnd_0: 6 - 6 + rorl $11, %ecx + addl %r9d, %r13d + xorl %r10d, %ecx + addl %ebx, %r9d + vpaddd %xmm8, %xmm4, %xmm4 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r13d, %edx + addl %ecx, %r9d + # rnd_1: 0 - 0 + rorl $14, %edx + vpshufd $0x50, %xmm4, %xmm6 + # rnd_1: 1 - 1 + movl %r10d, %ebx + movl %r14d, %ecx + addl 60(%rsp), %r8d + xorl %r15d, %ecx + vpsrlq $0x11, %xmm6, %xmm8 + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 2 - 3 + xorl %r13d, %edx + andl %r13d, %ecx + rorl $5, %edx + xorl %r15d, %ecx + xorl %r13d, %edx + addl %ecx, %r8d + vpsrld $10, %xmm6, %xmm9 + vpxor %xmm8, %xmm7, %xmm8 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r9d, %ebx + addl %edx, %r8d + movl %r9d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r9d, %ecx + xorl %r10d, %eax + vpxor %xmm9, %xmm8, %xmm9 + # rnd_1: 6 - 6 + rorl $11, %ecx + addl %r8d, %r12d + xorl %r9d, %ecx + addl %eax, %r8d + vpshufb %xmm12, %xmm9, %xmm9 + # rnd_1: 7 - 7 + rorl $2, %ecx + movl %r12d, %edx + addl %ecx, %r8d + vpaddd %xmm4, %xmm9, %xmm3 + # msg_sched done: 12-15 + # set_w_k_xfer_4: 12 + vpaddd 192+L_avx1_sha256_k(%rip), %xmm0, %xmm4 + vpaddd 208+L_avx1_sha256_k(%rip), %xmm1, %xmm5 + vmovdqu %xmm4, (%rsp) + vmovdqu %xmm5, 16(%rsp) + vpaddd 224+L_avx1_sha256_k(%rip), %xmm2, %xmm6 + vpaddd 240+L_avx1_sha256_k(%rip), %xmm3, %xmm7 + vmovdqu %xmm6, 32(%rsp) + vmovdqu %xmm7, 48(%rsp) + # rnd_all_4: 0-3 + addl (%rsp), %r15d + movl %r13d, %ecx + movl %r9d, %eax + xorl %r14d, %ecx + rorl $14, %edx + andl %r12d, %ecx + xorl %r12d, %edx + xorl %r14d, %ecx + rorl $5, %edx + addl %ecx, %r15d + xorl %r12d, %edx + xorl %r8d, %eax + rorl $6, %edx + movl %r8d, %ecx + addl %edx, %r15d + rorl $9, %ecx + andl %eax, %ebx + xorl %r8d, %ecx + xorl %r9d, %ebx + rorl $11, %ecx + addl %r15d, %r11d + xorl %r8d, %ecx + addl %ebx, %r15d + rorl $2, %ecx + movl %r11d, %edx + addl %ecx, %r15d + addl 4(%rsp), %r14d + movl %r12d, %ecx + movl %r8d, %ebx + xorl %r13d, %ecx + rorl $14, %edx + andl %r11d, %ecx + xorl %r11d, %edx + xorl %r13d, %ecx + rorl $5, %edx + addl %ecx, %r14d + xorl %r11d, %edx + xorl %r15d, %ebx + rorl $6, %edx + movl %r15d, %ecx + addl %edx, %r14d + rorl $9, %ecx + andl %ebx, %eax + xorl %r15d, %ecx + xorl %r8d, %eax + rorl $11, %ecx + addl %r14d, %r10d + xorl %r15d, %ecx + addl %eax, %r14d + rorl $2, %ecx + movl %r10d, %edx + addl %ecx, %r14d + addl 8(%rsp), %r13d + movl %r11d, %ecx + movl %r15d, %eax + xorl %r12d, %ecx + rorl $14, %edx + andl %r10d, %ecx + xorl %r10d, %edx + xorl %r12d, %ecx + rorl $5, %edx + addl %ecx, %r13d + xorl %r10d, %edx + xorl %r14d, %eax + rorl $6, %edx + movl %r14d, %ecx + addl %edx, %r13d + rorl $9, %ecx + andl %eax, %ebx + xorl %r14d, %ecx + xorl %r15d, %ebx + rorl $11, %ecx + addl %r13d, %r9d + xorl %r14d, %ecx + addl %ebx, %r13d + rorl $2, %ecx + movl %r9d, %edx + addl %ecx, %r13d + addl 12(%rsp), %r12d + movl %r10d, %ecx + movl %r14d, %ebx + xorl %r11d, %ecx + rorl $14, %edx + andl %r9d, %ecx + xorl %r9d, %edx + xorl %r11d, %ecx + rorl $5, %edx + addl %ecx, %r12d + xorl %r9d, %edx + xorl %r13d, %ebx + rorl $6, %edx + movl %r13d, %ecx + addl %edx, %r12d + rorl $9, %ecx + andl %ebx, %eax + xorl %r13d, %ecx + xorl %r14d, %eax + rorl $11, %ecx + addl %r12d, %r8d + xorl %r13d, %ecx + addl %eax, %r12d + rorl $2, %ecx + movl %r8d, %edx + addl %ecx, %r12d + # rnd_all_4: 1-4 + addl 16(%rsp), %r11d + movl %r9d, %ecx + movl %r13d, %eax + xorl %r10d, %ecx + rorl $14, %edx + andl %r8d, %ecx + xorl %r8d, %edx + xorl %r10d, %ecx + rorl $5, %edx + addl %ecx, %r11d + xorl %r8d, %edx + xorl %r12d, %eax + rorl $6, %edx + movl %r12d, %ecx + addl %edx, %r11d + rorl $9, %ecx + andl %eax, %ebx + xorl %r12d, %ecx + xorl %r13d, %ebx + rorl $11, %ecx + addl %r11d, %r15d + xorl %r12d, %ecx + addl %ebx, %r11d + rorl $2, %ecx + movl %r15d, %edx + addl %ecx, %r11d + addl 20(%rsp), %r10d + movl %r8d, %ecx + movl %r12d, %ebx + xorl %r9d, %ecx + rorl $14, %edx + andl %r15d, %ecx + xorl %r15d, %edx + xorl %r9d, %ecx + rorl $5, %edx + addl %ecx, %r10d + xorl %r15d, %edx + xorl %r11d, %ebx + rorl $6, %edx + movl %r11d, %ecx + addl %edx, %r10d + rorl $9, %ecx + andl %ebx, %eax + xorl %r11d, %ecx + xorl %r12d, %eax + rorl $11, %ecx + addl %r10d, %r14d + xorl %r11d, %ecx + addl %eax, %r10d + rorl $2, %ecx + movl %r14d, %edx + addl %ecx, %r10d + addl 24(%rsp), %r9d + movl %r15d, %ecx + movl %r11d, %eax + xorl %r8d, %ecx + rorl $14, %edx + andl %r14d, %ecx + xorl %r14d, %edx + xorl %r8d, %ecx + rorl $5, %edx + addl %ecx, %r9d + xorl %r14d, %edx + xorl %r10d, %eax + rorl $6, %edx + movl %r10d, %ecx + addl %edx, %r9d + rorl $9, %ecx + andl %eax, %ebx + xorl %r10d, %ecx + xorl %r11d, %ebx + rorl $11, %ecx + addl %r9d, %r13d + xorl %r10d, %ecx + addl %ebx, %r9d + rorl $2, %ecx + movl %r13d, %edx + addl %ecx, %r9d + addl 28(%rsp), %r8d + movl %r14d, %ecx + movl %r10d, %ebx + xorl %r15d, %ecx + rorl $14, %edx + andl %r13d, %ecx + xorl %r13d, %edx + xorl %r15d, %ecx + rorl $5, %edx + addl %ecx, %r8d + xorl %r13d, %edx + xorl %r9d, %ebx + rorl $6, %edx + movl %r9d, %ecx + addl %edx, %r8d + rorl $9, %ecx + andl %ebx, %eax + xorl %r9d, %ecx + xorl %r10d, %eax + rorl $11, %ecx + addl %r8d, %r12d + xorl %r9d, %ecx + addl %eax, %r8d + rorl $2, %ecx + movl %r12d, %edx + addl %ecx, %r8d + # rnd_all_4: 2-5 + addl 32(%rsp), %r15d + movl %r13d, %ecx + movl %r9d, %eax + xorl %r14d, %ecx + rorl $14, %edx + andl %r12d, %ecx + xorl %r12d, %edx + xorl %r14d, %ecx + rorl $5, %edx + addl %ecx, %r15d + xorl %r12d, %edx + xorl %r8d, %eax + rorl $6, %edx + movl %r8d, %ecx + addl %edx, %r15d + rorl $9, %ecx + andl %eax, %ebx + xorl %r8d, %ecx + xorl %r9d, %ebx + rorl $11, %ecx + addl %r15d, %r11d + xorl %r8d, %ecx + addl %ebx, %r15d + rorl $2, %ecx + movl %r11d, %edx + addl %ecx, %r15d + addl 36(%rsp), %r14d + movl %r12d, %ecx + movl %r8d, %ebx + xorl %r13d, %ecx + rorl $14, %edx + andl %r11d, %ecx + xorl %r11d, %edx + xorl %r13d, %ecx + rorl $5, %edx + addl %ecx, %r14d + xorl %r11d, %edx + xorl %r15d, %ebx + rorl $6, %edx + movl %r15d, %ecx + addl %edx, %r14d + rorl $9, %ecx + andl %ebx, %eax + xorl %r15d, %ecx + xorl %r8d, %eax + rorl $11, %ecx + addl %r14d, %r10d + xorl %r15d, %ecx + addl %eax, %r14d + rorl $2, %ecx + movl %r10d, %edx + addl %ecx, %r14d + addl 40(%rsp), %r13d + movl %r11d, %ecx + movl %r15d, %eax + xorl %r12d, %ecx + rorl $14, %edx + andl %r10d, %ecx + xorl %r10d, %edx + xorl %r12d, %ecx + rorl $5, %edx + addl %ecx, %r13d + xorl %r10d, %edx + xorl %r14d, %eax + rorl $6, %edx + movl %r14d, %ecx + addl %edx, %r13d + rorl $9, %ecx + andl %eax, %ebx + xorl %r14d, %ecx + xorl %r15d, %ebx + rorl $11, %ecx + addl %r13d, %r9d + xorl %r14d, %ecx + addl %ebx, %r13d + rorl $2, %ecx + movl %r9d, %edx + addl %ecx, %r13d + addl 44(%rsp), %r12d + movl %r10d, %ecx + movl %r14d, %ebx + xorl %r11d, %ecx + rorl $14, %edx + andl %r9d, %ecx + xorl %r9d, %edx + xorl %r11d, %ecx + rorl $5, %edx + addl %ecx, %r12d + xorl %r9d, %edx + xorl %r13d, %ebx + rorl $6, %edx + movl %r13d, %ecx + addl %edx, %r12d + rorl $9, %ecx + andl %ebx, %eax + xorl %r13d, %ecx + xorl %r14d, %eax + rorl $11, %ecx + addl %r12d, %r8d + xorl %r13d, %ecx + addl %eax, %r12d + rorl $2, %ecx + movl %r8d, %edx + addl %ecx, %r12d + # rnd_all_4: 3-6 + addl 48(%rsp), %r11d + movl %r9d, %ecx + movl %r13d, %eax + xorl %r10d, %ecx + rorl $14, %edx + andl %r8d, %ecx + xorl %r8d, %edx + xorl %r10d, %ecx + rorl $5, %edx + addl %ecx, %r11d + xorl %r8d, %edx + xorl %r12d, %eax + rorl $6, %edx + movl %r12d, %ecx + addl %edx, %r11d + rorl $9, %ecx + andl %eax, %ebx + xorl %r12d, %ecx + xorl %r13d, %ebx + rorl $11, %ecx + addl %r11d, %r15d + xorl %r12d, %ecx + addl %ebx, %r11d + rorl $2, %ecx + movl %r15d, %edx + addl %ecx, %r11d + addl 52(%rsp), %r10d + movl %r8d, %ecx + movl %r12d, %ebx + xorl %r9d, %ecx + rorl $14, %edx + andl %r15d, %ecx + xorl %r15d, %edx + xorl %r9d, %ecx + rorl $5, %edx + addl %ecx, %r10d + xorl %r15d, %edx + xorl %r11d, %ebx + rorl $6, %edx + movl %r11d, %ecx + addl %edx, %r10d + rorl $9, %ecx + andl %ebx, %eax + xorl %r11d, %ecx + xorl %r12d, %eax + rorl $11, %ecx + addl %r10d, %r14d + xorl %r11d, %ecx + addl %eax, %r10d + rorl $2, %ecx + movl %r14d, %edx + addl %ecx, %r10d + addl 56(%rsp), %r9d + movl %r15d, %ecx + movl %r11d, %eax + xorl %r8d, %ecx + rorl $14, %edx + andl %r14d, %ecx + xorl %r14d, %edx + xorl %r8d, %ecx + rorl $5, %edx + addl %ecx, %r9d + xorl %r14d, %edx + xorl %r10d, %eax + rorl $6, %edx + movl %r10d, %ecx + addl %edx, %r9d + rorl $9, %ecx + andl %eax, %ebx + xorl %r10d, %ecx + xorl %r11d, %ebx + rorl $11, %ecx + addl %r9d, %r13d + xorl %r10d, %ecx + addl %ebx, %r9d + rorl $2, %ecx + movl %r13d, %edx + addl %ecx, %r9d + addl 60(%rsp), %r8d + movl %r14d, %ecx + movl %r10d, %ebx + xorl %r15d, %ecx + rorl $14, %edx + andl %r13d, %ecx + xorl %r13d, %edx + xorl %r15d, %ecx + rorl $5, %edx + addl %ecx, %r8d + xorl %r13d, %edx + xorl %r9d, %ebx + rorl $6, %edx + movl %r9d, %ecx + addl %edx, %r8d + rorl $9, %ecx + andl %ebx, %eax + xorl %r9d, %ecx + xorl %r10d, %eax + rorl $11, %ecx + addl %r8d, %r12d + xorl %r9d, %ecx + addl %eax, %r8d + rorl $2, %ecx + movl %r12d, %edx + addl %ecx, %r8d + addl %r8d, (%rdi) + addl %r9d, 4(%rdi) + addl %r10d, 8(%rdi) + addl %r11d, 12(%rdi) + addl %r12d, 16(%rdi) + addl %r13d, 20(%rdi) + addl %r14d, 24(%rdi) + addl %r15d, 28(%rdi) + xorq %rax, %rax + vzeroupper + addq $0x40, %rsp + popq %r15 + popq %r14 + popq %r13 + popq %r12 + popq %rbx + repz retq +.size Transform_Sha256_AVX1,.-Transform_Sha256_AVX1 +.globl Transform_Sha256_AVX1_Len +.type Transform_Sha256_AVX1_Len,@function +.align 4 +Transform_Sha256_AVX1_Len: + pushq %rbx + pushq %r12 + pushq %r13 + pushq %r14 + pushq %r15 + pushq %rbp + movq %rsi, %rbp + movq %rdx, %rsi + subq $0x40, %rsp + vmovdqa L_avx1_sha256_flip_mask(%rip), %xmm13 + vmovdqa L_avx1_sha256_shuf_00BA(%rip), %xmm11 + vmovdqa L_avx1_sha256_shuf_DC00(%rip), %xmm12 + movl (%rdi), %r8d + movl 4(%rdi), %r9d + movl 8(%rdi), %r10d + movl 12(%rdi), %r11d + movl 16(%rdi), %r12d + movl 20(%rdi), %r13d + movl 24(%rdi), %r14d + movl 28(%rdi), %r15d + # Start of loop processing a block +L_sha256_len_avx1_start: + # X0, X1, X2, X3 = W[0..15] + vmovdqu (%rbp), %xmm0 + vmovdqu 16(%rbp), %xmm1 + vpshufb %xmm13, %xmm0, %xmm0 + vpshufb %xmm13, %xmm1, %xmm1 + vmovdqu 32(%rbp), %xmm2 + vmovdqu 48(%rbp), %xmm3 + vpshufb %xmm13, %xmm2, %xmm2 + vpshufb %xmm13, %xmm3, %xmm3 + movl %r9d, %ebx + movl %r12d, %edx + xorl %r10d, %ebx + # set_w_k_xfer_4: 0 + vpaddd 0+L_avx1_sha256_k(%rip), %xmm0, %xmm4 + vpaddd 16+L_avx1_sha256_k(%rip), %xmm1, %xmm5 + vmovdqu %xmm4, (%rsp) + vmovdqu %xmm5, 16(%rsp) + vpaddd 32+L_avx1_sha256_k(%rip), %xmm2, %xmm6 + vpaddd 48+L_avx1_sha256_k(%rip), %xmm3, %xmm7 + vmovdqu %xmm6, 32(%rsp) + vmovdqu %xmm7, 48(%rsp) + # msg_sched: 0-3 + # rnd_0: 0 - 0 + rorl $14, %edx + vpalignr $4, %xmm0, %xmm1, %xmm5 + vpalignr $4, %xmm2, %xmm3, %xmm4 + # rnd_0: 1 - 2 + movl %r9d, %eax + movl %r13d, %ecx + addl (%rsp), %r15d + xorl %r14d, %ecx + xorl %r12d, %edx + andl %r12d, %ecx + vpsrld $7, %xmm5, %xmm6 + vpslld $25, %xmm5, %xmm7 + # rnd_0: 3 - 4 + rorl $5, %edx + xorl %r14d, %ecx + xorl %r12d, %edx + addl %ecx, %r15d + rorl $6, %edx + xorl %r8d, %eax + addl %edx, %r15d + movl %r8d, %ecx + vpsrld $18, %xmm5, %xmm8 + vpslld $14, %xmm5, %xmm9 + # rnd_0: 5 - 6 + andl %eax, %ebx + rorl $9, %ecx + xorl %r8d, %ecx + xorl %r9d, %ebx + rorl $11, %ecx + addl %r15d, %r11d + xorl %r8d, %ecx + addl %ebx, %r15d + vpor %xmm6, %xmm7, %xmm6 + vpor %xmm8, %xmm9, %xmm8 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r11d, %edx + addl %ecx, %r15d + # rnd_1: 0 - 1 + rorl $14, %edx + movl %r8d, %ebx + movl %r12d, %ecx + addl 4(%rsp), %r14d + xorl %r13d, %ecx + vpsrld $3, %xmm5, %xmm9 + vpxor %xmm6, %xmm8, %xmm6 + # rnd_1: 2 - 3 + xorl %r11d, %edx + andl %r11d, %ecx + rorl $5, %edx + xorl %r13d, %ecx + xorl %r11d, %edx + addl %ecx, %r14d + vpxor %xmm6, %xmm9, %xmm5 + vpshufd $0xfa, %xmm3, %xmm6 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r15d, %ebx + addl %edx, %r14d + movl %r15d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r15d, %ecx + xorl %r8d, %eax + vpsrld $10, %xmm6, %xmm8 + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 6 - 7 + rorl $11, %ecx + addl %r14d, %r10d + xorl %r15d, %ecx + addl %eax, %r14d + rorl $2, %ecx + movl %r10d, %edx + addl %ecx, %r14d + # rnd_0: 0 - 0 + rorl $14, %edx + vpsrlq $0x11, %xmm6, %xmm6 + vpaddd %xmm0, %xmm4, %xmm4 + # rnd_0: 1 - 3 + movl %r15d, %eax + movl %r11d, %ecx + addl 8(%rsp), %r13d + xorl %r12d, %ecx + xorl %r10d, %edx + andl %r10d, %ecx + rorl $5, %edx + xorl %r12d, %ecx + xorl %r10d, %edx + addl %ecx, %r13d + vpxor %xmm6, %xmm7, %xmm6 + vpaddd %xmm5, %xmm4, %xmm4 + # rnd_0: 4 - 4 + rorl $6, %edx + xorl %r14d, %eax + addl %edx, %r13d + movl %r14d, %ecx + vpxor %xmm6, %xmm8, %xmm8 + # rnd_0: 5 - 5 + andl %eax, %ebx + rorl $9, %ecx + xorl %r14d, %ecx + xorl %r15d, %ebx + vpshufb %xmm11, %xmm8, %xmm8 + # rnd_0: 6 - 6 + rorl $11, %ecx + addl %r13d, %r9d + xorl %r14d, %ecx + addl %ebx, %r13d + vpaddd %xmm8, %xmm4, %xmm4 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r9d, %edx + addl %ecx, %r13d + # rnd_1: 0 - 0 + rorl $14, %edx + vpshufd $0x50, %xmm4, %xmm6 + # rnd_1: 1 - 1 + movl %r14d, %ebx + movl %r10d, %ecx + addl 12(%rsp), %r12d + xorl %r11d, %ecx + vpsrlq $0x11, %xmm6, %xmm8 + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 2 - 3 + xorl %r9d, %edx + andl %r9d, %ecx + rorl $5, %edx + xorl %r11d, %ecx + xorl %r9d, %edx + addl %ecx, %r12d + vpsrld $10, %xmm6, %xmm9 + vpxor %xmm8, %xmm7, %xmm8 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r13d, %ebx + addl %edx, %r12d + movl %r13d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r13d, %ecx + xorl %r14d, %eax + vpxor %xmm9, %xmm8, %xmm9 + # rnd_1: 6 - 6 + rorl $11, %ecx + addl %r12d, %r8d + xorl %r13d, %ecx + addl %eax, %r12d + vpshufb %xmm12, %xmm9, %xmm9 + # rnd_1: 7 - 7 + rorl $2, %ecx + movl %r8d, %edx + addl %ecx, %r12d + vpaddd %xmm4, %xmm9, %xmm0 + # msg_sched done: 0-3 + # msg_sched: 4-7 + # rnd_0: 0 - 0 + rorl $14, %edx + vpalignr $4, %xmm1, %xmm2, %xmm5 + vpalignr $4, %xmm3, %xmm0, %xmm4 + # rnd_0: 1 - 2 + movl %r13d, %eax + movl %r9d, %ecx + addl 16(%rsp), %r11d + xorl %r10d, %ecx + xorl %r8d, %edx + andl %r8d, %ecx + vpsrld $7, %xmm5, %xmm6 + vpslld $25, %xmm5, %xmm7 + # rnd_0: 3 - 4 + rorl $5, %edx + xorl %r10d, %ecx + xorl %r8d, %edx + addl %ecx, %r11d + rorl $6, %edx + xorl %r12d, %eax + addl %edx, %r11d + movl %r12d, %ecx + vpsrld $18, %xmm5, %xmm8 + vpslld $14, %xmm5, %xmm9 + # rnd_0: 5 - 6 + andl %eax, %ebx + rorl $9, %ecx + xorl %r12d, %ecx + xorl %r13d, %ebx + rorl $11, %ecx + addl %r11d, %r15d + xorl %r12d, %ecx + addl %ebx, %r11d + vpor %xmm6, %xmm7, %xmm6 + vpor %xmm8, %xmm9, %xmm8 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r15d, %edx + addl %ecx, %r11d + # rnd_1: 0 - 1 + rorl $14, %edx + movl %r12d, %ebx + movl %r8d, %ecx + addl 20(%rsp), %r10d + xorl %r9d, %ecx + vpsrld $3, %xmm5, %xmm9 + vpxor %xmm6, %xmm8, %xmm6 + # rnd_1: 2 - 3 + xorl %r15d, %edx + andl %r15d, %ecx + rorl $5, %edx + xorl %r9d, %ecx + xorl %r15d, %edx + addl %ecx, %r10d + vpxor %xmm6, %xmm9, %xmm5 + vpshufd $0xfa, %xmm0, %xmm6 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r11d, %ebx + addl %edx, %r10d + movl %r11d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r11d, %ecx + xorl %r12d, %eax + vpsrld $10, %xmm6, %xmm8 + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 6 - 7 + rorl $11, %ecx + addl %r10d, %r14d + xorl %r11d, %ecx + addl %eax, %r10d + rorl $2, %ecx + movl %r14d, %edx + addl %ecx, %r10d + # rnd_0: 0 - 0 + rorl $14, %edx + vpsrlq $0x11, %xmm6, %xmm6 + vpaddd %xmm1, %xmm4, %xmm4 + # rnd_0: 1 - 3 + movl %r11d, %eax + movl %r15d, %ecx + addl 24(%rsp), %r9d + xorl %r8d, %ecx + xorl %r14d, %edx + andl %r14d, %ecx + rorl $5, %edx + xorl %r8d, %ecx + xorl %r14d, %edx + addl %ecx, %r9d + vpxor %xmm6, %xmm7, %xmm6 + vpaddd %xmm5, %xmm4, %xmm4 + # rnd_0: 4 - 4 + rorl $6, %edx + xorl %r10d, %eax + addl %edx, %r9d + movl %r10d, %ecx + vpxor %xmm6, %xmm8, %xmm8 + # rnd_0: 5 - 5 + andl %eax, %ebx + rorl $9, %ecx + xorl %r10d, %ecx + xorl %r11d, %ebx + vpshufb %xmm11, %xmm8, %xmm8 + # rnd_0: 6 - 6 + rorl $11, %ecx + addl %r9d, %r13d + xorl %r10d, %ecx + addl %ebx, %r9d + vpaddd %xmm8, %xmm4, %xmm4 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r13d, %edx + addl %ecx, %r9d + # rnd_1: 0 - 0 + rorl $14, %edx + vpshufd $0x50, %xmm4, %xmm6 + # rnd_1: 1 - 1 + movl %r10d, %ebx + movl %r14d, %ecx + addl 28(%rsp), %r8d + xorl %r15d, %ecx + vpsrlq $0x11, %xmm6, %xmm8 + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 2 - 3 + xorl %r13d, %edx + andl %r13d, %ecx + rorl $5, %edx + xorl %r15d, %ecx + xorl %r13d, %edx + addl %ecx, %r8d + vpsrld $10, %xmm6, %xmm9 + vpxor %xmm8, %xmm7, %xmm8 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r9d, %ebx + addl %edx, %r8d + movl %r9d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r9d, %ecx + xorl %r10d, %eax + vpxor %xmm9, %xmm8, %xmm9 + # rnd_1: 6 - 6 + rorl $11, %ecx + addl %r8d, %r12d + xorl %r9d, %ecx + addl %eax, %r8d + vpshufb %xmm12, %xmm9, %xmm9 + # rnd_1: 7 - 7 + rorl $2, %ecx + movl %r12d, %edx + addl %ecx, %r8d + vpaddd %xmm4, %xmm9, %xmm1 + # msg_sched done: 4-7 + # msg_sched: 8-11 + # rnd_0: 0 - 0 + rorl $14, %edx + vpalignr $4, %xmm2, %xmm3, %xmm5 + vpalignr $4, %xmm0, %xmm1, %xmm4 + # rnd_0: 1 - 2 + movl %r9d, %eax + movl %r13d, %ecx + addl 32(%rsp), %r15d + xorl %r14d, %ecx + xorl %r12d, %edx + andl %r12d, %ecx + vpsrld $7, %xmm5, %xmm6 + vpslld $25, %xmm5, %xmm7 + # rnd_0: 3 - 4 + rorl $5, %edx + xorl %r14d, %ecx + xorl %r12d, %edx + addl %ecx, %r15d + rorl $6, %edx + xorl %r8d, %eax + addl %edx, %r15d + movl %r8d, %ecx + vpsrld $18, %xmm5, %xmm8 + vpslld $14, %xmm5, %xmm9 + # rnd_0: 5 - 6 + andl %eax, %ebx + rorl $9, %ecx + xorl %r8d, %ecx + xorl %r9d, %ebx + rorl $11, %ecx + addl %r15d, %r11d + xorl %r8d, %ecx + addl %ebx, %r15d + vpor %xmm6, %xmm7, %xmm6 + vpor %xmm8, %xmm9, %xmm8 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r11d, %edx + addl %ecx, %r15d + # rnd_1: 0 - 1 + rorl $14, %edx + movl %r8d, %ebx + movl %r12d, %ecx + addl 36(%rsp), %r14d + xorl %r13d, %ecx + vpsrld $3, %xmm5, %xmm9 + vpxor %xmm6, %xmm8, %xmm6 + # rnd_1: 2 - 3 + xorl %r11d, %edx + andl %r11d, %ecx + rorl $5, %edx + xorl %r13d, %ecx + xorl %r11d, %edx + addl %ecx, %r14d + vpxor %xmm6, %xmm9, %xmm5 + vpshufd $0xfa, %xmm1, %xmm6 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r15d, %ebx + addl %edx, %r14d + movl %r15d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r15d, %ecx + xorl %r8d, %eax + vpsrld $10, %xmm6, %xmm8 + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 6 - 7 + rorl $11, %ecx + addl %r14d, %r10d + xorl %r15d, %ecx + addl %eax, %r14d + rorl $2, %ecx + movl %r10d, %edx + addl %ecx, %r14d + # rnd_0: 0 - 0 + rorl $14, %edx + vpsrlq $0x11, %xmm6, %xmm6 + vpaddd %xmm2, %xmm4, %xmm4 + # rnd_0: 1 - 3 + movl %r15d, %eax + movl %r11d, %ecx + addl 40(%rsp), %r13d + xorl %r12d, %ecx + xorl %r10d, %edx + andl %r10d, %ecx + rorl $5, %edx + xorl %r12d, %ecx + xorl %r10d, %edx + addl %ecx, %r13d + vpxor %xmm6, %xmm7, %xmm6 + vpaddd %xmm5, %xmm4, %xmm4 + # rnd_0: 4 - 4 + rorl $6, %edx + xorl %r14d, %eax + addl %edx, %r13d + movl %r14d, %ecx + vpxor %xmm6, %xmm8, %xmm8 + # rnd_0: 5 - 5 + andl %eax, %ebx + rorl $9, %ecx + xorl %r14d, %ecx + xorl %r15d, %ebx + vpshufb %xmm11, %xmm8, %xmm8 + # rnd_0: 6 - 6 + rorl $11, %ecx + addl %r13d, %r9d + xorl %r14d, %ecx + addl %ebx, %r13d + vpaddd %xmm8, %xmm4, %xmm4 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r9d, %edx + addl %ecx, %r13d + # rnd_1: 0 - 0 + rorl $14, %edx + vpshufd $0x50, %xmm4, %xmm6 + # rnd_1: 1 - 1 + movl %r14d, %ebx + movl %r10d, %ecx + addl 44(%rsp), %r12d + xorl %r11d, %ecx + vpsrlq $0x11, %xmm6, %xmm8 + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 2 - 3 + xorl %r9d, %edx + andl %r9d, %ecx + rorl $5, %edx + xorl %r11d, %ecx + xorl %r9d, %edx + addl %ecx, %r12d + vpsrld $10, %xmm6, %xmm9 + vpxor %xmm8, %xmm7, %xmm8 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r13d, %ebx + addl %edx, %r12d + movl %r13d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r13d, %ecx + xorl %r14d, %eax + vpxor %xmm9, %xmm8, %xmm9 + # rnd_1: 6 - 6 + rorl $11, %ecx + addl %r12d, %r8d + xorl %r13d, %ecx + addl %eax, %r12d + vpshufb %xmm12, %xmm9, %xmm9 + # rnd_1: 7 - 7 + rorl $2, %ecx + movl %r8d, %edx + addl %ecx, %r12d + vpaddd %xmm4, %xmm9, %xmm2 + # msg_sched done: 8-11 + # msg_sched: 12-15 + # rnd_0: 0 - 0 + rorl $14, %edx + vpalignr $4, %xmm3, %xmm0, %xmm5 + vpalignr $4, %xmm1, %xmm2, %xmm4 + # rnd_0: 1 - 2 + movl %r13d, %eax + movl %r9d, %ecx + addl 48(%rsp), %r11d + xorl %r10d, %ecx + xorl %r8d, %edx + andl %r8d, %ecx + vpsrld $7, %xmm5, %xmm6 + vpslld $25, %xmm5, %xmm7 + # rnd_0: 3 - 4 + rorl $5, %edx + xorl %r10d, %ecx + xorl %r8d, %edx + addl %ecx, %r11d + rorl $6, %edx + xorl %r12d, %eax + addl %edx, %r11d + movl %r12d, %ecx + vpsrld $18, %xmm5, %xmm8 + vpslld $14, %xmm5, %xmm9 + # rnd_0: 5 - 6 + andl %eax, %ebx + rorl $9, %ecx + xorl %r12d, %ecx + xorl %r13d, %ebx + rorl $11, %ecx + addl %r11d, %r15d + xorl %r12d, %ecx + addl %ebx, %r11d + vpor %xmm6, %xmm7, %xmm6 + vpor %xmm8, %xmm9, %xmm8 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r15d, %edx + addl %ecx, %r11d + # rnd_1: 0 - 1 + rorl $14, %edx + movl %r12d, %ebx + movl %r8d, %ecx + addl 52(%rsp), %r10d + xorl %r9d, %ecx + vpsrld $3, %xmm5, %xmm9 + vpxor %xmm6, %xmm8, %xmm6 + # rnd_1: 2 - 3 + xorl %r15d, %edx + andl %r15d, %ecx + rorl $5, %edx + xorl %r9d, %ecx + xorl %r15d, %edx + addl %ecx, %r10d + vpxor %xmm6, %xmm9, %xmm5 + vpshufd $0xfa, %xmm2, %xmm6 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r11d, %ebx + addl %edx, %r10d + movl %r11d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r11d, %ecx + xorl %r12d, %eax + vpsrld $10, %xmm6, %xmm8 + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 6 - 7 + rorl $11, %ecx + addl %r10d, %r14d + xorl %r11d, %ecx + addl %eax, %r10d + rorl $2, %ecx + movl %r14d, %edx + addl %ecx, %r10d + # rnd_0: 0 - 0 + rorl $14, %edx + vpsrlq $0x11, %xmm6, %xmm6 + vpaddd %xmm3, %xmm4, %xmm4 + # rnd_0: 1 - 3 + movl %r11d, %eax + movl %r15d, %ecx + addl 56(%rsp), %r9d + xorl %r8d, %ecx + xorl %r14d, %edx + andl %r14d, %ecx + rorl $5, %edx + xorl %r8d, %ecx + xorl %r14d, %edx + addl %ecx, %r9d + vpxor %xmm6, %xmm7, %xmm6 + vpaddd %xmm5, %xmm4, %xmm4 + # rnd_0: 4 - 4 + rorl $6, %edx + xorl %r10d, %eax + addl %edx, %r9d + movl %r10d, %ecx + vpxor %xmm6, %xmm8, %xmm8 + # rnd_0: 5 - 5 + andl %eax, %ebx + rorl $9, %ecx + xorl %r10d, %ecx + xorl %r11d, %ebx + vpshufb %xmm11, %xmm8, %xmm8 + # rnd_0: 6 - 6 + rorl $11, %ecx + addl %r9d, %r13d + xorl %r10d, %ecx + addl %ebx, %r9d + vpaddd %xmm8, %xmm4, %xmm4 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r13d, %edx + addl %ecx, %r9d + # rnd_1: 0 - 0 + rorl $14, %edx + vpshufd $0x50, %xmm4, %xmm6 + # rnd_1: 1 - 1 + movl %r10d, %ebx + movl %r14d, %ecx + addl 60(%rsp), %r8d + xorl %r15d, %ecx + vpsrlq $0x11, %xmm6, %xmm8 + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 2 - 3 + xorl %r13d, %edx + andl %r13d, %ecx + rorl $5, %edx + xorl %r15d, %ecx + xorl %r13d, %edx + addl %ecx, %r8d + vpsrld $10, %xmm6, %xmm9 + vpxor %xmm8, %xmm7, %xmm8 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r9d, %ebx + addl %edx, %r8d + movl %r9d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r9d, %ecx + xorl %r10d, %eax + vpxor %xmm9, %xmm8, %xmm9 + # rnd_1: 6 - 6 + rorl $11, %ecx + addl %r8d, %r12d + xorl %r9d, %ecx + addl %eax, %r8d + vpshufb %xmm12, %xmm9, %xmm9 + # rnd_1: 7 - 7 + rorl $2, %ecx + movl %r12d, %edx + addl %ecx, %r8d + vpaddd %xmm4, %xmm9, %xmm3 + # msg_sched done: 12-15 + # set_w_k_xfer_4: 4 + vpaddd 64+L_avx1_sha256_k(%rip), %xmm0, %xmm4 + vpaddd 80+L_avx1_sha256_k(%rip), %xmm1, %xmm5 + vmovdqu %xmm4, (%rsp) + vmovdqu %xmm5, 16(%rsp) + vpaddd 96+L_avx1_sha256_k(%rip), %xmm2, %xmm6 + vpaddd 112+L_avx1_sha256_k(%rip), %xmm3, %xmm7 + vmovdqu %xmm6, 32(%rsp) + vmovdqu %xmm7, 48(%rsp) + # msg_sched: 0-3 + # rnd_0: 0 - 0 + rorl $14, %edx + vpalignr $4, %xmm0, %xmm1, %xmm5 + vpalignr $4, %xmm2, %xmm3, %xmm4 + # rnd_0: 1 - 2 + movl %r9d, %eax + movl %r13d, %ecx + addl (%rsp), %r15d + xorl %r14d, %ecx + xorl %r12d, %edx + andl %r12d, %ecx + vpsrld $7, %xmm5, %xmm6 + vpslld $25, %xmm5, %xmm7 + # rnd_0: 3 - 4 + rorl $5, %edx + xorl %r14d, %ecx + xorl %r12d, %edx + addl %ecx, %r15d + rorl $6, %edx + xorl %r8d, %eax + addl %edx, %r15d + movl %r8d, %ecx + vpsrld $18, %xmm5, %xmm8 + vpslld $14, %xmm5, %xmm9 + # rnd_0: 5 - 6 + andl %eax, %ebx + rorl $9, %ecx + xorl %r8d, %ecx + xorl %r9d, %ebx + rorl $11, %ecx + addl %r15d, %r11d + xorl %r8d, %ecx + addl %ebx, %r15d + vpor %xmm6, %xmm7, %xmm6 + vpor %xmm8, %xmm9, %xmm8 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r11d, %edx + addl %ecx, %r15d + # rnd_1: 0 - 1 + rorl $14, %edx + movl %r8d, %ebx + movl %r12d, %ecx + addl 4(%rsp), %r14d + xorl %r13d, %ecx + vpsrld $3, %xmm5, %xmm9 + vpxor %xmm6, %xmm8, %xmm6 + # rnd_1: 2 - 3 + xorl %r11d, %edx + andl %r11d, %ecx + rorl $5, %edx + xorl %r13d, %ecx + xorl %r11d, %edx + addl %ecx, %r14d + vpxor %xmm6, %xmm9, %xmm5 + vpshufd $0xfa, %xmm3, %xmm6 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r15d, %ebx + addl %edx, %r14d + movl %r15d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r15d, %ecx + xorl %r8d, %eax + vpsrld $10, %xmm6, %xmm8 + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 6 - 7 + rorl $11, %ecx + addl %r14d, %r10d + xorl %r15d, %ecx + addl %eax, %r14d + rorl $2, %ecx + movl %r10d, %edx + addl %ecx, %r14d + # rnd_0: 0 - 0 + rorl $14, %edx + vpsrlq $0x11, %xmm6, %xmm6 + vpaddd %xmm0, %xmm4, %xmm4 + # rnd_0: 1 - 3 + movl %r15d, %eax + movl %r11d, %ecx + addl 8(%rsp), %r13d + xorl %r12d, %ecx + xorl %r10d, %edx + andl %r10d, %ecx + rorl $5, %edx + xorl %r12d, %ecx + xorl %r10d, %edx + addl %ecx, %r13d + vpxor %xmm6, %xmm7, %xmm6 + vpaddd %xmm5, %xmm4, %xmm4 + # rnd_0: 4 - 4 + rorl $6, %edx + xorl %r14d, %eax + addl %edx, %r13d + movl %r14d, %ecx + vpxor %xmm6, %xmm8, %xmm8 + # rnd_0: 5 - 5 + andl %eax, %ebx + rorl $9, %ecx + xorl %r14d, %ecx + xorl %r15d, %ebx + vpshufb %xmm11, %xmm8, %xmm8 + # rnd_0: 6 - 6 + rorl $11, %ecx + addl %r13d, %r9d + xorl %r14d, %ecx + addl %ebx, %r13d + vpaddd %xmm8, %xmm4, %xmm4 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r9d, %edx + addl %ecx, %r13d + # rnd_1: 0 - 0 + rorl $14, %edx + vpshufd $0x50, %xmm4, %xmm6 + # rnd_1: 1 - 1 + movl %r14d, %ebx + movl %r10d, %ecx + addl 12(%rsp), %r12d + xorl %r11d, %ecx + vpsrlq $0x11, %xmm6, %xmm8 + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 2 - 3 + xorl %r9d, %edx + andl %r9d, %ecx + rorl $5, %edx + xorl %r11d, %ecx + xorl %r9d, %edx + addl %ecx, %r12d + vpsrld $10, %xmm6, %xmm9 + vpxor %xmm8, %xmm7, %xmm8 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r13d, %ebx + addl %edx, %r12d + movl %r13d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r13d, %ecx + xorl %r14d, %eax + vpxor %xmm9, %xmm8, %xmm9 + # rnd_1: 6 - 6 + rorl $11, %ecx + addl %r12d, %r8d + xorl %r13d, %ecx + addl %eax, %r12d + vpshufb %xmm12, %xmm9, %xmm9 + # rnd_1: 7 - 7 + rorl $2, %ecx + movl %r8d, %edx + addl %ecx, %r12d + vpaddd %xmm4, %xmm9, %xmm0 + # msg_sched done: 0-3 + # msg_sched: 4-7 + # rnd_0: 0 - 0 + rorl $14, %edx + vpalignr $4, %xmm1, %xmm2, %xmm5 + vpalignr $4, %xmm3, %xmm0, %xmm4 + # rnd_0: 1 - 2 + movl %r13d, %eax + movl %r9d, %ecx + addl 16(%rsp), %r11d + xorl %r10d, %ecx + xorl %r8d, %edx + andl %r8d, %ecx + vpsrld $7, %xmm5, %xmm6 + vpslld $25, %xmm5, %xmm7 + # rnd_0: 3 - 4 + rorl $5, %edx + xorl %r10d, %ecx + xorl %r8d, %edx + addl %ecx, %r11d + rorl $6, %edx + xorl %r12d, %eax + addl %edx, %r11d + movl %r12d, %ecx + vpsrld $18, %xmm5, %xmm8 + vpslld $14, %xmm5, %xmm9 + # rnd_0: 5 - 6 + andl %eax, %ebx + rorl $9, %ecx + xorl %r12d, %ecx + xorl %r13d, %ebx + rorl $11, %ecx + addl %r11d, %r15d + xorl %r12d, %ecx + addl %ebx, %r11d + vpor %xmm6, %xmm7, %xmm6 + vpor %xmm8, %xmm9, %xmm8 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r15d, %edx + addl %ecx, %r11d + # rnd_1: 0 - 1 + rorl $14, %edx + movl %r12d, %ebx + movl %r8d, %ecx + addl 20(%rsp), %r10d + xorl %r9d, %ecx + vpsrld $3, %xmm5, %xmm9 + vpxor %xmm6, %xmm8, %xmm6 + # rnd_1: 2 - 3 + xorl %r15d, %edx + andl %r15d, %ecx + rorl $5, %edx + xorl %r9d, %ecx + xorl %r15d, %edx + addl %ecx, %r10d + vpxor %xmm6, %xmm9, %xmm5 + vpshufd $0xfa, %xmm0, %xmm6 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r11d, %ebx + addl %edx, %r10d + movl %r11d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r11d, %ecx + xorl %r12d, %eax + vpsrld $10, %xmm6, %xmm8 + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 6 - 7 + rorl $11, %ecx + addl %r10d, %r14d + xorl %r11d, %ecx + addl %eax, %r10d + rorl $2, %ecx + movl %r14d, %edx + addl %ecx, %r10d + # rnd_0: 0 - 0 + rorl $14, %edx + vpsrlq $0x11, %xmm6, %xmm6 + vpaddd %xmm1, %xmm4, %xmm4 + # rnd_0: 1 - 3 + movl %r11d, %eax + movl %r15d, %ecx + addl 24(%rsp), %r9d + xorl %r8d, %ecx + xorl %r14d, %edx + andl %r14d, %ecx + rorl $5, %edx + xorl %r8d, %ecx + xorl %r14d, %edx + addl %ecx, %r9d + vpxor %xmm6, %xmm7, %xmm6 + vpaddd %xmm5, %xmm4, %xmm4 + # rnd_0: 4 - 4 + rorl $6, %edx + xorl %r10d, %eax + addl %edx, %r9d + movl %r10d, %ecx + vpxor %xmm6, %xmm8, %xmm8 + # rnd_0: 5 - 5 + andl %eax, %ebx + rorl $9, %ecx + xorl %r10d, %ecx + xorl %r11d, %ebx + vpshufb %xmm11, %xmm8, %xmm8 + # rnd_0: 6 - 6 + rorl $11, %ecx + addl %r9d, %r13d + xorl %r10d, %ecx + addl %ebx, %r9d + vpaddd %xmm8, %xmm4, %xmm4 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r13d, %edx + addl %ecx, %r9d + # rnd_1: 0 - 0 + rorl $14, %edx + vpshufd $0x50, %xmm4, %xmm6 + # rnd_1: 1 - 1 + movl %r10d, %ebx + movl %r14d, %ecx + addl 28(%rsp), %r8d + xorl %r15d, %ecx + vpsrlq $0x11, %xmm6, %xmm8 + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 2 - 3 + xorl %r13d, %edx + andl %r13d, %ecx + rorl $5, %edx + xorl %r15d, %ecx + xorl %r13d, %edx + addl %ecx, %r8d + vpsrld $10, %xmm6, %xmm9 + vpxor %xmm8, %xmm7, %xmm8 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r9d, %ebx + addl %edx, %r8d + movl %r9d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r9d, %ecx + xorl %r10d, %eax + vpxor %xmm9, %xmm8, %xmm9 + # rnd_1: 6 - 6 + rorl $11, %ecx + addl %r8d, %r12d + xorl %r9d, %ecx + addl %eax, %r8d + vpshufb %xmm12, %xmm9, %xmm9 + # rnd_1: 7 - 7 + rorl $2, %ecx + movl %r12d, %edx + addl %ecx, %r8d + vpaddd %xmm4, %xmm9, %xmm1 + # msg_sched done: 4-7 + # msg_sched: 8-11 + # rnd_0: 0 - 0 + rorl $14, %edx + vpalignr $4, %xmm2, %xmm3, %xmm5 + vpalignr $4, %xmm0, %xmm1, %xmm4 + # rnd_0: 1 - 2 + movl %r9d, %eax + movl %r13d, %ecx + addl 32(%rsp), %r15d + xorl %r14d, %ecx + xorl %r12d, %edx + andl %r12d, %ecx + vpsrld $7, %xmm5, %xmm6 + vpslld $25, %xmm5, %xmm7 + # rnd_0: 3 - 4 + rorl $5, %edx + xorl %r14d, %ecx + xorl %r12d, %edx + addl %ecx, %r15d + rorl $6, %edx + xorl %r8d, %eax + addl %edx, %r15d + movl %r8d, %ecx + vpsrld $18, %xmm5, %xmm8 + vpslld $14, %xmm5, %xmm9 + # rnd_0: 5 - 6 + andl %eax, %ebx + rorl $9, %ecx + xorl %r8d, %ecx + xorl %r9d, %ebx + rorl $11, %ecx + addl %r15d, %r11d + xorl %r8d, %ecx + addl %ebx, %r15d + vpor %xmm6, %xmm7, %xmm6 + vpor %xmm8, %xmm9, %xmm8 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r11d, %edx + addl %ecx, %r15d + # rnd_1: 0 - 1 + rorl $14, %edx + movl %r8d, %ebx + movl %r12d, %ecx + addl 36(%rsp), %r14d + xorl %r13d, %ecx + vpsrld $3, %xmm5, %xmm9 + vpxor %xmm6, %xmm8, %xmm6 + # rnd_1: 2 - 3 + xorl %r11d, %edx + andl %r11d, %ecx + rorl $5, %edx + xorl %r13d, %ecx + xorl %r11d, %edx + addl %ecx, %r14d + vpxor %xmm6, %xmm9, %xmm5 + vpshufd $0xfa, %xmm1, %xmm6 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r15d, %ebx + addl %edx, %r14d + movl %r15d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r15d, %ecx + xorl %r8d, %eax + vpsrld $10, %xmm6, %xmm8 + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 6 - 7 + rorl $11, %ecx + addl %r14d, %r10d + xorl %r15d, %ecx + addl %eax, %r14d + rorl $2, %ecx + movl %r10d, %edx + addl %ecx, %r14d + # rnd_0: 0 - 0 + rorl $14, %edx + vpsrlq $0x11, %xmm6, %xmm6 + vpaddd %xmm2, %xmm4, %xmm4 + # rnd_0: 1 - 3 + movl %r15d, %eax + movl %r11d, %ecx + addl 40(%rsp), %r13d + xorl %r12d, %ecx + xorl %r10d, %edx + andl %r10d, %ecx + rorl $5, %edx + xorl %r12d, %ecx + xorl %r10d, %edx + addl %ecx, %r13d + vpxor %xmm6, %xmm7, %xmm6 + vpaddd %xmm5, %xmm4, %xmm4 + # rnd_0: 4 - 4 + rorl $6, %edx + xorl %r14d, %eax + addl %edx, %r13d + movl %r14d, %ecx + vpxor %xmm6, %xmm8, %xmm8 + # rnd_0: 5 - 5 + andl %eax, %ebx + rorl $9, %ecx + xorl %r14d, %ecx + xorl %r15d, %ebx + vpshufb %xmm11, %xmm8, %xmm8 + # rnd_0: 6 - 6 + rorl $11, %ecx + addl %r13d, %r9d + xorl %r14d, %ecx + addl %ebx, %r13d + vpaddd %xmm8, %xmm4, %xmm4 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r9d, %edx + addl %ecx, %r13d + # rnd_1: 0 - 0 + rorl $14, %edx + vpshufd $0x50, %xmm4, %xmm6 + # rnd_1: 1 - 1 + movl %r14d, %ebx + movl %r10d, %ecx + addl 44(%rsp), %r12d + xorl %r11d, %ecx + vpsrlq $0x11, %xmm6, %xmm8 + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 2 - 3 + xorl %r9d, %edx + andl %r9d, %ecx + rorl $5, %edx + xorl %r11d, %ecx + xorl %r9d, %edx + addl %ecx, %r12d + vpsrld $10, %xmm6, %xmm9 + vpxor %xmm8, %xmm7, %xmm8 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r13d, %ebx + addl %edx, %r12d + movl %r13d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r13d, %ecx + xorl %r14d, %eax + vpxor %xmm9, %xmm8, %xmm9 + # rnd_1: 6 - 6 + rorl $11, %ecx + addl %r12d, %r8d + xorl %r13d, %ecx + addl %eax, %r12d + vpshufb %xmm12, %xmm9, %xmm9 + # rnd_1: 7 - 7 + rorl $2, %ecx + movl %r8d, %edx + addl %ecx, %r12d + vpaddd %xmm4, %xmm9, %xmm2 + # msg_sched done: 8-11 + # msg_sched: 12-15 + # rnd_0: 0 - 0 + rorl $14, %edx + vpalignr $4, %xmm3, %xmm0, %xmm5 + vpalignr $4, %xmm1, %xmm2, %xmm4 + # rnd_0: 1 - 2 + movl %r13d, %eax + movl %r9d, %ecx + addl 48(%rsp), %r11d + xorl %r10d, %ecx + xorl %r8d, %edx + andl %r8d, %ecx + vpsrld $7, %xmm5, %xmm6 + vpslld $25, %xmm5, %xmm7 + # rnd_0: 3 - 4 + rorl $5, %edx + xorl %r10d, %ecx + xorl %r8d, %edx + addl %ecx, %r11d + rorl $6, %edx + xorl %r12d, %eax + addl %edx, %r11d + movl %r12d, %ecx + vpsrld $18, %xmm5, %xmm8 + vpslld $14, %xmm5, %xmm9 + # rnd_0: 5 - 6 + andl %eax, %ebx + rorl $9, %ecx + xorl %r12d, %ecx + xorl %r13d, %ebx + rorl $11, %ecx + addl %r11d, %r15d + xorl %r12d, %ecx + addl %ebx, %r11d + vpor %xmm6, %xmm7, %xmm6 + vpor %xmm8, %xmm9, %xmm8 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r15d, %edx + addl %ecx, %r11d + # rnd_1: 0 - 1 + rorl $14, %edx + movl %r12d, %ebx + movl %r8d, %ecx + addl 52(%rsp), %r10d + xorl %r9d, %ecx + vpsrld $3, %xmm5, %xmm9 + vpxor %xmm6, %xmm8, %xmm6 + # rnd_1: 2 - 3 + xorl %r15d, %edx + andl %r15d, %ecx + rorl $5, %edx + xorl %r9d, %ecx + xorl %r15d, %edx + addl %ecx, %r10d + vpxor %xmm6, %xmm9, %xmm5 + vpshufd $0xfa, %xmm2, %xmm6 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r11d, %ebx + addl %edx, %r10d + movl %r11d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r11d, %ecx + xorl %r12d, %eax + vpsrld $10, %xmm6, %xmm8 + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 6 - 7 + rorl $11, %ecx + addl %r10d, %r14d + xorl %r11d, %ecx + addl %eax, %r10d + rorl $2, %ecx + movl %r14d, %edx + addl %ecx, %r10d + # rnd_0: 0 - 0 + rorl $14, %edx + vpsrlq $0x11, %xmm6, %xmm6 + vpaddd %xmm3, %xmm4, %xmm4 + # rnd_0: 1 - 3 + movl %r11d, %eax + movl %r15d, %ecx + addl 56(%rsp), %r9d + xorl %r8d, %ecx + xorl %r14d, %edx + andl %r14d, %ecx + rorl $5, %edx + xorl %r8d, %ecx + xorl %r14d, %edx + addl %ecx, %r9d + vpxor %xmm6, %xmm7, %xmm6 + vpaddd %xmm5, %xmm4, %xmm4 + # rnd_0: 4 - 4 + rorl $6, %edx + xorl %r10d, %eax + addl %edx, %r9d + movl %r10d, %ecx + vpxor %xmm6, %xmm8, %xmm8 + # rnd_0: 5 - 5 + andl %eax, %ebx + rorl $9, %ecx + xorl %r10d, %ecx + xorl %r11d, %ebx + vpshufb %xmm11, %xmm8, %xmm8 + # rnd_0: 6 - 6 + rorl $11, %ecx + addl %r9d, %r13d + xorl %r10d, %ecx + addl %ebx, %r9d + vpaddd %xmm8, %xmm4, %xmm4 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r13d, %edx + addl %ecx, %r9d + # rnd_1: 0 - 0 + rorl $14, %edx + vpshufd $0x50, %xmm4, %xmm6 + # rnd_1: 1 - 1 + movl %r10d, %ebx + movl %r14d, %ecx + addl 60(%rsp), %r8d + xorl %r15d, %ecx + vpsrlq $0x11, %xmm6, %xmm8 + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 2 - 3 + xorl %r13d, %edx + andl %r13d, %ecx + rorl $5, %edx + xorl %r15d, %ecx + xorl %r13d, %edx + addl %ecx, %r8d + vpsrld $10, %xmm6, %xmm9 + vpxor %xmm8, %xmm7, %xmm8 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r9d, %ebx + addl %edx, %r8d + movl %r9d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r9d, %ecx + xorl %r10d, %eax + vpxor %xmm9, %xmm8, %xmm9 + # rnd_1: 6 - 6 + rorl $11, %ecx + addl %r8d, %r12d + xorl %r9d, %ecx + addl %eax, %r8d + vpshufb %xmm12, %xmm9, %xmm9 + # rnd_1: 7 - 7 + rorl $2, %ecx + movl %r12d, %edx + addl %ecx, %r8d + vpaddd %xmm4, %xmm9, %xmm3 + # msg_sched done: 12-15 + # set_w_k_xfer_4: 8 + vpaddd 128+L_avx1_sha256_k(%rip), %xmm0, %xmm4 + vpaddd 144+L_avx1_sha256_k(%rip), %xmm1, %xmm5 + vmovdqu %xmm4, (%rsp) + vmovdqu %xmm5, 16(%rsp) + vpaddd 160+L_avx1_sha256_k(%rip), %xmm2, %xmm6 + vpaddd 176+L_avx1_sha256_k(%rip), %xmm3, %xmm7 + vmovdqu %xmm6, 32(%rsp) + vmovdqu %xmm7, 48(%rsp) + # msg_sched: 0-3 + # rnd_0: 0 - 0 + rorl $14, %edx + vpalignr $4, %xmm0, %xmm1, %xmm5 + vpalignr $4, %xmm2, %xmm3, %xmm4 + # rnd_0: 1 - 2 + movl %r9d, %eax + movl %r13d, %ecx + addl (%rsp), %r15d + xorl %r14d, %ecx + xorl %r12d, %edx + andl %r12d, %ecx + vpsrld $7, %xmm5, %xmm6 + vpslld $25, %xmm5, %xmm7 + # rnd_0: 3 - 4 + rorl $5, %edx + xorl %r14d, %ecx + xorl %r12d, %edx + addl %ecx, %r15d + rorl $6, %edx + xorl %r8d, %eax + addl %edx, %r15d + movl %r8d, %ecx + vpsrld $18, %xmm5, %xmm8 + vpslld $14, %xmm5, %xmm9 + # rnd_0: 5 - 6 + andl %eax, %ebx + rorl $9, %ecx + xorl %r8d, %ecx + xorl %r9d, %ebx + rorl $11, %ecx + addl %r15d, %r11d + xorl %r8d, %ecx + addl %ebx, %r15d + vpor %xmm6, %xmm7, %xmm6 + vpor %xmm8, %xmm9, %xmm8 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r11d, %edx + addl %ecx, %r15d + # rnd_1: 0 - 1 + rorl $14, %edx + movl %r8d, %ebx + movl %r12d, %ecx + addl 4(%rsp), %r14d + xorl %r13d, %ecx + vpsrld $3, %xmm5, %xmm9 + vpxor %xmm6, %xmm8, %xmm6 + # rnd_1: 2 - 3 + xorl %r11d, %edx + andl %r11d, %ecx + rorl $5, %edx + xorl %r13d, %ecx + xorl %r11d, %edx + addl %ecx, %r14d + vpxor %xmm6, %xmm9, %xmm5 + vpshufd $0xfa, %xmm3, %xmm6 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r15d, %ebx + addl %edx, %r14d + movl %r15d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r15d, %ecx + xorl %r8d, %eax + vpsrld $10, %xmm6, %xmm8 + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 6 - 7 + rorl $11, %ecx + addl %r14d, %r10d + xorl %r15d, %ecx + addl %eax, %r14d + rorl $2, %ecx + movl %r10d, %edx + addl %ecx, %r14d + # rnd_0: 0 - 0 + rorl $14, %edx + vpsrlq $0x11, %xmm6, %xmm6 + vpaddd %xmm0, %xmm4, %xmm4 + # rnd_0: 1 - 3 + movl %r15d, %eax + movl %r11d, %ecx + addl 8(%rsp), %r13d + xorl %r12d, %ecx + xorl %r10d, %edx + andl %r10d, %ecx + rorl $5, %edx + xorl %r12d, %ecx + xorl %r10d, %edx + addl %ecx, %r13d + vpxor %xmm6, %xmm7, %xmm6 + vpaddd %xmm5, %xmm4, %xmm4 + # rnd_0: 4 - 4 + rorl $6, %edx + xorl %r14d, %eax + addl %edx, %r13d + movl %r14d, %ecx + vpxor %xmm6, %xmm8, %xmm8 + # rnd_0: 5 - 5 + andl %eax, %ebx + rorl $9, %ecx + xorl %r14d, %ecx + xorl %r15d, %ebx + vpshufb %xmm11, %xmm8, %xmm8 + # rnd_0: 6 - 6 + rorl $11, %ecx + addl %r13d, %r9d + xorl %r14d, %ecx + addl %ebx, %r13d + vpaddd %xmm8, %xmm4, %xmm4 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r9d, %edx + addl %ecx, %r13d + # rnd_1: 0 - 0 + rorl $14, %edx + vpshufd $0x50, %xmm4, %xmm6 + # rnd_1: 1 - 1 + movl %r14d, %ebx + movl %r10d, %ecx + addl 12(%rsp), %r12d + xorl %r11d, %ecx + vpsrlq $0x11, %xmm6, %xmm8 + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 2 - 3 + xorl %r9d, %edx + andl %r9d, %ecx + rorl $5, %edx + xorl %r11d, %ecx + xorl %r9d, %edx + addl %ecx, %r12d + vpsrld $10, %xmm6, %xmm9 + vpxor %xmm8, %xmm7, %xmm8 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r13d, %ebx + addl %edx, %r12d + movl %r13d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r13d, %ecx + xorl %r14d, %eax + vpxor %xmm9, %xmm8, %xmm9 + # rnd_1: 6 - 6 + rorl $11, %ecx + addl %r12d, %r8d + xorl %r13d, %ecx + addl %eax, %r12d + vpshufb %xmm12, %xmm9, %xmm9 + # rnd_1: 7 - 7 + rorl $2, %ecx + movl %r8d, %edx + addl %ecx, %r12d + vpaddd %xmm4, %xmm9, %xmm0 + # msg_sched done: 0-3 + # msg_sched: 4-7 + # rnd_0: 0 - 0 + rorl $14, %edx + vpalignr $4, %xmm1, %xmm2, %xmm5 + vpalignr $4, %xmm3, %xmm0, %xmm4 + # rnd_0: 1 - 2 + movl %r13d, %eax + movl %r9d, %ecx + addl 16(%rsp), %r11d + xorl %r10d, %ecx + xorl %r8d, %edx + andl %r8d, %ecx + vpsrld $7, %xmm5, %xmm6 + vpslld $25, %xmm5, %xmm7 + # rnd_0: 3 - 4 + rorl $5, %edx + xorl %r10d, %ecx + xorl %r8d, %edx + addl %ecx, %r11d + rorl $6, %edx + xorl %r12d, %eax + addl %edx, %r11d + movl %r12d, %ecx + vpsrld $18, %xmm5, %xmm8 + vpslld $14, %xmm5, %xmm9 + # rnd_0: 5 - 6 + andl %eax, %ebx + rorl $9, %ecx + xorl %r12d, %ecx + xorl %r13d, %ebx + rorl $11, %ecx + addl %r11d, %r15d + xorl %r12d, %ecx + addl %ebx, %r11d + vpor %xmm6, %xmm7, %xmm6 + vpor %xmm8, %xmm9, %xmm8 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r15d, %edx + addl %ecx, %r11d + # rnd_1: 0 - 1 + rorl $14, %edx + movl %r12d, %ebx + movl %r8d, %ecx + addl 20(%rsp), %r10d + xorl %r9d, %ecx + vpsrld $3, %xmm5, %xmm9 + vpxor %xmm6, %xmm8, %xmm6 + # rnd_1: 2 - 3 + xorl %r15d, %edx + andl %r15d, %ecx + rorl $5, %edx + xorl %r9d, %ecx + xorl %r15d, %edx + addl %ecx, %r10d + vpxor %xmm6, %xmm9, %xmm5 + vpshufd $0xfa, %xmm0, %xmm6 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r11d, %ebx + addl %edx, %r10d + movl %r11d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r11d, %ecx + xorl %r12d, %eax + vpsrld $10, %xmm6, %xmm8 + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 6 - 7 + rorl $11, %ecx + addl %r10d, %r14d + xorl %r11d, %ecx + addl %eax, %r10d + rorl $2, %ecx + movl %r14d, %edx + addl %ecx, %r10d + # rnd_0: 0 - 0 + rorl $14, %edx + vpsrlq $0x11, %xmm6, %xmm6 + vpaddd %xmm1, %xmm4, %xmm4 + # rnd_0: 1 - 3 + movl %r11d, %eax + movl %r15d, %ecx + addl 24(%rsp), %r9d + xorl %r8d, %ecx + xorl %r14d, %edx + andl %r14d, %ecx + rorl $5, %edx + xorl %r8d, %ecx + xorl %r14d, %edx + addl %ecx, %r9d + vpxor %xmm6, %xmm7, %xmm6 + vpaddd %xmm5, %xmm4, %xmm4 + # rnd_0: 4 - 4 + rorl $6, %edx + xorl %r10d, %eax + addl %edx, %r9d + movl %r10d, %ecx + vpxor %xmm6, %xmm8, %xmm8 + # rnd_0: 5 - 5 + andl %eax, %ebx + rorl $9, %ecx + xorl %r10d, %ecx + xorl %r11d, %ebx + vpshufb %xmm11, %xmm8, %xmm8 + # rnd_0: 6 - 6 + rorl $11, %ecx + addl %r9d, %r13d + xorl %r10d, %ecx + addl %ebx, %r9d + vpaddd %xmm8, %xmm4, %xmm4 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r13d, %edx + addl %ecx, %r9d + # rnd_1: 0 - 0 + rorl $14, %edx + vpshufd $0x50, %xmm4, %xmm6 + # rnd_1: 1 - 1 + movl %r10d, %ebx + movl %r14d, %ecx + addl 28(%rsp), %r8d + xorl %r15d, %ecx + vpsrlq $0x11, %xmm6, %xmm8 + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 2 - 3 + xorl %r13d, %edx + andl %r13d, %ecx + rorl $5, %edx + xorl %r15d, %ecx + xorl %r13d, %edx + addl %ecx, %r8d + vpsrld $10, %xmm6, %xmm9 + vpxor %xmm8, %xmm7, %xmm8 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r9d, %ebx + addl %edx, %r8d + movl %r9d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r9d, %ecx + xorl %r10d, %eax + vpxor %xmm9, %xmm8, %xmm9 + # rnd_1: 6 - 6 + rorl $11, %ecx + addl %r8d, %r12d + xorl %r9d, %ecx + addl %eax, %r8d + vpshufb %xmm12, %xmm9, %xmm9 + # rnd_1: 7 - 7 + rorl $2, %ecx + movl %r12d, %edx + addl %ecx, %r8d + vpaddd %xmm4, %xmm9, %xmm1 + # msg_sched done: 4-7 + # msg_sched: 8-11 + # rnd_0: 0 - 0 + rorl $14, %edx + vpalignr $4, %xmm2, %xmm3, %xmm5 + vpalignr $4, %xmm0, %xmm1, %xmm4 + # rnd_0: 1 - 2 + movl %r9d, %eax + movl %r13d, %ecx + addl 32(%rsp), %r15d + xorl %r14d, %ecx + xorl %r12d, %edx + andl %r12d, %ecx + vpsrld $7, %xmm5, %xmm6 + vpslld $25, %xmm5, %xmm7 + # rnd_0: 3 - 4 + rorl $5, %edx + xorl %r14d, %ecx + xorl %r12d, %edx + addl %ecx, %r15d + rorl $6, %edx + xorl %r8d, %eax + addl %edx, %r15d + movl %r8d, %ecx + vpsrld $18, %xmm5, %xmm8 + vpslld $14, %xmm5, %xmm9 + # rnd_0: 5 - 6 + andl %eax, %ebx + rorl $9, %ecx + xorl %r8d, %ecx + xorl %r9d, %ebx + rorl $11, %ecx + addl %r15d, %r11d + xorl %r8d, %ecx + addl %ebx, %r15d + vpor %xmm6, %xmm7, %xmm6 + vpor %xmm8, %xmm9, %xmm8 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r11d, %edx + addl %ecx, %r15d + # rnd_1: 0 - 1 + rorl $14, %edx + movl %r8d, %ebx + movl %r12d, %ecx + addl 36(%rsp), %r14d + xorl %r13d, %ecx + vpsrld $3, %xmm5, %xmm9 + vpxor %xmm6, %xmm8, %xmm6 + # rnd_1: 2 - 3 + xorl %r11d, %edx + andl %r11d, %ecx + rorl $5, %edx + xorl %r13d, %ecx + xorl %r11d, %edx + addl %ecx, %r14d + vpxor %xmm6, %xmm9, %xmm5 + vpshufd $0xfa, %xmm1, %xmm6 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r15d, %ebx + addl %edx, %r14d + movl %r15d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r15d, %ecx + xorl %r8d, %eax + vpsrld $10, %xmm6, %xmm8 + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 6 - 7 + rorl $11, %ecx + addl %r14d, %r10d + xorl %r15d, %ecx + addl %eax, %r14d + rorl $2, %ecx + movl %r10d, %edx + addl %ecx, %r14d + # rnd_0: 0 - 0 + rorl $14, %edx + vpsrlq $0x11, %xmm6, %xmm6 + vpaddd %xmm2, %xmm4, %xmm4 + # rnd_0: 1 - 3 + movl %r15d, %eax + movl %r11d, %ecx + addl 40(%rsp), %r13d + xorl %r12d, %ecx + xorl %r10d, %edx + andl %r10d, %ecx + rorl $5, %edx + xorl %r12d, %ecx + xorl %r10d, %edx + addl %ecx, %r13d + vpxor %xmm6, %xmm7, %xmm6 + vpaddd %xmm5, %xmm4, %xmm4 + # rnd_0: 4 - 4 + rorl $6, %edx + xorl %r14d, %eax + addl %edx, %r13d + movl %r14d, %ecx + vpxor %xmm6, %xmm8, %xmm8 + # rnd_0: 5 - 5 + andl %eax, %ebx + rorl $9, %ecx + xorl %r14d, %ecx + xorl %r15d, %ebx + vpshufb %xmm11, %xmm8, %xmm8 + # rnd_0: 6 - 6 + rorl $11, %ecx + addl %r13d, %r9d + xorl %r14d, %ecx + addl %ebx, %r13d + vpaddd %xmm8, %xmm4, %xmm4 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r9d, %edx + addl %ecx, %r13d + # rnd_1: 0 - 0 + rorl $14, %edx + vpshufd $0x50, %xmm4, %xmm6 + # rnd_1: 1 - 1 + movl %r14d, %ebx + movl %r10d, %ecx + addl 44(%rsp), %r12d + xorl %r11d, %ecx + vpsrlq $0x11, %xmm6, %xmm8 + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 2 - 3 + xorl %r9d, %edx + andl %r9d, %ecx + rorl $5, %edx + xorl %r11d, %ecx + xorl %r9d, %edx + addl %ecx, %r12d + vpsrld $10, %xmm6, %xmm9 + vpxor %xmm8, %xmm7, %xmm8 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r13d, %ebx + addl %edx, %r12d + movl %r13d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r13d, %ecx + xorl %r14d, %eax + vpxor %xmm9, %xmm8, %xmm9 + # rnd_1: 6 - 6 + rorl $11, %ecx + addl %r12d, %r8d + xorl %r13d, %ecx + addl %eax, %r12d + vpshufb %xmm12, %xmm9, %xmm9 + # rnd_1: 7 - 7 + rorl $2, %ecx + movl %r8d, %edx + addl %ecx, %r12d + vpaddd %xmm4, %xmm9, %xmm2 + # msg_sched done: 8-11 + # msg_sched: 12-15 + # rnd_0: 0 - 0 + rorl $14, %edx + vpalignr $4, %xmm3, %xmm0, %xmm5 + vpalignr $4, %xmm1, %xmm2, %xmm4 + # rnd_0: 1 - 2 + movl %r13d, %eax + movl %r9d, %ecx + addl 48(%rsp), %r11d + xorl %r10d, %ecx + xorl %r8d, %edx + andl %r8d, %ecx + vpsrld $7, %xmm5, %xmm6 + vpslld $25, %xmm5, %xmm7 + # rnd_0: 3 - 4 + rorl $5, %edx + xorl %r10d, %ecx + xorl %r8d, %edx + addl %ecx, %r11d + rorl $6, %edx + xorl %r12d, %eax + addl %edx, %r11d + movl %r12d, %ecx + vpsrld $18, %xmm5, %xmm8 + vpslld $14, %xmm5, %xmm9 + # rnd_0: 5 - 6 + andl %eax, %ebx + rorl $9, %ecx + xorl %r12d, %ecx + xorl %r13d, %ebx + rorl $11, %ecx + addl %r11d, %r15d + xorl %r12d, %ecx + addl %ebx, %r11d + vpor %xmm6, %xmm7, %xmm6 + vpor %xmm8, %xmm9, %xmm8 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r15d, %edx + addl %ecx, %r11d + # rnd_1: 0 - 1 + rorl $14, %edx + movl %r12d, %ebx + movl %r8d, %ecx + addl 52(%rsp), %r10d + xorl %r9d, %ecx + vpsrld $3, %xmm5, %xmm9 + vpxor %xmm6, %xmm8, %xmm6 + # rnd_1: 2 - 3 + xorl %r15d, %edx + andl %r15d, %ecx + rorl $5, %edx + xorl %r9d, %ecx + xorl %r15d, %edx + addl %ecx, %r10d + vpxor %xmm6, %xmm9, %xmm5 + vpshufd $0xfa, %xmm2, %xmm6 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r11d, %ebx + addl %edx, %r10d + movl %r11d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r11d, %ecx + xorl %r12d, %eax + vpsrld $10, %xmm6, %xmm8 + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 6 - 7 + rorl $11, %ecx + addl %r10d, %r14d + xorl %r11d, %ecx + addl %eax, %r10d + rorl $2, %ecx + movl %r14d, %edx + addl %ecx, %r10d + # rnd_0: 0 - 0 + rorl $14, %edx + vpsrlq $0x11, %xmm6, %xmm6 + vpaddd %xmm3, %xmm4, %xmm4 + # rnd_0: 1 - 3 + movl %r11d, %eax + movl %r15d, %ecx + addl 56(%rsp), %r9d + xorl %r8d, %ecx + xorl %r14d, %edx + andl %r14d, %ecx + rorl $5, %edx + xorl %r8d, %ecx + xorl %r14d, %edx + addl %ecx, %r9d + vpxor %xmm6, %xmm7, %xmm6 + vpaddd %xmm5, %xmm4, %xmm4 + # rnd_0: 4 - 4 + rorl $6, %edx + xorl %r10d, %eax + addl %edx, %r9d + movl %r10d, %ecx + vpxor %xmm6, %xmm8, %xmm8 + # rnd_0: 5 - 5 + andl %eax, %ebx + rorl $9, %ecx + xorl %r10d, %ecx + xorl %r11d, %ebx + vpshufb %xmm11, %xmm8, %xmm8 + # rnd_0: 6 - 6 + rorl $11, %ecx + addl %r9d, %r13d + xorl %r10d, %ecx + addl %ebx, %r9d + vpaddd %xmm8, %xmm4, %xmm4 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r13d, %edx + addl %ecx, %r9d + # rnd_1: 0 - 0 + rorl $14, %edx + vpshufd $0x50, %xmm4, %xmm6 + # rnd_1: 1 - 1 + movl %r10d, %ebx + movl %r14d, %ecx + addl 60(%rsp), %r8d + xorl %r15d, %ecx + vpsrlq $0x11, %xmm6, %xmm8 + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 2 - 3 + xorl %r13d, %edx + andl %r13d, %ecx + rorl $5, %edx + xorl %r15d, %ecx + xorl %r13d, %edx + addl %ecx, %r8d + vpsrld $10, %xmm6, %xmm9 + vpxor %xmm8, %xmm7, %xmm8 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r9d, %ebx + addl %edx, %r8d + movl %r9d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r9d, %ecx + xorl %r10d, %eax + vpxor %xmm9, %xmm8, %xmm9 + # rnd_1: 6 - 6 + rorl $11, %ecx + addl %r8d, %r12d + xorl %r9d, %ecx + addl %eax, %r8d + vpshufb %xmm12, %xmm9, %xmm9 + # rnd_1: 7 - 7 + rorl $2, %ecx + movl %r12d, %edx + addl %ecx, %r8d + vpaddd %xmm4, %xmm9, %xmm3 + # msg_sched done: 12-15 + # set_w_k_xfer_4: 12 + vpaddd 192+L_avx1_sha256_k(%rip), %xmm0, %xmm4 + vpaddd 208+L_avx1_sha256_k(%rip), %xmm1, %xmm5 + vmovdqu %xmm4, (%rsp) + vmovdqu %xmm5, 16(%rsp) + vpaddd 224+L_avx1_sha256_k(%rip), %xmm2, %xmm6 + vpaddd 240+L_avx1_sha256_k(%rip), %xmm3, %xmm7 + vmovdqu %xmm6, 32(%rsp) + vmovdqu %xmm7, 48(%rsp) + # rnd_all_4: 0-3 + addl (%rsp), %r15d + movl %r13d, %ecx + movl %r9d, %eax + xorl %r14d, %ecx + rorl $14, %edx + andl %r12d, %ecx + xorl %r12d, %edx + xorl %r14d, %ecx + rorl $5, %edx + addl %ecx, %r15d + xorl %r12d, %edx + xorl %r8d, %eax + rorl $6, %edx + movl %r8d, %ecx + addl %edx, %r15d + rorl $9, %ecx + andl %eax, %ebx + xorl %r8d, %ecx + xorl %r9d, %ebx + rorl $11, %ecx + addl %r15d, %r11d + xorl %r8d, %ecx + addl %ebx, %r15d + rorl $2, %ecx + movl %r11d, %edx + addl %ecx, %r15d + addl 4(%rsp), %r14d + movl %r12d, %ecx + movl %r8d, %ebx + xorl %r13d, %ecx + rorl $14, %edx + andl %r11d, %ecx + xorl %r11d, %edx + xorl %r13d, %ecx + rorl $5, %edx + addl %ecx, %r14d + xorl %r11d, %edx + xorl %r15d, %ebx + rorl $6, %edx + movl %r15d, %ecx + addl %edx, %r14d + rorl $9, %ecx + andl %ebx, %eax + xorl %r15d, %ecx + xorl %r8d, %eax + rorl $11, %ecx + addl %r14d, %r10d + xorl %r15d, %ecx + addl %eax, %r14d + rorl $2, %ecx + movl %r10d, %edx + addl %ecx, %r14d + addl 8(%rsp), %r13d + movl %r11d, %ecx + movl %r15d, %eax + xorl %r12d, %ecx + rorl $14, %edx + andl %r10d, %ecx + xorl %r10d, %edx + xorl %r12d, %ecx + rorl $5, %edx + addl %ecx, %r13d + xorl %r10d, %edx + xorl %r14d, %eax + rorl $6, %edx + movl %r14d, %ecx + addl %edx, %r13d + rorl $9, %ecx + andl %eax, %ebx + xorl %r14d, %ecx + xorl %r15d, %ebx + rorl $11, %ecx + addl %r13d, %r9d + xorl %r14d, %ecx + addl %ebx, %r13d + rorl $2, %ecx + movl %r9d, %edx + addl %ecx, %r13d + addl 12(%rsp), %r12d + movl %r10d, %ecx + movl %r14d, %ebx + xorl %r11d, %ecx + rorl $14, %edx + andl %r9d, %ecx + xorl %r9d, %edx + xorl %r11d, %ecx + rorl $5, %edx + addl %ecx, %r12d + xorl %r9d, %edx + xorl %r13d, %ebx + rorl $6, %edx + movl %r13d, %ecx + addl %edx, %r12d + rorl $9, %ecx + andl %ebx, %eax + xorl %r13d, %ecx + xorl %r14d, %eax + rorl $11, %ecx + addl %r12d, %r8d + xorl %r13d, %ecx + addl %eax, %r12d + rorl $2, %ecx + movl %r8d, %edx + addl %ecx, %r12d + # rnd_all_4: 1-4 + addl 16(%rsp), %r11d + movl %r9d, %ecx + movl %r13d, %eax + xorl %r10d, %ecx + rorl $14, %edx + andl %r8d, %ecx + xorl %r8d, %edx + xorl %r10d, %ecx + rorl $5, %edx + addl %ecx, %r11d + xorl %r8d, %edx + xorl %r12d, %eax + rorl $6, %edx + movl %r12d, %ecx + addl %edx, %r11d + rorl $9, %ecx + andl %eax, %ebx + xorl %r12d, %ecx + xorl %r13d, %ebx + rorl $11, %ecx + addl %r11d, %r15d + xorl %r12d, %ecx + addl %ebx, %r11d + rorl $2, %ecx + movl %r15d, %edx + addl %ecx, %r11d + addl 20(%rsp), %r10d + movl %r8d, %ecx + movl %r12d, %ebx + xorl %r9d, %ecx + rorl $14, %edx + andl %r15d, %ecx + xorl %r15d, %edx + xorl %r9d, %ecx + rorl $5, %edx + addl %ecx, %r10d + xorl %r15d, %edx + xorl %r11d, %ebx + rorl $6, %edx + movl %r11d, %ecx + addl %edx, %r10d + rorl $9, %ecx + andl %ebx, %eax + xorl %r11d, %ecx + xorl %r12d, %eax + rorl $11, %ecx + addl %r10d, %r14d + xorl %r11d, %ecx + addl %eax, %r10d + rorl $2, %ecx + movl %r14d, %edx + addl %ecx, %r10d + addl 24(%rsp), %r9d + movl %r15d, %ecx + movl %r11d, %eax + xorl %r8d, %ecx + rorl $14, %edx + andl %r14d, %ecx + xorl %r14d, %edx + xorl %r8d, %ecx + rorl $5, %edx + addl %ecx, %r9d + xorl %r14d, %edx + xorl %r10d, %eax + rorl $6, %edx + movl %r10d, %ecx + addl %edx, %r9d + rorl $9, %ecx + andl %eax, %ebx + xorl %r10d, %ecx + xorl %r11d, %ebx + rorl $11, %ecx + addl %r9d, %r13d + xorl %r10d, %ecx + addl %ebx, %r9d + rorl $2, %ecx + movl %r13d, %edx + addl %ecx, %r9d + addl 28(%rsp), %r8d + movl %r14d, %ecx + movl %r10d, %ebx + xorl %r15d, %ecx + rorl $14, %edx + andl %r13d, %ecx + xorl %r13d, %edx + xorl %r15d, %ecx + rorl $5, %edx + addl %ecx, %r8d + xorl %r13d, %edx + xorl %r9d, %ebx + rorl $6, %edx + movl %r9d, %ecx + addl %edx, %r8d + rorl $9, %ecx + andl %ebx, %eax + xorl %r9d, %ecx + xorl %r10d, %eax + rorl $11, %ecx + addl %r8d, %r12d + xorl %r9d, %ecx + addl %eax, %r8d + rorl $2, %ecx + movl %r12d, %edx + addl %ecx, %r8d + # rnd_all_4: 2-5 + addl 32(%rsp), %r15d + movl %r13d, %ecx + movl %r9d, %eax + xorl %r14d, %ecx + rorl $14, %edx + andl %r12d, %ecx + xorl %r12d, %edx + xorl %r14d, %ecx + rorl $5, %edx + addl %ecx, %r15d + xorl %r12d, %edx + xorl %r8d, %eax + rorl $6, %edx + movl %r8d, %ecx + addl %edx, %r15d + rorl $9, %ecx + andl %eax, %ebx + xorl %r8d, %ecx + xorl %r9d, %ebx + rorl $11, %ecx + addl %r15d, %r11d + xorl %r8d, %ecx + addl %ebx, %r15d + rorl $2, %ecx + movl %r11d, %edx + addl %ecx, %r15d + addl 36(%rsp), %r14d + movl %r12d, %ecx + movl %r8d, %ebx + xorl %r13d, %ecx + rorl $14, %edx + andl %r11d, %ecx + xorl %r11d, %edx + xorl %r13d, %ecx + rorl $5, %edx + addl %ecx, %r14d + xorl %r11d, %edx + xorl %r15d, %ebx + rorl $6, %edx + movl %r15d, %ecx + addl %edx, %r14d + rorl $9, %ecx + andl %ebx, %eax + xorl %r15d, %ecx + xorl %r8d, %eax + rorl $11, %ecx + addl %r14d, %r10d + xorl %r15d, %ecx + addl %eax, %r14d + rorl $2, %ecx + movl %r10d, %edx + addl %ecx, %r14d + addl 40(%rsp), %r13d + movl %r11d, %ecx + movl %r15d, %eax + xorl %r12d, %ecx + rorl $14, %edx + andl %r10d, %ecx + xorl %r10d, %edx + xorl %r12d, %ecx + rorl $5, %edx + addl %ecx, %r13d + xorl %r10d, %edx + xorl %r14d, %eax + rorl $6, %edx + movl %r14d, %ecx + addl %edx, %r13d + rorl $9, %ecx + andl %eax, %ebx + xorl %r14d, %ecx + xorl %r15d, %ebx + rorl $11, %ecx + addl %r13d, %r9d + xorl %r14d, %ecx + addl %ebx, %r13d + rorl $2, %ecx + movl %r9d, %edx + addl %ecx, %r13d + addl 44(%rsp), %r12d + movl %r10d, %ecx + movl %r14d, %ebx + xorl %r11d, %ecx + rorl $14, %edx + andl %r9d, %ecx + xorl %r9d, %edx + xorl %r11d, %ecx + rorl $5, %edx + addl %ecx, %r12d + xorl %r9d, %edx + xorl %r13d, %ebx + rorl $6, %edx + movl %r13d, %ecx + addl %edx, %r12d + rorl $9, %ecx + andl %ebx, %eax + xorl %r13d, %ecx + xorl %r14d, %eax + rorl $11, %ecx + addl %r12d, %r8d + xorl %r13d, %ecx + addl %eax, %r12d + rorl $2, %ecx + movl %r8d, %edx + addl %ecx, %r12d + # rnd_all_4: 3-6 + addl 48(%rsp), %r11d + movl %r9d, %ecx + movl %r13d, %eax + xorl %r10d, %ecx + rorl $14, %edx + andl %r8d, %ecx + xorl %r8d, %edx + xorl %r10d, %ecx + rorl $5, %edx + addl %ecx, %r11d + xorl %r8d, %edx + xorl %r12d, %eax + rorl $6, %edx + movl %r12d, %ecx + addl %edx, %r11d + rorl $9, %ecx + andl %eax, %ebx + xorl %r12d, %ecx + xorl %r13d, %ebx + rorl $11, %ecx + addl %r11d, %r15d + xorl %r12d, %ecx + addl %ebx, %r11d + rorl $2, %ecx + movl %r15d, %edx + addl %ecx, %r11d + addl 52(%rsp), %r10d + movl %r8d, %ecx + movl %r12d, %ebx + xorl %r9d, %ecx + rorl $14, %edx + andl %r15d, %ecx + xorl %r15d, %edx + xorl %r9d, %ecx + rorl $5, %edx + addl %ecx, %r10d + xorl %r15d, %edx + xorl %r11d, %ebx + rorl $6, %edx + movl %r11d, %ecx + addl %edx, %r10d + rorl $9, %ecx + andl %ebx, %eax + xorl %r11d, %ecx + xorl %r12d, %eax + rorl $11, %ecx + addl %r10d, %r14d + xorl %r11d, %ecx + addl %eax, %r10d + rorl $2, %ecx + movl %r14d, %edx + addl %ecx, %r10d + addl 56(%rsp), %r9d + movl %r15d, %ecx + movl %r11d, %eax + xorl %r8d, %ecx + rorl $14, %edx + andl %r14d, %ecx + xorl %r14d, %edx + xorl %r8d, %ecx + rorl $5, %edx + addl %ecx, %r9d + xorl %r14d, %edx + xorl %r10d, %eax + rorl $6, %edx + movl %r10d, %ecx + addl %edx, %r9d + rorl $9, %ecx + andl %eax, %ebx + xorl %r10d, %ecx + xorl %r11d, %ebx + rorl $11, %ecx + addl %r9d, %r13d + xorl %r10d, %ecx + addl %ebx, %r9d + rorl $2, %ecx + movl %r13d, %edx + addl %ecx, %r9d + addl 60(%rsp), %r8d + movl %r14d, %ecx + movl %r10d, %ebx + xorl %r15d, %ecx + rorl $14, %edx + andl %r13d, %ecx + xorl %r13d, %edx + xorl %r15d, %ecx + rorl $5, %edx + addl %ecx, %r8d + xorl %r13d, %edx + xorl %r9d, %ebx + rorl $6, %edx + movl %r9d, %ecx + addl %edx, %r8d + rorl $9, %ecx + andl %ebx, %eax + xorl %r9d, %ecx + xorl %r10d, %eax + rorl $11, %ecx + addl %r8d, %r12d + xorl %r9d, %ecx + addl %eax, %r8d + rorl $2, %ecx + movl %r12d, %edx + addl %ecx, %r8d + addl (%rdi), %r8d + addl 4(%rdi), %r9d + addl 8(%rdi), %r10d + addl 12(%rdi), %r11d + addl 16(%rdi), %r12d + addl 20(%rdi), %r13d + addl 24(%rdi), %r14d + addl 28(%rdi), %r15d + addq $0x40, %rbp + subl $0x40, %esi + movl %r8d, (%rdi) + movl %r9d, 4(%rdi) + movl %r10d, 8(%rdi) + movl %r11d, 12(%rdi) + movl %r12d, 16(%rdi) + movl %r13d, 20(%rdi) + movl %r14d, 24(%rdi) + movl %r15d, 28(%rdi) + jnz L_sha256_len_avx1_start + xorq %rax, %rax + vzeroupper + addq $0x40, %rsp + popq %rbp + popq %r15 + popq %r14 + popq %r13 + popq %r12 + popq %rbx + repz retq +.size Transform_Sha256_AVX1_Len,.-Transform_Sha256_AVX1_Len +L_avx1_rorx_sha256_k: +.long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 +.long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 +.long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 +.long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 +.long 0xe49b69c1,0xefbe4786,0xfc19dc6,0x240ca1cc +.long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da +.long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 +.long 0xc6e00bf3,0xd5a79147,0x6ca6351,0x14292967 +.long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 +.long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 +.long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 +.long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 +.long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 +.long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 +.long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 +.long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 +.align 16 +L_avx1_rorx_sha256_shuf_00BA: +.quad 0xb0a090803020100, 0xffffffffffffffff +.align 16 +L_avx1_rorx_sha256_shuf_DC00: +.quad 0xffffffffffffffff, 0xb0a090803020100 +.align 16 +L_avx1_rorx_sha256_flip_mask: +.quad 0x405060700010203, 0xc0d0e0f08090a0b +.globl Transform_Sha256_AVX1_RORX +.type Transform_Sha256_AVX1_RORX,@function +.align 4 +Transform_Sha256_AVX1_RORX: + pushq %rbx + pushq %r12 + pushq %r13 + pushq %r14 + pushq %r15 + subq $0x40, %rsp + vmovdqa L_avx1_rorx_sha256_flip_mask(%rip), %xmm13 + vmovdqa L_avx1_rorx_sha256_shuf_00BA(%rip), %xmm11 + vmovdqa L_avx1_rorx_sha256_shuf_DC00(%rip), %xmm12 + leaq 32(%rdi), %rax + # X0, X1, X2, X3 = W[0..15] + vmovdqu (%rax), %xmm0 + vmovdqu 16(%rax), %xmm1 + vpshufb %xmm13, %xmm0, %xmm0 + vpshufb %xmm13, %xmm1, %xmm1 + vmovdqu 32(%rax), %xmm2 + vmovdqu 48(%rax), %xmm3 + vpshufb %xmm13, %xmm2, %xmm2 + vpshufb %xmm13, %xmm3, %xmm3 + movl (%rdi), %r8d + movl 4(%rdi), %r9d + movl 8(%rdi), %r10d + movl 12(%rdi), %r11d + movl 16(%rdi), %r12d + movl 20(%rdi), %r13d + movl 24(%rdi), %r14d + movl 28(%rdi), %r15d + # set_w_k_xfer_4: 0 + vpaddd 0+L_avx1_rorx_sha256_k(%rip), %xmm0, %xmm4 + vpaddd 16+L_avx1_rorx_sha256_k(%rip), %xmm1, %xmm5 + vmovdqu %xmm4, (%rsp) + vmovdqu %xmm5, 16(%rsp) + vpaddd 32+L_avx1_rorx_sha256_k(%rip), %xmm2, %xmm6 + vpaddd 48+L_avx1_rorx_sha256_k(%rip), %xmm3, %xmm7 + vmovdqu %xmm6, 32(%rsp) + vmovdqu %xmm7, 48(%rsp) + movl %r9d, %ebx + rorxl $6, %r12d, %edx + xorl %r10d, %ebx + # msg_sched: 0-3 + # rnd_0: 0 - 0 + movl %r13d, %eax + rorxl $11, %r12d, %ecx + addl (%rsp), %r15d + vpalignr $4, %xmm2, %xmm3, %xmm4 + vpalignr $4, %xmm0, %xmm1, %xmm5 + # rnd_0: 1 - 2 + xorl %edx, %ecx + xorl %r14d, %eax + rorxl $25, %r12d, %edx + andl %r12d, %eax + xorl %ecx, %edx + rorxl $13, %r8d, %ecx + vpsrld $7, %xmm5, %xmm6 + vpslld $25, %xmm5, %xmm7 + # rnd_0: 3 - 4 + addl %edx, %r15d + rorxl $2, %r8d, %edx + xorl %r14d, %eax + xorl %edx, %ecx + rorxl $22, %r8d, %edx + addl %eax, %r15d + vpsrld $3, %xmm5, %xmm8 + vpor %xmm6, %xmm7, %xmm7 + # rnd_0: 5 - 7 + xorl %ecx, %edx + movl %r9d, %eax + addl %r15d, %r11d + xorl %r8d, %eax + addl %edx, %r15d + andl %eax, %ebx + xorl %r9d, %ebx + rorxl $6, %r11d, %edx + addl %ebx, %r15d + # rnd_1: 0 - 0 + movl %r12d, %ebx + rorxl $11, %r11d, %ecx + addl 4(%rsp), %r14d + vpsrld $18, %xmm5, %xmm6 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r13d, %ebx + rorxl $25, %r11d, %edx + vpslld $14, %xmm5, %xmm5 + # rnd_1: 2 - 2 + andl %r11d, %ebx + xorl %ecx, %edx + rorxl $13, %r15d, %ecx + vpxor %xmm5, %xmm7, %xmm7 + # rnd_1: 3 - 3 + addl %edx, %r14d + rorxl $2, %r15d, %edx + xorl %r13d, %ebx + vpxor %xmm6, %xmm7, %xmm7 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r15d, %edx + addl %ebx, %r14d + vpshufd $0xfa, %xmm3, %xmm6 + # rnd_1: 5 - 5 + xorl %ecx, %edx + movl %r8d, %ebx + addl %r14d, %r10d + vpxor %xmm8, %xmm7, %xmm5 + # rnd_1: 6 - 6 + xorl %r15d, %ebx + addl %edx, %r14d + andl %ebx, %eax + vpsrld $10, %xmm6, %xmm8 + # rnd_1: 7 - 7 + xorl %r8d, %eax + rorxl $6, %r10d, %edx + addl %eax, %r14d + # rnd_0: 0 - 0 + movl %r11d, %eax + rorxl $11, %r10d, %ecx + addl 8(%rsp), %r13d + vpsrlq $19, %xmm6, %xmm7 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r12d, %eax + rorxl $25, %r10d, %edx + vpsrlq $0x11, %xmm6, %xmm6 + vpaddd %xmm0, %xmm4, %xmm4 + # rnd_0: 2 - 2 + andl %r10d, %eax + xorl %ecx, %edx + rorxl $13, %r14d, %ecx + vpaddd %xmm5, %xmm4, %xmm4 + # rnd_0: 3 - 3 + addl %edx, %r13d + rorxl $2, %r14d, %edx + xorl %r12d, %eax + vpxor %xmm7, %xmm6, %xmm6 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r14d, %edx + addl %eax, %r13d + vpxor %xmm6, %xmm8, %xmm8 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r15d, %eax + addl %r13d, %r9d + vpshufb %xmm11, %xmm8, %xmm8 + # rnd_0: 6 - 6 + xorl %r14d, %eax + addl %edx, %r13d + andl %eax, %ebx + vpaddd %xmm8, %xmm4, %xmm4 + # rnd_0: 7 - 7 + xorl %r15d, %ebx + rorxl $6, %r9d, %edx + addl %ebx, %r13d + # rnd_1: 0 - 0 + movl %r10d, %ebx + rorxl $11, %r9d, %ecx + addl 12(%rsp), %r12d + vpshufd $0x50, %xmm4, %xmm6 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r11d, %ebx + rorxl $25, %r9d, %edx + vpsrld $10, %xmm6, %xmm9 + # rnd_1: 2 - 2 + andl %r9d, %ebx + xorl %ecx, %edx + rorxl $13, %r13d, %ecx + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 3 - 3 + addl %edx, %r12d + rorxl $2, %r13d, %edx + xorl %r11d, %ebx + vpsrlq $0x11, %xmm6, %xmm6 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r13d, %edx + addl %ebx, %r12d + vpxor %xmm7, %xmm6, %xmm6 + # rnd_1: 5 - 5 + xorl %ecx, %edx + movl %r14d, %ebx + addl %r12d, %r8d + vpxor %xmm6, %xmm9, %xmm9 + # rnd_1: 6 - 6 + xorl %r13d, %ebx + addl %edx, %r12d + andl %ebx, %eax + vpshufb %xmm12, %xmm9, %xmm9 + # rnd_1: 7 - 7 + xorl %r14d, %eax + rorxl $6, %r8d, %edx + addl %eax, %r12d + vpaddd %xmm4, %xmm9, %xmm0 + # msg_sched done: 0-3 + # msg_sched: 4-7 + # rnd_0: 0 - 0 + movl %r9d, %eax + rorxl $11, %r8d, %ecx + addl 16(%rsp), %r11d + vpalignr $4, %xmm3, %xmm0, %xmm4 + vpalignr $4, %xmm1, %xmm2, %xmm5 + # rnd_0: 1 - 2 + xorl %edx, %ecx + xorl %r10d, %eax + rorxl $25, %r8d, %edx + andl %r8d, %eax + xorl %ecx, %edx + rorxl $13, %r12d, %ecx + vpsrld $7, %xmm5, %xmm6 + vpslld $25, %xmm5, %xmm7 + # rnd_0: 3 - 4 + addl %edx, %r11d + rorxl $2, %r12d, %edx + xorl %r10d, %eax + xorl %edx, %ecx + rorxl $22, %r12d, %edx + addl %eax, %r11d + vpsrld $3, %xmm5, %xmm8 + vpor %xmm6, %xmm7, %xmm7 + # rnd_0: 5 - 7 + xorl %ecx, %edx + movl %r13d, %eax + addl %r11d, %r15d + xorl %r12d, %eax + addl %edx, %r11d + andl %eax, %ebx + xorl %r13d, %ebx + rorxl $6, %r15d, %edx + addl %ebx, %r11d + # rnd_1: 0 - 0 + movl %r8d, %ebx + rorxl $11, %r15d, %ecx + addl 20(%rsp), %r10d + vpsrld $18, %xmm5, %xmm6 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r9d, %ebx + rorxl $25, %r15d, %edx + vpslld $14, %xmm5, %xmm5 + # rnd_1: 2 - 2 + andl %r15d, %ebx + xorl %ecx, %edx + rorxl $13, %r11d, %ecx + vpxor %xmm5, %xmm7, %xmm7 + # rnd_1: 3 - 3 + addl %edx, %r10d + rorxl $2, %r11d, %edx + xorl %r9d, %ebx + vpxor %xmm6, %xmm7, %xmm7 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r11d, %edx + addl %ebx, %r10d + vpshufd $0xfa, %xmm0, %xmm6 + # rnd_1: 5 - 5 + xorl %ecx, %edx + movl %r12d, %ebx + addl %r10d, %r14d + vpxor %xmm8, %xmm7, %xmm5 + # rnd_1: 6 - 6 + xorl %r11d, %ebx + addl %edx, %r10d + andl %ebx, %eax + vpsrld $10, %xmm6, %xmm8 + # rnd_1: 7 - 7 + xorl %r12d, %eax + rorxl $6, %r14d, %edx + addl %eax, %r10d + # rnd_0: 0 - 0 + movl %r15d, %eax + rorxl $11, %r14d, %ecx + addl 24(%rsp), %r9d + vpsrlq $19, %xmm6, %xmm7 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r8d, %eax + rorxl $25, %r14d, %edx + vpsrlq $0x11, %xmm6, %xmm6 + vpaddd %xmm1, %xmm4, %xmm4 + # rnd_0: 2 - 2 + andl %r14d, %eax + xorl %ecx, %edx + rorxl $13, %r10d, %ecx + vpaddd %xmm5, %xmm4, %xmm4 + # rnd_0: 3 - 3 + addl %edx, %r9d + rorxl $2, %r10d, %edx + xorl %r8d, %eax + vpxor %xmm7, %xmm6, %xmm6 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r10d, %edx + addl %eax, %r9d + vpxor %xmm6, %xmm8, %xmm8 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r11d, %eax + addl %r9d, %r13d + vpshufb %xmm11, %xmm8, %xmm8 + # rnd_0: 6 - 6 + xorl %r10d, %eax + addl %edx, %r9d + andl %eax, %ebx + vpaddd %xmm8, %xmm4, %xmm4 + # rnd_0: 7 - 7 + xorl %r11d, %ebx + rorxl $6, %r13d, %edx + addl %ebx, %r9d + # rnd_1: 0 - 0 + movl %r14d, %ebx + rorxl $11, %r13d, %ecx + addl 28(%rsp), %r8d + vpshufd $0x50, %xmm4, %xmm6 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r15d, %ebx + rorxl $25, %r13d, %edx + vpsrld $10, %xmm6, %xmm9 + # rnd_1: 2 - 2 + andl %r13d, %ebx + xorl %ecx, %edx + rorxl $13, %r9d, %ecx + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 3 - 3 + addl %edx, %r8d + rorxl $2, %r9d, %edx + xorl %r15d, %ebx + vpsrlq $0x11, %xmm6, %xmm6 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r9d, %edx + addl %ebx, %r8d + vpxor %xmm7, %xmm6, %xmm6 + # rnd_1: 5 - 5 + xorl %ecx, %edx + movl %r10d, %ebx + addl %r8d, %r12d + vpxor %xmm6, %xmm9, %xmm9 + # rnd_1: 6 - 6 + xorl %r9d, %ebx + addl %edx, %r8d + andl %ebx, %eax + vpshufb %xmm12, %xmm9, %xmm9 + # rnd_1: 7 - 7 + xorl %r10d, %eax + rorxl $6, %r12d, %edx + addl %eax, %r8d + vpaddd %xmm4, %xmm9, %xmm1 + # msg_sched done: 4-7 + # msg_sched: 8-11 + # rnd_0: 0 - 0 + movl %r13d, %eax + rorxl $11, %r12d, %ecx + addl 32(%rsp), %r15d + vpalignr $4, %xmm0, %xmm1, %xmm4 + vpalignr $4, %xmm2, %xmm3, %xmm5 + # rnd_0: 1 - 2 + xorl %edx, %ecx + xorl %r14d, %eax + rorxl $25, %r12d, %edx + andl %r12d, %eax + xorl %ecx, %edx + rorxl $13, %r8d, %ecx + vpsrld $7, %xmm5, %xmm6 + vpslld $25, %xmm5, %xmm7 + # rnd_0: 3 - 4 + addl %edx, %r15d + rorxl $2, %r8d, %edx + xorl %r14d, %eax + xorl %edx, %ecx + rorxl $22, %r8d, %edx + addl %eax, %r15d + vpsrld $3, %xmm5, %xmm8 + vpor %xmm6, %xmm7, %xmm7 + # rnd_0: 5 - 7 + xorl %ecx, %edx + movl %r9d, %eax + addl %r15d, %r11d + xorl %r8d, %eax + addl %edx, %r15d + andl %eax, %ebx + xorl %r9d, %ebx + rorxl $6, %r11d, %edx + addl %ebx, %r15d + # rnd_1: 0 - 0 + movl %r12d, %ebx + rorxl $11, %r11d, %ecx + addl 36(%rsp), %r14d + vpsrld $18, %xmm5, %xmm6 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r13d, %ebx + rorxl $25, %r11d, %edx + vpslld $14, %xmm5, %xmm5 + # rnd_1: 2 - 2 + andl %r11d, %ebx + xorl %ecx, %edx + rorxl $13, %r15d, %ecx + vpxor %xmm5, %xmm7, %xmm7 + # rnd_1: 3 - 3 + addl %edx, %r14d + rorxl $2, %r15d, %edx + xorl %r13d, %ebx + vpxor %xmm6, %xmm7, %xmm7 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r15d, %edx + addl %ebx, %r14d + vpshufd $0xfa, %xmm1, %xmm6 + # rnd_1: 5 - 5 + xorl %ecx, %edx + movl %r8d, %ebx + addl %r14d, %r10d + vpxor %xmm8, %xmm7, %xmm5 + # rnd_1: 6 - 6 + xorl %r15d, %ebx + addl %edx, %r14d + andl %ebx, %eax + vpsrld $10, %xmm6, %xmm8 + # rnd_1: 7 - 7 + xorl %r8d, %eax + rorxl $6, %r10d, %edx + addl %eax, %r14d + # rnd_0: 0 - 0 + movl %r11d, %eax + rorxl $11, %r10d, %ecx + addl 40(%rsp), %r13d + vpsrlq $19, %xmm6, %xmm7 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r12d, %eax + rorxl $25, %r10d, %edx + vpsrlq $0x11, %xmm6, %xmm6 + vpaddd %xmm2, %xmm4, %xmm4 + # rnd_0: 2 - 2 + andl %r10d, %eax + xorl %ecx, %edx + rorxl $13, %r14d, %ecx + vpaddd %xmm5, %xmm4, %xmm4 + # rnd_0: 3 - 3 + addl %edx, %r13d + rorxl $2, %r14d, %edx + xorl %r12d, %eax + vpxor %xmm7, %xmm6, %xmm6 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r14d, %edx + addl %eax, %r13d + vpxor %xmm6, %xmm8, %xmm8 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r15d, %eax + addl %r13d, %r9d + vpshufb %xmm11, %xmm8, %xmm8 + # rnd_0: 6 - 6 + xorl %r14d, %eax + addl %edx, %r13d + andl %eax, %ebx + vpaddd %xmm8, %xmm4, %xmm4 + # rnd_0: 7 - 7 + xorl %r15d, %ebx + rorxl $6, %r9d, %edx + addl %ebx, %r13d + # rnd_1: 0 - 0 + movl %r10d, %ebx + rorxl $11, %r9d, %ecx + addl 44(%rsp), %r12d + vpshufd $0x50, %xmm4, %xmm6 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r11d, %ebx + rorxl $25, %r9d, %edx + vpsrld $10, %xmm6, %xmm9 + # rnd_1: 2 - 2 + andl %r9d, %ebx + xorl %ecx, %edx + rorxl $13, %r13d, %ecx + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 3 - 3 + addl %edx, %r12d + rorxl $2, %r13d, %edx + xorl %r11d, %ebx + vpsrlq $0x11, %xmm6, %xmm6 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r13d, %edx + addl %ebx, %r12d + vpxor %xmm7, %xmm6, %xmm6 + # rnd_1: 5 - 5 + xorl %ecx, %edx + movl %r14d, %ebx + addl %r12d, %r8d + vpxor %xmm6, %xmm9, %xmm9 + # rnd_1: 6 - 6 + xorl %r13d, %ebx + addl %edx, %r12d + andl %ebx, %eax + vpshufb %xmm12, %xmm9, %xmm9 + # rnd_1: 7 - 7 + xorl %r14d, %eax + rorxl $6, %r8d, %edx + addl %eax, %r12d + vpaddd %xmm4, %xmm9, %xmm2 + # msg_sched done: 8-11 + # msg_sched: 12-15 + # rnd_0: 0 - 0 + movl %r9d, %eax + rorxl $11, %r8d, %ecx + addl 48(%rsp), %r11d + vpalignr $4, %xmm1, %xmm2, %xmm4 + vpalignr $4, %xmm3, %xmm0, %xmm5 + # rnd_0: 1 - 2 + xorl %edx, %ecx + xorl %r10d, %eax + rorxl $25, %r8d, %edx + andl %r8d, %eax + xorl %ecx, %edx + rorxl $13, %r12d, %ecx + vpsrld $7, %xmm5, %xmm6 + vpslld $25, %xmm5, %xmm7 + # rnd_0: 3 - 4 + addl %edx, %r11d + rorxl $2, %r12d, %edx + xorl %r10d, %eax + xorl %edx, %ecx + rorxl $22, %r12d, %edx + addl %eax, %r11d + vpsrld $3, %xmm5, %xmm8 + vpor %xmm6, %xmm7, %xmm7 + # rnd_0: 5 - 7 + xorl %ecx, %edx + movl %r13d, %eax + addl %r11d, %r15d + xorl %r12d, %eax + addl %edx, %r11d + andl %eax, %ebx + xorl %r13d, %ebx + rorxl $6, %r15d, %edx + addl %ebx, %r11d + # rnd_1: 0 - 0 + movl %r8d, %ebx + rorxl $11, %r15d, %ecx + addl 52(%rsp), %r10d + vpsrld $18, %xmm5, %xmm6 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r9d, %ebx + rorxl $25, %r15d, %edx + vpslld $14, %xmm5, %xmm5 + # rnd_1: 2 - 2 + andl %r15d, %ebx + xorl %ecx, %edx + rorxl $13, %r11d, %ecx + vpxor %xmm5, %xmm7, %xmm7 + # rnd_1: 3 - 3 + addl %edx, %r10d + rorxl $2, %r11d, %edx + xorl %r9d, %ebx + vpxor %xmm6, %xmm7, %xmm7 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r11d, %edx + addl %ebx, %r10d + vpshufd $0xfa, %xmm2, %xmm6 + # rnd_1: 5 - 5 + xorl %ecx, %edx + movl %r12d, %ebx + addl %r10d, %r14d + vpxor %xmm8, %xmm7, %xmm5 + # rnd_1: 6 - 6 + xorl %r11d, %ebx + addl %edx, %r10d + andl %ebx, %eax + vpsrld $10, %xmm6, %xmm8 + # rnd_1: 7 - 7 + xorl %r12d, %eax + rorxl $6, %r14d, %edx + addl %eax, %r10d + # rnd_0: 0 - 0 + movl %r15d, %eax + rorxl $11, %r14d, %ecx + addl 56(%rsp), %r9d + vpsrlq $19, %xmm6, %xmm7 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r8d, %eax + rorxl $25, %r14d, %edx + vpsrlq $0x11, %xmm6, %xmm6 + vpaddd %xmm3, %xmm4, %xmm4 + # rnd_0: 2 - 2 + andl %r14d, %eax + xorl %ecx, %edx + rorxl $13, %r10d, %ecx + vpaddd %xmm5, %xmm4, %xmm4 + # rnd_0: 3 - 3 + addl %edx, %r9d + rorxl $2, %r10d, %edx + xorl %r8d, %eax + vpxor %xmm7, %xmm6, %xmm6 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r10d, %edx + addl %eax, %r9d + vpxor %xmm6, %xmm8, %xmm8 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r11d, %eax + addl %r9d, %r13d + vpshufb %xmm11, %xmm8, %xmm8 + # rnd_0: 6 - 6 + xorl %r10d, %eax + addl %edx, %r9d + andl %eax, %ebx + vpaddd %xmm8, %xmm4, %xmm4 + # rnd_0: 7 - 7 + xorl %r11d, %ebx + rorxl $6, %r13d, %edx + addl %ebx, %r9d + # rnd_1: 0 - 0 + movl %r14d, %ebx + rorxl $11, %r13d, %ecx + addl 60(%rsp), %r8d + vpshufd $0x50, %xmm4, %xmm6 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r15d, %ebx + rorxl $25, %r13d, %edx + vpsrld $10, %xmm6, %xmm9 + # rnd_1: 2 - 2 + andl %r13d, %ebx + xorl %ecx, %edx + rorxl $13, %r9d, %ecx + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 3 - 3 + addl %edx, %r8d + rorxl $2, %r9d, %edx + xorl %r15d, %ebx + vpsrlq $0x11, %xmm6, %xmm6 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r9d, %edx + addl %ebx, %r8d + vpxor %xmm7, %xmm6, %xmm6 + # rnd_1: 5 - 5 + xorl %ecx, %edx + movl %r10d, %ebx + addl %r8d, %r12d + vpxor %xmm6, %xmm9, %xmm9 + # rnd_1: 6 - 6 + xorl %r9d, %ebx + addl %edx, %r8d + andl %ebx, %eax + vpshufb %xmm12, %xmm9, %xmm9 + # rnd_1: 7 - 7 + xorl %r10d, %eax + rorxl $6, %r12d, %edx + addl %eax, %r8d + vpaddd %xmm4, %xmm9, %xmm3 + # msg_sched done: 12-15 + # set_w_k_xfer_4: 4 + vpaddd 64+L_avx1_rorx_sha256_k(%rip), %xmm0, %xmm4 + vpaddd 80+L_avx1_rorx_sha256_k(%rip), %xmm1, %xmm5 + vmovdqu %xmm4, (%rsp) + vmovdqu %xmm5, 16(%rsp) + vpaddd 96+L_avx1_rorx_sha256_k(%rip), %xmm2, %xmm6 + vpaddd 112+L_avx1_rorx_sha256_k(%rip), %xmm3, %xmm7 + vmovdqu %xmm6, 32(%rsp) + vmovdqu %xmm7, 48(%rsp) + # msg_sched: 0-3 + # rnd_0: 0 - 0 + movl %r13d, %eax + rorxl $11, %r12d, %ecx + addl (%rsp), %r15d + vpalignr $4, %xmm2, %xmm3, %xmm4 + vpalignr $4, %xmm0, %xmm1, %xmm5 + # rnd_0: 1 - 2 + xorl %edx, %ecx + xorl %r14d, %eax + rorxl $25, %r12d, %edx + andl %r12d, %eax + xorl %ecx, %edx + rorxl $13, %r8d, %ecx + vpsrld $7, %xmm5, %xmm6 + vpslld $25, %xmm5, %xmm7 + # rnd_0: 3 - 4 + addl %edx, %r15d + rorxl $2, %r8d, %edx + xorl %r14d, %eax + xorl %edx, %ecx + rorxl $22, %r8d, %edx + addl %eax, %r15d + vpsrld $3, %xmm5, %xmm8 + vpor %xmm6, %xmm7, %xmm7 + # rnd_0: 5 - 7 + xorl %ecx, %edx + movl %r9d, %eax + addl %r15d, %r11d + xorl %r8d, %eax + addl %edx, %r15d + andl %eax, %ebx + xorl %r9d, %ebx + rorxl $6, %r11d, %edx + addl %ebx, %r15d + # rnd_1: 0 - 0 + movl %r12d, %ebx + rorxl $11, %r11d, %ecx + addl 4(%rsp), %r14d + vpsrld $18, %xmm5, %xmm6 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r13d, %ebx + rorxl $25, %r11d, %edx + vpslld $14, %xmm5, %xmm5 + # rnd_1: 2 - 2 + andl %r11d, %ebx + xorl %ecx, %edx + rorxl $13, %r15d, %ecx + vpxor %xmm5, %xmm7, %xmm7 + # rnd_1: 3 - 3 + addl %edx, %r14d + rorxl $2, %r15d, %edx + xorl %r13d, %ebx + vpxor %xmm6, %xmm7, %xmm7 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r15d, %edx + addl %ebx, %r14d + vpshufd $0xfa, %xmm3, %xmm6 + # rnd_1: 5 - 5 + xorl %ecx, %edx + movl %r8d, %ebx + addl %r14d, %r10d + vpxor %xmm8, %xmm7, %xmm5 + # rnd_1: 6 - 6 + xorl %r15d, %ebx + addl %edx, %r14d + andl %ebx, %eax + vpsrld $10, %xmm6, %xmm8 + # rnd_1: 7 - 7 + xorl %r8d, %eax + rorxl $6, %r10d, %edx + addl %eax, %r14d + # rnd_0: 0 - 0 + movl %r11d, %eax + rorxl $11, %r10d, %ecx + addl 8(%rsp), %r13d + vpsrlq $19, %xmm6, %xmm7 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r12d, %eax + rorxl $25, %r10d, %edx + vpsrlq $0x11, %xmm6, %xmm6 + vpaddd %xmm0, %xmm4, %xmm4 + # rnd_0: 2 - 2 + andl %r10d, %eax + xorl %ecx, %edx + rorxl $13, %r14d, %ecx + vpaddd %xmm5, %xmm4, %xmm4 + # rnd_0: 3 - 3 + addl %edx, %r13d + rorxl $2, %r14d, %edx + xorl %r12d, %eax + vpxor %xmm7, %xmm6, %xmm6 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r14d, %edx + addl %eax, %r13d + vpxor %xmm6, %xmm8, %xmm8 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r15d, %eax + addl %r13d, %r9d + vpshufb %xmm11, %xmm8, %xmm8 + # rnd_0: 6 - 6 + xorl %r14d, %eax + addl %edx, %r13d + andl %eax, %ebx + vpaddd %xmm8, %xmm4, %xmm4 + # rnd_0: 7 - 7 + xorl %r15d, %ebx + rorxl $6, %r9d, %edx + addl %ebx, %r13d + # rnd_1: 0 - 0 + movl %r10d, %ebx + rorxl $11, %r9d, %ecx + addl 12(%rsp), %r12d + vpshufd $0x50, %xmm4, %xmm6 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r11d, %ebx + rorxl $25, %r9d, %edx + vpsrld $10, %xmm6, %xmm9 + # rnd_1: 2 - 2 + andl %r9d, %ebx + xorl %ecx, %edx + rorxl $13, %r13d, %ecx + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 3 - 3 + addl %edx, %r12d + rorxl $2, %r13d, %edx + xorl %r11d, %ebx + vpsrlq $0x11, %xmm6, %xmm6 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r13d, %edx + addl %ebx, %r12d + vpxor %xmm7, %xmm6, %xmm6 + # rnd_1: 5 - 5 + xorl %ecx, %edx + movl %r14d, %ebx + addl %r12d, %r8d + vpxor %xmm6, %xmm9, %xmm9 + # rnd_1: 6 - 6 + xorl %r13d, %ebx + addl %edx, %r12d + andl %ebx, %eax + vpshufb %xmm12, %xmm9, %xmm9 + # rnd_1: 7 - 7 + xorl %r14d, %eax + rorxl $6, %r8d, %edx + addl %eax, %r12d + vpaddd %xmm4, %xmm9, %xmm0 + # msg_sched done: 0-3 + # msg_sched: 4-7 + # rnd_0: 0 - 0 + movl %r9d, %eax + rorxl $11, %r8d, %ecx + addl 16(%rsp), %r11d + vpalignr $4, %xmm3, %xmm0, %xmm4 + vpalignr $4, %xmm1, %xmm2, %xmm5 + # rnd_0: 1 - 2 + xorl %edx, %ecx + xorl %r10d, %eax + rorxl $25, %r8d, %edx + andl %r8d, %eax + xorl %ecx, %edx + rorxl $13, %r12d, %ecx + vpsrld $7, %xmm5, %xmm6 + vpslld $25, %xmm5, %xmm7 + # rnd_0: 3 - 4 + addl %edx, %r11d + rorxl $2, %r12d, %edx + xorl %r10d, %eax + xorl %edx, %ecx + rorxl $22, %r12d, %edx + addl %eax, %r11d + vpsrld $3, %xmm5, %xmm8 + vpor %xmm6, %xmm7, %xmm7 + # rnd_0: 5 - 7 + xorl %ecx, %edx + movl %r13d, %eax + addl %r11d, %r15d + xorl %r12d, %eax + addl %edx, %r11d + andl %eax, %ebx + xorl %r13d, %ebx + rorxl $6, %r15d, %edx + addl %ebx, %r11d + # rnd_1: 0 - 0 + movl %r8d, %ebx + rorxl $11, %r15d, %ecx + addl 20(%rsp), %r10d + vpsrld $18, %xmm5, %xmm6 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r9d, %ebx + rorxl $25, %r15d, %edx + vpslld $14, %xmm5, %xmm5 + # rnd_1: 2 - 2 + andl %r15d, %ebx + xorl %ecx, %edx + rorxl $13, %r11d, %ecx + vpxor %xmm5, %xmm7, %xmm7 + # rnd_1: 3 - 3 + addl %edx, %r10d + rorxl $2, %r11d, %edx + xorl %r9d, %ebx + vpxor %xmm6, %xmm7, %xmm7 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r11d, %edx + addl %ebx, %r10d + vpshufd $0xfa, %xmm0, %xmm6 + # rnd_1: 5 - 5 + xorl %ecx, %edx + movl %r12d, %ebx + addl %r10d, %r14d + vpxor %xmm8, %xmm7, %xmm5 + # rnd_1: 6 - 6 + xorl %r11d, %ebx + addl %edx, %r10d + andl %ebx, %eax + vpsrld $10, %xmm6, %xmm8 + # rnd_1: 7 - 7 + xorl %r12d, %eax + rorxl $6, %r14d, %edx + addl %eax, %r10d + # rnd_0: 0 - 0 + movl %r15d, %eax + rorxl $11, %r14d, %ecx + addl 24(%rsp), %r9d + vpsrlq $19, %xmm6, %xmm7 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r8d, %eax + rorxl $25, %r14d, %edx + vpsrlq $0x11, %xmm6, %xmm6 + vpaddd %xmm1, %xmm4, %xmm4 + # rnd_0: 2 - 2 + andl %r14d, %eax + xorl %ecx, %edx + rorxl $13, %r10d, %ecx + vpaddd %xmm5, %xmm4, %xmm4 + # rnd_0: 3 - 3 + addl %edx, %r9d + rorxl $2, %r10d, %edx + xorl %r8d, %eax + vpxor %xmm7, %xmm6, %xmm6 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r10d, %edx + addl %eax, %r9d + vpxor %xmm6, %xmm8, %xmm8 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r11d, %eax + addl %r9d, %r13d + vpshufb %xmm11, %xmm8, %xmm8 + # rnd_0: 6 - 6 + xorl %r10d, %eax + addl %edx, %r9d + andl %eax, %ebx + vpaddd %xmm8, %xmm4, %xmm4 + # rnd_0: 7 - 7 + xorl %r11d, %ebx + rorxl $6, %r13d, %edx + addl %ebx, %r9d + # rnd_1: 0 - 0 + movl %r14d, %ebx + rorxl $11, %r13d, %ecx + addl 28(%rsp), %r8d + vpshufd $0x50, %xmm4, %xmm6 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r15d, %ebx + rorxl $25, %r13d, %edx + vpsrld $10, %xmm6, %xmm9 + # rnd_1: 2 - 2 + andl %r13d, %ebx + xorl %ecx, %edx + rorxl $13, %r9d, %ecx + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 3 - 3 + addl %edx, %r8d + rorxl $2, %r9d, %edx + xorl %r15d, %ebx + vpsrlq $0x11, %xmm6, %xmm6 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r9d, %edx + addl %ebx, %r8d + vpxor %xmm7, %xmm6, %xmm6 + # rnd_1: 5 - 5 + xorl %ecx, %edx + movl %r10d, %ebx + addl %r8d, %r12d + vpxor %xmm6, %xmm9, %xmm9 + # rnd_1: 6 - 6 + xorl %r9d, %ebx + addl %edx, %r8d + andl %ebx, %eax + vpshufb %xmm12, %xmm9, %xmm9 + # rnd_1: 7 - 7 + xorl %r10d, %eax + rorxl $6, %r12d, %edx + addl %eax, %r8d + vpaddd %xmm4, %xmm9, %xmm1 + # msg_sched done: 4-7 + # msg_sched: 8-11 + # rnd_0: 0 - 0 + movl %r13d, %eax + rorxl $11, %r12d, %ecx + addl 32(%rsp), %r15d + vpalignr $4, %xmm0, %xmm1, %xmm4 + vpalignr $4, %xmm2, %xmm3, %xmm5 + # rnd_0: 1 - 2 + xorl %edx, %ecx + xorl %r14d, %eax + rorxl $25, %r12d, %edx + andl %r12d, %eax + xorl %ecx, %edx + rorxl $13, %r8d, %ecx + vpsrld $7, %xmm5, %xmm6 + vpslld $25, %xmm5, %xmm7 + # rnd_0: 3 - 4 + addl %edx, %r15d + rorxl $2, %r8d, %edx + xorl %r14d, %eax + xorl %edx, %ecx + rorxl $22, %r8d, %edx + addl %eax, %r15d + vpsrld $3, %xmm5, %xmm8 + vpor %xmm6, %xmm7, %xmm7 + # rnd_0: 5 - 7 + xorl %ecx, %edx + movl %r9d, %eax + addl %r15d, %r11d + xorl %r8d, %eax + addl %edx, %r15d + andl %eax, %ebx + xorl %r9d, %ebx + rorxl $6, %r11d, %edx + addl %ebx, %r15d + # rnd_1: 0 - 0 + movl %r12d, %ebx + rorxl $11, %r11d, %ecx + addl 36(%rsp), %r14d + vpsrld $18, %xmm5, %xmm6 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r13d, %ebx + rorxl $25, %r11d, %edx + vpslld $14, %xmm5, %xmm5 + # rnd_1: 2 - 2 + andl %r11d, %ebx + xorl %ecx, %edx + rorxl $13, %r15d, %ecx + vpxor %xmm5, %xmm7, %xmm7 + # rnd_1: 3 - 3 + addl %edx, %r14d + rorxl $2, %r15d, %edx + xorl %r13d, %ebx + vpxor %xmm6, %xmm7, %xmm7 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r15d, %edx + addl %ebx, %r14d + vpshufd $0xfa, %xmm1, %xmm6 + # rnd_1: 5 - 5 + xorl %ecx, %edx + movl %r8d, %ebx + addl %r14d, %r10d + vpxor %xmm8, %xmm7, %xmm5 + # rnd_1: 6 - 6 + xorl %r15d, %ebx + addl %edx, %r14d + andl %ebx, %eax + vpsrld $10, %xmm6, %xmm8 + # rnd_1: 7 - 7 + xorl %r8d, %eax + rorxl $6, %r10d, %edx + addl %eax, %r14d + # rnd_0: 0 - 0 + movl %r11d, %eax + rorxl $11, %r10d, %ecx + addl 40(%rsp), %r13d + vpsrlq $19, %xmm6, %xmm7 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r12d, %eax + rorxl $25, %r10d, %edx + vpsrlq $0x11, %xmm6, %xmm6 + vpaddd %xmm2, %xmm4, %xmm4 + # rnd_0: 2 - 2 + andl %r10d, %eax + xorl %ecx, %edx + rorxl $13, %r14d, %ecx + vpaddd %xmm5, %xmm4, %xmm4 + # rnd_0: 3 - 3 + addl %edx, %r13d + rorxl $2, %r14d, %edx + xorl %r12d, %eax + vpxor %xmm7, %xmm6, %xmm6 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r14d, %edx + addl %eax, %r13d + vpxor %xmm6, %xmm8, %xmm8 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r15d, %eax + addl %r13d, %r9d + vpshufb %xmm11, %xmm8, %xmm8 + # rnd_0: 6 - 6 + xorl %r14d, %eax + addl %edx, %r13d + andl %eax, %ebx + vpaddd %xmm8, %xmm4, %xmm4 + # rnd_0: 7 - 7 + xorl %r15d, %ebx + rorxl $6, %r9d, %edx + addl %ebx, %r13d + # rnd_1: 0 - 0 + movl %r10d, %ebx + rorxl $11, %r9d, %ecx + addl 44(%rsp), %r12d + vpshufd $0x50, %xmm4, %xmm6 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r11d, %ebx + rorxl $25, %r9d, %edx + vpsrld $10, %xmm6, %xmm9 + # rnd_1: 2 - 2 + andl %r9d, %ebx + xorl %ecx, %edx + rorxl $13, %r13d, %ecx + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 3 - 3 + addl %edx, %r12d + rorxl $2, %r13d, %edx + xorl %r11d, %ebx + vpsrlq $0x11, %xmm6, %xmm6 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r13d, %edx + addl %ebx, %r12d + vpxor %xmm7, %xmm6, %xmm6 + # rnd_1: 5 - 5 + xorl %ecx, %edx + movl %r14d, %ebx + addl %r12d, %r8d + vpxor %xmm6, %xmm9, %xmm9 + # rnd_1: 6 - 6 + xorl %r13d, %ebx + addl %edx, %r12d + andl %ebx, %eax + vpshufb %xmm12, %xmm9, %xmm9 + # rnd_1: 7 - 7 + xorl %r14d, %eax + rorxl $6, %r8d, %edx + addl %eax, %r12d + vpaddd %xmm4, %xmm9, %xmm2 + # msg_sched done: 8-11 + # msg_sched: 12-15 + # rnd_0: 0 - 0 + movl %r9d, %eax + rorxl $11, %r8d, %ecx + addl 48(%rsp), %r11d + vpalignr $4, %xmm1, %xmm2, %xmm4 + vpalignr $4, %xmm3, %xmm0, %xmm5 + # rnd_0: 1 - 2 + xorl %edx, %ecx + xorl %r10d, %eax + rorxl $25, %r8d, %edx + andl %r8d, %eax + xorl %ecx, %edx + rorxl $13, %r12d, %ecx + vpsrld $7, %xmm5, %xmm6 + vpslld $25, %xmm5, %xmm7 + # rnd_0: 3 - 4 + addl %edx, %r11d + rorxl $2, %r12d, %edx + xorl %r10d, %eax + xorl %edx, %ecx + rorxl $22, %r12d, %edx + addl %eax, %r11d + vpsrld $3, %xmm5, %xmm8 + vpor %xmm6, %xmm7, %xmm7 + # rnd_0: 5 - 7 + xorl %ecx, %edx + movl %r13d, %eax + addl %r11d, %r15d + xorl %r12d, %eax + addl %edx, %r11d + andl %eax, %ebx + xorl %r13d, %ebx + rorxl $6, %r15d, %edx + addl %ebx, %r11d + # rnd_1: 0 - 0 + movl %r8d, %ebx + rorxl $11, %r15d, %ecx + addl 52(%rsp), %r10d + vpsrld $18, %xmm5, %xmm6 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r9d, %ebx + rorxl $25, %r15d, %edx + vpslld $14, %xmm5, %xmm5 + # rnd_1: 2 - 2 + andl %r15d, %ebx + xorl %ecx, %edx + rorxl $13, %r11d, %ecx + vpxor %xmm5, %xmm7, %xmm7 + # rnd_1: 3 - 3 + addl %edx, %r10d + rorxl $2, %r11d, %edx + xorl %r9d, %ebx + vpxor %xmm6, %xmm7, %xmm7 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r11d, %edx + addl %ebx, %r10d + vpshufd $0xfa, %xmm2, %xmm6 + # rnd_1: 5 - 5 + xorl %ecx, %edx + movl %r12d, %ebx + addl %r10d, %r14d + vpxor %xmm8, %xmm7, %xmm5 + # rnd_1: 6 - 6 + xorl %r11d, %ebx + addl %edx, %r10d + andl %ebx, %eax + vpsrld $10, %xmm6, %xmm8 + # rnd_1: 7 - 7 + xorl %r12d, %eax + rorxl $6, %r14d, %edx + addl %eax, %r10d + # rnd_0: 0 - 0 + movl %r15d, %eax + rorxl $11, %r14d, %ecx + addl 56(%rsp), %r9d + vpsrlq $19, %xmm6, %xmm7 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r8d, %eax + rorxl $25, %r14d, %edx + vpsrlq $0x11, %xmm6, %xmm6 + vpaddd %xmm3, %xmm4, %xmm4 + # rnd_0: 2 - 2 + andl %r14d, %eax + xorl %ecx, %edx + rorxl $13, %r10d, %ecx + vpaddd %xmm5, %xmm4, %xmm4 + # rnd_0: 3 - 3 + addl %edx, %r9d + rorxl $2, %r10d, %edx + xorl %r8d, %eax + vpxor %xmm7, %xmm6, %xmm6 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r10d, %edx + addl %eax, %r9d + vpxor %xmm6, %xmm8, %xmm8 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r11d, %eax + addl %r9d, %r13d + vpshufb %xmm11, %xmm8, %xmm8 + # rnd_0: 6 - 6 + xorl %r10d, %eax + addl %edx, %r9d + andl %eax, %ebx + vpaddd %xmm8, %xmm4, %xmm4 + # rnd_0: 7 - 7 + xorl %r11d, %ebx + rorxl $6, %r13d, %edx + addl %ebx, %r9d + # rnd_1: 0 - 0 + movl %r14d, %ebx + rorxl $11, %r13d, %ecx + addl 60(%rsp), %r8d + vpshufd $0x50, %xmm4, %xmm6 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r15d, %ebx + rorxl $25, %r13d, %edx + vpsrld $10, %xmm6, %xmm9 + # rnd_1: 2 - 2 + andl %r13d, %ebx + xorl %ecx, %edx + rorxl $13, %r9d, %ecx + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 3 - 3 + addl %edx, %r8d + rorxl $2, %r9d, %edx + xorl %r15d, %ebx + vpsrlq $0x11, %xmm6, %xmm6 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r9d, %edx + addl %ebx, %r8d + vpxor %xmm7, %xmm6, %xmm6 + # rnd_1: 5 - 5 + xorl %ecx, %edx + movl %r10d, %ebx + addl %r8d, %r12d + vpxor %xmm6, %xmm9, %xmm9 + # rnd_1: 6 - 6 + xorl %r9d, %ebx + addl %edx, %r8d + andl %ebx, %eax + vpshufb %xmm12, %xmm9, %xmm9 + # rnd_1: 7 - 7 + xorl %r10d, %eax + rorxl $6, %r12d, %edx + addl %eax, %r8d + vpaddd %xmm4, %xmm9, %xmm3 + # msg_sched done: 12-15 + # set_w_k_xfer_4: 8 + vpaddd 128+L_avx1_rorx_sha256_k(%rip), %xmm0, %xmm4 + vpaddd 144+L_avx1_rorx_sha256_k(%rip), %xmm1, %xmm5 + vmovdqu %xmm4, (%rsp) + vmovdqu %xmm5, 16(%rsp) + vpaddd 160+L_avx1_rorx_sha256_k(%rip), %xmm2, %xmm6 + vpaddd 176+L_avx1_rorx_sha256_k(%rip), %xmm3, %xmm7 + vmovdqu %xmm6, 32(%rsp) + vmovdqu %xmm7, 48(%rsp) + # msg_sched: 0-3 + # rnd_0: 0 - 0 + movl %r13d, %eax + rorxl $11, %r12d, %ecx + addl (%rsp), %r15d + vpalignr $4, %xmm2, %xmm3, %xmm4 + vpalignr $4, %xmm0, %xmm1, %xmm5 + # rnd_0: 1 - 2 + xorl %edx, %ecx + xorl %r14d, %eax + rorxl $25, %r12d, %edx + andl %r12d, %eax + xorl %ecx, %edx + rorxl $13, %r8d, %ecx + vpsrld $7, %xmm5, %xmm6 + vpslld $25, %xmm5, %xmm7 + # rnd_0: 3 - 4 + addl %edx, %r15d + rorxl $2, %r8d, %edx + xorl %r14d, %eax + xorl %edx, %ecx + rorxl $22, %r8d, %edx + addl %eax, %r15d + vpsrld $3, %xmm5, %xmm8 + vpor %xmm6, %xmm7, %xmm7 + # rnd_0: 5 - 7 + xorl %ecx, %edx + movl %r9d, %eax + addl %r15d, %r11d + xorl %r8d, %eax + addl %edx, %r15d + andl %eax, %ebx + xorl %r9d, %ebx + rorxl $6, %r11d, %edx + addl %ebx, %r15d + # rnd_1: 0 - 0 + movl %r12d, %ebx + rorxl $11, %r11d, %ecx + addl 4(%rsp), %r14d + vpsrld $18, %xmm5, %xmm6 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r13d, %ebx + rorxl $25, %r11d, %edx + vpslld $14, %xmm5, %xmm5 + # rnd_1: 2 - 2 + andl %r11d, %ebx + xorl %ecx, %edx + rorxl $13, %r15d, %ecx + vpxor %xmm5, %xmm7, %xmm7 + # rnd_1: 3 - 3 + addl %edx, %r14d + rorxl $2, %r15d, %edx + xorl %r13d, %ebx + vpxor %xmm6, %xmm7, %xmm7 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r15d, %edx + addl %ebx, %r14d + vpshufd $0xfa, %xmm3, %xmm6 + # rnd_1: 5 - 5 + xorl %ecx, %edx + movl %r8d, %ebx + addl %r14d, %r10d + vpxor %xmm8, %xmm7, %xmm5 + # rnd_1: 6 - 6 + xorl %r15d, %ebx + addl %edx, %r14d + andl %ebx, %eax + vpsrld $10, %xmm6, %xmm8 + # rnd_1: 7 - 7 + xorl %r8d, %eax + rorxl $6, %r10d, %edx + addl %eax, %r14d + # rnd_0: 0 - 0 + movl %r11d, %eax + rorxl $11, %r10d, %ecx + addl 8(%rsp), %r13d + vpsrlq $19, %xmm6, %xmm7 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r12d, %eax + rorxl $25, %r10d, %edx + vpsrlq $0x11, %xmm6, %xmm6 + vpaddd %xmm0, %xmm4, %xmm4 + # rnd_0: 2 - 2 + andl %r10d, %eax + xorl %ecx, %edx + rorxl $13, %r14d, %ecx + vpaddd %xmm5, %xmm4, %xmm4 + # rnd_0: 3 - 3 + addl %edx, %r13d + rorxl $2, %r14d, %edx + xorl %r12d, %eax + vpxor %xmm7, %xmm6, %xmm6 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r14d, %edx + addl %eax, %r13d + vpxor %xmm6, %xmm8, %xmm8 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r15d, %eax + addl %r13d, %r9d + vpshufb %xmm11, %xmm8, %xmm8 + # rnd_0: 6 - 6 + xorl %r14d, %eax + addl %edx, %r13d + andl %eax, %ebx + vpaddd %xmm8, %xmm4, %xmm4 + # rnd_0: 7 - 7 + xorl %r15d, %ebx + rorxl $6, %r9d, %edx + addl %ebx, %r13d + # rnd_1: 0 - 0 + movl %r10d, %ebx + rorxl $11, %r9d, %ecx + addl 12(%rsp), %r12d + vpshufd $0x50, %xmm4, %xmm6 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r11d, %ebx + rorxl $25, %r9d, %edx + vpsrld $10, %xmm6, %xmm9 + # rnd_1: 2 - 2 + andl %r9d, %ebx + xorl %ecx, %edx + rorxl $13, %r13d, %ecx + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 3 - 3 + addl %edx, %r12d + rorxl $2, %r13d, %edx + xorl %r11d, %ebx + vpsrlq $0x11, %xmm6, %xmm6 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r13d, %edx + addl %ebx, %r12d + vpxor %xmm7, %xmm6, %xmm6 + # rnd_1: 5 - 5 + xorl %ecx, %edx + movl %r14d, %ebx + addl %r12d, %r8d + vpxor %xmm6, %xmm9, %xmm9 + # rnd_1: 6 - 6 + xorl %r13d, %ebx + addl %edx, %r12d + andl %ebx, %eax + vpshufb %xmm12, %xmm9, %xmm9 + # rnd_1: 7 - 7 + xorl %r14d, %eax + rorxl $6, %r8d, %edx + addl %eax, %r12d + vpaddd %xmm4, %xmm9, %xmm0 + # msg_sched done: 0-3 + # msg_sched: 4-7 + # rnd_0: 0 - 0 + movl %r9d, %eax + rorxl $11, %r8d, %ecx + addl 16(%rsp), %r11d + vpalignr $4, %xmm3, %xmm0, %xmm4 + vpalignr $4, %xmm1, %xmm2, %xmm5 + # rnd_0: 1 - 2 + xorl %edx, %ecx + xorl %r10d, %eax + rorxl $25, %r8d, %edx + andl %r8d, %eax + xorl %ecx, %edx + rorxl $13, %r12d, %ecx + vpsrld $7, %xmm5, %xmm6 + vpslld $25, %xmm5, %xmm7 + # rnd_0: 3 - 4 + addl %edx, %r11d + rorxl $2, %r12d, %edx + xorl %r10d, %eax + xorl %edx, %ecx + rorxl $22, %r12d, %edx + addl %eax, %r11d + vpsrld $3, %xmm5, %xmm8 + vpor %xmm6, %xmm7, %xmm7 + # rnd_0: 5 - 7 + xorl %ecx, %edx + movl %r13d, %eax + addl %r11d, %r15d + xorl %r12d, %eax + addl %edx, %r11d + andl %eax, %ebx + xorl %r13d, %ebx + rorxl $6, %r15d, %edx + addl %ebx, %r11d + # rnd_1: 0 - 0 + movl %r8d, %ebx + rorxl $11, %r15d, %ecx + addl 20(%rsp), %r10d + vpsrld $18, %xmm5, %xmm6 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r9d, %ebx + rorxl $25, %r15d, %edx + vpslld $14, %xmm5, %xmm5 + # rnd_1: 2 - 2 + andl %r15d, %ebx + xorl %ecx, %edx + rorxl $13, %r11d, %ecx + vpxor %xmm5, %xmm7, %xmm7 + # rnd_1: 3 - 3 + addl %edx, %r10d + rorxl $2, %r11d, %edx + xorl %r9d, %ebx + vpxor %xmm6, %xmm7, %xmm7 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r11d, %edx + addl %ebx, %r10d + vpshufd $0xfa, %xmm0, %xmm6 + # rnd_1: 5 - 5 + xorl %ecx, %edx + movl %r12d, %ebx + addl %r10d, %r14d + vpxor %xmm8, %xmm7, %xmm5 + # rnd_1: 6 - 6 + xorl %r11d, %ebx + addl %edx, %r10d + andl %ebx, %eax + vpsrld $10, %xmm6, %xmm8 + # rnd_1: 7 - 7 + xorl %r12d, %eax + rorxl $6, %r14d, %edx + addl %eax, %r10d + # rnd_0: 0 - 0 + movl %r15d, %eax + rorxl $11, %r14d, %ecx + addl 24(%rsp), %r9d + vpsrlq $19, %xmm6, %xmm7 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r8d, %eax + rorxl $25, %r14d, %edx + vpsrlq $0x11, %xmm6, %xmm6 + vpaddd %xmm1, %xmm4, %xmm4 + # rnd_0: 2 - 2 + andl %r14d, %eax + xorl %ecx, %edx + rorxl $13, %r10d, %ecx + vpaddd %xmm5, %xmm4, %xmm4 + # rnd_0: 3 - 3 + addl %edx, %r9d + rorxl $2, %r10d, %edx + xorl %r8d, %eax + vpxor %xmm7, %xmm6, %xmm6 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r10d, %edx + addl %eax, %r9d + vpxor %xmm6, %xmm8, %xmm8 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r11d, %eax + addl %r9d, %r13d + vpshufb %xmm11, %xmm8, %xmm8 + # rnd_0: 6 - 6 + xorl %r10d, %eax + addl %edx, %r9d + andl %eax, %ebx + vpaddd %xmm8, %xmm4, %xmm4 + # rnd_0: 7 - 7 + xorl %r11d, %ebx + rorxl $6, %r13d, %edx + addl %ebx, %r9d + # rnd_1: 0 - 0 + movl %r14d, %ebx + rorxl $11, %r13d, %ecx + addl 28(%rsp), %r8d + vpshufd $0x50, %xmm4, %xmm6 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r15d, %ebx + rorxl $25, %r13d, %edx + vpsrld $10, %xmm6, %xmm9 + # rnd_1: 2 - 2 + andl %r13d, %ebx + xorl %ecx, %edx + rorxl $13, %r9d, %ecx + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 3 - 3 + addl %edx, %r8d + rorxl $2, %r9d, %edx + xorl %r15d, %ebx + vpsrlq $0x11, %xmm6, %xmm6 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r9d, %edx + addl %ebx, %r8d + vpxor %xmm7, %xmm6, %xmm6 + # rnd_1: 5 - 5 + xorl %ecx, %edx + movl %r10d, %ebx + addl %r8d, %r12d + vpxor %xmm6, %xmm9, %xmm9 + # rnd_1: 6 - 6 + xorl %r9d, %ebx + addl %edx, %r8d + andl %ebx, %eax + vpshufb %xmm12, %xmm9, %xmm9 + # rnd_1: 7 - 7 + xorl %r10d, %eax + rorxl $6, %r12d, %edx + addl %eax, %r8d + vpaddd %xmm4, %xmm9, %xmm1 + # msg_sched done: 4-7 + # msg_sched: 8-11 + # rnd_0: 0 - 0 + movl %r13d, %eax + rorxl $11, %r12d, %ecx + addl 32(%rsp), %r15d + vpalignr $4, %xmm0, %xmm1, %xmm4 + vpalignr $4, %xmm2, %xmm3, %xmm5 + # rnd_0: 1 - 2 + xorl %edx, %ecx + xorl %r14d, %eax + rorxl $25, %r12d, %edx + andl %r12d, %eax + xorl %ecx, %edx + rorxl $13, %r8d, %ecx + vpsrld $7, %xmm5, %xmm6 + vpslld $25, %xmm5, %xmm7 + # rnd_0: 3 - 4 + addl %edx, %r15d + rorxl $2, %r8d, %edx + xorl %r14d, %eax + xorl %edx, %ecx + rorxl $22, %r8d, %edx + addl %eax, %r15d + vpsrld $3, %xmm5, %xmm8 + vpor %xmm6, %xmm7, %xmm7 + # rnd_0: 5 - 7 + xorl %ecx, %edx + movl %r9d, %eax + addl %r15d, %r11d + xorl %r8d, %eax + addl %edx, %r15d + andl %eax, %ebx + xorl %r9d, %ebx + rorxl $6, %r11d, %edx + addl %ebx, %r15d + # rnd_1: 0 - 0 + movl %r12d, %ebx + rorxl $11, %r11d, %ecx + addl 36(%rsp), %r14d + vpsrld $18, %xmm5, %xmm6 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r13d, %ebx + rorxl $25, %r11d, %edx + vpslld $14, %xmm5, %xmm5 + # rnd_1: 2 - 2 + andl %r11d, %ebx + xorl %ecx, %edx + rorxl $13, %r15d, %ecx + vpxor %xmm5, %xmm7, %xmm7 + # rnd_1: 3 - 3 + addl %edx, %r14d + rorxl $2, %r15d, %edx + xorl %r13d, %ebx + vpxor %xmm6, %xmm7, %xmm7 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r15d, %edx + addl %ebx, %r14d + vpshufd $0xfa, %xmm1, %xmm6 + # rnd_1: 5 - 5 + xorl %ecx, %edx + movl %r8d, %ebx + addl %r14d, %r10d + vpxor %xmm8, %xmm7, %xmm5 + # rnd_1: 6 - 6 + xorl %r15d, %ebx + addl %edx, %r14d + andl %ebx, %eax + vpsrld $10, %xmm6, %xmm8 + # rnd_1: 7 - 7 + xorl %r8d, %eax + rorxl $6, %r10d, %edx + addl %eax, %r14d + # rnd_0: 0 - 0 + movl %r11d, %eax + rorxl $11, %r10d, %ecx + addl 40(%rsp), %r13d + vpsrlq $19, %xmm6, %xmm7 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r12d, %eax + rorxl $25, %r10d, %edx + vpsrlq $0x11, %xmm6, %xmm6 + vpaddd %xmm2, %xmm4, %xmm4 + # rnd_0: 2 - 2 + andl %r10d, %eax + xorl %ecx, %edx + rorxl $13, %r14d, %ecx + vpaddd %xmm5, %xmm4, %xmm4 + # rnd_0: 3 - 3 + addl %edx, %r13d + rorxl $2, %r14d, %edx + xorl %r12d, %eax + vpxor %xmm7, %xmm6, %xmm6 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r14d, %edx + addl %eax, %r13d + vpxor %xmm6, %xmm8, %xmm8 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r15d, %eax + addl %r13d, %r9d + vpshufb %xmm11, %xmm8, %xmm8 + # rnd_0: 6 - 6 + xorl %r14d, %eax + addl %edx, %r13d + andl %eax, %ebx + vpaddd %xmm8, %xmm4, %xmm4 + # rnd_0: 7 - 7 + xorl %r15d, %ebx + rorxl $6, %r9d, %edx + addl %ebx, %r13d + # rnd_1: 0 - 0 + movl %r10d, %ebx + rorxl $11, %r9d, %ecx + addl 44(%rsp), %r12d + vpshufd $0x50, %xmm4, %xmm6 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r11d, %ebx + rorxl $25, %r9d, %edx + vpsrld $10, %xmm6, %xmm9 + # rnd_1: 2 - 2 + andl %r9d, %ebx + xorl %ecx, %edx + rorxl $13, %r13d, %ecx + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 3 - 3 + addl %edx, %r12d + rorxl $2, %r13d, %edx + xorl %r11d, %ebx + vpsrlq $0x11, %xmm6, %xmm6 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r13d, %edx + addl %ebx, %r12d + vpxor %xmm7, %xmm6, %xmm6 + # rnd_1: 5 - 5 + xorl %ecx, %edx + movl %r14d, %ebx + addl %r12d, %r8d + vpxor %xmm6, %xmm9, %xmm9 + # rnd_1: 6 - 6 + xorl %r13d, %ebx + addl %edx, %r12d + andl %ebx, %eax + vpshufb %xmm12, %xmm9, %xmm9 + # rnd_1: 7 - 7 + xorl %r14d, %eax + rorxl $6, %r8d, %edx + addl %eax, %r12d + vpaddd %xmm4, %xmm9, %xmm2 + # msg_sched done: 8-11 + # msg_sched: 12-15 + # rnd_0: 0 - 0 + movl %r9d, %eax + rorxl $11, %r8d, %ecx + addl 48(%rsp), %r11d + vpalignr $4, %xmm1, %xmm2, %xmm4 + vpalignr $4, %xmm3, %xmm0, %xmm5 + # rnd_0: 1 - 2 + xorl %edx, %ecx + xorl %r10d, %eax + rorxl $25, %r8d, %edx + andl %r8d, %eax + xorl %ecx, %edx + rorxl $13, %r12d, %ecx + vpsrld $7, %xmm5, %xmm6 + vpslld $25, %xmm5, %xmm7 + # rnd_0: 3 - 4 + addl %edx, %r11d + rorxl $2, %r12d, %edx + xorl %r10d, %eax + xorl %edx, %ecx + rorxl $22, %r12d, %edx + addl %eax, %r11d + vpsrld $3, %xmm5, %xmm8 + vpor %xmm6, %xmm7, %xmm7 + # rnd_0: 5 - 7 + xorl %ecx, %edx + movl %r13d, %eax + addl %r11d, %r15d + xorl %r12d, %eax + addl %edx, %r11d + andl %eax, %ebx + xorl %r13d, %ebx + rorxl $6, %r15d, %edx + addl %ebx, %r11d + # rnd_1: 0 - 0 + movl %r8d, %ebx + rorxl $11, %r15d, %ecx + addl 52(%rsp), %r10d + vpsrld $18, %xmm5, %xmm6 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r9d, %ebx + rorxl $25, %r15d, %edx + vpslld $14, %xmm5, %xmm5 + # rnd_1: 2 - 2 + andl %r15d, %ebx + xorl %ecx, %edx + rorxl $13, %r11d, %ecx + vpxor %xmm5, %xmm7, %xmm7 + # rnd_1: 3 - 3 + addl %edx, %r10d + rorxl $2, %r11d, %edx + xorl %r9d, %ebx + vpxor %xmm6, %xmm7, %xmm7 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r11d, %edx + addl %ebx, %r10d + vpshufd $0xfa, %xmm2, %xmm6 + # rnd_1: 5 - 5 + xorl %ecx, %edx + movl %r12d, %ebx + addl %r10d, %r14d + vpxor %xmm8, %xmm7, %xmm5 + # rnd_1: 6 - 6 + xorl %r11d, %ebx + addl %edx, %r10d + andl %ebx, %eax + vpsrld $10, %xmm6, %xmm8 + # rnd_1: 7 - 7 + xorl %r12d, %eax + rorxl $6, %r14d, %edx + addl %eax, %r10d + # rnd_0: 0 - 0 + movl %r15d, %eax + rorxl $11, %r14d, %ecx + addl 56(%rsp), %r9d + vpsrlq $19, %xmm6, %xmm7 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r8d, %eax + rorxl $25, %r14d, %edx + vpsrlq $0x11, %xmm6, %xmm6 + vpaddd %xmm3, %xmm4, %xmm4 + # rnd_0: 2 - 2 + andl %r14d, %eax + xorl %ecx, %edx + rorxl $13, %r10d, %ecx + vpaddd %xmm5, %xmm4, %xmm4 + # rnd_0: 3 - 3 + addl %edx, %r9d + rorxl $2, %r10d, %edx + xorl %r8d, %eax + vpxor %xmm7, %xmm6, %xmm6 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r10d, %edx + addl %eax, %r9d + vpxor %xmm6, %xmm8, %xmm8 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r11d, %eax + addl %r9d, %r13d + vpshufb %xmm11, %xmm8, %xmm8 + # rnd_0: 6 - 6 + xorl %r10d, %eax + addl %edx, %r9d + andl %eax, %ebx + vpaddd %xmm8, %xmm4, %xmm4 + # rnd_0: 7 - 7 + xorl %r11d, %ebx + rorxl $6, %r13d, %edx + addl %ebx, %r9d + # rnd_1: 0 - 0 + movl %r14d, %ebx + rorxl $11, %r13d, %ecx + addl 60(%rsp), %r8d + vpshufd $0x50, %xmm4, %xmm6 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r15d, %ebx + rorxl $25, %r13d, %edx + vpsrld $10, %xmm6, %xmm9 + # rnd_1: 2 - 2 + andl %r13d, %ebx + xorl %ecx, %edx + rorxl $13, %r9d, %ecx + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 3 - 3 + addl %edx, %r8d + rorxl $2, %r9d, %edx + xorl %r15d, %ebx + vpsrlq $0x11, %xmm6, %xmm6 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r9d, %edx + addl %ebx, %r8d + vpxor %xmm7, %xmm6, %xmm6 + # rnd_1: 5 - 5 + xorl %ecx, %edx + movl %r10d, %ebx + addl %r8d, %r12d + vpxor %xmm6, %xmm9, %xmm9 + # rnd_1: 6 - 6 + xorl %r9d, %ebx + addl %edx, %r8d + andl %ebx, %eax + vpshufb %xmm12, %xmm9, %xmm9 + # rnd_1: 7 - 7 + xorl %r10d, %eax + rorxl $6, %r12d, %edx + addl %eax, %r8d + vpaddd %xmm4, %xmm9, %xmm3 + # msg_sched done: 12-15 + # set_w_k_xfer_4: 12 + vpaddd 192+L_avx1_rorx_sha256_k(%rip), %xmm0, %xmm4 + vpaddd 208+L_avx1_rorx_sha256_k(%rip), %xmm1, %xmm5 + vmovdqu %xmm4, (%rsp) + vmovdqu %xmm5, 16(%rsp) + vpaddd 224+L_avx1_rorx_sha256_k(%rip), %xmm2, %xmm6 + vpaddd 240+L_avx1_rorx_sha256_k(%rip), %xmm3, %xmm7 + vmovdqu %xmm6, 32(%rsp) + vmovdqu %xmm7, 48(%rsp) + xorl %eax, %eax + # rnd_all_4: 0-3 + rorxl $6, %r12d, %edx + rorxl $11, %r12d, %ecx + addl %eax, %r8d + addl (%rsp), %r15d + movl %r13d, %eax + xorl %edx, %ecx + xorl %r14d, %eax + rorxl $25, %r12d, %edx + xorl %ecx, %edx + andl %r12d, %eax + addl %edx, %r15d + rorxl $2, %r8d, %edx + rorxl $13, %r8d, %ecx + xorl %r14d, %eax + xorl %edx, %ecx + rorxl $22, %r8d, %edx + addl %eax, %r15d + xorl %ecx, %edx + movl %r9d, %eax + addl %r15d, %r11d + xorl %r8d, %eax + andl %eax, %ebx + addl %edx, %r15d + xorl %r9d, %ebx + rorxl $6, %r11d, %edx + rorxl $11, %r11d, %ecx + addl %ebx, %r15d + addl 4(%rsp), %r14d + movl %r12d, %ebx + xorl %edx, %ecx + xorl %r13d, %ebx + rorxl $25, %r11d, %edx + xorl %ecx, %edx + andl %r11d, %ebx + addl %edx, %r14d + rorxl $2, %r15d, %edx + rorxl $13, %r15d, %ecx + xorl %r13d, %ebx + xorl %edx, %ecx + rorxl $22, %r15d, %edx + addl %ebx, %r14d + xorl %ecx, %edx + movl %r8d, %ebx + addl %r14d, %r10d + xorl %r15d, %ebx + andl %ebx, %eax + addl %edx, %r14d + xorl %r8d, %eax + rorxl $6, %r10d, %edx + rorxl $11, %r10d, %ecx + addl %eax, %r14d + addl 8(%rsp), %r13d + movl %r11d, %eax + xorl %edx, %ecx + xorl %r12d, %eax + rorxl $25, %r10d, %edx + xorl %ecx, %edx + andl %r10d, %eax + addl %edx, %r13d + rorxl $2, %r14d, %edx + rorxl $13, %r14d, %ecx + xorl %r12d, %eax + xorl %edx, %ecx + rorxl $22, %r14d, %edx + addl %eax, %r13d + xorl %ecx, %edx + movl %r15d, %eax + addl %r13d, %r9d + xorl %r14d, %eax + andl %eax, %ebx + addl %edx, %r13d + xorl %r15d, %ebx + rorxl $6, %r9d, %edx + rorxl $11, %r9d, %ecx + addl %ebx, %r13d + addl 12(%rsp), %r12d + movl %r10d, %ebx + xorl %edx, %ecx + xorl %r11d, %ebx + rorxl $25, %r9d, %edx + xorl %ecx, %edx + andl %r9d, %ebx + addl %edx, %r12d + rorxl $2, %r13d, %edx + rorxl $13, %r13d, %ecx + xorl %r11d, %ebx + xorl %edx, %ecx + rorxl $22, %r13d, %edx + addl %ebx, %r12d + xorl %ecx, %edx + movl %r14d, %ebx + addl %r12d, %r8d + xorl %r13d, %ebx + andl %ebx, %eax + addl %edx, %r12d + xorl %r14d, %eax + # rnd_all_4: 1-4 + rorxl $6, %r8d, %edx + rorxl $11, %r8d, %ecx + addl %eax, %r12d + addl 16(%rsp), %r11d + movl %r9d, %eax + xorl %edx, %ecx + xorl %r10d, %eax + rorxl $25, %r8d, %edx + xorl %ecx, %edx + andl %r8d, %eax + addl %edx, %r11d + rorxl $2, %r12d, %edx + rorxl $13, %r12d, %ecx + xorl %r10d, %eax + xorl %edx, %ecx + rorxl $22, %r12d, %edx + addl %eax, %r11d + xorl %ecx, %edx + movl %r13d, %eax + addl %r11d, %r15d + xorl %r12d, %eax + andl %eax, %ebx + addl %edx, %r11d + xorl %r13d, %ebx + rorxl $6, %r15d, %edx + rorxl $11, %r15d, %ecx + addl %ebx, %r11d + addl 20(%rsp), %r10d + movl %r8d, %ebx + xorl %edx, %ecx + xorl %r9d, %ebx + rorxl $25, %r15d, %edx + xorl %ecx, %edx + andl %r15d, %ebx + addl %edx, %r10d + rorxl $2, %r11d, %edx + rorxl $13, %r11d, %ecx + xorl %r9d, %ebx + xorl %edx, %ecx + rorxl $22, %r11d, %edx + addl %ebx, %r10d + xorl %ecx, %edx + movl %r12d, %ebx + addl %r10d, %r14d + xorl %r11d, %ebx + andl %ebx, %eax + addl %edx, %r10d + xorl %r12d, %eax + rorxl $6, %r14d, %edx + rorxl $11, %r14d, %ecx + addl %eax, %r10d + addl 24(%rsp), %r9d + movl %r15d, %eax + xorl %edx, %ecx + xorl %r8d, %eax + rorxl $25, %r14d, %edx + xorl %ecx, %edx + andl %r14d, %eax + addl %edx, %r9d + rorxl $2, %r10d, %edx + rorxl $13, %r10d, %ecx + xorl %r8d, %eax + xorl %edx, %ecx + rorxl $22, %r10d, %edx + addl %eax, %r9d + xorl %ecx, %edx + movl %r11d, %eax + addl %r9d, %r13d + xorl %r10d, %eax + andl %eax, %ebx + addl %edx, %r9d + xorl %r11d, %ebx + rorxl $6, %r13d, %edx + rorxl $11, %r13d, %ecx + addl %ebx, %r9d + addl 28(%rsp), %r8d + movl %r14d, %ebx + xorl %edx, %ecx + xorl %r15d, %ebx + rorxl $25, %r13d, %edx + xorl %ecx, %edx + andl %r13d, %ebx + addl %edx, %r8d + rorxl $2, %r9d, %edx + rorxl $13, %r9d, %ecx + xorl %r15d, %ebx + xorl %edx, %ecx + rorxl $22, %r9d, %edx + addl %ebx, %r8d + xorl %ecx, %edx + movl %r10d, %ebx + addl %r8d, %r12d + xorl %r9d, %ebx + andl %ebx, %eax + addl %edx, %r8d + xorl %r10d, %eax + # rnd_all_4: 2-5 + rorxl $6, %r12d, %edx + rorxl $11, %r12d, %ecx + addl %eax, %r8d + addl 32(%rsp), %r15d + movl %r13d, %eax + xorl %edx, %ecx + xorl %r14d, %eax + rorxl $25, %r12d, %edx + xorl %ecx, %edx + andl %r12d, %eax + addl %edx, %r15d + rorxl $2, %r8d, %edx + rorxl $13, %r8d, %ecx + xorl %r14d, %eax + xorl %edx, %ecx + rorxl $22, %r8d, %edx + addl %eax, %r15d + xorl %ecx, %edx + movl %r9d, %eax + addl %r15d, %r11d + xorl %r8d, %eax + andl %eax, %ebx + addl %edx, %r15d + xorl %r9d, %ebx + rorxl $6, %r11d, %edx + rorxl $11, %r11d, %ecx + addl %ebx, %r15d + addl 36(%rsp), %r14d + movl %r12d, %ebx + xorl %edx, %ecx + xorl %r13d, %ebx + rorxl $25, %r11d, %edx + xorl %ecx, %edx + andl %r11d, %ebx + addl %edx, %r14d + rorxl $2, %r15d, %edx + rorxl $13, %r15d, %ecx + xorl %r13d, %ebx + xorl %edx, %ecx + rorxl $22, %r15d, %edx + addl %ebx, %r14d + xorl %ecx, %edx + movl %r8d, %ebx + addl %r14d, %r10d + xorl %r15d, %ebx + andl %ebx, %eax + addl %edx, %r14d + xorl %r8d, %eax + rorxl $6, %r10d, %edx + rorxl $11, %r10d, %ecx + addl %eax, %r14d + addl 40(%rsp), %r13d + movl %r11d, %eax + xorl %edx, %ecx + xorl %r12d, %eax + rorxl $25, %r10d, %edx + xorl %ecx, %edx + andl %r10d, %eax + addl %edx, %r13d + rorxl $2, %r14d, %edx + rorxl $13, %r14d, %ecx + xorl %r12d, %eax + xorl %edx, %ecx + rorxl $22, %r14d, %edx + addl %eax, %r13d + xorl %ecx, %edx + movl %r15d, %eax + addl %r13d, %r9d + xorl %r14d, %eax + andl %eax, %ebx + addl %edx, %r13d + xorl %r15d, %ebx + rorxl $6, %r9d, %edx + rorxl $11, %r9d, %ecx + addl %ebx, %r13d + addl 44(%rsp), %r12d + movl %r10d, %ebx + xorl %edx, %ecx + xorl %r11d, %ebx + rorxl $25, %r9d, %edx + xorl %ecx, %edx + andl %r9d, %ebx + addl %edx, %r12d + rorxl $2, %r13d, %edx + rorxl $13, %r13d, %ecx + xorl %r11d, %ebx + xorl %edx, %ecx + rorxl $22, %r13d, %edx + addl %ebx, %r12d + xorl %ecx, %edx + movl %r14d, %ebx + addl %r12d, %r8d + xorl %r13d, %ebx + andl %ebx, %eax + addl %edx, %r12d + xorl %r14d, %eax + # rnd_all_4: 3-6 + rorxl $6, %r8d, %edx + rorxl $11, %r8d, %ecx + addl %eax, %r12d + addl 48(%rsp), %r11d + movl %r9d, %eax + xorl %edx, %ecx + xorl %r10d, %eax + rorxl $25, %r8d, %edx + xorl %ecx, %edx + andl %r8d, %eax + addl %edx, %r11d + rorxl $2, %r12d, %edx + rorxl $13, %r12d, %ecx + xorl %r10d, %eax + xorl %edx, %ecx + rorxl $22, %r12d, %edx + addl %eax, %r11d + xorl %ecx, %edx + movl %r13d, %eax + addl %r11d, %r15d + xorl %r12d, %eax + andl %eax, %ebx + addl %edx, %r11d + xorl %r13d, %ebx + rorxl $6, %r15d, %edx + rorxl $11, %r15d, %ecx + addl %ebx, %r11d + addl 52(%rsp), %r10d + movl %r8d, %ebx + xorl %edx, %ecx + xorl %r9d, %ebx + rorxl $25, %r15d, %edx + xorl %ecx, %edx + andl %r15d, %ebx + addl %edx, %r10d + rorxl $2, %r11d, %edx + rorxl $13, %r11d, %ecx + xorl %r9d, %ebx + xorl %edx, %ecx + rorxl $22, %r11d, %edx + addl %ebx, %r10d + xorl %ecx, %edx + movl %r12d, %ebx + addl %r10d, %r14d + xorl %r11d, %ebx + andl %ebx, %eax + addl %edx, %r10d + xorl %r12d, %eax + rorxl $6, %r14d, %edx + rorxl $11, %r14d, %ecx + addl %eax, %r10d + addl 56(%rsp), %r9d + movl %r15d, %eax + xorl %edx, %ecx + xorl %r8d, %eax + rorxl $25, %r14d, %edx + xorl %ecx, %edx + andl %r14d, %eax + addl %edx, %r9d + rorxl $2, %r10d, %edx + rorxl $13, %r10d, %ecx + xorl %r8d, %eax + xorl %edx, %ecx + rorxl $22, %r10d, %edx + addl %eax, %r9d + xorl %ecx, %edx + movl %r11d, %eax + addl %r9d, %r13d + xorl %r10d, %eax + andl %eax, %ebx + addl %edx, %r9d + xorl %r11d, %ebx + rorxl $6, %r13d, %edx + rorxl $11, %r13d, %ecx + addl %ebx, %r9d + addl 60(%rsp), %r8d + movl %r14d, %ebx + xorl %edx, %ecx + xorl %r15d, %ebx + rorxl $25, %r13d, %edx + xorl %ecx, %edx + andl %r13d, %ebx + addl %edx, %r8d + rorxl $2, %r9d, %edx + rorxl $13, %r9d, %ecx + xorl %r15d, %ebx + xorl %edx, %ecx + rorxl $22, %r9d, %edx + addl %ebx, %r8d + xorl %ecx, %edx + movl %r10d, %ebx + addl %r8d, %r12d + xorl %r9d, %ebx + andl %ebx, %eax + addl %edx, %r8d + xorl %r10d, %eax + addl %eax, %r8d + addl %r8d, (%rdi) + addl %r9d, 4(%rdi) + addl %r10d, 8(%rdi) + addl %r11d, 12(%rdi) + addl %r12d, 16(%rdi) + addl %r13d, 20(%rdi) + addl %r14d, 24(%rdi) + addl %r15d, 28(%rdi) + xorq %rax, %rax + vzeroupper + addq $0x40, %rsp + popq %r15 + popq %r14 + popq %r13 + popq %r12 + popq %rbx + repz retq +.size Transform_Sha256_AVX1_RORX,.-Transform_Sha256_AVX1_RORX +.globl Transform_Sha256_AVX1_RORX_Len +.type Transform_Sha256_AVX1_RORX_Len,@function +.align 4 +Transform_Sha256_AVX1_RORX_Len: + pushq %rbx + pushq %r12 + pushq %r13 + pushq %r14 + pushq %r15 + pushq %rbp + movq %rsi, %rbp + movq %rdx, %rsi + subq $0x40, %rsp + vmovdqa L_avx1_rorx_sha256_flip_mask(%rip), %xmm13 + vmovdqa L_avx1_rorx_sha256_shuf_00BA(%rip), %xmm11 + vmovdqa L_avx1_rorx_sha256_shuf_DC00(%rip), %xmm12 + movl (%rdi), %r8d + movl 4(%rdi), %r9d + movl 8(%rdi), %r10d + movl 12(%rdi), %r11d + movl 16(%rdi), %r12d + movl 20(%rdi), %r13d + movl 24(%rdi), %r14d + movl 28(%rdi), %r15d + # Start of loop processing a block +L_sha256_len_avx1_len_rorx_start: + # X0, X1, X2, X3 = W[0..15] + vmovdqu (%rbp), %xmm0 + vmovdqu 16(%rbp), %xmm1 + vpshufb %xmm13, %xmm0, %xmm0 + vpshufb %xmm13, %xmm1, %xmm1 + vmovdqu 32(%rbp), %xmm2 + vmovdqu 48(%rbp), %xmm3 + vpshufb %xmm13, %xmm2, %xmm2 + vpshufb %xmm13, %xmm3, %xmm3 + # set_w_k_xfer_4: 0 + vpaddd 0+L_avx1_rorx_sha256_k(%rip), %xmm0, %xmm4 + vpaddd 16+L_avx1_rorx_sha256_k(%rip), %xmm1, %xmm5 + vmovdqu %xmm4, (%rsp) + vmovdqu %xmm5, 16(%rsp) + vpaddd 32+L_avx1_rorx_sha256_k(%rip), %xmm2, %xmm6 + vpaddd 48+L_avx1_rorx_sha256_k(%rip), %xmm3, %xmm7 + vmovdqu %xmm6, 32(%rsp) + vmovdqu %xmm7, 48(%rsp) + movl %r9d, %ebx + rorxl $6, %r12d, %edx + xorl %r10d, %ebx + # msg_sched: 0-3 + # rnd_0: 0 - 0 + movl %r13d, %eax + rorxl $11, %r12d, %ecx + addl (%rsp), %r15d + vpalignr $4, %xmm2, %xmm3, %xmm4 + vpalignr $4, %xmm0, %xmm1, %xmm5 + # rnd_0: 1 - 2 + xorl %edx, %ecx + xorl %r14d, %eax + rorxl $25, %r12d, %edx + andl %r12d, %eax + xorl %ecx, %edx + rorxl $13, %r8d, %ecx + vpsrld $7, %xmm5, %xmm6 + vpslld $25, %xmm5, %xmm7 + # rnd_0: 3 - 4 + addl %edx, %r15d + rorxl $2, %r8d, %edx + xorl %r14d, %eax + xorl %edx, %ecx + rorxl $22, %r8d, %edx + addl %eax, %r15d + vpsrld $3, %xmm5, %xmm8 + vpor %xmm6, %xmm7, %xmm7 + # rnd_0: 5 - 7 + xorl %ecx, %edx + movl %r9d, %eax + addl %r15d, %r11d + xorl %r8d, %eax + addl %edx, %r15d + andl %eax, %ebx + xorl %r9d, %ebx + rorxl $6, %r11d, %edx + addl %ebx, %r15d + # rnd_1: 0 - 0 + movl %r12d, %ebx + rorxl $11, %r11d, %ecx + addl 4(%rsp), %r14d + vpsrld $18, %xmm5, %xmm6 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r13d, %ebx + rorxl $25, %r11d, %edx + vpslld $14, %xmm5, %xmm5 + # rnd_1: 2 - 2 + andl %r11d, %ebx + xorl %ecx, %edx + rorxl $13, %r15d, %ecx + vpxor %xmm5, %xmm7, %xmm7 + # rnd_1: 3 - 3 + addl %edx, %r14d + rorxl $2, %r15d, %edx + xorl %r13d, %ebx + vpxor %xmm6, %xmm7, %xmm7 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r15d, %edx + addl %ebx, %r14d + vpshufd $0xfa, %xmm3, %xmm6 + # rnd_1: 5 - 5 + xorl %ecx, %edx + movl %r8d, %ebx + addl %r14d, %r10d + vpxor %xmm8, %xmm7, %xmm5 + # rnd_1: 6 - 6 + xorl %r15d, %ebx + addl %edx, %r14d + andl %ebx, %eax + vpsrld $10, %xmm6, %xmm8 + # rnd_1: 7 - 7 + xorl %r8d, %eax + rorxl $6, %r10d, %edx + addl %eax, %r14d + # rnd_0: 0 - 0 + movl %r11d, %eax + rorxl $11, %r10d, %ecx + addl 8(%rsp), %r13d + vpsrlq $19, %xmm6, %xmm7 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r12d, %eax + rorxl $25, %r10d, %edx + vpsrlq $0x11, %xmm6, %xmm6 + vpaddd %xmm0, %xmm4, %xmm4 + # rnd_0: 2 - 2 + andl %r10d, %eax + xorl %ecx, %edx + rorxl $13, %r14d, %ecx + vpaddd %xmm5, %xmm4, %xmm4 + # rnd_0: 3 - 3 + addl %edx, %r13d + rorxl $2, %r14d, %edx + xorl %r12d, %eax + vpxor %xmm7, %xmm6, %xmm6 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r14d, %edx + addl %eax, %r13d + vpxor %xmm6, %xmm8, %xmm8 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r15d, %eax + addl %r13d, %r9d + vpshufb %xmm11, %xmm8, %xmm8 + # rnd_0: 6 - 6 + xorl %r14d, %eax + addl %edx, %r13d + andl %eax, %ebx + vpaddd %xmm8, %xmm4, %xmm4 + # rnd_0: 7 - 7 + xorl %r15d, %ebx + rorxl $6, %r9d, %edx + addl %ebx, %r13d + # rnd_1: 0 - 0 + movl %r10d, %ebx + rorxl $11, %r9d, %ecx + addl 12(%rsp), %r12d + vpshufd $0x50, %xmm4, %xmm6 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r11d, %ebx + rorxl $25, %r9d, %edx + vpsrld $10, %xmm6, %xmm9 + # rnd_1: 2 - 2 + andl %r9d, %ebx + xorl %ecx, %edx + rorxl $13, %r13d, %ecx + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 3 - 3 + addl %edx, %r12d + rorxl $2, %r13d, %edx + xorl %r11d, %ebx + vpsrlq $0x11, %xmm6, %xmm6 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r13d, %edx + addl %ebx, %r12d + vpxor %xmm7, %xmm6, %xmm6 + # rnd_1: 5 - 5 + xorl %ecx, %edx + movl %r14d, %ebx + addl %r12d, %r8d + vpxor %xmm6, %xmm9, %xmm9 + # rnd_1: 6 - 6 + xorl %r13d, %ebx + addl %edx, %r12d + andl %ebx, %eax + vpshufb %xmm12, %xmm9, %xmm9 + # rnd_1: 7 - 7 + xorl %r14d, %eax + rorxl $6, %r8d, %edx + addl %eax, %r12d + vpaddd %xmm4, %xmm9, %xmm0 + # msg_sched done: 0-3 + # msg_sched: 4-7 + # rnd_0: 0 - 0 + movl %r9d, %eax + rorxl $11, %r8d, %ecx + addl 16(%rsp), %r11d + vpalignr $4, %xmm3, %xmm0, %xmm4 + vpalignr $4, %xmm1, %xmm2, %xmm5 + # rnd_0: 1 - 2 + xorl %edx, %ecx + xorl %r10d, %eax + rorxl $25, %r8d, %edx + andl %r8d, %eax + xorl %ecx, %edx + rorxl $13, %r12d, %ecx + vpsrld $7, %xmm5, %xmm6 + vpslld $25, %xmm5, %xmm7 + # rnd_0: 3 - 4 + addl %edx, %r11d + rorxl $2, %r12d, %edx + xorl %r10d, %eax + xorl %edx, %ecx + rorxl $22, %r12d, %edx + addl %eax, %r11d + vpsrld $3, %xmm5, %xmm8 + vpor %xmm6, %xmm7, %xmm7 + # rnd_0: 5 - 7 + xorl %ecx, %edx + movl %r13d, %eax + addl %r11d, %r15d + xorl %r12d, %eax + addl %edx, %r11d + andl %eax, %ebx + xorl %r13d, %ebx + rorxl $6, %r15d, %edx + addl %ebx, %r11d + # rnd_1: 0 - 0 + movl %r8d, %ebx + rorxl $11, %r15d, %ecx + addl 20(%rsp), %r10d + vpsrld $18, %xmm5, %xmm6 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r9d, %ebx + rorxl $25, %r15d, %edx + vpslld $14, %xmm5, %xmm5 + # rnd_1: 2 - 2 + andl %r15d, %ebx + xorl %ecx, %edx + rorxl $13, %r11d, %ecx + vpxor %xmm5, %xmm7, %xmm7 + # rnd_1: 3 - 3 + addl %edx, %r10d + rorxl $2, %r11d, %edx + xorl %r9d, %ebx + vpxor %xmm6, %xmm7, %xmm7 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r11d, %edx + addl %ebx, %r10d + vpshufd $0xfa, %xmm0, %xmm6 + # rnd_1: 5 - 5 + xorl %ecx, %edx + movl %r12d, %ebx + addl %r10d, %r14d + vpxor %xmm8, %xmm7, %xmm5 + # rnd_1: 6 - 6 + xorl %r11d, %ebx + addl %edx, %r10d + andl %ebx, %eax + vpsrld $10, %xmm6, %xmm8 + # rnd_1: 7 - 7 + xorl %r12d, %eax + rorxl $6, %r14d, %edx + addl %eax, %r10d + # rnd_0: 0 - 0 + movl %r15d, %eax + rorxl $11, %r14d, %ecx + addl 24(%rsp), %r9d + vpsrlq $19, %xmm6, %xmm7 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r8d, %eax + rorxl $25, %r14d, %edx + vpsrlq $0x11, %xmm6, %xmm6 + vpaddd %xmm1, %xmm4, %xmm4 + # rnd_0: 2 - 2 + andl %r14d, %eax + xorl %ecx, %edx + rorxl $13, %r10d, %ecx + vpaddd %xmm5, %xmm4, %xmm4 + # rnd_0: 3 - 3 + addl %edx, %r9d + rorxl $2, %r10d, %edx + xorl %r8d, %eax + vpxor %xmm7, %xmm6, %xmm6 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r10d, %edx + addl %eax, %r9d + vpxor %xmm6, %xmm8, %xmm8 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r11d, %eax + addl %r9d, %r13d + vpshufb %xmm11, %xmm8, %xmm8 + # rnd_0: 6 - 6 + xorl %r10d, %eax + addl %edx, %r9d + andl %eax, %ebx + vpaddd %xmm8, %xmm4, %xmm4 + # rnd_0: 7 - 7 + xorl %r11d, %ebx + rorxl $6, %r13d, %edx + addl %ebx, %r9d + # rnd_1: 0 - 0 + movl %r14d, %ebx + rorxl $11, %r13d, %ecx + addl 28(%rsp), %r8d + vpshufd $0x50, %xmm4, %xmm6 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r15d, %ebx + rorxl $25, %r13d, %edx + vpsrld $10, %xmm6, %xmm9 + # rnd_1: 2 - 2 + andl %r13d, %ebx + xorl %ecx, %edx + rorxl $13, %r9d, %ecx + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 3 - 3 + addl %edx, %r8d + rorxl $2, %r9d, %edx + xorl %r15d, %ebx + vpsrlq $0x11, %xmm6, %xmm6 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r9d, %edx + addl %ebx, %r8d + vpxor %xmm7, %xmm6, %xmm6 + # rnd_1: 5 - 5 + xorl %ecx, %edx + movl %r10d, %ebx + addl %r8d, %r12d + vpxor %xmm6, %xmm9, %xmm9 + # rnd_1: 6 - 6 + xorl %r9d, %ebx + addl %edx, %r8d + andl %ebx, %eax + vpshufb %xmm12, %xmm9, %xmm9 + # rnd_1: 7 - 7 + xorl %r10d, %eax + rorxl $6, %r12d, %edx + addl %eax, %r8d + vpaddd %xmm4, %xmm9, %xmm1 + # msg_sched done: 4-7 + # msg_sched: 8-11 + # rnd_0: 0 - 0 + movl %r13d, %eax + rorxl $11, %r12d, %ecx + addl 32(%rsp), %r15d + vpalignr $4, %xmm0, %xmm1, %xmm4 + vpalignr $4, %xmm2, %xmm3, %xmm5 + # rnd_0: 1 - 2 + xorl %edx, %ecx + xorl %r14d, %eax + rorxl $25, %r12d, %edx + andl %r12d, %eax + xorl %ecx, %edx + rorxl $13, %r8d, %ecx + vpsrld $7, %xmm5, %xmm6 + vpslld $25, %xmm5, %xmm7 + # rnd_0: 3 - 4 + addl %edx, %r15d + rorxl $2, %r8d, %edx + xorl %r14d, %eax + xorl %edx, %ecx + rorxl $22, %r8d, %edx + addl %eax, %r15d + vpsrld $3, %xmm5, %xmm8 + vpor %xmm6, %xmm7, %xmm7 + # rnd_0: 5 - 7 + xorl %ecx, %edx + movl %r9d, %eax + addl %r15d, %r11d + xorl %r8d, %eax + addl %edx, %r15d + andl %eax, %ebx + xorl %r9d, %ebx + rorxl $6, %r11d, %edx + addl %ebx, %r15d + # rnd_1: 0 - 0 + movl %r12d, %ebx + rorxl $11, %r11d, %ecx + addl 36(%rsp), %r14d + vpsrld $18, %xmm5, %xmm6 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r13d, %ebx + rorxl $25, %r11d, %edx + vpslld $14, %xmm5, %xmm5 + # rnd_1: 2 - 2 + andl %r11d, %ebx + xorl %ecx, %edx + rorxl $13, %r15d, %ecx + vpxor %xmm5, %xmm7, %xmm7 + # rnd_1: 3 - 3 + addl %edx, %r14d + rorxl $2, %r15d, %edx + xorl %r13d, %ebx + vpxor %xmm6, %xmm7, %xmm7 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r15d, %edx + addl %ebx, %r14d + vpshufd $0xfa, %xmm1, %xmm6 + # rnd_1: 5 - 5 + xorl %ecx, %edx + movl %r8d, %ebx + addl %r14d, %r10d + vpxor %xmm8, %xmm7, %xmm5 + # rnd_1: 6 - 6 + xorl %r15d, %ebx + addl %edx, %r14d + andl %ebx, %eax + vpsrld $10, %xmm6, %xmm8 + # rnd_1: 7 - 7 + xorl %r8d, %eax + rorxl $6, %r10d, %edx + addl %eax, %r14d + # rnd_0: 0 - 0 + movl %r11d, %eax + rorxl $11, %r10d, %ecx + addl 40(%rsp), %r13d + vpsrlq $19, %xmm6, %xmm7 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r12d, %eax + rorxl $25, %r10d, %edx + vpsrlq $0x11, %xmm6, %xmm6 + vpaddd %xmm2, %xmm4, %xmm4 + # rnd_0: 2 - 2 + andl %r10d, %eax + xorl %ecx, %edx + rorxl $13, %r14d, %ecx + vpaddd %xmm5, %xmm4, %xmm4 + # rnd_0: 3 - 3 + addl %edx, %r13d + rorxl $2, %r14d, %edx + xorl %r12d, %eax + vpxor %xmm7, %xmm6, %xmm6 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r14d, %edx + addl %eax, %r13d + vpxor %xmm6, %xmm8, %xmm8 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r15d, %eax + addl %r13d, %r9d + vpshufb %xmm11, %xmm8, %xmm8 + # rnd_0: 6 - 6 + xorl %r14d, %eax + addl %edx, %r13d + andl %eax, %ebx + vpaddd %xmm8, %xmm4, %xmm4 + # rnd_0: 7 - 7 + xorl %r15d, %ebx + rorxl $6, %r9d, %edx + addl %ebx, %r13d + # rnd_1: 0 - 0 + movl %r10d, %ebx + rorxl $11, %r9d, %ecx + addl 44(%rsp), %r12d + vpshufd $0x50, %xmm4, %xmm6 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r11d, %ebx + rorxl $25, %r9d, %edx + vpsrld $10, %xmm6, %xmm9 + # rnd_1: 2 - 2 + andl %r9d, %ebx + xorl %ecx, %edx + rorxl $13, %r13d, %ecx + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 3 - 3 + addl %edx, %r12d + rorxl $2, %r13d, %edx + xorl %r11d, %ebx + vpsrlq $0x11, %xmm6, %xmm6 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r13d, %edx + addl %ebx, %r12d + vpxor %xmm7, %xmm6, %xmm6 + # rnd_1: 5 - 5 + xorl %ecx, %edx + movl %r14d, %ebx + addl %r12d, %r8d + vpxor %xmm6, %xmm9, %xmm9 + # rnd_1: 6 - 6 + xorl %r13d, %ebx + addl %edx, %r12d + andl %ebx, %eax + vpshufb %xmm12, %xmm9, %xmm9 + # rnd_1: 7 - 7 + xorl %r14d, %eax + rorxl $6, %r8d, %edx + addl %eax, %r12d + vpaddd %xmm4, %xmm9, %xmm2 + # msg_sched done: 8-11 + # msg_sched: 12-15 + # rnd_0: 0 - 0 + movl %r9d, %eax + rorxl $11, %r8d, %ecx + addl 48(%rsp), %r11d + vpalignr $4, %xmm1, %xmm2, %xmm4 + vpalignr $4, %xmm3, %xmm0, %xmm5 + # rnd_0: 1 - 2 + xorl %edx, %ecx + xorl %r10d, %eax + rorxl $25, %r8d, %edx + andl %r8d, %eax + xorl %ecx, %edx + rorxl $13, %r12d, %ecx + vpsrld $7, %xmm5, %xmm6 + vpslld $25, %xmm5, %xmm7 + # rnd_0: 3 - 4 + addl %edx, %r11d + rorxl $2, %r12d, %edx + xorl %r10d, %eax + xorl %edx, %ecx + rorxl $22, %r12d, %edx + addl %eax, %r11d + vpsrld $3, %xmm5, %xmm8 + vpor %xmm6, %xmm7, %xmm7 + # rnd_0: 5 - 7 + xorl %ecx, %edx + movl %r13d, %eax + addl %r11d, %r15d + xorl %r12d, %eax + addl %edx, %r11d + andl %eax, %ebx + xorl %r13d, %ebx + rorxl $6, %r15d, %edx + addl %ebx, %r11d + # rnd_1: 0 - 0 + movl %r8d, %ebx + rorxl $11, %r15d, %ecx + addl 52(%rsp), %r10d + vpsrld $18, %xmm5, %xmm6 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r9d, %ebx + rorxl $25, %r15d, %edx + vpslld $14, %xmm5, %xmm5 + # rnd_1: 2 - 2 + andl %r15d, %ebx + xorl %ecx, %edx + rorxl $13, %r11d, %ecx + vpxor %xmm5, %xmm7, %xmm7 + # rnd_1: 3 - 3 + addl %edx, %r10d + rorxl $2, %r11d, %edx + xorl %r9d, %ebx + vpxor %xmm6, %xmm7, %xmm7 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r11d, %edx + addl %ebx, %r10d + vpshufd $0xfa, %xmm2, %xmm6 + # rnd_1: 5 - 5 + xorl %ecx, %edx + movl %r12d, %ebx + addl %r10d, %r14d + vpxor %xmm8, %xmm7, %xmm5 + # rnd_1: 6 - 6 + xorl %r11d, %ebx + addl %edx, %r10d + andl %ebx, %eax + vpsrld $10, %xmm6, %xmm8 + # rnd_1: 7 - 7 + xorl %r12d, %eax + rorxl $6, %r14d, %edx + addl %eax, %r10d + # rnd_0: 0 - 0 + movl %r15d, %eax + rorxl $11, %r14d, %ecx + addl 56(%rsp), %r9d + vpsrlq $19, %xmm6, %xmm7 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r8d, %eax + rorxl $25, %r14d, %edx + vpsrlq $0x11, %xmm6, %xmm6 + vpaddd %xmm3, %xmm4, %xmm4 + # rnd_0: 2 - 2 + andl %r14d, %eax + xorl %ecx, %edx + rorxl $13, %r10d, %ecx + vpaddd %xmm5, %xmm4, %xmm4 + # rnd_0: 3 - 3 + addl %edx, %r9d + rorxl $2, %r10d, %edx + xorl %r8d, %eax + vpxor %xmm7, %xmm6, %xmm6 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r10d, %edx + addl %eax, %r9d + vpxor %xmm6, %xmm8, %xmm8 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r11d, %eax + addl %r9d, %r13d + vpshufb %xmm11, %xmm8, %xmm8 + # rnd_0: 6 - 6 + xorl %r10d, %eax + addl %edx, %r9d + andl %eax, %ebx + vpaddd %xmm8, %xmm4, %xmm4 + # rnd_0: 7 - 7 + xorl %r11d, %ebx + rorxl $6, %r13d, %edx + addl %ebx, %r9d + # rnd_1: 0 - 0 + movl %r14d, %ebx + rorxl $11, %r13d, %ecx + addl 60(%rsp), %r8d + vpshufd $0x50, %xmm4, %xmm6 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r15d, %ebx + rorxl $25, %r13d, %edx + vpsrld $10, %xmm6, %xmm9 + # rnd_1: 2 - 2 + andl %r13d, %ebx + xorl %ecx, %edx + rorxl $13, %r9d, %ecx + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 3 - 3 + addl %edx, %r8d + rorxl $2, %r9d, %edx + xorl %r15d, %ebx + vpsrlq $0x11, %xmm6, %xmm6 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r9d, %edx + addl %ebx, %r8d + vpxor %xmm7, %xmm6, %xmm6 + # rnd_1: 5 - 5 + xorl %ecx, %edx + movl %r10d, %ebx + addl %r8d, %r12d + vpxor %xmm6, %xmm9, %xmm9 + # rnd_1: 6 - 6 + xorl %r9d, %ebx + addl %edx, %r8d + andl %ebx, %eax + vpshufb %xmm12, %xmm9, %xmm9 + # rnd_1: 7 - 7 + xorl %r10d, %eax + rorxl $6, %r12d, %edx + addl %eax, %r8d + vpaddd %xmm4, %xmm9, %xmm3 + # msg_sched done: 12-15 + # set_w_k_xfer_4: 4 + vpaddd 64+L_avx1_rorx_sha256_k(%rip), %xmm0, %xmm4 + vpaddd 80+L_avx1_rorx_sha256_k(%rip), %xmm1, %xmm5 + vmovdqu %xmm4, (%rsp) + vmovdqu %xmm5, 16(%rsp) + vpaddd 96+L_avx1_rorx_sha256_k(%rip), %xmm2, %xmm6 + vpaddd 112+L_avx1_rorx_sha256_k(%rip), %xmm3, %xmm7 + vmovdqu %xmm6, 32(%rsp) + vmovdqu %xmm7, 48(%rsp) + # msg_sched: 0-3 + # rnd_0: 0 - 0 + movl %r13d, %eax + rorxl $11, %r12d, %ecx + addl (%rsp), %r15d + vpalignr $4, %xmm2, %xmm3, %xmm4 + vpalignr $4, %xmm0, %xmm1, %xmm5 + # rnd_0: 1 - 2 + xorl %edx, %ecx + xorl %r14d, %eax + rorxl $25, %r12d, %edx + andl %r12d, %eax + xorl %ecx, %edx + rorxl $13, %r8d, %ecx + vpsrld $7, %xmm5, %xmm6 + vpslld $25, %xmm5, %xmm7 + # rnd_0: 3 - 4 + addl %edx, %r15d + rorxl $2, %r8d, %edx + xorl %r14d, %eax + xorl %edx, %ecx + rorxl $22, %r8d, %edx + addl %eax, %r15d + vpsrld $3, %xmm5, %xmm8 + vpor %xmm6, %xmm7, %xmm7 + # rnd_0: 5 - 7 + xorl %ecx, %edx + movl %r9d, %eax + addl %r15d, %r11d + xorl %r8d, %eax + addl %edx, %r15d + andl %eax, %ebx + xorl %r9d, %ebx + rorxl $6, %r11d, %edx + addl %ebx, %r15d + # rnd_1: 0 - 0 + movl %r12d, %ebx + rorxl $11, %r11d, %ecx + addl 4(%rsp), %r14d + vpsrld $18, %xmm5, %xmm6 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r13d, %ebx + rorxl $25, %r11d, %edx + vpslld $14, %xmm5, %xmm5 + # rnd_1: 2 - 2 + andl %r11d, %ebx + xorl %ecx, %edx + rorxl $13, %r15d, %ecx + vpxor %xmm5, %xmm7, %xmm7 + # rnd_1: 3 - 3 + addl %edx, %r14d + rorxl $2, %r15d, %edx + xorl %r13d, %ebx + vpxor %xmm6, %xmm7, %xmm7 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r15d, %edx + addl %ebx, %r14d + vpshufd $0xfa, %xmm3, %xmm6 + # rnd_1: 5 - 5 + xorl %ecx, %edx + movl %r8d, %ebx + addl %r14d, %r10d + vpxor %xmm8, %xmm7, %xmm5 + # rnd_1: 6 - 6 + xorl %r15d, %ebx + addl %edx, %r14d + andl %ebx, %eax + vpsrld $10, %xmm6, %xmm8 + # rnd_1: 7 - 7 + xorl %r8d, %eax + rorxl $6, %r10d, %edx + addl %eax, %r14d + # rnd_0: 0 - 0 + movl %r11d, %eax + rorxl $11, %r10d, %ecx + addl 8(%rsp), %r13d + vpsrlq $19, %xmm6, %xmm7 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r12d, %eax + rorxl $25, %r10d, %edx + vpsrlq $0x11, %xmm6, %xmm6 + vpaddd %xmm0, %xmm4, %xmm4 + # rnd_0: 2 - 2 + andl %r10d, %eax + xorl %ecx, %edx + rorxl $13, %r14d, %ecx + vpaddd %xmm5, %xmm4, %xmm4 + # rnd_0: 3 - 3 + addl %edx, %r13d + rorxl $2, %r14d, %edx + xorl %r12d, %eax + vpxor %xmm7, %xmm6, %xmm6 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r14d, %edx + addl %eax, %r13d + vpxor %xmm6, %xmm8, %xmm8 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r15d, %eax + addl %r13d, %r9d + vpshufb %xmm11, %xmm8, %xmm8 + # rnd_0: 6 - 6 + xorl %r14d, %eax + addl %edx, %r13d + andl %eax, %ebx + vpaddd %xmm8, %xmm4, %xmm4 + # rnd_0: 7 - 7 + xorl %r15d, %ebx + rorxl $6, %r9d, %edx + addl %ebx, %r13d + # rnd_1: 0 - 0 + movl %r10d, %ebx + rorxl $11, %r9d, %ecx + addl 12(%rsp), %r12d + vpshufd $0x50, %xmm4, %xmm6 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r11d, %ebx + rorxl $25, %r9d, %edx + vpsrld $10, %xmm6, %xmm9 + # rnd_1: 2 - 2 + andl %r9d, %ebx + xorl %ecx, %edx + rorxl $13, %r13d, %ecx + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 3 - 3 + addl %edx, %r12d + rorxl $2, %r13d, %edx + xorl %r11d, %ebx + vpsrlq $0x11, %xmm6, %xmm6 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r13d, %edx + addl %ebx, %r12d + vpxor %xmm7, %xmm6, %xmm6 + # rnd_1: 5 - 5 + xorl %ecx, %edx + movl %r14d, %ebx + addl %r12d, %r8d + vpxor %xmm6, %xmm9, %xmm9 + # rnd_1: 6 - 6 + xorl %r13d, %ebx + addl %edx, %r12d + andl %ebx, %eax + vpshufb %xmm12, %xmm9, %xmm9 + # rnd_1: 7 - 7 + xorl %r14d, %eax + rorxl $6, %r8d, %edx + addl %eax, %r12d + vpaddd %xmm4, %xmm9, %xmm0 + # msg_sched done: 0-3 + # msg_sched: 4-7 + # rnd_0: 0 - 0 + movl %r9d, %eax + rorxl $11, %r8d, %ecx + addl 16(%rsp), %r11d + vpalignr $4, %xmm3, %xmm0, %xmm4 + vpalignr $4, %xmm1, %xmm2, %xmm5 + # rnd_0: 1 - 2 + xorl %edx, %ecx + xorl %r10d, %eax + rorxl $25, %r8d, %edx + andl %r8d, %eax + xorl %ecx, %edx + rorxl $13, %r12d, %ecx + vpsrld $7, %xmm5, %xmm6 + vpslld $25, %xmm5, %xmm7 + # rnd_0: 3 - 4 + addl %edx, %r11d + rorxl $2, %r12d, %edx + xorl %r10d, %eax + xorl %edx, %ecx + rorxl $22, %r12d, %edx + addl %eax, %r11d + vpsrld $3, %xmm5, %xmm8 + vpor %xmm6, %xmm7, %xmm7 + # rnd_0: 5 - 7 + xorl %ecx, %edx + movl %r13d, %eax + addl %r11d, %r15d + xorl %r12d, %eax + addl %edx, %r11d + andl %eax, %ebx + xorl %r13d, %ebx + rorxl $6, %r15d, %edx + addl %ebx, %r11d + # rnd_1: 0 - 0 + movl %r8d, %ebx + rorxl $11, %r15d, %ecx + addl 20(%rsp), %r10d + vpsrld $18, %xmm5, %xmm6 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r9d, %ebx + rorxl $25, %r15d, %edx + vpslld $14, %xmm5, %xmm5 + # rnd_1: 2 - 2 + andl %r15d, %ebx + xorl %ecx, %edx + rorxl $13, %r11d, %ecx + vpxor %xmm5, %xmm7, %xmm7 + # rnd_1: 3 - 3 + addl %edx, %r10d + rorxl $2, %r11d, %edx + xorl %r9d, %ebx + vpxor %xmm6, %xmm7, %xmm7 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r11d, %edx + addl %ebx, %r10d + vpshufd $0xfa, %xmm0, %xmm6 + # rnd_1: 5 - 5 + xorl %ecx, %edx + movl %r12d, %ebx + addl %r10d, %r14d + vpxor %xmm8, %xmm7, %xmm5 + # rnd_1: 6 - 6 + xorl %r11d, %ebx + addl %edx, %r10d + andl %ebx, %eax + vpsrld $10, %xmm6, %xmm8 + # rnd_1: 7 - 7 + xorl %r12d, %eax + rorxl $6, %r14d, %edx + addl %eax, %r10d + # rnd_0: 0 - 0 + movl %r15d, %eax + rorxl $11, %r14d, %ecx + addl 24(%rsp), %r9d + vpsrlq $19, %xmm6, %xmm7 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r8d, %eax + rorxl $25, %r14d, %edx + vpsrlq $0x11, %xmm6, %xmm6 + vpaddd %xmm1, %xmm4, %xmm4 + # rnd_0: 2 - 2 + andl %r14d, %eax + xorl %ecx, %edx + rorxl $13, %r10d, %ecx + vpaddd %xmm5, %xmm4, %xmm4 + # rnd_0: 3 - 3 + addl %edx, %r9d + rorxl $2, %r10d, %edx + xorl %r8d, %eax + vpxor %xmm7, %xmm6, %xmm6 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r10d, %edx + addl %eax, %r9d + vpxor %xmm6, %xmm8, %xmm8 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r11d, %eax + addl %r9d, %r13d + vpshufb %xmm11, %xmm8, %xmm8 + # rnd_0: 6 - 6 + xorl %r10d, %eax + addl %edx, %r9d + andl %eax, %ebx + vpaddd %xmm8, %xmm4, %xmm4 + # rnd_0: 7 - 7 + xorl %r11d, %ebx + rorxl $6, %r13d, %edx + addl %ebx, %r9d + # rnd_1: 0 - 0 + movl %r14d, %ebx + rorxl $11, %r13d, %ecx + addl 28(%rsp), %r8d + vpshufd $0x50, %xmm4, %xmm6 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r15d, %ebx + rorxl $25, %r13d, %edx + vpsrld $10, %xmm6, %xmm9 + # rnd_1: 2 - 2 + andl %r13d, %ebx + xorl %ecx, %edx + rorxl $13, %r9d, %ecx + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 3 - 3 + addl %edx, %r8d + rorxl $2, %r9d, %edx + xorl %r15d, %ebx + vpsrlq $0x11, %xmm6, %xmm6 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r9d, %edx + addl %ebx, %r8d + vpxor %xmm7, %xmm6, %xmm6 + # rnd_1: 5 - 5 + xorl %ecx, %edx + movl %r10d, %ebx + addl %r8d, %r12d + vpxor %xmm6, %xmm9, %xmm9 + # rnd_1: 6 - 6 + xorl %r9d, %ebx + addl %edx, %r8d + andl %ebx, %eax + vpshufb %xmm12, %xmm9, %xmm9 + # rnd_1: 7 - 7 + xorl %r10d, %eax + rorxl $6, %r12d, %edx + addl %eax, %r8d + vpaddd %xmm4, %xmm9, %xmm1 + # msg_sched done: 4-7 + # msg_sched: 8-11 + # rnd_0: 0 - 0 + movl %r13d, %eax + rorxl $11, %r12d, %ecx + addl 32(%rsp), %r15d + vpalignr $4, %xmm0, %xmm1, %xmm4 + vpalignr $4, %xmm2, %xmm3, %xmm5 + # rnd_0: 1 - 2 + xorl %edx, %ecx + xorl %r14d, %eax + rorxl $25, %r12d, %edx + andl %r12d, %eax + xorl %ecx, %edx + rorxl $13, %r8d, %ecx + vpsrld $7, %xmm5, %xmm6 + vpslld $25, %xmm5, %xmm7 + # rnd_0: 3 - 4 + addl %edx, %r15d + rorxl $2, %r8d, %edx + xorl %r14d, %eax + xorl %edx, %ecx + rorxl $22, %r8d, %edx + addl %eax, %r15d + vpsrld $3, %xmm5, %xmm8 + vpor %xmm6, %xmm7, %xmm7 + # rnd_0: 5 - 7 + xorl %ecx, %edx + movl %r9d, %eax + addl %r15d, %r11d + xorl %r8d, %eax + addl %edx, %r15d + andl %eax, %ebx + xorl %r9d, %ebx + rorxl $6, %r11d, %edx + addl %ebx, %r15d + # rnd_1: 0 - 0 + movl %r12d, %ebx + rorxl $11, %r11d, %ecx + addl 36(%rsp), %r14d + vpsrld $18, %xmm5, %xmm6 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r13d, %ebx + rorxl $25, %r11d, %edx + vpslld $14, %xmm5, %xmm5 + # rnd_1: 2 - 2 + andl %r11d, %ebx + xorl %ecx, %edx + rorxl $13, %r15d, %ecx + vpxor %xmm5, %xmm7, %xmm7 + # rnd_1: 3 - 3 + addl %edx, %r14d + rorxl $2, %r15d, %edx + xorl %r13d, %ebx + vpxor %xmm6, %xmm7, %xmm7 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r15d, %edx + addl %ebx, %r14d + vpshufd $0xfa, %xmm1, %xmm6 + # rnd_1: 5 - 5 + xorl %ecx, %edx + movl %r8d, %ebx + addl %r14d, %r10d + vpxor %xmm8, %xmm7, %xmm5 + # rnd_1: 6 - 6 + xorl %r15d, %ebx + addl %edx, %r14d + andl %ebx, %eax + vpsrld $10, %xmm6, %xmm8 + # rnd_1: 7 - 7 + xorl %r8d, %eax + rorxl $6, %r10d, %edx + addl %eax, %r14d + # rnd_0: 0 - 0 + movl %r11d, %eax + rorxl $11, %r10d, %ecx + addl 40(%rsp), %r13d + vpsrlq $19, %xmm6, %xmm7 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r12d, %eax + rorxl $25, %r10d, %edx + vpsrlq $0x11, %xmm6, %xmm6 + vpaddd %xmm2, %xmm4, %xmm4 + # rnd_0: 2 - 2 + andl %r10d, %eax + xorl %ecx, %edx + rorxl $13, %r14d, %ecx + vpaddd %xmm5, %xmm4, %xmm4 + # rnd_0: 3 - 3 + addl %edx, %r13d + rorxl $2, %r14d, %edx + xorl %r12d, %eax + vpxor %xmm7, %xmm6, %xmm6 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r14d, %edx + addl %eax, %r13d + vpxor %xmm6, %xmm8, %xmm8 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r15d, %eax + addl %r13d, %r9d + vpshufb %xmm11, %xmm8, %xmm8 + # rnd_0: 6 - 6 + xorl %r14d, %eax + addl %edx, %r13d + andl %eax, %ebx + vpaddd %xmm8, %xmm4, %xmm4 + # rnd_0: 7 - 7 + xorl %r15d, %ebx + rorxl $6, %r9d, %edx + addl %ebx, %r13d + # rnd_1: 0 - 0 + movl %r10d, %ebx + rorxl $11, %r9d, %ecx + addl 44(%rsp), %r12d + vpshufd $0x50, %xmm4, %xmm6 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r11d, %ebx + rorxl $25, %r9d, %edx + vpsrld $10, %xmm6, %xmm9 + # rnd_1: 2 - 2 + andl %r9d, %ebx + xorl %ecx, %edx + rorxl $13, %r13d, %ecx + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 3 - 3 + addl %edx, %r12d + rorxl $2, %r13d, %edx + xorl %r11d, %ebx + vpsrlq $0x11, %xmm6, %xmm6 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r13d, %edx + addl %ebx, %r12d + vpxor %xmm7, %xmm6, %xmm6 + # rnd_1: 5 - 5 + xorl %ecx, %edx + movl %r14d, %ebx + addl %r12d, %r8d + vpxor %xmm6, %xmm9, %xmm9 + # rnd_1: 6 - 6 + xorl %r13d, %ebx + addl %edx, %r12d + andl %ebx, %eax + vpshufb %xmm12, %xmm9, %xmm9 + # rnd_1: 7 - 7 + xorl %r14d, %eax + rorxl $6, %r8d, %edx + addl %eax, %r12d + vpaddd %xmm4, %xmm9, %xmm2 + # msg_sched done: 8-11 + # msg_sched: 12-15 + # rnd_0: 0 - 0 + movl %r9d, %eax + rorxl $11, %r8d, %ecx + addl 48(%rsp), %r11d + vpalignr $4, %xmm1, %xmm2, %xmm4 + vpalignr $4, %xmm3, %xmm0, %xmm5 + # rnd_0: 1 - 2 + xorl %edx, %ecx + xorl %r10d, %eax + rorxl $25, %r8d, %edx + andl %r8d, %eax + xorl %ecx, %edx + rorxl $13, %r12d, %ecx + vpsrld $7, %xmm5, %xmm6 + vpslld $25, %xmm5, %xmm7 + # rnd_0: 3 - 4 + addl %edx, %r11d + rorxl $2, %r12d, %edx + xorl %r10d, %eax + xorl %edx, %ecx + rorxl $22, %r12d, %edx + addl %eax, %r11d + vpsrld $3, %xmm5, %xmm8 + vpor %xmm6, %xmm7, %xmm7 + # rnd_0: 5 - 7 + xorl %ecx, %edx + movl %r13d, %eax + addl %r11d, %r15d + xorl %r12d, %eax + addl %edx, %r11d + andl %eax, %ebx + xorl %r13d, %ebx + rorxl $6, %r15d, %edx + addl %ebx, %r11d + # rnd_1: 0 - 0 + movl %r8d, %ebx + rorxl $11, %r15d, %ecx + addl 52(%rsp), %r10d + vpsrld $18, %xmm5, %xmm6 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r9d, %ebx + rorxl $25, %r15d, %edx + vpslld $14, %xmm5, %xmm5 + # rnd_1: 2 - 2 + andl %r15d, %ebx + xorl %ecx, %edx + rorxl $13, %r11d, %ecx + vpxor %xmm5, %xmm7, %xmm7 + # rnd_1: 3 - 3 + addl %edx, %r10d + rorxl $2, %r11d, %edx + xorl %r9d, %ebx + vpxor %xmm6, %xmm7, %xmm7 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r11d, %edx + addl %ebx, %r10d + vpshufd $0xfa, %xmm2, %xmm6 + # rnd_1: 5 - 5 + xorl %ecx, %edx + movl %r12d, %ebx + addl %r10d, %r14d + vpxor %xmm8, %xmm7, %xmm5 + # rnd_1: 6 - 6 + xorl %r11d, %ebx + addl %edx, %r10d + andl %ebx, %eax + vpsrld $10, %xmm6, %xmm8 + # rnd_1: 7 - 7 + xorl %r12d, %eax + rorxl $6, %r14d, %edx + addl %eax, %r10d + # rnd_0: 0 - 0 + movl %r15d, %eax + rorxl $11, %r14d, %ecx + addl 56(%rsp), %r9d + vpsrlq $19, %xmm6, %xmm7 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r8d, %eax + rorxl $25, %r14d, %edx + vpsrlq $0x11, %xmm6, %xmm6 + vpaddd %xmm3, %xmm4, %xmm4 + # rnd_0: 2 - 2 + andl %r14d, %eax + xorl %ecx, %edx + rorxl $13, %r10d, %ecx + vpaddd %xmm5, %xmm4, %xmm4 + # rnd_0: 3 - 3 + addl %edx, %r9d + rorxl $2, %r10d, %edx + xorl %r8d, %eax + vpxor %xmm7, %xmm6, %xmm6 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r10d, %edx + addl %eax, %r9d + vpxor %xmm6, %xmm8, %xmm8 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r11d, %eax + addl %r9d, %r13d + vpshufb %xmm11, %xmm8, %xmm8 + # rnd_0: 6 - 6 + xorl %r10d, %eax + addl %edx, %r9d + andl %eax, %ebx + vpaddd %xmm8, %xmm4, %xmm4 + # rnd_0: 7 - 7 + xorl %r11d, %ebx + rorxl $6, %r13d, %edx + addl %ebx, %r9d + # rnd_1: 0 - 0 + movl %r14d, %ebx + rorxl $11, %r13d, %ecx + addl 60(%rsp), %r8d + vpshufd $0x50, %xmm4, %xmm6 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r15d, %ebx + rorxl $25, %r13d, %edx + vpsrld $10, %xmm6, %xmm9 + # rnd_1: 2 - 2 + andl %r13d, %ebx + xorl %ecx, %edx + rorxl $13, %r9d, %ecx + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 3 - 3 + addl %edx, %r8d + rorxl $2, %r9d, %edx + xorl %r15d, %ebx + vpsrlq $0x11, %xmm6, %xmm6 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r9d, %edx + addl %ebx, %r8d + vpxor %xmm7, %xmm6, %xmm6 + # rnd_1: 5 - 5 + xorl %ecx, %edx + movl %r10d, %ebx + addl %r8d, %r12d + vpxor %xmm6, %xmm9, %xmm9 + # rnd_1: 6 - 6 + xorl %r9d, %ebx + addl %edx, %r8d + andl %ebx, %eax + vpshufb %xmm12, %xmm9, %xmm9 + # rnd_1: 7 - 7 + xorl %r10d, %eax + rorxl $6, %r12d, %edx + addl %eax, %r8d + vpaddd %xmm4, %xmm9, %xmm3 + # msg_sched done: 12-15 + # set_w_k_xfer_4: 8 + vpaddd 128+L_avx1_rorx_sha256_k(%rip), %xmm0, %xmm4 + vpaddd 144+L_avx1_rorx_sha256_k(%rip), %xmm1, %xmm5 + vmovdqu %xmm4, (%rsp) + vmovdqu %xmm5, 16(%rsp) + vpaddd 160+L_avx1_rorx_sha256_k(%rip), %xmm2, %xmm6 + vpaddd 176+L_avx1_rorx_sha256_k(%rip), %xmm3, %xmm7 + vmovdqu %xmm6, 32(%rsp) + vmovdqu %xmm7, 48(%rsp) + # msg_sched: 0-3 + # rnd_0: 0 - 0 + movl %r13d, %eax + rorxl $11, %r12d, %ecx + addl (%rsp), %r15d + vpalignr $4, %xmm2, %xmm3, %xmm4 + vpalignr $4, %xmm0, %xmm1, %xmm5 + # rnd_0: 1 - 2 + xorl %edx, %ecx + xorl %r14d, %eax + rorxl $25, %r12d, %edx + andl %r12d, %eax + xorl %ecx, %edx + rorxl $13, %r8d, %ecx + vpsrld $7, %xmm5, %xmm6 + vpslld $25, %xmm5, %xmm7 + # rnd_0: 3 - 4 + addl %edx, %r15d + rorxl $2, %r8d, %edx + xorl %r14d, %eax + xorl %edx, %ecx + rorxl $22, %r8d, %edx + addl %eax, %r15d + vpsrld $3, %xmm5, %xmm8 + vpor %xmm6, %xmm7, %xmm7 + # rnd_0: 5 - 7 + xorl %ecx, %edx + movl %r9d, %eax + addl %r15d, %r11d + xorl %r8d, %eax + addl %edx, %r15d + andl %eax, %ebx + xorl %r9d, %ebx + rorxl $6, %r11d, %edx + addl %ebx, %r15d + # rnd_1: 0 - 0 + movl %r12d, %ebx + rorxl $11, %r11d, %ecx + addl 4(%rsp), %r14d + vpsrld $18, %xmm5, %xmm6 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r13d, %ebx + rorxl $25, %r11d, %edx + vpslld $14, %xmm5, %xmm5 + # rnd_1: 2 - 2 + andl %r11d, %ebx + xorl %ecx, %edx + rorxl $13, %r15d, %ecx + vpxor %xmm5, %xmm7, %xmm7 + # rnd_1: 3 - 3 + addl %edx, %r14d + rorxl $2, %r15d, %edx + xorl %r13d, %ebx + vpxor %xmm6, %xmm7, %xmm7 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r15d, %edx + addl %ebx, %r14d + vpshufd $0xfa, %xmm3, %xmm6 + # rnd_1: 5 - 5 + xorl %ecx, %edx + movl %r8d, %ebx + addl %r14d, %r10d + vpxor %xmm8, %xmm7, %xmm5 + # rnd_1: 6 - 6 + xorl %r15d, %ebx + addl %edx, %r14d + andl %ebx, %eax + vpsrld $10, %xmm6, %xmm8 + # rnd_1: 7 - 7 + xorl %r8d, %eax + rorxl $6, %r10d, %edx + addl %eax, %r14d + # rnd_0: 0 - 0 + movl %r11d, %eax + rorxl $11, %r10d, %ecx + addl 8(%rsp), %r13d + vpsrlq $19, %xmm6, %xmm7 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r12d, %eax + rorxl $25, %r10d, %edx + vpsrlq $0x11, %xmm6, %xmm6 + vpaddd %xmm0, %xmm4, %xmm4 + # rnd_0: 2 - 2 + andl %r10d, %eax + xorl %ecx, %edx + rorxl $13, %r14d, %ecx + vpaddd %xmm5, %xmm4, %xmm4 + # rnd_0: 3 - 3 + addl %edx, %r13d + rorxl $2, %r14d, %edx + xorl %r12d, %eax + vpxor %xmm7, %xmm6, %xmm6 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r14d, %edx + addl %eax, %r13d + vpxor %xmm6, %xmm8, %xmm8 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r15d, %eax + addl %r13d, %r9d + vpshufb %xmm11, %xmm8, %xmm8 + # rnd_0: 6 - 6 + xorl %r14d, %eax + addl %edx, %r13d + andl %eax, %ebx + vpaddd %xmm8, %xmm4, %xmm4 + # rnd_0: 7 - 7 + xorl %r15d, %ebx + rorxl $6, %r9d, %edx + addl %ebx, %r13d + # rnd_1: 0 - 0 + movl %r10d, %ebx + rorxl $11, %r9d, %ecx + addl 12(%rsp), %r12d + vpshufd $0x50, %xmm4, %xmm6 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r11d, %ebx + rorxl $25, %r9d, %edx + vpsrld $10, %xmm6, %xmm9 + # rnd_1: 2 - 2 + andl %r9d, %ebx + xorl %ecx, %edx + rorxl $13, %r13d, %ecx + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 3 - 3 + addl %edx, %r12d + rorxl $2, %r13d, %edx + xorl %r11d, %ebx + vpsrlq $0x11, %xmm6, %xmm6 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r13d, %edx + addl %ebx, %r12d + vpxor %xmm7, %xmm6, %xmm6 + # rnd_1: 5 - 5 + xorl %ecx, %edx + movl %r14d, %ebx + addl %r12d, %r8d + vpxor %xmm6, %xmm9, %xmm9 + # rnd_1: 6 - 6 + xorl %r13d, %ebx + addl %edx, %r12d + andl %ebx, %eax + vpshufb %xmm12, %xmm9, %xmm9 + # rnd_1: 7 - 7 + xorl %r14d, %eax + rorxl $6, %r8d, %edx + addl %eax, %r12d + vpaddd %xmm4, %xmm9, %xmm0 + # msg_sched done: 0-3 + # msg_sched: 4-7 + # rnd_0: 0 - 0 + movl %r9d, %eax + rorxl $11, %r8d, %ecx + addl 16(%rsp), %r11d + vpalignr $4, %xmm3, %xmm0, %xmm4 + vpalignr $4, %xmm1, %xmm2, %xmm5 + # rnd_0: 1 - 2 + xorl %edx, %ecx + xorl %r10d, %eax + rorxl $25, %r8d, %edx + andl %r8d, %eax + xorl %ecx, %edx + rorxl $13, %r12d, %ecx + vpsrld $7, %xmm5, %xmm6 + vpslld $25, %xmm5, %xmm7 + # rnd_0: 3 - 4 + addl %edx, %r11d + rorxl $2, %r12d, %edx + xorl %r10d, %eax + xorl %edx, %ecx + rorxl $22, %r12d, %edx + addl %eax, %r11d + vpsrld $3, %xmm5, %xmm8 + vpor %xmm6, %xmm7, %xmm7 + # rnd_0: 5 - 7 + xorl %ecx, %edx + movl %r13d, %eax + addl %r11d, %r15d + xorl %r12d, %eax + addl %edx, %r11d + andl %eax, %ebx + xorl %r13d, %ebx + rorxl $6, %r15d, %edx + addl %ebx, %r11d + # rnd_1: 0 - 0 + movl %r8d, %ebx + rorxl $11, %r15d, %ecx + addl 20(%rsp), %r10d + vpsrld $18, %xmm5, %xmm6 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r9d, %ebx + rorxl $25, %r15d, %edx + vpslld $14, %xmm5, %xmm5 + # rnd_1: 2 - 2 + andl %r15d, %ebx + xorl %ecx, %edx + rorxl $13, %r11d, %ecx + vpxor %xmm5, %xmm7, %xmm7 + # rnd_1: 3 - 3 + addl %edx, %r10d + rorxl $2, %r11d, %edx + xorl %r9d, %ebx + vpxor %xmm6, %xmm7, %xmm7 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r11d, %edx + addl %ebx, %r10d + vpshufd $0xfa, %xmm0, %xmm6 + # rnd_1: 5 - 5 + xorl %ecx, %edx + movl %r12d, %ebx + addl %r10d, %r14d + vpxor %xmm8, %xmm7, %xmm5 + # rnd_1: 6 - 6 + xorl %r11d, %ebx + addl %edx, %r10d + andl %ebx, %eax + vpsrld $10, %xmm6, %xmm8 + # rnd_1: 7 - 7 + xorl %r12d, %eax + rorxl $6, %r14d, %edx + addl %eax, %r10d + # rnd_0: 0 - 0 + movl %r15d, %eax + rorxl $11, %r14d, %ecx + addl 24(%rsp), %r9d + vpsrlq $19, %xmm6, %xmm7 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r8d, %eax + rorxl $25, %r14d, %edx + vpsrlq $0x11, %xmm6, %xmm6 + vpaddd %xmm1, %xmm4, %xmm4 + # rnd_0: 2 - 2 + andl %r14d, %eax + xorl %ecx, %edx + rorxl $13, %r10d, %ecx + vpaddd %xmm5, %xmm4, %xmm4 + # rnd_0: 3 - 3 + addl %edx, %r9d + rorxl $2, %r10d, %edx + xorl %r8d, %eax + vpxor %xmm7, %xmm6, %xmm6 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r10d, %edx + addl %eax, %r9d + vpxor %xmm6, %xmm8, %xmm8 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r11d, %eax + addl %r9d, %r13d + vpshufb %xmm11, %xmm8, %xmm8 + # rnd_0: 6 - 6 + xorl %r10d, %eax + addl %edx, %r9d + andl %eax, %ebx + vpaddd %xmm8, %xmm4, %xmm4 + # rnd_0: 7 - 7 + xorl %r11d, %ebx + rorxl $6, %r13d, %edx + addl %ebx, %r9d + # rnd_1: 0 - 0 + movl %r14d, %ebx + rorxl $11, %r13d, %ecx + addl 28(%rsp), %r8d + vpshufd $0x50, %xmm4, %xmm6 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r15d, %ebx + rorxl $25, %r13d, %edx + vpsrld $10, %xmm6, %xmm9 + # rnd_1: 2 - 2 + andl %r13d, %ebx + xorl %ecx, %edx + rorxl $13, %r9d, %ecx + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 3 - 3 + addl %edx, %r8d + rorxl $2, %r9d, %edx + xorl %r15d, %ebx + vpsrlq $0x11, %xmm6, %xmm6 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r9d, %edx + addl %ebx, %r8d + vpxor %xmm7, %xmm6, %xmm6 + # rnd_1: 5 - 5 + xorl %ecx, %edx + movl %r10d, %ebx + addl %r8d, %r12d + vpxor %xmm6, %xmm9, %xmm9 + # rnd_1: 6 - 6 + xorl %r9d, %ebx + addl %edx, %r8d + andl %ebx, %eax + vpshufb %xmm12, %xmm9, %xmm9 + # rnd_1: 7 - 7 + xorl %r10d, %eax + rorxl $6, %r12d, %edx + addl %eax, %r8d + vpaddd %xmm4, %xmm9, %xmm1 + # msg_sched done: 4-7 + # msg_sched: 8-11 + # rnd_0: 0 - 0 + movl %r13d, %eax + rorxl $11, %r12d, %ecx + addl 32(%rsp), %r15d + vpalignr $4, %xmm0, %xmm1, %xmm4 + vpalignr $4, %xmm2, %xmm3, %xmm5 + # rnd_0: 1 - 2 + xorl %edx, %ecx + xorl %r14d, %eax + rorxl $25, %r12d, %edx + andl %r12d, %eax + xorl %ecx, %edx + rorxl $13, %r8d, %ecx + vpsrld $7, %xmm5, %xmm6 + vpslld $25, %xmm5, %xmm7 + # rnd_0: 3 - 4 + addl %edx, %r15d + rorxl $2, %r8d, %edx + xorl %r14d, %eax + xorl %edx, %ecx + rorxl $22, %r8d, %edx + addl %eax, %r15d + vpsrld $3, %xmm5, %xmm8 + vpor %xmm6, %xmm7, %xmm7 + # rnd_0: 5 - 7 + xorl %ecx, %edx + movl %r9d, %eax + addl %r15d, %r11d + xorl %r8d, %eax + addl %edx, %r15d + andl %eax, %ebx + xorl %r9d, %ebx + rorxl $6, %r11d, %edx + addl %ebx, %r15d + # rnd_1: 0 - 0 + movl %r12d, %ebx + rorxl $11, %r11d, %ecx + addl 36(%rsp), %r14d + vpsrld $18, %xmm5, %xmm6 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r13d, %ebx + rorxl $25, %r11d, %edx + vpslld $14, %xmm5, %xmm5 + # rnd_1: 2 - 2 + andl %r11d, %ebx + xorl %ecx, %edx + rorxl $13, %r15d, %ecx + vpxor %xmm5, %xmm7, %xmm7 + # rnd_1: 3 - 3 + addl %edx, %r14d + rorxl $2, %r15d, %edx + xorl %r13d, %ebx + vpxor %xmm6, %xmm7, %xmm7 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r15d, %edx + addl %ebx, %r14d + vpshufd $0xfa, %xmm1, %xmm6 + # rnd_1: 5 - 5 + xorl %ecx, %edx + movl %r8d, %ebx + addl %r14d, %r10d + vpxor %xmm8, %xmm7, %xmm5 + # rnd_1: 6 - 6 + xorl %r15d, %ebx + addl %edx, %r14d + andl %ebx, %eax + vpsrld $10, %xmm6, %xmm8 + # rnd_1: 7 - 7 + xorl %r8d, %eax + rorxl $6, %r10d, %edx + addl %eax, %r14d + # rnd_0: 0 - 0 + movl %r11d, %eax + rorxl $11, %r10d, %ecx + addl 40(%rsp), %r13d + vpsrlq $19, %xmm6, %xmm7 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r12d, %eax + rorxl $25, %r10d, %edx + vpsrlq $0x11, %xmm6, %xmm6 + vpaddd %xmm2, %xmm4, %xmm4 + # rnd_0: 2 - 2 + andl %r10d, %eax + xorl %ecx, %edx + rorxl $13, %r14d, %ecx + vpaddd %xmm5, %xmm4, %xmm4 + # rnd_0: 3 - 3 + addl %edx, %r13d + rorxl $2, %r14d, %edx + xorl %r12d, %eax + vpxor %xmm7, %xmm6, %xmm6 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r14d, %edx + addl %eax, %r13d + vpxor %xmm6, %xmm8, %xmm8 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r15d, %eax + addl %r13d, %r9d + vpshufb %xmm11, %xmm8, %xmm8 + # rnd_0: 6 - 6 + xorl %r14d, %eax + addl %edx, %r13d + andl %eax, %ebx + vpaddd %xmm8, %xmm4, %xmm4 + # rnd_0: 7 - 7 + xorl %r15d, %ebx + rorxl $6, %r9d, %edx + addl %ebx, %r13d + # rnd_1: 0 - 0 + movl %r10d, %ebx + rorxl $11, %r9d, %ecx + addl 44(%rsp), %r12d + vpshufd $0x50, %xmm4, %xmm6 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r11d, %ebx + rorxl $25, %r9d, %edx + vpsrld $10, %xmm6, %xmm9 + # rnd_1: 2 - 2 + andl %r9d, %ebx + xorl %ecx, %edx + rorxl $13, %r13d, %ecx + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 3 - 3 + addl %edx, %r12d + rorxl $2, %r13d, %edx + xorl %r11d, %ebx + vpsrlq $0x11, %xmm6, %xmm6 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r13d, %edx + addl %ebx, %r12d + vpxor %xmm7, %xmm6, %xmm6 + # rnd_1: 5 - 5 + xorl %ecx, %edx + movl %r14d, %ebx + addl %r12d, %r8d + vpxor %xmm6, %xmm9, %xmm9 + # rnd_1: 6 - 6 + xorl %r13d, %ebx + addl %edx, %r12d + andl %ebx, %eax + vpshufb %xmm12, %xmm9, %xmm9 + # rnd_1: 7 - 7 + xorl %r14d, %eax + rorxl $6, %r8d, %edx + addl %eax, %r12d + vpaddd %xmm4, %xmm9, %xmm2 + # msg_sched done: 8-11 + # msg_sched: 12-15 + # rnd_0: 0 - 0 + movl %r9d, %eax + rorxl $11, %r8d, %ecx + addl 48(%rsp), %r11d + vpalignr $4, %xmm1, %xmm2, %xmm4 + vpalignr $4, %xmm3, %xmm0, %xmm5 + # rnd_0: 1 - 2 + xorl %edx, %ecx + xorl %r10d, %eax + rorxl $25, %r8d, %edx + andl %r8d, %eax + xorl %ecx, %edx + rorxl $13, %r12d, %ecx + vpsrld $7, %xmm5, %xmm6 + vpslld $25, %xmm5, %xmm7 + # rnd_0: 3 - 4 + addl %edx, %r11d + rorxl $2, %r12d, %edx + xorl %r10d, %eax + xorl %edx, %ecx + rorxl $22, %r12d, %edx + addl %eax, %r11d + vpsrld $3, %xmm5, %xmm8 + vpor %xmm6, %xmm7, %xmm7 + # rnd_0: 5 - 7 + xorl %ecx, %edx + movl %r13d, %eax + addl %r11d, %r15d + xorl %r12d, %eax + addl %edx, %r11d + andl %eax, %ebx + xorl %r13d, %ebx + rorxl $6, %r15d, %edx + addl %ebx, %r11d + # rnd_1: 0 - 0 + movl %r8d, %ebx + rorxl $11, %r15d, %ecx + addl 52(%rsp), %r10d + vpsrld $18, %xmm5, %xmm6 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r9d, %ebx + rorxl $25, %r15d, %edx + vpslld $14, %xmm5, %xmm5 + # rnd_1: 2 - 2 + andl %r15d, %ebx + xorl %ecx, %edx + rorxl $13, %r11d, %ecx + vpxor %xmm5, %xmm7, %xmm7 + # rnd_1: 3 - 3 + addl %edx, %r10d + rorxl $2, %r11d, %edx + xorl %r9d, %ebx + vpxor %xmm6, %xmm7, %xmm7 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r11d, %edx + addl %ebx, %r10d + vpshufd $0xfa, %xmm2, %xmm6 + # rnd_1: 5 - 5 + xorl %ecx, %edx + movl %r12d, %ebx + addl %r10d, %r14d + vpxor %xmm8, %xmm7, %xmm5 + # rnd_1: 6 - 6 + xorl %r11d, %ebx + addl %edx, %r10d + andl %ebx, %eax + vpsrld $10, %xmm6, %xmm8 + # rnd_1: 7 - 7 + xorl %r12d, %eax + rorxl $6, %r14d, %edx + addl %eax, %r10d + # rnd_0: 0 - 0 + movl %r15d, %eax + rorxl $11, %r14d, %ecx + addl 56(%rsp), %r9d + vpsrlq $19, %xmm6, %xmm7 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r8d, %eax + rorxl $25, %r14d, %edx + vpsrlq $0x11, %xmm6, %xmm6 + vpaddd %xmm3, %xmm4, %xmm4 + # rnd_0: 2 - 2 + andl %r14d, %eax + xorl %ecx, %edx + rorxl $13, %r10d, %ecx + vpaddd %xmm5, %xmm4, %xmm4 + # rnd_0: 3 - 3 + addl %edx, %r9d + rorxl $2, %r10d, %edx + xorl %r8d, %eax + vpxor %xmm7, %xmm6, %xmm6 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r10d, %edx + addl %eax, %r9d + vpxor %xmm6, %xmm8, %xmm8 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r11d, %eax + addl %r9d, %r13d + vpshufb %xmm11, %xmm8, %xmm8 + # rnd_0: 6 - 6 + xorl %r10d, %eax + addl %edx, %r9d + andl %eax, %ebx + vpaddd %xmm8, %xmm4, %xmm4 + # rnd_0: 7 - 7 + xorl %r11d, %ebx + rorxl $6, %r13d, %edx + addl %ebx, %r9d + # rnd_1: 0 - 0 + movl %r14d, %ebx + rorxl $11, %r13d, %ecx + addl 60(%rsp), %r8d + vpshufd $0x50, %xmm4, %xmm6 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r15d, %ebx + rorxl $25, %r13d, %edx + vpsrld $10, %xmm6, %xmm9 + # rnd_1: 2 - 2 + andl %r13d, %ebx + xorl %ecx, %edx + rorxl $13, %r9d, %ecx + vpsrlq $19, %xmm6, %xmm7 + # rnd_1: 3 - 3 + addl %edx, %r8d + rorxl $2, %r9d, %edx + xorl %r15d, %ebx + vpsrlq $0x11, %xmm6, %xmm6 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r9d, %edx + addl %ebx, %r8d + vpxor %xmm7, %xmm6, %xmm6 + # rnd_1: 5 - 5 + xorl %ecx, %edx + movl %r10d, %ebx + addl %r8d, %r12d + vpxor %xmm6, %xmm9, %xmm9 + # rnd_1: 6 - 6 + xorl %r9d, %ebx + addl %edx, %r8d + andl %ebx, %eax + vpshufb %xmm12, %xmm9, %xmm9 + # rnd_1: 7 - 7 + xorl %r10d, %eax + rorxl $6, %r12d, %edx + addl %eax, %r8d + vpaddd %xmm4, %xmm9, %xmm3 + # msg_sched done: 12-15 + # set_w_k_xfer_4: 12 + vpaddd 192+L_avx1_rorx_sha256_k(%rip), %xmm0, %xmm4 + vpaddd 208+L_avx1_rorx_sha256_k(%rip), %xmm1, %xmm5 + vmovdqu %xmm4, (%rsp) + vmovdqu %xmm5, 16(%rsp) + vpaddd 224+L_avx1_rorx_sha256_k(%rip), %xmm2, %xmm6 + vpaddd 240+L_avx1_rorx_sha256_k(%rip), %xmm3, %xmm7 + vmovdqu %xmm6, 32(%rsp) + vmovdqu %xmm7, 48(%rsp) + xorl %eax, %eax + xorl %ecx, %ecx + # rnd_all_4: 0-3 + rorxl $6, %r12d, %edx + rorxl $11, %r12d, %ecx + addl %eax, %r8d + addl (%rsp), %r15d + movl %r13d, %eax + xorl %edx, %ecx + xorl %r14d, %eax + rorxl $25, %r12d, %edx + xorl %ecx, %edx + andl %r12d, %eax + addl %edx, %r15d + rorxl $2, %r8d, %edx + rorxl $13, %r8d, %ecx + xorl %r14d, %eax + xorl %edx, %ecx + rorxl $22, %r8d, %edx + addl %eax, %r15d + xorl %ecx, %edx + movl %r9d, %eax + addl %r15d, %r11d + xorl %r8d, %eax + andl %eax, %ebx + addl %edx, %r15d + xorl %r9d, %ebx + rorxl $6, %r11d, %edx + rorxl $11, %r11d, %ecx + addl %ebx, %r15d + addl 4(%rsp), %r14d + movl %r12d, %ebx + xorl %edx, %ecx + xorl %r13d, %ebx + rorxl $25, %r11d, %edx + xorl %ecx, %edx + andl %r11d, %ebx + addl %edx, %r14d + rorxl $2, %r15d, %edx + rorxl $13, %r15d, %ecx + xorl %r13d, %ebx + xorl %edx, %ecx + rorxl $22, %r15d, %edx + addl %ebx, %r14d + xorl %ecx, %edx + movl %r8d, %ebx + addl %r14d, %r10d + xorl %r15d, %ebx + andl %ebx, %eax + addl %edx, %r14d + xorl %r8d, %eax + rorxl $6, %r10d, %edx + rorxl $11, %r10d, %ecx + addl %eax, %r14d + addl 8(%rsp), %r13d + movl %r11d, %eax + xorl %edx, %ecx + xorl %r12d, %eax + rorxl $25, %r10d, %edx + xorl %ecx, %edx + andl %r10d, %eax + addl %edx, %r13d + rorxl $2, %r14d, %edx + rorxl $13, %r14d, %ecx + xorl %r12d, %eax + xorl %edx, %ecx + rorxl $22, %r14d, %edx + addl %eax, %r13d + xorl %ecx, %edx + movl %r15d, %eax + addl %r13d, %r9d + xorl %r14d, %eax + andl %eax, %ebx + addl %edx, %r13d + xorl %r15d, %ebx + rorxl $6, %r9d, %edx + rorxl $11, %r9d, %ecx + addl %ebx, %r13d + addl 12(%rsp), %r12d + movl %r10d, %ebx + xorl %edx, %ecx + xorl %r11d, %ebx + rorxl $25, %r9d, %edx + xorl %ecx, %edx + andl %r9d, %ebx + addl %edx, %r12d + rorxl $2, %r13d, %edx + rorxl $13, %r13d, %ecx + xorl %r11d, %ebx + xorl %edx, %ecx + rorxl $22, %r13d, %edx + addl %ebx, %r12d + xorl %ecx, %edx + movl %r14d, %ebx + addl %r12d, %r8d + xorl %r13d, %ebx + andl %ebx, %eax + addl %edx, %r12d + xorl %r14d, %eax + # rnd_all_4: 1-4 + rorxl $6, %r8d, %edx + rorxl $11, %r8d, %ecx + addl %eax, %r12d + addl 16(%rsp), %r11d + movl %r9d, %eax + xorl %edx, %ecx + xorl %r10d, %eax + rorxl $25, %r8d, %edx + xorl %ecx, %edx + andl %r8d, %eax + addl %edx, %r11d + rorxl $2, %r12d, %edx + rorxl $13, %r12d, %ecx + xorl %r10d, %eax + xorl %edx, %ecx + rorxl $22, %r12d, %edx + addl %eax, %r11d + xorl %ecx, %edx + movl %r13d, %eax + addl %r11d, %r15d + xorl %r12d, %eax + andl %eax, %ebx + addl %edx, %r11d + xorl %r13d, %ebx + rorxl $6, %r15d, %edx + rorxl $11, %r15d, %ecx + addl %ebx, %r11d + addl 20(%rsp), %r10d + movl %r8d, %ebx + xorl %edx, %ecx + xorl %r9d, %ebx + rorxl $25, %r15d, %edx + xorl %ecx, %edx + andl %r15d, %ebx + addl %edx, %r10d + rorxl $2, %r11d, %edx + rorxl $13, %r11d, %ecx + xorl %r9d, %ebx + xorl %edx, %ecx + rorxl $22, %r11d, %edx + addl %ebx, %r10d + xorl %ecx, %edx + movl %r12d, %ebx + addl %r10d, %r14d + xorl %r11d, %ebx + andl %ebx, %eax + addl %edx, %r10d + xorl %r12d, %eax + rorxl $6, %r14d, %edx + rorxl $11, %r14d, %ecx + addl %eax, %r10d + addl 24(%rsp), %r9d + movl %r15d, %eax + xorl %edx, %ecx + xorl %r8d, %eax + rorxl $25, %r14d, %edx + xorl %ecx, %edx + andl %r14d, %eax + addl %edx, %r9d + rorxl $2, %r10d, %edx + rorxl $13, %r10d, %ecx + xorl %r8d, %eax + xorl %edx, %ecx + rorxl $22, %r10d, %edx + addl %eax, %r9d + xorl %ecx, %edx + movl %r11d, %eax + addl %r9d, %r13d + xorl %r10d, %eax + andl %eax, %ebx + addl %edx, %r9d + xorl %r11d, %ebx + rorxl $6, %r13d, %edx + rorxl $11, %r13d, %ecx + addl %ebx, %r9d + addl 28(%rsp), %r8d + movl %r14d, %ebx + xorl %edx, %ecx + xorl %r15d, %ebx + rorxl $25, %r13d, %edx + xorl %ecx, %edx + andl %r13d, %ebx + addl %edx, %r8d + rorxl $2, %r9d, %edx + rorxl $13, %r9d, %ecx + xorl %r15d, %ebx + xorl %edx, %ecx + rorxl $22, %r9d, %edx + addl %ebx, %r8d + xorl %ecx, %edx + movl %r10d, %ebx + addl %r8d, %r12d + xorl %r9d, %ebx + andl %ebx, %eax + addl %edx, %r8d + xorl %r10d, %eax + # rnd_all_4: 2-5 + rorxl $6, %r12d, %edx + rorxl $11, %r12d, %ecx + addl %eax, %r8d + addl 32(%rsp), %r15d + movl %r13d, %eax + xorl %edx, %ecx + xorl %r14d, %eax + rorxl $25, %r12d, %edx + xorl %ecx, %edx + andl %r12d, %eax + addl %edx, %r15d + rorxl $2, %r8d, %edx + rorxl $13, %r8d, %ecx + xorl %r14d, %eax + xorl %edx, %ecx + rorxl $22, %r8d, %edx + addl %eax, %r15d + xorl %ecx, %edx + movl %r9d, %eax + addl %r15d, %r11d + xorl %r8d, %eax + andl %eax, %ebx + addl %edx, %r15d + xorl %r9d, %ebx + rorxl $6, %r11d, %edx + rorxl $11, %r11d, %ecx + addl %ebx, %r15d + addl 36(%rsp), %r14d + movl %r12d, %ebx + xorl %edx, %ecx + xorl %r13d, %ebx + rorxl $25, %r11d, %edx + xorl %ecx, %edx + andl %r11d, %ebx + addl %edx, %r14d + rorxl $2, %r15d, %edx + rorxl $13, %r15d, %ecx + xorl %r13d, %ebx + xorl %edx, %ecx + rorxl $22, %r15d, %edx + addl %ebx, %r14d + xorl %ecx, %edx + movl %r8d, %ebx + addl %r14d, %r10d + xorl %r15d, %ebx + andl %ebx, %eax + addl %edx, %r14d + xorl %r8d, %eax + rorxl $6, %r10d, %edx + rorxl $11, %r10d, %ecx + addl %eax, %r14d + addl 40(%rsp), %r13d + movl %r11d, %eax + xorl %edx, %ecx + xorl %r12d, %eax + rorxl $25, %r10d, %edx + xorl %ecx, %edx + andl %r10d, %eax + addl %edx, %r13d + rorxl $2, %r14d, %edx + rorxl $13, %r14d, %ecx + xorl %r12d, %eax + xorl %edx, %ecx + rorxl $22, %r14d, %edx + addl %eax, %r13d + xorl %ecx, %edx + movl %r15d, %eax + addl %r13d, %r9d + xorl %r14d, %eax + andl %eax, %ebx + addl %edx, %r13d + xorl %r15d, %ebx + rorxl $6, %r9d, %edx + rorxl $11, %r9d, %ecx + addl %ebx, %r13d + addl 44(%rsp), %r12d + movl %r10d, %ebx + xorl %edx, %ecx + xorl %r11d, %ebx + rorxl $25, %r9d, %edx + xorl %ecx, %edx + andl %r9d, %ebx + addl %edx, %r12d + rorxl $2, %r13d, %edx + rorxl $13, %r13d, %ecx + xorl %r11d, %ebx + xorl %edx, %ecx + rorxl $22, %r13d, %edx + addl %ebx, %r12d + xorl %ecx, %edx + movl %r14d, %ebx + addl %r12d, %r8d + xorl %r13d, %ebx + andl %ebx, %eax + addl %edx, %r12d + xorl %r14d, %eax + # rnd_all_4: 3-6 + rorxl $6, %r8d, %edx + rorxl $11, %r8d, %ecx + addl %eax, %r12d + addl 48(%rsp), %r11d + movl %r9d, %eax + xorl %edx, %ecx + xorl %r10d, %eax + rorxl $25, %r8d, %edx + xorl %ecx, %edx + andl %r8d, %eax + addl %edx, %r11d + rorxl $2, %r12d, %edx + rorxl $13, %r12d, %ecx + xorl %r10d, %eax + xorl %edx, %ecx + rorxl $22, %r12d, %edx + addl %eax, %r11d + xorl %ecx, %edx + movl %r13d, %eax + addl %r11d, %r15d + xorl %r12d, %eax + andl %eax, %ebx + addl %edx, %r11d + xorl %r13d, %ebx + rorxl $6, %r15d, %edx + rorxl $11, %r15d, %ecx + addl %ebx, %r11d + addl 52(%rsp), %r10d + movl %r8d, %ebx + xorl %edx, %ecx + xorl %r9d, %ebx + rorxl $25, %r15d, %edx + xorl %ecx, %edx + andl %r15d, %ebx + addl %edx, %r10d + rorxl $2, %r11d, %edx + rorxl $13, %r11d, %ecx + xorl %r9d, %ebx + xorl %edx, %ecx + rorxl $22, %r11d, %edx + addl %ebx, %r10d + xorl %ecx, %edx + movl %r12d, %ebx + addl %r10d, %r14d + xorl %r11d, %ebx + andl %ebx, %eax + addl %edx, %r10d + xorl %r12d, %eax + rorxl $6, %r14d, %edx + rorxl $11, %r14d, %ecx + addl %eax, %r10d + addl 56(%rsp), %r9d + movl %r15d, %eax + xorl %edx, %ecx + xorl %r8d, %eax + rorxl $25, %r14d, %edx + xorl %ecx, %edx + andl %r14d, %eax + addl %edx, %r9d + rorxl $2, %r10d, %edx + rorxl $13, %r10d, %ecx + xorl %r8d, %eax + xorl %edx, %ecx + rorxl $22, %r10d, %edx + addl %eax, %r9d + xorl %ecx, %edx + movl %r11d, %eax + addl %r9d, %r13d + xorl %r10d, %eax + andl %eax, %ebx + addl %edx, %r9d + xorl %r11d, %ebx + rorxl $6, %r13d, %edx + rorxl $11, %r13d, %ecx + addl %ebx, %r9d + addl 60(%rsp), %r8d + movl %r14d, %ebx + xorl %edx, %ecx + xorl %r15d, %ebx + rorxl $25, %r13d, %edx + xorl %ecx, %edx + andl %r13d, %ebx + addl %edx, %r8d + rorxl $2, %r9d, %edx + rorxl $13, %r9d, %ecx + xorl %r15d, %ebx + xorl %edx, %ecx + rorxl $22, %r9d, %edx + addl %ebx, %r8d + xorl %ecx, %edx + movl %r10d, %ebx + addl %r8d, %r12d + xorl %r9d, %ebx + andl %ebx, %eax + addl %edx, %r8d + xorl %r10d, %eax + addl %eax, %r8d + addl (%rdi), %r8d + addl 4(%rdi), %r9d + addl 8(%rdi), %r10d + addl 12(%rdi), %r11d + addl 16(%rdi), %r12d + addl 20(%rdi), %r13d + addl 24(%rdi), %r14d + addl 28(%rdi), %r15d + addq $0x40, %rbp + subl $0x40, %esi + movl %r8d, (%rdi) + movl %r9d, 4(%rdi) + movl %r10d, 8(%rdi) + movl %r11d, 12(%rdi) + movl %r12d, 16(%rdi) + movl %r13d, 20(%rdi) + movl %r14d, 24(%rdi) + movl %r15d, 28(%rdi) + jnz L_sha256_len_avx1_len_rorx_start + xorq %rax, %rax + vzeroupper + addq $0x40, %rsp + popq %rbp + popq %r15 + popq %r14 + popq %r13 + popq %r12 + popq %rbx + repz retq +.size Transform_Sha256_AVX1_RORX_Len,.-Transform_Sha256_AVX1_RORX_Len +#endif /* HAVE_INTEL_AVX1 */ +#ifdef HAVE_INTEL_AVX2 +L_avx2_sha256_k: +.long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 +.long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 +.long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 +.long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 +.long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 +.long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 +.long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 +.long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 +.long 0xe49b69c1,0xefbe4786,0xfc19dc6,0x240ca1cc +.long 0xe49b69c1,0xefbe4786,0xfc19dc6,0x240ca1cc +.long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da +.long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da +.long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 +.long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 +.long 0xc6e00bf3,0xd5a79147,0x6ca6351,0x14292967 +.long 0xc6e00bf3,0xd5a79147,0x6ca6351,0x14292967 +.long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 +.long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 +.long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 +.long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 +.long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 +.long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 +.long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 +.long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 +.long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 +.long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 +.long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 +.long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 +.long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 +.long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 +.long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 +.long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 +.align 32 +L_avx2_sha256_shuf_00BA: +.quad 0xb0a090803020100, 0xffffffffffffffff +.quad 0xb0a090803020100, 0xffffffffffffffff +.align 32 +L_avx2_sha256_shuf_DC00: +.quad 0xffffffffffffffff, 0xb0a090803020100 +.quad 0xffffffffffffffff, 0xb0a090803020100 +.align 32 +L_avx2_sha256_flip_mask: +.quad 0x405060700010203, 0xc0d0e0f08090a0b +.quad 0x405060700010203, 0xc0d0e0f08090a0b +.globl Transform_Sha256_AVX2 +.type Transform_Sha256_AVX2,@function +.align 4 +Transform_Sha256_AVX2: + pushq %rbx + pushq %r12 + pushq %r13 + pushq %r14 + pushq %r15 + subq $0x200, %rsp + leaq 32(%rdi), %rax + vmovdqa L_avx2_sha256_flip_mask(%rip), %xmm13 + vmovdqa L_avx2_sha256_shuf_00BA(%rip), %ymm11 + vmovdqa L_avx2_sha256_shuf_DC00(%rip), %ymm12 + movl (%rdi), %r8d + movl 4(%rdi), %r9d + movl 8(%rdi), %r10d + movl 12(%rdi), %r11d + movl 16(%rdi), %r12d + movl 20(%rdi), %r13d + movl 24(%rdi), %r14d + movl 28(%rdi), %r15d + # X0, X1, X2, X3 = W[0..15] + vmovdqu (%rax), %xmm0 + vmovdqu 16(%rax), %xmm1 + vpshufb %xmm13, %xmm0, %xmm0 + vpshufb %xmm13, %xmm1, %xmm1 + vmovdqu 32(%rax), %xmm2 + vmovdqu 48(%rax), %xmm3 + vpshufb %xmm13, %xmm2, %xmm2 + vpshufb %xmm13, %xmm3, %xmm3 + movl %r9d, %ebx + movl %r12d, %edx + xorl %r10d, %ebx + # set_w_k_xfer_4: 0 + vpaddd 0+L_avx2_sha256_k(%rip), %ymm0, %ymm4 + vpaddd 32+L_avx2_sha256_k(%rip), %ymm1, %ymm5 + vmovdqu %ymm4, (%rsp) + vmovdqu %ymm5, 32(%rsp) + vpaddd 64+L_avx2_sha256_k(%rip), %ymm2, %ymm4 + vpaddd 96+L_avx2_sha256_k(%rip), %ymm3, %ymm5 + vmovdqu %ymm4, 64(%rsp) + vmovdqu %ymm5, 96(%rsp) + # msg_sched: 0-3 + # rnd_0: 0 - 0 + rorl $14, %edx + vpalignr $4, %ymm0, %ymm1, %ymm5 + vpalignr $4, %ymm2, %ymm3, %ymm4 + # rnd_0: 1 - 2 + movl %r9d, %eax + movl %r13d, %ecx + addl (%rsp), %r15d + xorl %r14d, %ecx + xorl %r12d, %edx + andl %r12d, %ecx + vpsrld $7, %ymm5, %ymm6 + vpslld $25, %ymm5, %ymm7 + # rnd_0: 3 - 4 + rorl $5, %edx + xorl %r14d, %ecx + xorl %r12d, %edx + addl %ecx, %r15d + rorl $6, %edx + xorl %r8d, %eax + addl %edx, %r15d + movl %r8d, %ecx + vpsrld $18, %ymm5, %ymm8 + vpslld $14, %ymm5, %ymm9 + # rnd_0: 5 - 6 + andl %eax, %ebx + rorl $9, %ecx + xorl %r8d, %ecx + xorl %r9d, %ebx + rorl $11, %ecx + addl %r15d, %r11d + xorl %r8d, %ecx + addl %ebx, %r15d + vpor %ymm6, %ymm7, %ymm6 + vpor %ymm8, %ymm9, %ymm8 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r11d, %edx + addl %ecx, %r15d + # rnd_1: 0 - 1 + rorl $14, %edx + movl %r8d, %ebx + movl %r12d, %ecx + addl 4(%rsp), %r14d + xorl %r13d, %ecx + vpsrld $3, %ymm5, %ymm9 + vpxor %ymm6, %ymm8, %ymm6 + # rnd_1: 2 - 3 + xorl %r11d, %edx + andl %r11d, %ecx + rorl $5, %edx + xorl %r13d, %ecx + xorl %r11d, %edx + addl %ecx, %r14d + vpxor %ymm6, %ymm9, %ymm5 + vpshufd $0xfa, %ymm3, %ymm6 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r15d, %ebx + addl %edx, %r14d + movl %r15d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r15d, %ecx + xorl %r8d, %eax + vpsrld $10, %ymm6, %ymm8 + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 6 - 7 + rorl $11, %ecx + addl %r14d, %r10d + xorl %r15d, %ecx + addl %eax, %r14d + rorl $2, %ecx + movl %r10d, %edx + addl %ecx, %r14d + # rnd_0: 0 - 0 + rorl $14, %edx + vpsrlq $0x11, %ymm6, %ymm6 + vpaddd %ymm0, %ymm4, %ymm4 + # rnd_0: 1 - 3 + movl %r15d, %eax + movl %r11d, %ecx + addl 8(%rsp), %r13d + xorl %r12d, %ecx + xorl %r10d, %edx + andl %r10d, %ecx + rorl $5, %edx + xorl %r12d, %ecx + xorl %r10d, %edx + addl %ecx, %r13d + vpxor %ymm6, %ymm7, %ymm6 + vpaddd %ymm5, %ymm4, %ymm4 + # rnd_0: 4 - 4 + rorl $6, %edx + xorl %r14d, %eax + addl %edx, %r13d + movl %r14d, %ecx + vpxor %ymm6, %ymm8, %ymm8 + # rnd_0: 5 - 5 + andl %eax, %ebx + rorl $9, %ecx + xorl %r14d, %ecx + xorl %r15d, %ebx + vpshufb %ymm11, %ymm8, %ymm8 + # rnd_0: 6 - 6 + rorl $11, %ecx + addl %r13d, %r9d + xorl %r14d, %ecx + addl %ebx, %r13d + vpaddd %ymm8, %ymm4, %ymm4 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r9d, %edx + addl %ecx, %r13d + # rnd_1: 0 - 0 + rorl $14, %edx + vpshufd $0x50, %ymm4, %ymm6 + # rnd_1: 1 - 1 + movl %r14d, %ebx + movl %r10d, %ecx + addl 12(%rsp), %r12d + xorl %r11d, %ecx + vpsrlq $0x11, %ymm6, %ymm8 + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 2 - 3 + xorl %r9d, %edx + andl %r9d, %ecx + rorl $5, %edx + xorl %r11d, %ecx + xorl %r9d, %edx + addl %ecx, %r12d + vpsrld $10, %ymm6, %ymm9 + vpxor %ymm8, %ymm7, %ymm8 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r13d, %ebx + addl %edx, %r12d + movl %r13d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r13d, %ecx + xorl %r14d, %eax + vpxor %ymm9, %ymm8, %ymm9 + # rnd_1: 6 - 6 + rorl $11, %ecx + addl %r12d, %r8d + xorl %r13d, %ecx + addl %eax, %r12d + vpshufb %ymm12, %ymm9, %ymm9 + # rnd_1: 7 - 7 + rorl $2, %ecx + movl %r8d, %edx + addl %ecx, %r12d + vpaddd %ymm4, %ymm9, %ymm0 + # msg_sched done: 0-3 + # msg_sched: 8-11 + # rnd_0: 0 - 0 + rorl $14, %edx + vpalignr $4, %ymm1, %ymm2, %ymm5 + vpalignr $4, %ymm3, %ymm0, %ymm4 + # rnd_0: 1 - 2 + movl %r13d, %eax + movl %r9d, %ecx + addl 32(%rsp), %r11d + xorl %r10d, %ecx + xorl %r8d, %edx + andl %r8d, %ecx + vpsrld $7, %ymm5, %ymm6 + vpslld $25, %ymm5, %ymm7 + # rnd_0: 3 - 4 + rorl $5, %edx + xorl %r10d, %ecx + xorl %r8d, %edx + addl %ecx, %r11d + rorl $6, %edx + xorl %r12d, %eax + addl %edx, %r11d + movl %r12d, %ecx + vpsrld $18, %ymm5, %ymm8 + vpslld $14, %ymm5, %ymm9 + # rnd_0: 5 - 6 + andl %eax, %ebx + rorl $9, %ecx + xorl %r12d, %ecx + xorl %r13d, %ebx + rorl $11, %ecx + addl %r11d, %r15d + xorl %r12d, %ecx + addl %ebx, %r11d + vpor %ymm6, %ymm7, %ymm6 + vpor %ymm8, %ymm9, %ymm8 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r15d, %edx + addl %ecx, %r11d + # rnd_1: 0 - 1 + rorl $14, %edx + movl %r12d, %ebx + movl %r8d, %ecx + addl 36(%rsp), %r10d + xorl %r9d, %ecx + vpsrld $3, %ymm5, %ymm9 + vpxor %ymm6, %ymm8, %ymm6 + # rnd_1: 2 - 3 + xorl %r15d, %edx + andl %r15d, %ecx + rorl $5, %edx + xorl %r9d, %ecx + xorl %r15d, %edx + addl %ecx, %r10d + vpxor %ymm6, %ymm9, %ymm5 + vpshufd $0xfa, %ymm0, %ymm6 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r11d, %ebx + addl %edx, %r10d + movl %r11d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r11d, %ecx + xorl %r12d, %eax + vpsrld $10, %ymm6, %ymm8 + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 6 - 7 + rorl $11, %ecx + addl %r10d, %r14d + xorl %r11d, %ecx + addl %eax, %r10d + rorl $2, %ecx + movl %r14d, %edx + addl %ecx, %r10d + # rnd_0: 0 - 0 + rorl $14, %edx + vpsrlq $0x11, %ymm6, %ymm6 + vpaddd %ymm1, %ymm4, %ymm4 + # rnd_0: 1 - 3 + movl %r11d, %eax + movl %r15d, %ecx + addl 40(%rsp), %r9d + xorl %r8d, %ecx + xorl %r14d, %edx + andl %r14d, %ecx + rorl $5, %edx + xorl %r8d, %ecx + xorl %r14d, %edx + addl %ecx, %r9d + vpxor %ymm6, %ymm7, %ymm6 + vpaddd %ymm5, %ymm4, %ymm4 + # rnd_0: 4 - 4 + rorl $6, %edx + xorl %r10d, %eax + addl %edx, %r9d + movl %r10d, %ecx + vpxor %ymm6, %ymm8, %ymm8 + # rnd_0: 5 - 5 + andl %eax, %ebx + rorl $9, %ecx + xorl %r10d, %ecx + xorl %r11d, %ebx + vpshufb %ymm11, %ymm8, %ymm8 + # rnd_0: 6 - 6 + rorl $11, %ecx + addl %r9d, %r13d + xorl %r10d, %ecx + addl %ebx, %r9d + vpaddd %ymm8, %ymm4, %ymm4 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r13d, %edx + addl %ecx, %r9d + # rnd_1: 0 - 0 + rorl $14, %edx + vpshufd $0x50, %ymm4, %ymm6 + # rnd_1: 1 - 1 + movl %r10d, %ebx + movl %r14d, %ecx + addl 44(%rsp), %r8d + xorl %r15d, %ecx + vpsrlq $0x11, %ymm6, %ymm8 + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 2 - 3 + xorl %r13d, %edx + andl %r13d, %ecx + rorl $5, %edx + xorl %r15d, %ecx + xorl %r13d, %edx + addl %ecx, %r8d + vpsrld $10, %ymm6, %ymm9 + vpxor %ymm8, %ymm7, %ymm8 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r9d, %ebx + addl %edx, %r8d + movl %r9d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r9d, %ecx + xorl %r10d, %eax + vpxor %ymm9, %ymm8, %ymm9 + # rnd_1: 6 - 6 + rorl $11, %ecx + addl %r8d, %r12d + xorl %r9d, %ecx + addl %eax, %r8d + vpshufb %ymm12, %ymm9, %ymm9 + # rnd_1: 7 - 7 + rorl $2, %ecx + movl %r12d, %edx + addl %ecx, %r8d + vpaddd %ymm4, %ymm9, %ymm1 + # msg_sched done: 8-11 + # msg_sched: 16-19 + # rnd_0: 0 - 0 + rorl $14, %edx + vpalignr $4, %ymm2, %ymm3, %ymm5 + vpalignr $4, %ymm0, %ymm1, %ymm4 + # rnd_0: 1 - 2 + movl %r9d, %eax + movl %r13d, %ecx + addl 64(%rsp), %r15d + xorl %r14d, %ecx + xorl %r12d, %edx + andl %r12d, %ecx + vpsrld $7, %ymm5, %ymm6 + vpslld $25, %ymm5, %ymm7 + # rnd_0: 3 - 4 + rorl $5, %edx + xorl %r14d, %ecx + xorl %r12d, %edx + addl %ecx, %r15d + rorl $6, %edx + xorl %r8d, %eax + addl %edx, %r15d + movl %r8d, %ecx + vpsrld $18, %ymm5, %ymm8 + vpslld $14, %ymm5, %ymm9 + # rnd_0: 5 - 6 + andl %eax, %ebx + rorl $9, %ecx + xorl %r8d, %ecx + xorl %r9d, %ebx + rorl $11, %ecx + addl %r15d, %r11d + xorl %r8d, %ecx + addl %ebx, %r15d + vpor %ymm6, %ymm7, %ymm6 + vpor %ymm8, %ymm9, %ymm8 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r11d, %edx + addl %ecx, %r15d + # rnd_1: 0 - 1 + rorl $14, %edx + movl %r8d, %ebx + movl %r12d, %ecx + addl 68(%rsp), %r14d + xorl %r13d, %ecx + vpsrld $3, %ymm5, %ymm9 + vpxor %ymm6, %ymm8, %ymm6 + # rnd_1: 2 - 3 + xorl %r11d, %edx + andl %r11d, %ecx + rorl $5, %edx + xorl %r13d, %ecx + xorl %r11d, %edx + addl %ecx, %r14d + vpxor %ymm6, %ymm9, %ymm5 + vpshufd $0xfa, %ymm1, %ymm6 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r15d, %ebx + addl %edx, %r14d + movl %r15d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r15d, %ecx + xorl %r8d, %eax + vpsrld $10, %ymm6, %ymm8 + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 6 - 7 + rorl $11, %ecx + addl %r14d, %r10d + xorl %r15d, %ecx + addl %eax, %r14d + rorl $2, %ecx + movl %r10d, %edx + addl %ecx, %r14d + # rnd_0: 0 - 0 + rorl $14, %edx + vpsrlq $0x11, %ymm6, %ymm6 + vpaddd %ymm2, %ymm4, %ymm4 + # rnd_0: 1 - 3 + movl %r15d, %eax + movl %r11d, %ecx + addl 72(%rsp), %r13d + xorl %r12d, %ecx + xorl %r10d, %edx + andl %r10d, %ecx + rorl $5, %edx + xorl %r12d, %ecx + xorl %r10d, %edx + addl %ecx, %r13d + vpxor %ymm6, %ymm7, %ymm6 + vpaddd %ymm5, %ymm4, %ymm4 + # rnd_0: 4 - 4 + rorl $6, %edx + xorl %r14d, %eax + addl %edx, %r13d + movl %r14d, %ecx + vpxor %ymm6, %ymm8, %ymm8 + # rnd_0: 5 - 5 + andl %eax, %ebx + rorl $9, %ecx + xorl %r14d, %ecx + xorl %r15d, %ebx + vpshufb %ymm11, %ymm8, %ymm8 + # rnd_0: 6 - 6 + rorl $11, %ecx + addl %r13d, %r9d + xorl %r14d, %ecx + addl %ebx, %r13d + vpaddd %ymm8, %ymm4, %ymm4 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r9d, %edx + addl %ecx, %r13d + # rnd_1: 0 - 0 + rorl $14, %edx + vpshufd $0x50, %ymm4, %ymm6 + # rnd_1: 1 - 1 + movl %r14d, %ebx + movl %r10d, %ecx + addl 76(%rsp), %r12d + xorl %r11d, %ecx + vpsrlq $0x11, %ymm6, %ymm8 + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 2 - 3 + xorl %r9d, %edx + andl %r9d, %ecx + rorl $5, %edx + xorl %r11d, %ecx + xorl %r9d, %edx + addl %ecx, %r12d + vpsrld $10, %ymm6, %ymm9 + vpxor %ymm8, %ymm7, %ymm8 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r13d, %ebx + addl %edx, %r12d + movl %r13d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r13d, %ecx + xorl %r14d, %eax + vpxor %ymm9, %ymm8, %ymm9 + # rnd_1: 6 - 6 + rorl $11, %ecx + addl %r12d, %r8d + xorl %r13d, %ecx + addl %eax, %r12d + vpshufb %ymm12, %ymm9, %ymm9 + # rnd_1: 7 - 7 + rorl $2, %ecx + movl %r8d, %edx + addl %ecx, %r12d + vpaddd %ymm4, %ymm9, %ymm2 + # msg_sched done: 16-19 + # msg_sched: 24-27 + # rnd_0: 0 - 0 + rorl $14, %edx + vpalignr $4, %ymm3, %ymm0, %ymm5 + vpalignr $4, %ymm1, %ymm2, %ymm4 + # rnd_0: 1 - 2 + movl %r13d, %eax + movl %r9d, %ecx + addl 96(%rsp), %r11d + xorl %r10d, %ecx + xorl %r8d, %edx + andl %r8d, %ecx + vpsrld $7, %ymm5, %ymm6 + vpslld $25, %ymm5, %ymm7 + # rnd_0: 3 - 4 + rorl $5, %edx + xorl %r10d, %ecx + xorl %r8d, %edx + addl %ecx, %r11d + rorl $6, %edx + xorl %r12d, %eax + addl %edx, %r11d + movl %r12d, %ecx + vpsrld $18, %ymm5, %ymm8 + vpslld $14, %ymm5, %ymm9 + # rnd_0: 5 - 6 + andl %eax, %ebx + rorl $9, %ecx + xorl %r12d, %ecx + xorl %r13d, %ebx + rorl $11, %ecx + addl %r11d, %r15d + xorl %r12d, %ecx + addl %ebx, %r11d + vpor %ymm6, %ymm7, %ymm6 + vpor %ymm8, %ymm9, %ymm8 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r15d, %edx + addl %ecx, %r11d + # rnd_1: 0 - 1 + rorl $14, %edx + movl %r12d, %ebx + movl %r8d, %ecx + addl 100(%rsp), %r10d + xorl %r9d, %ecx + vpsrld $3, %ymm5, %ymm9 + vpxor %ymm6, %ymm8, %ymm6 + # rnd_1: 2 - 3 + xorl %r15d, %edx + andl %r15d, %ecx + rorl $5, %edx + xorl %r9d, %ecx + xorl %r15d, %edx + addl %ecx, %r10d + vpxor %ymm6, %ymm9, %ymm5 + vpshufd $0xfa, %ymm2, %ymm6 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r11d, %ebx + addl %edx, %r10d + movl %r11d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r11d, %ecx + xorl %r12d, %eax + vpsrld $10, %ymm6, %ymm8 + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 6 - 7 + rorl $11, %ecx + addl %r10d, %r14d + xorl %r11d, %ecx + addl %eax, %r10d + rorl $2, %ecx + movl %r14d, %edx + addl %ecx, %r10d + # rnd_0: 0 - 0 + rorl $14, %edx + vpsrlq $0x11, %ymm6, %ymm6 + vpaddd %ymm3, %ymm4, %ymm4 + # rnd_0: 1 - 3 + movl %r11d, %eax + movl %r15d, %ecx + addl 104(%rsp), %r9d + xorl %r8d, %ecx + xorl %r14d, %edx + andl %r14d, %ecx + rorl $5, %edx + xorl %r8d, %ecx + xorl %r14d, %edx + addl %ecx, %r9d + vpxor %ymm6, %ymm7, %ymm6 + vpaddd %ymm5, %ymm4, %ymm4 + # rnd_0: 4 - 4 + rorl $6, %edx + xorl %r10d, %eax + addl %edx, %r9d + movl %r10d, %ecx + vpxor %ymm6, %ymm8, %ymm8 + # rnd_0: 5 - 5 + andl %eax, %ebx + rorl $9, %ecx + xorl %r10d, %ecx + xorl %r11d, %ebx + vpshufb %ymm11, %ymm8, %ymm8 + # rnd_0: 6 - 6 + rorl $11, %ecx + addl %r9d, %r13d + xorl %r10d, %ecx + addl %ebx, %r9d + vpaddd %ymm8, %ymm4, %ymm4 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r13d, %edx + addl %ecx, %r9d + # rnd_1: 0 - 0 + rorl $14, %edx + vpshufd $0x50, %ymm4, %ymm6 + # rnd_1: 1 - 1 + movl %r10d, %ebx + movl %r14d, %ecx + addl 108(%rsp), %r8d + xorl %r15d, %ecx + vpsrlq $0x11, %ymm6, %ymm8 + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 2 - 3 + xorl %r13d, %edx + andl %r13d, %ecx + rorl $5, %edx + xorl %r15d, %ecx + xorl %r13d, %edx + addl %ecx, %r8d + vpsrld $10, %ymm6, %ymm9 + vpxor %ymm8, %ymm7, %ymm8 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r9d, %ebx + addl %edx, %r8d + movl %r9d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r9d, %ecx + xorl %r10d, %eax + vpxor %ymm9, %ymm8, %ymm9 + # rnd_1: 6 - 6 + rorl $11, %ecx + addl %r8d, %r12d + xorl %r9d, %ecx + addl %eax, %r8d + vpshufb %ymm12, %ymm9, %ymm9 + # rnd_1: 7 - 7 + rorl $2, %ecx + movl %r12d, %edx + addl %ecx, %r8d + vpaddd %ymm4, %ymm9, %ymm3 + # msg_sched done: 24-27 + # set_w_k_xfer_4: 4 + vpaddd 128+L_avx2_sha256_k(%rip), %ymm0, %ymm4 + vpaddd 160+L_avx2_sha256_k(%rip), %ymm1, %ymm5 + vmovdqu %ymm4, 128(%rsp) + vmovdqu %ymm5, 160(%rsp) + vpaddd 192+L_avx2_sha256_k(%rip), %ymm2, %ymm4 + vpaddd 224+L_avx2_sha256_k(%rip), %ymm3, %ymm5 + vmovdqu %ymm4, 192(%rsp) + vmovdqu %ymm5, 224(%rsp) + # msg_sched: 32-35 + # rnd_0: 0 - 0 + rorl $14, %edx + vpalignr $4, %ymm0, %ymm1, %ymm5 + vpalignr $4, %ymm2, %ymm3, %ymm4 + # rnd_0: 1 - 2 + movl %r9d, %eax + movl %r13d, %ecx + addl 128(%rsp), %r15d + xorl %r14d, %ecx + xorl %r12d, %edx + andl %r12d, %ecx + vpsrld $7, %ymm5, %ymm6 + vpslld $25, %ymm5, %ymm7 + # rnd_0: 3 - 4 + rorl $5, %edx + xorl %r14d, %ecx + xorl %r12d, %edx + addl %ecx, %r15d + rorl $6, %edx + xorl %r8d, %eax + addl %edx, %r15d + movl %r8d, %ecx + vpsrld $18, %ymm5, %ymm8 + vpslld $14, %ymm5, %ymm9 + # rnd_0: 5 - 6 + andl %eax, %ebx + rorl $9, %ecx + xorl %r8d, %ecx + xorl %r9d, %ebx + rorl $11, %ecx + addl %r15d, %r11d + xorl %r8d, %ecx + addl %ebx, %r15d + vpor %ymm6, %ymm7, %ymm6 + vpor %ymm8, %ymm9, %ymm8 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r11d, %edx + addl %ecx, %r15d + # rnd_1: 0 - 1 + rorl $14, %edx + movl %r8d, %ebx + movl %r12d, %ecx + addl 132(%rsp), %r14d + xorl %r13d, %ecx + vpsrld $3, %ymm5, %ymm9 + vpxor %ymm6, %ymm8, %ymm6 + # rnd_1: 2 - 3 + xorl %r11d, %edx + andl %r11d, %ecx + rorl $5, %edx + xorl %r13d, %ecx + xorl %r11d, %edx + addl %ecx, %r14d + vpxor %ymm6, %ymm9, %ymm5 + vpshufd $0xfa, %ymm3, %ymm6 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r15d, %ebx + addl %edx, %r14d + movl %r15d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r15d, %ecx + xorl %r8d, %eax + vpsrld $10, %ymm6, %ymm8 + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 6 - 7 + rorl $11, %ecx + addl %r14d, %r10d + xorl %r15d, %ecx + addl %eax, %r14d + rorl $2, %ecx + movl %r10d, %edx + addl %ecx, %r14d + # rnd_0: 0 - 0 + rorl $14, %edx + vpsrlq $0x11, %ymm6, %ymm6 + vpaddd %ymm0, %ymm4, %ymm4 + # rnd_0: 1 - 3 + movl %r15d, %eax + movl %r11d, %ecx + addl 136(%rsp), %r13d + xorl %r12d, %ecx + xorl %r10d, %edx + andl %r10d, %ecx + rorl $5, %edx + xorl %r12d, %ecx + xorl %r10d, %edx + addl %ecx, %r13d + vpxor %ymm6, %ymm7, %ymm6 + vpaddd %ymm5, %ymm4, %ymm4 + # rnd_0: 4 - 4 + rorl $6, %edx + xorl %r14d, %eax + addl %edx, %r13d + movl %r14d, %ecx + vpxor %ymm6, %ymm8, %ymm8 + # rnd_0: 5 - 5 + andl %eax, %ebx + rorl $9, %ecx + xorl %r14d, %ecx + xorl %r15d, %ebx + vpshufb %ymm11, %ymm8, %ymm8 + # rnd_0: 6 - 6 + rorl $11, %ecx + addl %r13d, %r9d + xorl %r14d, %ecx + addl %ebx, %r13d + vpaddd %ymm8, %ymm4, %ymm4 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r9d, %edx + addl %ecx, %r13d + # rnd_1: 0 - 0 + rorl $14, %edx + vpshufd $0x50, %ymm4, %ymm6 + # rnd_1: 1 - 1 + movl %r14d, %ebx + movl %r10d, %ecx + addl 140(%rsp), %r12d + xorl %r11d, %ecx + vpsrlq $0x11, %ymm6, %ymm8 + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 2 - 3 + xorl %r9d, %edx + andl %r9d, %ecx + rorl $5, %edx + xorl %r11d, %ecx + xorl %r9d, %edx + addl %ecx, %r12d + vpsrld $10, %ymm6, %ymm9 + vpxor %ymm8, %ymm7, %ymm8 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r13d, %ebx + addl %edx, %r12d + movl %r13d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r13d, %ecx + xorl %r14d, %eax + vpxor %ymm9, %ymm8, %ymm9 + # rnd_1: 6 - 6 + rorl $11, %ecx + addl %r12d, %r8d + xorl %r13d, %ecx + addl %eax, %r12d + vpshufb %ymm12, %ymm9, %ymm9 + # rnd_1: 7 - 7 + rorl $2, %ecx + movl %r8d, %edx + addl %ecx, %r12d + vpaddd %ymm4, %ymm9, %ymm0 + # msg_sched done: 32-35 + # msg_sched: 40-43 + # rnd_0: 0 - 0 + rorl $14, %edx + vpalignr $4, %ymm1, %ymm2, %ymm5 + vpalignr $4, %ymm3, %ymm0, %ymm4 + # rnd_0: 1 - 2 + movl %r13d, %eax + movl %r9d, %ecx + addl 160(%rsp), %r11d + xorl %r10d, %ecx + xorl %r8d, %edx + andl %r8d, %ecx + vpsrld $7, %ymm5, %ymm6 + vpslld $25, %ymm5, %ymm7 + # rnd_0: 3 - 4 + rorl $5, %edx + xorl %r10d, %ecx + xorl %r8d, %edx + addl %ecx, %r11d + rorl $6, %edx + xorl %r12d, %eax + addl %edx, %r11d + movl %r12d, %ecx + vpsrld $18, %ymm5, %ymm8 + vpslld $14, %ymm5, %ymm9 + # rnd_0: 5 - 6 + andl %eax, %ebx + rorl $9, %ecx + xorl %r12d, %ecx + xorl %r13d, %ebx + rorl $11, %ecx + addl %r11d, %r15d + xorl %r12d, %ecx + addl %ebx, %r11d + vpor %ymm6, %ymm7, %ymm6 + vpor %ymm8, %ymm9, %ymm8 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r15d, %edx + addl %ecx, %r11d + # rnd_1: 0 - 1 + rorl $14, %edx + movl %r12d, %ebx + movl %r8d, %ecx + addl 164(%rsp), %r10d + xorl %r9d, %ecx + vpsrld $3, %ymm5, %ymm9 + vpxor %ymm6, %ymm8, %ymm6 + # rnd_1: 2 - 3 + xorl %r15d, %edx + andl %r15d, %ecx + rorl $5, %edx + xorl %r9d, %ecx + xorl %r15d, %edx + addl %ecx, %r10d + vpxor %ymm6, %ymm9, %ymm5 + vpshufd $0xfa, %ymm0, %ymm6 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r11d, %ebx + addl %edx, %r10d + movl %r11d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r11d, %ecx + xorl %r12d, %eax + vpsrld $10, %ymm6, %ymm8 + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 6 - 7 + rorl $11, %ecx + addl %r10d, %r14d + xorl %r11d, %ecx + addl %eax, %r10d + rorl $2, %ecx + movl %r14d, %edx + addl %ecx, %r10d + # rnd_0: 0 - 0 + rorl $14, %edx + vpsrlq $0x11, %ymm6, %ymm6 + vpaddd %ymm1, %ymm4, %ymm4 + # rnd_0: 1 - 3 + movl %r11d, %eax + movl %r15d, %ecx + addl 168(%rsp), %r9d + xorl %r8d, %ecx + xorl %r14d, %edx + andl %r14d, %ecx + rorl $5, %edx + xorl %r8d, %ecx + xorl %r14d, %edx + addl %ecx, %r9d + vpxor %ymm6, %ymm7, %ymm6 + vpaddd %ymm5, %ymm4, %ymm4 + # rnd_0: 4 - 4 + rorl $6, %edx + xorl %r10d, %eax + addl %edx, %r9d + movl %r10d, %ecx + vpxor %ymm6, %ymm8, %ymm8 + # rnd_0: 5 - 5 + andl %eax, %ebx + rorl $9, %ecx + xorl %r10d, %ecx + xorl %r11d, %ebx + vpshufb %ymm11, %ymm8, %ymm8 + # rnd_0: 6 - 6 + rorl $11, %ecx + addl %r9d, %r13d + xorl %r10d, %ecx + addl %ebx, %r9d + vpaddd %ymm8, %ymm4, %ymm4 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r13d, %edx + addl %ecx, %r9d + # rnd_1: 0 - 0 + rorl $14, %edx + vpshufd $0x50, %ymm4, %ymm6 + # rnd_1: 1 - 1 + movl %r10d, %ebx + movl %r14d, %ecx + addl 172(%rsp), %r8d + xorl %r15d, %ecx + vpsrlq $0x11, %ymm6, %ymm8 + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 2 - 3 + xorl %r13d, %edx + andl %r13d, %ecx + rorl $5, %edx + xorl %r15d, %ecx + xorl %r13d, %edx + addl %ecx, %r8d + vpsrld $10, %ymm6, %ymm9 + vpxor %ymm8, %ymm7, %ymm8 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r9d, %ebx + addl %edx, %r8d + movl %r9d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r9d, %ecx + xorl %r10d, %eax + vpxor %ymm9, %ymm8, %ymm9 + # rnd_1: 6 - 6 + rorl $11, %ecx + addl %r8d, %r12d + xorl %r9d, %ecx + addl %eax, %r8d + vpshufb %ymm12, %ymm9, %ymm9 + # rnd_1: 7 - 7 + rorl $2, %ecx + movl %r12d, %edx + addl %ecx, %r8d + vpaddd %ymm4, %ymm9, %ymm1 + # msg_sched done: 40-43 + # msg_sched: 48-51 + # rnd_0: 0 - 0 + rorl $14, %edx + vpalignr $4, %ymm2, %ymm3, %ymm5 + vpalignr $4, %ymm0, %ymm1, %ymm4 + # rnd_0: 1 - 2 + movl %r9d, %eax + movl %r13d, %ecx + addl 192(%rsp), %r15d + xorl %r14d, %ecx + xorl %r12d, %edx + andl %r12d, %ecx + vpsrld $7, %ymm5, %ymm6 + vpslld $25, %ymm5, %ymm7 + # rnd_0: 3 - 4 + rorl $5, %edx + xorl %r14d, %ecx + xorl %r12d, %edx + addl %ecx, %r15d + rorl $6, %edx + xorl %r8d, %eax + addl %edx, %r15d + movl %r8d, %ecx + vpsrld $18, %ymm5, %ymm8 + vpslld $14, %ymm5, %ymm9 + # rnd_0: 5 - 6 + andl %eax, %ebx + rorl $9, %ecx + xorl %r8d, %ecx + xorl %r9d, %ebx + rorl $11, %ecx + addl %r15d, %r11d + xorl %r8d, %ecx + addl %ebx, %r15d + vpor %ymm6, %ymm7, %ymm6 + vpor %ymm8, %ymm9, %ymm8 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r11d, %edx + addl %ecx, %r15d + # rnd_1: 0 - 1 + rorl $14, %edx + movl %r8d, %ebx + movl %r12d, %ecx + addl 196(%rsp), %r14d + xorl %r13d, %ecx + vpsrld $3, %ymm5, %ymm9 + vpxor %ymm6, %ymm8, %ymm6 + # rnd_1: 2 - 3 + xorl %r11d, %edx + andl %r11d, %ecx + rorl $5, %edx + xorl %r13d, %ecx + xorl %r11d, %edx + addl %ecx, %r14d + vpxor %ymm6, %ymm9, %ymm5 + vpshufd $0xfa, %ymm1, %ymm6 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r15d, %ebx + addl %edx, %r14d + movl %r15d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r15d, %ecx + xorl %r8d, %eax + vpsrld $10, %ymm6, %ymm8 + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 6 - 7 + rorl $11, %ecx + addl %r14d, %r10d + xorl %r15d, %ecx + addl %eax, %r14d + rorl $2, %ecx + movl %r10d, %edx + addl %ecx, %r14d + # rnd_0: 0 - 0 + rorl $14, %edx + vpsrlq $0x11, %ymm6, %ymm6 + vpaddd %ymm2, %ymm4, %ymm4 + # rnd_0: 1 - 3 + movl %r15d, %eax + movl %r11d, %ecx + addl 200(%rsp), %r13d + xorl %r12d, %ecx + xorl %r10d, %edx + andl %r10d, %ecx + rorl $5, %edx + xorl %r12d, %ecx + xorl %r10d, %edx + addl %ecx, %r13d + vpxor %ymm6, %ymm7, %ymm6 + vpaddd %ymm5, %ymm4, %ymm4 + # rnd_0: 4 - 4 + rorl $6, %edx + xorl %r14d, %eax + addl %edx, %r13d + movl %r14d, %ecx + vpxor %ymm6, %ymm8, %ymm8 + # rnd_0: 5 - 5 + andl %eax, %ebx + rorl $9, %ecx + xorl %r14d, %ecx + xorl %r15d, %ebx + vpshufb %ymm11, %ymm8, %ymm8 + # rnd_0: 6 - 6 + rorl $11, %ecx + addl %r13d, %r9d + xorl %r14d, %ecx + addl %ebx, %r13d + vpaddd %ymm8, %ymm4, %ymm4 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r9d, %edx + addl %ecx, %r13d + # rnd_1: 0 - 0 + rorl $14, %edx + vpshufd $0x50, %ymm4, %ymm6 + # rnd_1: 1 - 1 + movl %r14d, %ebx + movl %r10d, %ecx + addl 204(%rsp), %r12d + xorl %r11d, %ecx + vpsrlq $0x11, %ymm6, %ymm8 + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 2 - 3 + xorl %r9d, %edx + andl %r9d, %ecx + rorl $5, %edx + xorl %r11d, %ecx + xorl %r9d, %edx + addl %ecx, %r12d + vpsrld $10, %ymm6, %ymm9 + vpxor %ymm8, %ymm7, %ymm8 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r13d, %ebx + addl %edx, %r12d + movl %r13d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r13d, %ecx + xorl %r14d, %eax + vpxor %ymm9, %ymm8, %ymm9 + # rnd_1: 6 - 6 + rorl $11, %ecx + addl %r12d, %r8d + xorl %r13d, %ecx + addl %eax, %r12d + vpshufb %ymm12, %ymm9, %ymm9 + # rnd_1: 7 - 7 + rorl $2, %ecx + movl %r8d, %edx + addl %ecx, %r12d + vpaddd %ymm4, %ymm9, %ymm2 + # msg_sched done: 48-51 + # msg_sched: 56-59 + # rnd_0: 0 - 0 + rorl $14, %edx + vpalignr $4, %ymm3, %ymm0, %ymm5 + vpalignr $4, %ymm1, %ymm2, %ymm4 + # rnd_0: 1 - 2 + movl %r13d, %eax + movl %r9d, %ecx + addl 224(%rsp), %r11d + xorl %r10d, %ecx + xorl %r8d, %edx + andl %r8d, %ecx + vpsrld $7, %ymm5, %ymm6 + vpslld $25, %ymm5, %ymm7 + # rnd_0: 3 - 4 + rorl $5, %edx + xorl %r10d, %ecx + xorl %r8d, %edx + addl %ecx, %r11d + rorl $6, %edx + xorl %r12d, %eax + addl %edx, %r11d + movl %r12d, %ecx + vpsrld $18, %ymm5, %ymm8 + vpslld $14, %ymm5, %ymm9 + # rnd_0: 5 - 6 + andl %eax, %ebx + rorl $9, %ecx + xorl %r12d, %ecx + xorl %r13d, %ebx + rorl $11, %ecx + addl %r11d, %r15d + xorl %r12d, %ecx + addl %ebx, %r11d + vpor %ymm6, %ymm7, %ymm6 + vpor %ymm8, %ymm9, %ymm8 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r15d, %edx + addl %ecx, %r11d + # rnd_1: 0 - 1 + rorl $14, %edx + movl %r12d, %ebx + movl %r8d, %ecx + addl 228(%rsp), %r10d + xorl %r9d, %ecx + vpsrld $3, %ymm5, %ymm9 + vpxor %ymm6, %ymm8, %ymm6 + # rnd_1: 2 - 3 + xorl %r15d, %edx + andl %r15d, %ecx + rorl $5, %edx + xorl %r9d, %ecx + xorl %r15d, %edx + addl %ecx, %r10d + vpxor %ymm6, %ymm9, %ymm5 + vpshufd $0xfa, %ymm2, %ymm6 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r11d, %ebx + addl %edx, %r10d + movl %r11d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r11d, %ecx + xorl %r12d, %eax + vpsrld $10, %ymm6, %ymm8 + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 6 - 7 + rorl $11, %ecx + addl %r10d, %r14d + xorl %r11d, %ecx + addl %eax, %r10d + rorl $2, %ecx + movl %r14d, %edx + addl %ecx, %r10d + # rnd_0: 0 - 0 + rorl $14, %edx + vpsrlq $0x11, %ymm6, %ymm6 + vpaddd %ymm3, %ymm4, %ymm4 + # rnd_0: 1 - 3 + movl %r11d, %eax + movl %r15d, %ecx + addl 232(%rsp), %r9d + xorl %r8d, %ecx + xorl %r14d, %edx + andl %r14d, %ecx + rorl $5, %edx + xorl %r8d, %ecx + xorl %r14d, %edx + addl %ecx, %r9d + vpxor %ymm6, %ymm7, %ymm6 + vpaddd %ymm5, %ymm4, %ymm4 + # rnd_0: 4 - 4 + rorl $6, %edx + xorl %r10d, %eax + addl %edx, %r9d + movl %r10d, %ecx + vpxor %ymm6, %ymm8, %ymm8 + # rnd_0: 5 - 5 + andl %eax, %ebx + rorl $9, %ecx + xorl %r10d, %ecx + xorl %r11d, %ebx + vpshufb %ymm11, %ymm8, %ymm8 + # rnd_0: 6 - 6 + rorl $11, %ecx + addl %r9d, %r13d + xorl %r10d, %ecx + addl %ebx, %r9d + vpaddd %ymm8, %ymm4, %ymm4 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r13d, %edx + addl %ecx, %r9d + # rnd_1: 0 - 0 + rorl $14, %edx + vpshufd $0x50, %ymm4, %ymm6 + # rnd_1: 1 - 1 + movl %r10d, %ebx + movl %r14d, %ecx + addl 236(%rsp), %r8d + xorl %r15d, %ecx + vpsrlq $0x11, %ymm6, %ymm8 + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 2 - 3 + xorl %r13d, %edx + andl %r13d, %ecx + rorl $5, %edx + xorl %r15d, %ecx + xorl %r13d, %edx + addl %ecx, %r8d + vpsrld $10, %ymm6, %ymm9 + vpxor %ymm8, %ymm7, %ymm8 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r9d, %ebx + addl %edx, %r8d + movl %r9d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r9d, %ecx + xorl %r10d, %eax + vpxor %ymm9, %ymm8, %ymm9 + # rnd_1: 6 - 6 + rorl $11, %ecx + addl %r8d, %r12d + xorl %r9d, %ecx + addl %eax, %r8d + vpshufb %ymm12, %ymm9, %ymm9 + # rnd_1: 7 - 7 + rorl $2, %ecx + movl %r12d, %edx + addl %ecx, %r8d + vpaddd %ymm4, %ymm9, %ymm3 + # msg_sched done: 56-59 + # set_w_k_xfer_4: 8 + vpaddd 256+L_avx2_sha256_k(%rip), %ymm0, %ymm4 + vpaddd 288+L_avx2_sha256_k(%rip), %ymm1, %ymm5 + vmovdqu %ymm4, 256(%rsp) + vmovdqu %ymm5, 288(%rsp) + vpaddd 320+L_avx2_sha256_k(%rip), %ymm2, %ymm4 + vpaddd 352+L_avx2_sha256_k(%rip), %ymm3, %ymm5 + vmovdqu %ymm4, 320(%rsp) + vmovdqu %ymm5, 352(%rsp) + # msg_sched: 64-67 + # rnd_0: 0 - 0 + rorl $14, %edx + vpalignr $4, %ymm0, %ymm1, %ymm5 + vpalignr $4, %ymm2, %ymm3, %ymm4 + # rnd_0: 1 - 2 + movl %r9d, %eax + movl %r13d, %ecx + addl 256(%rsp), %r15d + xorl %r14d, %ecx + xorl %r12d, %edx + andl %r12d, %ecx + vpsrld $7, %ymm5, %ymm6 + vpslld $25, %ymm5, %ymm7 + # rnd_0: 3 - 4 + rorl $5, %edx + xorl %r14d, %ecx + xorl %r12d, %edx + addl %ecx, %r15d + rorl $6, %edx + xorl %r8d, %eax + addl %edx, %r15d + movl %r8d, %ecx + vpsrld $18, %ymm5, %ymm8 + vpslld $14, %ymm5, %ymm9 + # rnd_0: 5 - 6 + andl %eax, %ebx + rorl $9, %ecx + xorl %r8d, %ecx + xorl %r9d, %ebx + rorl $11, %ecx + addl %r15d, %r11d + xorl %r8d, %ecx + addl %ebx, %r15d + vpor %ymm6, %ymm7, %ymm6 + vpor %ymm8, %ymm9, %ymm8 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r11d, %edx + addl %ecx, %r15d + # rnd_1: 0 - 1 + rorl $14, %edx + movl %r8d, %ebx + movl %r12d, %ecx + addl 260(%rsp), %r14d + xorl %r13d, %ecx + vpsrld $3, %ymm5, %ymm9 + vpxor %ymm6, %ymm8, %ymm6 + # rnd_1: 2 - 3 + xorl %r11d, %edx + andl %r11d, %ecx + rorl $5, %edx + xorl %r13d, %ecx + xorl %r11d, %edx + addl %ecx, %r14d + vpxor %ymm6, %ymm9, %ymm5 + vpshufd $0xfa, %ymm3, %ymm6 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r15d, %ebx + addl %edx, %r14d + movl %r15d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r15d, %ecx + xorl %r8d, %eax + vpsrld $10, %ymm6, %ymm8 + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 6 - 7 + rorl $11, %ecx + addl %r14d, %r10d + xorl %r15d, %ecx + addl %eax, %r14d + rorl $2, %ecx + movl %r10d, %edx + addl %ecx, %r14d + # rnd_0: 0 - 0 + rorl $14, %edx + vpsrlq $0x11, %ymm6, %ymm6 + vpaddd %ymm0, %ymm4, %ymm4 + # rnd_0: 1 - 3 + movl %r15d, %eax + movl %r11d, %ecx + addl 264(%rsp), %r13d + xorl %r12d, %ecx + xorl %r10d, %edx + andl %r10d, %ecx + rorl $5, %edx + xorl %r12d, %ecx + xorl %r10d, %edx + addl %ecx, %r13d + vpxor %ymm6, %ymm7, %ymm6 + vpaddd %ymm5, %ymm4, %ymm4 + # rnd_0: 4 - 4 + rorl $6, %edx + xorl %r14d, %eax + addl %edx, %r13d + movl %r14d, %ecx + vpxor %ymm6, %ymm8, %ymm8 + # rnd_0: 5 - 5 + andl %eax, %ebx + rorl $9, %ecx + xorl %r14d, %ecx + xorl %r15d, %ebx + vpshufb %ymm11, %ymm8, %ymm8 + # rnd_0: 6 - 6 + rorl $11, %ecx + addl %r13d, %r9d + xorl %r14d, %ecx + addl %ebx, %r13d + vpaddd %ymm8, %ymm4, %ymm4 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r9d, %edx + addl %ecx, %r13d + # rnd_1: 0 - 0 + rorl $14, %edx + vpshufd $0x50, %ymm4, %ymm6 + # rnd_1: 1 - 1 + movl %r14d, %ebx + movl %r10d, %ecx + addl 268(%rsp), %r12d + xorl %r11d, %ecx + vpsrlq $0x11, %ymm6, %ymm8 + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 2 - 3 + xorl %r9d, %edx + andl %r9d, %ecx + rorl $5, %edx + xorl %r11d, %ecx + xorl %r9d, %edx + addl %ecx, %r12d + vpsrld $10, %ymm6, %ymm9 + vpxor %ymm8, %ymm7, %ymm8 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r13d, %ebx + addl %edx, %r12d + movl %r13d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r13d, %ecx + xorl %r14d, %eax + vpxor %ymm9, %ymm8, %ymm9 + # rnd_1: 6 - 6 + rorl $11, %ecx + addl %r12d, %r8d + xorl %r13d, %ecx + addl %eax, %r12d + vpshufb %ymm12, %ymm9, %ymm9 + # rnd_1: 7 - 7 + rorl $2, %ecx + movl %r8d, %edx + addl %ecx, %r12d + vpaddd %ymm4, %ymm9, %ymm0 + # msg_sched done: 64-67 + # msg_sched: 72-75 + # rnd_0: 0 - 0 + rorl $14, %edx + vpalignr $4, %ymm1, %ymm2, %ymm5 + vpalignr $4, %ymm3, %ymm0, %ymm4 + # rnd_0: 1 - 2 + movl %r13d, %eax + movl %r9d, %ecx + addl 288(%rsp), %r11d + xorl %r10d, %ecx + xorl %r8d, %edx + andl %r8d, %ecx + vpsrld $7, %ymm5, %ymm6 + vpslld $25, %ymm5, %ymm7 + # rnd_0: 3 - 4 + rorl $5, %edx + xorl %r10d, %ecx + xorl %r8d, %edx + addl %ecx, %r11d + rorl $6, %edx + xorl %r12d, %eax + addl %edx, %r11d + movl %r12d, %ecx + vpsrld $18, %ymm5, %ymm8 + vpslld $14, %ymm5, %ymm9 + # rnd_0: 5 - 6 + andl %eax, %ebx + rorl $9, %ecx + xorl %r12d, %ecx + xorl %r13d, %ebx + rorl $11, %ecx + addl %r11d, %r15d + xorl %r12d, %ecx + addl %ebx, %r11d + vpor %ymm6, %ymm7, %ymm6 + vpor %ymm8, %ymm9, %ymm8 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r15d, %edx + addl %ecx, %r11d + # rnd_1: 0 - 1 + rorl $14, %edx + movl %r12d, %ebx + movl %r8d, %ecx + addl 292(%rsp), %r10d + xorl %r9d, %ecx + vpsrld $3, %ymm5, %ymm9 + vpxor %ymm6, %ymm8, %ymm6 + # rnd_1: 2 - 3 + xorl %r15d, %edx + andl %r15d, %ecx + rorl $5, %edx + xorl %r9d, %ecx + xorl %r15d, %edx + addl %ecx, %r10d + vpxor %ymm6, %ymm9, %ymm5 + vpshufd $0xfa, %ymm0, %ymm6 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r11d, %ebx + addl %edx, %r10d + movl %r11d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r11d, %ecx + xorl %r12d, %eax + vpsrld $10, %ymm6, %ymm8 + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 6 - 7 + rorl $11, %ecx + addl %r10d, %r14d + xorl %r11d, %ecx + addl %eax, %r10d + rorl $2, %ecx + movl %r14d, %edx + addl %ecx, %r10d + # rnd_0: 0 - 0 + rorl $14, %edx + vpsrlq $0x11, %ymm6, %ymm6 + vpaddd %ymm1, %ymm4, %ymm4 + # rnd_0: 1 - 3 + movl %r11d, %eax + movl %r15d, %ecx + addl 296(%rsp), %r9d + xorl %r8d, %ecx + xorl %r14d, %edx + andl %r14d, %ecx + rorl $5, %edx + xorl %r8d, %ecx + xorl %r14d, %edx + addl %ecx, %r9d + vpxor %ymm6, %ymm7, %ymm6 + vpaddd %ymm5, %ymm4, %ymm4 + # rnd_0: 4 - 4 + rorl $6, %edx + xorl %r10d, %eax + addl %edx, %r9d + movl %r10d, %ecx + vpxor %ymm6, %ymm8, %ymm8 + # rnd_0: 5 - 5 + andl %eax, %ebx + rorl $9, %ecx + xorl %r10d, %ecx + xorl %r11d, %ebx + vpshufb %ymm11, %ymm8, %ymm8 + # rnd_0: 6 - 6 + rorl $11, %ecx + addl %r9d, %r13d + xorl %r10d, %ecx + addl %ebx, %r9d + vpaddd %ymm8, %ymm4, %ymm4 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r13d, %edx + addl %ecx, %r9d + # rnd_1: 0 - 0 + rorl $14, %edx + vpshufd $0x50, %ymm4, %ymm6 + # rnd_1: 1 - 1 + movl %r10d, %ebx + movl %r14d, %ecx + addl 300(%rsp), %r8d + xorl %r15d, %ecx + vpsrlq $0x11, %ymm6, %ymm8 + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 2 - 3 + xorl %r13d, %edx + andl %r13d, %ecx + rorl $5, %edx + xorl %r15d, %ecx + xorl %r13d, %edx + addl %ecx, %r8d + vpsrld $10, %ymm6, %ymm9 + vpxor %ymm8, %ymm7, %ymm8 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r9d, %ebx + addl %edx, %r8d + movl %r9d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r9d, %ecx + xorl %r10d, %eax + vpxor %ymm9, %ymm8, %ymm9 + # rnd_1: 6 - 6 + rorl $11, %ecx + addl %r8d, %r12d + xorl %r9d, %ecx + addl %eax, %r8d + vpshufb %ymm12, %ymm9, %ymm9 + # rnd_1: 7 - 7 + rorl $2, %ecx + movl %r12d, %edx + addl %ecx, %r8d + vpaddd %ymm4, %ymm9, %ymm1 + # msg_sched done: 72-75 + # msg_sched: 80-83 + # rnd_0: 0 - 0 + rorl $14, %edx + vpalignr $4, %ymm2, %ymm3, %ymm5 + vpalignr $4, %ymm0, %ymm1, %ymm4 + # rnd_0: 1 - 2 + movl %r9d, %eax + movl %r13d, %ecx + addl 320(%rsp), %r15d + xorl %r14d, %ecx + xorl %r12d, %edx + andl %r12d, %ecx + vpsrld $7, %ymm5, %ymm6 + vpslld $25, %ymm5, %ymm7 + # rnd_0: 3 - 4 + rorl $5, %edx + xorl %r14d, %ecx + xorl %r12d, %edx + addl %ecx, %r15d + rorl $6, %edx + xorl %r8d, %eax + addl %edx, %r15d + movl %r8d, %ecx + vpsrld $18, %ymm5, %ymm8 + vpslld $14, %ymm5, %ymm9 + # rnd_0: 5 - 6 + andl %eax, %ebx + rorl $9, %ecx + xorl %r8d, %ecx + xorl %r9d, %ebx + rorl $11, %ecx + addl %r15d, %r11d + xorl %r8d, %ecx + addl %ebx, %r15d + vpor %ymm6, %ymm7, %ymm6 + vpor %ymm8, %ymm9, %ymm8 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r11d, %edx + addl %ecx, %r15d + # rnd_1: 0 - 1 + rorl $14, %edx + movl %r8d, %ebx + movl %r12d, %ecx + addl 324(%rsp), %r14d + xorl %r13d, %ecx + vpsrld $3, %ymm5, %ymm9 + vpxor %ymm6, %ymm8, %ymm6 + # rnd_1: 2 - 3 + xorl %r11d, %edx + andl %r11d, %ecx + rorl $5, %edx + xorl %r13d, %ecx + xorl %r11d, %edx + addl %ecx, %r14d + vpxor %ymm6, %ymm9, %ymm5 + vpshufd $0xfa, %ymm1, %ymm6 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r15d, %ebx + addl %edx, %r14d + movl %r15d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r15d, %ecx + xorl %r8d, %eax + vpsrld $10, %ymm6, %ymm8 + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 6 - 7 + rorl $11, %ecx + addl %r14d, %r10d + xorl %r15d, %ecx + addl %eax, %r14d + rorl $2, %ecx + movl %r10d, %edx + addl %ecx, %r14d + # rnd_0: 0 - 0 + rorl $14, %edx + vpsrlq $0x11, %ymm6, %ymm6 + vpaddd %ymm2, %ymm4, %ymm4 + # rnd_0: 1 - 3 + movl %r15d, %eax + movl %r11d, %ecx + addl 328(%rsp), %r13d + xorl %r12d, %ecx + xorl %r10d, %edx + andl %r10d, %ecx + rorl $5, %edx + xorl %r12d, %ecx + xorl %r10d, %edx + addl %ecx, %r13d + vpxor %ymm6, %ymm7, %ymm6 + vpaddd %ymm5, %ymm4, %ymm4 + # rnd_0: 4 - 4 + rorl $6, %edx + xorl %r14d, %eax + addl %edx, %r13d + movl %r14d, %ecx + vpxor %ymm6, %ymm8, %ymm8 + # rnd_0: 5 - 5 + andl %eax, %ebx + rorl $9, %ecx + xorl %r14d, %ecx + xorl %r15d, %ebx + vpshufb %ymm11, %ymm8, %ymm8 + # rnd_0: 6 - 6 + rorl $11, %ecx + addl %r13d, %r9d + xorl %r14d, %ecx + addl %ebx, %r13d + vpaddd %ymm8, %ymm4, %ymm4 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r9d, %edx + addl %ecx, %r13d + # rnd_1: 0 - 0 + rorl $14, %edx + vpshufd $0x50, %ymm4, %ymm6 + # rnd_1: 1 - 1 + movl %r14d, %ebx + movl %r10d, %ecx + addl 332(%rsp), %r12d + xorl %r11d, %ecx + vpsrlq $0x11, %ymm6, %ymm8 + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 2 - 3 + xorl %r9d, %edx + andl %r9d, %ecx + rorl $5, %edx + xorl %r11d, %ecx + xorl %r9d, %edx + addl %ecx, %r12d + vpsrld $10, %ymm6, %ymm9 + vpxor %ymm8, %ymm7, %ymm8 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r13d, %ebx + addl %edx, %r12d + movl %r13d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r13d, %ecx + xorl %r14d, %eax + vpxor %ymm9, %ymm8, %ymm9 + # rnd_1: 6 - 6 + rorl $11, %ecx + addl %r12d, %r8d + xorl %r13d, %ecx + addl %eax, %r12d + vpshufb %ymm12, %ymm9, %ymm9 + # rnd_1: 7 - 7 + rorl $2, %ecx + movl %r8d, %edx + addl %ecx, %r12d + vpaddd %ymm4, %ymm9, %ymm2 + # msg_sched done: 80-83 + # msg_sched: 88-91 + # rnd_0: 0 - 0 + rorl $14, %edx + vpalignr $4, %ymm3, %ymm0, %ymm5 + vpalignr $4, %ymm1, %ymm2, %ymm4 + # rnd_0: 1 - 2 + movl %r13d, %eax + movl %r9d, %ecx + addl 352(%rsp), %r11d + xorl %r10d, %ecx + xorl %r8d, %edx + andl %r8d, %ecx + vpsrld $7, %ymm5, %ymm6 + vpslld $25, %ymm5, %ymm7 + # rnd_0: 3 - 4 + rorl $5, %edx + xorl %r10d, %ecx + xorl %r8d, %edx + addl %ecx, %r11d + rorl $6, %edx + xorl %r12d, %eax + addl %edx, %r11d + movl %r12d, %ecx + vpsrld $18, %ymm5, %ymm8 + vpslld $14, %ymm5, %ymm9 + # rnd_0: 5 - 6 + andl %eax, %ebx + rorl $9, %ecx + xorl %r12d, %ecx + xorl %r13d, %ebx + rorl $11, %ecx + addl %r11d, %r15d + xorl %r12d, %ecx + addl %ebx, %r11d + vpor %ymm6, %ymm7, %ymm6 + vpor %ymm8, %ymm9, %ymm8 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r15d, %edx + addl %ecx, %r11d + # rnd_1: 0 - 1 + rorl $14, %edx + movl %r12d, %ebx + movl %r8d, %ecx + addl 356(%rsp), %r10d + xorl %r9d, %ecx + vpsrld $3, %ymm5, %ymm9 + vpxor %ymm6, %ymm8, %ymm6 + # rnd_1: 2 - 3 + xorl %r15d, %edx + andl %r15d, %ecx + rorl $5, %edx + xorl %r9d, %ecx + xorl %r15d, %edx + addl %ecx, %r10d + vpxor %ymm6, %ymm9, %ymm5 + vpshufd $0xfa, %ymm2, %ymm6 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r11d, %ebx + addl %edx, %r10d + movl %r11d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r11d, %ecx + xorl %r12d, %eax + vpsrld $10, %ymm6, %ymm8 + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 6 - 7 + rorl $11, %ecx + addl %r10d, %r14d + xorl %r11d, %ecx + addl %eax, %r10d + rorl $2, %ecx + movl %r14d, %edx + addl %ecx, %r10d + # rnd_0: 0 - 0 + rorl $14, %edx + vpsrlq $0x11, %ymm6, %ymm6 + vpaddd %ymm3, %ymm4, %ymm4 + # rnd_0: 1 - 3 + movl %r11d, %eax + movl %r15d, %ecx + addl 360(%rsp), %r9d + xorl %r8d, %ecx + xorl %r14d, %edx + andl %r14d, %ecx + rorl $5, %edx + xorl %r8d, %ecx + xorl %r14d, %edx + addl %ecx, %r9d + vpxor %ymm6, %ymm7, %ymm6 + vpaddd %ymm5, %ymm4, %ymm4 + # rnd_0: 4 - 4 + rorl $6, %edx + xorl %r10d, %eax + addl %edx, %r9d + movl %r10d, %ecx + vpxor %ymm6, %ymm8, %ymm8 + # rnd_0: 5 - 5 + andl %eax, %ebx + rorl $9, %ecx + xorl %r10d, %ecx + xorl %r11d, %ebx + vpshufb %ymm11, %ymm8, %ymm8 + # rnd_0: 6 - 6 + rorl $11, %ecx + addl %r9d, %r13d + xorl %r10d, %ecx + addl %ebx, %r9d + vpaddd %ymm8, %ymm4, %ymm4 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r13d, %edx + addl %ecx, %r9d + # rnd_1: 0 - 0 + rorl $14, %edx + vpshufd $0x50, %ymm4, %ymm6 + # rnd_1: 1 - 1 + movl %r10d, %ebx + movl %r14d, %ecx + addl 364(%rsp), %r8d + xorl %r15d, %ecx + vpsrlq $0x11, %ymm6, %ymm8 + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 2 - 3 + xorl %r13d, %edx + andl %r13d, %ecx + rorl $5, %edx + xorl %r15d, %ecx + xorl %r13d, %edx + addl %ecx, %r8d + vpsrld $10, %ymm6, %ymm9 + vpxor %ymm8, %ymm7, %ymm8 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r9d, %ebx + addl %edx, %r8d + movl %r9d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r9d, %ecx + xorl %r10d, %eax + vpxor %ymm9, %ymm8, %ymm9 + # rnd_1: 6 - 6 + rorl $11, %ecx + addl %r8d, %r12d + xorl %r9d, %ecx + addl %eax, %r8d + vpshufb %ymm12, %ymm9, %ymm9 + # rnd_1: 7 - 7 + rorl $2, %ecx + movl %r12d, %edx + addl %ecx, %r8d + vpaddd %ymm4, %ymm9, %ymm3 + # msg_sched done: 88-91 + # set_w_k_xfer_4: 12 + vpaddd 384+L_avx2_sha256_k(%rip), %ymm0, %ymm4 + vpaddd 416+L_avx2_sha256_k(%rip), %ymm1, %ymm5 + vmovdqu %ymm4, 384(%rsp) + vmovdqu %ymm5, 416(%rsp) + vpaddd 448+L_avx2_sha256_k(%rip), %ymm2, %ymm4 + vpaddd 480+L_avx2_sha256_k(%rip), %ymm3, %ymm5 + vmovdqu %ymm4, 448(%rsp) + vmovdqu %ymm5, 480(%rsp) + # rnd_all_4: 24-27 + addl 384(%rsp), %r15d + movl %r13d, %ecx + movl %r9d, %eax + xorl %r14d, %ecx + rorl $14, %edx + andl %r12d, %ecx + xorl %r12d, %edx + xorl %r14d, %ecx + rorl $5, %edx + addl %ecx, %r15d + xorl %r12d, %edx + xorl %r8d, %eax + rorl $6, %edx + movl %r8d, %ecx + addl %edx, %r15d + rorl $9, %ecx + andl %eax, %ebx + xorl %r8d, %ecx + xorl %r9d, %ebx + rorl $11, %ecx + addl %r15d, %r11d + xorl %r8d, %ecx + addl %ebx, %r15d + rorl $2, %ecx + movl %r11d, %edx + addl %ecx, %r15d + addl 388(%rsp), %r14d + movl %r12d, %ecx + movl %r8d, %ebx + xorl %r13d, %ecx + rorl $14, %edx + andl %r11d, %ecx + xorl %r11d, %edx + xorl %r13d, %ecx + rorl $5, %edx + addl %ecx, %r14d + xorl %r11d, %edx + xorl %r15d, %ebx + rorl $6, %edx + movl %r15d, %ecx + addl %edx, %r14d + rorl $9, %ecx + andl %ebx, %eax + xorl %r15d, %ecx + xorl %r8d, %eax + rorl $11, %ecx + addl %r14d, %r10d + xorl %r15d, %ecx + addl %eax, %r14d + rorl $2, %ecx + movl %r10d, %edx + addl %ecx, %r14d + addl 392(%rsp), %r13d + movl %r11d, %ecx + movl %r15d, %eax + xorl %r12d, %ecx + rorl $14, %edx + andl %r10d, %ecx + xorl %r10d, %edx + xorl %r12d, %ecx + rorl $5, %edx + addl %ecx, %r13d + xorl %r10d, %edx + xorl %r14d, %eax + rorl $6, %edx + movl %r14d, %ecx + addl %edx, %r13d + rorl $9, %ecx + andl %eax, %ebx + xorl %r14d, %ecx + xorl %r15d, %ebx + rorl $11, %ecx + addl %r13d, %r9d + xorl %r14d, %ecx + addl %ebx, %r13d + rorl $2, %ecx + movl %r9d, %edx + addl %ecx, %r13d + addl 396(%rsp), %r12d + movl %r10d, %ecx + movl %r14d, %ebx + xorl %r11d, %ecx + rorl $14, %edx + andl %r9d, %ecx + xorl %r9d, %edx + xorl %r11d, %ecx + rorl $5, %edx + addl %ecx, %r12d + xorl %r9d, %edx + xorl %r13d, %ebx + rorl $6, %edx + movl %r13d, %ecx + addl %edx, %r12d + rorl $9, %ecx + andl %ebx, %eax + xorl %r13d, %ecx + xorl %r14d, %eax + rorl $11, %ecx + addl %r12d, %r8d + xorl %r13d, %ecx + addl %eax, %r12d + rorl $2, %ecx + movl %r8d, %edx + addl %ecx, %r12d + # rnd_all_4: 26-29 + addl 416(%rsp), %r11d + movl %r9d, %ecx + movl %r13d, %eax + xorl %r10d, %ecx + rorl $14, %edx + andl %r8d, %ecx + xorl %r8d, %edx + xorl %r10d, %ecx + rorl $5, %edx + addl %ecx, %r11d + xorl %r8d, %edx + xorl %r12d, %eax + rorl $6, %edx + movl %r12d, %ecx + addl %edx, %r11d + rorl $9, %ecx + andl %eax, %ebx + xorl %r12d, %ecx + xorl %r13d, %ebx + rorl $11, %ecx + addl %r11d, %r15d + xorl %r12d, %ecx + addl %ebx, %r11d + rorl $2, %ecx + movl %r15d, %edx + addl %ecx, %r11d + addl 420(%rsp), %r10d + movl %r8d, %ecx + movl %r12d, %ebx + xorl %r9d, %ecx + rorl $14, %edx + andl %r15d, %ecx + xorl %r15d, %edx + xorl %r9d, %ecx + rorl $5, %edx + addl %ecx, %r10d + xorl %r15d, %edx + xorl %r11d, %ebx + rorl $6, %edx + movl %r11d, %ecx + addl %edx, %r10d + rorl $9, %ecx + andl %ebx, %eax + xorl %r11d, %ecx + xorl %r12d, %eax + rorl $11, %ecx + addl %r10d, %r14d + xorl %r11d, %ecx + addl %eax, %r10d + rorl $2, %ecx + movl %r14d, %edx + addl %ecx, %r10d + addl 424(%rsp), %r9d + movl %r15d, %ecx + movl %r11d, %eax + xorl %r8d, %ecx + rorl $14, %edx + andl %r14d, %ecx + xorl %r14d, %edx + xorl %r8d, %ecx + rorl $5, %edx + addl %ecx, %r9d + xorl %r14d, %edx + xorl %r10d, %eax + rorl $6, %edx + movl %r10d, %ecx + addl %edx, %r9d + rorl $9, %ecx + andl %eax, %ebx + xorl %r10d, %ecx + xorl %r11d, %ebx + rorl $11, %ecx + addl %r9d, %r13d + xorl %r10d, %ecx + addl %ebx, %r9d + rorl $2, %ecx + movl %r13d, %edx + addl %ecx, %r9d + addl 428(%rsp), %r8d + movl %r14d, %ecx + movl %r10d, %ebx + xorl %r15d, %ecx + rorl $14, %edx + andl %r13d, %ecx + xorl %r13d, %edx + xorl %r15d, %ecx + rorl $5, %edx + addl %ecx, %r8d + xorl %r13d, %edx + xorl %r9d, %ebx + rorl $6, %edx + movl %r9d, %ecx + addl %edx, %r8d + rorl $9, %ecx + andl %ebx, %eax + xorl %r9d, %ecx + xorl %r10d, %eax + rorl $11, %ecx + addl %r8d, %r12d + xorl %r9d, %ecx + addl %eax, %r8d + rorl $2, %ecx + movl %r12d, %edx + addl %ecx, %r8d + # rnd_all_4: 28-31 + addl 448(%rsp), %r15d + movl %r13d, %ecx + movl %r9d, %eax + xorl %r14d, %ecx + rorl $14, %edx + andl %r12d, %ecx + xorl %r12d, %edx + xorl %r14d, %ecx + rorl $5, %edx + addl %ecx, %r15d + xorl %r12d, %edx + xorl %r8d, %eax + rorl $6, %edx + movl %r8d, %ecx + addl %edx, %r15d + rorl $9, %ecx + andl %eax, %ebx + xorl %r8d, %ecx + xorl %r9d, %ebx + rorl $11, %ecx + addl %r15d, %r11d + xorl %r8d, %ecx + addl %ebx, %r15d + rorl $2, %ecx + movl %r11d, %edx + addl %ecx, %r15d + addl 452(%rsp), %r14d + movl %r12d, %ecx + movl %r8d, %ebx + xorl %r13d, %ecx + rorl $14, %edx + andl %r11d, %ecx + xorl %r11d, %edx + xorl %r13d, %ecx + rorl $5, %edx + addl %ecx, %r14d + xorl %r11d, %edx + xorl %r15d, %ebx + rorl $6, %edx + movl %r15d, %ecx + addl %edx, %r14d + rorl $9, %ecx + andl %ebx, %eax + xorl %r15d, %ecx + xorl %r8d, %eax + rorl $11, %ecx + addl %r14d, %r10d + xorl %r15d, %ecx + addl %eax, %r14d + rorl $2, %ecx + movl %r10d, %edx + addl %ecx, %r14d + addl 456(%rsp), %r13d + movl %r11d, %ecx + movl %r15d, %eax + xorl %r12d, %ecx + rorl $14, %edx + andl %r10d, %ecx + xorl %r10d, %edx + xorl %r12d, %ecx + rorl $5, %edx + addl %ecx, %r13d + xorl %r10d, %edx + xorl %r14d, %eax + rorl $6, %edx + movl %r14d, %ecx + addl %edx, %r13d + rorl $9, %ecx + andl %eax, %ebx + xorl %r14d, %ecx + xorl %r15d, %ebx + rorl $11, %ecx + addl %r13d, %r9d + xorl %r14d, %ecx + addl %ebx, %r13d + rorl $2, %ecx + movl %r9d, %edx + addl %ecx, %r13d + addl 460(%rsp), %r12d + movl %r10d, %ecx + movl %r14d, %ebx + xorl %r11d, %ecx + rorl $14, %edx + andl %r9d, %ecx + xorl %r9d, %edx + xorl %r11d, %ecx + rorl $5, %edx + addl %ecx, %r12d + xorl %r9d, %edx + xorl %r13d, %ebx + rorl $6, %edx + movl %r13d, %ecx + addl %edx, %r12d + rorl $9, %ecx + andl %ebx, %eax + xorl %r13d, %ecx + xorl %r14d, %eax + rorl $11, %ecx + addl %r12d, %r8d + xorl %r13d, %ecx + addl %eax, %r12d + rorl $2, %ecx + movl %r8d, %edx + addl %ecx, %r12d + # rnd_all_4: 30-33 + addl 480(%rsp), %r11d + movl %r9d, %ecx + movl %r13d, %eax + xorl %r10d, %ecx + rorl $14, %edx + andl %r8d, %ecx + xorl %r8d, %edx + xorl %r10d, %ecx + rorl $5, %edx + addl %ecx, %r11d + xorl %r8d, %edx + xorl %r12d, %eax + rorl $6, %edx + movl %r12d, %ecx + addl %edx, %r11d + rorl $9, %ecx + andl %eax, %ebx + xorl %r12d, %ecx + xorl %r13d, %ebx + rorl $11, %ecx + addl %r11d, %r15d + xorl %r12d, %ecx + addl %ebx, %r11d + rorl $2, %ecx + movl %r15d, %edx + addl %ecx, %r11d + addl 484(%rsp), %r10d + movl %r8d, %ecx + movl %r12d, %ebx + xorl %r9d, %ecx + rorl $14, %edx + andl %r15d, %ecx + xorl %r15d, %edx + xorl %r9d, %ecx + rorl $5, %edx + addl %ecx, %r10d + xorl %r15d, %edx + xorl %r11d, %ebx + rorl $6, %edx + movl %r11d, %ecx + addl %edx, %r10d + rorl $9, %ecx + andl %ebx, %eax + xorl %r11d, %ecx + xorl %r12d, %eax + rorl $11, %ecx + addl %r10d, %r14d + xorl %r11d, %ecx + addl %eax, %r10d + rorl $2, %ecx + movl %r14d, %edx + addl %ecx, %r10d + addl 488(%rsp), %r9d + movl %r15d, %ecx + movl %r11d, %eax + xorl %r8d, %ecx + rorl $14, %edx + andl %r14d, %ecx + xorl %r14d, %edx + xorl %r8d, %ecx + rorl $5, %edx + addl %ecx, %r9d + xorl %r14d, %edx + xorl %r10d, %eax + rorl $6, %edx + movl %r10d, %ecx + addl %edx, %r9d + rorl $9, %ecx + andl %eax, %ebx + xorl %r10d, %ecx + xorl %r11d, %ebx + rorl $11, %ecx + addl %r9d, %r13d + xorl %r10d, %ecx + addl %ebx, %r9d + rorl $2, %ecx + movl %r13d, %edx + addl %ecx, %r9d + addl 492(%rsp), %r8d + movl %r14d, %ecx + movl %r10d, %ebx + xorl %r15d, %ecx + rorl $14, %edx + andl %r13d, %ecx + xorl %r13d, %edx + xorl %r15d, %ecx + rorl $5, %edx + addl %ecx, %r8d + xorl %r13d, %edx + xorl %r9d, %ebx + rorl $6, %edx + movl %r9d, %ecx + addl %edx, %r8d + rorl $9, %ecx + andl %ebx, %eax + xorl %r9d, %ecx + xorl %r10d, %eax + rorl $11, %ecx + addl %r8d, %r12d + xorl %r9d, %ecx + addl %eax, %r8d + rorl $2, %ecx + movl %r12d, %edx + addl %ecx, %r8d + addl %r8d, (%rdi) + addl %r9d, 4(%rdi) + addl %r10d, 8(%rdi) + addl %r11d, 12(%rdi) + addl %r12d, 16(%rdi) + addl %r13d, 20(%rdi) + addl %r14d, 24(%rdi) + addl %r15d, 28(%rdi) + xorq %rax, %rax + vzeroupper + addq $0x200, %rsp + popq %r15 + popq %r14 + popq %r13 + popq %r12 + popq %rbx + repz retq +.size Transform_Sha256_AVX2,.-Transform_Sha256_AVX2 +.globl Transform_Sha256_AVX2_Len +.type Transform_Sha256_AVX2_Len,@function +.align 4 +Transform_Sha256_AVX2_Len: + pushq %rbx + pushq %r12 + pushq %r13 + pushq %r14 + pushq %r15 + pushq %rbp + movq %rsi, %rbp + movq %rdx, %rsi + subq $0x200, %rsp + testb $0x40, %sil + je L_sha256_len_avx2_block + vmovdqu (%rbp), %ymm0 + vmovdqu 32(%rbp), %ymm1 + vmovups %ymm0, 32(%rdi) + vmovups %ymm1, 64(%rdi) + call Transform_Sha256_AVX2@plt + addq $0x40, %rbp + subl $0x40, %esi + jz L_sha256_len_avx2_done +L_sha256_len_avx2_block: + vmovdqa L_avx2_sha256_flip_mask(%rip), %ymm13 + vmovdqa L_avx2_sha256_shuf_00BA(%rip), %ymm11 + vmovdqa L_avx2_sha256_shuf_DC00(%rip), %ymm12 + movl (%rdi), %r8d + movl 4(%rdi), %r9d + movl 8(%rdi), %r10d + movl 12(%rdi), %r11d + movl 16(%rdi), %r12d + movl 20(%rdi), %r13d + movl 24(%rdi), %r14d + movl 28(%rdi), %r15d + # Start of loop processing two blocks +L_sha256_len_avx2_start: + # X0, X1, X2, X3 = W[0..15] + vmovdqu (%rbp), %xmm0 + vmovdqu 16(%rbp), %xmm1 + vmovdqu 64(%rbp), %xmm4 + vmovdqu 80(%rbp), %xmm5 + vinserti128 $0x01, %xmm4, %ymm0, %ymm0 + vinserti128 $0x01, %xmm5, %ymm1, %ymm1 + vpshufb %ymm13, %ymm0, %ymm0 + vpshufb %ymm13, %ymm1, %ymm1 + vmovdqu 32(%rbp), %xmm2 + vmovdqu 48(%rbp), %xmm3 + vmovdqu 96(%rbp), %xmm6 + vmovdqu 112(%rbp), %xmm7 + vinserti128 $0x01, %xmm6, %ymm2, %ymm2 + vinserti128 $0x01, %xmm7, %ymm3, %ymm3 + vpshufb %ymm13, %ymm2, %ymm2 + vpshufb %ymm13, %ymm3, %ymm3 + movl %r9d, %ebx + movl %r12d, %edx + xorl %r10d, %ebx + # set_w_k_xfer_4: 0 + vpaddd 0+L_avx2_sha256_k(%rip), %ymm0, %ymm4 + vpaddd 32+L_avx2_sha256_k(%rip), %ymm1, %ymm5 + vmovdqu %ymm4, (%rsp) + vmovdqu %ymm5, 32(%rsp) + vpaddd 64+L_avx2_sha256_k(%rip), %ymm2, %ymm4 + vpaddd 96+L_avx2_sha256_k(%rip), %ymm3, %ymm5 + vmovdqu %ymm4, 64(%rsp) + vmovdqu %ymm5, 96(%rsp) + # msg_sched: 0-3 + # rnd_0: 0 - 0 + rorl $14, %edx + vpalignr $4, %ymm0, %ymm1, %ymm5 + vpalignr $4, %ymm2, %ymm3, %ymm4 + # rnd_0: 1 - 2 + movl %r9d, %eax + movl %r13d, %ecx + addl (%rsp), %r15d + xorl %r14d, %ecx + xorl %r12d, %edx + andl %r12d, %ecx + vpsrld $7, %ymm5, %ymm6 + vpslld $25, %ymm5, %ymm7 + # rnd_0: 3 - 4 + rorl $5, %edx + xorl %r14d, %ecx + xorl %r12d, %edx + addl %ecx, %r15d + rorl $6, %edx + xorl %r8d, %eax + addl %edx, %r15d + movl %r8d, %ecx + vpsrld $18, %ymm5, %ymm8 + vpslld $14, %ymm5, %ymm9 + # rnd_0: 5 - 6 + andl %eax, %ebx + rorl $9, %ecx + xorl %r8d, %ecx + xorl %r9d, %ebx + rorl $11, %ecx + addl %r15d, %r11d + xorl %r8d, %ecx + addl %ebx, %r15d + vpor %ymm6, %ymm7, %ymm6 + vpor %ymm8, %ymm9, %ymm8 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r11d, %edx + addl %ecx, %r15d + # rnd_1: 0 - 1 + rorl $14, %edx + movl %r8d, %ebx + movl %r12d, %ecx + addl 4(%rsp), %r14d + xorl %r13d, %ecx + vpsrld $3, %ymm5, %ymm9 + vpxor %ymm6, %ymm8, %ymm6 + # rnd_1: 2 - 3 + xorl %r11d, %edx + andl %r11d, %ecx + rorl $5, %edx + xorl %r13d, %ecx + xorl %r11d, %edx + addl %ecx, %r14d + vpxor %ymm6, %ymm9, %ymm5 + vpshufd $0xfa, %ymm3, %ymm6 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r15d, %ebx + addl %edx, %r14d + movl %r15d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r15d, %ecx + xorl %r8d, %eax + vpsrld $10, %ymm6, %ymm8 + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 6 - 7 + rorl $11, %ecx + addl %r14d, %r10d + xorl %r15d, %ecx + addl %eax, %r14d + rorl $2, %ecx + movl %r10d, %edx + addl %ecx, %r14d + # rnd_0: 0 - 0 + rorl $14, %edx + vpsrlq $0x11, %ymm6, %ymm6 + vpaddd %ymm0, %ymm4, %ymm4 + # rnd_0: 1 - 3 + movl %r15d, %eax + movl %r11d, %ecx + addl 8(%rsp), %r13d + xorl %r12d, %ecx + xorl %r10d, %edx + andl %r10d, %ecx + rorl $5, %edx + xorl %r12d, %ecx + xorl %r10d, %edx + addl %ecx, %r13d + vpxor %ymm6, %ymm7, %ymm6 + vpaddd %ymm5, %ymm4, %ymm4 + # rnd_0: 4 - 4 + rorl $6, %edx + xorl %r14d, %eax + addl %edx, %r13d + movl %r14d, %ecx + vpxor %ymm6, %ymm8, %ymm8 + # rnd_0: 5 - 5 + andl %eax, %ebx + rorl $9, %ecx + xorl %r14d, %ecx + xorl %r15d, %ebx + vpshufb %ymm11, %ymm8, %ymm8 + # rnd_0: 6 - 6 + rorl $11, %ecx + addl %r13d, %r9d + xorl %r14d, %ecx + addl %ebx, %r13d + vpaddd %ymm8, %ymm4, %ymm4 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r9d, %edx + addl %ecx, %r13d + # rnd_1: 0 - 0 + rorl $14, %edx + vpshufd $0x50, %ymm4, %ymm6 + # rnd_1: 1 - 1 + movl %r14d, %ebx + movl %r10d, %ecx + addl 12(%rsp), %r12d + xorl %r11d, %ecx + vpsrlq $0x11, %ymm6, %ymm8 + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 2 - 3 + xorl %r9d, %edx + andl %r9d, %ecx + rorl $5, %edx + xorl %r11d, %ecx + xorl %r9d, %edx + addl %ecx, %r12d + vpsrld $10, %ymm6, %ymm9 + vpxor %ymm8, %ymm7, %ymm8 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r13d, %ebx + addl %edx, %r12d + movl %r13d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r13d, %ecx + xorl %r14d, %eax + vpxor %ymm9, %ymm8, %ymm9 + # rnd_1: 6 - 6 + rorl $11, %ecx + addl %r12d, %r8d + xorl %r13d, %ecx + addl %eax, %r12d + vpshufb %ymm12, %ymm9, %ymm9 + # rnd_1: 7 - 7 + rorl $2, %ecx + movl %r8d, %edx + addl %ecx, %r12d + vpaddd %ymm4, %ymm9, %ymm0 + # msg_sched done: 0-3 + # msg_sched: 8-11 + # rnd_0: 0 - 0 + rorl $14, %edx + vpalignr $4, %ymm1, %ymm2, %ymm5 + vpalignr $4, %ymm3, %ymm0, %ymm4 + # rnd_0: 1 - 2 + movl %r13d, %eax + movl %r9d, %ecx + addl 32(%rsp), %r11d + xorl %r10d, %ecx + xorl %r8d, %edx + andl %r8d, %ecx + vpsrld $7, %ymm5, %ymm6 + vpslld $25, %ymm5, %ymm7 + # rnd_0: 3 - 4 + rorl $5, %edx + xorl %r10d, %ecx + xorl %r8d, %edx + addl %ecx, %r11d + rorl $6, %edx + xorl %r12d, %eax + addl %edx, %r11d + movl %r12d, %ecx + vpsrld $18, %ymm5, %ymm8 + vpslld $14, %ymm5, %ymm9 + # rnd_0: 5 - 6 + andl %eax, %ebx + rorl $9, %ecx + xorl %r12d, %ecx + xorl %r13d, %ebx + rorl $11, %ecx + addl %r11d, %r15d + xorl %r12d, %ecx + addl %ebx, %r11d + vpor %ymm6, %ymm7, %ymm6 + vpor %ymm8, %ymm9, %ymm8 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r15d, %edx + addl %ecx, %r11d + # rnd_1: 0 - 1 + rorl $14, %edx + movl %r12d, %ebx + movl %r8d, %ecx + addl 36(%rsp), %r10d + xorl %r9d, %ecx + vpsrld $3, %ymm5, %ymm9 + vpxor %ymm6, %ymm8, %ymm6 + # rnd_1: 2 - 3 + xorl %r15d, %edx + andl %r15d, %ecx + rorl $5, %edx + xorl %r9d, %ecx + xorl %r15d, %edx + addl %ecx, %r10d + vpxor %ymm6, %ymm9, %ymm5 + vpshufd $0xfa, %ymm0, %ymm6 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r11d, %ebx + addl %edx, %r10d + movl %r11d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r11d, %ecx + xorl %r12d, %eax + vpsrld $10, %ymm6, %ymm8 + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 6 - 7 + rorl $11, %ecx + addl %r10d, %r14d + xorl %r11d, %ecx + addl %eax, %r10d + rorl $2, %ecx + movl %r14d, %edx + addl %ecx, %r10d + # rnd_0: 0 - 0 + rorl $14, %edx + vpsrlq $0x11, %ymm6, %ymm6 + vpaddd %ymm1, %ymm4, %ymm4 + # rnd_0: 1 - 3 + movl %r11d, %eax + movl %r15d, %ecx + addl 40(%rsp), %r9d + xorl %r8d, %ecx + xorl %r14d, %edx + andl %r14d, %ecx + rorl $5, %edx + xorl %r8d, %ecx + xorl %r14d, %edx + addl %ecx, %r9d + vpxor %ymm6, %ymm7, %ymm6 + vpaddd %ymm5, %ymm4, %ymm4 + # rnd_0: 4 - 4 + rorl $6, %edx + xorl %r10d, %eax + addl %edx, %r9d + movl %r10d, %ecx + vpxor %ymm6, %ymm8, %ymm8 + # rnd_0: 5 - 5 + andl %eax, %ebx + rorl $9, %ecx + xorl %r10d, %ecx + xorl %r11d, %ebx + vpshufb %ymm11, %ymm8, %ymm8 + # rnd_0: 6 - 6 + rorl $11, %ecx + addl %r9d, %r13d + xorl %r10d, %ecx + addl %ebx, %r9d + vpaddd %ymm8, %ymm4, %ymm4 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r13d, %edx + addl %ecx, %r9d + # rnd_1: 0 - 0 + rorl $14, %edx + vpshufd $0x50, %ymm4, %ymm6 + # rnd_1: 1 - 1 + movl %r10d, %ebx + movl %r14d, %ecx + addl 44(%rsp), %r8d + xorl %r15d, %ecx + vpsrlq $0x11, %ymm6, %ymm8 + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 2 - 3 + xorl %r13d, %edx + andl %r13d, %ecx + rorl $5, %edx + xorl %r15d, %ecx + xorl %r13d, %edx + addl %ecx, %r8d + vpsrld $10, %ymm6, %ymm9 + vpxor %ymm8, %ymm7, %ymm8 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r9d, %ebx + addl %edx, %r8d + movl %r9d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r9d, %ecx + xorl %r10d, %eax + vpxor %ymm9, %ymm8, %ymm9 + # rnd_1: 6 - 6 + rorl $11, %ecx + addl %r8d, %r12d + xorl %r9d, %ecx + addl %eax, %r8d + vpshufb %ymm12, %ymm9, %ymm9 + # rnd_1: 7 - 7 + rorl $2, %ecx + movl %r12d, %edx + addl %ecx, %r8d + vpaddd %ymm4, %ymm9, %ymm1 + # msg_sched done: 8-11 + # msg_sched: 16-19 + # rnd_0: 0 - 0 + rorl $14, %edx + vpalignr $4, %ymm2, %ymm3, %ymm5 + vpalignr $4, %ymm0, %ymm1, %ymm4 + # rnd_0: 1 - 2 + movl %r9d, %eax + movl %r13d, %ecx + addl 64(%rsp), %r15d + xorl %r14d, %ecx + xorl %r12d, %edx + andl %r12d, %ecx + vpsrld $7, %ymm5, %ymm6 + vpslld $25, %ymm5, %ymm7 + # rnd_0: 3 - 4 + rorl $5, %edx + xorl %r14d, %ecx + xorl %r12d, %edx + addl %ecx, %r15d + rorl $6, %edx + xorl %r8d, %eax + addl %edx, %r15d + movl %r8d, %ecx + vpsrld $18, %ymm5, %ymm8 + vpslld $14, %ymm5, %ymm9 + # rnd_0: 5 - 6 + andl %eax, %ebx + rorl $9, %ecx + xorl %r8d, %ecx + xorl %r9d, %ebx + rorl $11, %ecx + addl %r15d, %r11d + xorl %r8d, %ecx + addl %ebx, %r15d + vpor %ymm6, %ymm7, %ymm6 + vpor %ymm8, %ymm9, %ymm8 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r11d, %edx + addl %ecx, %r15d + # rnd_1: 0 - 1 + rorl $14, %edx + movl %r8d, %ebx + movl %r12d, %ecx + addl 68(%rsp), %r14d + xorl %r13d, %ecx + vpsrld $3, %ymm5, %ymm9 + vpxor %ymm6, %ymm8, %ymm6 + # rnd_1: 2 - 3 + xorl %r11d, %edx + andl %r11d, %ecx + rorl $5, %edx + xorl %r13d, %ecx + xorl %r11d, %edx + addl %ecx, %r14d + vpxor %ymm6, %ymm9, %ymm5 + vpshufd $0xfa, %ymm1, %ymm6 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r15d, %ebx + addl %edx, %r14d + movl %r15d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r15d, %ecx + xorl %r8d, %eax + vpsrld $10, %ymm6, %ymm8 + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 6 - 7 + rorl $11, %ecx + addl %r14d, %r10d + xorl %r15d, %ecx + addl %eax, %r14d + rorl $2, %ecx + movl %r10d, %edx + addl %ecx, %r14d + # rnd_0: 0 - 0 + rorl $14, %edx + vpsrlq $0x11, %ymm6, %ymm6 + vpaddd %ymm2, %ymm4, %ymm4 + # rnd_0: 1 - 3 + movl %r15d, %eax + movl %r11d, %ecx + addl 72(%rsp), %r13d + xorl %r12d, %ecx + xorl %r10d, %edx + andl %r10d, %ecx + rorl $5, %edx + xorl %r12d, %ecx + xorl %r10d, %edx + addl %ecx, %r13d + vpxor %ymm6, %ymm7, %ymm6 + vpaddd %ymm5, %ymm4, %ymm4 + # rnd_0: 4 - 4 + rorl $6, %edx + xorl %r14d, %eax + addl %edx, %r13d + movl %r14d, %ecx + vpxor %ymm6, %ymm8, %ymm8 + # rnd_0: 5 - 5 + andl %eax, %ebx + rorl $9, %ecx + xorl %r14d, %ecx + xorl %r15d, %ebx + vpshufb %ymm11, %ymm8, %ymm8 + # rnd_0: 6 - 6 + rorl $11, %ecx + addl %r13d, %r9d + xorl %r14d, %ecx + addl %ebx, %r13d + vpaddd %ymm8, %ymm4, %ymm4 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r9d, %edx + addl %ecx, %r13d + # rnd_1: 0 - 0 + rorl $14, %edx + vpshufd $0x50, %ymm4, %ymm6 + # rnd_1: 1 - 1 + movl %r14d, %ebx + movl %r10d, %ecx + addl 76(%rsp), %r12d + xorl %r11d, %ecx + vpsrlq $0x11, %ymm6, %ymm8 + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 2 - 3 + xorl %r9d, %edx + andl %r9d, %ecx + rorl $5, %edx + xorl %r11d, %ecx + xorl %r9d, %edx + addl %ecx, %r12d + vpsrld $10, %ymm6, %ymm9 + vpxor %ymm8, %ymm7, %ymm8 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r13d, %ebx + addl %edx, %r12d + movl %r13d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r13d, %ecx + xorl %r14d, %eax + vpxor %ymm9, %ymm8, %ymm9 + # rnd_1: 6 - 6 + rorl $11, %ecx + addl %r12d, %r8d + xorl %r13d, %ecx + addl %eax, %r12d + vpshufb %ymm12, %ymm9, %ymm9 + # rnd_1: 7 - 7 + rorl $2, %ecx + movl %r8d, %edx + addl %ecx, %r12d + vpaddd %ymm4, %ymm9, %ymm2 + # msg_sched done: 16-19 + # msg_sched: 24-27 + # rnd_0: 0 - 0 + rorl $14, %edx + vpalignr $4, %ymm3, %ymm0, %ymm5 + vpalignr $4, %ymm1, %ymm2, %ymm4 + # rnd_0: 1 - 2 + movl %r13d, %eax + movl %r9d, %ecx + addl 96(%rsp), %r11d + xorl %r10d, %ecx + xorl %r8d, %edx + andl %r8d, %ecx + vpsrld $7, %ymm5, %ymm6 + vpslld $25, %ymm5, %ymm7 + # rnd_0: 3 - 4 + rorl $5, %edx + xorl %r10d, %ecx + xorl %r8d, %edx + addl %ecx, %r11d + rorl $6, %edx + xorl %r12d, %eax + addl %edx, %r11d + movl %r12d, %ecx + vpsrld $18, %ymm5, %ymm8 + vpslld $14, %ymm5, %ymm9 + # rnd_0: 5 - 6 + andl %eax, %ebx + rorl $9, %ecx + xorl %r12d, %ecx + xorl %r13d, %ebx + rorl $11, %ecx + addl %r11d, %r15d + xorl %r12d, %ecx + addl %ebx, %r11d + vpor %ymm6, %ymm7, %ymm6 + vpor %ymm8, %ymm9, %ymm8 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r15d, %edx + addl %ecx, %r11d + # rnd_1: 0 - 1 + rorl $14, %edx + movl %r12d, %ebx + movl %r8d, %ecx + addl 100(%rsp), %r10d + xorl %r9d, %ecx + vpsrld $3, %ymm5, %ymm9 + vpxor %ymm6, %ymm8, %ymm6 + # rnd_1: 2 - 3 + xorl %r15d, %edx + andl %r15d, %ecx + rorl $5, %edx + xorl %r9d, %ecx + xorl %r15d, %edx + addl %ecx, %r10d + vpxor %ymm6, %ymm9, %ymm5 + vpshufd $0xfa, %ymm2, %ymm6 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r11d, %ebx + addl %edx, %r10d + movl %r11d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r11d, %ecx + xorl %r12d, %eax + vpsrld $10, %ymm6, %ymm8 + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 6 - 7 + rorl $11, %ecx + addl %r10d, %r14d + xorl %r11d, %ecx + addl %eax, %r10d + rorl $2, %ecx + movl %r14d, %edx + addl %ecx, %r10d + # rnd_0: 0 - 0 + rorl $14, %edx + vpsrlq $0x11, %ymm6, %ymm6 + vpaddd %ymm3, %ymm4, %ymm4 + # rnd_0: 1 - 3 + movl %r11d, %eax + movl %r15d, %ecx + addl 104(%rsp), %r9d + xorl %r8d, %ecx + xorl %r14d, %edx + andl %r14d, %ecx + rorl $5, %edx + xorl %r8d, %ecx + xorl %r14d, %edx + addl %ecx, %r9d + vpxor %ymm6, %ymm7, %ymm6 + vpaddd %ymm5, %ymm4, %ymm4 + # rnd_0: 4 - 4 + rorl $6, %edx + xorl %r10d, %eax + addl %edx, %r9d + movl %r10d, %ecx + vpxor %ymm6, %ymm8, %ymm8 + # rnd_0: 5 - 5 + andl %eax, %ebx + rorl $9, %ecx + xorl %r10d, %ecx + xorl %r11d, %ebx + vpshufb %ymm11, %ymm8, %ymm8 + # rnd_0: 6 - 6 + rorl $11, %ecx + addl %r9d, %r13d + xorl %r10d, %ecx + addl %ebx, %r9d + vpaddd %ymm8, %ymm4, %ymm4 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r13d, %edx + addl %ecx, %r9d + # rnd_1: 0 - 0 + rorl $14, %edx + vpshufd $0x50, %ymm4, %ymm6 + # rnd_1: 1 - 1 + movl %r10d, %ebx + movl %r14d, %ecx + addl 108(%rsp), %r8d + xorl %r15d, %ecx + vpsrlq $0x11, %ymm6, %ymm8 + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 2 - 3 + xorl %r13d, %edx + andl %r13d, %ecx + rorl $5, %edx + xorl %r15d, %ecx + xorl %r13d, %edx + addl %ecx, %r8d + vpsrld $10, %ymm6, %ymm9 + vpxor %ymm8, %ymm7, %ymm8 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r9d, %ebx + addl %edx, %r8d + movl %r9d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r9d, %ecx + xorl %r10d, %eax + vpxor %ymm9, %ymm8, %ymm9 + # rnd_1: 6 - 6 + rorl $11, %ecx + addl %r8d, %r12d + xorl %r9d, %ecx + addl %eax, %r8d + vpshufb %ymm12, %ymm9, %ymm9 + # rnd_1: 7 - 7 + rorl $2, %ecx + movl %r12d, %edx + addl %ecx, %r8d + vpaddd %ymm4, %ymm9, %ymm3 + # msg_sched done: 24-27 + # set_w_k_xfer_4: 4 + vpaddd 128+L_avx2_sha256_k(%rip), %ymm0, %ymm4 + vpaddd 160+L_avx2_sha256_k(%rip), %ymm1, %ymm5 + vmovdqu %ymm4, 128(%rsp) + vmovdqu %ymm5, 160(%rsp) + vpaddd 192+L_avx2_sha256_k(%rip), %ymm2, %ymm4 + vpaddd 224+L_avx2_sha256_k(%rip), %ymm3, %ymm5 + vmovdqu %ymm4, 192(%rsp) + vmovdqu %ymm5, 224(%rsp) + # msg_sched: 32-35 + # rnd_0: 0 - 0 + rorl $14, %edx + vpalignr $4, %ymm0, %ymm1, %ymm5 + vpalignr $4, %ymm2, %ymm3, %ymm4 + # rnd_0: 1 - 2 + movl %r9d, %eax + movl %r13d, %ecx + addl 128(%rsp), %r15d + xorl %r14d, %ecx + xorl %r12d, %edx + andl %r12d, %ecx + vpsrld $7, %ymm5, %ymm6 + vpslld $25, %ymm5, %ymm7 + # rnd_0: 3 - 4 + rorl $5, %edx + xorl %r14d, %ecx + xorl %r12d, %edx + addl %ecx, %r15d + rorl $6, %edx + xorl %r8d, %eax + addl %edx, %r15d + movl %r8d, %ecx + vpsrld $18, %ymm5, %ymm8 + vpslld $14, %ymm5, %ymm9 + # rnd_0: 5 - 6 + andl %eax, %ebx + rorl $9, %ecx + xorl %r8d, %ecx + xorl %r9d, %ebx + rorl $11, %ecx + addl %r15d, %r11d + xorl %r8d, %ecx + addl %ebx, %r15d + vpor %ymm6, %ymm7, %ymm6 + vpor %ymm8, %ymm9, %ymm8 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r11d, %edx + addl %ecx, %r15d + # rnd_1: 0 - 1 + rorl $14, %edx + movl %r8d, %ebx + movl %r12d, %ecx + addl 132(%rsp), %r14d + xorl %r13d, %ecx + vpsrld $3, %ymm5, %ymm9 + vpxor %ymm6, %ymm8, %ymm6 + # rnd_1: 2 - 3 + xorl %r11d, %edx + andl %r11d, %ecx + rorl $5, %edx + xorl %r13d, %ecx + xorl %r11d, %edx + addl %ecx, %r14d + vpxor %ymm6, %ymm9, %ymm5 + vpshufd $0xfa, %ymm3, %ymm6 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r15d, %ebx + addl %edx, %r14d + movl %r15d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r15d, %ecx + xorl %r8d, %eax + vpsrld $10, %ymm6, %ymm8 + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 6 - 7 + rorl $11, %ecx + addl %r14d, %r10d + xorl %r15d, %ecx + addl %eax, %r14d + rorl $2, %ecx + movl %r10d, %edx + addl %ecx, %r14d + # rnd_0: 0 - 0 + rorl $14, %edx + vpsrlq $0x11, %ymm6, %ymm6 + vpaddd %ymm0, %ymm4, %ymm4 + # rnd_0: 1 - 3 + movl %r15d, %eax + movl %r11d, %ecx + addl 136(%rsp), %r13d + xorl %r12d, %ecx + xorl %r10d, %edx + andl %r10d, %ecx + rorl $5, %edx + xorl %r12d, %ecx + xorl %r10d, %edx + addl %ecx, %r13d + vpxor %ymm6, %ymm7, %ymm6 + vpaddd %ymm5, %ymm4, %ymm4 + # rnd_0: 4 - 4 + rorl $6, %edx + xorl %r14d, %eax + addl %edx, %r13d + movl %r14d, %ecx + vpxor %ymm6, %ymm8, %ymm8 + # rnd_0: 5 - 5 + andl %eax, %ebx + rorl $9, %ecx + xorl %r14d, %ecx + xorl %r15d, %ebx + vpshufb %ymm11, %ymm8, %ymm8 + # rnd_0: 6 - 6 + rorl $11, %ecx + addl %r13d, %r9d + xorl %r14d, %ecx + addl %ebx, %r13d + vpaddd %ymm8, %ymm4, %ymm4 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r9d, %edx + addl %ecx, %r13d + # rnd_1: 0 - 0 + rorl $14, %edx + vpshufd $0x50, %ymm4, %ymm6 + # rnd_1: 1 - 1 + movl %r14d, %ebx + movl %r10d, %ecx + addl 140(%rsp), %r12d + xorl %r11d, %ecx + vpsrlq $0x11, %ymm6, %ymm8 + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 2 - 3 + xorl %r9d, %edx + andl %r9d, %ecx + rorl $5, %edx + xorl %r11d, %ecx + xorl %r9d, %edx + addl %ecx, %r12d + vpsrld $10, %ymm6, %ymm9 + vpxor %ymm8, %ymm7, %ymm8 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r13d, %ebx + addl %edx, %r12d + movl %r13d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r13d, %ecx + xorl %r14d, %eax + vpxor %ymm9, %ymm8, %ymm9 + # rnd_1: 6 - 6 + rorl $11, %ecx + addl %r12d, %r8d + xorl %r13d, %ecx + addl %eax, %r12d + vpshufb %ymm12, %ymm9, %ymm9 + # rnd_1: 7 - 7 + rorl $2, %ecx + movl %r8d, %edx + addl %ecx, %r12d + vpaddd %ymm4, %ymm9, %ymm0 + # msg_sched done: 32-35 + # msg_sched: 40-43 + # rnd_0: 0 - 0 + rorl $14, %edx + vpalignr $4, %ymm1, %ymm2, %ymm5 + vpalignr $4, %ymm3, %ymm0, %ymm4 + # rnd_0: 1 - 2 + movl %r13d, %eax + movl %r9d, %ecx + addl 160(%rsp), %r11d + xorl %r10d, %ecx + xorl %r8d, %edx + andl %r8d, %ecx + vpsrld $7, %ymm5, %ymm6 + vpslld $25, %ymm5, %ymm7 + # rnd_0: 3 - 4 + rorl $5, %edx + xorl %r10d, %ecx + xorl %r8d, %edx + addl %ecx, %r11d + rorl $6, %edx + xorl %r12d, %eax + addl %edx, %r11d + movl %r12d, %ecx + vpsrld $18, %ymm5, %ymm8 + vpslld $14, %ymm5, %ymm9 + # rnd_0: 5 - 6 + andl %eax, %ebx + rorl $9, %ecx + xorl %r12d, %ecx + xorl %r13d, %ebx + rorl $11, %ecx + addl %r11d, %r15d + xorl %r12d, %ecx + addl %ebx, %r11d + vpor %ymm6, %ymm7, %ymm6 + vpor %ymm8, %ymm9, %ymm8 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r15d, %edx + addl %ecx, %r11d + # rnd_1: 0 - 1 + rorl $14, %edx + movl %r12d, %ebx + movl %r8d, %ecx + addl 164(%rsp), %r10d + xorl %r9d, %ecx + vpsrld $3, %ymm5, %ymm9 + vpxor %ymm6, %ymm8, %ymm6 + # rnd_1: 2 - 3 + xorl %r15d, %edx + andl %r15d, %ecx + rorl $5, %edx + xorl %r9d, %ecx + xorl %r15d, %edx + addl %ecx, %r10d + vpxor %ymm6, %ymm9, %ymm5 + vpshufd $0xfa, %ymm0, %ymm6 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r11d, %ebx + addl %edx, %r10d + movl %r11d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r11d, %ecx + xorl %r12d, %eax + vpsrld $10, %ymm6, %ymm8 + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 6 - 7 + rorl $11, %ecx + addl %r10d, %r14d + xorl %r11d, %ecx + addl %eax, %r10d + rorl $2, %ecx + movl %r14d, %edx + addl %ecx, %r10d + # rnd_0: 0 - 0 + rorl $14, %edx + vpsrlq $0x11, %ymm6, %ymm6 + vpaddd %ymm1, %ymm4, %ymm4 + # rnd_0: 1 - 3 + movl %r11d, %eax + movl %r15d, %ecx + addl 168(%rsp), %r9d + xorl %r8d, %ecx + xorl %r14d, %edx + andl %r14d, %ecx + rorl $5, %edx + xorl %r8d, %ecx + xorl %r14d, %edx + addl %ecx, %r9d + vpxor %ymm6, %ymm7, %ymm6 + vpaddd %ymm5, %ymm4, %ymm4 + # rnd_0: 4 - 4 + rorl $6, %edx + xorl %r10d, %eax + addl %edx, %r9d + movl %r10d, %ecx + vpxor %ymm6, %ymm8, %ymm8 + # rnd_0: 5 - 5 + andl %eax, %ebx + rorl $9, %ecx + xorl %r10d, %ecx + xorl %r11d, %ebx + vpshufb %ymm11, %ymm8, %ymm8 + # rnd_0: 6 - 6 + rorl $11, %ecx + addl %r9d, %r13d + xorl %r10d, %ecx + addl %ebx, %r9d + vpaddd %ymm8, %ymm4, %ymm4 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r13d, %edx + addl %ecx, %r9d + # rnd_1: 0 - 0 + rorl $14, %edx + vpshufd $0x50, %ymm4, %ymm6 + # rnd_1: 1 - 1 + movl %r10d, %ebx + movl %r14d, %ecx + addl 172(%rsp), %r8d + xorl %r15d, %ecx + vpsrlq $0x11, %ymm6, %ymm8 + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 2 - 3 + xorl %r13d, %edx + andl %r13d, %ecx + rorl $5, %edx + xorl %r15d, %ecx + xorl %r13d, %edx + addl %ecx, %r8d + vpsrld $10, %ymm6, %ymm9 + vpxor %ymm8, %ymm7, %ymm8 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r9d, %ebx + addl %edx, %r8d + movl %r9d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r9d, %ecx + xorl %r10d, %eax + vpxor %ymm9, %ymm8, %ymm9 + # rnd_1: 6 - 6 + rorl $11, %ecx + addl %r8d, %r12d + xorl %r9d, %ecx + addl %eax, %r8d + vpshufb %ymm12, %ymm9, %ymm9 + # rnd_1: 7 - 7 + rorl $2, %ecx + movl %r12d, %edx + addl %ecx, %r8d + vpaddd %ymm4, %ymm9, %ymm1 + # msg_sched done: 40-43 + # msg_sched: 48-51 + # rnd_0: 0 - 0 + rorl $14, %edx + vpalignr $4, %ymm2, %ymm3, %ymm5 + vpalignr $4, %ymm0, %ymm1, %ymm4 + # rnd_0: 1 - 2 + movl %r9d, %eax + movl %r13d, %ecx + addl 192(%rsp), %r15d + xorl %r14d, %ecx + xorl %r12d, %edx + andl %r12d, %ecx + vpsrld $7, %ymm5, %ymm6 + vpslld $25, %ymm5, %ymm7 + # rnd_0: 3 - 4 + rorl $5, %edx + xorl %r14d, %ecx + xorl %r12d, %edx + addl %ecx, %r15d + rorl $6, %edx + xorl %r8d, %eax + addl %edx, %r15d + movl %r8d, %ecx + vpsrld $18, %ymm5, %ymm8 + vpslld $14, %ymm5, %ymm9 + # rnd_0: 5 - 6 + andl %eax, %ebx + rorl $9, %ecx + xorl %r8d, %ecx + xorl %r9d, %ebx + rorl $11, %ecx + addl %r15d, %r11d + xorl %r8d, %ecx + addl %ebx, %r15d + vpor %ymm6, %ymm7, %ymm6 + vpor %ymm8, %ymm9, %ymm8 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r11d, %edx + addl %ecx, %r15d + # rnd_1: 0 - 1 + rorl $14, %edx + movl %r8d, %ebx + movl %r12d, %ecx + addl 196(%rsp), %r14d + xorl %r13d, %ecx + vpsrld $3, %ymm5, %ymm9 + vpxor %ymm6, %ymm8, %ymm6 + # rnd_1: 2 - 3 + xorl %r11d, %edx + andl %r11d, %ecx + rorl $5, %edx + xorl %r13d, %ecx + xorl %r11d, %edx + addl %ecx, %r14d + vpxor %ymm6, %ymm9, %ymm5 + vpshufd $0xfa, %ymm1, %ymm6 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r15d, %ebx + addl %edx, %r14d + movl %r15d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r15d, %ecx + xorl %r8d, %eax + vpsrld $10, %ymm6, %ymm8 + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 6 - 7 + rorl $11, %ecx + addl %r14d, %r10d + xorl %r15d, %ecx + addl %eax, %r14d + rorl $2, %ecx + movl %r10d, %edx + addl %ecx, %r14d + # rnd_0: 0 - 0 + rorl $14, %edx + vpsrlq $0x11, %ymm6, %ymm6 + vpaddd %ymm2, %ymm4, %ymm4 + # rnd_0: 1 - 3 + movl %r15d, %eax + movl %r11d, %ecx + addl 200(%rsp), %r13d + xorl %r12d, %ecx + xorl %r10d, %edx + andl %r10d, %ecx + rorl $5, %edx + xorl %r12d, %ecx + xorl %r10d, %edx + addl %ecx, %r13d + vpxor %ymm6, %ymm7, %ymm6 + vpaddd %ymm5, %ymm4, %ymm4 + # rnd_0: 4 - 4 + rorl $6, %edx + xorl %r14d, %eax + addl %edx, %r13d + movl %r14d, %ecx + vpxor %ymm6, %ymm8, %ymm8 + # rnd_0: 5 - 5 + andl %eax, %ebx + rorl $9, %ecx + xorl %r14d, %ecx + xorl %r15d, %ebx + vpshufb %ymm11, %ymm8, %ymm8 + # rnd_0: 6 - 6 + rorl $11, %ecx + addl %r13d, %r9d + xorl %r14d, %ecx + addl %ebx, %r13d + vpaddd %ymm8, %ymm4, %ymm4 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r9d, %edx + addl %ecx, %r13d + # rnd_1: 0 - 0 + rorl $14, %edx + vpshufd $0x50, %ymm4, %ymm6 + # rnd_1: 1 - 1 + movl %r14d, %ebx + movl %r10d, %ecx + addl 204(%rsp), %r12d + xorl %r11d, %ecx + vpsrlq $0x11, %ymm6, %ymm8 + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 2 - 3 + xorl %r9d, %edx + andl %r9d, %ecx + rorl $5, %edx + xorl %r11d, %ecx + xorl %r9d, %edx + addl %ecx, %r12d + vpsrld $10, %ymm6, %ymm9 + vpxor %ymm8, %ymm7, %ymm8 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r13d, %ebx + addl %edx, %r12d + movl %r13d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r13d, %ecx + xorl %r14d, %eax + vpxor %ymm9, %ymm8, %ymm9 + # rnd_1: 6 - 6 + rorl $11, %ecx + addl %r12d, %r8d + xorl %r13d, %ecx + addl %eax, %r12d + vpshufb %ymm12, %ymm9, %ymm9 + # rnd_1: 7 - 7 + rorl $2, %ecx + movl %r8d, %edx + addl %ecx, %r12d + vpaddd %ymm4, %ymm9, %ymm2 + # msg_sched done: 48-51 + # msg_sched: 56-59 + # rnd_0: 0 - 0 + rorl $14, %edx + vpalignr $4, %ymm3, %ymm0, %ymm5 + vpalignr $4, %ymm1, %ymm2, %ymm4 + # rnd_0: 1 - 2 + movl %r13d, %eax + movl %r9d, %ecx + addl 224(%rsp), %r11d + xorl %r10d, %ecx + xorl %r8d, %edx + andl %r8d, %ecx + vpsrld $7, %ymm5, %ymm6 + vpslld $25, %ymm5, %ymm7 + # rnd_0: 3 - 4 + rorl $5, %edx + xorl %r10d, %ecx + xorl %r8d, %edx + addl %ecx, %r11d + rorl $6, %edx + xorl %r12d, %eax + addl %edx, %r11d + movl %r12d, %ecx + vpsrld $18, %ymm5, %ymm8 + vpslld $14, %ymm5, %ymm9 + # rnd_0: 5 - 6 + andl %eax, %ebx + rorl $9, %ecx + xorl %r12d, %ecx + xorl %r13d, %ebx + rorl $11, %ecx + addl %r11d, %r15d + xorl %r12d, %ecx + addl %ebx, %r11d + vpor %ymm6, %ymm7, %ymm6 + vpor %ymm8, %ymm9, %ymm8 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r15d, %edx + addl %ecx, %r11d + # rnd_1: 0 - 1 + rorl $14, %edx + movl %r12d, %ebx + movl %r8d, %ecx + addl 228(%rsp), %r10d + xorl %r9d, %ecx + vpsrld $3, %ymm5, %ymm9 + vpxor %ymm6, %ymm8, %ymm6 + # rnd_1: 2 - 3 + xorl %r15d, %edx + andl %r15d, %ecx + rorl $5, %edx + xorl %r9d, %ecx + xorl %r15d, %edx + addl %ecx, %r10d + vpxor %ymm6, %ymm9, %ymm5 + vpshufd $0xfa, %ymm2, %ymm6 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r11d, %ebx + addl %edx, %r10d + movl %r11d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r11d, %ecx + xorl %r12d, %eax + vpsrld $10, %ymm6, %ymm8 + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 6 - 7 + rorl $11, %ecx + addl %r10d, %r14d + xorl %r11d, %ecx + addl %eax, %r10d + rorl $2, %ecx + movl %r14d, %edx + addl %ecx, %r10d + # rnd_0: 0 - 0 + rorl $14, %edx + vpsrlq $0x11, %ymm6, %ymm6 + vpaddd %ymm3, %ymm4, %ymm4 + # rnd_0: 1 - 3 + movl %r11d, %eax + movl %r15d, %ecx + addl 232(%rsp), %r9d + xorl %r8d, %ecx + xorl %r14d, %edx + andl %r14d, %ecx + rorl $5, %edx + xorl %r8d, %ecx + xorl %r14d, %edx + addl %ecx, %r9d + vpxor %ymm6, %ymm7, %ymm6 + vpaddd %ymm5, %ymm4, %ymm4 + # rnd_0: 4 - 4 + rorl $6, %edx + xorl %r10d, %eax + addl %edx, %r9d + movl %r10d, %ecx + vpxor %ymm6, %ymm8, %ymm8 + # rnd_0: 5 - 5 + andl %eax, %ebx + rorl $9, %ecx + xorl %r10d, %ecx + xorl %r11d, %ebx + vpshufb %ymm11, %ymm8, %ymm8 + # rnd_0: 6 - 6 + rorl $11, %ecx + addl %r9d, %r13d + xorl %r10d, %ecx + addl %ebx, %r9d + vpaddd %ymm8, %ymm4, %ymm4 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r13d, %edx + addl %ecx, %r9d + # rnd_1: 0 - 0 + rorl $14, %edx + vpshufd $0x50, %ymm4, %ymm6 + # rnd_1: 1 - 1 + movl %r10d, %ebx + movl %r14d, %ecx + addl 236(%rsp), %r8d + xorl %r15d, %ecx + vpsrlq $0x11, %ymm6, %ymm8 + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 2 - 3 + xorl %r13d, %edx + andl %r13d, %ecx + rorl $5, %edx + xorl %r15d, %ecx + xorl %r13d, %edx + addl %ecx, %r8d + vpsrld $10, %ymm6, %ymm9 + vpxor %ymm8, %ymm7, %ymm8 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r9d, %ebx + addl %edx, %r8d + movl %r9d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r9d, %ecx + xorl %r10d, %eax + vpxor %ymm9, %ymm8, %ymm9 + # rnd_1: 6 - 6 + rorl $11, %ecx + addl %r8d, %r12d + xorl %r9d, %ecx + addl %eax, %r8d + vpshufb %ymm12, %ymm9, %ymm9 + # rnd_1: 7 - 7 + rorl $2, %ecx + movl %r12d, %edx + addl %ecx, %r8d + vpaddd %ymm4, %ymm9, %ymm3 + # msg_sched done: 56-59 + # set_w_k_xfer_4: 8 + vpaddd 256+L_avx2_sha256_k(%rip), %ymm0, %ymm4 + vpaddd 288+L_avx2_sha256_k(%rip), %ymm1, %ymm5 + vmovdqu %ymm4, 256(%rsp) + vmovdqu %ymm5, 288(%rsp) + vpaddd 320+L_avx2_sha256_k(%rip), %ymm2, %ymm4 + vpaddd 352+L_avx2_sha256_k(%rip), %ymm3, %ymm5 + vmovdqu %ymm4, 320(%rsp) + vmovdqu %ymm5, 352(%rsp) + # msg_sched: 64-67 + # rnd_0: 0 - 0 + rorl $14, %edx + vpalignr $4, %ymm0, %ymm1, %ymm5 + vpalignr $4, %ymm2, %ymm3, %ymm4 + # rnd_0: 1 - 2 + movl %r9d, %eax + movl %r13d, %ecx + addl 256(%rsp), %r15d + xorl %r14d, %ecx + xorl %r12d, %edx + andl %r12d, %ecx + vpsrld $7, %ymm5, %ymm6 + vpslld $25, %ymm5, %ymm7 + # rnd_0: 3 - 4 + rorl $5, %edx + xorl %r14d, %ecx + xorl %r12d, %edx + addl %ecx, %r15d + rorl $6, %edx + xorl %r8d, %eax + addl %edx, %r15d + movl %r8d, %ecx + vpsrld $18, %ymm5, %ymm8 + vpslld $14, %ymm5, %ymm9 + # rnd_0: 5 - 6 + andl %eax, %ebx + rorl $9, %ecx + xorl %r8d, %ecx + xorl %r9d, %ebx + rorl $11, %ecx + addl %r15d, %r11d + xorl %r8d, %ecx + addl %ebx, %r15d + vpor %ymm6, %ymm7, %ymm6 + vpor %ymm8, %ymm9, %ymm8 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r11d, %edx + addl %ecx, %r15d + # rnd_1: 0 - 1 + rorl $14, %edx + movl %r8d, %ebx + movl %r12d, %ecx + addl 260(%rsp), %r14d + xorl %r13d, %ecx + vpsrld $3, %ymm5, %ymm9 + vpxor %ymm6, %ymm8, %ymm6 + # rnd_1: 2 - 3 + xorl %r11d, %edx + andl %r11d, %ecx + rorl $5, %edx + xorl %r13d, %ecx + xorl %r11d, %edx + addl %ecx, %r14d + vpxor %ymm6, %ymm9, %ymm5 + vpshufd $0xfa, %ymm3, %ymm6 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r15d, %ebx + addl %edx, %r14d + movl %r15d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r15d, %ecx + xorl %r8d, %eax + vpsrld $10, %ymm6, %ymm8 + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 6 - 7 + rorl $11, %ecx + addl %r14d, %r10d + xorl %r15d, %ecx + addl %eax, %r14d + rorl $2, %ecx + movl %r10d, %edx + addl %ecx, %r14d + # rnd_0: 0 - 0 + rorl $14, %edx + vpsrlq $0x11, %ymm6, %ymm6 + vpaddd %ymm0, %ymm4, %ymm4 + # rnd_0: 1 - 3 + movl %r15d, %eax + movl %r11d, %ecx + addl 264(%rsp), %r13d + xorl %r12d, %ecx + xorl %r10d, %edx + andl %r10d, %ecx + rorl $5, %edx + xorl %r12d, %ecx + xorl %r10d, %edx + addl %ecx, %r13d + vpxor %ymm6, %ymm7, %ymm6 + vpaddd %ymm5, %ymm4, %ymm4 + # rnd_0: 4 - 4 + rorl $6, %edx + xorl %r14d, %eax + addl %edx, %r13d + movl %r14d, %ecx + vpxor %ymm6, %ymm8, %ymm8 + # rnd_0: 5 - 5 + andl %eax, %ebx + rorl $9, %ecx + xorl %r14d, %ecx + xorl %r15d, %ebx + vpshufb %ymm11, %ymm8, %ymm8 + # rnd_0: 6 - 6 + rorl $11, %ecx + addl %r13d, %r9d + xorl %r14d, %ecx + addl %ebx, %r13d + vpaddd %ymm8, %ymm4, %ymm4 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r9d, %edx + addl %ecx, %r13d + # rnd_1: 0 - 0 + rorl $14, %edx + vpshufd $0x50, %ymm4, %ymm6 + # rnd_1: 1 - 1 + movl %r14d, %ebx + movl %r10d, %ecx + addl 268(%rsp), %r12d + xorl %r11d, %ecx + vpsrlq $0x11, %ymm6, %ymm8 + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 2 - 3 + xorl %r9d, %edx + andl %r9d, %ecx + rorl $5, %edx + xorl %r11d, %ecx + xorl %r9d, %edx + addl %ecx, %r12d + vpsrld $10, %ymm6, %ymm9 + vpxor %ymm8, %ymm7, %ymm8 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r13d, %ebx + addl %edx, %r12d + movl %r13d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r13d, %ecx + xorl %r14d, %eax + vpxor %ymm9, %ymm8, %ymm9 + # rnd_1: 6 - 6 + rorl $11, %ecx + addl %r12d, %r8d + xorl %r13d, %ecx + addl %eax, %r12d + vpshufb %ymm12, %ymm9, %ymm9 + # rnd_1: 7 - 7 + rorl $2, %ecx + movl %r8d, %edx + addl %ecx, %r12d + vpaddd %ymm4, %ymm9, %ymm0 + # msg_sched done: 64-67 + # msg_sched: 72-75 + # rnd_0: 0 - 0 + rorl $14, %edx + vpalignr $4, %ymm1, %ymm2, %ymm5 + vpalignr $4, %ymm3, %ymm0, %ymm4 + # rnd_0: 1 - 2 + movl %r13d, %eax + movl %r9d, %ecx + addl 288(%rsp), %r11d + xorl %r10d, %ecx + xorl %r8d, %edx + andl %r8d, %ecx + vpsrld $7, %ymm5, %ymm6 + vpslld $25, %ymm5, %ymm7 + # rnd_0: 3 - 4 + rorl $5, %edx + xorl %r10d, %ecx + xorl %r8d, %edx + addl %ecx, %r11d + rorl $6, %edx + xorl %r12d, %eax + addl %edx, %r11d + movl %r12d, %ecx + vpsrld $18, %ymm5, %ymm8 + vpslld $14, %ymm5, %ymm9 + # rnd_0: 5 - 6 + andl %eax, %ebx + rorl $9, %ecx + xorl %r12d, %ecx + xorl %r13d, %ebx + rorl $11, %ecx + addl %r11d, %r15d + xorl %r12d, %ecx + addl %ebx, %r11d + vpor %ymm6, %ymm7, %ymm6 + vpor %ymm8, %ymm9, %ymm8 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r15d, %edx + addl %ecx, %r11d + # rnd_1: 0 - 1 + rorl $14, %edx + movl %r12d, %ebx + movl %r8d, %ecx + addl 292(%rsp), %r10d + xorl %r9d, %ecx + vpsrld $3, %ymm5, %ymm9 + vpxor %ymm6, %ymm8, %ymm6 + # rnd_1: 2 - 3 + xorl %r15d, %edx + andl %r15d, %ecx + rorl $5, %edx + xorl %r9d, %ecx + xorl %r15d, %edx + addl %ecx, %r10d + vpxor %ymm6, %ymm9, %ymm5 + vpshufd $0xfa, %ymm0, %ymm6 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r11d, %ebx + addl %edx, %r10d + movl %r11d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r11d, %ecx + xorl %r12d, %eax + vpsrld $10, %ymm6, %ymm8 + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 6 - 7 + rorl $11, %ecx + addl %r10d, %r14d + xorl %r11d, %ecx + addl %eax, %r10d + rorl $2, %ecx + movl %r14d, %edx + addl %ecx, %r10d + # rnd_0: 0 - 0 + rorl $14, %edx + vpsrlq $0x11, %ymm6, %ymm6 + vpaddd %ymm1, %ymm4, %ymm4 + # rnd_0: 1 - 3 + movl %r11d, %eax + movl %r15d, %ecx + addl 296(%rsp), %r9d + xorl %r8d, %ecx + xorl %r14d, %edx + andl %r14d, %ecx + rorl $5, %edx + xorl %r8d, %ecx + xorl %r14d, %edx + addl %ecx, %r9d + vpxor %ymm6, %ymm7, %ymm6 + vpaddd %ymm5, %ymm4, %ymm4 + # rnd_0: 4 - 4 + rorl $6, %edx + xorl %r10d, %eax + addl %edx, %r9d + movl %r10d, %ecx + vpxor %ymm6, %ymm8, %ymm8 + # rnd_0: 5 - 5 + andl %eax, %ebx + rorl $9, %ecx + xorl %r10d, %ecx + xorl %r11d, %ebx + vpshufb %ymm11, %ymm8, %ymm8 + # rnd_0: 6 - 6 + rorl $11, %ecx + addl %r9d, %r13d + xorl %r10d, %ecx + addl %ebx, %r9d + vpaddd %ymm8, %ymm4, %ymm4 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r13d, %edx + addl %ecx, %r9d + # rnd_1: 0 - 0 + rorl $14, %edx + vpshufd $0x50, %ymm4, %ymm6 + # rnd_1: 1 - 1 + movl %r10d, %ebx + movl %r14d, %ecx + addl 300(%rsp), %r8d + xorl %r15d, %ecx + vpsrlq $0x11, %ymm6, %ymm8 + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 2 - 3 + xorl %r13d, %edx + andl %r13d, %ecx + rorl $5, %edx + xorl %r15d, %ecx + xorl %r13d, %edx + addl %ecx, %r8d + vpsrld $10, %ymm6, %ymm9 + vpxor %ymm8, %ymm7, %ymm8 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r9d, %ebx + addl %edx, %r8d + movl %r9d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r9d, %ecx + xorl %r10d, %eax + vpxor %ymm9, %ymm8, %ymm9 + # rnd_1: 6 - 6 + rorl $11, %ecx + addl %r8d, %r12d + xorl %r9d, %ecx + addl %eax, %r8d + vpshufb %ymm12, %ymm9, %ymm9 + # rnd_1: 7 - 7 + rorl $2, %ecx + movl %r12d, %edx + addl %ecx, %r8d + vpaddd %ymm4, %ymm9, %ymm1 + # msg_sched done: 72-75 + # msg_sched: 80-83 + # rnd_0: 0 - 0 + rorl $14, %edx + vpalignr $4, %ymm2, %ymm3, %ymm5 + vpalignr $4, %ymm0, %ymm1, %ymm4 + # rnd_0: 1 - 2 + movl %r9d, %eax + movl %r13d, %ecx + addl 320(%rsp), %r15d + xorl %r14d, %ecx + xorl %r12d, %edx + andl %r12d, %ecx + vpsrld $7, %ymm5, %ymm6 + vpslld $25, %ymm5, %ymm7 + # rnd_0: 3 - 4 + rorl $5, %edx + xorl %r14d, %ecx + xorl %r12d, %edx + addl %ecx, %r15d + rorl $6, %edx + xorl %r8d, %eax + addl %edx, %r15d + movl %r8d, %ecx + vpsrld $18, %ymm5, %ymm8 + vpslld $14, %ymm5, %ymm9 + # rnd_0: 5 - 6 + andl %eax, %ebx + rorl $9, %ecx + xorl %r8d, %ecx + xorl %r9d, %ebx + rorl $11, %ecx + addl %r15d, %r11d + xorl %r8d, %ecx + addl %ebx, %r15d + vpor %ymm6, %ymm7, %ymm6 + vpor %ymm8, %ymm9, %ymm8 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r11d, %edx + addl %ecx, %r15d + # rnd_1: 0 - 1 + rorl $14, %edx + movl %r8d, %ebx + movl %r12d, %ecx + addl 324(%rsp), %r14d + xorl %r13d, %ecx + vpsrld $3, %ymm5, %ymm9 + vpxor %ymm6, %ymm8, %ymm6 + # rnd_1: 2 - 3 + xorl %r11d, %edx + andl %r11d, %ecx + rorl $5, %edx + xorl %r13d, %ecx + xorl %r11d, %edx + addl %ecx, %r14d + vpxor %ymm6, %ymm9, %ymm5 + vpshufd $0xfa, %ymm1, %ymm6 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r15d, %ebx + addl %edx, %r14d + movl %r15d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r15d, %ecx + xorl %r8d, %eax + vpsrld $10, %ymm6, %ymm8 + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 6 - 7 + rorl $11, %ecx + addl %r14d, %r10d + xorl %r15d, %ecx + addl %eax, %r14d + rorl $2, %ecx + movl %r10d, %edx + addl %ecx, %r14d + # rnd_0: 0 - 0 + rorl $14, %edx + vpsrlq $0x11, %ymm6, %ymm6 + vpaddd %ymm2, %ymm4, %ymm4 + # rnd_0: 1 - 3 + movl %r15d, %eax + movl %r11d, %ecx + addl 328(%rsp), %r13d + xorl %r12d, %ecx + xorl %r10d, %edx + andl %r10d, %ecx + rorl $5, %edx + xorl %r12d, %ecx + xorl %r10d, %edx + addl %ecx, %r13d + vpxor %ymm6, %ymm7, %ymm6 + vpaddd %ymm5, %ymm4, %ymm4 + # rnd_0: 4 - 4 + rorl $6, %edx + xorl %r14d, %eax + addl %edx, %r13d + movl %r14d, %ecx + vpxor %ymm6, %ymm8, %ymm8 + # rnd_0: 5 - 5 + andl %eax, %ebx + rorl $9, %ecx + xorl %r14d, %ecx + xorl %r15d, %ebx + vpshufb %ymm11, %ymm8, %ymm8 + # rnd_0: 6 - 6 + rorl $11, %ecx + addl %r13d, %r9d + xorl %r14d, %ecx + addl %ebx, %r13d + vpaddd %ymm8, %ymm4, %ymm4 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r9d, %edx + addl %ecx, %r13d + # rnd_1: 0 - 0 + rorl $14, %edx + vpshufd $0x50, %ymm4, %ymm6 + # rnd_1: 1 - 1 + movl %r14d, %ebx + movl %r10d, %ecx + addl 332(%rsp), %r12d + xorl %r11d, %ecx + vpsrlq $0x11, %ymm6, %ymm8 + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 2 - 3 + xorl %r9d, %edx + andl %r9d, %ecx + rorl $5, %edx + xorl %r11d, %ecx + xorl %r9d, %edx + addl %ecx, %r12d + vpsrld $10, %ymm6, %ymm9 + vpxor %ymm8, %ymm7, %ymm8 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r13d, %ebx + addl %edx, %r12d + movl %r13d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r13d, %ecx + xorl %r14d, %eax + vpxor %ymm9, %ymm8, %ymm9 + # rnd_1: 6 - 6 + rorl $11, %ecx + addl %r12d, %r8d + xorl %r13d, %ecx + addl %eax, %r12d + vpshufb %ymm12, %ymm9, %ymm9 + # rnd_1: 7 - 7 + rorl $2, %ecx + movl %r8d, %edx + addl %ecx, %r12d + vpaddd %ymm4, %ymm9, %ymm2 + # msg_sched done: 80-83 + # msg_sched: 88-91 + # rnd_0: 0 - 0 + rorl $14, %edx + vpalignr $4, %ymm3, %ymm0, %ymm5 + vpalignr $4, %ymm1, %ymm2, %ymm4 + # rnd_0: 1 - 2 + movl %r13d, %eax + movl %r9d, %ecx + addl 352(%rsp), %r11d + xorl %r10d, %ecx + xorl %r8d, %edx + andl %r8d, %ecx + vpsrld $7, %ymm5, %ymm6 + vpslld $25, %ymm5, %ymm7 + # rnd_0: 3 - 4 + rorl $5, %edx + xorl %r10d, %ecx + xorl %r8d, %edx + addl %ecx, %r11d + rorl $6, %edx + xorl %r12d, %eax + addl %edx, %r11d + movl %r12d, %ecx + vpsrld $18, %ymm5, %ymm8 + vpslld $14, %ymm5, %ymm9 + # rnd_0: 5 - 6 + andl %eax, %ebx + rorl $9, %ecx + xorl %r12d, %ecx + xorl %r13d, %ebx + rorl $11, %ecx + addl %r11d, %r15d + xorl %r12d, %ecx + addl %ebx, %r11d + vpor %ymm6, %ymm7, %ymm6 + vpor %ymm8, %ymm9, %ymm8 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r15d, %edx + addl %ecx, %r11d + # rnd_1: 0 - 1 + rorl $14, %edx + movl %r12d, %ebx + movl %r8d, %ecx + addl 356(%rsp), %r10d + xorl %r9d, %ecx + vpsrld $3, %ymm5, %ymm9 + vpxor %ymm6, %ymm8, %ymm6 + # rnd_1: 2 - 3 + xorl %r15d, %edx + andl %r15d, %ecx + rorl $5, %edx + xorl %r9d, %ecx + xorl %r15d, %edx + addl %ecx, %r10d + vpxor %ymm6, %ymm9, %ymm5 + vpshufd $0xfa, %ymm2, %ymm6 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r11d, %ebx + addl %edx, %r10d + movl %r11d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r11d, %ecx + xorl %r12d, %eax + vpsrld $10, %ymm6, %ymm8 + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 6 - 7 + rorl $11, %ecx + addl %r10d, %r14d + xorl %r11d, %ecx + addl %eax, %r10d + rorl $2, %ecx + movl %r14d, %edx + addl %ecx, %r10d + # rnd_0: 0 - 0 + rorl $14, %edx + vpsrlq $0x11, %ymm6, %ymm6 + vpaddd %ymm3, %ymm4, %ymm4 + # rnd_0: 1 - 3 + movl %r11d, %eax + movl %r15d, %ecx + addl 360(%rsp), %r9d + xorl %r8d, %ecx + xorl %r14d, %edx + andl %r14d, %ecx + rorl $5, %edx + xorl %r8d, %ecx + xorl %r14d, %edx + addl %ecx, %r9d + vpxor %ymm6, %ymm7, %ymm6 + vpaddd %ymm5, %ymm4, %ymm4 + # rnd_0: 4 - 4 + rorl $6, %edx + xorl %r10d, %eax + addl %edx, %r9d + movl %r10d, %ecx + vpxor %ymm6, %ymm8, %ymm8 + # rnd_0: 5 - 5 + andl %eax, %ebx + rorl $9, %ecx + xorl %r10d, %ecx + xorl %r11d, %ebx + vpshufb %ymm11, %ymm8, %ymm8 + # rnd_0: 6 - 6 + rorl $11, %ecx + addl %r9d, %r13d + xorl %r10d, %ecx + addl %ebx, %r9d + vpaddd %ymm8, %ymm4, %ymm4 + # rnd_0: 7 - 7 + rorl $2, %ecx + movl %r13d, %edx + addl %ecx, %r9d + # rnd_1: 0 - 0 + rorl $14, %edx + vpshufd $0x50, %ymm4, %ymm6 + # rnd_1: 1 - 1 + movl %r10d, %ebx + movl %r14d, %ecx + addl 364(%rsp), %r8d + xorl %r15d, %ecx + vpsrlq $0x11, %ymm6, %ymm8 + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 2 - 3 + xorl %r13d, %edx + andl %r13d, %ecx + rorl $5, %edx + xorl %r15d, %ecx + xorl %r13d, %edx + addl %ecx, %r8d + vpsrld $10, %ymm6, %ymm9 + vpxor %ymm8, %ymm7, %ymm8 + # rnd_1: 4 - 5 + rorl $6, %edx + xorl %r9d, %ebx + addl %edx, %r8d + movl %r9d, %ecx + andl %ebx, %eax + rorl $9, %ecx + xorl %r9d, %ecx + xorl %r10d, %eax + vpxor %ymm9, %ymm8, %ymm9 + # rnd_1: 6 - 6 + rorl $11, %ecx + addl %r8d, %r12d + xorl %r9d, %ecx + addl %eax, %r8d + vpshufb %ymm12, %ymm9, %ymm9 + # rnd_1: 7 - 7 + rorl $2, %ecx + movl %r12d, %edx + addl %ecx, %r8d + vpaddd %ymm4, %ymm9, %ymm3 + # msg_sched done: 88-91 + # set_w_k_xfer_4: 12 + vpaddd 384+L_avx2_sha256_k(%rip), %ymm0, %ymm4 + vpaddd 416+L_avx2_sha256_k(%rip), %ymm1, %ymm5 + vmovdqu %ymm4, 384(%rsp) + vmovdqu %ymm5, 416(%rsp) + vpaddd 448+L_avx2_sha256_k(%rip), %ymm2, %ymm4 + vpaddd 480+L_avx2_sha256_k(%rip), %ymm3, %ymm5 + vmovdqu %ymm4, 448(%rsp) + vmovdqu %ymm5, 480(%rsp) + # rnd_all_4: 24-27 + addl 384(%rsp), %r15d + movl %r13d, %ecx + movl %r9d, %eax + xorl %r14d, %ecx + rorl $14, %edx + andl %r12d, %ecx + xorl %r12d, %edx + xorl %r14d, %ecx + rorl $5, %edx + addl %ecx, %r15d + xorl %r12d, %edx + xorl %r8d, %eax + rorl $6, %edx + movl %r8d, %ecx + addl %edx, %r15d + rorl $9, %ecx + andl %eax, %ebx + xorl %r8d, %ecx + xorl %r9d, %ebx + rorl $11, %ecx + addl %r15d, %r11d + xorl %r8d, %ecx + addl %ebx, %r15d + rorl $2, %ecx + movl %r11d, %edx + addl %ecx, %r15d + addl 388(%rsp), %r14d + movl %r12d, %ecx + movl %r8d, %ebx + xorl %r13d, %ecx + rorl $14, %edx + andl %r11d, %ecx + xorl %r11d, %edx + xorl %r13d, %ecx + rorl $5, %edx + addl %ecx, %r14d + xorl %r11d, %edx + xorl %r15d, %ebx + rorl $6, %edx + movl %r15d, %ecx + addl %edx, %r14d + rorl $9, %ecx + andl %ebx, %eax + xorl %r15d, %ecx + xorl %r8d, %eax + rorl $11, %ecx + addl %r14d, %r10d + xorl %r15d, %ecx + addl %eax, %r14d + rorl $2, %ecx + movl %r10d, %edx + addl %ecx, %r14d + addl 392(%rsp), %r13d + movl %r11d, %ecx + movl %r15d, %eax + xorl %r12d, %ecx + rorl $14, %edx + andl %r10d, %ecx + xorl %r10d, %edx + xorl %r12d, %ecx + rorl $5, %edx + addl %ecx, %r13d + xorl %r10d, %edx + xorl %r14d, %eax + rorl $6, %edx + movl %r14d, %ecx + addl %edx, %r13d + rorl $9, %ecx + andl %eax, %ebx + xorl %r14d, %ecx + xorl %r15d, %ebx + rorl $11, %ecx + addl %r13d, %r9d + xorl %r14d, %ecx + addl %ebx, %r13d + rorl $2, %ecx + movl %r9d, %edx + addl %ecx, %r13d + addl 396(%rsp), %r12d + movl %r10d, %ecx + movl %r14d, %ebx + xorl %r11d, %ecx + rorl $14, %edx + andl %r9d, %ecx + xorl %r9d, %edx + xorl %r11d, %ecx + rorl $5, %edx + addl %ecx, %r12d + xorl %r9d, %edx + xorl %r13d, %ebx + rorl $6, %edx + movl %r13d, %ecx + addl %edx, %r12d + rorl $9, %ecx + andl %ebx, %eax + xorl %r13d, %ecx + xorl %r14d, %eax + rorl $11, %ecx + addl %r12d, %r8d + xorl %r13d, %ecx + addl %eax, %r12d + rorl $2, %ecx + movl %r8d, %edx + addl %ecx, %r12d + # rnd_all_4: 26-29 + addl 416(%rsp), %r11d + movl %r9d, %ecx + movl %r13d, %eax + xorl %r10d, %ecx + rorl $14, %edx + andl %r8d, %ecx + xorl %r8d, %edx + xorl %r10d, %ecx + rorl $5, %edx + addl %ecx, %r11d + xorl %r8d, %edx + xorl %r12d, %eax + rorl $6, %edx + movl %r12d, %ecx + addl %edx, %r11d + rorl $9, %ecx + andl %eax, %ebx + xorl %r12d, %ecx + xorl %r13d, %ebx + rorl $11, %ecx + addl %r11d, %r15d + xorl %r12d, %ecx + addl %ebx, %r11d + rorl $2, %ecx + movl %r15d, %edx + addl %ecx, %r11d + addl 420(%rsp), %r10d + movl %r8d, %ecx + movl %r12d, %ebx + xorl %r9d, %ecx + rorl $14, %edx + andl %r15d, %ecx + xorl %r15d, %edx + xorl %r9d, %ecx + rorl $5, %edx + addl %ecx, %r10d + xorl %r15d, %edx + xorl %r11d, %ebx + rorl $6, %edx + movl %r11d, %ecx + addl %edx, %r10d + rorl $9, %ecx + andl %ebx, %eax + xorl %r11d, %ecx + xorl %r12d, %eax + rorl $11, %ecx + addl %r10d, %r14d + xorl %r11d, %ecx + addl %eax, %r10d + rorl $2, %ecx + movl %r14d, %edx + addl %ecx, %r10d + addl 424(%rsp), %r9d + movl %r15d, %ecx + movl %r11d, %eax + xorl %r8d, %ecx + rorl $14, %edx + andl %r14d, %ecx + xorl %r14d, %edx + xorl %r8d, %ecx + rorl $5, %edx + addl %ecx, %r9d + xorl %r14d, %edx + xorl %r10d, %eax + rorl $6, %edx + movl %r10d, %ecx + addl %edx, %r9d + rorl $9, %ecx + andl %eax, %ebx + xorl %r10d, %ecx + xorl %r11d, %ebx + rorl $11, %ecx + addl %r9d, %r13d + xorl %r10d, %ecx + addl %ebx, %r9d + rorl $2, %ecx + movl %r13d, %edx + addl %ecx, %r9d + addl 428(%rsp), %r8d + movl %r14d, %ecx + movl %r10d, %ebx + xorl %r15d, %ecx + rorl $14, %edx + andl %r13d, %ecx + xorl %r13d, %edx + xorl %r15d, %ecx + rorl $5, %edx + addl %ecx, %r8d + xorl %r13d, %edx + xorl %r9d, %ebx + rorl $6, %edx + movl %r9d, %ecx + addl %edx, %r8d + rorl $9, %ecx + andl %ebx, %eax + xorl %r9d, %ecx + xorl %r10d, %eax + rorl $11, %ecx + addl %r8d, %r12d + xorl %r9d, %ecx + addl %eax, %r8d + rorl $2, %ecx + movl %r12d, %edx + addl %ecx, %r8d + # rnd_all_4: 28-31 + addl 448(%rsp), %r15d + movl %r13d, %ecx + movl %r9d, %eax + xorl %r14d, %ecx + rorl $14, %edx + andl %r12d, %ecx + xorl %r12d, %edx + xorl %r14d, %ecx + rorl $5, %edx + addl %ecx, %r15d + xorl %r12d, %edx + xorl %r8d, %eax + rorl $6, %edx + movl %r8d, %ecx + addl %edx, %r15d + rorl $9, %ecx + andl %eax, %ebx + xorl %r8d, %ecx + xorl %r9d, %ebx + rorl $11, %ecx + addl %r15d, %r11d + xorl %r8d, %ecx + addl %ebx, %r15d + rorl $2, %ecx + movl %r11d, %edx + addl %ecx, %r15d + addl 452(%rsp), %r14d + movl %r12d, %ecx + movl %r8d, %ebx + xorl %r13d, %ecx + rorl $14, %edx + andl %r11d, %ecx + xorl %r11d, %edx + xorl %r13d, %ecx + rorl $5, %edx + addl %ecx, %r14d + xorl %r11d, %edx + xorl %r15d, %ebx + rorl $6, %edx + movl %r15d, %ecx + addl %edx, %r14d + rorl $9, %ecx + andl %ebx, %eax + xorl %r15d, %ecx + xorl %r8d, %eax + rorl $11, %ecx + addl %r14d, %r10d + xorl %r15d, %ecx + addl %eax, %r14d + rorl $2, %ecx + movl %r10d, %edx + addl %ecx, %r14d + addl 456(%rsp), %r13d + movl %r11d, %ecx + movl %r15d, %eax + xorl %r12d, %ecx + rorl $14, %edx + andl %r10d, %ecx + xorl %r10d, %edx + xorl %r12d, %ecx + rorl $5, %edx + addl %ecx, %r13d + xorl %r10d, %edx + xorl %r14d, %eax + rorl $6, %edx + movl %r14d, %ecx + addl %edx, %r13d + rorl $9, %ecx + andl %eax, %ebx + xorl %r14d, %ecx + xorl %r15d, %ebx + rorl $11, %ecx + addl %r13d, %r9d + xorl %r14d, %ecx + addl %ebx, %r13d + rorl $2, %ecx + movl %r9d, %edx + addl %ecx, %r13d + addl 460(%rsp), %r12d + movl %r10d, %ecx + movl %r14d, %ebx + xorl %r11d, %ecx + rorl $14, %edx + andl %r9d, %ecx + xorl %r9d, %edx + xorl %r11d, %ecx + rorl $5, %edx + addl %ecx, %r12d + xorl %r9d, %edx + xorl %r13d, %ebx + rorl $6, %edx + movl %r13d, %ecx + addl %edx, %r12d + rorl $9, %ecx + andl %ebx, %eax + xorl %r13d, %ecx + xorl %r14d, %eax + rorl $11, %ecx + addl %r12d, %r8d + xorl %r13d, %ecx + addl %eax, %r12d + rorl $2, %ecx + movl %r8d, %edx + addl %ecx, %r12d + # rnd_all_4: 30-33 + addl 480(%rsp), %r11d + movl %r9d, %ecx + movl %r13d, %eax + xorl %r10d, %ecx + rorl $14, %edx + andl %r8d, %ecx + xorl %r8d, %edx + xorl %r10d, %ecx + rorl $5, %edx + addl %ecx, %r11d + xorl %r8d, %edx + xorl %r12d, %eax + rorl $6, %edx + movl %r12d, %ecx + addl %edx, %r11d + rorl $9, %ecx + andl %eax, %ebx + xorl %r12d, %ecx + xorl %r13d, %ebx + rorl $11, %ecx + addl %r11d, %r15d + xorl %r12d, %ecx + addl %ebx, %r11d + rorl $2, %ecx + movl %r15d, %edx + addl %ecx, %r11d + addl 484(%rsp), %r10d + movl %r8d, %ecx + movl %r12d, %ebx + xorl %r9d, %ecx + rorl $14, %edx + andl %r15d, %ecx + xorl %r15d, %edx + xorl %r9d, %ecx + rorl $5, %edx + addl %ecx, %r10d + xorl %r15d, %edx + xorl %r11d, %ebx + rorl $6, %edx + movl %r11d, %ecx + addl %edx, %r10d + rorl $9, %ecx + andl %ebx, %eax + xorl %r11d, %ecx + xorl %r12d, %eax + rorl $11, %ecx + addl %r10d, %r14d + xorl %r11d, %ecx + addl %eax, %r10d + rorl $2, %ecx + movl %r14d, %edx + addl %ecx, %r10d + addl 488(%rsp), %r9d + movl %r15d, %ecx + movl %r11d, %eax + xorl %r8d, %ecx + rorl $14, %edx + andl %r14d, %ecx + xorl %r14d, %edx + xorl %r8d, %ecx + rorl $5, %edx + addl %ecx, %r9d + xorl %r14d, %edx + xorl %r10d, %eax + rorl $6, %edx + movl %r10d, %ecx + addl %edx, %r9d + rorl $9, %ecx + andl %eax, %ebx + xorl %r10d, %ecx + xorl %r11d, %ebx + rorl $11, %ecx + addl %r9d, %r13d + xorl %r10d, %ecx + addl %ebx, %r9d + rorl $2, %ecx + movl %r13d, %edx + addl %ecx, %r9d + addl 492(%rsp), %r8d + movl %r14d, %ecx + movl %r10d, %ebx + xorl %r15d, %ecx + rorl $14, %edx + andl %r13d, %ecx + xorl %r13d, %edx + xorl %r15d, %ecx + rorl $5, %edx + addl %ecx, %r8d + xorl %r13d, %edx + xorl %r9d, %ebx + rorl $6, %edx + movl %r9d, %ecx + addl %edx, %r8d + rorl $9, %ecx + andl %ebx, %eax + xorl %r9d, %ecx + xorl %r10d, %eax + rorl $11, %ecx + addl %r8d, %r12d + xorl %r9d, %ecx + addl %eax, %r8d + rorl $2, %ecx + movl %r12d, %edx + addl %ecx, %r8d + addl (%rdi), %r8d + addl 4(%rdi), %r9d + addl 8(%rdi), %r10d + addl 12(%rdi), %r11d + addl 16(%rdi), %r12d + addl 20(%rdi), %r13d + addl 24(%rdi), %r14d + addl 28(%rdi), %r15d + movl %r8d, (%rdi) + movl %r9d, 4(%rdi) + movl %r10d, 8(%rdi) + movl %r11d, 12(%rdi) + movl %r12d, 16(%rdi) + movl %r13d, 20(%rdi) + movl %r14d, 24(%rdi) + movl %r15d, 28(%rdi) + movl %r9d, %ebx + movl %r12d, %edx + xorl %r10d, %ebx + # rnd_all_4: 1-4 + addl 16(%rsp), %r15d + movl %r13d, %ecx + movl %r9d, %eax + xorl %r14d, %ecx + rorl $14, %edx + andl %r12d, %ecx + xorl %r12d, %edx + xorl %r14d, %ecx + rorl $5, %edx + addl %ecx, %r15d + xorl %r12d, %edx + xorl %r8d, %eax + rorl $6, %edx + movl %r8d, %ecx + addl %edx, %r15d + rorl $9, %ecx + andl %eax, %ebx + xorl %r8d, %ecx + xorl %r9d, %ebx + rorl $11, %ecx + addl %r15d, %r11d + xorl %r8d, %ecx + addl %ebx, %r15d + rorl $2, %ecx + movl %r11d, %edx + addl %ecx, %r15d + addl 20(%rsp), %r14d + movl %r12d, %ecx + movl %r8d, %ebx + xorl %r13d, %ecx + rorl $14, %edx + andl %r11d, %ecx + xorl %r11d, %edx + xorl %r13d, %ecx + rorl $5, %edx + addl %ecx, %r14d + xorl %r11d, %edx + xorl %r15d, %ebx + rorl $6, %edx + movl %r15d, %ecx + addl %edx, %r14d + rorl $9, %ecx + andl %ebx, %eax + xorl %r15d, %ecx + xorl %r8d, %eax + rorl $11, %ecx + addl %r14d, %r10d + xorl %r15d, %ecx + addl %eax, %r14d + rorl $2, %ecx + movl %r10d, %edx + addl %ecx, %r14d + addl 24(%rsp), %r13d + movl %r11d, %ecx + movl %r15d, %eax + xorl %r12d, %ecx + rorl $14, %edx + andl %r10d, %ecx + xorl %r10d, %edx + xorl %r12d, %ecx + rorl $5, %edx + addl %ecx, %r13d + xorl %r10d, %edx + xorl %r14d, %eax + rorl $6, %edx + movl %r14d, %ecx + addl %edx, %r13d + rorl $9, %ecx + andl %eax, %ebx + xorl %r14d, %ecx + xorl %r15d, %ebx + rorl $11, %ecx + addl %r13d, %r9d + xorl %r14d, %ecx + addl %ebx, %r13d + rorl $2, %ecx + movl %r9d, %edx + addl %ecx, %r13d + addl 28(%rsp), %r12d + movl %r10d, %ecx + movl %r14d, %ebx + xorl %r11d, %ecx + rorl $14, %edx + andl %r9d, %ecx + xorl %r9d, %edx + xorl %r11d, %ecx + rorl $5, %edx + addl %ecx, %r12d + xorl %r9d, %edx + xorl %r13d, %ebx + rorl $6, %edx + movl %r13d, %ecx + addl %edx, %r12d + rorl $9, %ecx + andl %ebx, %eax + xorl %r13d, %ecx + xorl %r14d, %eax + rorl $11, %ecx + addl %r12d, %r8d + xorl %r13d, %ecx + addl %eax, %r12d + rorl $2, %ecx + movl %r8d, %edx + addl %ecx, %r12d + # rnd_all_4: 3-6 + addl 48(%rsp), %r11d + movl %r9d, %ecx + movl %r13d, %eax + xorl %r10d, %ecx + rorl $14, %edx + andl %r8d, %ecx + xorl %r8d, %edx + xorl %r10d, %ecx + rorl $5, %edx + addl %ecx, %r11d + xorl %r8d, %edx + xorl %r12d, %eax + rorl $6, %edx + movl %r12d, %ecx + addl %edx, %r11d + rorl $9, %ecx + andl %eax, %ebx + xorl %r12d, %ecx + xorl %r13d, %ebx + rorl $11, %ecx + addl %r11d, %r15d + xorl %r12d, %ecx + addl %ebx, %r11d + rorl $2, %ecx + movl %r15d, %edx + addl %ecx, %r11d + addl 52(%rsp), %r10d + movl %r8d, %ecx + movl %r12d, %ebx + xorl %r9d, %ecx + rorl $14, %edx + andl %r15d, %ecx + xorl %r15d, %edx + xorl %r9d, %ecx + rorl $5, %edx + addl %ecx, %r10d + xorl %r15d, %edx + xorl %r11d, %ebx + rorl $6, %edx + movl %r11d, %ecx + addl %edx, %r10d + rorl $9, %ecx + andl %ebx, %eax + xorl %r11d, %ecx + xorl %r12d, %eax + rorl $11, %ecx + addl %r10d, %r14d + xorl %r11d, %ecx + addl %eax, %r10d + rorl $2, %ecx + movl %r14d, %edx + addl %ecx, %r10d + addl 56(%rsp), %r9d + movl %r15d, %ecx + movl %r11d, %eax + xorl %r8d, %ecx + rorl $14, %edx + andl %r14d, %ecx + xorl %r14d, %edx + xorl %r8d, %ecx + rorl $5, %edx + addl %ecx, %r9d + xorl %r14d, %edx + xorl %r10d, %eax + rorl $6, %edx + movl %r10d, %ecx + addl %edx, %r9d + rorl $9, %ecx + andl %eax, %ebx + xorl %r10d, %ecx + xorl %r11d, %ebx + rorl $11, %ecx + addl %r9d, %r13d + xorl %r10d, %ecx + addl %ebx, %r9d + rorl $2, %ecx + movl %r13d, %edx + addl %ecx, %r9d + addl 60(%rsp), %r8d + movl %r14d, %ecx + movl %r10d, %ebx + xorl %r15d, %ecx + rorl $14, %edx + andl %r13d, %ecx + xorl %r13d, %edx + xorl %r15d, %ecx + rorl $5, %edx + addl %ecx, %r8d + xorl %r13d, %edx + xorl %r9d, %ebx + rorl $6, %edx + movl %r9d, %ecx + addl %edx, %r8d + rorl $9, %ecx + andl %ebx, %eax + xorl %r9d, %ecx + xorl %r10d, %eax + rorl $11, %ecx + addl %r8d, %r12d + xorl %r9d, %ecx + addl %eax, %r8d + rorl $2, %ecx + movl %r12d, %edx + addl %ecx, %r8d + # rnd_all_4: 5-8 + addl 80(%rsp), %r15d + movl %r13d, %ecx + movl %r9d, %eax + xorl %r14d, %ecx + rorl $14, %edx + andl %r12d, %ecx + xorl %r12d, %edx + xorl %r14d, %ecx + rorl $5, %edx + addl %ecx, %r15d + xorl %r12d, %edx + xorl %r8d, %eax + rorl $6, %edx + movl %r8d, %ecx + addl %edx, %r15d + rorl $9, %ecx + andl %eax, %ebx + xorl %r8d, %ecx + xorl %r9d, %ebx + rorl $11, %ecx + addl %r15d, %r11d + xorl %r8d, %ecx + addl %ebx, %r15d + rorl $2, %ecx + movl %r11d, %edx + addl %ecx, %r15d + addl 84(%rsp), %r14d + movl %r12d, %ecx + movl %r8d, %ebx + xorl %r13d, %ecx + rorl $14, %edx + andl %r11d, %ecx + xorl %r11d, %edx + xorl %r13d, %ecx + rorl $5, %edx + addl %ecx, %r14d + xorl %r11d, %edx + xorl %r15d, %ebx + rorl $6, %edx + movl %r15d, %ecx + addl %edx, %r14d + rorl $9, %ecx + andl %ebx, %eax + xorl %r15d, %ecx + xorl %r8d, %eax + rorl $11, %ecx + addl %r14d, %r10d + xorl %r15d, %ecx + addl %eax, %r14d + rorl $2, %ecx + movl %r10d, %edx + addl %ecx, %r14d + addl 88(%rsp), %r13d + movl %r11d, %ecx + movl %r15d, %eax + xorl %r12d, %ecx + rorl $14, %edx + andl %r10d, %ecx + xorl %r10d, %edx + xorl %r12d, %ecx + rorl $5, %edx + addl %ecx, %r13d + xorl %r10d, %edx + xorl %r14d, %eax + rorl $6, %edx + movl %r14d, %ecx + addl %edx, %r13d + rorl $9, %ecx + andl %eax, %ebx + xorl %r14d, %ecx + xorl %r15d, %ebx + rorl $11, %ecx + addl %r13d, %r9d + xorl %r14d, %ecx + addl %ebx, %r13d + rorl $2, %ecx + movl %r9d, %edx + addl %ecx, %r13d + addl 92(%rsp), %r12d + movl %r10d, %ecx + movl %r14d, %ebx + xorl %r11d, %ecx + rorl $14, %edx + andl %r9d, %ecx + xorl %r9d, %edx + xorl %r11d, %ecx + rorl $5, %edx + addl %ecx, %r12d + xorl %r9d, %edx + xorl %r13d, %ebx + rorl $6, %edx + movl %r13d, %ecx + addl %edx, %r12d + rorl $9, %ecx + andl %ebx, %eax + xorl %r13d, %ecx + xorl %r14d, %eax + rorl $11, %ecx + addl %r12d, %r8d + xorl %r13d, %ecx + addl %eax, %r12d + rorl $2, %ecx + movl %r8d, %edx + addl %ecx, %r12d + # rnd_all_4: 7-10 + addl 112(%rsp), %r11d + movl %r9d, %ecx + movl %r13d, %eax + xorl %r10d, %ecx + rorl $14, %edx + andl %r8d, %ecx + xorl %r8d, %edx + xorl %r10d, %ecx + rorl $5, %edx + addl %ecx, %r11d + xorl %r8d, %edx + xorl %r12d, %eax + rorl $6, %edx + movl %r12d, %ecx + addl %edx, %r11d + rorl $9, %ecx + andl %eax, %ebx + xorl %r12d, %ecx + xorl %r13d, %ebx + rorl $11, %ecx + addl %r11d, %r15d + xorl %r12d, %ecx + addl %ebx, %r11d + rorl $2, %ecx + movl %r15d, %edx + addl %ecx, %r11d + addl 116(%rsp), %r10d + movl %r8d, %ecx + movl %r12d, %ebx + xorl %r9d, %ecx + rorl $14, %edx + andl %r15d, %ecx + xorl %r15d, %edx + xorl %r9d, %ecx + rorl $5, %edx + addl %ecx, %r10d + xorl %r15d, %edx + xorl %r11d, %ebx + rorl $6, %edx + movl %r11d, %ecx + addl %edx, %r10d + rorl $9, %ecx + andl %ebx, %eax + xorl %r11d, %ecx + xorl %r12d, %eax + rorl $11, %ecx + addl %r10d, %r14d + xorl %r11d, %ecx + addl %eax, %r10d + rorl $2, %ecx + movl %r14d, %edx + addl %ecx, %r10d + addl 120(%rsp), %r9d + movl %r15d, %ecx + movl %r11d, %eax + xorl %r8d, %ecx + rorl $14, %edx + andl %r14d, %ecx + xorl %r14d, %edx + xorl %r8d, %ecx + rorl $5, %edx + addl %ecx, %r9d + xorl %r14d, %edx + xorl %r10d, %eax + rorl $6, %edx + movl %r10d, %ecx + addl %edx, %r9d + rorl $9, %ecx + andl %eax, %ebx + xorl %r10d, %ecx + xorl %r11d, %ebx + rorl $11, %ecx + addl %r9d, %r13d + xorl %r10d, %ecx + addl %ebx, %r9d + rorl $2, %ecx + movl %r13d, %edx + addl %ecx, %r9d + addl 124(%rsp), %r8d + movl %r14d, %ecx + movl %r10d, %ebx + xorl %r15d, %ecx + rorl $14, %edx + andl %r13d, %ecx + xorl %r13d, %edx + xorl %r15d, %ecx + rorl $5, %edx + addl %ecx, %r8d + xorl %r13d, %edx + xorl %r9d, %ebx + rorl $6, %edx + movl %r9d, %ecx + addl %edx, %r8d + rorl $9, %ecx + andl %ebx, %eax + xorl %r9d, %ecx + xorl %r10d, %eax + rorl $11, %ecx + addl %r8d, %r12d + xorl %r9d, %ecx + addl %eax, %r8d + rorl $2, %ecx + movl %r12d, %edx + addl %ecx, %r8d + # rnd_all_4: 9-12 + addl 144(%rsp), %r15d + movl %r13d, %ecx + movl %r9d, %eax + xorl %r14d, %ecx + rorl $14, %edx + andl %r12d, %ecx + xorl %r12d, %edx + xorl %r14d, %ecx + rorl $5, %edx + addl %ecx, %r15d + xorl %r12d, %edx + xorl %r8d, %eax + rorl $6, %edx + movl %r8d, %ecx + addl %edx, %r15d + rorl $9, %ecx + andl %eax, %ebx + xorl %r8d, %ecx + xorl %r9d, %ebx + rorl $11, %ecx + addl %r15d, %r11d + xorl %r8d, %ecx + addl %ebx, %r15d + rorl $2, %ecx + movl %r11d, %edx + addl %ecx, %r15d + addl 148(%rsp), %r14d + movl %r12d, %ecx + movl %r8d, %ebx + xorl %r13d, %ecx + rorl $14, %edx + andl %r11d, %ecx + xorl %r11d, %edx + xorl %r13d, %ecx + rorl $5, %edx + addl %ecx, %r14d + xorl %r11d, %edx + xorl %r15d, %ebx + rorl $6, %edx + movl %r15d, %ecx + addl %edx, %r14d + rorl $9, %ecx + andl %ebx, %eax + xorl %r15d, %ecx + xorl %r8d, %eax + rorl $11, %ecx + addl %r14d, %r10d + xorl %r15d, %ecx + addl %eax, %r14d + rorl $2, %ecx + movl %r10d, %edx + addl %ecx, %r14d + addl 152(%rsp), %r13d + movl %r11d, %ecx + movl %r15d, %eax + xorl %r12d, %ecx + rorl $14, %edx + andl %r10d, %ecx + xorl %r10d, %edx + xorl %r12d, %ecx + rorl $5, %edx + addl %ecx, %r13d + xorl %r10d, %edx + xorl %r14d, %eax + rorl $6, %edx + movl %r14d, %ecx + addl %edx, %r13d + rorl $9, %ecx + andl %eax, %ebx + xorl %r14d, %ecx + xorl %r15d, %ebx + rorl $11, %ecx + addl %r13d, %r9d + xorl %r14d, %ecx + addl %ebx, %r13d + rorl $2, %ecx + movl %r9d, %edx + addl %ecx, %r13d + addl 156(%rsp), %r12d + movl %r10d, %ecx + movl %r14d, %ebx + xorl %r11d, %ecx + rorl $14, %edx + andl %r9d, %ecx + xorl %r9d, %edx + xorl %r11d, %ecx + rorl $5, %edx + addl %ecx, %r12d + xorl %r9d, %edx + xorl %r13d, %ebx + rorl $6, %edx + movl %r13d, %ecx + addl %edx, %r12d + rorl $9, %ecx + andl %ebx, %eax + xorl %r13d, %ecx + xorl %r14d, %eax + rorl $11, %ecx + addl %r12d, %r8d + xorl %r13d, %ecx + addl %eax, %r12d + rorl $2, %ecx + movl %r8d, %edx + addl %ecx, %r12d + # rnd_all_4: 11-14 + addl 176(%rsp), %r11d + movl %r9d, %ecx + movl %r13d, %eax + xorl %r10d, %ecx + rorl $14, %edx + andl %r8d, %ecx + xorl %r8d, %edx + xorl %r10d, %ecx + rorl $5, %edx + addl %ecx, %r11d + xorl %r8d, %edx + xorl %r12d, %eax + rorl $6, %edx + movl %r12d, %ecx + addl %edx, %r11d + rorl $9, %ecx + andl %eax, %ebx + xorl %r12d, %ecx + xorl %r13d, %ebx + rorl $11, %ecx + addl %r11d, %r15d + xorl %r12d, %ecx + addl %ebx, %r11d + rorl $2, %ecx + movl %r15d, %edx + addl %ecx, %r11d + addl 180(%rsp), %r10d + movl %r8d, %ecx + movl %r12d, %ebx + xorl %r9d, %ecx + rorl $14, %edx + andl %r15d, %ecx + xorl %r15d, %edx + xorl %r9d, %ecx + rorl $5, %edx + addl %ecx, %r10d + xorl %r15d, %edx + xorl %r11d, %ebx + rorl $6, %edx + movl %r11d, %ecx + addl %edx, %r10d + rorl $9, %ecx + andl %ebx, %eax + xorl %r11d, %ecx + xorl %r12d, %eax + rorl $11, %ecx + addl %r10d, %r14d + xorl %r11d, %ecx + addl %eax, %r10d + rorl $2, %ecx + movl %r14d, %edx + addl %ecx, %r10d + addl 184(%rsp), %r9d + movl %r15d, %ecx + movl %r11d, %eax + xorl %r8d, %ecx + rorl $14, %edx + andl %r14d, %ecx + xorl %r14d, %edx + xorl %r8d, %ecx + rorl $5, %edx + addl %ecx, %r9d + xorl %r14d, %edx + xorl %r10d, %eax + rorl $6, %edx + movl %r10d, %ecx + addl %edx, %r9d + rorl $9, %ecx + andl %eax, %ebx + xorl %r10d, %ecx + xorl %r11d, %ebx + rorl $11, %ecx + addl %r9d, %r13d + xorl %r10d, %ecx + addl %ebx, %r9d + rorl $2, %ecx + movl %r13d, %edx + addl %ecx, %r9d + addl 188(%rsp), %r8d + movl %r14d, %ecx + movl %r10d, %ebx + xorl %r15d, %ecx + rorl $14, %edx + andl %r13d, %ecx + xorl %r13d, %edx + xorl %r15d, %ecx + rorl $5, %edx + addl %ecx, %r8d + xorl %r13d, %edx + xorl %r9d, %ebx + rorl $6, %edx + movl %r9d, %ecx + addl %edx, %r8d + rorl $9, %ecx + andl %ebx, %eax + xorl %r9d, %ecx + xorl %r10d, %eax + rorl $11, %ecx + addl %r8d, %r12d + xorl %r9d, %ecx + addl %eax, %r8d + rorl $2, %ecx + movl %r12d, %edx + addl %ecx, %r8d + # rnd_all_4: 13-16 + addl 208(%rsp), %r15d + movl %r13d, %ecx + movl %r9d, %eax + xorl %r14d, %ecx + rorl $14, %edx + andl %r12d, %ecx + xorl %r12d, %edx + xorl %r14d, %ecx + rorl $5, %edx + addl %ecx, %r15d + xorl %r12d, %edx + xorl %r8d, %eax + rorl $6, %edx + movl %r8d, %ecx + addl %edx, %r15d + rorl $9, %ecx + andl %eax, %ebx + xorl %r8d, %ecx + xorl %r9d, %ebx + rorl $11, %ecx + addl %r15d, %r11d + xorl %r8d, %ecx + addl %ebx, %r15d + rorl $2, %ecx + movl %r11d, %edx + addl %ecx, %r15d + addl 212(%rsp), %r14d + movl %r12d, %ecx + movl %r8d, %ebx + xorl %r13d, %ecx + rorl $14, %edx + andl %r11d, %ecx + xorl %r11d, %edx + xorl %r13d, %ecx + rorl $5, %edx + addl %ecx, %r14d + xorl %r11d, %edx + xorl %r15d, %ebx + rorl $6, %edx + movl %r15d, %ecx + addl %edx, %r14d + rorl $9, %ecx + andl %ebx, %eax + xorl %r15d, %ecx + xorl %r8d, %eax + rorl $11, %ecx + addl %r14d, %r10d + xorl %r15d, %ecx + addl %eax, %r14d + rorl $2, %ecx + movl %r10d, %edx + addl %ecx, %r14d + addl 216(%rsp), %r13d + movl %r11d, %ecx + movl %r15d, %eax + xorl %r12d, %ecx + rorl $14, %edx + andl %r10d, %ecx + xorl %r10d, %edx + xorl %r12d, %ecx + rorl $5, %edx + addl %ecx, %r13d + xorl %r10d, %edx + xorl %r14d, %eax + rorl $6, %edx + movl %r14d, %ecx + addl %edx, %r13d + rorl $9, %ecx + andl %eax, %ebx + xorl %r14d, %ecx + xorl %r15d, %ebx + rorl $11, %ecx + addl %r13d, %r9d + xorl %r14d, %ecx + addl %ebx, %r13d + rorl $2, %ecx + movl %r9d, %edx + addl %ecx, %r13d + addl 220(%rsp), %r12d + movl %r10d, %ecx + movl %r14d, %ebx + xorl %r11d, %ecx + rorl $14, %edx + andl %r9d, %ecx + xorl %r9d, %edx + xorl %r11d, %ecx + rorl $5, %edx + addl %ecx, %r12d + xorl %r9d, %edx + xorl %r13d, %ebx + rorl $6, %edx + movl %r13d, %ecx + addl %edx, %r12d + rorl $9, %ecx + andl %ebx, %eax + xorl %r13d, %ecx + xorl %r14d, %eax + rorl $11, %ecx + addl %r12d, %r8d + xorl %r13d, %ecx + addl %eax, %r12d + rorl $2, %ecx + movl %r8d, %edx + addl %ecx, %r12d + # rnd_all_4: 15-18 + addl 240(%rsp), %r11d + movl %r9d, %ecx + movl %r13d, %eax + xorl %r10d, %ecx + rorl $14, %edx + andl %r8d, %ecx + xorl %r8d, %edx + xorl %r10d, %ecx + rorl $5, %edx + addl %ecx, %r11d + xorl %r8d, %edx + xorl %r12d, %eax + rorl $6, %edx + movl %r12d, %ecx + addl %edx, %r11d + rorl $9, %ecx + andl %eax, %ebx + xorl %r12d, %ecx + xorl %r13d, %ebx + rorl $11, %ecx + addl %r11d, %r15d + xorl %r12d, %ecx + addl %ebx, %r11d + rorl $2, %ecx + movl %r15d, %edx + addl %ecx, %r11d + addl 244(%rsp), %r10d + movl %r8d, %ecx + movl %r12d, %ebx + xorl %r9d, %ecx + rorl $14, %edx + andl %r15d, %ecx + xorl %r15d, %edx + xorl %r9d, %ecx + rorl $5, %edx + addl %ecx, %r10d + xorl %r15d, %edx + xorl %r11d, %ebx + rorl $6, %edx + movl %r11d, %ecx + addl %edx, %r10d + rorl $9, %ecx + andl %ebx, %eax + xorl %r11d, %ecx + xorl %r12d, %eax + rorl $11, %ecx + addl %r10d, %r14d + xorl %r11d, %ecx + addl %eax, %r10d + rorl $2, %ecx + movl %r14d, %edx + addl %ecx, %r10d + addl 248(%rsp), %r9d + movl %r15d, %ecx + movl %r11d, %eax + xorl %r8d, %ecx + rorl $14, %edx + andl %r14d, %ecx + xorl %r14d, %edx + xorl %r8d, %ecx + rorl $5, %edx + addl %ecx, %r9d + xorl %r14d, %edx + xorl %r10d, %eax + rorl $6, %edx + movl %r10d, %ecx + addl %edx, %r9d + rorl $9, %ecx + andl %eax, %ebx + xorl %r10d, %ecx + xorl %r11d, %ebx + rorl $11, %ecx + addl %r9d, %r13d + xorl %r10d, %ecx + addl %ebx, %r9d + rorl $2, %ecx + movl %r13d, %edx + addl %ecx, %r9d + addl 252(%rsp), %r8d + movl %r14d, %ecx + movl %r10d, %ebx + xorl %r15d, %ecx + rorl $14, %edx + andl %r13d, %ecx + xorl %r13d, %edx + xorl %r15d, %ecx + rorl $5, %edx + addl %ecx, %r8d + xorl %r13d, %edx + xorl %r9d, %ebx + rorl $6, %edx + movl %r9d, %ecx + addl %edx, %r8d + rorl $9, %ecx + andl %ebx, %eax + xorl %r9d, %ecx + xorl %r10d, %eax + rorl $11, %ecx + addl %r8d, %r12d + xorl %r9d, %ecx + addl %eax, %r8d + rorl $2, %ecx + movl %r12d, %edx + addl %ecx, %r8d + # rnd_all_4: 17-20 + addl 272(%rsp), %r15d + movl %r13d, %ecx + movl %r9d, %eax + xorl %r14d, %ecx + rorl $14, %edx + andl %r12d, %ecx + xorl %r12d, %edx + xorl %r14d, %ecx + rorl $5, %edx + addl %ecx, %r15d + xorl %r12d, %edx + xorl %r8d, %eax + rorl $6, %edx + movl %r8d, %ecx + addl %edx, %r15d + rorl $9, %ecx + andl %eax, %ebx + xorl %r8d, %ecx + xorl %r9d, %ebx + rorl $11, %ecx + addl %r15d, %r11d + xorl %r8d, %ecx + addl %ebx, %r15d + rorl $2, %ecx + movl %r11d, %edx + addl %ecx, %r15d + addl 276(%rsp), %r14d + movl %r12d, %ecx + movl %r8d, %ebx + xorl %r13d, %ecx + rorl $14, %edx + andl %r11d, %ecx + xorl %r11d, %edx + xorl %r13d, %ecx + rorl $5, %edx + addl %ecx, %r14d + xorl %r11d, %edx + xorl %r15d, %ebx + rorl $6, %edx + movl %r15d, %ecx + addl %edx, %r14d + rorl $9, %ecx + andl %ebx, %eax + xorl %r15d, %ecx + xorl %r8d, %eax + rorl $11, %ecx + addl %r14d, %r10d + xorl %r15d, %ecx + addl %eax, %r14d + rorl $2, %ecx + movl %r10d, %edx + addl %ecx, %r14d + addl 280(%rsp), %r13d + movl %r11d, %ecx + movl %r15d, %eax + xorl %r12d, %ecx + rorl $14, %edx + andl %r10d, %ecx + xorl %r10d, %edx + xorl %r12d, %ecx + rorl $5, %edx + addl %ecx, %r13d + xorl %r10d, %edx + xorl %r14d, %eax + rorl $6, %edx + movl %r14d, %ecx + addl %edx, %r13d + rorl $9, %ecx + andl %eax, %ebx + xorl %r14d, %ecx + xorl %r15d, %ebx + rorl $11, %ecx + addl %r13d, %r9d + xorl %r14d, %ecx + addl %ebx, %r13d + rorl $2, %ecx + movl %r9d, %edx + addl %ecx, %r13d + addl 284(%rsp), %r12d + movl %r10d, %ecx + movl %r14d, %ebx + xorl %r11d, %ecx + rorl $14, %edx + andl %r9d, %ecx + xorl %r9d, %edx + xorl %r11d, %ecx + rorl $5, %edx + addl %ecx, %r12d + xorl %r9d, %edx + xorl %r13d, %ebx + rorl $6, %edx + movl %r13d, %ecx + addl %edx, %r12d + rorl $9, %ecx + andl %ebx, %eax + xorl %r13d, %ecx + xorl %r14d, %eax + rorl $11, %ecx + addl %r12d, %r8d + xorl %r13d, %ecx + addl %eax, %r12d + rorl $2, %ecx + movl %r8d, %edx + addl %ecx, %r12d + # rnd_all_4: 19-22 + addl 304(%rsp), %r11d + movl %r9d, %ecx + movl %r13d, %eax + xorl %r10d, %ecx + rorl $14, %edx + andl %r8d, %ecx + xorl %r8d, %edx + xorl %r10d, %ecx + rorl $5, %edx + addl %ecx, %r11d + xorl %r8d, %edx + xorl %r12d, %eax + rorl $6, %edx + movl %r12d, %ecx + addl %edx, %r11d + rorl $9, %ecx + andl %eax, %ebx + xorl %r12d, %ecx + xorl %r13d, %ebx + rorl $11, %ecx + addl %r11d, %r15d + xorl %r12d, %ecx + addl %ebx, %r11d + rorl $2, %ecx + movl %r15d, %edx + addl %ecx, %r11d + addl 308(%rsp), %r10d + movl %r8d, %ecx + movl %r12d, %ebx + xorl %r9d, %ecx + rorl $14, %edx + andl %r15d, %ecx + xorl %r15d, %edx + xorl %r9d, %ecx + rorl $5, %edx + addl %ecx, %r10d + xorl %r15d, %edx + xorl %r11d, %ebx + rorl $6, %edx + movl %r11d, %ecx + addl %edx, %r10d + rorl $9, %ecx + andl %ebx, %eax + xorl %r11d, %ecx + xorl %r12d, %eax + rorl $11, %ecx + addl %r10d, %r14d + xorl %r11d, %ecx + addl %eax, %r10d + rorl $2, %ecx + movl %r14d, %edx + addl %ecx, %r10d + addl 312(%rsp), %r9d + movl %r15d, %ecx + movl %r11d, %eax + xorl %r8d, %ecx + rorl $14, %edx + andl %r14d, %ecx + xorl %r14d, %edx + xorl %r8d, %ecx + rorl $5, %edx + addl %ecx, %r9d + xorl %r14d, %edx + xorl %r10d, %eax + rorl $6, %edx + movl %r10d, %ecx + addl %edx, %r9d + rorl $9, %ecx + andl %eax, %ebx + xorl %r10d, %ecx + xorl %r11d, %ebx + rorl $11, %ecx + addl %r9d, %r13d + xorl %r10d, %ecx + addl %ebx, %r9d + rorl $2, %ecx + movl %r13d, %edx + addl %ecx, %r9d + addl 316(%rsp), %r8d + movl %r14d, %ecx + movl %r10d, %ebx + xorl %r15d, %ecx + rorl $14, %edx + andl %r13d, %ecx + xorl %r13d, %edx + xorl %r15d, %ecx + rorl $5, %edx + addl %ecx, %r8d + xorl %r13d, %edx + xorl %r9d, %ebx + rorl $6, %edx + movl %r9d, %ecx + addl %edx, %r8d + rorl $9, %ecx + andl %ebx, %eax + xorl %r9d, %ecx + xorl %r10d, %eax + rorl $11, %ecx + addl %r8d, %r12d + xorl %r9d, %ecx + addl %eax, %r8d + rorl $2, %ecx + movl %r12d, %edx + addl %ecx, %r8d + # rnd_all_4: 21-24 + addl 336(%rsp), %r15d + movl %r13d, %ecx + movl %r9d, %eax + xorl %r14d, %ecx + rorl $14, %edx + andl %r12d, %ecx + xorl %r12d, %edx + xorl %r14d, %ecx + rorl $5, %edx + addl %ecx, %r15d + xorl %r12d, %edx + xorl %r8d, %eax + rorl $6, %edx + movl %r8d, %ecx + addl %edx, %r15d + rorl $9, %ecx + andl %eax, %ebx + xorl %r8d, %ecx + xorl %r9d, %ebx + rorl $11, %ecx + addl %r15d, %r11d + xorl %r8d, %ecx + addl %ebx, %r15d + rorl $2, %ecx + movl %r11d, %edx + addl %ecx, %r15d + addl 340(%rsp), %r14d + movl %r12d, %ecx + movl %r8d, %ebx + xorl %r13d, %ecx + rorl $14, %edx + andl %r11d, %ecx + xorl %r11d, %edx + xorl %r13d, %ecx + rorl $5, %edx + addl %ecx, %r14d + xorl %r11d, %edx + xorl %r15d, %ebx + rorl $6, %edx + movl %r15d, %ecx + addl %edx, %r14d + rorl $9, %ecx + andl %ebx, %eax + xorl %r15d, %ecx + xorl %r8d, %eax + rorl $11, %ecx + addl %r14d, %r10d + xorl %r15d, %ecx + addl %eax, %r14d + rorl $2, %ecx + movl %r10d, %edx + addl %ecx, %r14d + addl 344(%rsp), %r13d + movl %r11d, %ecx + movl %r15d, %eax + xorl %r12d, %ecx + rorl $14, %edx + andl %r10d, %ecx + xorl %r10d, %edx + xorl %r12d, %ecx + rorl $5, %edx + addl %ecx, %r13d + xorl %r10d, %edx + xorl %r14d, %eax + rorl $6, %edx + movl %r14d, %ecx + addl %edx, %r13d + rorl $9, %ecx + andl %eax, %ebx + xorl %r14d, %ecx + xorl %r15d, %ebx + rorl $11, %ecx + addl %r13d, %r9d + xorl %r14d, %ecx + addl %ebx, %r13d + rorl $2, %ecx + movl %r9d, %edx + addl %ecx, %r13d + addl 348(%rsp), %r12d + movl %r10d, %ecx + movl %r14d, %ebx + xorl %r11d, %ecx + rorl $14, %edx + andl %r9d, %ecx + xorl %r9d, %edx + xorl %r11d, %ecx + rorl $5, %edx + addl %ecx, %r12d + xorl %r9d, %edx + xorl %r13d, %ebx + rorl $6, %edx + movl %r13d, %ecx + addl %edx, %r12d + rorl $9, %ecx + andl %ebx, %eax + xorl %r13d, %ecx + xorl %r14d, %eax + rorl $11, %ecx + addl %r12d, %r8d + xorl %r13d, %ecx + addl %eax, %r12d + rorl $2, %ecx + movl %r8d, %edx + addl %ecx, %r12d + # rnd_all_4: 23-26 + addl 368(%rsp), %r11d + movl %r9d, %ecx + movl %r13d, %eax + xorl %r10d, %ecx + rorl $14, %edx + andl %r8d, %ecx + xorl %r8d, %edx + xorl %r10d, %ecx + rorl $5, %edx + addl %ecx, %r11d + xorl %r8d, %edx + xorl %r12d, %eax + rorl $6, %edx + movl %r12d, %ecx + addl %edx, %r11d + rorl $9, %ecx + andl %eax, %ebx + xorl %r12d, %ecx + xorl %r13d, %ebx + rorl $11, %ecx + addl %r11d, %r15d + xorl %r12d, %ecx + addl %ebx, %r11d + rorl $2, %ecx + movl %r15d, %edx + addl %ecx, %r11d + addl 372(%rsp), %r10d + movl %r8d, %ecx + movl %r12d, %ebx + xorl %r9d, %ecx + rorl $14, %edx + andl %r15d, %ecx + xorl %r15d, %edx + xorl %r9d, %ecx + rorl $5, %edx + addl %ecx, %r10d + xorl %r15d, %edx + xorl %r11d, %ebx + rorl $6, %edx + movl %r11d, %ecx + addl %edx, %r10d + rorl $9, %ecx + andl %ebx, %eax + xorl %r11d, %ecx + xorl %r12d, %eax + rorl $11, %ecx + addl %r10d, %r14d + xorl %r11d, %ecx + addl %eax, %r10d + rorl $2, %ecx + movl %r14d, %edx + addl %ecx, %r10d + addl 376(%rsp), %r9d + movl %r15d, %ecx + movl %r11d, %eax + xorl %r8d, %ecx + rorl $14, %edx + andl %r14d, %ecx + xorl %r14d, %edx + xorl %r8d, %ecx + rorl $5, %edx + addl %ecx, %r9d + xorl %r14d, %edx + xorl %r10d, %eax + rorl $6, %edx + movl %r10d, %ecx + addl %edx, %r9d + rorl $9, %ecx + andl %eax, %ebx + xorl %r10d, %ecx + xorl %r11d, %ebx + rorl $11, %ecx + addl %r9d, %r13d + xorl %r10d, %ecx + addl %ebx, %r9d + rorl $2, %ecx + movl %r13d, %edx + addl %ecx, %r9d + addl 380(%rsp), %r8d + movl %r14d, %ecx + movl %r10d, %ebx + xorl %r15d, %ecx + rorl $14, %edx + andl %r13d, %ecx + xorl %r13d, %edx + xorl %r15d, %ecx + rorl $5, %edx + addl %ecx, %r8d + xorl %r13d, %edx + xorl %r9d, %ebx + rorl $6, %edx + movl %r9d, %ecx + addl %edx, %r8d + rorl $9, %ecx + andl %ebx, %eax + xorl %r9d, %ecx + xorl %r10d, %eax + rorl $11, %ecx + addl %r8d, %r12d + xorl %r9d, %ecx + addl %eax, %r8d + rorl $2, %ecx + movl %r12d, %edx + addl %ecx, %r8d + # rnd_all_4: 25-28 + addl 400(%rsp), %r15d + movl %r13d, %ecx + movl %r9d, %eax + xorl %r14d, %ecx + rorl $14, %edx + andl %r12d, %ecx + xorl %r12d, %edx + xorl %r14d, %ecx + rorl $5, %edx + addl %ecx, %r15d + xorl %r12d, %edx + xorl %r8d, %eax + rorl $6, %edx + movl %r8d, %ecx + addl %edx, %r15d + rorl $9, %ecx + andl %eax, %ebx + xorl %r8d, %ecx + xorl %r9d, %ebx + rorl $11, %ecx + addl %r15d, %r11d + xorl %r8d, %ecx + addl %ebx, %r15d + rorl $2, %ecx + movl %r11d, %edx + addl %ecx, %r15d + addl 404(%rsp), %r14d + movl %r12d, %ecx + movl %r8d, %ebx + xorl %r13d, %ecx + rorl $14, %edx + andl %r11d, %ecx + xorl %r11d, %edx + xorl %r13d, %ecx + rorl $5, %edx + addl %ecx, %r14d + xorl %r11d, %edx + xorl %r15d, %ebx + rorl $6, %edx + movl %r15d, %ecx + addl %edx, %r14d + rorl $9, %ecx + andl %ebx, %eax + xorl %r15d, %ecx + xorl %r8d, %eax + rorl $11, %ecx + addl %r14d, %r10d + xorl %r15d, %ecx + addl %eax, %r14d + rorl $2, %ecx + movl %r10d, %edx + addl %ecx, %r14d + addl 408(%rsp), %r13d + movl %r11d, %ecx + movl %r15d, %eax + xorl %r12d, %ecx + rorl $14, %edx + andl %r10d, %ecx + xorl %r10d, %edx + xorl %r12d, %ecx + rorl $5, %edx + addl %ecx, %r13d + xorl %r10d, %edx + xorl %r14d, %eax + rorl $6, %edx + movl %r14d, %ecx + addl %edx, %r13d + rorl $9, %ecx + andl %eax, %ebx + xorl %r14d, %ecx + xorl %r15d, %ebx + rorl $11, %ecx + addl %r13d, %r9d + xorl %r14d, %ecx + addl %ebx, %r13d + rorl $2, %ecx + movl %r9d, %edx + addl %ecx, %r13d + addl 412(%rsp), %r12d + movl %r10d, %ecx + movl %r14d, %ebx + xorl %r11d, %ecx + rorl $14, %edx + andl %r9d, %ecx + xorl %r9d, %edx + xorl %r11d, %ecx + rorl $5, %edx + addl %ecx, %r12d + xorl %r9d, %edx + xorl %r13d, %ebx + rorl $6, %edx + movl %r13d, %ecx + addl %edx, %r12d + rorl $9, %ecx + andl %ebx, %eax + xorl %r13d, %ecx + xorl %r14d, %eax + rorl $11, %ecx + addl %r12d, %r8d + xorl %r13d, %ecx + addl %eax, %r12d + rorl $2, %ecx + movl %r8d, %edx + addl %ecx, %r12d + # rnd_all_4: 27-30 + addl 432(%rsp), %r11d + movl %r9d, %ecx + movl %r13d, %eax + xorl %r10d, %ecx + rorl $14, %edx + andl %r8d, %ecx + xorl %r8d, %edx + xorl %r10d, %ecx + rorl $5, %edx + addl %ecx, %r11d + xorl %r8d, %edx + xorl %r12d, %eax + rorl $6, %edx + movl %r12d, %ecx + addl %edx, %r11d + rorl $9, %ecx + andl %eax, %ebx + xorl %r12d, %ecx + xorl %r13d, %ebx + rorl $11, %ecx + addl %r11d, %r15d + xorl %r12d, %ecx + addl %ebx, %r11d + rorl $2, %ecx + movl %r15d, %edx + addl %ecx, %r11d + addl 436(%rsp), %r10d + movl %r8d, %ecx + movl %r12d, %ebx + xorl %r9d, %ecx + rorl $14, %edx + andl %r15d, %ecx + xorl %r15d, %edx + xorl %r9d, %ecx + rorl $5, %edx + addl %ecx, %r10d + xorl %r15d, %edx + xorl %r11d, %ebx + rorl $6, %edx + movl %r11d, %ecx + addl %edx, %r10d + rorl $9, %ecx + andl %ebx, %eax + xorl %r11d, %ecx + xorl %r12d, %eax + rorl $11, %ecx + addl %r10d, %r14d + xorl %r11d, %ecx + addl %eax, %r10d + rorl $2, %ecx + movl %r14d, %edx + addl %ecx, %r10d + addl 440(%rsp), %r9d + movl %r15d, %ecx + movl %r11d, %eax + xorl %r8d, %ecx + rorl $14, %edx + andl %r14d, %ecx + xorl %r14d, %edx + xorl %r8d, %ecx + rorl $5, %edx + addl %ecx, %r9d + xorl %r14d, %edx + xorl %r10d, %eax + rorl $6, %edx + movl %r10d, %ecx + addl %edx, %r9d + rorl $9, %ecx + andl %eax, %ebx + xorl %r10d, %ecx + xorl %r11d, %ebx + rorl $11, %ecx + addl %r9d, %r13d + xorl %r10d, %ecx + addl %ebx, %r9d + rorl $2, %ecx + movl %r13d, %edx + addl %ecx, %r9d + addl 444(%rsp), %r8d + movl %r14d, %ecx + movl %r10d, %ebx + xorl %r15d, %ecx + rorl $14, %edx + andl %r13d, %ecx + xorl %r13d, %edx + xorl %r15d, %ecx + rorl $5, %edx + addl %ecx, %r8d + xorl %r13d, %edx + xorl %r9d, %ebx + rorl $6, %edx + movl %r9d, %ecx + addl %edx, %r8d + rorl $9, %ecx + andl %ebx, %eax + xorl %r9d, %ecx + xorl %r10d, %eax + rorl $11, %ecx + addl %r8d, %r12d + xorl %r9d, %ecx + addl %eax, %r8d + rorl $2, %ecx + movl %r12d, %edx + addl %ecx, %r8d + # rnd_all_4: 29-32 + addl 464(%rsp), %r15d + movl %r13d, %ecx + movl %r9d, %eax + xorl %r14d, %ecx + rorl $14, %edx + andl %r12d, %ecx + xorl %r12d, %edx + xorl %r14d, %ecx + rorl $5, %edx + addl %ecx, %r15d + xorl %r12d, %edx + xorl %r8d, %eax + rorl $6, %edx + movl %r8d, %ecx + addl %edx, %r15d + rorl $9, %ecx + andl %eax, %ebx + xorl %r8d, %ecx + xorl %r9d, %ebx + rorl $11, %ecx + addl %r15d, %r11d + xorl %r8d, %ecx + addl %ebx, %r15d + rorl $2, %ecx + movl %r11d, %edx + addl %ecx, %r15d + addl 468(%rsp), %r14d + movl %r12d, %ecx + movl %r8d, %ebx + xorl %r13d, %ecx + rorl $14, %edx + andl %r11d, %ecx + xorl %r11d, %edx + xorl %r13d, %ecx + rorl $5, %edx + addl %ecx, %r14d + xorl %r11d, %edx + xorl %r15d, %ebx + rorl $6, %edx + movl %r15d, %ecx + addl %edx, %r14d + rorl $9, %ecx + andl %ebx, %eax + xorl %r15d, %ecx + xorl %r8d, %eax + rorl $11, %ecx + addl %r14d, %r10d + xorl %r15d, %ecx + addl %eax, %r14d + rorl $2, %ecx + movl %r10d, %edx + addl %ecx, %r14d + addl 472(%rsp), %r13d + movl %r11d, %ecx + movl %r15d, %eax + xorl %r12d, %ecx + rorl $14, %edx + andl %r10d, %ecx + xorl %r10d, %edx + xorl %r12d, %ecx + rorl $5, %edx + addl %ecx, %r13d + xorl %r10d, %edx + xorl %r14d, %eax + rorl $6, %edx + movl %r14d, %ecx + addl %edx, %r13d + rorl $9, %ecx + andl %eax, %ebx + xorl %r14d, %ecx + xorl %r15d, %ebx + rorl $11, %ecx + addl %r13d, %r9d + xorl %r14d, %ecx + addl %ebx, %r13d + rorl $2, %ecx + movl %r9d, %edx + addl %ecx, %r13d + addl 476(%rsp), %r12d + movl %r10d, %ecx + movl %r14d, %ebx + xorl %r11d, %ecx + rorl $14, %edx + andl %r9d, %ecx + xorl %r9d, %edx + xorl %r11d, %ecx + rorl $5, %edx + addl %ecx, %r12d + xorl %r9d, %edx + xorl %r13d, %ebx + rorl $6, %edx + movl %r13d, %ecx + addl %edx, %r12d + rorl $9, %ecx + andl %ebx, %eax + xorl %r13d, %ecx + xorl %r14d, %eax + rorl $11, %ecx + addl %r12d, %r8d + xorl %r13d, %ecx + addl %eax, %r12d + rorl $2, %ecx + movl %r8d, %edx + addl %ecx, %r12d + # rnd_all_4: 31-34 + addl 496(%rsp), %r11d + movl %r9d, %ecx + movl %r13d, %eax + xorl %r10d, %ecx + rorl $14, %edx + andl %r8d, %ecx + xorl %r8d, %edx + xorl %r10d, %ecx + rorl $5, %edx + addl %ecx, %r11d + xorl %r8d, %edx + xorl %r12d, %eax + rorl $6, %edx + movl %r12d, %ecx + addl %edx, %r11d + rorl $9, %ecx + andl %eax, %ebx + xorl %r12d, %ecx + xorl %r13d, %ebx + rorl $11, %ecx + addl %r11d, %r15d + xorl %r12d, %ecx + addl %ebx, %r11d + rorl $2, %ecx + movl %r15d, %edx + addl %ecx, %r11d + addl 500(%rsp), %r10d + movl %r8d, %ecx + movl %r12d, %ebx + xorl %r9d, %ecx + rorl $14, %edx + andl %r15d, %ecx + xorl %r15d, %edx + xorl %r9d, %ecx + rorl $5, %edx + addl %ecx, %r10d + xorl %r15d, %edx + xorl %r11d, %ebx + rorl $6, %edx + movl %r11d, %ecx + addl %edx, %r10d + rorl $9, %ecx + andl %ebx, %eax + xorl %r11d, %ecx + xorl %r12d, %eax + rorl $11, %ecx + addl %r10d, %r14d + xorl %r11d, %ecx + addl %eax, %r10d + rorl $2, %ecx + movl %r14d, %edx + addl %ecx, %r10d + addl 504(%rsp), %r9d + movl %r15d, %ecx + movl %r11d, %eax + xorl %r8d, %ecx + rorl $14, %edx + andl %r14d, %ecx + xorl %r14d, %edx + xorl %r8d, %ecx + rorl $5, %edx + addl %ecx, %r9d + xorl %r14d, %edx + xorl %r10d, %eax + rorl $6, %edx + movl %r10d, %ecx + addl %edx, %r9d + rorl $9, %ecx + andl %eax, %ebx + xorl %r10d, %ecx + xorl %r11d, %ebx + rorl $11, %ecx + addl %r9d, %r13d + xorl %r10d, %ecx + addl %ebx, %r9d + rorl $2, %ecx + movl %r13d, %edx + addl %ecx, %r9d + addl 508(%rsp), %r8d + movl %r14d, %ecx + movl %r10d, %ebx + xorl %r15d, %ecx + rorl $14, %edx + andl %r13d, %ecx + xorl %r13d, %edx + xorl %r15d, %ecx + rorl $5, %edx + addl %ecx, %r8d + xorl %r13d, %edx + xorl %r9d, %ebx + rorl $6, %edx + movl %r9d, %ecx + addl %edx, %r8d + rorl $9, %ecx + andl %ebx, %eax + xorl %r9d, %ecx + xorl %r10d, %eax + rorl $11, %ecx + addl %r8d, %r12d + xorl %r9d, %ecx + addl %eax, %r8d + rorl $2, %ecx + movl %r12d, %edx + addl %ecx, %r8d + addl (%rdi), %r8d + addl 4(%rdi), %r9d + addl 8(%rdi), %r10d + addl 12(%rdi), %r11d + addl 16(%rdi), %r12d + addl 20(%rdi), %r13d + addl 24(%rdi), %r14d + addl 28(%rdi), %r15d + addq $0x80, %rbp + subl $0x80, %esi + movl %r8d, (%rdi) + movl %r9d, 4(%rdi) + movl %r10d, 8(%rdi) + movl %r11d, 12(%rdi) + movl %r12d, 16(%rdi) + movl %r13d, 20(%rdi) + movl %r14d, 24(%rdi) + movl %r15d, 28(%rdi) + jnz L_sha256_len_avx2_start +L_sha256_len_avx2_done: + xorq %rax, %rax + vzeroupper + addq $0x200, %rsp + popq %rbp + popq %r15 + popq %r14 + popq %r13 + popq %r12 + popq %rbx + repz retq +.size Transform_Sha256_AVX2_Len,.-Transform_Sha256_AVX2_Len +L_avx2_rorx_sha256_k: +.long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 +.long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 +.long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 +.long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 +.long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 +.long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 +.long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 +.long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 +.long 0xe49b69c1,0xefbe4786,0xfc19dc6,0x240ca1cc +.long 0xe49b69c1,0xefbe4786,0xfc19dc6,0x240ca1cc +.long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da +.long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da +.long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 +.long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 +.long 0xc6e00bf3,0xd5a79147,0x6ca6351,0x14292967 +.long 0xc6e00bf3,0xd5a79147,0x6ca6351,0x14292967 +.long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 +.long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 +.long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 +.long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 +.long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 +.long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 +.long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 +.long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 +.long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 +.long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 +.long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 +.long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 +.long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 +.long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 +.long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 +.long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 +.align 32 +L_avx2_rorx_sha256_flip_mask: +.quad 0x405060700010203, 0xc0d0e0f08090a0b +.quad 0x405060700010203, 0xc0d0e0f08090a0b +.align 32 +L_avx2_rorx_sha256_shuf_00BA: +.quad 0xb0a090803020100, 0xffffffffffffffff +.quad 0xb0a090803020100, 0xffffffffffffffff +.align 32 +L_avx2_rorx_sha256_shuf_DC00: +.quad 0xffffffffffffffff, 0xb0a090803020100 +.quad 0xffffffffffffffff, 0xb0a090803020100 +.globl Transform_Sha256_AVX2_RORX +.type Transform_Sha256_AVX2_RORX,@function +.align 4 +Transform_Sha256_AVX2_RORX: + pushq %rbx + pushq %r12 + pushq %r13 + pushq %r14 + pushq %r15 + subq $0x200, %rsp + leaq 32(%rdi), %rax + vmovdqa L_avx2_rorx_sha256_flip_mask(%rip), %xmm13 + vmovdqa L_avx2_rorx_sha256_shuf_00BA(%rip), %ymm11 + vmovdqa L_avx2_rorx_sha256_shuf_DC00(%rip), %ymm12 + # X0, X1, X2, X3 = W[0..15] + vmovdqu (%rax), %xmm0 + vmovdqu 16(%rax), %xmm1 + vpshufb %xmm13, %xmm0, %xmm0 + vpshufb %xmm13, %xmm1, %xmm1 + vpaddd 0+L_avx2_rorx_sha256_k(%rip), %ymm0, %ymm4 + vpaddd 32+L_avx2_rorx_sha256_k(%rip), %ymm1, %ymm5 + vmovdqu %ymm4, (%rsp) + vmovdqu %ymm5, 32(%rsp) + vmovdqu 32(%rax), %xmm2 + vmovdqu 48(%rax), %xmm3 + vpshufb %xmm13, %xmm2, %xmm2 + vpshufb %xmm13, %xmm3, %xmm3 + vpaddd 64+L_avx2_rorx_sha256_k(%rip), %ymm2, %ymm4 + vpaddd 96+L_avx2_rorx_sha256_k(%rip), %ymm3, %ymm5 + vmovdqu %ymm4, 64(%rsp) + vmovdqu %ymm5, 96(%rsp) + movl (%rdi), %r8d + movl 4(%rdi), %r9d + movl 8(%rdi), %r10d + movl 12(%rdi), %r11d + movl 16(%rdi), %r12d + movl 20(%rdi), %r13d + movl 24(%rdi), %r14d + movl 28(%rdi), %r15d + movl %r9d, %ebx + rorxl $6, %r12d, %edx + xorl %r10d, %ebx + # rnd_0: 0 - 0 + movl %r13d, %eax + rorxl $11, %r12d, %ecx + addl (%rsp), %r15d + vpalignr $4, %ymm0, %ymm1, %ymm5 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r14d, %eax + rorxl $25, %r12d, %edx + vpalignr $4, %ymm2, %ymm3, %ymm4 + # rnd_0: 2 - 2 + andl %r12d, %eax + xorl %ecx, %edx + rorxl $13, %r8d, %ecx + vpsrld $7, %ymm5, %ymm6 + # rnd_0: 3 - 3 + addl %edx, %r15d + rorxl $2, %r8d, %edx + xorl %r14d, %eax + vpslld $25, %ymm5, %ymm7 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r8d, %edx + addl %eax, %r15d + vpsrld $18, %ymm5, %ymm8 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r9d, %eax + addl %r15d, %r11d + vpslld $14, %ymm5, %ymm9 + # rnd_0: 6 - 6 + xorl %r8d, %eax + addl %edx, %r15d + andl %eax, %ebx + vpor %ymm7, %ymm6, %ymm6 + # rnd_0: 7 - 7 + xorl %r9d, %ebx + rorxl $6, %r11d, %edx + addl %ebx, %r15d + vpor %ymm9, %ymm8, %ymm8 + # rnd_1: 0 - 0 + movl %r12d, %ebx + rorxl $11, %r11d, %ecx + addl 4(%rsp), %r14d + vpsrld $3, %ymm5, %ymm9 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r13d, %ebx + rorxl $25, %r11d, %edx + vpxor %ymm8, %ymm6, %ymm6 + # rnd_1: 2 - 2 + andl %r11d, %ebx + xorl %ecx, %edx + rorxl $13, %r15d, %ecx + vpshufd $0xfa, %ymm3, %ymm7 + # rnd_1: 3 - 3 + addl %edx, %r14d + rorxl $2, %r15d, %edx + xorl %r13d, %ebx + vpxor %ymm6, %ymm9, %ymm5 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r15d, %edx + addl %ebx, %r14d + vpsrld $10, %ymm7, %ymm8 + # rnd_1: 5 - 5 + xorl %ecx, %edx + addl %r14d, %r10d + movl %r8d, %ebx + vpsrlq $19, %ymm7, %ymm6 + # rnd_1: 6 - 6 + xorl %r15d, %ebx + addl %edx, %r14d + andl %ebx, %eax + vpsrlq $0x11, %ymm7, %ymm7 + # rnd_1: 7 - 7 + xorl %r8d, %eax + rorxl $6, %r10d, %edx + addl %eax, %r14d + vpaddd %ymm0, %ymm4, %ymm4 + # rnd_0: 0 - 0 + movl %r11d, %eax + rorxl $11, %r10d, %ecx + addl 8(%rsp), %r13d + vpxor %ymm7, %ymm6, %ymm6 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r12d, %eax + rorxl $25, %r10d, %edx + vpxor %ymm6, %ymm8, %ymm8 + # rnd_0: 2 - 2 + andl %r10d, %eax + xorl %ecx, %edx + rorxl $13, %r14d, %ecx + vpaddd %ymm5, %ymm4, %ymm4 + # rnd_0: 3 - 3 + addl %edx, %r13d + rorxl $2, %r14d, %edx + xorl %r12d, %eax + vpshufb %ymm11, %ymm8, %ymm8 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r14d, %edx + addl %eax, %r13d + vpaddd %ymm8, %ymm4, %ymm4 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r15d, %eax + addl %r13d, %r9d + vpshufd $0x50, %ymm4, %ymm6 + # rnd_0: 6 - 6 + xorl %r14d, %eax + addl %edx, %r13d + andl %eax, %ebx + vpsrlq $0x11, %ymm6, %ymm8 + # rnd_0: 7 - 7 + xorl %r15d, %ebx + rorxl $6, %r9d, %edx + addl %ebx, %r13d + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 0 - 0 + movl %r10d, %ebx + rorxl $11, %r9d, %ecx + addl 12(%rsp), %r12d + vpsrld $10, %ymm6, %ymm9 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r11d, %ebx + rorxl $25, %r9d, %edx + vpxor %ymm7, %ymm8, %ymm8 + # rnd_1: 2 - 2 + andl %r9d, %ebx + xorl %ecx, %edx + rorxl $13, %r13d, %ecx + vpxor %ymm8, %ymm9, %ymm9 + # rnd_1: 3 - 3 + addl %edx, %r12d + rorxl $2, %r13d, %edx + xorl %r11d, %ebx + vpshufb %ymm12, %ymm9, %ymm9 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r13d, %edx + addl %ebx, %r12d + vpaddd %ymm4, %ymm9, %ymm0 + # rnd_1: 5 - 5 + xorl %ecx, %edx + addl %r12d, %r8d + movl %r14d, %ebx + vpaddd 128+L_avx2_rorx_sha256_k(%rip), %ymm0, %ymm4 + # rnd_1: 6 - 6 + xorl %r13d, %ebx + addl %edx, %r12d + andl %ebx, %eax + # rnd_1: 7 - 7 + xorl %r14d, %eax + rorxl $6, %r8d, %edx + addl %eax, %r12d + vmovdqu %ymm4, 128(%rsp) + # rnd_0: 0 - 0 + movl %r9d, %eax + rorxl $11, %r8d, %ecx + addl 32(%rsp), %r11d + vpalignr $4, %ymm1, %ymm2, %ymm5 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r10d, %eax + rorxl $25, %r8d, %edx + vpalignr $4, %ymm3, %ymm0, %ymm4 + # rnd_0: 2 - 2 + andl %r8d, %eax + xorl %ecx, %edx + rorxl $13, %r12d, %ecx + vpsrld $7, %ymm5, %ymm6 + # rnd_0: 3 - 3 + addl %edx, %r11d + rorxl $2, %r12d, %edx + xorl %r10d, %eax + vpslld $25, %ymm5, %ymm7 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r12d, %edx + addl %eax, %r11d + vpsrld $18, %ymm5, %ymm8 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r13d, %eax + addl %r11d, %r15d + vpslld $14, %ymm5, %ymm9 + # rnd_0: 6 - 6 + xorl %r12d, %eax + addl %edx, %r11d + andl %eax, %ebx + vpor %ymm7, %ymm6, %ymm6 + # rnd_0: 7 - 7 + xorl %r13d, %ebx + rorxl $6, %r15d, %edx + addl %ebx, %r11d + vpor %ymm9, %ymm8, %ymm8 + # rnd_1: 0 - 0 + movl %r8d, %ebx + rorxl $11, %r15d, %ecx + addl 36(%rsp), %r10d + vpsrld $3, %ymm5, %ymm9 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r9d, %ebx + rorxl $25, %r15d, %edx + vpxor %ymm8, %ymm6, %ymm6 + # rnd_1: 2 - 2 + andl %r15d, %ebx + xorl %ecx, %edx + rorxl $13, %r11d, %ecx + vpshufd $0xfa, %ymm0, %ymm7 + # rnd_1: 3 - 3 + addl %edx, %r10d + rorxl $2, %r11d, %edx + xorl %r9d, %ebx + vpxor %ymm6, %ymm9, %ymm5 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r11d, %edx + addl %ebx, %r10d + vpsrld $10, %ymm7, %ymm8 + # rnd_1: 5 - 5 + xorl %ecx, %edx + addl %r10d, %r14d + movl %r12d, %ebx + vpsrlq $19, %ymm7, %ymm6 + # rnd_1: 6 - 6 + xorl %r11d, %ebx + addl %edx, %r10d + andl %ebx, %eax + vpsrlq $0x11, %ymm7, %ymm7 + # rnd_1: 7 - 7 + xorl %r12d, %eax + rorxl $6, %r14d, %edx + addl %eax, %r10d + vpaddd %ymm1, %ymm4, %ymm4 + # rnd_0: 0 - 0 + movl %r15d, %eax + rorxl $11, %r14d, %ecx + addl 40(%rsp), %r9d + vpxor %ymm7, %ymm6, %ymm6 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r8d, %eax + rorxl $25, %r14d, %edx + vpxor %ymm6, %ymm8, %ymm8 + # rnd_0: 2 - 2 + andl %r14d, %eax + xorl %ecx, %edx + rorxl $13, %r10d, %ecx + vpaddd %ymm5, %ymm4, %ymm4 + # rnd_0: 3 - 3 + addl %edx, %r9d + rorxl $2, %r10d, %edx + xorl %r8d, %eax + vpshufb %ymm11, %ymm8, %ymm8 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r10d, %edx + addl %eax, %r9d + vpaddd %ymm8, %ymm4, %ymm4 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r11d, %eax + addl %r9d, %r13d + vpshufd $0x50, %ymm4, %ymm6 + # rnd_0: 6 - 6 + xorl %r10d, %eax + addl %edx, %r9d + andl %eax, %ebx + vpsrlq $0x11, %ymm6, %ymm8 + # rnd_0: 7 - 7 + xorl %r11d, %ebx + rorxl $6, %r13d, %edx + addl %ebx, %r9d + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 0 - 0 + movl %r14d, %ebx + rorxl $11, %r13d, %ecx + addl 44(%rsp), %r8d + vpsrld $10, %ymm6, %ymm9 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r15d, %ebx + rorxl $25, %r13d, %edx + vpxor %ymm7, %ymm8, %ymm8 + # rnd_1: 2 - 2 + andl %r13d, %ebx + xorl %ecx, %edx + rorxl $13, %r9d, %ecx + vpxor %ymm8, %ymm9, %ymm9 + # rnd_1: 3 - 3 + addl %edx, %r8d + rorxl $2, %r9d, %edx + xorl %r15d, %ebx + vpshufb %ymm12, %ymm9, %ymm9 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r9d, %edx + addl %ebx, %r8d + vpaddd %ymm4, %ymm9, %ymm1 + # rnd_1: 5 - 5 + xorl %ecx, %edx + addl %r8d, %r12d + movl %r10d, %ebx + vpaddd 160+L_avx2_rorx_sha256_k(%rip), %ymm1, %ymm4 + # rnd_1: 6 - 6 + xorl %r9d, %ebx + addl %edx, %r8d + andl %ebx, %eax + # rnd_1: 7 - 7 + xorl %r10d, %eax + rorxl $6, %r12d, %edx + addl %eax, %r8d + vmovdqu %ymm4, 160(%rsp) + # rnd_0: 0 - 0 + movl %r13d, %eax + rorxl $11, %r12d, %ecx + addl 64(%rsp), %r15d + vpalignr $4, %ymm2, %ymm3, %ymm5 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r14d, %eax + rorxl $25, %r12d, %edx + vpalignr $4, %ymm0, %ymm1, %ymm4 + # rnd_0: 2 - 2 + andl %r12d, %eax + xorl %ecx, %edx + rorxl $13, %r8d, %ecx + vpsrld $7, %ymm5, %ymm6 + # rnd_0: 3 - 3 + addl %edx, %r15d + rorxl $2, %r8d, %edx + xorl %r14d, %eax + vpslld $25, %ymm5, %ymm7 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r8d, %edx + addl %eax, %r15d + vpsrld $18, %ymm5, %ymm8 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r9d, %eax + addl %r15d, %r11d + vpslld $14, %ymm5, %ymm9 + # rnd_0: 6 - 6 + xorl %r8d, %eax + addl %edx, %r15d + andl %eax, %ebx + vpor %ymm7, %ymm6, %ymm6 + # rnd_0: 7 - 7 + xorl %r9d, %ebx + rorxl $6, %r11d, %edx + addl %ebx, %r15d + vpor %ymm9, %ymm8, %ymm8 + # rnd_1: 0 - 0 + movl %r12d, %ebx + rorxl $11, %r11d, %ecx + addl 68(%rsp), %r14d + vpsrld $3, %ymm5, %ymm9 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r13d, %ebx + rorxl $25, %r11d, %edx + vpxor %ymm8, %ymm6, %ymm6 + # rnd_1: 2 - 2 + andl %r11d, %ebx + xorl %ecx, %edx + rorxl $13, %r15d, %ecx + vpshufd $0xfa, %ymm1, %ymm7 + # rnd_1: 3 - 3 + addl %edx, %r14d + rorxl $2, %r15d, %edx + xorl %r13d, %ebx + vpxor %ymm6, %ymm9, %ymm5 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r15d, %edx + addl %ebx, %r14d + vpsrld $10, %ymm7, %ymm8 + # rnd_1: 5 - 5 + xorl %ecx, %edx + addl %r14d, %r10d + movl %r8d, %ebx + vpsrlq $19, %ymm7, %ymm6 + # rnd_1: 6 - 6 + xorl %r15d, %ebx + addl %edx, %r14d + andl %ebx, %eax + vpsrlq $0x11, %ymm7, %ymm7 + # rnd_1: 7 - 7 + xorl %r8d, %eax + rorxl $6, %r10d, %edx + addl %eax, %r14d + vpaddd %ymm2, %ymm4, %ymm4 + # rnd_0: 0 - 0 + movl %r11d, %eax + rorxl $11, %r10d, %ecx + addl 72(%rsp), %r13d + vpxor %ymm7, %ymm6, %ymm6 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r12d, %eax + rorxl $25, %r10d, %edx + vpxor %ymm6, %ymm8, %ymm8 + # rnd_0: 2 - 2 + andl %r10d, %eax + xorl %ecx, %edx + rorxl $13, %r14d, %ecx + vpaddd %ymm5, %ymm4, %ymm4 + # rnd_0: 3 - 3 + addl %edx, %r13d + rorxl $2, %r14d, %edx + xorl %r12d, %eax + vpshufb %ymm11, %ymm8, %ymm8 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r14d, %edx + addl %eax, %r13d + vpaddd %ymm8, %ymm4, %ymm4 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r15d, %eax + addl %r13d, %r9d + vpshufd $0x50, %ymm4, %ymm6 + # rnd_0: 6 - 6 + xorl %r14d, %eax + addl %edx, %r13d + andl %eax, %ebx + vpsrlq $0x11, %ymm6, %ymm8 + # rnd_0: 7 - 7 + xorl %r15d, %ebx + rorxl $6, %r9d, %edx + addl %ebx, %r13d + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 0 - 0 + movl %r10d, %ebx + rorxl $11, %r9d, %ecx + addl 76(%rsp), %r12d + vpsrld $10, %ymm6, %ymm9 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r11d, %ebx + rorxl $25, %r9d, %edx + vpxor %ymm7, %ymm8, %ymm8 + # rnd_1: 2 - 2 + andl %r9d, %ebx + xorl %ecx, %edx + rorxl $13, %r13d, %ecx + vpxor %ymm8, %ymm9, %ymm9 + # rnd_1: 3 - 3 + addl %edx, %r12d + rorxl $2, %r13d, %edx + xorl %r11d, %ebx + vpshufb %ymm12, %ymm9, %ymm9 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r13d, %edx + addl %ebx, %r12d + vpaddd %ymm4, %ymm9, %ymm2 + # rnd_1: 5 - 5 + xorl %ecx, %edx + addl %r12d, %r8d + movl %r14d, %ebx + vpaddd 192+L_avx2_rorx_sha256_k(%rip), %ymm2, %ymm4 + # rnd_1: 6 - 6 + xorl %r13d, %ebx + addl %edx, %r12d + andl %ebx, %eax + # rnd_1: 7 - 7 + xorl %r14d, %eax + rorxl $6, %r8d, %edx + addl %eax, %r12d + vmovdqu %ymm4, 192(%rsp) + # rnd_0: 0 - 0 + movl %r9d, %eax + rorxl $11, %r8d, %ecx + addl 96(%rsp), %r11d + vpalignr $4, %ymm3, %ymm0, %ymm5 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r10d, %eax + rorxl $25, %r8d, %edx + vpalignr $4, %ymm1, %ymm2, %ymm4 + # rnd_0: 2 - 2 + andl %r8d, %eax + xorl %ecx, %edx + rorxl $13, %r12d, %ecx + vpsrld $7, %ymm5, %ymm6 + # rnd_0: 3 - 3 + addl %edx, %r11d + rorxl $2, %r12d, %edx + xorl %r10d, %eax + vpslld $25, %ymm5, %ymm7 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r12d, %edx + addl %eax, %r11d + vpsrld $18, %ymm5, %ymm8 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r13d, %eax + addl %r11d, %r15d + vpslld $14, %ymm5, %ymm9 + # rnd_0: 6 - 6 + xorl %r12d, %eax + addl %edx, %r11d + andl %eax, %ebx + vpor %ymm7, %ymm6, %ymm6 + # rnd_0: 7 - 7 + xorl %r13d, %ebx + rorxl $6, %r15d, %edx + addl %ebx, %r11d + vpor %ymm9, %ymm8, %ymm8 + # rnd_1: 0 - 0 + movl %r8d, %ebx + rorxl $11, %r15d, %ecx + addl 100(%rsp), %r10d + vpsrld $3, %ymm5, %ymm9 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r9d, %ebx + rorxl $25, %r15d, %edx + vpxor %ymm8, %ymm6, %ymm6 + # rnd_1: 2 - 2 + andl %r15d, %ebx + xorl %ecx, %edx + rorxl $13, %r11d, %ecx + vpshufd $0xfa, %ymm2, %ymm7 + # rnd_1: 3 - 3 + addl %edx, %r10d + rorxl $2, %r11d, %edx + xorl %r9d, %ebx + vpxor %ymm6, %ymm9, %ymm5 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r11d, %edx + addl %ebx, %r10d + vpsrld $10, %ymm7, %ymm8 + # rnd_1: 5 - 5 + xorl %ecx, %edx + addl %r10d, %r14d + movl %r12d, %ebx + vpsrlq $19, %ymm7, %ymm6 + # rnd_1: 6 - 6 + xorl %r11d, %ebx + addl %edx, %r10d + andl %ebx, %eax + vpsrlq $0x11, %ymm7, %ymm7 + # rnd_1: 7 - 7 + xorl %r12d, %eax + rorxl $6, %r14d, %edx + addl %eax, %r10d + vpaddd %ymm3, %ymm4, %ymm4 + # rnd_0: 0 - 0 + movl %r15d, %eax + rorxl $11, %r14d, %ecx + addl 104(%rsp), %r9d + vpxor %ymm7, %ymm6, %ymm6 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r8d, %eax + rorxl $25, %r14d, %edx + vpxor %ymm6, %ymm8, %ymm8 + # rnd_0: 2 - 2 + andl %r14d, %eax + xorl %ecx, %edx + rorxl $13, %r10d, %ecx + vpaddd %ymm5, %ymm4, %ymm4 + # rnd_0: 3 - 3 + addl %edx, %r9d + rorxl $2, %r10d, %edx + xorl %r8d, %eax + vpshufb %ymm11, %ymm8, %ymm8 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r10d, %edx + addl %eax, %r9d + vpaddd %ymm8, %ymm4, %ymm4 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r11d, %eax + addl %r9d, %r13d + vpshufd $0x50, %ymm4, %ymm6 + # rnd_0: 6 - 6 + xorl %r10d, %eax + addl %edx, %r9d + andl %eax, %ebx + vpsrlq $0x11, %ymm6, %ymm8 + # rnd_0: 7 - 7 + xorl %r11d, %ebx + rorxl $6, %r13d, %edx + addl %ebx, %r9d + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 0 - 0 + movl %r14d, %ebx + rorxl $11, %r13d, %ecx + addl 108(%rsp), %r8d + vpsrld $10, %ymm6, %ymm9 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r15d, %ebx + rorxl $25, %r13d, %edx + vpxor %ymm7, %ymm8, %ymm8 + # rnd_1: 2 - 2 + andl %r13d, %ebx + xorl %ecx, %edx + rorxl $13, %r9d, %ecx + vpxor %ymm8, %ymm9, %ymm9 + # rnd_1: 3 - 3 + addl %edx, %r8d + rorxl $2, %r9d, %edx + xorl %r15d, %ebx + vpshufb %ymm12, %ymm9, %ymm9 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r9d, %edx + addl %ebx, %r8d + vpaddd %ymm4, %ymm9, %ymm3 + # rnd_1: 5 - 5 + xorl %ecx, %edx + addl %r8d, %r12d + movl %r10d, %ebx + vpaddd 224+L_avx2_rorx_sha256_k(%rip), %ymm3, %ymm4 + # rnd_1: 6 - 6 + xorl %r9d, %ebx + addl %edx, %r8d + andl %ebx, %eax + # rnd_1: 7 - 7 + xorl %r10d, %eax + rorxl $6, %r12d, %edx + addl %eax, %r8d + vmovdqu %ymm4, 224(%rsp) + # rnd_0: 0 - 0 + movl %r13d, %eax + rorxl $11, %r12d, %ecx + addl 128(%rsp), %r15d + vpalignr $4, %ymm0, %ymm1, %ymm5 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r14d, %eax + rorxl $25, %r12d, %edx + vpalignr $4, %ymm2, %ymm3, %ymm4 + # rnd_0: 2 - 2 + andl %r12d, %eax + xorl %ecx, %edx + rorxl $13, %r8d, %ecx + vpsrld $7, %ymm5, %ymm6 + # rnd_0: 3 - 3 + addl %edx, %r15d + rorxl $2, %r8d, %edx + xorl %r14d, %eax + vpslld $25, %ymm5, %ymm7 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r8d, %edx + addl %eax, %r15d + vpsrld $18, %ymm5, %ymm8 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r9d, %eax + addl %r15d, %r11d + vpslld $14, %ymm5, %ymm9 + # rnd_0: 6 - 6 + xorl %r8d, %eax + addl %edx, %r15d + andl %eax, %ebx + vpor %ymm7, %ymm6, %ymm6 + # rnd_0: 7 - 7 + xorl %r9d, %ebx + rorxl $6, %r11d, %edx + addl %ebx, %r15d + vpor %ymm9, %ymm8, %ymm8 + # rnd_1: 0 - 0 + movl %r12d, %ebx + rorxl $11, %r11d, %ecx + addl 132(%rsp), %r14d + vpsrld $3, %ymm5, %ymm9 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r13d, %ebx + rorxl $25, %r11d, %edx + vpxor %ymm8, %ymm6, %ymm6 + # rnd_1: 2 - 2 + andl %r11d, %ebx + xorl %ecx, %edx + rorxl $13, %r15d, %ecx + vpshufd $0xfa, %ymm3, %ymm7 + # rnd_1: 3 - 3 + addl %edx, %r14d + rorxl $2, %r15d, %edx + xorl %r13d, %ebx + vpxor %ymm6, %ymm9, %ymm5 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r15d, %edx + addl %ebx, %r14d + vpsrld $10, %ymm7, %ymm8 + # rnd_1: 5 - 5 + xorl %ecx, %edx + addl %r14d, %r10d + movl %r8d, %ebx + vpsrlq $19, %ymm7, %ymm6 + # rnd_1: 6 - 6 + xorl %r15d, %ebx + addl %edx, %r14d + andl %ebx, %eax + vpsrlq $0x11, %ymm7, %ymm7 + # rnd_1: 7 - 7 + xorl %r8d, %eax + rorxl $6, %r10d, %edx + addl %eax, %r14d + vpaddd %ymm0, %ymm4, %ymm4 + # rnd_0: 0 - 0 + movl %r11d, %eax + rorxl $11, %r10d, %ecx + addl 136(%rsp), %r13d + vpxor %ymm7, %ymm6, %ymm6 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r12d, %eax + rorxl $25, %r10d, %edx + vpxor %ymm6, %ymm8, %ymm8 + # rnd_0: 2 - 2 + andl %r10d, %eax + xorl %ecx, %edx + rorxl $13, %r14d, %ecx + vpaddd %ymm5, %ymm4, %ymm4 + # rnd_0: 3 - 3 + addl %edx, %r13d + rorxl $2, %r14d, %edx + xorl %r12d, %eax + vpshufb %ymm11, %ymm8, %ymm8 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r14d, %edx + addl %eax, %r13d + vpaddd %ymm8, %ymm4, %ymm4 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r15d, %eax + addl %r13d, %r9d + vpshufd $0x50, %ymm4, %ymm6 + # rnd_0: 6 - 6 + xorl %r14d, %eax + addl %edx, %r13d + andl %eax, %ebx + vpsrlq $0x11, %ymm6, %ymm8 + # rnd_0: 7 - 7 + xorl %r15d, %ebx + rorxl $6, %r9d, %edx + addl %ebx, %r13d + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 0 - 0 + movl %r10d, %ebx + rorxl $11, %r9d, %ecx + addl 140(%rsp), %r12d + vpsrld $10, %ymm6, %ymm9 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r11d, %ebx + rorxl $25, %r9d, %edx + vpxor %ymm7, %ymm8, %ymm8 + # rnd_1: 2 - 2 + andl %r9d, %ebx + xorl %ecx, %edx + rorxl $13, %r13d, %ecx + vpxor %ymm8, %ymm9, %ymm9 + # rnd_1: 3 - 3 + addl %edx, %r12d + rorxl $2, %r13d, %edx + xorl %r11d, %ebx + vpshufb %ymm12, %ymm9, %ymm9 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r13d, %edx + addl %ebx, %r12d + vpaddd %ymm4, %ymm9, %ymm0 + # rnd_1: 5 - 5 + xorl %ecx, %edx + addl %r12d, %r8d + movl %r14d, %ebx + vpaddd 256+L_avx2_rorx_sha256_k(%rip), %ymm0, %ymm4 + # rnd_1: 6 - 6 + xorl %r13d, %ebx + addl %edx, %r12d + andl %ebx, %eax + # rnd_1: 7 - 7 + xorl %r14d, %eax + rorxl $6, %r8d, %edx + addl %eax, %r12d + vmovdqu %ymm4, 256(%rsp) + # rnd_0: 0 - 0 + movl %r9d, %eax + rorxl $11, %r8d, %ecx + addl 160(%rsp), %r11d + vpalignr $4, %ymm1, %ymm2, %ymm5 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r10d, %eax + rorxl $25, %r8d, %edx + vpalignr $4, %ymm3, %ymm0, %ymm4 + # rnd_0: 2 - 2 + andl %r8d, %eax + xorl %ecx, %edx + rorxl $13, %r12d, %ecx + vpsrld $7, %ymm5, %ymm6 + # rnd_0: 3 - 3 + addl %edx, %r11d + rorxl $2, %r12d, %edx + xorl %r10d, %eax + vpslld $25, %ymm5, %ymm7 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r12d, %edx + addl %eax, %r11d + vpsrld $18, %ymm5, %ymm8 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r13d, %eax + addl %r11d, %r15d + vpslld $14, %ymm5, %ymm9 + # rnd_0: 6 - 6 + xorl %r12d, %eax + addl %edx, %r11d + andl %eax, %ebx + vpor %ymm7, %ymm6, %ymm6 + # rnd_0: 7 - 7 + xorl %r13d, %ebx + rorxl $6, %r15d, %edx + addl %ebx, %r11d + vpor %ymm9, %ymm8, %ymm8 + # rnd_1: 0 - 0 + movl %r8d, %ebx + rorxl $11, %r15d, %ecx + addl 164(%rsp), %r10d + vpsrld $3, %ymm5, %ymm9 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r9d, %ebx + rorxl $25, %r15d, %edx + vpxor %ymm8, %ymm6, %ymm6 + # rnd_1: 2 - 2 + andl %r15d, %ebx + xorl %ecx, %edx + rorxl $13, %r11d, %ecx + vpshufd $0xfa, %ymm0, %ymm7 + # rnd_1: 3 - 3 + addl %edx, %r10d + rorxl $2, %r11d, %edx + xorl %r9d, %ebx + vpxor %ymm6, %ymm9, %ymm5 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r11d, %edx + addl %ebx, %r10d + vpsrld $10, %ymm7, %ymm8 + # rnd_1: 5 - 5 + xorl %ecx, %edx + addl %r10d, %r14d + movl %r12d, %ebx + vpsrlq $19, %ymm7, %ymm6 + # rnd_1: 6 - 6 + xorl %r11d, %ebx + addl %edx, %r10d + andl %ebx, %eax + vpsrlq $0x11, %ymm7, %ymm7 + # rnd_1: 7 - 7 + xorl %r12d, %eax + rorxl $6, %r14d, %edx + addl %eax, %r10d + vpaddd %ymm1, %ymm4, %ymm4 + # rnd_0: 0 - 0 + movl %r15d, %eax + rorxl $11, %r14d, %ecx + addl 168(%rsp), %r9d + vpxor %ymm7, %ymm6, %ymm6 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r8d, %eax + rorxl $25, %r14d, %edx + vpxor %ymm6, %ymm8, %ymm8 + # rnd_0: 2 - 2 + andl %r14d, %eax + xorl %ecx, %edx + rorxl $13, %r10d, %ecx + vpaddd %ymm5, %ymm4, %ymm4 + # rnd_0: 3 - 3 + addl %edx, %r9d + rorxl $2, %r10d, %edx + xorl %r8d, %eax + vpshufb %ymm11, %ymm8, %ymm8 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r10d, %edx + addl %eax, %r9d + vpaddd %ymm8, %ymm4, %ymm4 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r11d, %eax + addl %r9d, %r13d + vpshufd $0x50, %ymm4, %ymm6 + # rnd_0: 6 - 6 + xorl %r10d, %eax + addl %edx, %r9d + andl %eax, %ebx + vpsrlq $0x11, %ymm6, %ymm8 + # rnd_0: 7 - 7 + xorl %r11d, %ebx + rorxl $6, %r13d, %edx + addl %ebx, %r9d + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 0 - 0 + movl %r14d, %ebx + rorxl $11, %r13d, %ecx + addl 172(%rsp), %r8d + vpsrld $10, %ymm6, %ymm9 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r15d, %ebx + rorxl $25, %r13d, %edx + vpxor %ymm7, %ymm8, %ymm8 + # rnd_1: 2 - 2 + andl %r13d, %ebx + xorl %ecx, %edx + rorxl $13, %r9d, %ecx + vpxor %ymm8, %ymm9, %ymm9 + # rnd_1: 3 - 3 + addl %edx, %r8d + rorxl $2, %r9d, %edx + xorl %r15d, %ebx + vpshufb %ymm12, %ymm9, %ymm9 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r9d, %edx + addl %ebx, %r8d + vpaddd %ymm4, %ymm9, %ymm1 + # rnd_1: 5 - 5 + xorl %ecx, %edx + addl %r8d, %r12d + movl %r10d, %ebx + vpaddd 288+L_avx2_rorx_sha256_k(%rip), %ymm1, %ymm4 + # rnd_1: 6 - 6 + xorl %r9d, %ebx + addl %edx, %r8d + andl %ebx, %eax + # rnd_1: 7 - 7 + xorl %r10d, %eax + rorxl $6, %r12d, %edx + addl %eax, %r8d + vmovdqu %ymm4, 288(%rsp) + # rnd_0: 0 - 0 + movl %r13d, %eax + rorxl $11, %r12d, %ecx + addl 192(%rsp), %r15d + vpalignr $4, %ymm2, %ymm3, %ymm5 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r14d, %eax + rorxl $25, %r12d, %edx + vpalignr $4, %ymm0, %ymm1, %ymm4 + # rnd_0: 2 - 2 + andl %r12d, %eax + xorl %ecx, %edx + rorxl $13, %r8d, %ecx + vpsrld $7, %ymm5, %ymm6 + # rnd_0: 3 - 3 + addl %edx, %r15d + rorxl $2, %r8d, %edx + xorl %r14d, %eax + vpslld $25, %ymm5, %ymm7 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r8d, %edx + addl %eax, %r15d + vpsrld $18, %ymm5, %ymm8 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r9d, %eax + addl %r15d, %r11d + vpslld $14, %ymm5, %ymm9 + # rnd_0: 6 - 6 + xorl %r8d, %eax + addl %edx, %r15d + andl %eax, %ebx + vpor %ymm7, %ymm6, %ymm6 + # rnd_0: 7 - 7 + xorl %r9d, %ebx + rorxl $6, %r11d, %edx + addl %ebx, %r15d + vpor %ymm9, %ymm8, %ymm8 + # rnd_1: 0 - 0 + movl %r12d, %ebx + rorxl $11, %r11d, %ecx + addl 196(%rsp), %r14d + vpsrld $3, %ymm5, %ymm9 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r13d, %ebx + rorxl $25, %r11d, %edx + vpxor %ymm8, %ymm6, %ymm6 + # rnd_1: 2 - 2 + andl %r11d, %ebx + xorl %ecx, %edx + rorxl $13, %r15d, %ecx + vpshufd $0xfa, %ymm1, %ymm7 + # rnd_1: 3 - 3 + addl %edx, %r14d + rorxl $2, %r15d, %edx + xorl %r13d, %ebx + vpxor %ymm6, %ymm9, %ymm5 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r15d, %edx + addl %ebx, %r14d + vpsrld $10, %ymm7, %ymm8 + # rnd_1: 5 - 5 + xorl %ecx, %edx + addl %r14d, %r10d + movl %r8d, %ebx + vpsrlq $19, %ymm7, %ymm6 + # rnd_1: 6 - 6 + xorl %r15d, %ebx + addl %edx, %r14d + andl %ebx, %eax + vpsrlq $0x11, %ymm7, %ymm7 + # rnd_1: 7 - 7 + xorl %r8d, %eax + rorxl $6, %r10d, %edx + addl %eax, %r14d + vpaddd %ymm2, %ymm4, %ymm4 + # rnd_0: 0 - 0 + movl %r11d, %eax + rorxl $11, %r10d, %ecx + addl 200(%rsp), %r13d + vpxor %ymm7, %ymm6, %ymm6 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r12d, %eax + rorxl $25, %r10d, %edx + vpxor %ymm6, %ymm8, %ymm8 + # rnd_0: 2 - 2 + andl %r10d, %eax + xorl %ecx, %edx + rorxl $13, %r14d, %ecx + vpaddd %ymm5, %ymm4, %ymm4 + # rnd_0: 3 - 3 + addl %edx, %r13d + rorxl $2, %r14d, %edx + xorl %r12d, %eax + vpshufb %ymm11, %ymm8, %ymm8 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r14d, %edx + addl %eax, %r13d + vpaddd %ymm8, %ymm4, %ymm4 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r15d, %eax + addl %r13d, %r9d + vpshufd $0x50, %ymm4, %ymm6 + # rnd_0: 6 - 6 + xorl %r14d, %eax + addl %edx, %r13d + andl %eax, %ebx + vpsrlq $0x11, %ymm6, %ymm8 + # rnd_0: 7 - 7 + xorl %r15d, %ebx + rorxl $6, %r9d, %edx + addl %ebx, %r13d + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 0 - 0 + movl %r10d, %ebx + rorxl $11, %r9d, %ecx + addl 204(%rsp), %r12d + vpsrld $10, %ymm6, %ymm9 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r11d, %ebx + rorxl $25, %r9d, %edx + vpxor %ymm7, %ymm8, %ymm8 + # rnd_1: 2 - 2 + andl %r9d, %ebx + xorl %ecx, %edx + rorxl $13, %r13d, %ecx + vpxor %ymm8, %ymm9, %ymm9 + # rnd_1: 3 - 3 + addl %edx, %r12d + rorxl $2, %r13d, %edx + xorl %r11d, %ebx + vpshufb %ymm12, %ymm9, %ymm9 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r13d, %edx + addl %ebx, %r12d + vpaddd %ymm4, %ymm9, %ymm2 + # rnd_1: 5 - 5 + xorl %ecx, %edx + addl %r12d, %r8d + movl %r14d, %ebx + vpaddd 320+L_avx2_rorx_sha256_k(%rip), %ymm2, %ymm4 + # rnd_1: 6 - 6 + xorl %r13d, %ebx + addl %edx, %r12d + andl %ebx, %eax + # rnd_1: 7 - 7 + xorl %r14d, %eax + rorxl $6, %r8d, %edx + addl %eax, %r12d + vmovdqu %ymm4, 320(%rsp) + # rnd_0: 0 - 0 + movl %r9d, %eax + rorxl $11, %r8d, %ecx + addl 224(%rsp), %r11d + vpalignr $4, %ymm3, %ymm0, %ymm5 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r10d, %eax + rorxl $25, %r8d, %edx + vpalignr $4, %ymm1, %ymm2, %ymm4 + # rnd_0: 2 - 2 + andl %r8d, %eax + xorl %ecx, %edx + rorxl $13, %r12d, %ecx + vpsrld $7, %ymm5, %ymm6 + # rnd_0: 3 - 3 + addl %edx, %r11d + rorxl $2, %r12d, %edx + xorl %r10d, %eax + vpslld $25, %ymm5, %ymm7 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r12d, %edx + addl %eax, %r11d + vpsrld $18, %ymm5, %ymm8 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r13d, %eax + addl %r11d, %r15d + vpslld $14, %ymm5, %ymm9 + # rnd_0: 6 - 6 + xorl %r12d, %eax + addl %edx, %r11d + andl %eax, %ebx + vpor %ymm7, %ymm6, %ymm6 + # rnd_0: 7 - 7 + xorl %r13d, %ebx + rorxl $6, %r15d, %edx + addl %ebx, %r11d + vpor %ymm9, %ymm8, %ymm8 + # rnd_1: 0 - 0 + movl %r8d, %ebx + rorxl $11, %r15d, %ecx + addl 228(%rsp), %r10d + vpsrld $3, %ymm5, %ymm9 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r9d, %ebx + rorxl $25, %r15d, %edx + vpxor %ymm8, %ymm6, %ymm6 + # rnd_1: 2 - 2 + andl %r15d, %ebx + xorl %ecx, %edx + rorxl $13, %r11d, %ecx + vpshufd $0xfa, %ymm2, %ymm7 + # rnd_1: 3 - 3 + addl %edx, %r10d + rorxl $2, %r11d, %edx + xorl %r9d, %ebx + vpxor %ymm6, %ymm9, %ymm5 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r11d, %edx + addl %ebx, %r10d + vpsrld $10, %ymm7, %ymm8 + # rnd_1: 5 - 5 + xorl %ecx, %edx + addl %r10d, %r14d + movl %r12d, %ebx + vpsrlq $19, %ymm7, %ymm6 + # rnd_1: 6 - 6 + xorl %r11d, %ebx + addl %edx, %r10d + andl %ebx, %eax + vpsrlq $0x11, %ymm7, %ymm7 + # rnd_1: 7 - 7 + xorl %r12d, %eax + rorxl $6, %r14d, %edx + addl %eax, %r10d + vpaddd %ymm3, %ymm4, %ymm4 + # rnd_0: 0 - 0 + movl %r15d, %eax + rorxl $11, %r14d, %ecx + addl 232(%rsp), %r9d + vpxor %ymm7, %ymm6, %ymm6 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r8d, %eax + rorxl $25, %r14d, %edx + vpxor %ymm6, %ymm8, %ymm8 + # rnd_0: 2 - 2 + andl %r14d, %eax + xorl %ecx, %edx + rorxl $13, %r10d, %ecx + vpaddd %ymm5, %ymm4, %ymm4 + # rnd_0: 3 - 3 + addl %edx, %r9d + rorxl $2, %r10d, %edx + xorl %r8d, %eax + vpshufb %ymm11, %ymm8, %ymm8 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r10d, %edx + addl %eax, %r9d + vpaddd %ymm8, %ymm4, %ymm4 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r11d, %eax + addl %r9d, %r13d + vpshufd $0x50, %ymm4, %ymm6 + # rnd_0: 6 - 6 + xorl %r10d, %eax + addl %edx, %r9d + andl %eax, %ebx + vpsrlq $0x11, %ymm6, %ymm8 + # rnd_0: 7 - 7 + xorl %r11d, %ebx + rorxl $6, %r13d, %edx + addl %ebx, %r9d + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 0 - 0 + movl %r14d, %ebx + rorxl $11, %r13d, %ecx + addl 236(%rsp), %r8d + vpsrld $10, %ymm6, %ymm9 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r15d, %ebx + rorxl $25, %r13d, %edx + vpxor %ymm7, %ymm8, %ymm8 + # rnd_1: 2 - 2 + andl %r13d, %ebx + xorl %ecx, %edx + rorxl $13, %r9d, %ecx + vpxor %ymm8, %ymm9, %ymm9 + # rnd_1: 3 - 3 + addl %edx, %r8d + rorxl $2, %r9d, %edx + xorl %r15d, %ebx + vpshufb %ymm12, %ymm9, %ymm9 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r9d, %edx + addl %ebx, %r8d + vpaddd %ymm4, %ymm9, %ymm3 + # rnd_1: 5 - 5 + xorl %ecx, %edx + addl %r8d, %r12d + movl %r10d, %ebx + vpaddd 352+L_avx2_rorx_sha256_k(%rip), %ymm3, %ymm4 + # rnd_1: 6 - 6 + xorl %r9d, %ebx + addl %edx, %r8d + andl %ebx, %eax + # rnd_1: 7 - 7 + xorl %r10d, %eax + rorxl $6, %r12d, %edx + addl %eax, %r8d + vmovdqu %ymm4, 352(%rsp) + # rnd_0: 0 - 0 + movl %r13d, %eax + rorxl $11, %r12d, %ecx + addl 256(%rsp), %r15d + vpalignr $4, %ymm0, %ymm1, %ymm5 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r14d, %eax + rorxl $25, %r12d, %edx + vpalignr $4, %ymm2, %ymm3, %ymm4 + # rnd_0: 2 - 2 + andl %r12d, %eax + xorl %ecx, %edx + rorxl $13, %r8d, %ecx + vpsrld $7, %ymm5, %ymm6 + # rnd_0: 3 - 3 + addl %edx, %r15d + rorxl $2, %r8d, %edx + xorl %r14d, %eax + vpslld $25, %ymm5, %ymm7 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r8d, %edx + addl %eax, %r15d + vpsrld $18, %ymm5, %ymm8 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r9d, %eax + addl %r15d, %r11d + vpslld $14, %ymm5, %ymm9 + # rnd_0: 6 - 6 + xorl %r8d, %eax + addl %edx, %r15d + andl %eax, %ebx + vpor %ymm7, %ymm6, %ymm6 + # rnd_0: 7 - 7 + xorl %r9d, %ebx + rorxl $6, %r11d, %edx + addl %ebx, %r15d + vpor %ymm9, %ymm8, %ymm8 + # rnd_1: 0 - 0 + movl %r12d, %ebx + rorxl $11, %r11d, %ecx + addl 260(%rsp), %r14d + vpsrld $3, %ymm5, %ymm9 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r13d, %ebx + rorxl $25, %r11d, %edx + vpxor %ymm8, %ymm6, %ymm6 + # rnd_1: 2 - 2 + andl %r11d, %ebx + xorl %ecx, %edx + rorxl $13, %r15d, %ecx + vpshufd $0xfa, %ymm3, %ymm7 + # rnd_1: 3 - 3 + addl %edx, %r14d + rorxl $2, %r15d, %edx + xorl %r13d, %ebx + vpxor %ymm6, %ymm9, %ymm5 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r15d, %edx + addl %ebx, %r14d + vpsrld $10, %ymm7, %ymm8 + # rnd_1: 5 - 5 + xorl %ecx, %edx + addl %r14d, %r10d + movl %r8d, %ebx + vpsrlq $19, %ymm7, %ymm6 + # rnd_1: 6 - 6 + xorl %r15d, %ebx + addl %edx, %r14d + andl %ebx, %eax + vpsrlq $0x11, %ymm7, %ymm7 + # rnd_1: 7 - 7 + xorl %r8d, %eax + rorxl $6, %r10d, %edx + addl %eax, %r14d + vpaddd %ymm0, %ymm4, %ymm4 + # rnd_0: 0 - 0 + movl %r11d, %eax + rorxl $11, %r10d, %ecx + addl 264(%rsp), %r13d + vpxor %ymm7, %ymm6, %ymm6 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r12d, %eax + rorxl $25, %r10d, %edx + vpxor %ymm6, %ymm8, %ymm8 + # rnd_0: 2 - 2 + andl %r10d, %eax + xorl %ecx, %edx + rorxl $13, %r14d, %ecx + vpaddd %ymm5, %ymm4, %ymm4 + # rnd_0: 3 - 3 + addl %edx, %r13d + rorxl $2, %r14d, %edx + xorl %r12d, %eax + vpshufb %ymm11, %ymm8, %ymm8 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r14d, %edx + addl %eax, %r13d + vpaddd %ymm8, %ymm4, %ymm4 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r15d, %eax + addl %r13d, %r9d + vpshufd $0x50, %ymm4, %ymm6 + # rnd_0: 6 - 6 + xorl %r14d, %eax + addl %edx, %r13d + andl %eax, %ebx + vpsrlq $0x11, %ymm6, %ymm8 + # rnd_0: 7 - 7 + xorl %r15d, %ebx + rorxl $6, %r9d, %edx + addl %ebx, %r13d + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 0 - 0 + movl %r10d, %ebx + rorxl $11, %r9d, %ecx + addl 268(%rsp), %r12d + vpsrld $10, %ymm6, %ymm9 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r11d, %ebx + rorxl $25, %r9d, %edx + vpxor %ymm7, %ymm8, %ymm8 + # rnd_1: 2 - 2 + andl %r9d, %ebx + xorl %ecx, %edx + rorxl $13, %r13d, %ecx + vpxor %ymm8, %ymm9, %ymm9 + # rnd_1: 3 - 3 + addl %edx, %r12d + rorxl $2, %r13d, %edx + xorl %r11d, %ebx + vpshufb %ymm12, %ymm9, %ymm9 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r13d, %edx + addl %ebx, %r12d + vpaddd %ymm4, %ymm9, %ymm0 + # rnd_1: 5 - 5 + xorl %ecx, %edx + addl %r12d, %r8d + movl %r14d, %ebx + vpaddd 384+L_avx2_rorx_sha256_k(%rip), %ymm0, %ymm4 + # rnd_1: 6 - 6 + xorl %r13d, %ebx + addl %edx, %r12d + andl %ebx, %eax + # rnd_1: 7 - 7 + xorl %r14d, %eax + rorxl $6, %r8d, %edx + addl %eax, %r12d + vmovdqu %ymm4, 384(%rsp) + # rnd_0: 0 - 0 + movl %r9d, %eax + rorxl $11, %r8d, %ecx + addl 288(%rsp), %r11d + vpalignr $4, %ymm1, %ymm2, %ymm5 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r10d, %eax + rorxl $25, %r8d, %edx + vpalignr $4, %ymm3, %ymm0, %ymm4 + # rnd_0: 2 - 2 + andl %r8d, %eax + xorl %ecx, %edx + rorxl $13, %r12d, %ecx + vpsrld $7, %ymm5, %ymm6 + # rnd_0: 3 - 3 + addl %edx, %r11d + rorxl $2, %r12d, %edx + xorl %r10d, %eax + vpslld $25, %ymm5, %ymm7 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r12d, %edx + addl %eax, %r11d + vpsrld $18, %ymm5, %ymm8 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r13d, %eax + addl %r11d, %r15d + vpslld $14, %ymm5, %ymm9 + # rnd_0: 6 - 6 + xorl %r12d, %eax + addl %edx, %r11d + andl %eax, %ebx + vpor %ymm7, %ymm6, %ymm6 + # rnd_0: 7 - 7 + xorl %r13d, %ebx + rorxl $6, %r15d, %edx + addl %ebx, %r11d + vpor %ymm9, %ymm8, %ymm8 + # rnd_1: 0 - 0 + movl %r8d, %ebx + rorxl $11, %r15d, %ecx + addl 292(%rsp), %r10d + vpsrld $3, %ymm5, %ymm9 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r9d, %ebx + rorxl $25, %r15d, %edx + vpxor %ymm8, %ymm6, %ymm6 + # rnd_1: 2 - 2 + andl %r15d, %ebx + xorl %ecx, %edx + rorxl $13, %r11d, %ecx + vpshufd $0xfa, %ymm0, %ymm7 + # rnd_1: 3 - 3 + addl %edx, %r10d + rorxl $2, %r11d, %edx + xorl %r9d, %ebx + vpxor %ymm6, %ymm9, %ymm5 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r11d, %edx + addl %ebx, %r10d + vpsrld $10, %ymm7, %ymm8 + # rnd_1: 5 - 5 + xorl %ecx, %edx + addl %r10d, %r14d + movl %r12d, %ebx + vpsrlq $19, %ymm7, %ymm6 + # rnd_1: 6 - 6 + xorl %r11d, %ebx + addl %edx, %r10d + andl %ebx, %eax + vpsrlq $0x11, %ymm7, %ymm7 + # rnd_1: 7 - 7 + xorl %r12d, %eax + rorxl $6, %r14d, %edx + addl %eax, %r10d + vpaddd %ymm1, %ymm4, %ymm4 + # rnd_0: 0 - 0 + movl %r15d, %eax + rorxl $11, %r14d, %ecx + addl 296(%rsp), %r9d + vpxor %ymm7, %ymm6, %ymm6 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r8d, %eax + rorxl $25, %r14d, %edx + vpxor %ymm6, %ymm8, %ymm8 + # rnd_0: 2 - 2 + andl %r14d, %eax + xorl %ecx, %edx + rorxl $13, %r10d, %ecx + vpaddd %ymm5, %ymm4, %ymm4 + # rnd_0: 3 - 3 + addl %edx, %r9d + rorxl $2, %r10d, %edx + xorl %r8d, %eax + vpshufb %ymm11, %ymm8, %ymm8 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r10d, %edx + addl %eax, %r9d + vpaddd %ymm8, %ymm4, %ymm4 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r11d, %eax + addl %r9d, %r13d + vpshufd $0x50, %ymm4, %ymm6 + # rnd_0: 6 - 6 + xorl %r10d, %eax + addl %edx, %r9d + andl %eax, %ebx + vpsrlq $0x11, %ymm6, %ymm8 + # rnd_0: 7 - 7 + xorl %r11d, %ebx + rorxl $6, %r13d, %edx + addl %ebx, %r9d + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 0 - 0 + movl %r14d, %ebx + rorxl $11, %r13d, %ecx + addl 300(%rsp), %r8d + vpsrld $10, %ymm6, %ymm9 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r15d, %ebx + rorxl $25, %r13d, %edx + vpxor %ymm7, %ymm8, %ymm8 + # rnd_1: 2 - 2 + andl %r13d, %ebx + xorl %ecx, %edx + rorxl $13, %r9d, %ecx + vpxor %ymm8, %ymm9, %ymm9 + # rnd_1: 3 - 3 + addl %edx, %r8d + rorxl $2, %r9d, %edx + xorl %r15d, %ebx + vpshufb %ymm12, %ymm9, %ymm9 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r9d, %edx + addl %ebx, %r8d + vpaddd %ymm4, %ymm9, %ymm1 + # rnd_1: 5 - 5 + xorl %ecx, %edx + addl %r8d, %r12d + movl %r10d, %ebx + vpaddd 416+L_avx2_rorx_sha256_k(%rip), %ymm1, %ymm4 + # rnd_1: 6 - 6 + xorl %r9d, %ebx + addl %edx, %r8d + andl %ebx, %eax + # rnd_1: 7 - 7 + xorl %r10d, %eax + rorxl $6, %r12d, %edx + addl %eax, %r8d + vmovdqu %ymm4, 416(%rsp) + # rnd_0: 0 - 0 + movl %r13d, %eax + rorxl $11, %r12d, %ecx + addl 320(%rsp), %r15d + vpalignr $4, %ymm2, %ymm3, %ymm5 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r14d, %eax + rorxl $25, %r12d, %edx + vpalignr $4, %ymm0, %ymm1, %ymm4 + # rnd_0: 2 - 2 + andl %r12d, %eax + xorl %ecx, %edx + rorxl $13, %r8d, %ecx + vpsrld $7, %ymm5, %ymm6 + # rnd_0: 3 - 3 + addl %edx, %r15d + rorxl $2, %r8d, %edx + xorl %r14d, %eax + vpslld $25, %ymm5, %ymm7 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r8d, %edx + addl %eax, %r15d + vpsrld $18, %ymm5, %ymm8 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r9d, %eax + addl %r15d, %r11d + vpslld $14, %ymm5, %ymm9 + # rnd_0: 6 - 6 + xorl %r8d, %eax + addl %edx, %r15d + andl %eax, %ebx + vpor %ymm7, %ymm6, %ymm6 + # rnd_0: 7 - 7 + xorl %r9d, %ebx + rorxl $6, %r11d, %edx + addl %ebx, %r15d + vpor %ymm9, %ymm8, %ymm8 + # rnd_1: 0 - 0 + movl %r12d, %ebx + rorxl $11, %r11d, %ecx + addl 324(%rsp), %r14d + vpsrld $3, %ymm5, %ymm9 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r13d, %ebx + rorxl $25, %r11d, %edx + vpxor %ymm8, %ymm6, %ymm6 + # rnd_1: 2 - 2 + andl %r11d, %ebx + xorl %ecx, %edx + rorxl $13, %r15d, %ecx + vpshufd $0xfa, %ymm1, %ymm7 + # rnd_1: 3 - 3 + addl %edx, %r14d + rorxl $2, %r15d, %edx + xorl %r13d, %ebx + vpxor %ymm6, %ymm9, %ymm5 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r15d, %edx + addl %ebx, %r14d + vpsrld $10, %ymm7, %ymm8 + # rnd_1: 5 - 5 + xorl %ecx, %edx + addl %r14d, %r10d + movl %r8d, %ebx + vpsrlq $19, %ymm7, %ymm6 + # rnd_1: 6 - 6 + xorl %r15d, %ebx + addl %edx, %r14d + andl %ebx, %eax + vpsrlq $0x11, %ymm7, %ymm7 + # rnd_1: 7 - 7 + xorl %r8d, %eax + rorxl $6, %r10d, %edx + addl %eax, %r14d + vpaddd %ymm2, %ymm4, %ymm4 + # rnd_0: 0 - 0 + movl %r11d, %eax + rorxl $11, %r10d, %ecx + addl 328(%rsp), %r13d + vpxor %ymm7, %ymm6, %ymm6 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r12d, %eax + rorxl $25, %r10d, %edx + vpxor %ymm6, %ymm8, %ymm8 + # rnd_0: 2 - 2 + andl %r10d, %eax + xorl %ecx, %edx + rorxl $13, %r14d, %ecx + vpaddd %ymm5, %ymm4, %ymm4 + # rnd_0: 3 - 3 + addl %edx, %r13d + rorxl $2, %r14d, %edx + xorl %r12d, %eax + vpshufb %ymm11, %ymm8, %ymm8 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r14d, %edx + addl %eax, %r13d + vpaddd %ymm8, %ymm4, %ymm4 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r15d, %eax + addl %r13d, %r9d + vpshufd $0x50, %ymm4, %ymm6 + # rnd_0: 6 - 6 + xorl %r14d, %eax + addl %edx, %r13d + andl %eax, %ebx + vpsrlq $0x11, %ymm6, %ymm8 + # rnd_0: 7 - 7 + xorl %r15d, %ebx + rorxl $6, %r9d, %edx + addl %ebx, %r13d + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 0 - 0 + movl %r10d, %ebx + rorxl $11, %r9d, %ecx + addl 332(%rsp), %r12d + vpsrld $10, %ymm6, %ymm9 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r11d, %ebx + rorxl $25, %r9d, %edx + vpxor %ymm7, %ymm8, %ymm8 + # rnd_1: 2 - 2 + andl %r9d, %ebx + xorl %ecx, %edx + rorxl $13, %r13d, %ecx + vpxor %ymm8, %ymm9, %ymm9 + # rnd_1: 3 - 3 + addl %edx, %r12d + rorxl $2, %r13d, %edx + xorl %r11d, %ebx + vpshufb %ymm12, %ymm9, %ymm9 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r13d, %edx + addl %ebx, %r12d + vpaddd %ymm4, %ymm9, %ymm2 + # rnd_1: 5 - 5 + xorl %ecx, %edx + addl %r12d, %r8d + movl %r14d, %ebx + vpaddd 448+L_avx2_rorx_sha256_k(%rip), %ymm2, %ymm4 + # rnd_1: 6 - 6 + xorl %r13d, %ebx + addl %edx, %r12d + andl %ebx, %eax + # rnd_1: 7 - 7 + xorl %r14d, %eax + rorxl $6, %r8d, %edx + addl %eax, %r12d + vmovdqu %ymm4, 448(%rsp) + # rnd_0: 0 - 0 + movl %r9d, %eax + rorxl $11, %r8d, %ecx + addl 352(%rsp), %r11d + vpalignr $4, %ymm3, %ymm0, %ymm5 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r10d, %eax + rorxl $25, %r8d, %edx + vpalignr $4, %ymm1, %ymm2, %ymm4 + # rnd_0: 2 - 2 + andl %r8d, %eax + xorl %ecx, %edx + rorxl $13, %r12d, %ecx + vpsrld $7, %ymm5, %ymm6 + # rnd_0: 3 - 3 + addl %edx, %r11d + rorxl $2, %r12d, %edx + xorl %r10d, %eax + vpslld $25, %ymm5, %ymm7 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r12d, %edx + addl %eax, %r11d + vpsrld $18, %ymm5, %ymm8 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r13d, %eax + addl %r11d, %r15d + vpslld $14, %ymm5, %ymm9 + # rnd_0: 6 - 6 + xorl %r12d, %eax + addl %edx, %r11d + andl %eax, %ebx + vpor %ymm7, %ymm6, %ymm6 + # rnd_0: 7 - 7 + xorl %r13d, %ebx + rorxl $6, %r15d, %edx + addl %ebx, %r11d + vpor %ymm9, %ymm8, %ymm8 + # rnd_1: 0 - 0 + movl %r8d, %ebx + rorxl $11, %r15d, %ecx + addl 356(%rsp), %r10d + vpsrld $3, %ymm5, %ymm9 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r9d, %ebx + rorxl $25, %r15d, %edx + vpxor %ymm8, %ymm6, %ymm6 + # rnd_1: 2 - 2 + andl %r15d, %ebx + xorl %ecx, %edx + rorxl $13, %r11d, %ecx + vpshufd $0xfa, %ymm2, %ymm7 + # rnd_1: 3 - 3 + addl %edx, %r10d + rorxl $2, %r11d, %edx + xorl %r9d, %ebx + vpxor %ymm6, %ymm9, %ymm5 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r11d, %edx + addl %ebx, %r10d + vpsrld $10, %ymm7, %ymm8 + # rnd_1: 5 - 5 + xorl %ecx, %edx + addl %r10d, %r14d + movl %r12d, %ebx + vpsrlq $19, %ymm7, %ymm6 + # rnd_1: 6 - 6 + xorl %r11d, %ebx + addl %edx, %r10d + andl %ebx, %eax + vpsrlq $0x11, %ymm7, %ymm7 + # rnd_1: 7 - 7 + xorl %r12d, %eax + rorxl $6, %r14d, %edx + addl %eax, %r10d + vpaddd %ymm3, %ymm4, %ymm4 + # rnd_0: 0 - 0 + movl %r15d, %eax + rorxl $11, %r14d, %ecx + addl 360(%rsp), %r9d + vpxor %ymm7, %ymm6, %ymm6 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r8d, %eax + rorxl $25, %r14d, %edx + vpxor %ymm6, %ymm8, %ymm8 + # rnd_0: 2 - 2 + andl %r14d, %eax + xorl %ecx, %edx + rorxl $13, %r10d, %ecx + vpaddd %ymm5, %ymm4, %ymm4 + # rnd_0: 3 - 3 + addl %edx, %r9d + rorxl $2, %r10d, %edx + xorl %r8d, %eax + vpshufb %ymm11, %ymm8, %ymm8 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r10d, %edx + addl %eax, %r9d + vpaddd %ymm8, %ymm4, %ymm4 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r11d, %eax + addl %r9d, %r13d + vpshufd $0x50, %ymm4, %ymm6 + # rnd_0: 6 - 6 + xorl %r10d, %eax + addl %edx, %r9d + andl %eax, %ebx + vpsrlq $0x11, %ymm6, %ymm8 + # rnd_0: 7 - 7 + xorl %r11d, %ebx + rorxl $6, %r13d, %edx + addl %ebx, %r9d + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 0 - 0 + movl %r14d, %ebx + rorxl $11, %r13d, %ecx + addl 364(%rsp), %r8d + vpsrld $10, %ymm6, %ymm9 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r15d, %ebx + rorxl $25, %r13d, %edx + vpxor %ymm7, %ymm8, %ymm8 + # rnd_1: 2 - 2 + andl %r13d, %ebx + xorl %ecx, %edx + rorxl $13, %r9d, %ecx + vpxor %ymm8, %ymm9, %ymm9 + # rnd_1: 3 - 3 + addl %edx, %r8d + rorxl $2, %r9d, %edx + xorl %r15d, %ebx + vpshufb %ymm12, %ymm9, %ymm9 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r9d, %edx + addl %ebx, %r8d + vpaddd %ymm4, %ymm9, %ymm3 + # rnd_1: 5 - 5 + xorl %ecx, %edx + addl %r8d, %r12d + movl %r10d, %ebx + vpaddd 480+L_avx2_rorx_sha256_k(%rip), %ymm3, %ymm4 + # rnd_1: 6 - 6 + xorl %r9d, %ebx + addl %edx, %r8d + andl %ebx, %eax + # rnd_1: 7 - 7 + xorl %r10d, %eax + rorxl $6, %r12d, %edx + addl %eax, %r8d + vmovdqu %ymm4, 480(%rsp) + xorl %eax, %eax + xorl %ecx, %ecx + rorxl $6, %r12d, %edx + rorxl $11, %r12d, %ecx + leal (%r8,%rax,1), %r8d + addl 384(%rsp), %r15d + movl %r13d, %eax + xorl %edx, %ecx + xorl %r14d, %eax + rorxl $25, %r12d, %edx + xorl %ecx, %edx + andl %r12d, %eax + addl %edx, %r15d + rorxl $2, %r8d, %edx + rorxl $13, %r8d, %ecx + xorl %r14d, %eax + xorl %edx, %ecx + rorxl $22, %r8d, %edx + addl %eax, %r15d + xorl %ecx, %edx + movl %r9d, %eax + addl %r15d, %r11d + xorl %r8d, %eax + andl %eax, %ebx + addl %edx, %r15d + xorl %r9d, %ebx + rorxl $6, %r11d, %edx + rorxl $11, %r11d, %ecx + addl %ebx, %r15d + addl 388(%rsp), %r14d + movl %r12d, %ebx + xorl %edx, %ecx + xorl %r13d, %ebx + rorxl $25, %r11d, %edx + xorl %ecx, %edx + andl %r11d, %ebx + addl %edx, %r14d + rorxl $2, %r15d, %edx + rorxl $13, %r15d, %ecx + xorl %r13d, %ebx + xorl %edx, %ecx + rorxl $22, %r15d, %edx + addl %ebx, %r14d + xorl %ecx, %edx + movl %r8d, %ebx + leal (%r10,%r14,1), %r10d + xorl %r15d, %ebx + andl %ebx, %eax + addl %edx, %r14d + xorl %r8d, %eax + rorxl $6, %r10d, %edx + rorxl $11, %r10d, %ecx + leal (%r14,%rax,1), %r14d + addl 392(%rsp), %r13d + movl %r11d, %eax + xorl %edx, %ecx + xorl %r12d, %eax + rorxl $25, %r10d, %edx + xorl %ecx, %edx + andl %r10d, %eax + addl %edx, %r13d + rorxl $2, %r14d, %edx + rorxl $13, %r14d, %ecx + xorl %r12d, %eax + xorl %edx, %ecx + rorxl $22, %r14d, %edx + addl %eax, %r13d + xorl %ecx, %edx + movl %r15d, %eax + addl %r13d, %r9d + xorl %r14d, %eax + andl %eax, %ebx + addl %edx, %r13d + xorl %r15d, %ebx + rorxl $6, %r9d, %edx + rorxl $11, %r9d, %ecx + addl %ebx, %r13d + addl 396(%rsp), %r12d + movl %r10d, %ebx + xorl %edx, %ecx + xorl %r11d, %ebx + rorxl $25, %r9d, %edx + xorl %ecx, %edx + andl %r9d, %ebx + addl %edx, %r12d + rorxl $2, %r13d, %edx + rorxl $13, %r13d, %ecx + xorl %r11d, %ebx + xorl %edx, %ecx + rorxl $22, %r13d, %edx + addl %ebx, %r12d + xorl %ecx, %edx + movl %r14d, %ebx + leal (%r8,%r12,1), %r8d + xorl %r13d, %ebx + andl %ebx, %eax + addl %edx, %r12d + xorl %r14d, %eax + rorxl $6, %r8d, %edx + rorxl $11, %r8d, %ecx + leal (%r12,%rax,1), %r12d + addl 416(%rsp), %r11d + movl %r9d, %eax + xorl %edx, %ecx + xorl %r10d, %eax + rorxl $25, %r8d, %edx + xorl %ecx, %edx + andl %r8d, %eax + addl %edx, %r11d + rorxl $2, %r12d, %edx + rorxl $13, %r12d, %ecx + xorl %r10d, %eax + xorl %edx, %ecx + rorxl $22, %r12d, %edx + addl %eax, %r11d + xorl %ecx, %edx + movl %r13d, %eax + addl %r11d, %r15d + xorl %r12d, %eax + andl %eax, %ebx + addl %edx, %r11d + xorl %r13d, %ebx + rorxl $6, %r15d, %edx + rorxl $11, %r15d, %ecx + addl %ebx, %r11d + addl 420(%rsp), %r10d + movl %r8d, %ebx + xorl %edx, %ecx + xorl %r9d, %ebx + rorxl $25, %r15d, %edx + xorl %ecx, %edx + andl %r15d, %ebx + addl %edx, %r10d + rorxl $2, %r11d, %edx + rorxl $13, %r11d, %ecx + xorl %r9d, %ebx + xorl %edx, %ecx + rorxl $22, %r11d, %edx + addl %ebx, %r10d + xorl %ecx, %edx + movl %r12d, %ebx + leal (%r14,%r10,1), %r14d + xorl %r11d, %ebx + andl %ebx, %eax + addl %edx, %r10d + xorl %r12d, %eax + rorxl $6, %r14d, %edx + rorxl $11, %r14d, %ecx + leal (%r10,%rax,1), %r10d + addl 424(%rsp), %r9d + movl %r15d, %eax + xorl %edx, %ecx + xorl %r8d, %eax + rorxl $25, %r14d, %edx + xorl %ecx, %edx + andl %r14d, %eax + addl %edx, %r9d + rorxl $2, %r10d, %edx + rorxl $13, %r10d, %ecx + xorl %r8d, %eax + xorl %edx, %ecx + rorxl $22, %r10d, %edx + addl %eax, %r9d + xorl %ecx, %edx + movl %r11d, %eax + addl %r9d, %r13d + xorl %r10d, %eax + andl %eax, %ebx + addl %edx, %r9d + xorl %r11d, %ebx + rorxl $6, %r13d, %edx + rorxl $11, %r13d, %ecx + addl %ebx, %r9d + addl 428(%rsp), %r8d + movl %r14d, %ebx + xorl %edx, %ecx + xorl %r15d, %ebx + rorxl $25, %r13d, %edx + xorl %ecx, %edx + andl %r13d, %ebx + addl %edx, %r8d + rorxl $2, %r9d, %edx + rorxl $13, %r9d, %ecx + xorl %r15d, %ebx + xorl %edx, %ecx + rorxl $22, %r9d, %edx + addl %ebx, %r8d + xorl %ecx, %edx + movl %r10d, %ebx + leal (%r12,%r8,1), %r12d + xorl %r9d, %ebx + andl %ebx, %eax + addl %edx, %r8d + xorl %r10d, %eax + rorxl $6, %r12d, %edx + rorxl $11, %r12d, %ecx + leal (%r8,%rax,1), %r8d + addl 448(%rsp), %r15d + movl %r13d, %eax + xorl %edx, %ecx + xorl %r14d, %eax + rorxl $25, %r12d, %edx + xorl %ecx, %edx + andl %r12d, %eax + addl %edx, %r15d + rorxl $2, %r8d, %edx + rorxl $13, %r8d, %ecx + xorl %r14d, %eax + xorl %edx, %ecx + rorxl $22, %r8d, %edx + addl %eax, %r15d + xorl %ecx, %edx + movl %r9d, %eax + addl %r15d, %r11d + xorl %r8d, %eax + andl %eax, %ebx + addl %edx, %r15d + xorl %r9d, %ebx + rorxl $6, %r11d, %edx + rorxl $11, %r11d, %ecx + addl %ebx, %r15d + addl 452(%rsp), %r14d + movl %r12d, %ebx + xorl %edx, %ecx + xorl %r13d, %ebx + rorxl $25, %r11d, %edx + xorl %ecx, %edx + andl %r11d, %ebx + addl %edx, %r14d + rorxl $2, %r15d, %edx + rorxl $13, %r15d, %ecx + xorl %r13d, %ebx + xorl %edx, %ecx + rorxl $22, %r15d, %edx + addl %ebx, %r14d + xorl %ecx, %edx + movl %r8d, %ebx + leal (%r10,%r14,1), %r10d + xorl %r15d, %ebx + andl %ebx, %eax + addl %edx, %r14d + xorl %r8d, %eax + rorxl $6, %r10d, %edx + rorxl $11, %r10d, %ecx + leal (%r14,%rax,1), %r14d + addl 456(%rsp), %r13d + movl %r11d, %eax + xorl %edx, %ecx + xorl %r12d, %eax + rorxl $25, %r10d, %edx + xorl %ecx, %edx + andl %r10d, %eax + addl %edx, %r13d + rorxl $2, %r14d, %edx + rorxl $13, %r14d, %ecx + xorl %r12d, %eax + xorl %edx, %ecx + rorxl $22, %r14d, %edx + addl %eax, %r13d + xorl %ecx, %edx + movl %r15d, %eax + addl %r13d, %r9d + xorl %r14d, %eax + andl %eax, %ebx + addl %edx, %r13d + xorl %r15d, %ebx + rorxl $6, %r9d, %edx + rorxl $11, %r9d, %ecx + addl %ebx, %r13d + addl 460(%rsp), %r12d + movl %r10d, %ebx + xorl %edx, %ecx + xorl %r11d, %ebx + rorxl $25, %r9d, %edx + xorl %ecx, %edx + andl %r9d, %ebx + addl %edx, %r12d + rorxl $2, %r13d, %edx + rorxl $13, %r13d, %ecx + xorl %r11d, %ebx + xorl %edx, %ecx + rorxl $22, %r13d, %edx + addl %ebx, %r12d + xorl %ecx, %edx + movl %r14d, %ebx + leal (%r8,%r12,1), %r8d + xorl %r13d, %ebx + andl %ebx, %eax + addl %edx, %r12d + xorl %r14d, %eax + rorxl $6, %r8d, %edx + rorxl $11, %r8d, %ecx + leal (%r12,%rax,1), %r12d + addl 480(%rsp), %r11d + movl %r9d, %eax + xorl %edx, %ecx + xorl %r10d, %eax + rorxl $25, %r8d, %edx + xorl %ecx, %edx + andl %r8d, %eax + addl %edx, %r11d + rorxl $2, %r12d, %edx + rorxl $13, %r12d, %ecx + xorl %r10d, %eax + xorl %edx, %ecx + rorxl $22, %r12d, %edx + addl %eax, %r11d + xorl %ecx, %edx + movl %r13d, %eax + addl %r11d, %r15d + xorl %r12d, %eax + andl %eax, %ebx + addl %edx, %r11d + xorl %r13d, %ebx + rorxl $6, %r15d, %edx + rorxl $11, %r15d, %ecx + addl %ebx, %r11d + addl 484(%rsp), %r10d + movl %r8d, %ebx + xorl %edx, %ecx + xorl %r9d, %ebx + rorxl $25, %r15d, %edx + xorl %ecx, %edx + andl %r15d, %ebx + addl %edx, %r10d + rorxl $2, %r11d, %edx + rorxl $13, %r11d, %ecx + xorl %r9d, %ebx + xorl %edx, %ecx + rorxl $22, %r11d, %edx + addl %ebx, %r10d + xorl %ecx, %edx + movl %r12d, %ebx + leal (%r14,%r10,1), %r14d + xorl %r11d, %ebx + andl %ebx, %eax + addl %edx, %r10d + xorl %r12d, %eax + rorxl $6, %r14d, %edx + rorxl $11, %r14d, %ecx + leal (%r10,%rax,1), %r10d + addl 488(%rsp), %r9d + movl %r15d, %eax + xorl %edx, %ecx + xorl %r8d, %eax + rorxl $25, %r14d, %edx + xorl %ecx, %edx + andl %r14d, %eax + addl %edx, %r9d + rorxl $2, %r10d, %edx + rorxl $13, %r10d, %ecx + xorl %r8d, %eax + xorl %edx, %ecx + rorxl $22, %r10d, %edx + addl %eax, %r9d + xorl %ecx, %edx + movl %r11d, %eax + addl %r9d, %r13d + xorl %r10d, %eax + andl %eax, %ebx + addl %edx, %r9d + xorl %r11d, %ebx + rorxl $6, %r13d, %edx + rorxl $11, %r13d, %ecx + addl %ebx, %r9d + addl 492(%rsp), %r8d + movl %r14d, %ebx + xorl %edx, %ecx + xorl %r15d, %ebx + rorxl $25, %r13d, %edx + xorl %ecx, %edx + andl %r13d, %ebx + addl %edx, %r8d + rorxl $2, %r9d, %edx + rorxl $13, %r9d, %ecx + xorl %r15d, %ebx + xorl %edx, %ecx + rorxl $22, %r9d, %edx + addl %ebx, %r8d + xorl %ecx, %edx + movl %r10d, %ebx + leal (%r12,%r8,1), %r12d + xorl %r9d, %ebx + andl %ebx, %eax + addl %edx, %r8d + xorl %r10d, %eax + addl %eax, %r8d + addl %r8d, (%rdi) + addl %r9d, 4(%rdi) + addl %r10d, 8(%rdi) + addl %r11d, 12(%rdi) + addl %r12d, 16(%rdi) + addl %r13d, 20(%rdi) + addl %r14d, 24(%rdi) + addl %r15d, 28(%rdi) + xorq %rax, %rax + vzeroupper + addq $0x200, %rsp + popq %r15 + popq %r14 + popq %r13 + popq %r12 + popq %rbx + repz retq +.size Transform_Sha256_AVX2_RORX,.-Transform_Sha256_AVX2_RORX +.globl Transform_Sha256_AVX2_RORX_Len +.type Transform_Sha256_AVX2_RORX_Len,@function +.align 4 +Transform_Sha256_AVX2_RORX_Len: + pushq %rbx + pushq %r12 + pushq %r13 + pushq %r14 + pushq %r15 + pushq %rbp + movq %rsi, %rbp + movq %rdx, %rsi + subq $0x200, %rsp + testb $0x40, %sil + je L_sha256_len_avx2_rorx_block + vmovdqu (%rbp), %ymm0 + vmovdqu 32(%rbp), %ymm1 + vmovups %ymm0, 32(%rdi) + vmovups %ymm1, 64(%rdi) + call Transform_Sha256_AVX2_RORX@plt + addq $0x40, %rbp + subl $0x40, %esi + jz L_sha256_len_avx2_rorx_done +L_sha256_len_avx2_rorx_block: + vmovdqa L_avx2_rorx_sha256_flip_mask(%rip), %ymm13 + vmovdqa L_avx2_rorx_sha256_shuf_00BA(%rip), %ymm11 + vmovdqa L_avx2_rorx_sha256_shuf_DC00(%rip), %ymm12 + movl (%rdi), %r8d + movl 4(%rdi), %r9d + movl 8(%rdi), %r10d + movl 12(%rdi), %r11d + movl 16(%rdi), %r12d + movl 20(%rdi), %r13d + movl 24(%rdi), %r14d + movl 28(%rdi), %r15d + # Start of loop processing two blocks +L_sha256_len_avx2_rorx_start: + # X0, X1, X2, X3 = W[0..15] + vmovdqu (%rbp), %xmm0 + vmovdqu 16(%rbp), %xmm1 + vinserti128 $0x01, 64(%rbp), %ymm0, %ymm0 + vinserti128 $0x01, 80(%rbp), %ymm1, %ymm1 + vpshufb %ymm13, %ymm0, %ymm0 + vpshufb %ymm13, %ymm1, %ymm1 + vpaddd 0+L_avx2_rorx_sha256_k(%rip), %ymm0, %ymm4 + vpaddd 32+L_avx2_rorx_sha256_k(%rip), %ymm1, %ymm5 + vmovdqu %ymm4, (%rsp) + vmovdqu %ymm5, 32(%rsp) + vmovdqu 32(%rbp), %xmm2 + vmovdqu 48(%rbp), %xmm3 + vinserti128 $0x01, 96(%rbp), %ymm2, %ymm2 + vinserti128 $0x01, 112(%rbp), %ymm3, %ymm3 + vpshufb %ymm13, %ymm2, %ymm2 + vpshufb %ymm13, %ymm3, %ymm3 + vpaddd 64+L_avx2_rorx_sha256_k(%rip), %ymm2, %ymm4 + vpaddd 96+L_avx2_rorx_sha256_k(%rip), %ymm3, %ymm5 + vmovdqu %ymm4, 64(%rsp) + vmovdqu %ymm5, 96(%rsp) + movl %r9d, %ebx + rorxl $6, %r12d, %edx + xorl %r10d, %ebx + # rnd_0: 0 - 0 + movl %r13d, %eax + rorxl $11, %r12d, %ecx + addl (%rsp), %r15d + vpalignr $4, %ymm0, %ymm1, %ymm5 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r14d, %eax + rorxl $25, %r12d, %edx + vpalignr $4, %ymm2, %ymm3, %ymm4 + # rnd_0: 2 - 2 + andl %r12d, %eax + xorl %ecx, %edx + rorxl $13, %r8d, %ecx + vpsrld $7, %ymm5, %ymm6 + # rnd_0: 3 - 3 + addl %edx, %r15d + rorxl $2, %r8d, %edx + xorl %r14d, %eax + vpslld $25, %ymm5, %ymm7 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r8d, %edx + addl %eax, %r15d + vpsrld $18, %ymm5, %ymm8 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r9d, %eax + addl %r15d, %r11d + vpslld $14, %ymm5, %ymm9 + # rnd_0: 6 - 6 + xorl %r8d, %eax + addl %edx, %r15d + andl %eax, %ebx + vpor %ymm7, %ymm6, %ymm6 + # rnd_0: 7 - 7 + xorl %r9d, %ebx + rorxl $6, %r11d, %edx + addl %ebx, %r15d + vpor %ymm9, %ymm8, %ymm8 + # rnd_1: 0 - 0 + movl %r12d, %ebx + rorxl $11, %r11d, %ecx + addl 4(%rsp), %r14d + vpsrld $3, %ymm5, %ymm9 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r13d, %ebx + rorxl $25, %r11d, %edx + vpxor %ymm8, %ymm6, %ymm6 + # rnd_1: 2 - 2 + andl %r11d, %ebx + xorl %ecx, %edx + rorxl $13, %r15d, %ecx + vpshufd $0xfa, %ymm3, %ymm7 + # rnd_1: 3 - 3 + addl %edx, %r14d + rorxl $2, %r15d, %edx + xorl %r13d, %ebx + vpxor %ymm6, %ymm9, %ymm5 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r15d, %edx + addl %ebx, %r14d + vpsrld $10, %ymm7, %ymm8 + # rnd_1: 5 - 5 + xorl %ecx, %edx + addl %r14d, %r10d + movl %r8d, %ebx + vpsrlq $19, %ymm7, %ymm6 + # rnd_1: 6 - 6 + xorl %r15d, %ebx + addl %edx, %r14d + andl %ebx, %eax + vpsrlq $0x11, %ymm7, %ymm7 + # rnd_1: 7 - 7 + xorl %r8d, %eax + rorxl $6, %r10d, %edx + addl %eax, %r14d + vpaddd %ymm0, %ymm4, %ymm4 + # rnd_0: 0 - 0 + movl %r11d, %eax + rorxl $11, %r10d, %ecx + addl 8(%rsp), %r13d + vpxor %ymm7, %ymm6, %ymm6 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r12d, %eax + rorxl $25, %r10d, %edx + vpxor %ymm6, %ymm8, %ymm8 + # rnd_0: 2 - 2 + andl %r10d, %eax + xorl %ecx, %edx + rorxl $13, %r14d, %ecx + vpaddd %ymm5, %ymm4, %ymm4 + # rnd_0: 3 - 3 + addl %edx, %r13d + rorxl $2, %r14d, %edx + xorl %r12d, %eax + vpshufb %ymm11, %ymm8, %ymm8 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r14d, %edx + addl %eax, %r13d + vpaddd %ymm8, %ymm4, %ymm4 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r15d, %eax + addl %r13d, %r9d + vpshufd $0x50, %ymm4, %ymm6 + # rnd_0: 6 - 6 + xorl %r14d, %eax + addl %edx, %r13d + andl %eax, %ebx + vpsrlq $0x11, %ymm6, %ymm8 + # rnd_0: 7 - 7 + xorl %r15d, %ebx + rorxl $6, %r9d, %edx + addl %ebx, %r13d + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 0 - 0 + movl %r10d, %ebx + rorxl $11, %r9d, %ecx + addl 12(%rsp), %r12d + vpsrld $10, %ymm6, %ymm9 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r11d, %ebx + rorxl $25, %r9d, %edx + vpxor %ymm7, %ymm8, %ymm8 + # rnd_1: 2 - 2 + andl %r9d, %ebx + xorl %ecx, %edx + rorxl $13, %r13d, %ecx + vpxor %ymm8, %ymm9, %ymm9 + # rnd_1: 3 - 3 + addl %edx, %r12d + rorxl $2, %r13d, %edx + xorl %r11d, %ebx + vpshufb %ymm12, %ymm9, %ymm9 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r13d, %edx + addl %ebx, %r12d + vpaddd %ymm4, %ymm9, %ymm0 + # rnd_1: 5 - 5 + xorl %ecx, %edx + addl %r12d, %r8d + movl %r14d, %ebx + vpaddd 128+L_avx2_rorx_sha256_k(%rip), %ymm0, %ymm4 + # rnd_1: 6 - 6 + xorl %r13d, %ebx + addl %edx, %r12d + andl %ebx, %eax + # rnd_1: 7 - 7 + xorl %r14d, %eax + rorxl $6, %r8d, %edx + addl %eax, %r12d + vmovdqu %ymm4, 128(%rsp) + # rnd_0: 0 - 0 + movl %r9d, %eax + rorxl $11, %r8d, %ecx + addl 32(%rsp), %r11d + vpalignr $4, %ymm1, %ymm2, %ymm5 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r10d, %eax + rorxl $25, %r8d, %edx + vpalignr $4, %ymm3, %ymm0, %ymm4 + # rnd_0: 2 - 2 + andl %r8d, %eax + xorl %ecx, %edx + rorxl $13, %r12d, %ecx + vpsrld $7, %ymm5, %ymm6 + # rnd_0: 3 - 3 + addl %edx, %r11d + rorxl $2, %r12d, %edx + xorl %r10d, %eax + vpslld $25, %ymm5, %ymm7 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r12d, %edx + addl %eax, %r11d + vpsrld $18, %ymm5, %ymm8 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r13d, %eax + addl %r11d, %r15d + vpslld $14, %ymm5, %ymm9 + # rnd_0: 6 - 6 + xorl %r12d, %eax + addl %edx, %r11d + andl %eax, %ebx + vpor %ymm7, %ymm6, %ymm6 + # rnd_0: 7 - 7 + xorl %r13d, %ebx + rorxl $6, %r15d, %edx + addl %ebx, %r11d + vpor %ymm9, %ymm8, %ymm8 + # rnd_1: 0 - 0 + movl %r8d, %ebx + rorxl $11, %r15d, %ecx + addl 36(%rsp), %r10d + vpsrld $3, %ymm5, %ymm9 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r9d, %ebx + rorxl $25, %r15d, %edx + vpxor %ymm8, %ymm6, %ymm6 + # rnd_1: 2 - 2 + andl %r15d, %ebx + xorl %ecx, %edx + rorxl $13, %r11d, %ecx + vpshufd $0xfa, %ymm0, %ymm7 + # rnd_1: 3 - 3 + addl %edx, %r10d + rorxl $2, %r11d, %edx + xorl %r9d, %ebx + vpxor %ymm6, %ymm9, %ymm5 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r11d, %edx + addl %ebx, %r10d + vpsrld $10, %ymm7, %ymm8 + # rnd_1: 5 - 5 + xorl %ecx, %edx + addl %r10d, %r14d + movl %r12d, %ebx + vpsrlq $19, %ymm7, %ymm6 + # rnd_1: 6 - 6 + xorl %r11d, %ebx + addl %edx, %r10d + andl %ebx, %eax + vpsrlq $0x11, %ymm7, %ymm7 + # rnd_1: 7 - 7 + xorl %r12d, %eax + rorxl $6, %r14d, %edx + addl %eax, %r10d + vpaddd %ymm1, %ymm4, %ymm4 + # rnd_0: 0 - 0 + movl %r15d, %eax + rorxl $11, %r14d, %ecx + addl 40(%rsp), %r9d + vpxor %ymm7, %ymm6, %ymm6 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r8d, %eax + rorxl $25, %r14d, %edx + vpxor %ymm6, %ymm8, %ymm8 + # rnd_0: 2 - 2 + andl %r14d, %eax + xorl %ecx, %edx + rorxl $13, %r10d, %ecx + vpaddd %ymm5, %ymm4, %ymm4 + # rnd_0: 3 - 3 + addl %edx, %r9d + rorxl $2, %r10d, %edx + xorl %r8d, %eax + vpshufb %ymm11, %ymm8, %ymm8 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r10d, %edx + addl %eax, %r9d + vpaddd %ymm8, %ymm4, %ymm4 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r11d, %eax + addl %r9d, %r13d + vpshufd $0x50, %ymm4, %ymm6 + # rnd_0: 6 - 6 + xorl %r10d, %eax + addl %edx, %r9d + andl %eax, %ebx + vpsrlq $0x11, %ymm6, %ymm8 + # rnd_0: 7 - 7 + xorl %r11d, %ebx + rorxl $6, %r13d, %edx + addl %ebx, %r9d + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 0 - 0 + movl %r14d, %ebx + rorxl $11, %r13d, %ecx + addl 44(%rsp), %r8d + vpsrld $10, %ymm6, %ymm9 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r15d, %ebx + rorxl $25, %r13d, %edx + vpxor %ymm7, %ymm8, %ymm8 + # rnd_1: 2 - 2 + andl %r13d, %ebx + xorl %ecx, %edx + rorxl $13, %r9d, %ecx + vpxor %ymm8, %ymm9, %ymm9 + # rnd_1: 3 - 3 + addl %edx, %r8d + rorxl $2, %r9d, %edx + xorl %r15d, %ebx + vpshufb %ymm12, %ymm9, %ymm9 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r9d, %edx + addl %ebx, %r8d + vpaddd %ymm4, %ymm9, %ymm1 + # rnd_1: 5 - 5 + xorl %ecx, %edx + addl %r8d, %r12d + movl %r10d, %ebx + vpaddd 160+L_avx2_rorx_sha256_k(%rip), %ymm1, %ymm4 + # rnd_1: 6 - 6 + xorl %r9d, %ebx + addl %edx, %r8d + andl %ebx, %eax + # rnd_1: 7 - 7 + xorl %r10d, %eax + rorxl $6, %r12d, %edx + addl %eax, %r8d + vmovdqu %ymm4, 160(%rsp) + # rnd_0: 0 - 0 + movl %r13d, %eax + rorxl $11, %r12d, %ecx + addl 64(%rsp), %r15d + vpalignr $4, %ymm2, %ymm3, %ymm5 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r14d, %eax + rorxl $25, %r12d, %edx + vpalignr $4, %ymm0, %ymm1, %ymm4 + # rnd_0: 2 - 2 + andl %r12d, %eax + xorl %ecx, %edx + rorxl $13, %r8d, %ecx + vpsrld $7, %ymm5, %ymm6 + # rnd_0: 3 - 3 + addl %edx, %r15d + rorxl $2, %r8d, %edx + xorl %r14d, %eax + vpslld $25, %ymm5, %ymm7 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r8d, %edx + addl %eax, %r15d + vpsrld $18, %ymm5, %ymm8 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r9d, %eax + addl %r15d, %r11d + vpslld $14, %ymm5, %ymm9 + # rnd_0: 6 - 6 + xorl %r8d, %eax + addl %edx, %r15d + andl %eax, %ebx + vpor %ymm7, %ymm6, %ymm6 + # rnd_0: 7 - 7 + xorl %r9d, %ebx + rorxl $6, %r11d, %edx + addl %ebx, %r15d + vpor %ymm9, %ymm8, %ymm8 + # rnd_1: 0 - 0 + movl %r12d, %ebx + rorxl $11, %r11d, %ecx + addl 68(%rsp), %r14d + vpsrld $3, %ymm5, %ymm9 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r13d, %ebx + rorxl $25, %r11d, %edx + vpxor %ymm8, %ymm6, %ymm6 + # rnd_1: 2 - 2 + andl %r11d, %ebx + xorl %ecx, %edx + rorxl $13, %r15d, %ecx + vpshufd $0xfa, %ymm1, %ymm7 + # rnd_1: 3 - 3 + addl %edx, %r14d + rorxl $2, %r15d, %edx + xorl %r13d, %ebx + vpxor %ymm6, %ymm9, %ymm5 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r15d, %edx + addl %ebx, %r14d + vpsrld $10, %ymm7, %ymm8 + # rnd_1: 5 - 5 + xorl %ecx, %edx + addl %r14d, %r10d + movl %r8d, %ebx + vpsrlq $19, %ymm7, %ymm6 + # rnd_1: 6 - 6 + xorl %r15d, %ebx + addl %edx, %r14d + andl %ebx, %eax + vpsrlq $0x11, %ymm7, %ymm7 + # rnd_1: 7 - 7 + xorl %r8d, %eax + rorxl $6, %r10d, %edx + addl %eax, %r14d + vpaddd %ymm2, %ymm4, %ymm4 + # rnd_0: 0 - 0 + movl %r11d, %eax + rorxl $11, %r10d, %ecx + addl 72(%rsp), %r13d + vpxor %ymm7, %ymm6, %ymm6 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r12d, %eax + rorxl $25, %r10d, %edx + vpxor %ymm6, %ymm8, %ymm8 + # rnd_0: 2 - 2 + andl %r10d, %eax + xorl %ecx, %edx + rorxl $13, %r14d, %ecx + vpaddd %ymm5, %ymm4, %ymm4 + # rnd_0: 3 - 3 + addl %edx, %r13d + rorxl $2, %r14d, %edx + xorl %r12d, %eax + vpshufb %ymm11, %ymm8, %ymm8 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r14d, %edx + addl %eax, %r13d + vpaddd %ymm8, %ymm4, %ymm4 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r15d, %eax + addl %r13d, %r9d + vpshufd $0x50, %ymm4, %ymm6 + # rnd_0: 6 - 6 + xorl %r14d, %eax + addl %edx, %r13d + andl %eax, %ebx + vpsrlq $0x11, %ymm6, %ymm8 + # rnd_0: 7 - 7 + xorl %r15d, %ebx + rorxl $6, %r9d, %edx + addl %ebx, %r13d + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 0 - 0 + movl %r10d, %ebx + rorxl $11, %r9d, %ecx + addl 76(%rsp), %r12d + vpsrld $10, %ymm6, %ymm9 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r11d, %ebx + rorxl $25, %r9d, %edx + vpxor %ymm7, %ymm8, %ymm8 + # rnd_1: 2 - 2 + andl %r9d, %ebx + xorl %ecx, %edx + rorxl $13, %r13d, %ecx + vpxor %ymm8, %ymm9, %ymm9 + # rnd_1: 3 - 3 + addl %edx, %r12d + rorxl $2, %r13d, %edx + xorl %r11d, %ebx + vpshufb %ymm12, %ymm9, %ymm9 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r13d, %edx + addl %ebx, %r12d + vpaddd %ymm4, %ymm9, %ymm2 + # rnd_1: 5 - 5 + xorl %ecx, %edx + addl %r12d, %r8d + movl %r14d, %ebx + vpaddd 192+L_avx2_rorx_sha256_k(%rip), %ymm2, %ymm4 + # rnd_1: 6 - 6 + xorl %r13d, %ebx + addl %edx, %r12d + andl %ebx, %eax + # rnd_1: 7 - 7 + xorl %r14d, %eax + rorxl $6, %r8d, %edx + addl %eax, %r12d + vmovdqu %ymm4, 192(%rsp) + # rnd_0: 0 - 0 + movl %r9d, %eax + rorxl $11, %r8d, %ecx + addl 96(%rsp), %r11d + vpalignr $4, %ymm3, %ymm0, %ymm5 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r10d, %eax + rorxl $25, %r8d, %edx + vpalignr $4, %ymm1, %ymm2, %ymm4 + # rnd_0: 2 - 2 + andl %r8d, %eax + xorl %ecx, %edx + rorxl $13, %r12d, %ecx + vpsrld $7, %ymm5, %ymm6 + # rnd_0: 3 - 3 + addl %edx, %r11d + rorxl $2, %r12d, %edx + xorl %r10d, %eax + vpslld $25, %ymm5, %ymm7 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r12d, %edx + addl %eax, %r11d + vpsrld $18, %ymm5, %ymm8 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r13d, %eax + addl %r11d, %r15d + vpslld $14, %ymm5, %ymm9 + # rnd_0: 6 - 6 + xorl %r12d, %eax + addl %edx, %r11d + andl %eax, %ebx + vpor %ymm7, %ymm6, %ymm6 + # rnd_0: 7 - 7 + xorl %r13d, %ebx + rorxl $6, %r15d, %edx + addl %ebx, %r11d + vpor %ymm9, %ymm8, %ymm8 + # rnd_1: 0 - 0 + movl %r8d, %ebx + rorxl $11, %r15d, %ecx + addl 100(%rsp), %r10d + vpsrld $3, %ymm5, %ymm9 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r9d, %ebx + rorxl $25, %r15d, %edx + vpxor %ymm8, %ymm6, %ymm6 + # rnd_1: 2 - 2 + andl %r15d, %ebx + xorl %ecx, %edx + rorxl $13, %r11d, %ecx + vpshufd $0xfa, %ymm2, %ymm7 + # rnd_1: 3 - 3 + addl %edx, %r10d + rorxl $2, %r11d, %edx + xorl %r9d, %ebx + vpxor %ymm6, %ymm9, %ymm5 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r11d, %edx + addl %ebx, %r10d + vpsrld $10, %ymm7, %ymm8 + # rnd_1: 5 - 5 + xorl %ecx, %edx + addl %r10d, %r14d + movl %r12d, %ebx + vpsrlq $19, %ymm7, %ymm6 + # rnd_1: 6 - 6 + xorl %r11d, %ebx + addl %edx, %r10d + andl %ebx, %eax + vpsrlq $0x11, %ymm7, %ymm7 + # rnd_1: 7 - 7 + xorl %r12d, %eax + rorxl $6, %r14d, %edx + addl %eax, %r10d + vpaddd %ymm3, %ymm4, %ymm4 + # rnd_0: 0 - 0 + movl %r15d, %eax + rorxl $11, %r14d, %ecx + addl 104(%rsp), %r9d + vpxor %ymm7, %ymm6, %ymm6 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r8d, %eax + rorxl $25, %r14d, %edx + vpxor %ymm6, %ymm8, %ymm8 + # rnd_0: 2 - 2 + andl %r14d, %eax + xorl %ecx, %edx + rorxl $13, %r10d, %ecx + vpaddd %ymm5, %ymm4, %ymm4 + # rnd_0: 3 - 3 + addl %edx, %r9d + rorxl $2, %r10d, %edx + xorl %r8d, %eax + vpshufb %ymm11, %ymm8, %ymm8 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r10d, %edx + addl %eax, %r9d + vpaddd %ymm8, %ymm4, %ymm4 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r11d, %eax + addl %r9d, %r13d + vpshufd $0x50, %ymm4, %ymm6 + # rnd_0: 6 - 6 + xorl %r10d, %eax + addl %edx, %r9d + andl %eax, %ebx + vpsrlq $0x11, %ymm6, %ymm8 + # rnd_0: 7 - 7 + xorl %r11d, %ebx + rorxl $6, %r13d, %edx + addl %ebx, %r9d + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 0 - 0 + movl %r14d, %ebx + rorxl $11, %r13d, %ecx + addl 108(%rsp), %r8d + vpsrld $10, %ymm6, %ymm9 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r15d, %ebx + rorxl $25, %r13d, %edx + vpxor %ymm7, %ymm8, %ymm8 + # rnd_1: 2 - 2 + andl %r13d, %ebx + xorl %ecx, %edx + rorxl $13, %r9d, %ecx + vpxor %ymm8, %ymm9, %ymm9 + # rnd_1: 3 - 3 + addl %edx, %r8d + rorxl $2, %r9d, %edx + xorl %r15d, %ebx + vpshufb %ymm12, %ymm9, %ymm9 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r9d, %edx + addl %ebx, %r8d + vpaddd %ymm4, %ymm9, %ymm3 + # rnd_1: 5 - 5 + xorl %ecx, %edx + addl %r8d, %r12d + movl %r10d, %ebx + vpaddd 224+L_avx2_rorx_sha256_k(%rip), %ymm3, %ymm4 + # rnd_1: 6 - 6 + xorl %r9d, %ebx + addl %edx, %r8d + andl %ebx, %eax + # rnd_1: 7 - 7 + xorl %r10d, %eax + rorxl $6, %r12d, %edx + addl %eax, %r8d + vmovdqu %ymm4, 224(%rsp) + # rnd_0: 0 - 0 + movl %r13d, %eax + rorxl $11, %r12d, %ecx + addl 128(%rsp), %r15d + vpalignr $4, %ymm0, %ymm1, %ymm5 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r14d, %eax + rorxl $25, %r12d, %edx + vpalignr $4, %ymm2, %ymm3, %ymm4 + # rnd_0: 2 - 2 + andl %r12d, %eax + xorl %ecx, %edx + rorxl $13, %r8d, %ecx + vpsrld $7, %ymm5, %ymm6 + # rnd_0: 3 - 3 + addl %edx, %r15d + rorxl $2, %r8d, %edx + xorl %r14d, %eax + vpslld $25, %ymm5, %ymm7 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r8d, %edx + addl %eax, %r15d + vpsrld $18, %ymm5, %ymm8 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r9d, %eax + addl %r15d, %r11d + vpslld $14, %ymm5, %ymm9 + # rnd_0: 6 - 6 + xorl %r8d, %eax + addl %edx, %r15d + andl %eax, %ebx + vpor %ymm7, %ymm6, %ymm6 + # rnd_0: 7 - 7 + xorl %r9d, %ebx + rorxl $6, %r11d, %edx + addl %ebx, %r15d + vpor %ymm9, %ymm8, %ymm8 + # rnd_1: 0 - 0 + movl %r12d, %ebx + rorxl $11, %r11d, %ecx + addl 132(%rsp), %r14d + vpsrld $3, %ymm5, %ymm9 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r13d, %ebx + rorxl $25, %r11d, %edx + vpxor %ymm8, %ymm6, %ymm6 + # rnd_1: 2 - 2 + andl %r11d, %ebx + xorl %ecx, %edx + rorxl $13, %r15d, %ecx + vpshufd $0xfa, %ymm3, %ymm7 + # rnd_1: 3 - 3 + addl %edx, %r14d + rorxl $2, %r15d, %edx + xorl %r13d, %ebx + vpxor %ymm6, %ymm9, %ymm5 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r15d, %edx + addl %ebx, %r14d + vpsrld $10, %ymm7, %ymm8 + # rnd_1: 5 - 5 + xorl %ecx, %edx + addl %r14d, %r10d + movl %r8d, %ebx + vpsrlq $19, %ymm7, %ymm6 + # rnd_1: 6 - 6 + xorl %r15d, %ebx + addl %edx, %r14d + andl %ebx, %eax + vpsrlq $0x11, %ymm7, %ymm7 + # rnd_1: 7 - 7 + xorl %r8d, %eax + rorxl $6, %r10d, %edx + addl %eax, %r14d + vpaddd %ymm0, %ymm4, %ymm4 + # rnd_0: 0 - 0 + movl %r11d, %eax + rorxl $11, %r10d, %ecx + addl 136(%rsp), %r13d + vpxor %ymm7, %ymm6, %ymm6 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r12d, %eax + rorxl $25, %r10d, %edx + vpxor %ymm6, %ymm8, %ymm8 + # rnd_0: 2 - 2 + andl %r10d, %eax + xorl %ecx, %edx + rorxl $13, %r14d, %ecx + vpaddd %ymm5, %ymm4, %ymm4 + # rnd_0: 3 - 3 + addl %edx, %r13d + rorxl $2, %r14d, %edx + xorl %r12d, %eax + vpshufb %ymm11, %ymm8, %ymm8 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r14d, %edx + addl %eax, %r13d + vpaddd %ymm8, %ymm4, %ymm4 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r15d, %eax + addl %r13d, %r9d + vpshufd $0x50, %ymm4, %ymm6 + # rnd_0: 6 - 6 + xorl %r14d, %eax + addl %edx, %r13d + andl %eax, %ebx + vpsrlq $0x11, %ymm6, %ymm8 + # rnd_0: 7 - 7 + xorl %r15d, %ebx + rorxl $6, %r9d, %edx + addl %ebx, %r13d + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 0 - 0 + movl %r10d, %ebx + rorxl $11, %r9d, %ecx + addl 140(%rsp), %r12d + vpsrld $10, %ymm6, %ymm9 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r11d, %ebx + rorxl $25, %r9d, %edx + vpxor %ymm7, %ymm8, %ymm8 + # rnd_1: 2 - 2 + andl %r9d, %ebx + xorl %ecx, %edx + rorxl $13, %r13d, %ecx + vpxor %ymm8, %ymm9, %ymm9 + # rnd_1: 3 - 3 + addl %edx, %r12d + rorxl $2, %r13d, %edx + xorl %r11d, %ebx + vpshufb %ymm12, %ymm9, %ymm9 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r13d, %edx + addl %ebx, %r12d + vpaddd %ymm4, %ymm9, %ymm0 + # rnd_1: 5 - 5 + xorl %ecx, %edx + addl %r12d, %r8d + movl %r14d, %ebx + vpaddd 256+L_avx2_rorx_sha256_k(%rip), %ymm0, %ymm4 + # rnd_1: 6 - 6 + xorl %r13d, %ebx + addl %edx, %r12d + andl %ebx, %eax + # rnd_1: 7 - 7 + xorl %r14d, %eax + rorxl $6, %r8d, %edx + addl %eax, %r12d + vmovdqu %ymm4, 256(%rsp) + # rnd_0: 0 - 0 + movl %r9d, %eax + rorxl $11, %r8d, %ecx + addl 160(%rsp), %r11d + vpalignr $4, %ymm1, %ymm2, %ymm5 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r10d, %eax + rorxl $25, %r8d, %edx + vpalignr $4, %ymm3, %ymm0, %ymm4 + # rnd_0: 2 - 2 + andl %r8d, %eax + xorl %ecx, %edx + rorxl $13, %r12d, %ecx + vpsrld $7, %ymm5, %ymm6 + # rnd_0: 3 - 3 + addl %edx, %r11d + rorxl $2, %r12d, %edx + xorl %r10d, %eax + vpslld $25, %ymm5, %ymm7 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r12d, %edx + addl %eax, %r11d + vpsrld $18, %ymm5, %ymm8 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r13d, %eax + addl %r11d, %r15d + vpslld $14, %ymm5, %ymm9 + # rnd_0: 6 - 6 + xorl %r12d, %eax + addl %edx, %r11d + andl %eax, %ebx + vpor %ymm7, %ymm6, %ymm6 + # rnd_0: 7 - 7 + xorl %r13d, %ebx + rorxl $6, %r15d, %edx + addl %ebx, %r11d + vpor %ymm9, %ymm8, %ymm8 + # rnd_1: 0 - 0 + movl %r8d, %ebx + rorxl $11, %r15d, %ecx + addl 164(%rsp), %r10d + vpsrld $3, %ymm5, %ymm9 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r9d, %ebx + rorxl $25, %r15d, %edx + vpxor %ymm8, %ymm6, %ymm6 + # rnd_1: 2 - 2 + andl %r15d, %ebx + xorl %ecx, %edx + rorxl $13, %r11d, %ecx + vpshufd $0xfa, %ymm0, %ymm7 + # rnd_1: 3 - 3 + addl %edx, %r10d + rorxl $2, %r11d, %edx + xorl %r9d, %ebx + vpxor %ymm6, %ymm9, %ymm5 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r11d, %edx + addl %ebx, %r10d + vpsrld $10, %ymm7, %ymm8 + # rnd_1: 5 - 5 + xorl %ecx, %edx + addl %r10d, %r14d + movl %r12d, %ebx + vpsrlq $19, %ymm7, %ymm6 + # rnd_1: 6 - 6 + xorl %r11d, %ebx + addl %edx, %r10d + andl %ebx, %eax + vpsrlq $0x11, %ymm7, %ymm7 + # rnd_1: 7 - 7 + xorl %r12d, %eax + rorxl $6, %r14d, %edx + addl %eax, %r10d + vpaddd %ymm1, %ymm4, %ymm4 + # rnd_0: 0 - 0 + movl %r15d, %eax + rorxl $11, %r14d, %ecx + addl 168(%rsp), %r9d + vpxor %ymm7, %ymm6, %ymm6 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r8d, %eax + rorxl $25, %r14d, %edx + vpxor %ymm6, %ymm8, %ymm8 + # rnd_0: 2 - 2 + andl %r14d, %eax + xorl %ecx, %edx + rorxl $13, %r10d, %ecx + vpaddd %ymm5, %ymm4, %ymm4 + # rnd_0: 3 - 3 + addl %edx, %r9d + rorxl $2, %r10d, %edx + xorl %r8d, %eax + vpshufb %ymm11, %ymm8, %ymm8 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r10d, %edx + addl %eax, %r9d + vpaddd %ymm8, %ymm4, %ymm4 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r11d, %eax + addl %r9d, %r13d + vpshufd $0x50, %ymm4, %ymm6 + # rnd_0: 6 - 6 + xorl %r10d, %eax + addl %edx, %r9d + andl %eax, %ebx + vpsrlq $0x11, %ymm6, %ymm8 + # rnd_0: 7 - 7 + xorl %r11d, %ebx + rorxl $6, %r13d, %edx + addl %ebx, %r9d + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 0 - 0 + movl %r14d, %ebx + rorxl $11, %r13d, %ecx + addl 172(%rsp), %r8d + vpsrld $10, %ymm6, %ymm9 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r15d, %ebx + rorxl $25, %r13d, %edx + vpxor %ymm7, %ymm8, %ymm8 + # rnd_1: 2 - 2 + andl %r13d, %ebx + xorl %ecx, %edx + rorxl $13, %r9d, %ecx + vpxor %ymm8, %ymm9, %ymm9 + # rnd_1: 3 - 3 + addl %edx, %r8d + rorxl $2, %r9d, %edx + xorl %r15d, %ebx + vpshufb %ymm12, %ymm9, %ymm9 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r9d, %edx + addl %ebx, %r8d + vpaddd %ymm4, %ymm9, %ymm1 + # rnd_1: 5 - 5 + xorl %ecx, %edx + addl %r8d, %r12d + movl %r10d, %ebx + vpaddd 288+L_avx2_rorx_sha256_k(%rip), %ymm1, %ymm4 + # rnd_1: 6 - 6 + xorl %r9d, %ebx + addl %edx, %r8d + andl %ebx, %eax + # rnd_1: 7 - 7 + xorl %r10d, %eax + rorxl $6, %r12d, %edx + addl %eax, %r8d + vmovdqu %ymm4, 288(%rsp) + # rnd_0: 0 - 0 + movl %r13d, %eax + rorxl $11, %r12d, %ecx + addl 192(%rsp), %r15d + vpalignr $4, %ymm2, %ymm3, %ymm5 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r14d, %eax + rorxl $25, %r12d, %edx + vpalignr $4, %ymm0, %ymm1, %ymm4 + # rnd_0: 2 - 2 + andl %r12d, %eax + xorl %ecx, %edx + rorxl $13, %r8d, %ecx + vpsrld $7, %ymm5, %ymm6 + # rnd_0: 3 - 3 + addl %edx, %r15d + rorxl $2, %r8d, %edx + xorl %r14d, %eax + vpslld $25, %ymm5, %ymm7 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r8d, %edx + addl %eax, %r15d + vpsrld $18, %ymm5, %ymm8 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r9d, %eax + addl %r15d, %r11d + vpslld $14, %ymm5, %ymm9 + # rnd_0: 6 - 6 + xorl %r8d, %eax + addl %edx, %r15d + andl %eax, %ebx + vpor %ymm7, %ymm6, %ymm6 + # rnd_0: 7 - 7 + xorl %r9d, %ebx + rorxl $6, %r11d, %edx + addl %ebx, %r15d + vpor %ymm9, %ymm8, %ymm8 + # rnd_1: 0 - 0 + movl %r12d, %ebx + rorxl $11, %r11d, %ecx + addl 196(%rsp), %r14d + vpsrld $3, %ymm5, %ymm9 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r13d, %ebx + rorxl $25, %r11d, %edx + vpxor %ymm8, %ymm6, %ymm6 + # rnd_1: 2 - 2 + andl %r11d, %ebx + xorl %ecx, %edx + rorxl $13, %r15d, %ecx + vpshufd $0xfa, %ymm1, %ymm7 + # rnd_1: 3 - 3 + addl %edx, %r14d + rorxl $2, %r15d, %edx + xorl %r13d, %ebx + vpxor %ymm6, %ymm9, %ymm5 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r15d, %edx + addl %ebx, %r14d + vpsrld $10, %ymm7, %ymm8 + # rnd_1: 5 - 5 + xorl %ecx, %edx + addl %r14d, %r10d + movl %r8d, %ebx + vpsrlq $19, %ymm7, %ymm6 + # rnd_1: 6 - 6 + xorl %r15d, %ebx + addl %edx, %r14d + andl %ebx, %eax + vpsrlq $0x11, %ymm7, %ymm7 + # rnd_1: 7 - 7 + xorl %r8d, %eax + rorxl $6, %r10d, %edx + addl %eax, %r14d + vpaddd %ymm2, %ymm4, %ymm4 + # rnd_0: 0 - 0 + movl %r11d, %eax + rorxl $11, %r10d, %ecx + addl 200(%rsp), %r13d + vpxor %ymm7, %ymm6, %ymm6 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r12d, %eax + rorxl $25, %r10d, %edx + vpxor %ymm6, %ymm8, %ymm8 + # rnd_0: 2 - 2 + andl %r10d, %eax + xorl %ecx, %edx + rorxl $13, %r14d, %ecx + vpaddd %ymm5, %ymm4, %ymm4 + # rnd_0: 3 - 3 + addl %edx, %r13d + rorxl $2, %r14d, %edx + xorl %r12d, %eax + vpshufb %ymm11, %ymm8, %ymm8 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r14d, %edx + addl %eax, %r13d + vpaddd %ymm8, %ymm4, %ymm4 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r15d, %eax + addl %r13d, %r9d + vpshufd $0x50, %ymm4, %ymm6 + # rnd_0: 6 - 6 + xorl %r14d, %eax + addl %edx, %r13d + andl %eax, %ebx + vpsrlq $0x11, %ymm6, %ymm8 + # rnd_0: 7 - 7 + xorl %r15d, %ebx + rorxl $6, %r9d, %edx + addl %ebx, %r13d + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 0 - 0 + movl %r10d, %ebx + rorxl $11, %r9d, %ecx + addl 204(%rsp), %r12d + vpsrld $10, %ymm6, %ymm9 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r11d, %ebx + rorxl $25, %r9d, %edx + vpxor %ymm7, %ymm8, %ymm8 + # rnd_1: 2 - 2 + andl %r9d, %ebx + xorl %ecx, %edx + rorxl $13, %r13d, %ecx + vpxor %ymm8, %ymm9, %ymm9 + # rnd_1: 3 - 3 + addl %edx, %r12d + rorxl $2, %r13d, %edx + xorl %r11d, %ebx + vpshufb %ymm12, %ymm9, %ymm9 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r13d, %edx + addl %ebx, %r12d + vpaddd %ymm4, %ymm9, %ymm2 + # rnd_1: 5 - 5 + xorl %ecx, %edx + addl %r12d, %r8d + movl %r14d, %ebx + vpaddd 320+L_avx2_rorx_sha256_k(%rip), %ymm2, %ymm4 + # rnd_1: 6 - 6 + xorl %r13d, %ebx + addl %edx, %r12d + andl %ebx, %eax + # rnd_1: 7 - 7 + xorl %r14d, %eax + rorxl $6, %r8d, %edx + addl %eax, %r12d + vmovdqu %ymm4, 320(%rsp) + # rnd_0: 0 - 0 + movl %r9d, %eax + rorxl $11, %r8d, %ecx + addl 224(%rsp), %r11d + vpalignr $4, %ymm3, %ymm0, %ymm5 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r10d, %eax + rorxl $25, %r8d, %edx + vpalignr $4, %ymm1, %ymm2, %ymm4 + # rnd_0: 2 - 2 + andl %r8d, %eax + xorl %ecx, %edx + rorxl $13, %r12d, %ecx + vpsrld $7, %ymm5, %ymm6 + # rnd_0: 3 - 3 + addl %edx, %r11d + rorxl $2, %r12d, %edx + xorl %r10d, %eax + vpslld $25, %ymm5, %ymm7 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r12d, %edx + addl %eax, %r11d + vpsrld $18, %ymm5, %ymm8 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r13d, %eax + addl %r11d, %r15d + vpslld $14, %ymm5, %ymm9 + # rnd_0: 6 - 6 + xorl %r12d, %eax + addl %edx, %r11d + andl %eax, %ebx + vpor %ymm7, %ymm6, %ymm6 + # rnd_0: 7 - 7 + xorl %r13d, %ebx + rorxl $6, %r15d, %edx + addl %ebx, %r11d + vpor %ymm9, %ymm8, %ymm8 + # rnd_1: 0 - 0 + movl %r8d, %ebx + rorxl $11, %r15d, %ecx + addl 228(%rsp), %r10d + vpsrld $3, %ymm5, %ymm9 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r9d, %ebx + rorxl $25, %r15d, %edx + vpxor %ymm8, %ymm6, %ymm6 + # rnd_1: 2 - 2 + andl %r15d, %ebx + xorl %ecx, %edx + rorxl $13, %r11d, %ecx + vpshufd $0xfa, %ymm2, %ymm7 + # rnd_1: 3 - 3 + addl %edx, %r10d + rorxl $2, %r11d, %edx + xorl %r9d, %ebx + vpxor %ymm6, %ymm9, %ymm5 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r11d, %edx + addl %ebx, %r10d + vpsrld $10, %ymm7, %ymm8 + # rnd_1: 5 - 5 + xorl %ecx, %edx + addl %r10d, %r14d + movl %r12d, %ebx + vpsrlq $19, %ymm7, %ymm6 + # rnd_1: 6 - 6 + xorl %r11d, %ebx + addl %edx, %r10d + andl %ebx, %eax + vpsrlq $0x11, %ymm7, %ymm7 + # rnd_1: 7 - 7 + xorl %r12d, %eax + rorxl $6, %r14d, %edx + addl %eax, %r10d + vpaddd %ymm3, %ymm4, %ymm4 + # rnd_0: 0 - 0 + movl %r15d, %eax + rorxl $11, %r14d, %ecx + addl 232(%rsp), %r9d + vpxor %ymm7, %ymm6, %ymm6 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r8d, %eax + rorxl $25, %r14d, %edx + vpxor %ymm6, %ymm8, %ymm8 + # rnd_0: 2 - 2 + andl %r14d, %eax + xorl %ecx, %edx + rorxl $13, %r10d, %ecx + vpaddd %ymm5, %ymm4, %ymm4 + # rnd_0: 3 - 3 + addl %edx, %r9d + rorxl $2, %r10d, %edx + xorl %r8d, %eax + vpshufb %ymm11, %ymm8, %ymm8 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r10d, %edx + addl %eax, %r9d + vpaddd %ymm8, %ymm4, %ymm4 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r11d, %eax + addl %r9d, %r13d + vpshufd $0x50, %ymm4, %ymm6 + # rnd_0: 6 - 6 + xorl %r10d, %eax + addl %edx, %r9d + andl %eax, %ebx + vpsrlq $0x11, %ymm6, %ymm8 + # rnd_0: 7 - 7 + xorl %r11d, %ebx + rorxl $6, %r13d, %edx + addl %ebx, %r9d + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 0 - 0 + movl %r14d, %ebx + rorxl $11, %r13d, %ecx + addl 236(%rsp), %r8d + vpsrld $10, %ymm6, %ymm9 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r15d, %ebx + rorxl $25, %r13d, %edx + vpxor %ymm7, %ymm8, %ymm8 + # rnd_1: 2 - 2 + andl %r13d, %ebx + xorl %ecx, %edx + rorxl $13, %r9d, %ecx + vpxor %ymm8, %ymm9, %ymm9 + # rnd_1: 3 - 3 + addl %edx, %r8d + rorxl $2, %r9d, %edx + xorl %r15d, %ebx + vpshufb %ymm12, %ymm9, %ymm9 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r9d, %edx + addl %ebx, %r8d + vpaddd %ymm4, %ymm9, %ymm3 + # rnd_1: 5 - 5 + xorl %ecx, %edx + addl %r8d, %r12d + movl %r10d, %ebx + vpaddd 352+L_avx2_rorx_sha256_k(%rip), %ymm3, %ymm4 + # rnd_1: 6 - 6 + xorl %r9d, %ebx + addl %edx, %r8d + andl %ebx, %eax + # rnd_1: 7 - 7 + xorl %r10d, %eax + rorxl $6, %r12d, %edx + addl %eax, %r8d + vmovdqu %ymm4, 352(%rsp) + # rnd_0: 0 - 0 + movl %r13d, %eax + rorxl $11, %r12d, %ecx + addl 256(%rsp), %r15d + vpalignr $4, %ymm0, %ymm1, %ymm5 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r14d, %eax + rorxl $25, %r12d, %edx + vpalignr $4, %ymm2, %ymm3, %ymm4 + # rnd_0: 2 - 2 + andl %r12d, %eax + xorl %ecx, %edx + rorxl $13, %r8d, %ecx + vpsrld $7, %ymm5, %ymm6 + # rnd_0: 3 - 3 + addl %edx, %r15d + rorxl $2, %r8d, %edx + xorl %r14d, %eax + vpslld $25, %ymm5, %ymm7 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r8d, %edx + addl %eax, %r15d + vpsrld $18, %ymm5, %ymm8 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r9d, %eax + addl %r15d, %r11d + vpslld $14, %ymm5, %ymm9 + # rnd_0: 6 - 6 + xorl %r8d, %eax + addl %edx, %r15d + andl %eax, %ebx + vpor %ymm7, %ymm6, %ymm6 + # rnd_0: 7 - 7 + xorl %r9d, %ebx + rorxl $6, %r11d, %edx + addl %ebx, %r15d + vpor %ymm9, %ymm8, %ymm8 + # rnd_1: 0 - 0 + movl %r12d, %ebx + rorxl $11, %r11d, %ecx + addl 260(%rsp), %r14d + vpsrld $3, %ymm5, %ymm9 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r13d, %ebx + rorxl $25, %r11d, %edx + vpxor %ymm8, %ymm6, %ymm6 + # rnd_1: 2 - 2 + andl %r11d, %ebx + xorl %ecx, %edx + rorxl $13, %r15d, %ecx + vpshufd $0xfa, %ymm3, %ymm7 + # rnd_1: 3 - 3 + addl %edx, %r14d + rorxl $2, %r15d, %edx + xorl %r13d, %ebx + vpxor %ymm6, %ymm9, %ymm5 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r15d, %edx + addl %ebx, %r14d + vpsrld $10, %ymm7, %ymm8 + # rnd_1: 5 - 5 + xorl %ecx, %edx + addl %r14d, %r10d + movl %r8d, %ebx + vpsrlq $19, %ymm7, %ymm6 + # rnd_1: 6 - 6 + xorl %r15d, %ebx + addl %edx, %r14d + andl %ebx, %eax + vpsrlq $0x11, %ymm7, %ymm7 + # rnd_1: 7 - 7 + xorl %r8d, %eax + rorxl $6, %r10d, %edx + addl %eax, %r14d + vpaddd %ymm0, %ymm4, %ymm4 + # rnd_0: 0 - 0 + movl %r11d, %eax + rorxl $11, %r10d, %ecx + addl 264(%rsp), %r13d + vpxor %ymm7, %ymm6, %ymm6 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r12d, %eax + rorxl $25, %r10d, %edx + vpxor %ymm6, %ymm8, %ymm8 + # rnd_0: 2 - 2 + andl %r10d, %eax + xorl %ecx, %edx + rorxl $13, %r14d, %ecx + vpaddd %ymm5, %ymm4, %ymm4 + # rnd_0: 3 - 3 + addl %edx, %r13d + rorxl $2, %r14d, %edx + xorl %r12d, %eax + vpshufb %ymm11, %ymm8, %ymm8 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r14d, %edx + addl %eax, %r13d + vpaddd %ymm8, %ymm4, %ymm4 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r15d, %eax + addl %r13d, %r9d + vpshufd $0x50, %ymm4, %ymm6 + # rnd_0: 6 - 6 + xorl %r14d, %eax + addl %edx, %r13d + andl %eax, %ebx + vpsrlq $0x11, %ymm6, %ymm8 + # rnd_0: 7 - 7 + xorl %r15d, %ebx + rorxl $6, %r9d, %edx + addl %ebx, %r13d + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 0 - 0 + movl %r10d, %ebx + rorxl $11, %r9d, %ecx + addl 268(%rsp), %r12d + vpsrld $10, %ymm6, %ymm9 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r11d, %ebx + rorxl $25, %r9d, %edx + vpxor %ymm7, %ymm8, %ymm8 + # rnd_1: 2 - 2 + andl %r9d, %ebx + xorl %ecx, %edx + rorxl $13, %r13d, %ecx + vpxor %ymm8, %ymm9, %ymm9 + # rnd_1: 3 - 3 + addl %edx, %r12d + rorxl $2, %r13d, %edx + xorl %r11d, %ebx + vpshufb %ymm12, %ymm9, %ymm9 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r13d, %edx + addl %ebx, %r12d + vpaddd %ymm4, %ymm9, %ymm0 + # rnd_1: 5 - 5 + xorl %ecx, %edx + addl %r12d, %r8d + movl %r14d, %ebx + vpaddd 384+L_avx2_rorx_sha256_k(%rip), %ymm0, %ymm4 + # rnd_1: 6 - 6 + xorl %r13d, %ebx + addl %edx, %r12d + andl %ebx, %eax + # rnd_1: 7 - 7 + xorl %r14d, %eax + rorxl $6, %r8d, %edx + addl %eax, %r12d + vmovdqu %ymm4, 384(%rsp) + # rnd_0: 0 - 0 + movl %r9d, %eax + rorxl $11, %r8d, %ecx + addl 288(%rsp), %r11d + vpalignr $4, %ymm1, %ymm2, %ymm5 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r10d, %eax + rorxl $25, %r8d, %edx + vpalignr $4, %ymm3, %ymm0, %ymm4 + # rnd_0: 2 - 2 + andl %r8d, %eax + xorl %ecx, %edx + rorxl $13, %r12d, %ecx + vpsrld $7, %ymm5, %ymm6 + # rnd_0: 3 - 3 + addl %edx, %r11d + rorxl $2, %r12d, %edx + xorl %r10d, %eax + vpslld $25, %ymm5, %ymm7 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r12d, %edx + addl %eax, %r11d + vpsrld $18, %ymm5, %ymm8 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r13d, %eax + addl %r11d, %r15d + vpslld $14, %ymm5, %ymm9 + # rnd_0: 6 - 6 + xorl %r12d, %eax + addl %edx, %r11d + andl %eax, %ebx + vpor %ymm7, %ymm6, %ymm6 + # rnd_0: 7 - 7 + xorl %r13d, %ebx + rorxl $6, %r15d, %edx + addl %ebx, %r11d + vpor %ymm9, %ymm8, %ymm8 + # rnd_1: 0 - 0 + movl %r8d, %ebx + rorxl $11, %r15d, %ecx + addl 292(%rsp), %r10d + vpsrld $3, %ymm5, %ymm9 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r9d, %ebx + rorxl $25, %r15d, %edx + vpxor %ymm8, %ymm6, %ymm6 + # rnd_1: 2 - 2 + andl %r15d, %ebx + xorl %ecx, %edx + rorxl $13, %r11d, %ecx + vpshufd $0xfa, %ymm0, %ymm7 + # rnd_1: 3 - 3 + addl %edx, %r10d + rorxl $2, %r11d, %edx + xorl %r9d, %ebx + vpxor %ymm6, %ymm9, %ymm5 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r11d, %edx + addl %ebx, %r10d + vpsrld $10, %ymm7, %ymm8 + # rnd_1: 5 - 5 + xorl %ecx, %edx + addl %r10d, %r14d + movl %r12d, %ebx + vpsrlq $19, %ymm7, %ymm6 + # rnd_1: 6 - 6 + xorl %r11d, %ebx + addl %edx, %r10d + andl %ebx, %eax + vpsrlq $0x11, %ymm7, %ymm7 + # rnd_1: 7 - 7 + xorl %r12d, %eax + rorxl $6, %r14d, %edx + addl %eax, %r10d + vpaddd %ymm1, %ymm4, %ymm4 + # rnd_0: 0 - 0 + movl %r15d, %eax + rorxl $11, %r14d, %ecx + addl 296(%rsp), %r9d + vpxor %ymm7, %ymm6, %ymm6 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r8d, %eax + rorxl $25, %r14d, %edx + vpxor %ymm6, %ymm8, %ymm8 + # rnd_0: 2 - 2 + andl %r14d, %eax + xorl %ecx, %edx + rorxl $13, %r10d, %ecx + vpaddd %ymm5, %ymm4, %ymm4 + # rnd_0: 3 - 3 + addl %edx, %r9d + rorxl $2, %r10d, %edx + xorl %r8d, %eax + vpshufb %ymm11, %ymm8, %ymm8 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r10d, %edx + addl %eax, %r9d + vpaddd %ymm8, %ymm4, %ymm4 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r11d, %eax + addl %r9d, %r13d + vpshufd $0x50, %ymm4, %ymm6 + # rnd_0: 6 - 6 + xorl %r10d, %eax + addl %edx, %r9d + andl %eax, %ebx + vpsrlq $0x11, %ymm6, %ymm8 + # rnd_0: 7 - 7 + xorl %r11d, %ebx + rorxl $6, %r13d, %edx + addl %ebx, %r9d + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 0 - 0 + movl %r14d, %ebx + rorxl $11, %r13d, %ecx + addl 300(%rsp), %r8d + vpsrld $10, %ymm6, %ymm9 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r15d, %ebx + rorxl $25, %r13d, %edx + vpxor %ymm7, %ymm8, %ymm8 + # rnd_1: 2 - 2 + andl %r13d, %ebx + xorl %ecx, %edx + rorxl $13, %r9d, %ecx + vpxor %ymm8, %ymm9, %ymm9 + # rnd_1: 3 - 3 + addl %edx, %r8d + rorxl $2, %r9d, %edx + xorl %r15d, %ebx + vpshufb %ymm12, %ymm9, %ymm9 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r9d, %edx + addl %ebx, %r8d + vpaddd %ymm4, %ymm9, %ymm1 + # rnd_1: 5 - 5 + xorl %ecx, %edx + addl %r8d, %r12d + movl %r10d, %ebx + vpaddd 416+L_avx2_rorx_sha256_k(%rip), %ymm1, %ymm4 + # rnd_1: 6 - 6 + xorl %r9d, %ebx + addl %edx, %r8d + andl %ebx, %eax + # rnd_1: 7 - 7 + xorl %r10d, %eax + rorxl $6, %r12d, %edx + addl %eax, %r8d + vmovdqu %ymm4, 416(%rsp) + # rnd_0: 0 - 0 + movl %r13d, %eax + rorxl $11, %r12d, %ecx + addl 320(%rsp), %r15d + vpalignr $4, %ymm2, %ymm3, %ymm5 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r14d, %eax + rorxl $25, %r12d, %edx + vpalignr $4, %ymm0, %ymm1, %ymm4 + # rnd_0: 2 - 2 + andl %r12d, %eax + xorl %ecx, %edx + rorxl $13, %r8d, %ecx + vpsrld $7, %ymm5, %ymm6 + # rnd_0: 3 - 3 + addl %edx, %r15d + rorxl $2, %r8d, %edx + xorl %r14d, %eax + vpslld $25, %ymm5, %ymm7 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r8d, %edx + addl %eax, %r15d + vpsrld $18, %ymm5, %ymm8 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r9d, %eax + addl %r15d, %r11d + vpslld $14, %ymm5, %ymm9 + # rnd_0: 6 - 6 + xorl %r8d, %eax + addl %edx, %r15d + andl %eax, %ebx + vpor %ymm7, %ymm6, %ymm6 + # rnd_0: 7 - 7 + xorl %r9d, %ebx + rorxl $6, %r11d, %edx + addl %ebx, %r15d + vpor %ymm9, %ymm8, %ymm8 + # rnd_1: 0 - 0 + movl %r12d, %ebx + rorxl $11, %r11d, %ecx + addl 324(%rsp), %r14d + vpsrld $3, %ymm5, %ymm9 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r13d, %ebx + rorxl $25, %r11d, %edx + vpxor %ymm8, %ymm6, %ymm6 + # rnd_1: 2 - 2 + andl %r11d, %ebx + xorl %ecx, %edx + rorxl $13, %r15d, %ecx + vpshufd $0xfa, %ymm1, %ymm7 + # rnd_1: 3 - 3 + addl %edx, %r14d + rorxl $2, %r15d, %edx + xorl %r13d, %ebx + vpxor %ymm6, %ymm9, %ymm5 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r15d, %edx + addl %ebx, %r14d + vpsrld $10, %ymm7, %ymm8 + # rnd_1: 5 - 5 + xorl %ecx, %edx + addl %r14d, %r10d + movl %r8d, %ebx + vpsrlq $19, %ymm7, %ymm6 + # rnd_1: 6 - 6 + xorl %r15d, %ebx + addl %edx, %r14d + andl %ebx, %eax + vpsrlq $0x11, %ymm7, %ymm7 + # rnd_1: 7 - 7 + xorl %r8d, %eax + rorxl $6, %r10d, %edx + addl %eax, %r14d + vpaddd %ymm2, %ymm4, %ymm4 + # rnd_0: 0 - 0 + movl %r11d, %eax + rorxl $11, %r10d, %ecx + addl 328(%rsp), %r13d + vpxor %ymm7, %ymm6, %ymm6 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r12d, %eax + rorxl $25, %r10d, %edx + vpxor %ymm6, %ymm8, %ymm8 + # rnd_0: 2 - 2 + andl %r10d, %eax + xorl %ecx, %edx + rorxl $13, %r14d, %ecx + vpaddd %ymm5, %ymm4, %ymm4 + # rnd_0: 3 - 3 + addl %edx, %r13d + rorxl $2, %r14d, %edx + xorl %r12d, %eax + vpshufb %ymm11, %ymm8, %ymm8 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r14d, %edx + addl %eax, %r13d + vpaddd %ymm8, %ymm4, %ymm4 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r15d, %eax + addl %r13d, %r9d + vpshufd $0x50, %ymm4, %ymm6 + # rnd_0: 6 - 6 + xorl %r14d, %eax + addl %edx, %r13d + andl %eax, %ebx + vpsrlq $0x11, %ymm6, %ymm8 + # rnd_0: 7 - 7 + xorl %r15d, %ebx + rorxl $6, %r9d, %edx + addl %ebx, %r13d + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 0 - 0 + movl %r10d, %ebx + rorxl $11, %r9d, %ecx + addl 332(%rsp), %r12d + vpsrld $10, %ymm6, %ymm9 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r11d, %ebx + rorxl $25, %r9d, %edx + vpxor %ymm7, %ymm8, %ymm8 + # rnd_1: 2 - 2 + andl %r9d, %ebx + xorl %ecx, %edx + rorxl $13, %r13d, %ecx + vpxor %ymm8, %ymm9, %ymm9 + # rnd_1: 3 - 3 + addl %edx, %r12d + rorxl $2, %r13d, %edx + xorl %r11d, %ebx + vpshufb %ymm12, %ymm9, %ymm9 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r13d, %edx + addl %ebx, %r12d + vpaddd %ymm4, %ymm9, %ymm2 + # rnd_1: 5 - 5 + xorl %ecx, %edx + addl %r12d, %r8d + movl %r14d, %ebx + vpaddd 448+L_avx2_rorx_sha256_k(%rip), %ymm2, %ymm4 + # rnd_1: 6 - 6 + xorl %r13d, %ebx + addl %edx, %r12d + andl %ebx, %eax + # rnd_1: 7 - 7 + xorl %r14d, %eax + rorxl $6, %r8d, %edx + addl %eax, %r12d + vmovdqu %ymm4, 448(%rsp) + # rnd_0: 0 - 0 + movl %r9d, %eax + rorxl $11, %r8d, %ecx + addl 352(%rsp), %r11d + vpalignr $4, %ymm3, %ymm0, %ymm5 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r10d, %eax + rorxl $25, %r8d, %edx + vpalignr $4, %ymm1, %ymm2, %ymm4 + # rnd_0: 2 - 2 + andl %r8d, %eax + xorl %ecx, %edx + rorxl $13, %r12d, %ecx + vpsrld $7, %ymm5, %ymm6 + # rnd_0: 3 - 3 + addl %edx, %r11d + rorxl $2, %r12d, %edx + xorl %r10d, %eax + vpslld $25, %ymm5, %ymm7 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r12d, %edx + addl %eax, %r11d + vpsrld $18, %ymm5, %ymm8 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r13d, %eax + addl %r11d, %r15d + vpslld $14, %ymm5, %ymm9 + # rnd_0: 6 - 6 + xorl %r12d, %eax + addl %edx, %r11d + andl %eax, %ebx + vpor %ymm7, %ymm6, %ymm6 + # rnd_0: 7 - 7 + xorl %r13d, %ebx + rorxl $6, %r15d, %edx + addl %ebx, %r11d + vpor %ymm9, %ymm8, %ymm8 + # rnd_1: 0 - 0 + movl %r8d, %ebx + rorxl $11, %r15d, %ecx + addl 356(%rsp), %r10d + vpsrld $3, %ymm5, %ymm9 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r9d, %ebx + rorxl $25, %r15d, %edx + vpxor %ymm8, %ymm6, %ymm6 + # rnd_1: 2 - 2 + andl %r15d, %ebx + xorl %ecx, %edx + rorxl $13, %r11d, %ecx + vpshufd $0xfa, %ymm2, %ymm7 + # rnd_1: 3 - 3 + addl %edx, %r10d + rorxl $2, %r11d, %edx + xorl %r9d, %ebx + vpxor %ymm6, %ymm9, %ymm5 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r11d, %edx + addl %ebx, %r10d + vpsrld $10, %ymm7, %ymm8 + # rnd_1: 5 - 5 + xorl %ecx, %edx + addl %r10d, %r14d + movl %r12d, %ebx + vpsrlq $19, %ymm7, %ymm6 + # rnd_1: 6 - 6 + xorl %r11d, %ebx + addl %edx, %r10d + andl %ebx, %eax + vpsrlq $0x11, %ymm7, %ymm7 + # rnd_1: 7 - 7 + xorl %r12d, %eax + rorxl $6, %r14d, %edx + addl %eax, %r10d + vpaddd %ymm3, %ymm4, %ymm4 + # rnd_0: 0 - 0 + movl %r15d, %eax + rorxl $11, %r14d, %ecx + addl 360(%rsp), %r9d + vpxor %ymm7, %ymm6, %ymm6 + # rnd_0: 1 - 1 + xorl %edx, %ecx + xorl %r8d, %eax + rorxl $25, %r14d, %edx + vpxor %ymm6, %ymm8, %ymm8 + # rnd_0: 2 - 2 + andl %r14d, %eax + xorl %ecx, %edx + rorxl $13, %r10d, %ecx + vpaddd %ymm5, %ymm4, %ymm4 + # rnd_0: 3 - 3 + addl %edx, %r9d + rorxl $2, %r10d, %edx + xorl %r8d, %eax + vpshufb %ymm11, %ymm8, %ymm8 + # rnd_0: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r10d, %edx + addl %eax, %r9d + vpaddd %ymm8, %ymm4, %ymm4 + # rnd_0: 5 - 5 + xorl %ecx, %edx + movl %r11d, %eax + addl %r9d, %r13d + vpshufd $0x50, %ymm4, %ymm6 + # rnd_0: 6 - 6 + xorl %r10d, %eax + addl %edx, %r9d + andl %eax, %ebx + vpsrlq $0x11, %ymm6, %ymm8 + # rnd_0: 7 - 7 + xorl %r11d, %ebx + rorxl $6, %r13d, %edx + addl %ebx, %r9d + vpsrlq $19, %ymm6, %ymm7 + # rnd_1: 0 - 0 + movl %r14d, %ebx + rorxl $11, %r13d, %ecx + addl 364(%rsp), %r8d + vpsrld $10, %ymm6, %ymm9 + # rnd_1: 1 - 1 + xorl %edx, %ecx + xorl %r15d, %ebx + rorxl $25, %r13d, %edx + vpxor %ymm7, %ymm8, %ymm8 + # rnd_1: 2 - 2 + andl %r13d, %ebx + xorl %ecx, %edx + rorxl $13, %r9d, %ecx + vpxor %ymm8, %ymm9, %ymm9 + # rnd_1: 3 - 3 + addl %edx, %r8d + rorxl $2, %r9d, %edx + xorl %r15d, %ebx + vpshufb %ymm12, %ymm9, %ymm9 + # rnd_1: 4 - 4 + xorl %edx, %ecx + rorxl $22, %r9d, %edx + addl %ebx, %r8d + vpaddd %ymm4, %ymm9, %ymm3 + # rnd_1: 5 - 5 + xorl %ecx, %edx + addl %r8d, %r12d + movl %r10d, %ebx + vpaddd 480+L_avx2_rorx_sha256_k(%rip), %ymm3, %ymm4 + # rnd_1: 6 - 6 + xorl %r9d, %ebx + addl %edx, %r8d + andl %ebx, %eax + # rnd_1: 7 - 7 + xorl %r10d, %eax + rorxl $6, %r12d, %edx + addl %eax, %r8d + vmovdqu %ymm4, 480(%rsp) + xorl %eax, %eax + xorl %ecx, %ecx + rorxl $6, %r12d, %edx + rorxl $11, %r12d, %ecx + leal (%r8,%rax,1), %r8d + addl 384(%rsp), %r15d + movl %r13d, %eax + xorl %edx, %ecx + xorl %r14d, %eax + rorxl $25, %r12d, %edx + xorl %ecx, %edx + andl %r12d, %eax + addl %edx, %r15d + rorxl $2, %r8d, %edx + rorxl $13, %r8d, %ecx + xorl %r14d, %eax + xorl %edx, %ecx + rorxl $22, %r8d, %edx + addl %eax, %r15d + xorl %ecx, %edx + movl %r9d, %eax + addl %r15d, %r11d + xorl %r8d, %eax + andl %eax, %ebx + addl %edx, %r15d + xorl %r9d, %ebx + rorxl $6, %r11d, %edx + rorxl $11, %r11d, %ecx + addl %ebx, %r15d + addl 388(%rsp), %r14d + movl %r12d, %ebx + xorl %edx, %ecx + xorl %r13d, %ebx + rorxl $25, %r11d, %edx + xorl %ecx, %edx + andl %r11d, %ebx + addl %edx, %r14d + rorxl $2, %r15d, %edx + rorxl $13, %r15d, %ecx + xorl %r13d, %ebx + xorl %edx, %ecx + rorxl $22, %r15d, %edx + addl %ebx, %r14d + xorl %ecx, %edx + movl %r8d, %ebx + leal (%r10,%r14,1), %r10d + xorl %r15d, %ebx + andl %ebx, %eax + addl %edx, %r14d + xorl %r8d, %eax + rorxl $6, %r10d, %edx + rorxl $11, %r10d, %ecx + leal (%r14,%rax,1), %r14d + addl 392(%rsp), %r13d + movl %r11d, %eax + xorl %edx, %ecx + xorl %r12d, %eax + rorxl $25, %r10d, %edx + xorl %ecx, %edx + andl %r10d, %eax + addl %edx, %r13d + rorxl $2, %r14d, %edx + rorxl $13, %r14d, %ecx + xorl %r12d, %eax + xorl %edx, %ecx + rorxl $22, %r14d, %edx + addl %eax, %r13d + xorl %ecx, %edx + movl %r15d, %eax + addl %r13d, %r9d + xorl %r14d, %eax + andl %eax, %ebx + addl %edx, %r13d + xorl %r15d, %ebx + rorxl $6, %r9d, %edx + rorxl $11, %r9d, %ecx + addl %ebx, %r13d + addl 396(%rsp), %r12d + movl %r10d, %ebx + xorl %edx, %ecx + xorl %r11d, %ebx + rorxl $25, %r9d, %edx + xorl %ecx, %edx + andl %r9d, %ebx + addl %edx, %r12d + rorxl $2, %r13d, %edx + rorxl $13, %r13d, %ecx + xorl %r11d, %ebx + xorl %edx, %ecx + rorxl $22, %r13d, %edx + addl %ebx, %r12d + xorl %ecx, %edx + movl %r14d, %ebx + leal (%r8,%r12,1), %r8d + xorl %r13d, %ebx + andl %ebx, %eax + addl %edx, %r12d + xorl %r14d, %eax + rorxl $6, %r8d, %edx + rorxl $11, %r8d, %ecx + leal (%r12,%rax,1), %r12d + addl 416(%rsp), %r11d + movl %r9d, %eax + xorl %edx, %ecx + xorl %r10d, %eax + rorxl $25, %r8d, %edx + xorl %ecx, %edx + andl %r8d, %eax + addl %edx, %r11d + rorxl $2, %r12d, %edx + rorxl $13, %r12d, %ecx + xorl %r10d, %eax + xorl %edx, %ecx + rorxl $22, %r12d, %edx + addl %eax, %r11d + xorl %ecx, %edx + movl %r13d, %eax + addl %r11d, %r15d + xorl %r12d, %eax + andl %eax, %ebx + addl %edx, %r11d + xorl %r13d, %ebx + rorxl $6, %r15d, %edx + rorxl $11, %r15d, %ecx + addl %ebx, %r11d + addl 420(%rsp), %r10d + movl %r8d, %ebx + xorl %edx, %ecx + xorl %r9d, %ebx + rorxl $25, %r15d, %edx + xorl %ecx, %edx + andl %r15d, %ebx + addl %edx, %r10d + rorxl $2, %r11d, %edx + rorxl $13, %r11d, %ecx + xorl %r9d, %ebx + xorl %edx, %ecx + rorxl $22, %r11d, %edx + addl %ebx, %r10d + xorl %ecx, %edx + movl %r12d, %ebx + leal (%r14,%r10,1), %r14d + xorl %r11d, %ebx + andl %ebx, %eax + addl %edx, %r10d + xorl %r12d, %eax + rorxl $6, %r14d, %edx + rorxl $11, %r14d, %ecx + leal (%r10,%rax,1), %r10d + addl 424(%rsp), %r9d + movl %r15d, %eax + xorl %edx, %ecx + xorl %r8d, %eax + rorxl $25, %r14d, %edx + xorl %ecx, %edx + andl %r14d, %eax + addl %edx, %r9d + rorxl $2, %r10d, %edx + rorxl $13, %r10d, %ecx + xorl %r8d, %eax + xorl %edx, %ecx + rorxl $22, %r10d, %edx + addl %eax, %r9d + xorl %ecx, %edx + movl %r11d, %eax + addl %r9d, %r13d + xorl %r10d, %eax + andl %eax, %ebx + addl %edx, %r9d + xorl %r11d, %ebx + rorxl $6, %r13d, %edx + rorxl $11, %r13d, %ecx + addl %ebx, %r9d + addl 428(%rsp), %r8d + movl %r14d, %ebx + xorl %edx, %ecx + xorl %r15d, %ebx + rorxl $25, %r13d, %edx + xorl %ecx, %edx + andl %r13d, %ebx + addl %edx, %r8d + rorxl $2, %r9d, %edx + rorxl $13, %r9d, %ecx + xorl %r15d, %ebx + xorl %edx, %ecx + rorxl $22, %r9d, %edx + addl %ebx, %r8d + xorl %ecx, %edx + movl %r10d, %ebx + leal (%r12,%r8,1), %r12d + xorl %r9d, %ebx + andl %ebx, %eax + addl %edx, %r8d + xorl %r10d, %eax + rorxl $6, %r12d, %edx + rorxl $11, %r12d, %ecx + leal (%r8,%rax,1), %r8d + addl 448(%rsp), %r15d + movl %r13d, %eax + xorl %edx, %ecx + xorl %r14d, %eax + rorxl $25, %r12d, %edx + xorl %ecx, %edx + andl %r12d, %eax + addl %edx, %r15d + rorxl $2, %r8d, %edx + rorxl $13, %r8d, %ecx + xorl %r14d, %eax + xorl %edx, %ecx + rorxl $22, %r8d, %edx + addl %eax, %r15d + xorl %ecx, %edx + movl %r9d, %eax + addl %r15d, %r11d + xorl %r8d, %eax + andl %eax, %ebx + addl %edx, %r15d + xorl %r9d, %ebx + rorxl $6, %r11d, %edx + rorxl $11, %r11d, %ecx + addl %ebx, %r15d + addl 452(%rsp), %r14d + movl %r12d, %ebx + xorl %edx, %ecx + xorl %r13d, %ebx + rorxl $25, %r11d, %edx + xorl %ecx, %edx + andl %r11d, %ebx + addl %edx, %r14d + rorxl $2, %r15d, %edx + rorxl $13, %r15d, %ecx + xorl %r13d, %ebx + xorl %edx, %ecx + rorxl $22, %r15d, %edx + addl %ebx, %r14d + xorl %ecx, %edx + movl %r8d, %ebx + leal (%r10,%r14,1), %r10d + xorl %r15d, %ebx + andl %ebx, %eax + addl %edx, %r14d + xorl %r8d, %eax + rorxl $6, %r10d, %edx + rorxl $11, %r10d, %ecx + leal (%r14,%rax,1), %r14d + addl 456(%rsp), %r13d + movl %r11d, %eax + xorl %edx, %ecx + xorl %r12d, %eax + rorxl $25, %r10d, %edx + xorl %ecx, %edx + andl %r10d, %eax + addl %edx, %r13d + rorxl $2, %r14d, %edx + rorxl $13, %r14d, %ecx + xorl %r12d, %eax + xorl %edx, %ecx + rorxl $22, %r14d, %edx + addl %eax, %r13d + xorl %ecx, %edx + movl %r15d, %eax + addl %r13d, %r9d + xorl %r14d, %eax + andl %eax, %ebx + addl %edx, %r13d + xorl %r15d, %ebx + rorxl $6, %r9d, %edx + rorxl $11, %r9d, %ecx + addl %ebx, %r13d + addl 460(%rsp), %r12d + movl %r10d, %ebx + xorl %edx, %ecx + xorl %r11d, %ebx + rorxl $25, %r9d, %edx + xorl %ecx, %edx + andl %r9d, %ebx + addl %edx, %r12d + rorxl $2, %r13d, %edx + rorxl $13, %r13d, %ecx + xorl %r11d, %ebx + xorl %edx, %ecx + rorxl $22, %r13d, %edx + addl %ebx, %r12d + xorl %ecx, %edx + movl %r14d, %ebx + leal (%r8,%r12,1), %r8d + xorl %r13d, %ebx + andl %ebx, %eax + addl %edx, %r12d + xorl %r14d, %eax + rorxl $6, %r8d, %edx + rorxl $11, %r8d, %ecx + leal (%r12,%rax,1), %r12d + addl 480(%rsp), %r11d + movl %r9d, %eax + xorl %edx, %ecx + xorl %r10d, %eax + rorxl $25, %r8d, %edx + xorl %ecx, %edx + andl %r8d, %eax + addl %edx, %r11d + rorxl $2, %r12d, %edx + rorxl $13, %r12d, %ecx + xorl %r10d, %eax + xorl %edx, %ecx + rorxl $22, %r12d, %edx + addl %eax, %r11d + xorl %ecx, %edx + movl %r13d, %eax + addl %r11d, %r15d + xorl %r12d, %eax + andl %eax, %ebx + addl %edx, %r11d + xorl %r13d, %ebx + rorxl $6, %r15d, %edx + rorxl $11, %r15d, %ecx + addl %ebx, %r11d + addl 484(%rsp), %r10d + movl %r8d, %ebx + xorl %edx, %ecx + xorl %r9d, %ebx + rorxl $25, %r15d, %edx + xorl %ecx, %edx + andl %r15d, %ebx + addl %edx, %r10d + rorxl $2, %r11d, %edx + rorxl $13, %r11d, %ecx + xorl %r9d, %ebx + xorl %edx, %ecx + rorxl $22, %r11d, %edx + addl %ebx, %r10d + xorl %ecx, %edx + movl %r12d, %ebx + leal (%r14,%r10,1), %r14d + xorl %r11d, %ebx + andl %ebx, %eax + addl %edx, %r10d + xorl %r12d, %eax + rorxl $6, %r14d, %edx + rorxl $11, %r14d, %ecx + leal (%r10,%rax,1), %r10d + addl 488(%rsp), %r9d + movl %r15d, %eax + xorl %edx, %ecx + xorl %r8d, %eax + rorxl $25, %r14d, %edx + xorl %ecx, %edx + andl %r14d, %eax + addl %edx, %r9d + rorxl $2, %r10d, %edx + rorxl $13, %r10d, %ecx + xorl %r8d, %eax + xorl %edx, %ecx + rorxl $22, %r10d, %edx + addl %eax, %r9d + xorl %ecx, %edx + movl %r11d, %eax + addl %r9d, %r13d + xorl %r10d, %eax + andl %eax, %ebx + addl %edx, %r9d + xorl %r11d, %ebx + rorxl $6, %r13d, %edx + rorxl $11, %r13d, %ecx + addl %ebx, %r9d + addl 492(%rsp), %r8d + movl %r14d, %ebx + xorl %edx, %ecx + xorl %r15d, %ebx + rorxl $25, %r13d, %edx + xorl %ecx, %edx + andl %r13d, %ebx + addl %edx, %r8d + rorxl $2, %r9d, %edx + rorxl $13, %r9d, %ecx + xorl %r15d, %ebx + xorl %edx, %ecx + rorxl $22, %r9d, %edx + addl %ebx, %r8d + xorl %ecx, %edx + movl %r10d, %ebx + leal (%r12,%r8,1), %r12d + xorl %r9d, %ebx + andl %ebx, %eax + addl %edx, %r8d + xorl %r10d, %eax + addl %eax, %r8d + xorl %ecx, %ecx + addl (%rdi), %r8d + addl 4(%rdi), %r9d + addl 8(%rdi), %r10d + addl 12(%rdi), %r11d + addl 16(%rdi), %r12d + addl 20(%rdi), %r13d + addl 24(%rdi), %r14d + addl 28(%rdi), %r15d + movl %r8d, (%rdi) + movl %r9d, 4(%rdi) + movl %r10d, 8(%rdi) + movl %r11d, 12(%rdi) + movl %r12d, 16(%rdi) + movl %r13d, 20(%rdi) + movl %r14d, 24(%rdi) + movl %r15d, 28(%rdi) + movl %r9d, %ebx + xorl %eax, %eax + xorl %r10d, %ebx + rorxl $6, %r12d, %edx + rorxl $11, %r12d, %ecx + leal (%r8,%rax,1), %r8d + addl 16(%rsp), %r15d + movl %r13d, %eax + xorl %edx, %ecx + xorl %r14d, %eax + rorxl $25, %r12d, %edx + xorl %ecx, %edx + andl %r12d, %eax + addl %edx, %r15d + rorxl $2, %r8d, %edx + rorxl $13, %r8d, %ecx + xorl %r14d, %eax + xorl %edx, %ecx + rorxl $22, %r8d, %edx + addl %eax, %r15d + xorl %ecx, %edx + movl %r9d, %eax + addl %r15d, %r11d + xorl %r8d, %eax + andl %eax, %ebx + addl %edx, %r15d + xorl %r9d, %ebx + rorxl $6, %r11d, %edx + rorxl $11, %r11d, %ecx + addl %ebx, %r15d + addl 20(%rsp), %r14d + movl %r12d, %ebx + xorl %edx, %ecx + xorl %r13d, %ebx + rorxl $25, %r11d, %edx + xorl %ecx, %edx + andl %r11d, %ebx + addl %edx, %r14d + rorxl $2, %r15d, %edx + rorxl $13, %r15d, %ecx + xorl %r13d, %ebx + xorl %edx, %ecx + rorxl $22, %r15d, %edx + addl %ebx, %r14d + xorl %ecx, %edx + movl %r8d, %ebx + leal (%r10,%r14,1), %r10d + xorl %r15d, %ebx + andl %ebx, %eax + addl %edx, %r14d + xorl %r8d, %eax + rorxl $6, %r10d, %edx + rorxl $11, %r10d, %ecx + leal (%r14,%rax,1), %r14d + addl 24(%rsp), %r13d + movl %r11d, %eax + xorl %edx, %ecx + xorl %r12d, %eax + rorxl $25, %r10d, %edx + xorl %ecx, %edx + andl %r10d, %eax + addl %edx, %r13d + rorxl $2, %r14d, %edx + rorxl $13, %r14d, %ecx + xorl %r12d, %eax + xorl %edx, %ecx + rorxl $22, %r14d, %edx + addl %eax, %r13d + xorl %ecx, %edx + movl %r15d, %eax + addl %r13d, %r9d + xorl %r14d, %eax + andl %eax, %ebx + addl %edx, %r13d + xorl %r15d, %ebx + rorxl $6, %r9d, %edx + rorxl $11, %r9d, %ecx + addl %ebx, %r13d + addl 28(%rsp), %r12d + movl %r10d, %ebx + xorl %edx, %ecx + xorl %r11d, %ebx + rorxl $25, %r9d, %edx + xorl %ecx, %edx + andl %r9d, %ebx + addl %edx, %r12d + rorxl $2, %r13d, %edx + rorxl $13, %r13d, %ecx + xorl %r11d, %ebx + xorl %edx, %ecx + rorxl $22, %r13d, %edx + addl %ebx, %r12d + xorl %ecx, %edx + movl %r14d, %ebx + leal (%r8,%r12,1), %r8d + xorl %r13d, %ebx + andl %ebx, %eax + addl %edx, %r12d + xorl %r14d, %eax + rorxl $6, %r8d, %edx + rorxl $11, %r8d, %ecx + leal (%r12,%rax,1), %r12d + addl 48(%rsp), %r11d + movl %r9d, %eax + xorl %edx, %ecx + xorl %r10d, %eax + rorxl $25, %r8d, %edx + xorl %ecx, %edx + andl %r8d, %eax + addl %edx, %r11d + rorxl $2, %r12d, %edx + rorxl $13, %r12d, %ecx + xorl %r10d, %eax + xorl %edx, %ecx + rorxl $22, %r12d, %edx + addl %eax, %r11d + xorl %ecx, %edx + movl %r13d, %eax + addl %r11d, %r15d + xorl %r12d, %eax + andl %eax, %ebx + addl %edx, %r11d + xorl %r13d, %ebx + rorxl $6, %r15d, %edx + rorxl $11, %r15d, %ecx + addl %ebx, %r11d + addl 52(%rsp), %r10d + movl %r8d, %ebx + xorl %edx, %ecx + xorl %r9d, %ebx + rorxl $25, %r15d, %edx + xorl %ecx, %edx + andl %r15d, %ebx + addl %edx, %r10d + rorxl $2, %r11d, %edx + rorxl $13, %r11d, %ecx + xorl %r9d, %ebx + xorl %edx, %ecx + rorxl $22, %r11d, %edx + addl %ebx, %r10d + xorl %ecx, %edx + movl %r12d, %ebx + leal (%r14,%r10,1), %r14d + xorl %r11d, %ebx + andl %ebx, %eax + addl %edx, %r10d + xorl %r12d, %eax + rorxl $6, %r14d, %edx + rorxl $11, %r14d, %ecx + leal (%r10,%rax,1), %r10d + addl 56(%rsp), %r9d + movl %r15d, %eax + xorl %edx, %ecx + xorl %r8d, %eax + rorxl $25, %r14d, %edx + xorl %ecx, %edx + andl %r14d, %eax + addl %edx, %r9d + rorxl $2, %r10d, %edx + rorxl $13, %r10d, %ecx + xorl %r8d, %eax + xorl %edx, %ecx + rorxl $22, %r10d, %edx + addl %eax, %r9d + xorl %ecx, %edx + movl %r11d, %eax + addl %r9d, %r13d + xorl %r10d, %eax + andl %eax, %ebx + addl %edx, %r9d + xorl %r11d, %ebx + rorxl $6, %r13d, %edx + rorxl $11, %r13d, %ecx + addl %ebx, %r9d + addl 60(%rsp), %r8d + movl %r14d, %ebx + xorl %edx, %ecx + xorl %r15d, %ebx + rorxl $25, %r13d, %edx + xorl %ecx, %edx + andl %r13d, %ebx + addl %edx, %r8d + rorxl $2, %r9d, %edx + rorxl $13, %r9d, %ecx + xorl %r15d, %ebx + xorl %edx, %ecx + rorxl $22, %r9d, %edx + addl %ebx, %r8d + xorl %ecx, %edx + movl %r10d, %ebx + leal (%r12,%r8,1), %r12d + xorl %r9d, %ebx + andl %ebx, %eax + addl %edx, %r8d + xorl %r10d, %eax + rorxl $6, %r12d, %edx + rorxl $11, %r12d, %ecx + leal (%r8,%rax,1), %r8d + addl 80(%rsp), %r15d + movl %r13d, %eax + xorl %edx, %ecx + xorl %r14d, %eax + rorxl $25, %r12d, %edx + xorl %ecx, %edx + andl %r12d, %eax + addl %edx, %r15d + rorxl $2, %r8d, %edx + rorxl $13, %r8d, %ecx + xorl %r14d, %eax + xorl %edx, %ecx + rorxl $22, %r8d, %edx + addl %eax, %r15d + xorl %ecx, %edx + movl %r9d, %eax + addl %r15d, %r11d + xorl %r8d, %eax + andl %eax, %ebx + addl %edx, %r15d + xorl %r9d, %ebx + rorxl $6, %r11d, %edx + rorxl $11, %r11d, %ecx + addl %ebx, %r15d + addl 84(%rsp), %r14d + movl %r12d, %ebx + xorl %edx, %ecx + xorl %r13d, %ebx + rorxl $25, %r11d, %edx + xorl %ecx, %edx + andl %r11d, %ebx + addl %edx, %r14d + rorxl $2, %r15d, %edx + rorxl $13, %r15d, %ecx + xorl %r13d, %ebx + xorl %edx, %ecx + rorxl $22, %r15d, %edx + addl %ebx, %r14d + xorl %ecx, %edx + movl %r8d, %ebx + leal (%r10,%r14,1), %r10d + xorl %r15d, %ebx + andl %ebx, %eax + addl %edx, %r14d + xorl %r8d, %eax + rorxl $6, %r10d, %edx + rorxl $11, %r10d, %ecx + leal (%r14,%rax,1), %r14d + addl 88(%rsp), %r13d + movl %r11d, %eax + xorl %edx, %ecx + xorl %r12d, %eax + rorxl $25, %r10d, %edx + xorl %ecx, %edx + andl %r10d, %eax + addl %edx, %r13d + rorxl $2, %r14d, %edx + rorxl $13, %r14d, %ecx + xorl %r12d, %eax + xorl %edx, %ecx + rorxl $22, %r14d, %edx + addl %eax, %r13d + xorl %ecx, %edx + movl %r15d, %eax + addl %r13d, %r9d + xorl %r14d, %eax + andl %eax, %ebx + addl %edx, %r13d + xorl %r15d, %ebx + rorxl $6, %r9d, %edx + rorxl $11, %r9d, %ecx + addl %ebx, %r13d + addl 92(%rsp), %r12d + movl %r10d, %ebx + xorl %edx, %ecx + xorl %r11d, %ebx + rorxl $25, %r9d, %edx + xorl %ecx, %edx + andl %r9d, %ebx + addl %edx, %r12d + rorxl $2, %r13d, %edx + rorxl $13, %r13d, %ecx + xorl %r11d, %ebx + xorl %edx, %ecx + rorxl $22, %r13d, %edx + addl %ebx, %r12d + xorl %ecx, %edx + movl %r14d, %ebx + leal (%r8,%r12,1), %r8d + xorl %r13d, %ebx + andl %ebx, %eax + addl %edx, %r12d + xorl %r14d, %eax + rorxl $6, %r8d, %edx + rorxl $11, %r8d, %ecx + leal (%r12,%rax,1), %r12d + addl 112(%rsp), %r11d + movl %r9d, %eax + xorl %edx, %ecx + xorl %r10d, %eax + rorxl $25, %r8d, %edx + xorl %ecx, %edx + andl %r8d, %eax + addl %edx, %r11d + rorxl $2, %r12d, %edx + rorxl $13, %r12d, %ecx + xorl %r10d, %eax + xorl %edx, %ecx + rorxl $22, %r12d, %edx + addl %eax, %r11d + xorl %ecx, %edx + movl %r13d, %eax + addl %r11d, %r15d + xorl %r12d, %eax + andl %eax, %ebx + addl %edx, %r11d + xorl %r13d, %ebx + rorxl $6, %r15d, %edx + rorxl $11, %r15d, %ecx + addl %ebx, %r11d + addl 116(%rsp), %r10d + movl %r8d, %ebx + xorl %edx, %ecx + xorl %r9d, %ebx + rorxl $25, %r15d, %edx + xorl %ecx, %edx + andl %r15d, %ebx + addl %edx, %r10d + rorxl $2, %r11d, %edx + rorxl $13, %r11d, %ecx + xorl %r9d, %ebx + xorl %edx, %ecx + rorxl $22, %r11d, %edx + addl %ebx, %r10d + xorl %ecx, %edx + movl %r12d, %ebx + leal (%r14,%r10,1), %r14d + xorl %r11d, %ebx + andl %ebx, %eax + addl %edx, %r10d + xorl %r12d, %eax + rorxl $6, %r14d, %edx + rorxl $11, %r14d, %ecx + leal (%r10,%rax,1), %r10d + addl 120(%rsp), %r9d + movl %r15d, %eax + xorl %edx, %ecx + xorl %r8d, %eax + rorxl $25, %r14d, %edx + xorl %ecx, %edx + andl %r14d, %eax + addl %edx, %r9d + rorxl $2, %r10d, %edx + rorxl $13, %r10d, %ecx + xorl %r8d, %eax + xorl %edx, %ecx + rorxl $22, %r10d, %edx + addl %eax, %r9d + xorl %ecx, %edx + movl %r11d, %eax + addl %r9d, %r13d + xorl %r10d, %eax + andl %eax, %ebx + addl %edx, %r9d + xorl %r11d, %ebx + rorxl $6, %r13d, %edx + rorxl $11, %r13d, %ecx + addl %ebx, %r9d + addl 124(%rsp), %r8d + movl %r14d, %ebx + xorl %edx, %ecx + xorl %r15d, %ebx + rorxl $25, %r13d, %edx + xorl %ecx, %edx + andl %r13d, %ebx + addl %edx, %r8d + rorxl $2, %r9d, %edx + rorxl $13, %r9d, %ecx + xorl %r15d, %ebx + xorl %edx, %ecx + rorxl $22, %r9d, %edx + addl %ebx, %r8d + xorl %ecx, %edx + movl %r10d, %ebx + leal (%r12,%r8,1), %r12d + xorl %r9d, %ebx + andl %ebx, %eax + addl %edx, %r8d + xorl %r10d, %eax + rorxl $6, %r12d, %edx + rorxl $11, %r12d, %ecx + leal (%r8,%rax,1), %r8d + addl 144(%rsp), %r15d + movl %r13d, %eax + xorl %edx, %ecx + xorl %r14d, %eax + rorxl $25, %r12d, %edx + xorl %ecx, %edx + andl %r12d, %eax + addl %edx, %r15d + rorxl $2, %r8d, %edx + rorxl $13, %r8d, %ecx + xorl %r14d, %eax + xorl %edx, %ecx + rorxl $22, %r8d, %edx + addl %eax, %r15d + xorl %ecx, %edx + movl %r9d, %eax + addl %r15d, %r11d + xorl %r8d, %eax + andl %eax, %ebx + addl %edx, %r15d + xorl %r9d, %ebx + rorxl $6, %r11d, %edx + rorxl $11, %r11d, %ecx + addl %ebx, %r15d + addl 148(%rsp), %r14d + movl %r12d, %ebx + xorl %edx, %ecx + xorl %r13d, %ebx + rorxl $25, %r11d, %edx + xorl %ecx, %edx + andl %r11d, %ebx + addl %edx, %r14d + rorxl $2, %r15d, %edx + rorxl $13, %r15d, %ecx + xorl %r13d, %ebx + xorl %edx, %ecx + rorxl $22, %r15d, %edx + addl %ebx, %r14d + xorl %ecx, %edx + movl %r8d, %ebx + leal (%r10,%r14,1), %r10d + xorl %r15d, %ebx + andl %ebx, %eax + addl %edx, %r14d + xorl %r8d, %eax + rorxl $6, %r10d, %edx + rorxl $11, %r10d, %ecx + leal (%r14,%rax,1), %r14d + addl 152(%rsp), %r13d + movl %r11d, %eax + xorl %edx, %ecx + xorl %r12d, %eax + rorxl $25, %r10d, %edx + xorl %ecx, %edx + andl %r10d, %eax + addl %edx, %r13d + rorxl $2, %r14d, %edx + rorxl $13, %r14d, %ecx + xorl %r12d, %eax + xorl %edx, %ecx + rorxl $22, %r14d, %edx + addl %eax, %r13d + xorl %ecx, %edx + movl %r15d, %eax + addl %r13d, %r9d + xorl %r14d, %eax + andl %eax, %ebx + addl %edx, %r13d + xorl %r15d, %ebx + rorxl $6, %r9d, %edx + rorxl $11, %r9d, %ecx + addl %ebx, %r13d + addl 156(%rsp), %r12d + movl %r10d, %ebx + xorl %edx, %ecx + xorl %r11d, %ebx + rorxl $25, %r9d, %edx + xorl %ecx, %edx + andl %r9d, %ebx + addl %edx, %r12d + rorxl $2, %r13d, %edx + rorxl $13, %r13d, %ecx + xorl %r11d, %ebx + xorl %edx, %ecx + rorxl $22, %r13d, %edx + addl %ebx, %r12d + xorl %ecx, %edx + movl %r14d, %ebx + leal (%r8,%r12,1), %r8d + xorl %r13d, %ebx + andl %ebx, %eax + addl %edx, %r12d + xorl %r14d, %eax + rorxl $6, %r8d, %edx + rorxl $11, %r8d, %ecx + leal (%r12,%rax,1), %r12d + addl 176(%rsp), %r11d + movl %r9d, %eax + xorl %edx, %ecx + xorl %r10d, %eax + rorxl $25, %r8d, %edx + xorl %ecx, %edx + andl %r8d, %eax + addl %edx, %r11d + rorxl $2, %r12d, %edx + rorxl $13, %r12d, %ecx + xorl %r10d, %eax + xorl %edx, %ecx + rorxl $22, %r12d, %edx + addl %eax, %r11d + xorl %ecx, %edx + movl %r13d, %eax + addl %r11d, %r15d + xorl %r12d, %eax + andl %eax, %ebx + addl %edx, %r11d + xorl %r13d, %ebx + rorxl $6, %r15d, %edx + rorxl $11, %r15d, %ecx + addl %ebx, %r11d + addl 180(%rsp), %r10d + movl %r8d, %ebx + xorl %edx, %ecx + xorl %r9d, %ebx + rorxl $25, %r15d, %edx + xorl %ecx, %edx + andl %r15d, %ebx + addl %edx, %r10d + rorxl $2, %r11d, %edx + rorxl $13, %r11d, %ecx + xorl %r9d, %ebx + xorl %edx, %ecx + rorxl $22, %r11d, %edx + addl %ebx, %r10d + xorl %ecx, %edx + movl %r12d, %ebx + leal (%r14,%r10,1), %r14d + xorl %r11d, %ebx + andl %ebx, %eax + addl %edx, %r10d + xorl %r12d, %eax + rorxl $6, %r14d, %edx + rorxl $11, %r14d, %ecx + leal (%r10,%rax,1), %r10d + addl 184(%rsp), %r9d + movl %r15d, %eax + xorl %edx, %ecx + xorl %r8d, %eax + rorxl $25, %r14d, %edx + xorl %ecx, %edx + andl %r14d, %eax + addl %edx, %r9d + rorxl $2, %r10d, %edx + rorxl $13, %r10d, %ecx + xorl %r8d, %eax + xorl %edx, %ecx + rorxl $22, %r10d, %edx + addl %eax, %r9d + xorl %ecx, %edx + movl %r11d, %eax + addl %r9d, %r13d + xorl %r10d, %eax + andl %eax, %ebx + addl %edx, %r9d + xorl %r11d, %ebx + rorxl $6, %r13d, %edx + rorxl $11, %r13d, %ecx + addl %ebx, %r9d + addl 188(%rsp), %r8d + movl %r14d, %ebx + xorl %edx, %ecx + xorl %r15d, %ebx + rorxl $25, %r13d, %edx + xorl %ecx, %edx + andl %r13d, %ebx + addl %edx, %r8d + rorxl $2, %r9d, %edx + rorxl $13, %r9d, %ecx + xorl %r15d, %ebx + xorl %edx, %ecx + rorxl $22, %r9d, %edx + addl %ebx, %r8d + xorl %ecx, %edx + movl %r10d, %ebx + leal (%r12,%r8,1), %r12d + xorl %r9d, %ebx + andl %ebx, %eax + addl %edx, %r8d + xorl %r10d, %eax + rorxl $6, %r12d, %edx + rorxl $11, %r12d, %ecx + leal (%r8,%rax,1), %r8d + addl 208(%rsp), %r15d + movl %r13d, %eax + xorl %edx, %ecx + xorl %r14d, %eax + rorxl $25, %r12d, %edx + xorl %ecx, %edx + andl %r12d, %eax + addl %edx, %r15d + rorxl $2, %r8d, %edx + rorxl $13, %r8d, %ecx + xorl %r14d, %eax + xorl %edx, %ecx + rorxl $22, %r8d, %edx + addl %eax, %r15d + xorl %ecx, %edx + movl %r9d, %eax + addl %r15d, %r11d + xorl %r8d, %eax + andl %eax, %ebx + addl %edx, %r15d + xorl %r9d, %ebx + rorxl $6, %r11d, %edx + rorxl $11, %r11d, %ecx + addl %ebx, %r15d + addl 212(%rsp), %r14d + movl %r12d, %ebx + xorl %edx, %ecx + xorl %r13d, %ebx + rorxl $25, %r11d, %edx + xorl %ecx, %edx + andl %r11d, %ebx + addl %edx, %r14d + rorxl $2, %r15d, %edx + rorxl $13, %r15d, %ecx + xorl %r13d, %ebx + xorl %edx, %ecx + rorxl $22, %r15d, %edx + addl %ebx, %r14d + xorl %ecx, %edx + movl %r8d, %ebx + leal (%r10,%r14,1), %r10d + xorl %r15d, %ebx + andl %ebx, %eax + addl %edx, %r14d + xorl %r8d, %eax + rorxl $6, %r10d, %edx + rorxl $11, %r10d, %ecx + leal (%r14,%rax,1), %r14d + addl 216(%rsp), %r13d + movl %r11d, %eax + xorl %edx, %ecx + xorl %r12d, %eax + rorxl $25, %r10d, %edx + xorl %ecx, %edx + andl %r10d, %eax + addl %edx, %r13d + rorxl $2, %r14d, %edx + rorxl $13, %r14d, %ecx + xorl %r12d, %eax + xorl %edx, %ecx + rorxl $22, %r14d, %edx + addl %eax, %r13d + xorl %ecx, %edx + movl %r15d, %eax + addl %r13d, %r9d + xorl %r14d, %eax + andl %eax, %ebx + addl %edx, %r13d + xorl %r15d, %ebx + rorxl $6, %r9d, %edx + rorxl $11, %r9d, %ecx + addl %ebx, %r13d + addl 220(%rsp), %r12d + movl %r10d, %ebx + xorl %edx, %ecx + xorl %r11d, %ebx + rorxl $25, %r9d, %edx + xorl %ecx, %edx + andl %r9d, %ebx + addl %edx, %r12d + rorxl $2, %r13d, %edx + rorxl $13, %r13d, %ecx + xorl %r11d, %ebx + xorl %edx, %ecx + rorxl $22, %r13d, %edx + addl %ebx, %r12d + xorl %ecx, %edx + movl %r14d, %ebx + leal (%r8,%r12,1), %r8d + xorl %r13d, %ebx + andl %ebx, %eax + addl %edx, %r12d + xorl %r14d, %eax + rorxl $6, %r8d, %edx + rorxl $11, %r8d, %ecx + leal (%r12,%rax,1), %r12d + addl 240(%rsp), %r11d + movl %r9d, %eax + xorl %edx, %ecx + xorl %r10d, %eax + rorxl $25, %r8d, %edx + xorl %ecx, %edx + andl %r8d, %eax + addl %edx, %r11d + rorxl $2, %r12d, %edx + rorxl $13, %r12d, %ecx + xorl %r10d, %eax + xorl %edx, %ecx + rorxl $22, %r12d, %edx + addl %eax, %r11d + xorl %ecx, %edx + movl %r13d, %eax + addl %r11d, %r15d + xorl %r12d, %eax + andl %eax, %ebx + addl %edx, %r11d + xorl %r13d, %ebx + rorxl $6, %r15d, %edx + rorxl $11, %r15d, %ecx + addl %ebx, %r11d + addl 244(%rsp), %r10d + movl %r8d, %ebx + xorl %edx, %ecx + xorl %r9d, %ebx + rorxl $25, %r15d, %edx + xorl %ecx, %edx + andl %r15d, %ebx + addl %edx, %r10d + rorxl $2, %r11d, %edx + rorxl $13, %r11d, %ecx + xorl %r9d, %ebx + xorl %edx, %ecx + rorxl $22, %r11d, %edx + addl %ebx, %r10d + xorl %ecx, %edx + movl %r12d, %ebx + leal (%r14,%r10,1), %r14d + xorl %r11d, %ebx + andl %ebx, %eax + addl %edx, %r10d + xorl %r12d, %eax + rorxl $6, %r14d, %edx + rorxl $11, %r14d, %ecx + leal (%r10,%rax,1), %r10d + addl 248(%rsp), %r9d + movl %r15d, %eax + xorl %edx, %ecx + xorl %r8d, %eax + rorxl $25, %r14d, %edx + xorl %ecx, %edx + andl %r14d, %eax + addl %edx, %r9d + rorxl $2, %r10d, %edx + rorxl $13, %r10d, %ecx + xorl %r8d, %eax + xorl %edx, %ecx + rorxl $22, %r10d, %edx + addl %eax, %r9d + xorl %ecx, %edx + movl %r11d, %eax + addl %r9d, %r13d + xorl %r10d, %eax + andl %eax, %ebx + addl %edx, %r9d + xorl %r11d, %ebx + rorxl $6, %r13d, %edx + rorxl $11, %r13d, %ecx + addl %ebx, %r9d + addl 252(%rsp), %r8d + movl %r14d, %ebx + xorl %edx, %ecx + xorl %r15d, %ebx + rorxl $25, %r13d, %edx + xorl %ecx, %edx + andl %r13d, %ebx + addl %edx, %r8d + rorxl $2, %r9d, %edx + rorxl $13, %r9d, %ecx + xorl %r15d, %ebx + xorl %edx, %ecx + rorxl $22, %r9d, %edx + addl %ebx, %r8d + xorl %ecx, %edx + movl %r10d, %ebx + leal (%r12,%r8,1), %r12d + xorl %r9d, %ebx + andl %ebx, %eax + addl %edx, %r8d + xorl %r10d, %eax + rorxl $6, %r12d, %edx + rorxl $11, %r12d, %ecx + leal (%r8,%rax,1), %r8d + addl 272(%rsp), %r15d + movl %r13d, %eax + xorl %edx, %ecx + xorl %r14d, %eax + rorxl $25, %r12d, %edx + xorl %ecx, %edx + andl %r12d, %eax + addl %edx, %r15d + rorxl $2, %r8d, %edx + rorxl $13, %r8d, %ecx + xorl %r14d, %eax + xorl %edx, %ecx + rorxl $22, %r8d, %edx + addl %eax, %r15d + xorl %ecx, %edx + movl %r9d, %eax + addl %r15d, %r11d + xorl %r8d, %eax + andl %eax, %ebx + addl %edx, %r15d + xorl %r9d, %ebx + rorxl $6, %r11d, %edx + rorxl $11, %r11d, %ecx + addl %ebx, %r15d + addl 276(%rsp), %r14d + movl %r12d, %ebx + xorl %edx, %ecx + xorl %r13d, %ebx + rorxl $25, %r11d, %edx + xorl %ecx, %edx + andl %r11d, %ebx + addl %edx, %r14d + rorxl $2, %r15d, %edx + rorxl $13, %r15d, %ecx + xorl %r13d, %ebx + xorl %edx, %ecx + rorxl $22, %r15d, %edx + addl %ebx, %r14d + xorl %ecx, %edx + movl %r8d, %ebx + leal (%r10,%r14,1), %r10d + xorl %r15d, %ebx + andl %ebx, %eax + addl %edx, %r14d + xorl %r8d, %eax + rorxl $6, %r10d, %edx + rorxl $11, %r10d, %ecx + leal (%r14,%rax,1), %r14d + addl 280(%rsp), %r13d + movl %r11d, %eax + xorl %edx, %ecx + xorl %r12d, %eax + rorxl $25, %r10d, %edx + xorl %ecx, %edx + andl %r10d, %eax + addl %edx, %r13d + rorxl $2, %r14d, %edx + rorxl $13, %r14d, %ecx + xorl %r12d, %eax + xorl %edx, %ecx + rorxl $22, %r14d, %edx + addl %eax, %r13d + xorl %ecx, %edx + movl %r15d, %eax + addl %r13d, %r9d + xorl %r14d, %eax + andl %eax, %ebx + addl %edx, %r13d + xorl %r15d, %ebx + rorxl $6, %r9d, %edx + rorxl $11, %r9d, %ecx + addl %ebx, %r13d + addl 284(%rsp), %r12d + movl %r10d, %ebx + xorl %edx, %ecx + xorl %r11d, %ebx + rorxl $25, %r9d, %edx + xorl %ecx, %edx + andl %r9d, %ebx + addl %edx, %r12d + rorxl $2, %r13d, %edx + rorxl $13, %r13d, %ecx + xorl %r11d, %ebx + xorl %edx, %ecx + rorxl $22, %r13d, %edx + addl %ebx, %r12d + xorl %ecx, %edx + movl %r14d, %ebx + leal (%r8,%r12,1), %r8d + xorl %r13d, %ebx + andl %ebx, %eax + addl %edx, %r12d + xorl %r14d, %eax + rorxl $6, %r8d, %edx + rorxl $11, %r8d, %ecx + leal (%r12,%rax,1), %r12d + addl 304(%rsp), %r11d + movl %r9d, %eax + xorl %edx, %ecx + xorl %r10d, %eax + rorxl $25, %r8d, %edx + xorl %ecx, %edx + andl %r8d, %eax + addl %edx, %r11d + rorxl $2, %r12d, %edx + rorxl $13, %r12d, %ecx + xorl %r10d, %eax + xorl %edx, %ecx + rorxl $22, %r12d, %edx + addl %eax, %r11d + xorl %ecx, %edx + movl %r13d, %eax + addl %r11d, %r15d + xorl %r12d, %eax + andl %eax, %ebx + addl %edx, %r11d + xorl %r13d, %ebx + rorxl $6, %r15d, %edx + rorxl $11, %r15d, %ecx + addl %ebx, %r11d + addl 308(%rsp), %r10d + movl %r8d, %ebx + xorl %edx, %ecx + xorl %r9d, %ebx + rorxl $25, %r15d, %edx + xorl %ecx, %edx + andl %r15d, %ebx + addl %edx, %r10d + rorxl $2, %r11d, %edx + rorxl $13, %r11d, %ecx + xorl %r9d, %ebx + xorl %edx, %ecx + rorxl $22, %r11d, %edx + addl %ebx, %r10d + xorl %ecx, %edx + movl %r12d, %ebx + leal (%r14,%r10,1), %r14d + xorl %r11d, %ebx + andl %ebx, %eax + addl %edx, %r10d + xorl %r12d, %eax + rorxl $6, %r14d, %edx + rorxl $11, %r14d, %ecx + leal (%r10,%rax,1), %r10d + addl 312(%rsp), %r9d + movl %r15d, %eax + xorl %edx, %ecx + xorl %r8d, %eax + rorxl $25, %r14d, %edx + xorl %ecx, %edx + andl %r14d, %eax + addl %edx, %r9d + rorxl $2, %r10d, %edx + rorxl $13, %r10d, %ecx + xorl %r8d, %eax + xorl %edx, %ecx + rorxl $22, %r10d, %edx + addl %eax, %r9d + xorl %ecx, %edx + movl %r11d, %eax + addl %r9d, %r13d + xorl %r10d, %eax + andl %eax, %ebx + addl %edx, %r9d + xorl %r11d, %ebx + rorxl $6, %r13d, %edx + rorxl $11, %r13d, %ecx + addl %ebx, %r9d + addl 316(%rsp), %r8d + movl %r14d, %ebx + xorl %edx, %ecx + xorl %r15d, %ebx + rorxl $25, %r13d, %edx + xorl %ecx, %edx + andl %r13d, %ebx + addl %edx, %r8d + rorxl $2, %r9d, %edx + rorxl $13, %r9d, %ecx + xorl %r15d, %ebx + xorl %edx, %ecx + rorxl $22, %r9d, %edx + addl %ebx, %r8d + xorl %ecx, %edx + movl %r10d, %ebx + leal (%r12,%r8,1), %r12d + xorl %r9d, %ebx + andl %ebx, %eax + addl %edx, %r8d + xorl %r10d, %eax + rorxl $6, %r12d, %edx + rorxl $11, %r12d, %ecx + leal (%r8,%rax,1), %r8d + addl 336(%rsp), %r15d + movl %r13d, %eax + xorl %edx, %ecx + xorl %r14d, %eax + rorxl $25, %r12d, %edx + xorl %ecx, %edx + andl %r12d, %eax + addl %edx, %r15d + rorxl $2, %r8d, %edx + rorxl $13, %r8d, %ecx + xorl %r14d, %eax + xorl %edx, %ecx + rorxl $22, %r8d, %edx + addl %eax, %r15d + xorl %ecx, %edx + movl %r9d, %eax + addl %r15d, %r11d + xorl %r8d, %eax + andl %eax, %ebx + addl %edx, %r15d + xorl %r9d, %ebx + rorxl $6, %r11d, %edx + rorxl $11, %r11d, %ecx + addl %ebx, %r15d + addl 340(%rsp), %r14d + movl %r12d, %ebx + xorl %edx, %ecx + xorl %r13d, %ebx + rorxl $25, %r11d, %edx + xorl %ecx, %edx + andl %r11d, %ebx + addl %edx, %r14d + rorxl $2, %r15d, %edx + rorxl $13, %r15d, %ecx + xorl %r13d, %ebx + xorl %edx, %ecx + rorxl $22, %r15d, %edx + addl %ebx, %r14d + xorl %ecx, %edx + movl %r8d, %ebx + leal (%r10,%r14,1), %r10d + xorl %r15d, %ebx + andl %ebx, %eax + addl %edx, %r14d + xorl %r8d, %eax + rorxl $6, %r10d, %edx + rorxl $11, %r10d, %ecx + leal (%r14,%rax,1), %r14d + addl 344(%rsp), %r13d + movl %r11d, %eax + xorl %edx, %ecx + xorl %r12d, %eax + rorxl $25, %r10d, %edx + xorl %ecx, %edx + andl %r10d, %eax + addl %edx, %r13d + rorxl $2, %r14d, %edx + rorxl $13, %r14d, %ecx + xorl %r12d, %eax + xorl %edx, %ecx + rorxl $22, %r14d, %edx + addl %eax, %r13d + xorl %ecx, %edx + movl %r15d, %eax + addl %r13d, %r9d + xorl %r14d, %eax + andl %eax, %ebx + addl %edx, %r13d + xorl %r15d, %ebx + rorxl $6, %r9d, %edx + rorxl $11, %r9d, %ecx + addl %ebx, %r13d + addl 348(%rsp), %r12d + movl %r10d, %ebx + xorl %edx, %ecx + xorl %r11d, %ebx + rorxl $25, %r9d, %edx + xorl %ecx, %edx + andl %r9d, %ebx + addl %edx, %r12d + rorxl $2, %r13d, %edx + rorxl $13, %r13d, %ecx + xorl %r11d, %ebx + xorl %edx, %ecx + rorxl $22, %r13d, %edx + addl %ebx, %r12d + xorl %ecx, %edx + movl %r14d, %ebx + leal (%r8,%r12,1), %r8d + xorl %r13d, %ebx + andl %ebx, %eax + addl %edx, %r12d + xorl %r14d, %eax + rorxl $6, %r8d, %edx + rorxl $11, %r8d, %ecx + leal (%r12,%rax,1), %r12d + addl 368(%rsp), %r11d + movl %r9d, %eax + xorl %edx, %ecx + xorl %r10d, %eax + rorxl $25, %r8d, %edx + xorl %ecx, %edx + andl %r8d, %eax + addl %edx, %r11d + rorxl $2, %r12d, %edx + rorxl $13, %r12d, %ecx + xorl %r10d, %eax + xorl %edx, %ecx + rorxl $22, %r12d, %edx + addl %eax, %r11d + xorl %ecx, %edx + movl %r13d, %eax + addl %r11d, %r15d + xorl %r12d, %eax + andl %eax, %ebx + addl %edx, %r11d + xorl %r13d, %ebx + rorxl $6, %r15d, %edx + rorxl $11, %r15d, %ecx + addl %ebx, %r11d + addl 372(%rsp), %r10d + movl %r8d, %ebx + xorl %edx, %ecx + xorl %r9d, %ebx + rorxl $25, %r15d, %edx + xorl %ecx, %edx + andl %r15d, %ebx + addl %edx, %r10d + rorxl $2, %r11d, %edx + rorxl $13, %r11d, %ecx + xorl %r9d, %ebx + xorl %edx, %ecx + rorxl $22, %r11d, %edx + addl %ebx, %r10d + xorl %ecx, %edx + movl %r12d, %ebx + leal (%r14,%r10,1), %r14d + xorl %r11d, %ebx + andl %ebx, %eax + addl %edx, %r10d + xorl %r12d, %eax + rorxl $6, %r14d, %edx + rorxl $11, %r14d, %ecx + leal (%r10,%rax,1), %r10d + addl 376(%rsp), %r9d + movl %r15d, %eax + xorl %edx, %ecx + xorl %r8d, %eax + rorxl $25, %r14d, %edx + xorl %ecx, %edx + andl %r14d, %eax + addl %edx, %r9d + rorxl $2, %r10d, %edx + rorxl $13, %r10d, %ecx + xorl %r8d, %eax + xorl %edx, %ecx + rorxl $22, %r10d, %edx + addl %eax, %r9d + xorl %ecx, %edx + movl %r11d, %eax + addl %r9d, %r13d + xorl %r10d, %eax + andl %eax, %ebx + addl %edx, %r9d + xorl %r11d, %ebx + rorxl $6, %r13d, %edx + rorxl $11, %r13d, %ecx + addl %ebx, %r9d + addl 380(%rsp), %r8d + movl %r14d, %ebx + xorl %edx, %ecx + xorl %r15d, %ebx + rorxl $25, %r13d, %edx + xorl %ecx, %edx + andl %r13d, %ebx + addl %edx, %r8d + rorxl $2, %r9d, %edx + rorxl $13, %r9d, %ecx + xorl %r15d, %ebx + xorl %edx, %ecx + rorxl $22, %r9d, %edx + addl %ebx, %r8d + xorl %ecx, %edx + movl %r10d, %ebx + leal (%r12,%r8,1), %r12d + xorl %r9d, %ebx + andl %ebx, %eax + addl %edx, %r8d + xorl %r10d, %eax + rorxl $6, %r12d, %edx + rorxl $11, %r12d, %ecx + leal (%r8,%rax,1), %r8d + addl 400(%rsp), %r15d + movl %r13d, %eax + xorl %edx, %ecx + xorl %r14d, %eax + rorxl $25, %r12d, %edx + xorl %ecx, %edx + andl %r12d, %eax + addl %edx, %r15d + rorxl $2, %r8d, %edx + rorxl $13, %r8d, %ecx + xorl %r14d, %eax + xorl %edx, %ecx + rorxl $22, %r8d, %edx + addl %eax, %r15d + xorl %ecx, %edx + movl %r9d, %eax + addl %r15d, %r11d + xorl %r8d, %eax + andl %eax, %ebx + addl %edx, %r15d + xorl %r9d, %ebx + rorxl $6, %r11d, %edx + rorxl $11, %r11d, %ecx + addl %ebx, %r15d + addl 404(%rsp), %r14d + movl %r12d, %ebx + xorl %edx, %ecx + xorl %r13d, %ebx + rorxl $25, %r11d, %edx + xorl %ecx, %edx + andl %r11d, %ebx + addl %edx, %r14d + rorxl $2, %r15d, %edx + rorxl $13, %r15d, %ecx + xorl %r13d, %ebx + xorl %edx, %ecx + rorxl $22, %r15d, %edx + addl %ebx, %r14d + xorl %ecx, %edx + movl %r8d, %ebx + leal (%r10,%r14,1), %r10d + xorl %r15d, %ebx + andl %ebx, %eax + addl %edx, %r14d + xorl %r8d, %eax + rorxl $6, %r10d, %edx + rorxl $11, %r10d, %ecx + leal (%r14,%rax,1), %r14d + addl 408(%rsp), %r13d + movl %r11d, %eax + xorl %edx, %ecx + xorl %r12d, %eax + rorxl $25, %r10d, %edx + xorl %ecx, %edx + andl %r10d, %eax + addl %edx, %r13d + rorxl $2, %r14d, %edx + rorxl $13, %r14d, %ecx + xorl %r12d, %eax + xorl %edx, %ecx + rorxl $22, %r14d, %edx + addl %eax, %r13d + xorl %ecx, %edx + movl %r15d, %eax + addl %r13d, %r9d + xorl %r14d, %eax + andl %eax, %ebx + addl %edx, %r13d + xorl %r15d, %ebx + rorxl $6, %r9d, %edx + rorxl $11, %r9d, %ecx + addl %ebx, %r13d + addl 412(%rsp), %r12d + movl %r10d, %ebx + xorl %edx, %ecx + xorl %r11d, %ebx + rorxl $25, %r9d, %edx + xorl %ecx, %edx + andl %r9d, %ebx + addl %edx, %r12d + rorxl $2, %r13d, %edx + rorxl $13, %r13d, %ecx + xorl %r11d, %ebx + xorl %edx, %ecx + rorxl $22, %r13d, %edx + addl %ebx, %r12d + xorl %ecx, %edx + movl %r14d, %ebx + leal (%r8,%r12,1), %r8d + xorl %r13d, %ebx + andl %ebx, %eax + addl %edx, %r12d + xorl %r14d, %eax + rorxl $6, %r8d, %edx + rorxl $11, %r8d, %ecx + leal (%r12,%rax,1), %r12d + addl 432(%rsp), %r11d + movl %r9d, %eax + xorl %edx, %ecx + xorl %r10d, %eax + rorxl $25, %r8d, %edx + xorl %ecx, %edx + andl %r8d, %eax + addl %edx, %r11d + rorxl $2, %r12d, %edx + rorxl $13, %r12d, %ecx + xorl %r10d, %eax + xorl %edx, %ecx + rorxl $22, %r12d, %edx + addl %eax, %r11d + xorl %ecx, %edx + movl %r13d, %eax + addl %r11d, %r15d + xorl %r12d, %eax + andl %eax, %ebx + addl %edx, %r11d + xorl %r13d, %ebx + rorxl $6, %r15d, %edx + rorxl $11, %r15d, %ecx + addl %ebx, %r11d + addl 436(%rsp), %r10d + movl %r8d, %ebx + xorl %edx, %ecx + xorl %r9d, %ebx + rorxl $25, %r15d, %edx + xorl %ecx, %edx + andl %r15d, %ebx + addl %edx, %r10d + rorxl $2, %r11d, %edx + rorxl $13, %r11d, %ecx + xorl %r9d, %ebx + xorl %edx, %ecx + rorxl $22, %r11d, %edx + addl %ebx, %r10d + xorl %ecx, %edx + movl %r12d, %ebx + leal (%r14,%r10,1), %r14d + xorl %r11d, %ebx + andl %ebx, %eax + addl %edx, %r10d + xorl %r12d, %eax + rorxl $6, %r14d, %edx + rorxl $11, %r14d, %ecx + leal (%r10,%rax,1), %r10d + addl 440(%rsp), %r9d + movl %r15d, %eax + xorl %edx, %ecx + xorl %r8d, %eax + rorxl $25, %r14d, %edx + xorl %ecx, %edx + andl %r14d, %eax + addl %edx, %r9d + rorxl $2, %r10d, %edx + rorxl $13, %r10d, %ecx + xorl %r8d, %eax + xorl %edx, %ecx + rorxl $22, %r10d, %edx + addl %eax, %r9d + xorl %ecx, %edx + movl %r11d, %eax + addl %r9d, %r13d + xorl %r10d, %eax + andl %eax, %ebx + addl %edx, %r9d + xorl %r11d, %ebx + rorxl $6, %r13d, %edx + rorxl $11, %r13d, %ecx + addl %ebx, %r9d + addl 444(%rsp), %r8d + movl %r14d, %ebx + xorl %edx, %ecx + xorl %r15d, %ebx + rorxl $25, %r13d, %edx + xorl %ecx, %edx + andl %r13d, %ebx + addl %edx, %r8d + rorxl $2, %r9d, %edx + rorxl $13, %r9d, %ecx + xorl %r15d, %ebx + xorl %edx, %ecx + rorxl $22, %r9d, %edx + addl %ebx, %r8d + xorl %ecx, %edx + movl %r10d, %ebx + leal (%r12,%r8,1), %r12d + xorl %r9d, %ebx + andl %ebx, %eax + addl %edx, %r8d + xorl %r10d, %eax + rorxl $6, %r12d, %edx + rorxl $11, %r12d, %ecx + leal (%r8,%rax,1), %r8d + addl 464(%rsp), %r15d + movl %r13d, %eax + xorl %edx, %ecx + xorl %r14d, %eax + rorxl $25, %r12d, %edx + xorl %ecx, %edx + andl %r12d, %eax + addl %edx, %r15d + rorxl $2, %r8d, %edx + rorxl $13, %r8d, %ecx + xorl %r14d, %eax + xorl %edx, %ecx + rorxl $22, %r8d, %edx + addl %eax, %r15d + xorl %ecx, %edx + movl %r9d, %eax + addl %r15d, %r11d + xorl %r8d, %eax + andl %eax, %ebx + addl %edx, %r15d + xorl %r9d, %ebx + rorxl $6, %r11d, %edx + rorxl $11, %r11d, %ecx + addl %ebx, %r15d + addl 468(%rsp), %r14d + movl %r12d, %ebx + xorl %edx, %ecx + xorl %r13d, %ebx + rorxl $25, %r11d, %edx + xorl %ecx, %edx + andl %r11d, %ebx + addl %edx, %r14d + rorxl $2, %r15d, %edx + rorxl $13, %r15d, %ecx + xorl %r13d, %ebx + xorl %edx, %ecx + rorxl $22, %r15d, %edx + addl %ebx, %r14d + xorl %ecx, %edx + movl %r8d, %ebx + leal (%r10,%r14,1), %r10d + xorl %r15d, %ebx + andl %ebx, %eax + addl %edx, %r14d + xorl %r8d, %eax + rorxl $6, %r10d, %edx + rorxl $11, %r10d, %ecx + leal (%r14,%rax,1), %r14d + addl 472(%rsp), %r13d + movl %r11d, %eax + xorl %edx, %ecx + xorl %r12d, %eax + rorxl $25, %r10d, %edx + xorl %ecx, %edx + andl %r10d, %eax + addl %edx, %r13d + rorxl $2, %r14d, %edx + rorxl $13, %r14d, %ecx + xorl %r12d, %eax + xorl %edx, %ecx + rorxl $22, %r14d, %edx + addl %eax, %r13d + xorl %ecx, %edx + movl %r15d, %eax + addl %r13d, %r9d + xorl %r14d, %eax + andl %eax, %ebx + addl %edx, %r13d + xorl %r15d, %ebx + rorxl $6, %r9d, %edx + rorxl $11, %r9d, %ecx + addl %ebx, %r13d + addl 476(%rsp), %r12d + movl %r10d, %ebx + xorl %edx, %ecx + xorl %r11d, %ebx + rorxl $25, %r9d, %edx + xorl %ecx, %edx + andl %r9d, %ebx + addl %edx, %r12d + rorxl $2, %r13d, %edx + rorxl $13, %r13d, %ecx + xorl %r11d, %ebx + xorl %edx, %ecx + rorxl $22, %r13d, %edx + addl %ebx, %r12d + xorl %ecx, %edx + movl %r14d, %ebx + leal (%r8,%r12,1), %r8d + xorl %r13d, %ebx + andl %ebx, %eax + addl %edx, %r12d + xorl %r14d, %eax + rorxl $6, %r8d, %edx + rorxl $11, %r8d, %ecx + leal (%r12,%rax,1), %r12d + addl 496(%rsp), %r11d + movl %r9d, %eax + xorl %edx, %ecx + xorl %r10d, %eax + rorxl $25, %r8d, %edx + xorl %ecx, %edx + andl %r8d, %eax + addl %edx, %r11d + rorxl $2, %r12d, %edx + rorxl $13, %r12d, %ecx + xorl %r10d, %eax + xorl %edx, %ecx + rorxl $22, %r12d, %edx + addl %eax, %r11d + xorl %ecx, %edx + movl %r13d, %eax + addl %r11d, %r15d + xorl %r12d, %eax + andl %eax, %ebx + addl %edx, %r11d + xorl %r13d, %ebx + rorxl $6, %r15d, %edx + rorxl $11, %r15d, %ecx + addl %ebx, %r11d + addl 500(%rsp), %r10d + movl %r8d, %ebx + xorl %edx, %ecx + xorl %r9d, %ebx + rorxl $25, %r15d, %edx + xorl %ecx, %edx + andl %r15d, %ebx + addl %edx, %r10d + rorxl $2, %r11d, %edx + rorxl $13, %r11d, %ecx + xorl %r9d, %ebx + xorl %edx, %ecx + rorxl $22, %r11d, %edx + addl %ebx, %r10d + xorl %ecx, %edx + movl %r12d, %ebx + leal (%r14,%r10,1), %r14d + xorl %r11d, %ebx + andl %ebx, %eax + addl %edx, %r10d + xorl %r12d, %eax + rorxl $6, %r14d, %edx + rorxl $11, %r14d, %ecx + leal (%r10,%rax,1), %r10d + addl 504(%rsp), %r9d + movl %r15d, %eax + xorl %edx, %ecx + xorl %r8d, %eax + rorxl $25, %r14d, %edx + xorl %ecx, %edx + andl %r14d, %eax + addl %edx, %r9d + rorxl $2, %r10d, %edx + rorxl $13, %r10d, %ecx + xorl %r8d, %eax + xorl %edx, %ecx + rorxl $22, %r10d, %edx + addl %eax, %r9d + xorl %ecx, %edx + movl %r11d, %eax + addl %r9d, %r13d + xorl %r10d, %eax + andl %eax, %ebx + addl %edx, %r9d + xorl %r11d, %ebx + rorxl $6, %r13d, %edx + rorxl $11, %r13d, %ecx + addl %ebx, %r9d + addl 508(%rsp), %r8d + movl %r14d, %ebx + xorl %edx, %ecx + xorl %r15d, %ebx + rorxl $25, %r13d, %edx + xorl %ecx, %edx + andl %r13d, %ebx + addl %edx, %r8d + rorxl $2, %r9d, %edx + rorxl $13, %r9d, %ecx + xorl %r15d, %ebx + xorl %edx, %ecx + rorxl $22, %r9d, %edx + addl %ebx, %r8d + xorl %ecx, %edx + movl %r10d, %ebx + leal (%r12,%r8,1), %r12d + xorl %r9d, %ebx + andl %ebx, %eax + addl %edx, %r8d + xorl %r10d, %eax + addl %eax, %r8d + addq $0x80, %rbp + addl (%rdi), %r8d + addl 4(%rdi), %r9d + addl 8(%rdi), %r10d + addl 12(%rdi), %r11d + addl 16(%rdi), %r12d + addl 20(%rdi), %r13d + addl 24(%rdi), %r14d + addl 28(%rdi), %r15d + subl $0x80, %esi + movl %r8d, (%rdi) + movl %r9d, 4(%rdi) + movl %r10d, 8(%rdi) + movl %r11d, 12(%rdi) + movl %r12d, 16(%rdi) + movl %r13d, 20(%rdi) + movl %r14d, 24(%rdi) + movl %r15d, 28(%rdi) + jnz L_sha256_len_avx2_rorx_start +L_sha256_len_avx2_rorx_done: + xorq %rax, %rax + vzeroupper + addq $0x200, %rsp + popq %rbp + popq %r15 + popq %r14 + popq %r13 + popq %r12 + popq %rbx + repz retq +.size Transform_Sha256_AVX2_RORX_Len,.-Transform_Sha256_AVX2_RORX_Len +#endif /* HAVE_INTEL_AVX2 */ diff --git a/wolfcrypt/src/sha512.c b/wolfcrypt/src/sha512.c index 5fa3b90487..584a02ae74 100644 --- a/wolfcrypt/src/sha512.c +++ b/wolfcrypt/src/sha512.c @@ -303,22 +303,31 @@ static int InitSha512(wc_Sha512* sha512) * supported. Also, let's setup a macro for proper linkage w/o ABI conflicts */ +#ifdef __cplusplus + extern "C" { +#endif + #if defined(HAVE_INTEL_AVX1) - static int Transform_Sha512_AVX1(wc_Sha512 *sha512); - static int Transform_Sha512_AVX1_Len(wc_Sha512 *sha512, word32 len); + extern int Transform_Sha512_AVX1(wc_Sha512 *sha512); + extern int Transform_Sha512_AVX1_Len(wc_Sha512 *sha512, word32 len); #endif #if defined(HAVE_INTEL_AVX2) - static int Transform_Sha512_AVX2(wc_Sha512 *sha512); - static int Transform_Sha512_AVX2_Len(wc_Sha512 *sha512, word32 len); + extern int Transform_Sha512_AVX2(wc_Sha512 *sha512); + extern int Transform_Sha512_AVX2_Len(wc_Sha512 *sha512, word32 len); #if defined(HAVE_INTEL_RORX) - static int Transform_Sha512_AVX1_RORX(wc_Sha512 *sha512); - static int Transform_Sha512_AVX1_RORX_Len(wc_Sha512 *sha512, + extern int Transform_Sha512_AVX1_RORX(wc_Sha512 *sha512); + extern int Transform_Sha512_AVX1_RORX_Len(wc_Sha512 *sha512, word32 len); - static int Transform_Sha512_AVX2_RORX(wc_Sha512 *sha512); - static int Transform_Sha512_AVX2_RORX_Len(wc_Sha512 *sha512, + extern int Transform_Sha512_AVX2_RORX(wc_Sha512 *sha512); + extern int Transform_Sha512_AVX2_RORX_Len(wc_Sha512 *sha512, word32 len); #endif #endif + +#ifdef __cplusplus + } /* extern "C" */ +#endif + static int _Transform_Sha512(wc_Sha512 *sha512); static int (*Transform_Sha512_p)(wc_Sha512* sha512) = _Transform_Sha512; static int (*Transform_Sha512_Len_p)(wc_Sha512* sha512, word32 len) = NULL; @@ -326,7 +335,7 @@ static int InitSha512(wc_Sha512* sha512) static int intel_flags; #define Transform_Sha512(sha512) (*Transform_Sha512_p)(sha512) #define Transform_Sha512_Len(sha512, len) \ - (*Transform_Sha512_Len_p)(sha512, len) + (*Transform_Sha512_Len_p)(sha512, len) static void Sha512_SetTransform() { @@ -570,6 +579,8 @@ static WC_INLINE int Sha512Update(wc_Sha512* sha512, const byte* data, word32 le if (sha512->buffLen >= WC_SHA512_BLOCK_SIZE) return BUFFER_E; + AddLength(sha512, len); + if (sha512->buffLen > 0) { word32 add = min(len, WC_SHA512_BLOCK_SIZE - sha512->buffLen); if (add > 0) { @@ -605,10 +616,8 @@ static WC_INLINE int Sha512Update(wc_Sha512* sha512, const byte* data, word32 le ret = Transform_Sha512(sha512); } #endif - if (ret == 0) { - AddLength(sha512, WC_SHA512_BLOCK_SIZE); + if (ret == 0) sha512->buffLen = 0; - } else len = 0; } @@ -619,7 +628,6 @@ static WC_INLINE int Sha512Update(wc_Sha512* sha512, const byte* data, word32 le word32 blocksLen = len & ~(WC_SHA512_BLOCK_SIZE-1); if (blocksLen > 0) { - AddLength(sha512, blocksLen); sha512->data = data; /* Byte reversal performed in function if required. */ Transform_Sha512_Len(sha512, blocksLen); @@ -631,15 +639,19 @@ static WC_INLINE int Sha512Update(wc_Sha512* sha512, const byte* data, word32 le #endif #if !defined(LITTLE_ENDIAN_ORDER) || defined(HAVE_INTEL_AVX1) || defined(HAVE_INTEL_AVX2) { - word32 blocksLen = len & ~(WC_SHA512_BLOCK_SIZE-1); - - AddLength(sha512, blocksLen); while (len >= WC_SHA512_BLOCK_SIZE) { XMEMCPY(local, data, WC_SHA512_BLOCK_SIZE); data += WC_SHA512_BLOCK_SIZE; len -= WC_SHA512_BLOCK_SIZE; + #if defined(HAVE_INTEL_AVX1) || defined(HAVE_INTEL_AVX2) + if (!IS_INTEL_AVX1(intel_flags) && !IS_INTEL_AVX2(intel_flags)) + { + ByteReverseWords64(sha512->buffer, sha512->buffer, + WC_SHA512_BLOCK_SIZE); + } + #endif /* Byte reversal performed in function if required. */ ret = Transform_Sha512(sha512); if (ret != 0) @@ -648,9 +660,6 @@ static WC_INLINE int Sha512Update(wc_Sha512* sha512, const byte* data, word32 le } #else { - word32 blocksLen = len & ~(WC_SHA512_BLOCK_SIZE-1); - - AddLength(sha512, blocksLen); while (len >= WC_SHA512_BLOCK_SIZE) { XMEMCPY(local, data, WC_SHA512_BLOCK_SIZE); @@ -719,8 +728,6 @@ static WC_INLINE int Sha512Final(wc_Sha512* sha512) return BAD_FUNC_ARG; } - AddLength(sha512, sha512->buffLen); /* before adding pads */ - local[sha512->buffLen++] = 0x80; /* add 1 */ /* pad with zeros */ @@ -878,1809 +885,8 @@ void wc_Sha512Free(wc_Sha512* sha512) #endif /* WOLFSSL_ASYNC_CRYPT */ } - -#if defined(HAVE_INTEL_AVX1) - -static word64 mBYTE_FLIP_MASK[] = { 0x0001020304050607, 0x08090a0b0c0d0e0f }; - -#define W_0 xmm0 -#define W_2 xmm1 -#define W_4 xmm2 -#define W_6 xmm3 -#define W_8 xmm4 -#define W_10 xmm5 -#define W_12 xmm6 -#define W_14 xmm7 - -#define W_M15 xmm12 -#define W_M7 xmm13 -#define MASK xmm14 - -#define XTMP1 xmm8 -#define XTMP2 xmm9 -#define XTMP3 xmm10 -#define XTMP4 xmm11 - -#define XMM_REGS \ - "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", \ - "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", "xmm15" - -#define _VPALIGNR(dest, src1, src2, bits) \ - "vpalignr $" #bits ", %%" #src2 ", %%" #src1 ", %%" #dest "\n\t" -#define VPALIGNR(dest, src1, src2, bits) \ - _VPALIGNR(dest, src1, src2, bits) - -#define _V_SHIFT_R(dest, src, bits) \ - "vpsrlq $" #bits ", %%" #src ", %%" #dest "\n\t" -#define V_SHIFT_R(dest, src, bits) \ - _V_SHIFT_R(dest, src, bits) - -#define _V_SHIFT_L(dest, src, bits) \ - "vpsllq $" #bits ", %%" #src ", %%" #dest "\n\t" -#define V_SHIFT_L(dest, src, bits) \ - _V_SHIFT_L(dest, src, bits) - -#define _V_ADD(dest, src1, src2) \ - "vpaddq %%" #src1 ", %%" #src2 ", %%" #dest "\n\t" -#define V_ADD(dest, src1, src2) \ - _V_ADD(dest, src1, src2) - -#define _V_XOR(dest, src1, src2) \ - "vpxor %%" #src1 ", %%" #src2 ", %%" #dest "\n\t" -#define V_XOR(dest, src1, src2) \ - _V_XOR(dest, src1, src2) - -#define _V_OR(dest, src1, src2) \ - "vpor %%" #src1 ", %%" #src2 ", %%" #dest "\n\t" -#define V_OR(dest, src1, src2) \ - _V_OR(dest, src1, src2) - -#define RA %%r8 -#define RB %%r9 -#define RC %%r10 -#define RD %%r11 -#define RE %%r12 -#define RF %%r13 -#define RG %%r14 -#define RH %%r15 - -#define STATE_REGS "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15" - -#define L1 "%%rax" -#define L2 "%%rcx" -#define L3 "%%rdx" -#define L4 "%%rbx" -#define WX "%%rsp" - -#define WORK_REGS "rax", "rbx", "rcx", "rdx" - -#define RND_0_1(a,b,c,d,e,f,g,h,i) \ - /* L1 = e >>> 23 */ \ - "rorq $23, " L1 "\n\t" \ - -#define RND_0_2(a,b,c,d,e,f,g,h,i) \ - /* L3 = a */ \ - "movq "#a", " L3 "\n\t" \ - /* L2 = f */ \ - "movq "#f", " L2 "\n\t" \ - /* h += W_X[i] */ \ - "addq ("#i")*8(" WX "), "#h"\n\t" \ - /* L2 = f ^ g */ \ - "xorq "#g", " L2 "\n\t" \ - -#define RND_0_2_A(a,b,c,d,e,f,g,h,i) \ - /* L3 = a */ \ - "movq "#a", " L3 "\n\t" \ - /* L2 = f */ \ - "movq "#f", " L2 "\n\t" \ - -#define RND_0_2_B(a,b,c,d,e,f,g,h,i) \ - /* h += W_X[i] */ \ - "addq ("#i")*8(" WX "), "#h"\n\t" \ - /* L2 = f ^ g */ \ - "xorq "#g", " L2 "\n\t" \ - -#define RND_0_3(a,b,c,d,e,f,g,h,i) \ - /* L1 = (e >>> 23) ^ e */ \ - "xorq "#e", " L1 "\n\t" \ - /* L2 = (f ^ g) & e */ \ - "andq "#e", " L2 "\n\t" \ - -#define RND_0_4(a,b,c,d,e,f,g,h,i) \ - /* L1 = ((e >>> 23) ^ e) >>> 4 */ \ - "rorq $4, " L1 "\n\t" \ - /* L2 = ((f ^ g) & e) ^ g */ \ - "xorq "#g", " L2 "\n\t" \ - -#define RND_0_5(a,b,c,d,e,f,g,h,i) \ - /* L1 = (((e >>> 23) ^ e) >>> 4) ^ e */ \ - "xorq "#e", " L1 "\n\t" \ - /* h += Ch(e,f,g) */ \ - "addq " L2 ", "#h"\n\t" \ - -#define RND_0_6(a,b,c,d,e,f,g,h,i) \ - /* L1 = ((((e >>> 23) ^ e) >>> 4) ^ e) >>> 14 */ \ - "rorq $14, " L1 "\n\t" \ - /* L3 = a ^ b */ \ - "xorq "#b", " L3 "\n\t" \ - -#define RND_0_7(a,b,c,d,e,f,g,h,i) \ - /* h += Sigma1(e) */ \ - "addq " L1 ", "#h"\n\t" \ - /* L2 = a */ \ - "movq "#a", " L2 "\n\t" \ - -#define RND_0_8(a,b,c,d,e,f,g,h,i) \ - /* L4 = (a ^ b) & (b ^ c) */ \ - "andq " L3 ", " L4 "\n\t" \ - /* L2 = a >>> 5 */ \ - "rorq $5, " L2 "\n\t" \ - -#define RND_0_9(a,b,c,d,e,f,g,h,i) \ - /* L2 = (a >>> 5) ^ a */ \ - "xorq "#a", " L2 "\n\t" \ - /* L4 = ((a ^ b) & (b ^ c) ^ b */ \ - "xorq "#b", " L4 "\n\t" \ - -#define RND_0_10(a,b,c,d,e,f,g,h,i) \ - /* L2 = ((a >>> 5) ^ a) >>> 6 */ \ - "rorq $6, " L2 "\n\t" \ - /* d += h */ \ - "addq "#h", "#d"\n\t" \ - -#define RND_0_11(a,b,c,d,e,f,g,h,i) \ - /* L2 = (((a >>> 5) ^ a) >>> 6) ^ a */ \ - "xorq "#a", " L2 "\n\t" \ - /* h += Sigma0(a) */ \ - "addq " L4 ", "#h"\n\t" \ - -#define RND_0_12(a,b,c,d,e,f,g,h,i) \ - /* L2 = ((((a >>> 5) ^ a) >>> 6) ^ a) >>> 28 */ \ - "rorq $28, " L2 "\n\t" \ - /* d (= e next RND) */ \ - "movq "#d", " L1 "\n\t" \ - /* h += Maj(a,b,c) */ \ - "addq " L2 ", "#h"\n\t" \ - -#define RND_1_1(a,b,c,d,e,f,g,h,i) \ - /* L1 = e >>> 23 */ \ - "rorq $23, " L1 "\n\t" \ - -#define RND_1_2(a,b,c,d,e,f,g,h,i) \ - /* L4 = a */ \ - "movq "#a", " L4 "\n\t" \ - /* L2 = f */ \ - "movq "#f", " L2 "\n\t" \ - /* h += W_X[i] */ \ - "addq ("#i")*8(" WX "), "#h"\n\t" \ - /* L2 = f ^ g */ \ - "xorq "#g", " L2 "\n\t" \ - -#define RND_1_2_A(a,b,c,d,e,f,g,h,i) \ - /* L4 = a */ \ - "movq "#a", " L4 "\n\t" \ - /* L2 = f */ \ - "movq "#f", " L2 "\n\t" \ - -#define RND_1_2_B(a,b,c,d,e,f,g,h,i) \ - /* h += W_X[i] */ \ - "addq ("#i")*8(" WX "), "#h"\n\t" \ - /* L2 = f ^ g */ \ - "xorq "#g", " L2 "\n\t" \ - -#define RND_1_3(a,b,c,d,e,f,g,h,i) \ - /* L1 = (e >>> 23) ^ e */ \ - "xorq "#e", " L1 "\n\t" \ - /* L2 = (f ^ g) & e */ \ - "andq "#e", " L2 "\n\t" \ - -#define RND_1_4(a,b,c,d,e,f,g,h,i) \ - /* ((e >>> 23) ^ e) >>> 4 */ \ - "rorq $4, " L1 "\n\t" \ - /* ((f ^ g) & e) ^ g */ \ - "xorq "#g", " L2 "\n\t" \ - -#define RND_1_5(a,b,c,d,e,f,g,h,i) \ - /* (((e >>> 23) ^ e) >>> 4) ^ e */ \ - "xorq "#e", " L1 "\n\t" \ - /* h += Ch(e,f,g) */ \ - "addq " L2 ", "#h"\n\t" \ - -#define RND_1_6(a,b,c,d,e,f,g,h,i) \ - /* L1 = ((((e >>> 23) ^ e) >>> 4) ^ e) >>> 14 */ \ - "rorq $14, " L1 "\n\t" \ - /* L4 = a ^ b */ \ - "xorq "#b", " L4 "\n\t" \ - -#define RND_1_7(a,b,c,d,e,f,g,h,i) \ - /* h += Sigma1(e) */ \ - "addq " L1 ", "#h"\n\t" \ - /* L2 = a */ \ - "movq "#a", " L2 "\n\t" \ - -#define RND_1_8(a,b,c,d,e,f,g,h,i) \ - /* L3 = (a ^ b) & (b ^ c) */ \ - "andq " L4 ", " L3 "\n\t" \ - /* L2 = a >>> 5 */ \ - "rorq $5, " L2 "\n\t" \ - -#define RND_1_9(a,b,c,d,e,f,g,h,i) \ - /* L2 = (a >>> 5) ^ a */ \ - "xorq "#a", " L2 "\n\t" \ - /* L3 = ((a ^ b) & (b ^ c) ^ b */ \ - "xorq "#b", " L3 "\n\t" \ - -#define RND_1_10(a,b,c,d,e,f,g,h,i) \ - /* L2 = ((a >>> 5) ^ a) >>> 6 */ \ - "rorq $6, " L2 "\n\t" \ - /* d += h */ \ - "addq "#h", "#d"\n\t" \ - -#define RND_1_11(a,b,c,d,e,f,g,h,i) \ - /* L2 = (((a >>> 5) ^ a) >>> 6) ^ a */ \ - "xorq "#a", " L2 "\n\t" \ - /* h += Sigma0(a) */ \ - "addq " L3 ", "#h"\n\t" \ - -#define RND_1_12(a,b,c,d,e,f,g,h,i) \ - /* L2 = ((((a >>> 5) ^ a) >>> 6) ^ a) >>> 28 */ \ - "rorq $28, " L2 "\n\t" \ - /* d (= e next RND) */ \ - "movq "#d", " L1 "\n\t" \ - /* h += Maj(a,b,c) */ \ - "addq " L2 ", "#h"\n\t" \ - - -#define MsgSched2(W_0,W_2,W_4,W_6,W_8,W_10,W_12,W_14,a,b,c,d,e,f,g,h,i) \ - RND_0_1(a,b,c,d,e,f,g,h,i) \ - VPALIGNR(W_M15, W_2, W_0, 8) \ - VPALIGNR(W_M7, W_10, W_8, 8) \ - RND_0_2(a,b,c,d,e,f,g,h,i) \ - V_SHIFT_R(XTMP1, W_M15, 1) \ - V_SHIFT_L(XTMP2, W_M15, 63) \ - RND_0_3(a,b,c,d,e,f,g,h,i) \ - RND_0_4(a,b,c,d,e,f,g,h,i) \ - V_SHIFT_R(XTMP3, W_M15, 8) \ - V_SHIFT_L(XTMP4, W_M15, 56) \ - RND_0_5(a,b,c,d,e,f,g,h,i) \ - RND_0_6(a,b,c,d,e,f,g,h,i) \ - V_OR(XTMP1, XTMP2, XTMP1) \ - V_OR(XTMP3, XTMP4, XTMP3) \ - RND_0_7(a,b,c,d,e,f,g,h,i) \ - RND_0_8(a,b,c,d,e,f,g,h,i) \ - V_SHIFT_R(XTMP4, W_M15, 7) \ - V_XOR(XTMP1, XTMP3, XTMP1) \ - RND_0_9(a,b,c,d,e,f,g,h,i) \ - RND_0_10(a,b,c,d,e,f,g,h,i) \ - V_XOR(XTMP1, XTMP4, XTMP1) \ - V_ADD(W_0, W_0, W_M7) \ - RND_0_11(a,b,c,d,e,f,g,h,i) \ - RND_0_12(a,b,c,d,e,f,g,h,i) \ - RND_1_1(h,a,b,c,d,e,f,g,i+1) \ - V_ADD(W_0, W_0, XTMP1) \ - RND_1_2(h,a,b,c,d,e,f,g,i+1) \ - V_SHIFT_R(XTMP1, W_14, 19) \ - V_SHIFT_L(XTMP2, W_14, 45) \ - RND_1_3(h,a,b,c,d,e,f,g,i+1) \ - RND_1_4(h,a,b,c,d,e,f,g,i+1) \ - V_SHIFT_R(XTMP3, W_14, 61) \ - V_SHIFT_L(XTMP4, W_14, 3) \ - RND_1_5(h,a,b,c,d,e,f,g,i+1) \ - RND_1_6(h,a,b,c,d,e,f,g,i+1) \ - RND_1_7(h,a,b,c,d,e,f,g,i+1) \ - V_OR(XTMP1, XTMP2, XTMP1) \ - V_OR(XTMP3, XTMP4, XTMP3) \ - RND_1_8(h,a,b,c,d,e,f,g,i+1) \ - RND_1_9(h,a,b,c,d,e,f,g,i+1) \ - V_XOR(XTMP1, XTMP3, XTMP1) \ - V_SHIFT_R(XTMP4, W_14, 6) \ - RND_1_10(h,a,b,c,d,e,f,g,i+1) \ - RND_1_11(h,a,b,c,d,e,f,g,i+1) \ - V_XOR(XTMP1, XTMP4, XTMP1) \ - RND_1_12(h,a,b,c,d,e,f,g,i+1) \ - V_ADD(W_0, W_0, XTMP1) \ - -#define RND_ALL_2(a, b, c, d, e, f, g, h, i) \ - RND_0_1 (a, b, c, d, e, f, g, h, i ) \ - RND_0_2 (a, b, c, d, e, f, g, h, i ) \ - RND_0_3 (a, b, c, d, e, f, g, h, i ) \ - RND_0_4 (a, b, c, d, e, f, g, h, i ) \ - RND_0_5 (a, b, c, d, e, f, g, h, i ) \ - RND_0_6 (a, b, c, d, e, f, g, h, i ) \ - RND_0_7 (a, b, c, d, e, f, g, h, i ) \ - RND_0_8 (a, b, c, d, e, f, g, h, i ) \ - RND_0_9 (a, b, c, d, e, f, g, h, i ) \ - RND_0_10(a, b, c, d, e, f, g, h, i ) \ - RND_0_11(a, b, c, d, e, f, g, h, i ) \ - RND_0_12(a, b, c, d, e, f, g, h, i ) \ - RND_1_1 (h, a, b, c, d, e, f, g, i+1) \ - RND_1_2 (h, a, b, c, d, e, f, g, i+1) \ - RND_1_3 (h, a, b, c, d, e, f, g, i+1) \ - RND_1_4 (h, a, b, c, d, e, f, g, i+1) \ - RND_1_5 (h, a, b, c, d, e, f, g, i+1) \ - RND_1_6 (h, a, b, c, d, e, f, g, i+1) \ - RND_1_7 (h, a, b, c, d, e, f, g, i+1) \ - RND_1_8 (h, a, b, c, d, e, f, g, i+1) \ - RND_1_9 (h, a, b, c, d, e, f, g, i+1) \ - RND_1_10(h, a, b, c, d, e, f, g, i+1) \ - RND_1_11(h, a, b, c, d, e, f, g, i+1) \ - RND_1_12(h, a, b, c, d, e, f, g, i+1) - - -#if defined(HAVE_INTEL_RORX) - -#define RND_RORX_0_1(a, b, c, d, e, f, g, h, i) \ - /* L1 = e>>>14 */ \ - "rorxq $14, "#e", " L1 "\n\t" \ - /* L2 = e>>>18 */ \ - "rorxq $18, "#e", " L2 "\n\t" \ - /* Prev RND: h += Maj(a,b,c) */ \ - "addq " L3 ", "#a"\n\t" \ - -#define RND_RORX_0_2(a, b, c, d, e, f, g, h, i) \ - /* h += w_k */ \ - "addq ("#i")*8(" WX "), "#h"\n\t" \ - /* L3 = f */ \ - "movq "#f", " L3 "\n\t" \ - /* L2 = (e>>>14) ^ (e>>>18) */ \ - "xorq " L1 ", " L2 "\n\t" \ - -#define RND_RORX_0_3(a, b, c, d, e, f, g, h, i) \ - /* L3 = f ^ g */ \ - "xorq "#g", " L3 "\n\t" \ - /* L1 = e>>>41 */ \ - "rorxq $41, "#e", " L1 "\n\t" \ - /* L1 = Sigma1(e) */ \ - "xorq " L2 ", " L1 "\n\t" \ - -#define RND_RORX_0_4(a, b, c, d, e, f, g, h, i) \ - /* L3 = (f ^ g) & e */ \ - "andq "#e", " L3 "\n\t" \ - /* h += Sigma1(e) */ \ - "addq " L1 ", "#h"\n\t" \ - /* L1 = a>>>28 */ \ - "rorxq $28, "#a", " L1 "\n\t" \ - -#define RND_RORX_0_5(a, b, c, d, e, f, g, h, i) \ - /* L2 = a>>>34 */ \ - "rorxq $34, "#a", " L2 "\n\t" \ - /* L3 = Ch(e,f,g) */ \ - "xorq "#g", " L3 "\n\t" \ - /* L2 = (a>>>28) ^ (a>>>34) */ \ - "xorq " L1 ", " L2 "\n\t" \ - -#define RND_RORX_0_6(a, b, c, d, e, f, g, h, i) \ - /* L1 = a>>>39 */ \ - "rorxq $39, "#a", " L1 "\n\t" \ - /* h += Ch(e,f,g) */ \ - "addq " L3 ", "#h"\n\t" \ - /* L1 = Sigma0(a) */ \ - "xorq " L2 ", " L1 "\n\t" \ - -#define RND_RORX_0_7(a, b, c, d, e, f, g, h, i) \ - /* L3 = b */ \ - "movq "#b", " L3 "\n\t" \ - /* d += h + w_k + Sigma1(e) + Ch(e,f,g) */ \ - "addq "#h", "#d"\n\t" \ - /* L3 = a ^ b */ \ - "xorq "#a", " L3 "\n\t" \ - -#define RND_RORX_0_8(a, b, c, d, e, f, g, h, i) \ - /* L4 = (a ^ b) & (b ^ c) */ \ - "andq " L3 ", " L4 "\n\t" \ - /* h += Sigma0(a) */ \ - "addq " L1 ", "#h"\n\t" \ - /* L4 = Maj(a,b,c) */ \ - "xorq "#b", " L4 "\n\t" \ - -#define RND_RORX_1_1(a, b, c, d, e, f, g, h, i) \ - /* L1 = e>>>14 */ \ - "rorxq $14, "#e", " L1 "\n\t" \ - /* L2 = e>>>18 */ \ - "rorxq $18, "#e", " L2 "\n\t" \ - /* Prev RND: h += Maj(a,b,c) */ \ - "addq " L4 ", "#a"\n\t" \ - -#define RND_RORX_1_2(a, b, c, d, e, f, g, h, i) \ - /* h += w_k */ \ - "addq ("#i")*8(" WX "), "#h"\n\t" \ - /* L4 = f */ \ - "movq "#f", " L4 "\n\t" \ - /* L2 = (e>>>14) ^ (e>>>18) */ \ - "xorq " L1 ", " L2 "\n\t" \ - -#define RND_RORX_1_3(a, b, c, d, e, f, g, h, i) \ - /* L4 = f ^ g */ \ - "xorq "#g", " L4 "\n\t" \ - /* L1 = e>>>41 */ \ - "rorxq $41, "#e", " L1 "\n\t" \ - /* L1 = Sigma1(e) */ \ - "xorq " L2 ", " L1 "\n\t" \ - -#define RND_RORX_1_4(a, b, c, d, e, f, g, h, i) \ - /* L4 = (f ^ g) & e */ \ - "andq "#e", " L4 "\n\t" \ - /* h += Sigma1(e) */ \ - "addq " L1 ", "#h"\n\t" \ - /* L1 = a>>>28 */ \ - "rorxq $28, "#a", " L1 "\n\t" \ - -#define RND_RORX_1_5(a, b, c, d, e, f, g, h, i) \ - /* L2 = a>>>34 */ \ - "rorxq $34, "#a", " L2 "\n\t" \ - /* L4 = Ch(e,f,g) */ \ - "xorq "#g", " L4 "\n\t" \ - /* L2 = (a>>>28) ^ (a>>>34) */ \ - "xorq " L1 ", " L2 "\n\t" \ - -#define RND_RORX_1_6(a, b, c, d, e, f, g, h, i) \ - /* L1 = a>>>39 */ \ - "rorxq $39, "#a", " L1 "\n\t" \ - /* h += Ch(e,f,g) */ \ - "addq " L4 ", "#h"\n\t" \ - /* L1 = Sigma0(a) */ \ - "xorq " L2 ", " L1 "\n\t" \ - -#define RND_RORX_1_7(a, b, c, d, e, f, g, h, i) \ - /* L4 = b */ \ - "movq "#b", " L4 "\n\t" \ - /* d += h + w_k + Sigma1(e) + Ch(e,f,g) */ \ - "addq "#h", "#d"\n\t" \ - /* L4 = a ^ b */ \ - "xorq "#a", " L4 "\n\t" \ - -#define RND_RORX_1_8(a, b, c, d, e, f, g, h, i) \ - /* L2 = (a ^ b) & (b ^ c) */ \ - "andq " L4 ", " L3 "\n\t" \ - /* h += Sigma0(a) */ \ - "addq " L1 ", "#h"\n\t" \ - /* L3 = Maj(a,b,c) */ \ - "xorq "#b", " L3 "\n\t" \ - -#define RND_RORX_ALL_2(a, b, c, d, e, f, g, h, i) \ - RND_RORX_0_1(a, b, c, d, e, f, g, h, i+0) \ - RND_RORX_0_2(a, b, c, d, e, f, g, h, i+0) \ - RND_RORX_0_3(a, b, c, d, e, f, g, h, i+0) \ - RND_RORX_0_4(a, b, c, d, e, f, g, h, i+0) \ - RND_RORX_0_5(a, b, c, d, e, f, g, h, i+0) \ - RND_RORX_0_6(a, b, c, d, e, f, g, h, i+0) \ - RND_RORX_0_7(a, b, c, d, e, f, g, h, i+0) \ - RND_RORX_0_8(a, b, c, d, e, f, g, h, i+0) \ - RND_RORX_1_1(h, a, b, c, d, e, f, g, i+1) \ - RND_RORX_1_2(h, a, b, c, d, e, f, g, i+1) \ - RND_RORX_1_3(h, a, b, c, d, e, f, g, i+1) \ - RND_RORX_1_4(h, a, b, c, d, e, f, g, i+1) \ - RND_RORX_1_5(h, a, b, c, d, e, f, g, i+1) \ - RND_RORX_1_6(h, a, b, c, d, e, f, g, i+1) \ - RND_RORX_1_7(h, a, b, c, d, e, f, g, i+1) \ - RND_RORX_1_8(h, a, b, c, d, e, f, g, i+1) \ - -#define RND_RORX_ALL_4(a, b, c, d, e, f, g, h, i) \ - RND_RORX_ALL_2(a, b, c, d, e, f, g, h, i+0) \ - RND_RORX_ALL_2(g, h, a, b, c, d, e, f, i+2) - -#define MsgSched_RORX(W_0,W_2,W_4,W_6,W_8,W_10,W_12,W_14,a,b,c,d,e,f,g,h,i) \ - RND_RORX_0_1(a,b,c,d,e,f,g,h,i) \ - VPALIGNR(W_M15, W_2, W_0, 8) \ - VPALIGNR(W_M7, W_10, W_8, 8) \ - RND_RORX_0_2(a,b,c,d,e,f,g,h,i) \ - V_SHIFT_R(XTMP1, W_M15, 1) \ - V_SHIFT_L(XTMP2, W_M15, 63) \ - RND_RORX_0_3(a,b,c,d,e,f,g,h,i) \ - V_SHIFT_R(XTMP3, W_M15, 8) \ - V_SHIFT_L(XTMP4, W_M15, 56) \ - RND_RORX_0_4(a,b,c,d,e,f,g,h,i) \ - V_OR(XTMP1, XTMP2, XTMP1) \ - V_OR(XTMP3, XTMP4, XTMP3) \ - RND_RORX_0_5(a,b,c,d,e,f,g,h,i) \ - V_SHIFT_R(XTMP4, W_M15, 7) \ - V_XOR(XTMP1, XTMP3, XTMP1) \ - RND_RORX_0_6(a,b,c,d,e,f,g,h,i) \ - V_XOR(XTMP1, XTMP4, XTMP1) \ - V_ADD(W_0, W_0, W_M7) \ - RND_RORX_0_7(a,b,c,d,e,f,g,h,i) \ - RND_RORX_0_8(a,b,c,d,e,f,g,h,i) \ - V_ADD(W_0, W_0, XTMP1) \ - RND_RORX_1_1(h,a,b,c,d,e,f,g,i+1) \ - V_SHIFT_R(XTMP1, W_14, 19) \ - V_SHIFT_L(XTMP2, W_14, 45) \ - RND_RORX_1_2(h,a,b,c,d,e,f,g,i+1) \ - V_SHIFT_R(XTMP3, W_14, 61) \ - V_SHIFT_L(XTMP4, W_14, 3) \ - RND_RORX_1_3(h,a,b,c,d,e,f,g,i+1) \ - V_OR(XTMP1, XTMP2, XTMP1) \ - V_OR(XTMP3, XTMP4, XTMP3) \ - RND_RORX_1_4(h,a,b,c,d,e,f,g,i+1) \ - RND_RORX_1_5(h,a,b,c,d,e,f,g,i+1) \ - V_XOR(XTMP1, XTMP3, XTMP1) \ - V_SHIFT_R(XTMP4, W_14, 6) \ - RND_RORX_1_6(h,a,b,c,d,e,f,g,i+1) \ - RND_RORX_1_7(h,a,b,c,d,e,f,g,i+1) \ - V_XOR(XTMP1, XTMP4, XTMP1) \ - RND_RORX_1_8(h,a,b,c,d,e,f,g,i+1) \ - V_ADD(W_0, W_0, XTMP1) \ - -#endif - -#define _INIT_MASK(mask) \ - "vmovdqu %[mask], %%" #mask "\n\t" -#define INIT_MASK(mask) \ - _INIT_MASK(mask) - -#define _LOAD_W_2(i1, i2, xmm1, xmm2, mask, reg) \ - "vmovdqu " #i1 "*16(%%" #reg "), %%" #xmm1 "\n\t" \ - "vmovdqu " #i2 "*16(%%" #reg "), %%" #xmm2 "\n\t" \ - "vpshufb %%" #mask ", %%" #xmm1 ", %%" #xmm1 "\n\t" \ - "vpshufb %%" #mask ", %%" #xmm2 ", %%" #xmm2 "\n\t" -#define LOAD_W_2(i1, i2, xmm1, xmm2, mask, reg) \ - _LOAD_W_2(i1, i2, xmm1, xmm2, mask, reg) - -#define LOAD_W(mask, reg) \ - /* X0..3(xmm4..7), W[0..15] = buffer[0.15]; */ \ - LOAD_W_2(0, 1, W_0 , W_2 , mask, reg) \ - LOAD_W_2(2, 3, W_4 , W_6 , mask, reg) \ - LOAD_W_2(4, 5, W_8 , W_10, mask, reg) \ - LOAD_W_2(6, 7, W_12, W_14, mask, reg) - -#define _SET_W_X_2(xmm0, xmm1, reg, i) \ - "vpaddq " #i "+ 0(%%" #reg "), %%" #xmm0 ", %%xmm8\n\t" \ - "vpaddq " #i "+16(%%" #reg "), %%" #xmm1 ", %%xmm9\n\t" \ - "vmovdqu %%xmm8, " #i "+ 0(" WX ")\n\t" \ - "vmovdqu %%xmm9, " #i "+16(" WX ")\n\t" \ - -#define SET_W_X_2(xmm0, xmm1, reg, i) \ - _SET_W_X_2(xmm0, xmm1, reg, i) - -#define SET_W_X(reg) \ - SET_W_X_2(W_0 , W_2 , reg, 0) \ - SET_W_X_2(W_4 , W_6 , reg, 32) \ - SET_W_X_2(W_8 , W_10, reg, 64) \ - SET_W_X_2(W_12, W_14, reg, 96) - -#define LOAD_DIGEST() \ - "movq (%[sha512]), %%r8 \n\t" \ - "movq 8(%[sha512]), %%r9 \n\t" \ - "movq 16(%[sha512]), %%r10\n\t" \ - "movq 24(%[sha512]), %%r11\n\t" \ - "movq 32(%[sha512]), %%r12\n\t" \ - "movq 40(%[sha512]), %%r13\n\t" \ - "movq 48(%[sha512]), %%r14\n\t" \ - "movq 56(%[sha512]), %%r15\n\t" - -#define STORE_ADD_DIGEST() \ - "addq %%r8, (%[sha512])\n\t" \ - "addq %%r9, 8(%[sha512])\n\t" \ - "addq %%r10, 16(%[sha512])\n\t" \ - "addq %%r11, 24(%[sha512])\n\t" \ - "addq %%r12, 32(%[sha512])\n\t" \ - "addq %%r13, 40(%[sha512])\n\t" \ - "addq %%r14, 48(%[sha512])\n\t" \ - "addq %%r15, 56(%[sha512])\n\t" - -#define ADD_DIGEST() \ - "addq (%[sha512]), %%r8 \n\t" \ - "addq 8(%[sha512]), %%r9 \n\t" \ - "addq 16(%[sha512]), %%r10\n\t" \ - "addq 24(%[sha512]), %%r11\n\t" \ - "addq 32(%[sha512]), %%r12\n\t" \ - "addq 40(%[sha512]), %%r13\n\t" \ - "addq 48(%[sha512]), %%r14\n\t" \ - "addq 56(%[sha512]), %%r15\n\t" - -#define STORE_DIGEST() \ - "movq %%r8, (%[sha512])\n\t" \ - "movq %%r9, 8(%[sha512])\n\t" \ - "movq %%r10, 16(%[sha512])\n\t" \ - "movq %%r11, 24(%[sha512])\n\t" \ - "movq %%r12, 32(%[sha512])\n\t" \ - "movq %%r13, 40(%[sha512])\n\t" \ - "movq %%r14, 48(%[sha512])\n\t" \ - "movq %%r15, 56(%[sha512])\n\t" - -#endif /* HAVE_INTEL_AVX1 */ - - -/*** Transform Body ***/ -#if defined(HAVE_INTEL_AVX1) -static int Transform_Sha512_AVX1(wc_Sha512* sha512) -{ - __asm__ __volatile__ ( - - /* 16 Ws plus loop counter. */ - "subq $136, %%rsp\n\t" - "leaq 64(%[sha512]), %%rax\n\t" - - INIT_MASK(MASK) - LOAD_DIGEST() - - LOAD_W(MASK, rax) - - "movl $4, 16*8(" WX ")\n\t" - "leaq %[K512], %%rsi\n\t" - /* b */ - "movq %%r9, " L4 "\n\t" - /* e */ - "movq %%r12, " L1 "\n\t" - /* b ^ c */ - "xorq %%r10, " L4 "\n\t" - - "# Start of 16 rounds\n" - "1:\n\t" - - SET_W_X(rsi) - - "addq $128, %%rsi\n\t" - - MsgSched2(W_0,W_2,W_4,W_6,W_8,W_10,W_12,W_14,RA,RB,RC,RD,RE,RF,RG,RH, 0) - MsgSched2(W_2,W_4,W_6,W_8,W_10,W_12,W_14,W_0,RG,RH,RA,RB,RC,RD,RE,RF, 2) - MsgSched2(W_4,W_6,W_8,W_10,W_12,W_14,W_0,W_2,RE,RF,RG,RH,RA,RB,RC,RD, 4) - MsgSched2(W_6,W_8,W_10,W_12,W_14,W_0,W_2,W_4,RC,RD,RE,RF,RG,RH,RA,RB, 6) - MsgSched2(W_8,W_10,W_12,W_14,W_0,W_2,W_4,W_6,RA,RB,RC,RD,RE,RF,RG,RH, 8) - MsgSched2(W_10,W_12,W_14,W_0,W_2,W_4,W_6,W_8,RG,RH,RA,RB,RC,RD,RE,RF,10) - MsgSched2(W_12,W_14,W_0,W_2,W_4,W_6,W_8,W_10,RE,RF,RG,RH,RA,RB,RC,RD,12) - MsgSched2(W_14,W_0,W_2,W_4,W_6,W_8,W_10,W_12,RC,RD,RE,RF,RG,RH,RA,RB,14) - - "subl $1, 16*8(" WX ")\n\t" - "jne 1b\n\t" - - SET_W_X(rsi) - - RND_ALL_2(RA,RB,RC,RD,RE,RF,RG,RH, 0) - RND_ALL_2(RG,RH,RA,RB,RC,RD,RE,RF, 2) - RND_ALL_2(RE,RF,RG,RH,RA,RB,RC,RD, 4) - RND_ALL_2(RC,RD,RE,RF,RG,RH,RA,RB, 6) - - RND_ALL_2(RA,RB,RC,RD,RE,RF,RG,RH, 8) - RND_ALL_2(RG,RH,RA,RB,RC,RD,RE,RF,10) - RND_ALL_2(RE,RF,RG,RH,RA,RB,RC,RD,12) - RND_ALL_2(RC,RD,RE,RF,RG,RH,RA,RB,14) - - STORE_ADD_DIGEST() - - "addq $136, %%rsp\n\t" - - : - : [mask] "m" (mBYTE_FLIP_MASK), - [sha512] "r" (sha512), - [K512] "m" (K512) - : WORK_REGS, STATE_REGS, XMM_REGS, "memory", "rsi" - ); - - return 0; -} - -static int Transform_Sha512_AVX1_Len(wc_Sha512* sha512, word32 len) -{ - __asm__ __volatile__ ( - - "movq 224(%[sha512]), %%rsi\n\t" - "leaq %[K512], %%rdx\n\t" - - INIT_MASK(MASK) - LOAD_DIGEST() - - "# Start of processing a block\n" - "2:\n\t" - - /* 16 Ws plus loop counter and K512. len goes into -4(%rsp). - * Debug needs more stack space. */ - "subq $256, %%rsp\n\t" - - LOAD_W(MASK, rsi) - - "movl $4, 16*8(" WX ")\n\t" - /* b */ - "movq %%r9, " L4 "\n\t" - /* e */ - "movq %%r12, " L1 "\n\t" - /* b ^ c */ - "xorq %%r10, " L4 "\n\t" - - SET_W_X(rdx) - - "# Start of 16 rounds\n" - "1:\n\t" - - "addq $128, %%rdx\n\t" - "movq %%rdx, 17*8(%%rsp)\n\t" - - MsgSched2(W_0,W_2,W_4,W_6,W_8,W_10,W_12,W_14,RA,RB,RC,RD,RE,RF,RG,RH, 0) - MsgSched2(W_2,W_4,W_6,W_8,W_10,W_12,W_14,W_0,RG,RH,RA,RB,RC,RD,RE,RF, 2) - MsgSched2(W_4,W_6,W_8,W_10,W_12,W_14,W_0,W_2,RE,RF,RG,RH,RA,RB,RC,RD, 4) - MsgSched2(W_6,W_8,W_10,W_12,W_14,W_0,W_2,W_4,RC,RD,RE,RF,RG,RH,RA,RB, 6) - MsgSched2(W_8,W_10,W_12,W_14,W_0,W_2,W_4,W_6,RA,RB,RC,RD,RE,RF,RG,RH, 8) - MsgSched2(W_10,W_12,W_14,W_0,W_2,W_4,W_6,W_8,RG,RH,RA,RB,RC,RD,RE,RF,10) - MsgSched2(W_12,W_14,W_0,W_2,W_4,W_6,W_8,W_10,RE,RF,RG,RH,RA,RB,RC,RD,12) - MsgSched2(W_14,W_0,W_2,W_4,W_6,W_8,W_10,W_12,RC,RD,RE,RF,RG,RH,RA,RB,14) - - "movq 17*8(%%rsp), %%rdx\n\t" - - SET_W_X(rdx) - - "subl $1, 16*8(" WX ")\n\t" - "jne 1b\n\t" - - RND_ALL_2(RA,RB,RC,RD,RE,RF,RG,RH, 0) - RND_ALL_2(RG,RH,RA,RB,RC,RD,RE,RF, 2) - RND_ALL_2(RE,RF,RG,RH,RA,RB,RC,RD, 4) - RND_ALL_2(RC,RD,RE,RF,RG,RH,RA,RB, 6) - - RND_ALL_2(RA,RB,RC,RD,RE,RF,RG,RH, 8) - RND_ALL_2(RG,RH,RA,RB,RC,RD,RE,RF,10) - RND_ALL_2(RE,RF,RG,RH,RA,RB,RC,RD,12) - RND_ALL_2(RC,RD,RE,RF,RG,RH,RA,RB,14) - - ADD_DIGEST() - - "addq $256, %%rsp\n\t" - "leaq %[K512], %%rdx\n\t" - "addq $128, %%rsi\n\t" - "subl $128, %[len]\n\t" - - STORE_DIGEST() - - "jnz 2b\n\t" - - : - : [mask] "m" (mBYTE_FLIP_MASK), - [len] "m" (len), - [sha512] "r" (sha512), - [K512] "m" (K512) - : WORK_REGS, STATE_REGS, XMM_REGS, "memory", "rsi" - ); - - return 0; -} -#endif /* HAVE_INTEL_AVX1 */ - -#if defined(HAVE_INTEL_AVX2) && defined(HAVE_INTEL_RORX) -static int Transform_Sha512_AVX1_RORX(wc_Sha512* sha512) -{ - __asm__ __volatile__ ( - - /* 16 Ws plus loop counter and K512. */ - "subq $144, %%rsp\n\t" - "leaq 64(%[sha512]), %%rax\n\t" - - INIT_MASK(MASK) - LOAD_DIGEST() - - LOAD_W(MASK, rax) - - "movl $4, 16*8(" WX ")\n\t" - "leaq %[K512], %%rsi\n\t" - /* L4 = b */ - "movq %%r9, " L4 "\n\t" - /* L3 = 0 (add to prev h) */ - "xorq " L3 ", " L3 "\n\t" - /* L4 = b ^ c */ - "xorq %%r10, " L4 "\n\t" - - SET_W_X(rsi) - - "# Start of 16 rounds\n" - "1:\n\t" - - "addq $128, %%rsi\n\t" - - MsgSched_RORX(W_0,W_2,W_4,W_6,W_8,W_10,W_12,W_14,RA,RB,RC,RD,RE,RF,RG,RH, 0) - MsgSched_RORX(W_2,W_4,W_6,W_8,W_10,W_12,W_14,W_0,RG,RH,RA,RB,RC,RD,RE,RF, 2) - MsgSched_RORX(W_4,W_6,W_8,W_10,W_12,W_14,W_0,W_2,RE,RF,RG,RH,RA,RB,RC,RD, 4) - MsgSched_RORX(W_6,W_8,W_10,W_12,W_14,W_0,W_2,W_4,RC,RD,RE,RF,RG,RH,RA,RB, 6) - MsgSched_RORX(W_8,W_10,W_12,W_14,W_0,W_2,W_4,W_6,RA,RB,RC,RD,RE,RF,RG,RH, 8) - MsgSched_RORX(W_10,W_12,W_14,W_0,W_2,W_4,W_6,W_8,RG,RH,RA,RB,RC,RD,RE,RF,10) - MsgSched_RORX(W_12,W_14,W_0,W_2,W_4,W_6,W_8,W_10,RE,RF,RG,RH,RA,RB,RC,RD,12) - MsgSched_RORX(W_14,W_0,W_2,W_4,W_6,W_8,W_10,W_12,RC,RD,RE,RF,RG,RH,RA,RB,14) - - SET_W_X(rsi) - - "subl $1, 16*8(" WX ")\n\t" - "jne 1b\n\t" - - RND_RORX_ALL_2(RA,RB,RC,RD,RE,RF,RG,RH, 0) - RND_RORX_ALL_2(RG,RH,RA,RB,RC,RD,RE,RF, 2) - RND_RORX_ALL_2(RE,RF,RG,RH,RA,RB,RC,RD, 4) - RND_RORX_ALL_2(RC,RD,RE,RF,RG,RH,RA,RB, 6) - - RND_RORX_ALL_2(RA,RB,RC,RD,RE,RF,RG,RH, 8) - RND_RORX_ALL_2(RG,RH,RA,RB,RC,RD,RE,RF,10) - RND_RORX_ALL_2(RE,RF,RG,RH,RA,RB,RC,RD,12) - RND_RORX_ALL_2(RC,RD,RE,RF,RG,RH,RA,RB,14) - - /* Prev RND: h += Maj(a,b,c) */ - "addq " L3 ", %%r8\n\t" - "addq $144, %%rsp\n\t" - - STORE_ADD_DIGEST() - - : - : [mask] "m" (mBYTE_FLIP_MASK), - [sha512] "r" (sha512), - [K512] "m" (K512) - : WORK_REGS, STATE_REGS, XMM_REGS, "memory", "rsi" - ); - - return 0; -} - -static int Transform_Sha512_AVX1_RORX_Len(wc_Sha512* sha512, word32 len) -{ - __asm__ __volatile__ ( - - "movq 224(%[sha512]), %%rsi\n\t" - "leaq %[K512], %%rcx\n\t" - - INIT_MASK(MASK) - LOAD_DIGEST() - - "# Start of processing a block\n" - "2:\n\t" - - /* 16 Ws plus loop counter and K512. len goes into -4(%rsp). - * Debug needs more stack space. */ - "subq $256, %%rsp\n\t" - - LOAD_W(MASK, rsi) - - "movl $4, 16*8(" WX ")\n\t" - /* L4 = b */ - "movq %%r9, " L4 "\n\t" - /* L3 = 0 (add to prev h) */ - "xorq " L3 ", " L3 "\n\t" - /* L4 = b ^ c */ - "xorq %%r10, " L4 "\n\t" - - SET_W_X(rcx) - - "# Start of 16 rounds\n" - "1:\n\t" - - "addq $128, %%rcx\n\t" - "movq %%rcx, 17*8(%%rsp)\n\t" - - MsgSched_RORX(W_0,W_2,W_4,W_6,W_8,W_10,W_12,W_14,RA,RB,RC,RD,RE,RF,RG,RH, 0) - MsgSched_RORX(W_2,W_4,W_6,W_8,W_10,W_12,W_14,W_0,RG,RH,RA,RB,RC,RD,RE,RF, 2) - MsgSched_RORX(W_4,W_6,W_8,W_10,W_12,W_14,W_0,W_2,RE,RF,RG,RH,RA,RB,RC,RD, 4) - MsgSched_RORX(W_6,W_8,W_10,W_12,W_14,W_0,W_2,W_4,RC,RD,RE,RF,RG,RH,RA,RB, 6) - MsgSched_RORX(W_8,W_10,W_12,W_14,W_0,W_2,W_4,W_6,RA,RB,RC,RD,RE,RF,RG,RH, 8) - MsgSched_RORX(W_10,W_12,W_14,W_0,W_2,W_4,W_6,W_8,RG,RH,RA,RB,RC,RD,RE,RF,10) - MsgSched_RORX(W_12,W_14,W_0,W_2,W_4,W_6,W_8,W_10,RE,RF,RG,RH,RA,RB,RC,RD,12) - MsgSched_RORX(W_14,W_0,W_2,W_4,W_6,W_8,W_10,W_12,RC,RD,RE,RF,RG,RH,RA,RB,14) - - "movq 17*8(%%rsp), %%rcx\n\t" - - SET_W_X(rcx) - - "subl $1, 16*8(" WX ")\n\t" - "jne 1b\n\t" - - SET_W_X(rcx) - - RND_RORX_ALL_2(RA,RB,RC,RD,RE,RF,RG,RH, 0) - RND_RORX_ALL_2(RG,RH,RA,RB,RC,RD,RE,RF, 2) - RND_RORX_ALL_2(RE,RF,RG,RH,RA,RB,RC,RD, 4) - RND_RORX_ALL_2(RC,RD,RE,RF,RG,RH,RA,RB, 6) - - RND_RORX_ALL_2(RA,RB,RC,RD,RE,RF,RG,RH, 8) - RND_RORX_ALL_2(RG,RH,RA,RB,RC,RD,RE,RF,10) - RND_RORX_ALL_2(RE,RF,RG,RH,RA,RB,RC,RD,12) - RND_RORX_ALL_2(RC,RD,RE,RF,RG,RH,RA,RB,14) - - /* Prev RND: h += Maj(a,b,c) */ - "addq " L3 ", %%r8\n\t" - "addq $256, %%rsp\n\t" - - ADD_DIGEST() - - "leaq %[K512], %%rcx\n\t" - "addq $128, %%rsi\n\t" - "subl $128, %[len]\n\t" - - STORE_DIGEST() - - "jnz 2b\n\t" - - : - : [mask] "m" (mBYTE_FLIP_MASK), - [len] "m" (len), - [sha512] "r" (sha512), - [K512] "m" (K512) - : WORK_REGS, STATE_REGS, XMM_REGS, "memory", "rsi" - ); - - return 0; -} -#endif /* HAVE_INTEL_AVX2 && HAVE_INTEL_RORX */ - -#if defined(HAVE_INTEL_AVX2) -static const unsigned long mBYTE_FLIP_MASK_Y[] = - { 0x0001020304050607, 0x08090a0b0c0d0e0f, - 0x0001020304050607, 0x08090a0b0c0d0e0f }; - -#define W_Y_0 ymm0 -#define W_Y_4 ymm1 -#define W_Y_8 ymm2 -#define W_Y_12 ymm3 - -#define X0 xmm0 -#define X1 xmm1 -#define X2 xmm2 -#define X3 xmm3 -#define X4 xmm4 -#define X5 xmm5 -#define X6 xmm6 -#define X7 xmm7 -#define X8 xmm8 -#define X9 xmm9 -#define Y0 ymm0 -#define Y1 ymm1 -#define Y2 ymm2 -#define Y3 ymm3 -#define Y4 ymm4 -#define Y5 ymm5 -#define Y6 ymm6 -#define Y7 ymm7 - -#define W_Y_M15 ymm12 -#define W_Y_M7 ymm13 -#define W_Y_M2 ymm14 -#define MASK_Y ymm15 - -#define YTMP1 ymm8 -#define YTMP2 ymm9 -#define YTMP3 ymm10 -#define YTMP4 ymm11 - -#define YMM_REGS \ - "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", "ymm7", \ - "xmm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", "ymm14", "ymm15" - -#define _VPERM2I128(dest, src1, src2, sel) \ - "vperm2I128 $" #sel ", %%" #src2 ", %%" #src1 ", %%" #dest "\n\t" -#define VPERM2I128(dest, src1, src2, sel) \ - _VPERM2I128(dest, src1, src2, sel) - -#define _VPERMQ(dest, src, sel) \ - "vpermq $" #sel ", %%" #src ", %%" #dest "\n\t" -#define VPERMQ(dest, src, sel) \ - _VPERMQ(dest, src, sel) - -#define _VPBLENDD(dest, src1, src2, sel) \ - "vpblendd $" #sel ", %%" #src2 ", %%" #src1 ", %%" #dest "\n\t" -#define VPBLENDD(dest, src1, src2, sel) \ - _VPBLENDD(dest, src1, src2, sel) - -#define _V_ADD_I(dest, src1, addr, i) \ - "vpaddq "#i"*8(%%" #addr "), %%" #src1 ", %%" #dest "\n\t" -#define V_ADD_I(dest, src1, addr, i) \ - _V_ADD_I(dest, src1, addr, i) - -#define _VMOVDQU_I(addr, i, src) \ - "vmovdqu %%" #src ", " #i "*8(%%" #addr ")\n\t" -#define VMOVDQU_I(addr, i, src) \ - _VMOVDQU_I(addr, i, src) - -#define MsgSched4_AVX2(W_Y_0,W_Y_4,W_Y_8,W_Y_12,a,b,c,d,e,f,g,h,i) \ - RND_0_1(a,b,c,d,e,f,g,h,i) \ - /* W[-13]..W[-15], W[-12] */ \ - VPBLENDD(W_Y_M15, W_Y_0, W_Y_4, 0x03) \ - /* W[-5]..W[-7], W[-4] */ \ - VPBLENDD(W_Y_M7, W_Y_8, W_Y_12, 0x03) \ - RND_0_2(a,b,c,d,e,f,g,h,i) \ - RND_0_3(a,b,c,d,e,f,g,h,i) \ - /* W_Y_M15 = W[-12]..W[-15] */ \ - VPERMQ(W_Y_M15, W_Y_M15, 0x39) \ - RND_0_4(a,b,c,d,e,f,g,h,i) \ - /* W_Y_M7 = W[-4]..W[-7] */ \ - VPERMQ(W_Y_M7, W_Y_M7, 0x39) \ - RND_0_5(a,b,c,d,e,f,g,h,i) \ - RND_0_6(a,b,c,d,e,f,g,h,i) \ - /* W[-15] >> 1 */ \ - V_SHIFT_R(YTMP1, W_Y_M15, 1) \ - RND_0_7(a,b,c,d,e,f,g,h,i) \ - /* W[-15] << 63 */ \ - V_SHIFT_L(YTMP2, W_Y_M15, 63) \ - RND_0_8(a,b,c,d,e,f,g,h,i) \ - /* W[-15] >> 8 */ \ - V_SHIFT_R(YTMP3, W_Y_M15, 8) \ - RND_0_9(a,b,c,d,e,f,g,h,i) \ - /* W[-15] << 56 */ \ - V_SHIFT_L(YTMP4, W_Y_M15, 56) \ - RND_0_10(a,b,c,d,e,f,g,h,i) \ - /* W[-15] >>> 1 */ \ - V_OR(YTMP1, YTMP2, YTMP1) \ - RND_0_11(a,b,c,d,e,f,g,h,i) \ - /* W[-15] >>> 8 */ \ - V_OR(YTMP3, YTMP4, YTMP3) \ - RND_0_12(a,b,c,d,e,f,g,h,i) \ - RND_1_1(h,a,b,c,d,e,f,g,i+1) \ - /* W[-15] >> 7 */ \ - V_SHIFT_R(YTMP4, W_Y_M15, 7) \ - RND_1_2_A(h,a,b,c,d,e,f,g,i+1) \ - /* (W[-15] >>> 1) ^ (W[-15] >>> 8) */ \ - V_XOR(YTMP1, YTMP3, YTMP1) \ - RND_1_2_B(h,a,b,c,d,e,f,g,i+1) \ - /* (W[-15] >>> 1) ^ (W[-15] >>> 8) ^ (W[-15] >> 7) */ \ - V_XOR(YTMP1, YTMP4, YTMP1) \ - RND_1_3(h,a,b,c,d,e,f,g,i+1) \ - /* W[0] = W[-16] + W[-7] */ \ - V_ADD(W_Y_0, W_Y_0, W_Y_M7) \ - RND_1_4(h,a,b,c,d,e,f,g,i+1) \ - /* W[0] = W[-16] + W[-7] + s0(W[-15]) */ \ - V_ADD(W_Y_0, W_Y_0, YTMP1) \ - RND_1_5(h,a,b,c,d,e,f,g,i+1) \ - /* 0, 0, W[-1], W[-2] */ \ - VPERM2I128(W_Y_M2, W_Y_12, W_Y_12, 0x81) \ - RND_1_6(h,a,b,c,d,e,f,g,i+1) \ - RND_1_7(h,a,b,c,d,e,f,g,i+1) \ - RND_1_8(h,a,b,c,d,e,f,g,i+1) \ - /* W[-2] >> 19 */ \ - V_SHIFT_R(YTMP1, W_Y_M2, 19) \ - RND_1_9(h,a,b,c,d,e,f,g,i+1) \ - /* W[-2] << 45 */ \ - V_SHIFT_L(YTMP2, W_Y_M2, 45) \ - RND_1_10(h,a,b,c,d,e,f,g,i+1) \ - /* W[-2] >> 61 */ \ - V_SHIFT_R(YTMP3, W_Y_M2, 61) \ - RND_1_11(h,a,b,c,d,e,f,g,i+1) \ - /* W[-2] << 3 */ \ - V_SHIFT_L(YTMP4, W_Y_M2, 3) \ - RND_1_12(h,a,b,c,d,e,f,g,i+1) \ - RND_0_1(g,h,a,b,c,d,e,f,i+2) \ - /* W[-2] >>> 19 */ \ - V_OR(YTMP1, YTMP2, YTMP1) \ - RND_0_2(g,h,a,b,c,d,e,f,i+2) \ - /* W[-2] >>> 61 */ \ - V_OR(YTMP3, YTMP4, YTMP3) \ - RND_0_3(g,h,a,b,c,d,e,f,i+2) \ - /* (W[-2] >>> 19) ^ (W[-2] >>> 61) */ \ - V_XOR(YTMP1, YTMP3, YTMP1) \ - RND_0_4(g,h,a,b,c,d,e,f,i+2) \ - /* W[-2] >> 6 */ \ - V_SHIFT_R(YTMP4, W_Y_M2, 6) \ - RND_0_5(g,h,a,b,c,d,e,f,i+2) \ - /* (W[-2] >>> 19) ^ (W[-2] >>> 61) ^ (W[-2] >> 6) */ \ - V_XOR(YTMP1, YTMP4, YTMP1) \ - RND_0_6(g,h,a,b,c,d,e,f,i+2) \ - /* W[0] = W[-16] + W[-7] + s0(W[-15]) + s1(W[-2]) */ \ - V_ADD(W_Y_0, W_Y_0, YTMP1) \ - RND_0_7(g,h,a,b,c,d,e,f,i+2) \ - RND_0_8(g,h,a,b,c,d,e,f,i+2) \ - /* W[1], W[0], 0, 0 */ \ - VPERM2I128(W_Y_M2, W_Y_0, W_Y_0, 0x08) \ - RND_0_9(g,h,a,b,c,d,e,f,i+2) \ - RND_0_10(g,h,a,b,c,d,e,f,i+2) \ - /* W[-2] >> 19 */ \ - V_SHIFT_R(YTMP1, W_Y_M2, 19) \ - RND_0_11(g,h,a,b,c,d,e,f,i+2) \ - /* W[-2] << 45 */ \ - V_SHIFT_L(YTMP2, W_Y_M2, 45) \ - RND_0_12(g,h,a,b,c,d,e,f,i+2) \ - RND_1_1(f,g,h,a,b,c,d,e,i+3) \ - /* W[-2] >> 61 */ \ - V_SHIFT_R(YTMP3, W_Y_M2, 61) \ - RND_1_2(f,g,h,a,b,c,d,e,i+3) \ - /* W[-2] << 3 */ \ - V_SHIFT_L(YTMP4, W_Y_M2, 3) \ - RND_1_3(f,g,h,a,b,c,d,e,i+3) \ - /* W[-2] >>> 19 */ \ - V_OR(YTMP1, YTMP2, YTMP1) \ - RND_1_4(f,g,h,a,b,c,d,e,i+3) \ - /* W[-2] >>> 61 */ \ - V_OR(YTMP3, YTMP4, YTMP3) \ - RND_1_5(f,g,h,a,b,c,d,e,i+3) \ - /* (W[-2] >>> 19) ^ (W[-2] >>> 61) */ \ - V_XOR(YTMP1, YTMP3, YTMP1) \ - RND_1_6(f,g,h,a,b,c,d,e,i+3) \ - /* W[-2] >> 6 */ \ - V_SHIFT_R(YTMP4, W_Y_M2, 6) \ - RND_1_7(f,g,h,a,b,c,d,e,i+3) \ - /* (W[-2] >>> 19) ^ (W[-2] >>> 61) ^ (W[-2] >> 6) */ \ - V_XOR(YTMP1, YTMP4, YTMP1) \ - RND_1_8(f,g,h,a,b,c,d,e,i+3) \ - /* W[0] = W[-16] + W[-7] + s0(W[-15]) + s1(W[-2]) */ \ - V_ADD(W_Y_0, W_Y_0, YTMP1) \ - RND_1_9(f,g,h,a,b,c,d,e,i+3) \ - RND_1_10(f,g,h,a,b,c,d,e,i+3) \ - RND_1_11(f,g,h,a,b,c,d,e,i+3) \ - RND_1_12(f,g,h,a,b,c,d,e,i+3) \ - -#define MsgSched2_AVX2(W_0,W_2,W_4,W_6,W_8,W_10,W_12,W_14,a,b,c,d,e,f,g,h,i) \ - RND_0_1(a,b,c,d,e,f,g,h,i) \ - VPALIGNR(W_Y_M15, W_2, W_0, 8) \ - VPALIGNR(W_Y_M7, W_10, W_8, 8) \ - RND_0_2(a,b,c,d,e,f,g,h,i) \ - V_SHIFT_R(YTMP1, W_Y_M15, 1) \ - V_SHIFT_L(YTMP2, W_Y_M15, 63) \ - RND_0_3(a,b,c,d,e,f,g,h,i) \ - RND_0_4(a,b,c,d,e,f,g,h,i) \ - V_SHIFT_R(YTMP3, W_Y_M15, 8) \ - V_SHIFT_L(YTMP4, W_Y_M15, 56) \ - RND_0_5(a,b,c,d,e,f,g,h,i) \ - RND_0_6(a,b,c,d,e,f,g,h,i) \ - V_OR(YTMP1, YTMP2, YTMP1) \ - V_OR(YTMP3, YTMP4, YTMP3) \ - RND_0_7(a,b,c,d,e,f,g,h,i) \ - RND_0_8(a,b,c,d,e,f,g,h,i) \ - V_SHIFT_R(YTMP4, W_Y_M15, 7) \ - V_XOR(YTMP1, YTMP3, YTMP1) \ - RND_0_9(a,b,c,d,e,f,g,h,i) \ - RND_0_10(a,b,c,d,e,f,g,h,i) \ - V_XOR(YTMP1, YTMP4, YTMP1) \ - V_ADD(W_0, W_0, W_Y_M7) \ - RND_0_11(a,b,c,d,e,f,g,h,i) \ - RND_0_12(a,b,c,d,e,f,g,h,i) \ - RND_1_1(h,a,b,c,d,e,f,g,i+1) \ - V_ADD(W_0, W_0, YTMP1) \ - RND_1_2(h,a,b,c,d,e,f,g,i+1) \ - V_SHIFT_R(YTMP1, W_14, 19) \ - V_SHIFT_L(YTMP2, W_14, 45) \ - RND_1_3(h,a,b,c,d,e,f,g,i+1) \ - RND_1_4(h,a,b,c,d,e,f,g,i+1) \ - V_SHIFT_R(YTMP3, W_14, 61) \ - V_SHIFT_L(YTMP4, W_14, 3) \ - RND_1_5(h,a,b,c,d,e,f,g,i+1) \ - RND_1_6(h,a,b,c,d,e,f,g,i+1) \ - RND_1_7(h,a,b,c,d,e,f,g,i+1) \ - V_OR(YTMP1, YTMP2, YTMP1) \ - V_OR(YTMP3, YTMP4, YTMP3) \ - RND_1_8(h,a,b,c,d,e,f,g,i+1) \ - RND_1_9(h,a,b,c,d,e,f,g,i+1) \ - V_XOR(YTMP1, YTMP3, YTMP1) \ - V_SHIFT_R(YTMP4, W_14, 6) \ - RND_1_10(h,a,b,c,d,e,f,g,i+1) \ - RND_1_11(h,a,b,c,d,e,f,g,i+1) \ - V_XOR(YTMP1, YTMP4, YTMP1) \ - RND_1_12(h,a,b,c,d,e,f,g,i+1) \ - V_ADD(W_0, W_0, YTMP1) \ - -#define MsgSched4_AVX2_RORX_SET(W_Y_0,W_Y_4,W_Y_8,W_Y_12,a,b,c,d,e,f,g,h,i) \ - RND_RORX_0_1(a,b,c,d,e,f,g,h,i) \ - /* W[-13]..W[-15], W[-12] */ \ - VPBLENDD(W_Y_M15, W_Y_0, W_Y_4, 0x03) \ - /* W[-5]..W[-7], W[-4] */ \ - VPBLENDD(W_Y_M7, W_Y_8, W_Y_12, 0x03) \ - RND_RORX_0_2(a,b,c,d,e,f,g,h,i) \ - /* W_Y_M15 = W[-12]..W[-15] */ \ - VPERMQ(W_Y_M15, W_Y_M15, 0x39) \ - RND_RORX_0_3(a,b,c,d,e,f,g,h,i) \ - /* W_Y_M7 = W[-4]..W[-7] */ \ - VPERMQ(W_Y_M7, W_Y_M7, 0x39) \ - RND_RORX_0_4(a,b,c,d,e,f,g,h,i) \ - /* W[-15] >> 1 */ \ - V_SHIFT_R(YTMP1, W_Y_M15, 1) \ - /* W[-15] << 63 */ \ - V_SHIFT_L(YTMP2, W_Y_M15, 63) \ - RND_RORX_0_5(a,b,c,d,e,f,g,h,i) \ - /* W[-15] >> 8 */ \ - V_SHIFT_R(YTMP3, W_Y_M15, 8) \ - /* W[-15] << 56 */ \ - V_SHIFT_L(YTMP4, W_Y_M15, 56) \ - /* W[-15] >>> 1 */ \ - V_OR(YTMP1, YTMP2, YTMP1) \ - /* W[-15] >>> 8 */ \ - V_OR(YTMP3, YTMP4, YTMP3) \ - RND_RORX_0_6(a,b,c,d,e,f,g,h,i) \ - /* W[-15] >> 7 */ \ - V_SHIFT_R(YTMP4, W_Y_M15, 7) \ - RND_RORX_0_7(a,b,c,d,e,f,g,h,i) \ - /* 0, 0, W[-1], W[-2] */ \ - VPERM2I128(W_Y_M2, W_Y_12, W_Y_12, 0x81) \ - RND_RORX_0_8(a,b,c,d,e,f,g,h,i) \ - RND_RORX_1_1(h,a,b,c,d,e,f,g,i+1) \ - /* (W[-15] >>> 1) ^ (W[-15] >>> 8) */ \ - V_XOR(YTMP1, YTMP3, YTMP1) \ - RND_RORX_1_2(h,a,b,c,d,e,f,g,i+1) \ - /* (W[-15] >>> 1) ^ (W[-15] >>> 8) ^ (W[-15] >> 7) */ \ - V_XOR(YTMP1, YTMP4, YTMP1) \ - RND_RORX_1_3(h,a,b,c,d,e,f,g,i+1) \ - /* W[0] = W[-16] + W[-7] */ \ - V_ADD(W_Y_0, W_Y_0, W_Y_M7) \ - /* W[0] = W[-16] + W[-7] + s0(W[-15]) */ \ - V_ADD(W_Y_0, W_Y_0, YTMP1) \ - RND_RORX_1_4(h,a,b,c,d,e,f,g,i+1) \ - /* W[-2] >> 19 */ \ - V_SHIFT_R(YTMP1, W_Y_M2, 19) \ - /* W[-2] << 45 */ \ - V_SHIFT_L(YTMP2, W_Y_M2, 45) \ - RND_RORX_1_5(h,a,b,c,d,e,f,g,i+1) \ - /* W[-2] >> 61 */ \ - V_SHIFT_R(YTMP3, W_Y_M2, 61) \ - /* W[-2] << 3 */ \ - V_SHIFT_L(YTMP4, W_Y_M2, 3) \ - /* W[-2] >>> 19 */ \ - V_OR(YTMP1, YTMP2, YTMP1) \ - RND_RORX_1_6(h,a,b,c,d,e,f,g,i+1) \ - /* W[-2] >>> 61 */ \ - V_OR(YTMP3, YTMP4, YTMP3) \ - RND_RORX_1_7(h,a,b,c,d,e,f,g,i+1) \ - /* (W[-2] >>> 19) ^ (W[-2] >>> 61) */ \ - V_XOR(YTMP1, YTMP3, YTMP1) \ - RND_RORX_1_8(h,a,b,c,d,e,f,g,i+1) \ - /* W[-2] >> 6 */ \ - V_SHIFT_R(YTMP4, W_Y_M2, 6) \ - RND_RORX_0_1(g,h,a,b,c,d,e,f,i+2) \ - /* (W[-2] >>> 19) ^ (W[-2] >>> 61) ^ (W[-2] >> 6) */ \ - V_XOR(YTMP1, YTMP4, YTMP1) \ - RND_RORX_0_2(g,h,a,b,c,d,e,f,i+2) \ - /* W[0] = W[-16] + W[-7] + s0(W[-15]) + s1(W[-2]) */ \ - V_ADD(W_Y_0, W_Y_0, YTMP1) \ - RND_RORX_0_3(g,h,a,b,c,d,e,f,i+2) \ - /* W[1], W[0], 0, 0 */ \ - VPERM2I128(W_Y_M2, W_Y_0, W_Y_0, 0x08) \ - RND_RORX_0_4(g,h,a,b,c,d,e,f,i+2) \ - RND_RORX_0_5(g,h,a,b,c,d,e,f,i+2) \ - /* W[-2] >> 19 */ \ - V_SHIFT_R(YTMP1, W_Y_M2, 19) \ - /* W[-2] << 45 */ \ - V_SHIFT_L(YTMP2, W_Y_M2, 45) \ - RND_RORX_0_6(g,h,a,b,c,d,e,f,i+2) \ - /* W[-2] >> 61 */ \ - V_SHIFT_R(YTMP3, W_Y_M2, 61) \ - /* W[-2] << 3 */ \ - V_SHIFT_L(YTMP4, W_Y_M2, 3) \ - /* W[-2] >>> 19 */ \ - V_OR(YTMP1, YTMP2, YTMP1) \ - RND_RORX_0_7(g,h,a,b,c,d,e,f,i+2) \ - /* W[-2] >>> 61 */ \ - V_OR(YTMP3, YTMP4, YTMP3) \ - RND_RORX_0_8(g,h,a,b,c,d,e,f,i+2) \ - /* (W[-2] >>> 19) ^ (W[-2] >>> 61) */ \ - V_XOR(YTMP1, YTMP3, YTMP1) \ - RND_RORX_1_1(f,g,h,a,b,c,d,e,i+3) \ - /* W[-2] >> 6 */ \ - V_SHIFT_R(YTMP4, W_Y_M2, 6) \ - RND_RORX_1_2(f,g,h,a,b,c,d,e,i+3) \ - RND_RORX_1_3(f,g,h,a,b,c,d,e,i+3) \ - /* (W[-2] >>> 19) ^ (W[-2] >>> 61) ^ (W[-2] >> 6) */ \ - V_XOR(YTMP1, YTMP4, YTMP1) \ - RND_RORX_1_4(f,g,h,a,b,c,d,e,i+3) \ - RND_RORX_1_5(f,g,h,a,b,c,d,e,i+3) \ - /* W[0] = W[-16] + W[-7] + s0(W[-15]) + s1(W[-2]) */ \ - V_ADD(W_Y_0, W_Y_0, YTMP1) \ - RND_RORX_1_6(f,g,h,a,b,c,d,e,i+3) \ - V_ADD_I(YTMP1, W_Y_0, rsi, i) \ - RND_RORX_1_7(f,g,h,a,b,c,d,e,i+3) \ - RND_RORX_1_8(f,g,h,a,b,c,d,e,i+3) \ - VMOVDQU_I(rsp, i, YTMP1) \ - -#define MsgSched2_AVX2_RORX(W_0,W_2,W_4,W_6,W_8,W_10,W_12,W_14,a,b,c,d,e, \ - f,g,h,i) \ - RND_RORX_0_1(a,b,c,d,e,f,g,h,i) \ - VPALIGNR(W_Y_M15, W_2, W_0, 8) \ - VPALIGNR(W_Y_M7, W_10, W_8, 8) \ - RND_RORX_0_2(a,b,c,d,e,f,g,h,i) \ - V_SHIFT_R(YTMP1, W_Y_M15, 1) \ - V_SHIFT_L(YTMP2, W_Y_M15, 63) \ - RND_RORX_0_3(a,b,c,d,e,f,g,h,i) \ - V_SHIFT_R(YTMP3, W_Y_M15, 8) \ - V_SHIFT_L(YTMP4, W_Y_M15, 56) \ - RND_RORX_0_4(a,b,c,d,e,f,g,h,i) \ - V_OR(YTMP1, YTMP2, YTMP1) \ - V_OR(YTMP3, YTMP4, YTMP3) \ - RND_RORX_0_5(a,b,c,d,e,f,g,h,i) \ - V_SHIFT_R(YTMP4, W_Y_M15, 7) \ - V_XOR(YTMP1, YTMP3, YTMP1) \ - RND_RORX_0_6(a,b,c,d,e,f,g,h,i) \ - V_XOR(YTMP1, YTMP4, YTMP1) \ - V_ADD(W_0, W_0, W_Y_M7) \ - RND_RORX_0_7(a,b,c,d,e,f,g,h,i) \ - RND_RORX_0_8(a,b,c,d,e,f,g,h,i) \ - V_ADD(W_0, W_0, YTMP1) \ - RND_RORX_1_1(h,a,b,c,d,e,f,g,i+1) \ - V_SHIFT_R(YTMP1, W_14, 19) \ - V_SHIFT_L(YTMP2, W_14, 45) \ - RND_RORX_1_2(h,a,b,c,d,e,f,g,i+1) \ - V_SHIFT_R(YTMP3, W_14, 61) \ - V_SHIFT_L(YTMP4, W_14, 3) \ - RND_RORX_1_3(h,a,b,c,d,e,f,g,i+1) \ - V_OR(YTMP1, YTMP2, YTMP1) \ - V_OR(YTMP3, YTMP4, YTMP3) \ - RND_RORX_1_4(h,a,b,c,d,e,f,g,i+1) \ - RND_RORX_1_5(h,a,b,c,d,e,f,g,i+1) \ - V_XOR(YTMP1, YTMP3, YTMP1) \ - V_SHIFT_R(YTMP4, W_14, 6) \ - RND_RORX_1_6(h,a,b,c,d,e,f,g,i+1) \ - RND_RORX_1_7(h,a,b,c,d,e,f,g,i+1) \ - V_XOR(YTMP1, YTMP4, YTMP1) \ - RND_RORX_1_8(h,a,b,c,d,e,f,g,i+1) \ - V_ADD(W_0, W_0, YTMP1) \ - - -#define _INIT_MASK_Y(mask) \ - "vmovdqu %[mask], %%"#mask"\n\t" -#define INIT_MASK_Y(mask) \ - _INIT_MASK_Y(mask) - -/* Load into YMM registers and swap endian. */ -#define _LOAD_BLOCK_W_Y_2(mask, ymm0, ymm1, reg, i) \ - /* buffer[0..15] => ymm0..ymm3; */ \ - "vmovdqu " #i "+ 0(%%" #reg "), %%" #ymm0 "\n\t" \ - "vmovdqu " #i "+32(%%" #reg "), %%" #ymm1 "\n\t" \ - "vpshufb %%" #mask ", %%" #ymm0 ", %%" #ymm0 "\n\t" \ - "vpshufb %%" #mask ", %%" #ymm1 ", %%" #ymm1 "\n\t" - -#define LOAD_BLOCK_W_Y_2(mask, ymm1, ymm2, reg, i) \ - _LOAD_BLOCK_W_Y_2(mask, ymm1, ymm2, reg, i) - -#define LOAD_BLOCK_W_Y(mask, reg) \ - LOAD_BLOCK_W_Y_2(mask, W_Y_0, W_Y_4 , reg, 0) \ - LOAD_BLOCK_W_Y_2(mask, W_Y_8, W_Y_12, reg, 64) - -#define _SET_W_Y_2(ymm0, ymm1, ymm2, ymm3, reg, i) \ - "vpaddq " #i "+ 0(%%" #reg "), %%" #ymm0 ", %%" #ymm2 "\n\t" \ - "vpaddq " #i "+32(%%" #reg "), %%" #ymm1 ", %%" #ymm3 "\n\t" \ - "vmovdqu %%" #ymm2 ", " #i "+ 0(" WX ")\n\t" \ - "vmovdqu %%" #ymm3 ", " #i "+32(" WX ")\n\t" - -#define SET_W_Y_2(ymm0, ymm1, ymm2, ymm3, reg, i) \ - _SET_W_Y_2(ymm0, ymm1, ymm2, ymm3, reg, i) - -#define SET_BLOCK_W_Y(reg) \ - SET_W_Y_2(W_Y_0, W_Y_4 , YTMP1, YTMP2, reg, 0) \ - SET_W_Y_2(W_Y_8, W_Y_12, YTMP1, YTMP2, reg, 64) - -/* Load into YMM registers and swap endian. */ -#define _LOAD_BLOCK2_W_Y_2(mask, Y0, Y1, X0, X1, X8, X9, reg, i) \ - "vmovdqu " #i "+ 0(%%" #reg "), %%" #X0 "\n\t" \ - "vmovdqu " #i "+ 16(%%" #reg "), %%" #X1 "\n\t" \ - "vmovdqu " #i "+128(%%" #reg "), %%" #X8 "\n\t" \ - "vmovdqu " #i "+144(%%" #reg "), %%" #X9 "\n\t" \ - "vinserti128 $1, %%" #X8 ", %%" #Y0 ", %%" #Y0 "\n\t" \ - "vinserti128 $1, %%" #X9 ", %%" #Y1 ", %%" #Y1 "\n\t" \ - "vpshufb %%" #mask ", %%" #Y0 ", %%" #Y0 "\n\t" \ - "vpshufb %%" #mask ", %%" #Y1 ", %%" #Y1 "\n\t" - -#define LOAD_BLOCK2_W_Y_2(mask, Y0, Y1, X0, X1, X8, X9, reg, i) \ - _LOAD_BLOCK2_W_Y_2(mask, Y0, Y1, X0, X1, X8, X9, reg, i) - -#define LOAD_BLOCK2_W_Y(mask, reg) \ - LOAD_BLOCK2_W_Y_2(mask, Y0, Y1, X0, X1, X8, X9, reg, 0) \ - LOAD_BLOCK2_W_Y_2(mask, Y2, Y3, X2, X3, X8, X9, reg, 32) \ - LOAD_BLOCK2_W_Y_2(mask, Y4, Y5, X4, X5, X8, X9, reg, 64) \ - LOAD_BLOCK2_W_Y_2(mask, Y6, Y7, X6, X7, X8, X9, reg, 96) \ - -#define SET_BLOCK2_W_Y(reg) \ - SET_W_Y_2(Y0, Y1, YTMP1, YTMP2, reg, 0) \ - SET_W_Y_2(Y2, Y3, YTMP1, YTMP2, reg, 64) \ - SET_W_Y_2(Y4, Y5, YTMP1, YTMP2, reg, 128) \ - SET_W_Y_2(Y6, Y7, YTMP1, YTMP2, reg, 192) - -static const word64 K512_AVX2[160] = { - W64LIT(0x428a2f98d728ae22), W64LIT(0x7137449123ef65cd), - W64LIT(0x428a2f98d728ae22), W64LIT(0x7137449123ef65cd), - W64LIT(0xb5c0fbcfec4d3b2f), W64LIT(0xe9b5dba58189dbbc), - W64LIT(0xb5c0fbcfec4d3b2f), W64LIT(0xe9b5dba58189dbbc), - W64LIT(0x3956c25bf348b538), W64LIT(0x59f111f1b605d019), - W64LIT(0x3956c25bf348b538), W64LIT(0x59f111f1b605d019), - W64LIT(0x923f82a4af194f9b), W64LIT(0xab1c5ed5da6d8118), - W64LIT(0x923f82a4af194f9b), W64LIT(0xab1c5ed5da6d8118), - W64LIT(0xd807aa98a3030242), W64LIT(0x12835b0145706fbe), - W64LIT(0xd807aa98a3030242), W64LIT(0x12835b0145706fbe), - W64LIT(0x243185be4ee4b28c), W64LIT(0x550c7dc3d5ffb4e2), - W64LIT(0x243185be4ee4b28c), W64LIT(0x550c7dc3d5ffb4e2), - W64LIT(0x72be5d74f27b896f), W64LIT(0x80deb1fe3b1696b1), - W64LIT(0x72be5d74f27b896f), W64LIT(0x80deb1fe3b1696b1), - W64LIT(0x9bdc06a725c71235), W64LIT(0xc19bf174cf692694), - W64LIT(0x9bdc06a725c71235), W64LIT(0xc19bf174cf692694), - W64LIT(0xe49b69c19ef14ad2), W64LIT(0xefbe4786384f25e3), - W64LIT(0xe49b69c19ef14ad2), W64LIT(0xefbe4786384f25e3), - W64LIT(0x0fc19dc68b8cd5b5), W64LIT(0x240ca1cc77ac9c65), - W64LIT(0x0fc19dc68b8cd5b5), W64LIT(0x240ca1cc77ac9c65), - W64LIT(0x2de92c6f592b0275), W64LIT(0x4a7484aa6ea6e483), - W64LIT(0x2de92c6f592b0275), W64LIT(0x4a7484aa6ea6e483), - W64LIT(0x5cb0a9dcbd41fbd4), W64LIT(0x76f988da831153b5), - W64LIT(0x5cb0a9dcbd41fbd4), W64LIT(0x76f988da831153b5), - W64LIT(0x983e5152ee66dfab), W64LIT(0xa831c66d2db43210), - W64LIT(0x983e5152ee66dfab), W64LIT(0xa831c66d2db43210), - W64LIT(0xb00327c898fb213f), W64LIT(0xbf597fc7beef0ee4), - W64LIT(0xb00327c898fb213f), W64LIT(0xbf597fc7beef0ee4), - W64LIT(0xc6e00bf33da88fc2), W64LIT(0xd5a79147930aa725), - W64LIT(0xc6e00bf33da88fc2), W64LIT(0xd5a79147930aa725), - W64LIT(0x06ca6351e003826f), W64LIT(0x142929670a0e6e70), - W64LIT(0x06ca6351e003826f), W64LIT(0x142929670a0e6e70), - W64LIT(0x27b70a8546d22ffc), W64LIT(0x2e1b21385c26c926), - W64LIT(0x27b70a8546d22ffc), W64LIT(0x2e1b21385c26c926), - W64LIT(0x4d2c6dfc5ac42aed), W64LIT(0x53380d139d95b3df), - W64LIT(0x4d2c6dfc5ac42aed), W64LIT(0x53380d139d95b3df), - W64LIT(0x650a73548baf63de), W64LIT(0x766a0abb3c77b2a8), - W64LIT(0x650a73548baf63de), W64LIT(0x766a0abb3c77b2a8), - W64LIT(0x81c2c92e47edaee6), W64LIT(0x92722c851482353b), - W64LIT(0x81c2c92e47edaee6), W64LIT(0x92722c851482353b), - W64LIT(0xa2bfe8a14cf10364), W64LIT(0xa81a664bbc423001), - W64LIT(0xa2bfe8a14cf10364), W64LIT(0xa81a664bbc423001), - W64LIT(0xc24b8b70d0f89791), W64LIT(0xc76c51a30654be30), - W64LIT(0xc24b8b70d0f89791), W64LIT(0xc76c51a30654be30), - W64LIT(0xd192e819d6ef5218), W64LIT(0xd69906245565a910), - W64LIT(0xd192e819d6ef5218), W64LIT(0xd69906245565a910), - W64LIT(0xf40e35855771202a), W64LIT(0x106aa07032bbd1b8), - W64LIT(0xf40e35855771202a), W64LIT(0x106aa07032bbd1b8), - W64LIT(0x19a4c116b8d2d0c8), W64LIT(0x1e376c085141ab53), - W64LIT(0x19a4c116b8d2d0c8), W64LIT(0x1e376c085141ab53), - W64LIT(0x2748774cdf8eeb99), W64LIT(0x34b0bcb5e19b48a8), - W64LIT(0x2748774cdf8eeb99), W64LIT(0x34b0bcb5e19b48a8), - W64LIT(0x391c0cb3c5c95a63), W64LIT(0x4ed8aa4ae3418acb), - W64LIT(0x391c0cb3c5c95a63), W64LIT(0x4ed8aa4ae3418acb), - W64LIT(0x5b9cca4f7763e373), W64LIT(0x682e6ff3d6b2b8a3), - W64LIT(0x5b9cca4f7763e373), W64LIT(0x682e6ff3d6b2b8a3), - W64LIT(0x748f82ee5defb2fc), W64LIT(0x78a5636f43172f60), - W64LIT(0x748f82ee5defb2fc), W64LIT(0x78a5636f43172f60), - W64LIT(0x84c87814a1f0ab72), W64LIT(0x8cc702081a6439ec), - W64LIT(0x84c87814a1f0ab72), W64LIT(0x8cc702081a6439ec), - W64LIT(0x90befffa23631e28), W64LIT(0xa4506cebde82bde9), - W64LIT(0x90befffa23631e28), W64LIT(0xa4506cebde82bde9), - W64LIT(0xbef9a3f7b2c67915), W64LIT(0xc67178f2e372532b), - W64LIT(0xbef9a3f7b2c67915), W64LIT(0xc67178f2e372532b), - W64LIT(0xca273eceea26619c), W64LIT(0xd186b8c721c0c207), - W64LIT(0xca273eceea26619c), W64LIT(0xd186b8c721c0c207), - W64LIT(0xeada7dd6cde0eb1e), W64LIT(0xf57d4f7fee6ed178), - W64LIT(0xeada7dd6cde0eb1e), W64LIT(0xf57d4f7fee6ed178), - W64LIT(0x06f067aa72176fba), W64LIT(0x0a637dc5a2c898a6), - W64LIT(0x06f067aa72176fba), W64LIT(0x0a637dc5a2c898a6), - W64LIT(0x113f9804bef90dae), W64LIT(0x1b710b35131c471b), - W64LIT(0x113f9804bef90dae), W64LIT(0x1b710b35131c471b), - W64LIT(0x28db77f523047d84), W64LIT(0x32caab7b40c72493), - W64LIT(0x28db77f523047d84), W64LIT(0x32caab7b40c72493), - W64LIT(0x3c9ebe0a15c9bebc), W64LIT(0x431d67c49c100d4c), - W64LIT(0x3c9ebe0a15c9bebc), W64LIT(0x431d67c49c100d4c), - W64LIT(0x4cc5d4becb3e42b6), W64LIT(0x597f299cfc657e2a), - W64LIT(0x4cc5d4becb3e42b6), W64LIT(0x597f299cfc657e2a), - W64LIT(0x5fcb6fab3ad6faec), W64LIT(0x6c44198c4a475817), - W64LIT(0x5fcb6fab3ad6faec), W64LIT(0x6c44198c4a475817) -}; -static const word64* K512_AVX2_END = &K512_AVX2[128]; - -static int Transform_Sha512_AVX2(wc_Sha512* sha512) -{ - __asm__ __volatile__ ( - - /* 16 Ws plus loop counter and K512. */ - "subq $136, %%rsp\n\t" - "leaq 64(%[sha512]), %%rax\n\t" - - INIT_MASK(MASK_Y) - LOAD_DIGEST() - - LOAD_BLOCK_W_Y(MASK_Y, rax) - - "movl $4, 16*8(" WX ")\n\t" - "leaq %[K512], %%rsi\n\t" - /* b */ - "movq %%r9, " L4 "\n\t" - /* e */ - "movq %%r12, " L1 "\n\t" - /* b ^ c */ - "xorq %%r10, " L4 "\n\t" - - SET_BLOCK_W_Y(rsi) - - "# Start of 16 rounds\n" - "1:\n\t" - - "addq $128, %%rsi\n\t" - - MsgSched4_AVX2(W_Y_0,W_Y_4,W_Y_8,W_Y_12,RA,RB,RC,RD,RE,RF,RG,RH, 0) - MsgSched4_AVX2(W_Y_4,W_Y_8,W_Y_12,W_Y_0,RE,RF,RG,RH,RA,RB,RC,RD, 4) - MsgSched4_AVX2(W_Y_8,W_Y_12,W_Y_0,W_Y_4,RA,RB,RC,RD,RE,RF,RG,RH, 8) - MsgSched4_AVX2(W_Y_12,W_Y_0,W_Y_4,W_Y_8,RE,RF,RG,RH,RA,RB,RC,RD,12) - - SET_BLOCK_W_Y(rsi) - - "subl $1, 16*8(" WX ")\n\t" - "jne 1b\n\t" - - RND_ALL_2(RA,RB,RC,RD,RE,RF,RG,RH, 0) - RND_ALL_2(RG,RH,RA,RB,RC,RD,RE,RF, 2) - RND_ALL_2(RE,RF,RG,RH,RA,RB,RC,RD, 4) - RND_ALL_2(RC,RD,RE,RF,RG,RH,RA,RB, 6) - - RND_ALL_2(RA,RB,RC,RD,RE,RF,RG,RH, 8) - RND_ALL_2(RG,RH,RA,RB,RC,RD,RE,RF,10) - RND_ALL_2(RE,RF,RG,RH,RA,RB,RC,RD,12) - RND_ALL_2(RC,RD,RE,RF,RG,RH,RA,RB,14) - - STORE_ADD_DIGEST() - - "addq $136, %%rsp\n\t" - - : - : [mask] "m" (mBYTE_FLIP_MASK_Y), - [sha512] "r" (sha512), - [K512] "m" (K512) - : WORK_REGS, STATE_REGS, YMM_REGS, "memory", "rsi" - ); - - return 0; -} - -static int Transform_Sha512_AVX2_Len(wc_Sha512* sha512, word32 len) -{ - if ((len & WC_SHA512_BLOCK_SIZE) != 0) { - XMEMCPY(sha512->buffer, sha512->data, WC_SHA512_BLOCK_SIZE); - Transform_Sha512_AVX2(sha512); - sha512->data += WC_SHA512_BLOCK_SIZE; - len -= WC_SHA512_BLOCK_SIZE; - if (len == 0) - return 0; - } - - __asm__ __volatile__ ( - - "movq 224(%[sha512]), %%rcx\n\t" - - INIT_MASK(MASK_Y) - LOAD_DIGEST() - - "# Start of processing two blocks\n" - "2:\n\t" - - "subq $1344, %%rsp\n\t" - "leaq %[K512], %%rsi\n\t" - - /* L4 = b */ - "movq %%r9, " L4 "\n\t" - /* e */ - "movq %%r12, " L1 "\n\t" - - LOAD_BLOCK2_W_Y(MASK_Y, rcx) - - /* L4 = b ^ c */ - "xorq %%r10, " L4 "\n\t" - "\n" - "1:\n\t" - SET_BLOCK2_W_Y(rsi) - MsgSched2_AVX2(Y0,Y1,Y2,Y3,Y4,Y5,Y6,Y7,RA,RB,RC,RD,RE,RF,RG,RH, 0) - MsgSched2_AVX2(Y1,Y2,Y3,Y4,Y5,Y6,Y7,Y0,RG,RH,RA,RB,RC,RD,RE,RF, 4) - MsgSched2_AVX2(Y2,Y3,Y4,Y5,Y6,Y7,Y0,Y1,RE,RF,RG,RH,RA,RB,RC,RD, 8) - MsgSched2_AVX2(Y3,Y4,Y5,Y6,Y7,Y0,Y1,Y2,RC,RD,RE,RF,RG,RH,RA,RB,12) - MsgSched2_AVX2(Y4,Y5,Y6,Y7,Y0,Y1,Y2,Y3,RA,RB,RC,RD,RE,RF,RG,RH,16) - MsgSched2_AVX2(Y5,Y6,Y7,Y0,Y1,Y2,Y3,Y4,RG,RH,RA,RB,RC,RD,RE,RF,20) - MsgSched2_AVX2(Y6,Y7,Y0,Y1,Y2,Y3,Y4,Y5,RE,RF,RG,RH,RA,RB,RC,RD,24) - MsgSched2_AVX2(Y7,Y0,Y1,Y2,Y3,Y4,Y5,Y6,RC,RD,RE,RF,RG,RH,RA,RB,28) - "addq $256, %%rsi\n\t" - "addq $256, %%rsp\n\t" - "cmpq %[K512_END], %%rsi\n\t" - "jne 1b\n\t" - - SET_BLOCK2_W_Y(rsi) - RND_ALL_2(RA,RB,RC,RD,RE,RF,RG,RH, 0) - RND_ALL_2(RG,RH,RA,RB,RC,RD,RE,RF, 4) - RND_ALL_2(RE,RF,RG,RH,RA,RB,RC,RD, 8) - RND_ALL_2(RC,RD,RE,RF,RG,RH,RA,RB,12) - - RND_ALL_2(RA,RB,RC,RD,RE,RF,RG,RH,16) - RND_ALL_2(RG,RH,RA,RB,RC,RD,RE,RF,20) - RND_ALL_2(RE,RF,RG,RH,RA,RB,RC,RD,24) - RND_ALL_2(RC,RD,RE,RF,RG,RH,RA,RB,28) - "subq $1024, %%rsp\n\t" - - ADD_DIGEST() - STORE_DIGEST() - - /* L4 = b */ - "movq %%r9, " L4 "\n\t" - /* e */ - "movq %%r12, " L1 "\n\t" - /* L4 = b ^ c */ - "xorq %%r10, " L4 "\n\t" - - "movq $5, %%rsi\n\t" - "\n" - "3:\n\t" - RND_ALL_2(RA,RB,RC,RD,RE,RF,RG,RH, 2) - RND_ALL_2(RG,RH,RA,RB,RC,RD,RE,RF, 6) - RND_ALL_2(RE,RF,RG,RH,RA,RB,RC,RD,10) - RND_ALL_2(RC,RD,RE,RF,RG,RH,RA,RB,14) - - RND_ALL_2(RA,RB,RC,RD,RE,RF,RG,RH,18) - RND_ALL_2(RG,RH,RA,RB,RC,RD,RE,RF,22) - RND_ALL_2(RE,RF,RG,RH,RA,RB,RC,RD,26) - RND_ALL_2(RC,RD,RE,RF,RG,RH,RA,RB,30) - "addq $256, %%rsp\n\t" - "subq $1, %%rsi\n\t" - "jnz 3b\n\t" - - ADD_DIGEST() - - "movq 224(%[sha512]), %%rcx\n\t" - "addq $64, %%rsp\n\t" - "addq $256, %%rcx\n\t" - "subl $256, %[len]\n\t" - "movq %%rcx, 224(%[sha512])\n\t" - - STORE_DIGEST() - - "jnz 2b\n\t" - - : - : [mask] "m" (mBYTE_FLIP_MASK_Y), - [len] "m" (len), - [sha512] "r" (sha512), - [K512] "m" (K512_AVX2), - [K512_END] "m" (K512_AVX2_END) - : WORK_REGS, STATE_REGS, YMM_REGS, "memory", "rsi" - ); - - return 0; -} - -#ifdef HAVE_INTEL_RORX -static int Transform_Sha512_AVX2_RORX(wc_Sha512* sha512) -{ - __asm__ __volatile__ ( - - /* 16 Ws plus loop counter. */ - "subq $136, %%rsp\n\t" - "leaq 64(%[sha512]), " L2 "\n\t" - - INIT_MASK(MASK_Y) - LOAD_DIGEST() - - LOAD_BLOCK_W_Y(MASK_Y, rcx) - - "movl $4, 16*8(" WX ")\n\t" - "leaq %[K512], %%rsi\n\t" - /* b */ - "movq %%r9, " L4 "\n\t" - /* L3 = 0 (add to prev h) */ - "xorq " L3 ", " L3 "\n\t" - /* b ^ c */ - "xorq %%r10, " L4 "\n\t" - - SET_BLOCK_W_Y(rsi) - - "# Start of 16 rounds\n" - "1:\n\t" - - "addq $128, %%rsi\n\t" - - MsgSched4_AVX2_RORX_SET(W_Y_0,W_Y_4,W_Y_8,W_Y_12,RA,RB,RC,RD,RE,RF,RG,RH, 0) - MsgSched4_AVX2_RORX_SET(W_Y_4,W_Y_8,W_Y_12,W_Y_0,RE,RF,RG,RH,RA,RB,RC,RD, 4) - MsgSched4_AVX2_RORX_SET(W_Y_8,W_Y_12,W_Y_0,W_Y_4,RA,RB,RC,RD,RE,RF,RG,RH, 8) - MsgSched4_AVX2_RORX_SET(W_Y_12,W_Y_0,W_Y_4,W_Y_8,RE,RF,RG,RH,RA,RB,RC,RD,12) - - "subl $1, 16*8(%%rsp)\n\t" - "jnz 1b\n\t" - - RND_RORX_ALL_4(RA,RB,RC,RD,RE,RF,RG,RH, 0) - RND_RORX_ALL_4(RE,RF,RG,RH,RA,RB,RC,RD, 4) - RND_RORX_ALL_4(RA,RB,RC,RD,RE,RF,RG,RH, 8) - RND_RORX_ALL_4(RE,RF,RG,RH,RA,RB,RC,RD,12) - /* Prev RND: h += Maj(a,b,c) */ - "addq " L3 ", %%r8\n\t" - "addq $136, %%rsp\n\t" - - STORE_ADD_DIGEST() - - : - : [mask] "m" (mBYTE_FLIP_MASK_Y), - [sha512] "r" (sha512), - [K512] "m" (K512) - : WORK_REGS, STATE_REGS, YMM_REGS, "memory", "rsi" - ); - - return 0; -} - -static int Transform_Sha512_AVX2_RORX_Len(wc_Sha512* sha512, word32 len) -{ - if ((len & WC_SHA512_BLOCK_SIZE) != 0) { - XMEMCPY(sha512->buffer, sha512->data, WC_SHA512_BLOCK_SIZE); - Transform_Sha512_AVX2_RORX(sha512); - sha512->data += WC_SHA512_BLOCK_SIZE; - len -= WC_SHA512_BLOCK_SIZE; - if (len == 0) - return 0; - } - - __asm__ __volatile__ ( - - "movq 224(%[sha512]), %%rax\n\t" - - INIT_MASK(MASK_Y) - LOAD_DIGEST() - - "# Start of processing two blocks\n" - "2:\n\t" - - "subq $1344, %%rsp\n\t" - "leaq %[K512], %%rsi\n\t" - - /* L4 = b */ - "movq %%r9, " L4 "\n\t" - /* L3 = 0 (add to prev h) */ - "xorq " L3 ", " L3 "\n\t" - - LOAD_BLOCK2_W_Y(MASK_Y, rax) - - /* L4 = b ^ c */ - "xorq %%r10, " L4 "\n\t" - "\n" - "1:\n\t" - SET_BLOCK2_W_Y(rsi) - MsgSched2_AVX2_RORX(Y0,Y1,Y2,Y3,Y4,Y5,Y6,Y7,RA,RB,RC,RD,RE,RF,RG,RH, 0) - MsgSched2_AVX2_RORX(Y1,Y2,Y3,Y4,Y5,Y6,Y7,Y0,RG,RH,RA,RB,RC,RD,RE,RF, 4) - MsgSched2_AVX2_RORX(Y2,Y3,Y4,Y5,Y6,Y7,Y0,Y1,RE,RF,RG,RH,RA,RB,RC,RD, 8) - MsgSched2_AVX2_RORX(Y3,Y4,Y5,Y6,Y7,Y0,Y1,Y2,RC,RD,RE,RF,RG,RH,RA,RB,12) - MsgSched2_AVX2_RORX(Y4,Y5,Y6,Y7,Y0,Y1,Y2,Y3,RA,RB,RC,RD,RE,RF,RG,RH,16) - MsgSched2_AVX2_RORX(Y5,Y6,Y7,Y0,Y1,Y2,Y3,Y4,RG,RH,RA,RB,RC,RD,RE,RF,20) - MsgSched2_AVX2_RORX(Y6,Y7,Y0,Y1,Y2,Y3,Y4,Y5,RE,RF,RG,RH,RA,RB,RC,RD,24) - MsgSched2_AVX2_RORX(Y7,Y0,Y1,Y2,Y3,Y4,Y5,Y6,RC,RD,RE,RF,RG,RH,RA,RB,28) - "addq $256, %%rsi\n\t" - "addq $256, %%rsp\n\t" - "cmpq %[K512_END], %%rsi\n\t" - "jne 1b\n\t" - - SET_BLOCK2_W_Y(rsi) - RND_RORX_ALL_2(RA,RB,RC,RD,RE,RF,RG,RH, 0) - RND_RORX_ALL_2(RG,RH,RA,RB,RC,RD,RE,RF, 4) - RND_RORX_ALL_2(RE,RF,RG,RH,RA,RB,RC,RD, 8) - RND_RORX_ALL_2(RC,RD,RE,RF,RG,RH,RA,RB,12) - - RND_RORX_ALL_2(RA,RB,RC,RD,RE,RF,RG,RH,16) - RND_RORX_ALL_2(RG,RH,RA,RB,RC,RD,RE,RF,20) - RND_RORX_ALL_2(RE,RF,RG,RH,RA,RB,RC,RD,24) - RND_RORX_ALL_2(RC,RD,RE,RF,RG,RH,RA,RB,28) - "addq " L3 ", %%r8\n\t" - "subq $1024, %%rsp\n\t" - - ADD_DIGEST() - STORE_DIGEST() - - /* L4 = b */ - "movq %%r9, " L4 "\n\t" - /* L3 = 0 (add to prev h) */ - "xorq " L3 ", " L3 "\n\t" - /* L4 = b ^ c */ - "xorq %%r10, " L4 "\n\t" - - "movq $5, %%rsi\n\t" - "\n" - "3:\n\t" - RND_RORX_ALL_2(RA,RB,RC,RD,RE,RF,RG,RH, 2) - RND_RORX_ALL_2(RG,RH,RA,RB,RC,RD,RE,RF, 6) - RND_RORX_ALL_2(RE,RF,RG,RH,RA,RB,RC,RD,10) - RND_RORX_ALL_2(RC,RD,RE,RF,RG,RH,RA,RB,14) - - RND_RORX_ALL_2(RA,RB,RC,RD,RE,RF,RG,RH,18) - RND_RORX_ALL_2(RG,RH,RA,RB,RC,RD,RE,RF,22) - RND_RORX_ALL_2(RE,RF,RG,RH,RA,RB,RC,RD,26) - RND_RORX_ALL_2(RC,RD,RE,RF,RG,RH,RA,RB,30) - "addq $256, %%rsp\n\t" - "subq $1, %%rsi\n\t" - "jnz 3b\n\t" - - "addq " L3 ", %%r8\n\t" - - ADD_DIGEST() - - "movq 224(%[sha512]), %%rax\n\t" - "addq $64, %%rsp\n\t" - "addq $256, %%rax\n\t" - "subl $256, %[len]\n\t" - "movq %%rax, 224(%[sha512])\n\t" - - STORE_DIGEST() - - "jnz 2b\n\t" - - : - : [mask] "m" (mBYTE_FLIP_MASK_Y), - [len] "m" (len), - [sha512] "r" (sha512), - [K512] "m" (K512_AVX2), - [K512_END] "m" (K512_AVX2_END) - : WORK_REGS, STATE_REGS, YMM_REGS, "memory", "rsi" - ); - - return 0; -} -#endif /* HAVE_INTEL_RORX */ -#endif /* HAVE_INTEL_AVX2 */ - #endif /* WOLFSSL_SHA512 */ - /* -------------------------------------------------------------------------- */ /* SHA384 */ /* -------------------------------------------------------------------------- */ diff --git a/wolfcrypt/src/sha512_asm.S b/wolfcrypt/src/sha512_asm.S new file mode 100644 index 0000000000..dded1c009b --- /dev/null +++ b/wolfcrypt/src/sha512_asm.S @@ -0,0 +1,10545 @@ +/* sha512_asm + * + * Copyright (C) 2006-2018 wolfSSL Inc. + * + * This file is part of wolfSSL. + * + * wolfSSL is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * wolfSSL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA + */ + +#ifndef HAVE_INTEL_AVX1 +#define HAVE_INTEL_AVX1 +#endif /* HAVE_INTEL_AVX1 */ +#ifndef HAVE_INTEL_AVX2 +#define HAVE_INTEL_AVX2 +#endif /* HAVE_INTEL_AVX2 */ + +#ifdef HAVE_INTEL_AVX1 +.align 16 +L_avx1_sha512_k: +.quad 0x428a2f98d728ae22,0x7137449123ef65cd +.quad 0xb5c0fbcfec4d3b2f,0xe9b5dba58189dbbc +.quad 0x3956c25bf348b538,0x59f111f1b605d019 +.quad 0x923f82a4af194f9b,0xab1c5ed5da6d8118 +.quad 0xd807aa98a3030242,0x12835b0145706fbe +.quad 0x243185be4ee4b28c,0x550c7dc3d5ffb4e2 +.quad 0x72be5d74f27b896f,0x80deb1fe3b1696b1 +.quad 0x9bdc06a725c71235,0xc19bf174cf692694 +.quad 0xe49b69c19ef14ad2,0xefbe4786384f25e3 +.quad 0xfc19dc68b8cd5b5,0x240ca1cc77ac9c65 +.quad 0x2de92c6f592b0275,0x4a7484aa6ea6e483 +.quad 0x5cb0a9dcbd41fbd4,0x76f988da831153b5 +.quad 0x983e5152ee66dfab,0xa831c66d2db43210 +.quad 0xb00327c898fb213f,0xbf597fc7beef0ee4 +.quad 0xc6e00bf33da88fc2,0xd5a79147930aa725 +.quad 0x6ca6351e003826f,0x142929670a0e6e70 +.quad 0x27b70a8546d22ffc,0x2e1b21385c26c926 +.quad 0x4d2c6dfc5ac42aed,0x53380d139d95b3df +.quad 0x650a73548baf63de,0x766a0abb3c77b2a8 +.quad 0x81c2c92e47edaee6,0x92722c851482353b +.quad 0xa2bfe8a14cf10364,0xa81a664bbc423001 +.quad 0xc24b8b70d0f89791,0xc76c51a30654be30 +.quad 0xd192e819d6ef5218,0xd69906245565a910 +.quad 0xf40e35855771202a,0x106aa07032bbd1b8 +.quad 0x19a4c116b8d2d0c8,0x1e376c085141ab53 +.quad 0x2748774cdf8eeb99,0x34b0bcb5e19b48a8 +.quad 0x391c0cb3c5c95a63,0x4ed8aa4ae3418acb +.quad 0x5b9cca4f7763e373,0x682e6ff3d6b2b8a3 +.quad 0x748f82ee5defb2fc,0x78a5636f43172f60 +.quad 0x84c87814a1f0ab72,0x8cc702081a6439ec +.quad 0x90befffa23631e28,0xa4506cebde82bde9 +.quad 0xbef9a3f7b2c67915,0xc67178f2e372532b +.quad 0xca273eceea26619c,0xd186b8c721c0c207 +.quad 0xeada7dd6cde0eb1e,0xf57d4f7fee6ed178 +.quad 0x6f067aa72176fba,0xa637dc5a2c898a6 +.quad 0x113f9804bef90dae,0x1b710b35131c471b +.quad 0x28db77f523047d84,0x32caab7b40c72493 +.quad 0x3c9ebe0a15c9bebc,0x431d67c49c100d4c +.quad 0x4cc5d4becb3e42b6,0x597f299cfc657e2a +.quad 0x5fcb6fab3ad6faec,0x6c44198c4a475817 +.align 16 +L_avx1_sha512_flip_mask: +.quad 0x1020304050607, 0x8090a0b0c0d0e0f +.globl Transform_Sha512_AVX1 +.type Transform_Sha512_AVX1,@function +.align 4 +Transform_Sha512_AVX1: + pushq %rbx + pushq %r12 + pushq %r13 + pushq %r14 + pushq %r15 + subq $0x88, %rsp + leaq 64(%rdi), %rax + vmovdqa L_avx1_sha512_flip_mask(%rip), %xmm14 + movq (%rdi), %r8 + movq 8(%rdi), %r9 + movq 16(%rdi), %r10 + movq 24(%rdi), %r11 + movq 32(%rdi), %r12 + movq 40(%rdi), %r13 + movq 48(%rdi), %r14 + movq 56(%rdi), %r15 + vmovdqu (%rax), %xmm0 + vmovdqu 16(%rax), %xmm1 + vpshufb %xmm14, %xmm0, %xmm0 + vpshufb %xmm14, %xmm1, %xmm1 + vmovdqu 32(%rax), %xmm2 + vmovdqu 48(%rax), %xmm3 + vpshufb %xmm14, %xmm2, %xmm2 + vpshufb %xmm14, %xmm3, %xmm3 + vmovdqu 64(%rax), %xmm4 + vmovdqu 80(%rax), %xmm5 + vpshufb %xmm14, %xmm4, %xmm4 + vpshufb %xmm14, %xmm5, %xmm5 + vmovdqu 96(%rax), %xmm6 + vmovdqu 112(%rax), %xmm7 + vpshufb %xmm14, %xmm6, %xmm6 + vpshufb %xmm14, %xmm7, %xmm7 + movl $4, 128(%rsp) + leaq L_avx1_sha512_k(%rip), %rsi + movq %r9, %rbx + movq %r12, %rax + xorq %r10, %rbx + # Start of 16 rounds +L_sha256_len_avx1_start: + vpaddq (%rsi), %xmm0, %xmm8 + vpaddq 16(%rsi), %xmm1, %xmm9 + vmovdqu %xmm8, (%rsp) + vmovdqu %xmm9, 16(%rsp) + vpaddq 32(%rsi), %xmm2, %xmm8 + vpaddq 48(%rsi), %xmm3, %xmm9 + vmovdqu %xmm8, 32(%rsp) + vmovdqu %xmm9, 48(%rsp) + vpaddq 64(%rsi), %xmm4, %xmm8 + vpaddq 80(%rsi), %xmm5, %xmm9 + vmovdqu %xmm8, 64(%rsp) + vmovdqu %xmm9, 80(%rsp) + vpaddq 96(%rsi), %xmm6, %xmm8 + vpaddq 112(%rsi), %xmm7, %xmm9 + vmovdqu %xmm8, 96(%rsp) + vmovdqu %xmm9, 112(%rsp) + addq $0x80, %rsi + # msg_sched: 0-1 + # rnd_0: 0 - 0 + rorq $23, %rax + vpalignr $8, %xmm0, %xmm1, %xmm12 + vpalignr $8, %xmm4, %xmm5, %xmm13 + # rnd_0: 1 - 1 + movq %r8, %rdx + movq %r13, %rcx + addq (%rsp), %r15 + xorq %r14, %rcx + vpsrlq $0x01, %xmm12, %xmm8 + vpsllq $63, %xmm12, %xmm9 + # rnd_0: 2 - 3 + xorq %r12, %rax + andq %r12, %rcx + rorq $4, %rax + xorq %r14, %rcx + vpsrlq $8, %xmm12, %xmm10 + vpsllq $56, %xmm12, %xmm11 + # rnd_0: 4 - 5 + xorq %r12, %rax + addq %rcx, %r15 + rorq $14, %rax + xorq %r9, %rdx + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_0: 6 - 7 + addq %rax, %r15 + movq %r8, %rcx + andq %rdx, %rbx + rorq $5, %rcx + vpsrlq $7, %xmm12, %xmm11 + vpxor %xmm10, %xmm8, %xmm8 + # rnd_0: 8 - 9 + xorq %r8, %rcx + xorq %r9, %rbx + rorq $6, %rcx + addq %r15, %r11 + vpxor %xmm11, %xmm8, %xmm8 + vpaddq %xmm0, %xmm13, %xmm0 + # rnd_0: 10 - 11 + xorq %r8, %rcx + addq %rbx, %r15 + rorq $28, %rcx + movq %r11, %rax + addq %rcx, %r15 + # rnd_1: 0 - 0 + rorq $23, %rax + vpaddq %xmm0, %xmm8, %xmm0 + # rnd_1: 1 - 1 + movq %r15, %rbx + movq %r12, %rcx + addq 8(%rsp), %r14 + xorq %r13, %rcx + vpsrlq $19, %xmm7, %xmm8 + vpsllq $45, %xmm7, %xmm9 + # rnd_1: 2 - 3 + xorq %r11, %rax + andq %r11, %rcx + rorq $4, %rax + xorq %r13, %rcx + vpsrlq $61, %xmm7, %xmm10 + vpsllq $3, %xmm7, %xmm11 + # rnd_1: 4 - 6 + xorq %r11, %rax + addq %rcx, %r14 + rorq $14, %rax + xorq %r8, %rbx + addq %rax, %r14 + movq %r15, %rcx + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_1: 7 - 8 + andq %rbx, %rdx + rorq $5, %rcx + xorq %r15, %rcx + xorq %r8, %rdx + vpxor %xmm10, %xmm8, %xmm8 + vpsrlq $6, %xmm7, %xmm11 + # rnd_1: 9 - 10 + rorq $6, %rcx + addq %r14, %r10 + xorq %r15, %rcx + addq %rdx, %r14 + vpxor %xmm11, %xmm8, %xmm8 + # rnd_1: 11 - 11 + rorq $28, %rcx + movq %r10, %rax + addq %rcx, %r14 + vpaddq %xmm0, %xmm8, %xmm0 + # msg_sched done: 0-3 + # msg_sched: 2-3 + # rnd_0: 0 - 0 + rorq $23, %rax + vpalignr $8, %xmm1, %xmm2, %xmm12 + vpalignr $8, %xmm5, %xmm6, %xmm13 + # rnd_0: 1 - 1 + movq %r14, %rdx + movq %r11, %rcx + addq 16(%rsp), %r13 + xorq %r12, %rcx + vpsrlq $0x01, %xmm12, %xmm8 + vpsllq $63, %xmm12, %xmm9 + # rnd_0: 2 - 3 + xorq %r10, %rax + andq %r10, %rcx + rorq $4, %rax + xorq %r12, %rcx + vpsrlq $8, %xmm12, %xmm10 + vpsllq $56, %xmm12, %xmm11 + # rnd_0: 4 - 5 + xorq %r10, %rax + addq %rcx, %r13 + rorq $14, %rax + xorq %r15, %rdx + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_0: 6 - 7 + addq %rax, %r13 + movq %r14, %rcx + andq %rdx, %rbx + rorq $5, %rcx + vpsrlq $7, %xmm12, %xmm11 + vpxor %xmm10, %xmm8, %xmm8 + # rnd_0: 8 - 9 + xorq %r14, %rcx + xorq %r15, %rbx + rorq $6, %rcx + addq %r13, %r9 + vpxor %xmm11, %xmm8, %xmm8 + vpaddq %xmm1, %xmm13, %xmm1 + # rnd_0: 10 - 11 + xorq %r14, %rcx + addq %rbx, %r13 + rorq $28, %rcx + movq %r9, %rax + addq %rcx, %r13 + # rnd_1: 0 - 0 + rorq $23, %rax + vpaddq %xmm1, %xmm8, %xmm1 + # rnd_1: 1 - 1 + movq %r13, %rbx + movq %r10, %rcx + addq 24(%rsp), %r12 + xorq %r11, %rcx + vpsrlq $19, %xmm0, %xmm8 + vpsllq $45, %xmm0, %xmm9 + # rnd_1: 2 - 3 + xorq %r9, %rax + andq %r9, %rcx + rorq $4, %rax + xorq %r11, %rcx + vpsrlq $61, %xmm0, %xmm10 + vpsllq $3, %xmm0, %xmm11 + # rnd_1: 4 - 6 + xorq %r9, %rax + addq %rcx, %r12 + rorq $14, %rax + xorq %r14, %rbx + addq %rax, %r12 + movq %r13, %rcx + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_1: 7 - 8 + andq %rbx, %rdx + rorq $5, %rcx + xorq %r13, %rcx + xorq %r14, %rdx + vpxor %xmm10, %xmm8, %xmm8 + vpsrlq $6, %xmm0, %xmm11 + # rnd_1: 9 - 10 + rorq $6, %rcx + addq %r12, %r8 + xorq %r13, %rcx + addq %rdx, %r12 + vpxor %xmm11, %xmm8, %xmm8 + # rnd_1: 11 - 11 + rorq $28, %rcx + movq %r8, %rax + addq %rcx, %r12 + vpaddq %xmm1, %xmm8, %xmm1 + # msg_sched done: 2-5 + # msg_sched: 4-5 + # rnd_0: 0 - 0 + rorq $23, %rax + vpalignr $8, %xmm2, %xmm3, %xmm12 + vpalignr $8, %xmm6, %xmm7, %xmm13 + # rnd_0: 1 - 1 + movq %r12, %rdx + movq %r9, %rcx + addq 32(%rsp), %r11 + xorq %r10, %rcx + vpsrlq $0x01, %xmm12, %xmm8 + vpsllq $63, %xmm12, %xmm9 + # rnd_0: 2 - 3 + xorq %r8, %rax + andq %r8, %rcx + rorq $4, %rax + xorq %r10, %rcx + vpsrlq $8, %xmm12, %xmm10 + vpsllq $56, %xmm12, %xmm11 + # rnd_0: 4 - 5 + xorq %r8, %rax + addq %rcx, %r11 + rorq $14, %rax + xorq %r13, %rdx + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_0: 6 - 7 + addq %rax, %r11 + movq %r12, %rcx + andq %rdx, %rbx + rorq $5, %rcx + vpsrlq $7, %xmm12, %xmm11 + vpxor %xmm10, %xmm8, %xmm8 + # rnd_0: 8 - 9 + xorq %r12, %rcx + xorq %r13, %rbx + rorq $6, %rcx + addq %r11, %r15 + vpxor %xmm11, %xmm8, %xmm8 + vpaddq %xmm2, %xmm13, %xmm2 + # rnd_0: 10 - 11 + xorq %r12, %rcx + addq %rbx, %r11 + rorq $28, %rcx + movq %r15, %rax + addq %rcx, %r11 + # rnd_1: 0 - 0 + rorq $23, %rax + vpaddq %xmm2, %xmm8, %xmm2 + # rnd_1: 1 - 1 + movq %r11, %rbx + movq %r8, %rcx + addq 40(%rsp), %r10 + xorq %r9, %rcx + vpsrlq $19, %xmm1, %xmm8 + vpsllq $45, %xmm1, %xmm9 + # rnd_1: 2 - 3 + xorq %r15, %rax + andq %r15, %rcx + rorq $4, %rax + xorq %r9, %rcx + vpsrlq $61, %xmm1, %xmm10 + vpsllq $3, %xmm1, %xmm11 + # rnd_1: 4 - 6 + xorq %r15, %rax + addq %rcx, %r10 + rorq $14, %rax + xorq %r12, %rbx + addq %rax, %r10 + movq %r11, %rcx + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_1: 7 - 8 + andq %rbx, %rdx + rorq $5, %rcx + xorq %r11, %rcx + xorq %r12, %rdx + vpxor %xmm10, %xmm8, %xmm8 + vpsrlq $6, %xmm1, %xmm11 + # rnd_1: 9 - 10 + rorq $6, %rcx + addq %r10, %r14 + xorq %r11, %rcx + addq %rdx, %r10 + vpxor %xmm11, %xmm8, %xmm8 + # rnd_1: 11 - 11 + rorq $28, %rcx + movq %r14, %rax + addq %rcx, %r10 + vpaddq %xmm2, %xmm8, %xmm2 + # msg_sched done: 4-7 + # msg_sched: 6-7 + # rnd_0: 0 - 0 + rorq $23, %rax + vpalignr $8, %xmm3, %xmm4, %xmm12 + vpalignr $8, %xmm7, %xmm0, %xmm13 + # rnd_0: 1 - 1 + movq %r10, %rdx + movq %r15, %rcx + addq 48(%rsp), %r9 + xorq %r8, %rcx + vpsrlq $0x01, %xmm12, %xmm8 + vpsllq $63, %xmm12, %xmm9 + # rnd_0: 2 - 3 + xorq %r14, %rax + andq %r14, %rcx + rorq $4, %rax + xorq %r8, %rcx + vpsrlq $8, %xmm12, %xmm10 + vpsllq $56, %xmm12, %xmm11 + # rnd_0: 4 - 5 + xorq %r14, %rax + addq %rcx, %r9 + rorq $14, %rax + xorq %r11, %rdx + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_0: 6 - 7 + addq %rax, %r9 + movq %r10, %rcx + andq %rdx, %rbx + rorq $5, %rcx + vpsrlq $7, %xmm12, %xmm11 + vpxor %xmm10, %xmm8, %xmm8 + # rnd_0: 8 - 9 + xorq %r10, %rcx + xorq %r11, %rbx + rorq $6, %rcx + addq %r9, %r13 + vpxor %xmm11, %xmm8, %xmm8 + vpaddq %xmm3, %xmm13, %xmm3 + # rnd_0: 10 - 11 + xorq %r10, %rcx + addq %rbx, %r9 + rorq $28, %rcx + movq %r13, %rax + addq %rcx, %r9 + # rnd_1: 0 - 0 + rorq $23, %rax + vpaddq %xmm3, %xmm8, %xmm3 + # rnd_1: 1 - 1 + movq %r9, %rbx + movq %r14, %rcx + addq 56(%rsp), %r8 + xorq %r15, %rcx + vpsrlq $19, %xmm2, %xmm8 + vpsllq $45, %xmm2, %xmm9 + # rnd_1: 2 - 3 + xorq %r13, %rax + andq %r13, %rcx + rorq $4, %rax + xorq %r15, %rcx + vpsrlq $61, %xmm2, %xmm10 + vpsllq $3, %xmm2, %xmm11 + # rnd_1: 4 - 6 + xorq %r13, %rax + addq %rcx, %r8 + rorq $14, %rax + xorq %r10, %rbx + addq %rax, %r8 + movq %r9, %rcx + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_1: 7 - 8 + andq %rbx, %rdx + rorq $5, %rcx + xorq %r9, %rcx + xorq %r10, %rdx + vpxor %xmm10, %xmm8, %xmm8 + vpsrlq $6, %xmm2, %xmm11 + # rnd_1: 9 - 10 + rorq $6, %rcx + addq %r8, %r12 + xorq %r9, %rcx + addq %rdx, %r8 + vpxor %xmm11, %xmm8, %xmm8 + # rnd_1: 11 - 11 + rorq $28, %rcx + movq %r12, %rax + addq %rcx, %r8 + vpaddq %xmm3, %xmm8, %xmm3 + # msg_sched done: 6-9 + # msg_sched: 8-9 + # rnd_0: 0 - 0 + rorq $23, %rax + vpalignr $8, %xmm4, %xmm5, %xmm12 + vpalignr $8, %xmm0, %xmm1, %xmm13 + # rnd_0: 1 - 1 + movq %r8, %rdx + movq %r13, %rcx + addq 64(%rsp), %r15 + xorq %r14, %rcx + vpsrlq $0x01, %xmm12, %xmm8 + vpsllq $63, %xmm12, %xmm9 + # rnd_0: 2 - 3 + xorq %r12, %rax + andq %r12, %rcx + rorq $4, %rax + xorq %r14, %rcx + vpsrlq $8, %xmm12, %xmm10 + vpsllq $56, %xmm12, %xmm11 + # rnd_0: 4 - 5 + xorq %r12, %rax + addq %rcx, %r15 + rorq $14, %rax + xorq %r9, %rdx + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_0: 6 - 7 + addq %rax, %r15 + movq %r8, %rcx + andq %rdx, %rbx + rorq $5, %rcx + vpsrlq $7, %xmm12, %xmm11 + vpxor %xmm10, %xmm8, %xmm8 + # rnd_0: 8 - 9 + xorq %r8, %rcx + xorq %r9, %rbx + rorq $6, %rcx + addq %r15, %r11 + vpxor %xmm11, %xmm8, %xmm8 + vpaddq %xmm4, %xmm13, %xmm4 + # rnd_0: 10 - 11 + xorq %r8, %rcx + addq %rbx, %r15 + rorq $28, %rcx + movq %r11, %rax + addq %rcx, %r15 + # rnd_1: 0 - 0 + rorq $23, %rax + vpaddq %xmm4, %xmm8, %xmm4 + # rnd_1: 1 - 1 + movq %r15, %rbx + movq %r12, %rcx + addq 72(%rsp), %r14 + xorq %r13, %rcx + vpsrlq $19, %xmm3, %xmm8 + vpsllq $45, %xmm3, %xmm9 + # rnd_1: 2 - 3 + xorq %r11, %rax + andq %r11, %rcx + rorq $4, %rax + xorq %r13, %rcx + vpsrlq $61, %xmm3, %xmm10 + vpsllq $3, %xmm3, %xmm11 + # rnd_1: 4 - 6 + xorq %r11, %rax + addq %rcx, %r14 + rorq $14, %rax + xorq %r8, %rbx + addq %rax, %r14 + movq %r15, %rcx + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_1: 7 - 8 + andq %rbx, %rdx + rorq $5, %rcx + xorq %r15, %rcx + xorq %r8, %rdx + vpxor %xmm10, %xmm8, %xmm8 + vpsrlq $6, %xmm3, %xmm11 + # rnd_1: 9 - 10 + rorq $6, %rcx + addq %r14, %r10 + xorq %r15, %rcx + addq %rdx, %r14 + vpxor %xmm11, %xmm8, %xmm8 + # rnd_1: 11 - 11 + rorq $28, %rcx + movq %r10, %rax + addq %rcx, %r14 + vpaddq %xmm4, %xmm8, %xmm4 + # msg_sched done: 8-11 + # msg_sched: 10-11 + # rnd_0: 0 - 0 + rorq $23, %rax + vpalignr $8, %xmm5, %xmm6, %xmm12 + vpalignr $8, %xmm1, %xmm2, %xmm13 + # rnd_0: 1 - 1 + movq %r14, %rdx + movq %r11, %rcx + addq 80(%rsp), %r13 + xorq %r12, %rcx + vpsrlq $0x01, %xmm12, %xmm8 + vpsllq $63, %xmm12, %xmm9 + # rnd_0: 2 - 3 + xorq %r10, %rax + andq %r10, %rcx + rorq $4, %rax + xorq %r12, %rcx + vpsrlq $8, %xmm12, %xmm10 + vpsllq $56, %xmm12, %xmm11 + # rnd_0: 4 - 5 + xorq %r10, %rax + addq %rcx, %r13 + rorq $14, %rax + xorq %r15, %rdx + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_0: 6 - 7 + addq %rax, %r13 + movq %r14, %rcx + andq %rdx, %rbx + rorq $5, %rcx + vpsrlq $7, %xmm12, %xmm11 + vpxor %xmm10, %xmm8, %xmm8 + # rnd_0: 8 - 9 + xorq %r14, %rcx + xorq %r15, %rbx + rorq $6, %rcx + addq %r13, %r9 + vpxor %xmm11, %xmm8, %xmm8 + vpaddq %xmm5, %xmm13, %xmm5 + # rnd_0: 10 - 11 + xorq %r14, %rcx + addq %rbx, %r13 + rorq $28, %rcx + movq %r9, %rax + addq %rcx, %r13 + # rnd_1: 0 - 0 + rorq $23, %rax + vpaddq %xmm5, %xmm8, %xmm5 + # rnd_1: 1 - 1 + movq %r13, %rbx + movq %r10, %rcx + addq 88(%rsp), %r12 + xorq %r11, %rcx + vpsrlq $19, %xmm4, %xmm8 + vpsllq $45, %xmm4, %xmm9 + # rnd_1: 2 - 3 + xorq %r9, %rax + andq %r9, %rcx + rorq $4, %rax + xorq %r11, %rcx + vpsrlq $61, %xmm4, %xmm10 + vpsllq $3, %xmm4, %xmm11 + # rnd_1: 4 - 6 + xorq %r9, %rax + addq %rcx, %r12 + rorq $14, %rax + xorq %r14, %rbx + addq %rax, %r12 + movq %r13, %rcx + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_1: 7 - 8 + andq %rbx, %rdx + rorq $5, %rcx + xorq %r13, %rcx + xorq %r14, %rdx + vpxor %xmm10, %xmm8, %xmm8 + vpsrlq $6, %xmm4, %xmm11 + # rnd_1: 9 - 10 + rorq $6, %rcx + addq %r12, %r8 + xorq %r13, %rcx + addq %rdx, %r12 + vpxor %xmm11, %xmm8, %xmm8 + # rnd_1: 11 - 11 + rorq $28, %rcx + movq %r8, %rax + addq %rcx, %r12 + vpaddq %xmm5, %xmm8, %xmm5 + # msg_sched done: 10-13 + # msg_sched: 12-13 + # rnd_0: 0 - 0 + rorq $23, %rax + vpalignr $8, %xmm6, %xmm7, %xmm12 + vpalignr $8, %xmm2, %xmm3, %xmm13 + # rnd_0: 1 - 1 + movq %r12, %rdx + movq %r9, %rcx + addq 96(%rsp), %r11 + xorq %r10, %rcx + vpsrlq $0x01, %xmm12, %xmm8 + vpsllq $63, %xmm12, %xmm9 + # rnd_0: 2 - 3 + xorq %r8, %rax + andq %r8, %rcx + rorq $4, %rax + xorq %r10, %rcx + vpsrlq $8, %xmm12, %xmm10 + vpsllq $56, %xmm12, %xmm11 + # rnd_0: 4 - 5 + xorq %r8, %rax + addq %rcx, %r11 + rorq $14, %rax + xorq %r13, %rdx + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_0: 6 - 7 + addq %rax, %r11 + movq %r12, %rcx + andq %rdx, %rbx + rorq $5, %rcx + vpsrlq $7, %xmm12, %xmm11 + vpxor %xmm10, %xmm8, %xmm8 + # rnd_0: 8 - 9 + xorq %r12, %rcx + xorq %r13, %rbx + rorq $6, %rcx + addq %r11, %r15 + vpxor %xmm11, %xmm8, %xmm8 + vpaddq %xmm6, %xmm13, %xmm6 + # rnd_0: 10 - 11 + xorq %r12, %rcx + addq %rbx, %r11 + rorq $28, %rcx + movq %r15, %rax + addq %rcx, %r11 + # rnd_1: 0 - 0 + rorq $23, %rax + vpaddq %xmm6, %xmm8, %xmm6 + # rnd_1: 1 - 1 + movq %r11, %rbx + movq %r8, %rcx + addq 104(%rsp), %r10 + xorq %r9, %rcx + vpsrlq $19, %xmm5, %xmm8 + vpsllq $45, %xmm5, %xmm9 + # rnd_1: 2 - 3 + xorq %r15, %rax + andq %r15, %rcx + rorq $4, %rax + xorq %r9, %rcx + vpsrlq $61, %xmm5, %xmm10 + vpsllq $3, %xmm5, %xmm11 + # rnd_1: 4 - 6 + xorq %r15, %rax + addq %rcx, %r10 + rorq $14, %rax + xorq %r12, %rbx + addq %rax, %r10 + movq %r11, %rcx + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_1: 7 - 8 + andq %rbx, %rdx + rorq $5, %rcx + xorq %r11, %rcx + xorq %r12, %rdx + vpxor %xmm10, %xmm8, %xmm8 + vpsrlq $6, %xmm5, %xmm11 + # rnd_1: 9 - 10 + rorq $6, %rcx + addq %r10, %r14 + xorq %r11, %rcx + addq %rdx, %r10 + vpxor %xmm11, %xmm8, %xmm8 + # rnd_1: 11 - 11 + rorq $28, %rcx + movq %r14, %rax + addq %rcx, %r10 + vpaddq %xmm6, %xmm8, %xmm6 + # msg_sched done: 12-15 + # msg_sched: 14-15 + # rnd_0: 0 - 0 + rorq $23, %rax + vpalignr $8, %xmm7, %xmm0, %xmm12 + vpalignr $8, %xmm3, %xmm4, %xmm13 + # rnd_0: 1 - 1 + movq %r10, %rdx + movq %r15, %rcx + addq 112(%rsp), %r9 + xorq %r8, %rcx + vpsrlq $0x01, %xmm12, %xmm8 + vpsllq $63, %xmm12, %xmm9 + # rnd_0: 2 - 3 + xorq %r14, %rax + andq %r14, %rcx + rorq $4, %rax + xorq %r8, %rcx + vpsrlq $8, %xmm12, %xmm10 + vpsllq $56, %xmm12, %xmm11 + # rnd_0: 4 - 5 + xorq %r14, %rax + addq %rcx, %r9 + rorq $14, %rax + xorq %r11, %rdx + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_0: 6 - 7 + addq %rax, %r9 + movq %r10, %rcx + andq %rdx, %rbx + rorq $5, %rcx + vpsrlq $7, %xmm12, %xmm11 + vpxor %xmm10, %xmm8, %xmm8 + # rnd_0: 8 - 9 + xorq %r10, %rcx + xorq %r11, %rbx + rorq $6, %rcx + addq %r9, %r13 + vpxor %xmm11, %xmm8, %xmm8 + vpaddq %xmm7, %xmm13, %xmm7 + # rnd_0: 10 - 11 + xorq %r10, %rcx + addq %rbx, %r9 + rorq $28, %rcx + movq %r13, %rax + addq %rcx, %r9 + # rnd_1: 0 - 0 + rorq $23, %rax + vpaddq %xmm7, %xmm8, %xmm7 + # rnd_1: 1 - 1 + movq %r9, %rbx + movq %r14, %rcx + addq 120(%rsp), %r8 + xorq %r15, %rcx + vpsrlq $19, %xmm6, %xmm8 + vpsllq $45, %xmm6, %xmm9 + # rnd_1: 2 - 3 + xorq %r13, %rax + andq %r13, %rcx + rorq $4, %rax + xorq %r15, %rcx + vpsrlq $61, %xmm6, %xmm10 + vpsllq $3, %xmm6, %xmm11 + # rnd_1: 4 - 6 + xorq %r13, %rax + addq %rcx, %r8 + rorq $14, %rax + xorq %r10, %rbx + addq %rax, %r8 + movq %r9, %rcx + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_1: 7 - 8 + andq %rbx, %rdx + rorq $5, %rcx + xorq %r9, %rcx + xorq %r10, %rdx + vpxor %xmm10, %xmm8, %xmm8 + vpsrlq $6, %xmm6, %xmm11 + # rnd_1: 9 - 10 + rorq $6, %rcx + addq %r8, %r12 + xorq %r9, %rcx + addq %rdx, %r8 + vpxor %xmm11, %xmm8, %xmm8 + # rnd_1: 11 - 11 + rorq $28, %rcx + movq %r12, %rax + addq %rcx, %r8 + vpaddq %xmm7, %xmm8, %xmm7 + # msg_sched done: 14-17 + subl $0x01, 128(%rsp) + jne L_sha256_len_avx1_start + vpaddq (%rsi), %xmm0, %xmm8 + vpaddq 16(%rsi), %xmm1, %xmm9 + vmovdqu %xmm8, (%rsp) + vmovdqu %xmm9, 16(%rsp) + vpaddq 32(%rsi), %xmm2, %xmm8 + vpaddq 48(%rsi), %xmm3, %xmm9 + vmovdqu %xmm8, 32(%rsp) + vmovdqu %xmm9, 48(%rsp) + vpaddq 64(%rsi), %xmm4, %xmm8 + vpaddq 80(%rsi), %xmm5, %xmm9 + vmovdqu %xmm8, 64(%rsp) + vmovdqu %xmm9, 80(%rsp) + vpaddq 96(%rsi), %xmm6, %xmm8 + vpaddq 112(%rsi), %xmm7, %xmm9 + vmovdqu %xmm8, 96(%rsp) + vmovdqu %xmm9, 112(%rsp) + # rnd_all_2: 0-1 + # rnd_0: 0 - 11 + rorq $23, %rax + movq %r8, %rdx + movq %r13, %rcx + addq (%rsp), %r15 + xorq %r14, %rcx + xorq %r12, %rax + andq %r12, %rcx + rorq $4, %rax + xorq %r14, %rcx + xorq %r12, %rax + addq %rcx, %r15 + rorq $14, %rax + xorq %r9, %rdx + addq %rax, %r15 + movq %r8, %rcx + andq %rdx, %rbx + rorq $5, %rcx + xorq %r8, %rcx + xorq %r9, %rbx + rorq $6, %rcx + addq %r15, %r11 + xorq %r8, %rcx + addq %rbx, %r15 + rorq $28, %rcx + movq %r11, %rax + addq %rcx, %r15 + # rnd_1: 0 - 11 + rorq $23, %rax + movq %r15, %rbx + movq %r12, %rcx + addq 8(%rsp), %r14 + xorq %r13, %rcx + xorq %r11, %rax + andq %r11, %rcx + rorq $4, %rax + xorq %r13, %rcx + xorq %r11, %rax + addq %rcx, %r14 + rorq $14, %rax + xorq %r8, %rbx + addq %rax, %r14 + movq %r15, %rcx + andq %rbx, %rdx + rorq $5, %rcx + xorq %r15, %rcx + xorq %r8, %rdx + rorq $6, %rcx + addq %r14, %r10 + xorq %r15, %rcx + addq %rdx, %r14 + rorq $28, %rcx + movq %r10, %rax + addq %rcx, %r14 + # rnd_all_2: 2-3 + # rnd_0: 0 - 11 + rorq $23, %rax + movq %r14, %rdx + movq %r11, %rcx + addq 16(%rsp), %r13 + xorq %r12, %rcx + xorq %r10, %rax + andq %r10, %rcx + rorq $4, %rax + xorq %r12, %rcx + xorq %r10, %rax + addq %rcx, %r13 + rorq $14, %rax + xorq %r15, %rdx + addq %rax, %r13 + movq %r14, %rcx + andq %rdx, %rbx + rorq $5, %rcx + xorq %r14, %rcx + xorq %r15, %rbx + rorq $6, %rcx + addq %r13, %r9 + xorq %r14, %rcx + addq %rbx, %r13 + rorq $28, %rcx + movq %r9, %rax + addq %rcx, %r13 + # rnd_1: 0 - 11 + rorq $23, %rax + movq %r13, %rbx + movq %r10, %rcx + addq 24(%rsp), %r12 + xorq %r11, %rcx + xorq %r9, %rax + andq %r9, %rcx + rorq $4, %rax + xorq %r11, %rcx + xorq %r9, %rax + addq %rcx, %r12 + rorq $14, %rax + xorq %r14, %rbx + addq %rax, %r12 + movq %r13, %rcx + andq %rbx, %rdx + rorq $5, %rcx + xorq %r13, %rcx + xorq %r14, %rdx + rorq $6, %rcx + addq %r12, %r8 + xorq %r13, %rcx + addq %rdx, %r12 + rorq $28, %rcx + movq %r8, %rax + addq %rcx, %r12 + # rnd_all_2: 4-5 + # rnd_0: 0 - 11 + rorq $23, %rax + movq %r12, %rdx + movq %r9, %rcx + addq 32(%rsp), %r11 + xorq %r10, %rcx + xorq %r8, %rax + andq %r8, %rcx + rorq $4, %rax + xorq %r10, %rcx + xorq %r8, %rax + addq %rcx, %r11 + rorq $14, %rax + xorq %r13, %rdx + addq %rax, %r11 + movq %r12, %rcx + andq %rdx, %rbx + rorq $5, %rcx + xorq %r12, %rcx + xorq %r13, %rbx + rorq $6, %rcx + addq %r11, %r15 + xorq %r12, %rcx + addq %rbx, %r11 + rorq $28, %rcx + movq %r15, %rax + addq %rcx, %r11 + # rnd_1: 0 - 11 + rorq $23, %rax + movq %r11, %rbx + movq %r8, %rcx + addq 40(%rsp), %r10 + xorq %r9, %rcx + xorq %r15, %rax + andq %r15, %rcx + rorq $4, %rax + xorq %r9, %rcx + xorq %r15, %rax + addq %rcx, %r10 + rorq $14, %rax + xorq %r12, %rbx + addq %rax, %r10 + movq %r11, %rcx + andq %rbx, %rdx + rorq $5, %rcx + xorq %r11, %rcx + xorq %r12, %rdx + rorq $6, %rcx + addq %r10, %r14 + xorq %r11, %rcx + addq %rdx, %r10 + rorq $28, %rcx + movq %r14, %rax + addq %rcx, %r10 + # rnd_all_2: 6-7 + # rnd_0: 0 - 11 + rorq $23, %rax + movq %r10, %rdx + movq %r15, %rcx + addq 48(%rsp), %r9 + xorq %r8, %rcx + xorq %r14, %rax + andq %r14, %rcx + rorq $4, %rax + xorq %r8, %rcx + xorq %r14, %rax + addq %rcx, %r9 + rorq $14, %rax + xorq %r11, %rdx + addq %rax, %r9 + movq %r10, %rcx + andq %rdx, %rbx + rorq $5, %rcx + xorq %r10, %rcx + xorq %r11, %rbx + rorq $6, %rcx + addq %r9, %r13 + xorq %r10, %rcx + addq %rbx, %r9 + rorq $28, %rcx + movq %r13, %rax + addq %rcx, %r9 + # rnd_1: 0 - 11 + rorq $23, %rax + movq %r9, %rbx + movq %r14, %rcx + addq 56(%rsp), %r8 + xorq %r15, %rcx + xorq %r13, %rax + andq %r13, %rcx + rorq $4, %rax + xorq %r15, %rcx + xorq %r13, %rax + addq %rcx, %r8 + rorq $14, %rax + xorq %r10, %rbx + addq %rax, %r8 + movq %r9, %rcx + andq %rbx, %rdx + rorq $5, %rcx + xorq %r9, %rcx + xorq %r10, %rdx + rorq $6, %rcx + addq %r8, %r12 + xorq %r9, %rcx + addq %rdx, %r8 + rorq $28, %rcx + movq %r12, %rax + addq %rcx, %r8 + # rnd_all_2: 8-9 + # rnd_0: 0 - 11 + rorq $23, %rax + movq %r8, %rdx + movq %r13, %rcx + addq 64(%rsp), %r15 + xorq %r14, %rcx + xorq %r12, %rax + andq %r12, %rcx + rorq $4, %rax + xorq %r14, %rcx + xorq %r12, %rax + addq %rcx, %r15 + rorq $14, %rax + xorq %r9, %rdx + addq %rax, %r15 + movq %r8, %rcx + andq %rdx, %rbx + rorq $5, %rcx + xorq %r8, %rcx + xorq %r9, %rbx + rorq $6, %rcx + addq %r15, %r11 + xorq %r8, %rcx + addq %rbx, %r15 + rorq $28, %rcx + movq %r11, %rax + addq %rcx, %r15 + # rnd_1: 0 - 11 + rorq $23, %rax + movq %r15, %rbx + movq %r12, %rcx + addq 72(%rsp), %r14 + xorq %r13, %rcx + xorq %r11, %rax + andq %r11, %rcx + rorq $4, %rax + xorq %r13, %rcx + xorq %r11, %rax + addq %rcx, %r14 + rorq $14, %rax + xorq %r8, %rbx + addq %rax, %r14 + movq %r15, %rcx + andq %rbx, %rdx + rorq $5, %rcx + xorq %r15, %rcx + xorq %r8, %rdx + rorq $6, %rcx + addq %r14, %r10 + xorq %r15, %rcx + addq %rdx, %r14 + rorq $28, %rcx + movq %r10, %rax + addq %rcx, %r14 + # rnd_all_2: 10-11 + # rnd_0: 0 - 11 + rorq $23, %rax + movq %r14, %rdx + movq %r11, %rcx + addq 80(%rsp), %r13 + xorq %r12, %rcx + xorq %r10, %rax + andq %r10, %rcx + rorq $4, %rax + xorq %r12, %rcx + xorq %r10, %rax + addq %rcx, %r13 + rorq $14, %rax + xorq %r15, %rdx + addq %rax, %r13 + movq %r14, %rcx + andq %rdx, %rbx + rorq $5, %rcx + xorq %r14, %rcx + xorq %r15, %rbx + rorq $6, %rcx + addq %r13, %r9 + xorq %r14, %rcx + addq %rbx, %r13 + rorq $28, %rcx + movq %r9, %rax + addq %rcx, %r13 + # rnd_1: 0 - 11 + rorq $23, %rax + movq %r13, %rbx + movq %r10, %rcx + addq 88(%rsp), %r12 + xorq %r11, %rcx + xorq %r9, %rax + andq %r9, %rcx + rorq $4, %rax + xorq %r11, %rcx + xorq %r9, %rax + addq %rcx, %r12 + rorq $14, %rax + xorq %r14, %rbx + addq %rax, %r12 + movq %r13, %rcx + andq %rbx, %rdx + rorq $5, %rcx + xorq %r13, %rcx + xorq %r14, %rdx + rorq $6, %rcx + addq %r12, %r8 + xorq %r13, %rcx + addq %rdx, %r12 + rorq $28, %rcx + movq %r8, %rax + addq %rcx, %r12 + # rnd_all_2: 12-13 + # rnd_0: 0 - 11 + rorq $23, %rax + movq %r12, %rdx + movq %r9, %rcx + addq 96(%rsp), %r11 + xorq %r10, %rcx + xorq %r8, %rax + andq %r8, %rcx + rorq $4, %rax + xorq %r10, %rcx + xorq %r8, %rax + addq %rcx, %r11 + rorq $14, %rax + xorq %r13, %rdx + addq %rax, %r11 + movq %r12, %rcx + andq %rdx, %rbx + rorq $5, %rcx + xorq %r12, %rcx + xorq %r13, %rbx + rorq $6, %rcx + addq %r11, %r15 + xorq %r12, %rcx + addq %rbx, %r11 + rorq $28, %rcx + movq %r15, %rax + addq %rcx, %r11 + # rnd_1: 0 - 11 + rorq $23, %rax + movq %r11, %rbx + movq %r8, %rcx + addq 104(%rsp), %r10 + xorq %r9, %rcx + xorq %r15, %rax + andq %r15, %rcx + rorq $4, %rax + xorq %r9, %rcx + xorq %r15, %rax + addq %rcx, %r10 + rorq $14, %rax + xorq %r12, %rbx + addq %rax, %r10 + movq %r11, %rcx + andq %rbx, %rdx + rorq $5, %rcx + xorq %r11, %rcx + xorq %r12, %rdx + rorq $6, %rcx + addq %r10, %r14 + xorq %r11, %rcx + addq %rdx, %r10 + rorq $28, %rcx + movq %r14, %rax + addq %rcx, %r10 + # rnd_all_2: 14-15 + # rnd_0: 0 - 11 + rorq $23, %rax + movq %r10, %rdx + movq %r15, %rcx + addq 112(%rsp), %r9 + xorq %r8, %rcx + xorq %r14, %rax + andq %r14, %rcx + rorq $4, %rax + xorq %r8, %rcx + xorq %r14, %rax + addq %rcx, %r9 + rorq $14, %rax + xorq %r11, %rdx + addq %rax, %r9 + movq %r10, %rcx + andq %rdx, %rbx + rorq $5, %rcx + xorq %r10, %rcx + xorq %r11, %rbx + rorq $6, %rcx + addq %r9, %r13 + xorq %r10, %rcx + addq %rbx, %r9 + rorq $28, %rcx + movq %r13, %rax + addq %rcx, %r9 + # rnd_1: 0 - 11 + rorq $23, %rax + movq %r9, %rbx + movq %r14, %rcx + addq 120(%rsp), %r8 + xorq %r15, %rcx + xorq %r13, %rax + andq %r13, %rcx + rorq $4, %rax + xorq %r15, %rcx + xorq %r13, %rax + addq %rcx, %r8 + rorq $14, %rax + xorq %r10, %rbx + addq %rax, %r8 + movq %r9, %rcx + andq %rbx, %rdx + rorq $5, %rcx + xorq %r9, %rcx + xorq %r10, %rdx + rorq $6, %rcx + addq %r8, %r12 + xorq %r9, %rcx + addq %rdx, %r8 + rorq $28, %rcx + movq %r12, %rax + addq %rcx, %r8 + addq %r8, (%rdi) + addq %r9, 8(%rdi) + addq %r10, 16(%rdi) + addq %r11, 24(%rdi) + addq %r12, 32(%rdi) + addq %r13, 40(%rdi) + addq %r14, 48(%rdi) + addq %r15, 56(%rdi) + xorq %rax, %rax + vzeroupper + addq $0x88, %rsp + popq %r15 + popq %r14 + popq %r13 + popq %r12 + popq %rbx + repz retq +.size Transform_Sha512_AVX1,.-Transform_Sha512_AVX1 +.globl Transform_Sha512_AVX1_Len +.type Transform_Sha512_AVX1_Len,@function +.align 4 +Transform_Sha512_AVX1_Len: + pushq %rbx + pushq %r12 + pushq %r13 + pushq %r14 + pushq %r15 + pushq %rbp + movq %rsi, %rbp + subq $0x90, %rsp + movq 224(%rdi), %rsi + leaq L_avx1_sha512_k(%rip), %rdx + vmovdqa L_avx1_sha512_flip_mask(%rip), %xmm14 + movq (%rdi), %r8 + movq 8(%rdi), %r9 + movq 16(%rdi), %r10 + movq 24(%rdi), %r11 + movq 32(%rdi), %r12 + movq 40(%rdi), %r13 + movq 48(%rdi), %r14 + movq 56(%rdi), %r15 + # Start of loop processing a block +L_sha512_len_avx1_begin: + vmovdqu (%rsi), %xmm0 + vmovdqu 16(%rsi), %xmm1 + vpshufb %xmm14, %xmm0, %xmm0 + vpshufb %xmm14, %xmm1, %xmm1 + vmovdqu 32(%rsi), %xmm2 + vmovdqu 48(%rsi), %xmm3 + vpshufb %xmm14, %xmm2, %xmm2 + vpshufb %xmm14, %xmm3, %xmm3 + vmovdqu 64(%rsi), %xmm4 + vmovdqu 80(%rsi), %xmm5 + vpshufb %xmm14, %xmm4, %xmm4 + vpshufb %xmm14, %xmm5, %xmm5 + vmovdqu 96(%rsi), %xmm6 + vmovdqu 112(%rsi), %xmm7 + vpshufb %xmm14, %xmm6, %xmm6 + vpshufb %xmm14, %xmm7, %xmm7 + movl $4, 128(%rsp) + movq %r9, %rbx + movq %r12, %rax + xorq %r10, %rbx + vpaddq (%rdx), %xmm0, %xmm8 + vpaddq 16(%rdx), %xmm1, %xmm9 + vmovdqu %xmm8, (%rsp) + vmovdqu %xmm9, 16(%rsp) + vpaddq 32(%rdx), %xmm2, %xmm8 + vpaddq 48(%rdx), %xmm3, %xmm9 + vmovdqu %xmm8, 32(%rsp) + vmovdqu %xmm9, 48(%rsp) + vpaddq 64(%rdx), %xmm4, %xmm8 + vpaddq 80(%rdx), %xmm5, %xmm9 + vmovdqu %xmm8, 64(%rsp) + vmovdqu %xmm9, 80(%rsp) + vpaddq 96(%rdx), %xmm6, %xmm8 + vpaddq 112(%rdx), %xmm7, %xmm9 + vmovdqu %xmm8, 96(%rsp) + vmovdqu %xmm9, 112(%rsp) + # Start of 16 rounds +L_sha512_len_avx1_start: + addq $0x80, %rdx + movq %rdx, 136(%rsp) + # msg_sched: 0-1 + # rnd_0: 0 - 0 + rorq $23, %rax + vpalignr $8, %xmm0, %xmm1, %xmm12 + vpalignr $8, %xmm4, %xmm5, %xmm13 + # rnd_0: 1 - 1 + movq %r8, %rdx + movq %r13, %rcx + addq (%rsp), %r15 + xorq %r14, %rcx + vpsrlq $0x01, %xmm12, %xmm8 + vpsllq $63, %xmm12, %xmm9 + # rnd_0: 2 - 3 + xorq %r12, %rax + andq %r12, %rcx + rorq $4, %rax + xorq %r14, %rcx + vpsrlq $8, %xmm12, %xmm10 + vpsllq $56, %xmm12, %xmm11 + # rnd_0: 4 - 5 + xorq %r12, %rax + addq %rcx, %r15 + rorq $14, %rax + xorq %r9, %rdx + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_0: 6 - 7 + addq %rax, %r15 + movq %r8, %rcx + andq %rdx, %rbx + rorq $5, %rcx + vpsrlq $7, %xmm12, %xmm11 + vpxor %xmm10, %xmm8, %xmm8 + # rnd_0: 8 - 9 + xorq %r8, %rcx + xorq %r9, %rbx + rorq $6, %rcx + addq %r15, %r11 + vpxor %xmm11, %xmm8, %xmm8 + vpaddq %xmm0, %xmm13, %xmm0 + # rnd_0: 10 - 11 + xorq %r8, %rcx + addq %rbx, %r15 + rorq $28, %rcx + movq %r11, %rax + addq %rcx, %r15 + # rnd_1: 0 - 0 + rorq $23, %rax + vpaddq %xmm0, %xmm8, %xmm0 + # rnd_1: 1 - 1 + movq %r15, %rbx + movq %r12, %rcx + addq 8(%rsp), %r14 + xorq %r13, %rcx + vpsrlq $19, %xmm7, %xmm8 + vpsllq $45, %xmm7, %xmm9 + # rnd_1: 2 - 3 + xorq %r11, %rax + andq %r11, %rcx + rorq $4, %rax + xorq %r13, %rcx + vpsrlq $61, %xmm7, %xmm10 + vpsllq $3, %xmm7, %xmm11 + # rnd_1: 4 - 6 + xorq %r11, %rax + addq %rcx, %r14 + rorq $14, %rax + xorq %r8, %rbx + addq %rax, %r14 + movq %r15, %rcx + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_1: 7 - 8 + andq %rbx, %rdx + rorq $5, %rcx + xorq %r15, %rcx + xorq %r8, %rdx + vpxor %xmm10, %xmm8, %xmm8 + vpsrlq $6, %xmm7, %xmm11 + # rnd_1: 9 - 10 + rorq $6, %rcx + addq %r14, %r10 + xorq %r15, %rcx + addq %rdx, %r14 + vpxor %xmm11, %xmm8, %xmm8 + # rnd_1: 11 - 11 + rorq $28, %rcx + movq %r10, %rax + addq %rcx, %r14 + vpaddq %xmm0, %xmm8, %xmm0 + # msg_sched done: 0-3 + # msg_sched: 2-3 + # rnd_0: 0 - 0 + rorq $23, %rax + vpalignr $8, %xmm1, %xmm2, %xmm12 + vpalignr $8, %xmm5, %xmm6, %xmm13 + # rnd_0: 1 - 1 + movq %r14, %rdx + movq %r11, %rcx + addq 16(%rsp), %r13 + xorq %r12, %rcx + vpsrlq $0x01, %xmm12, %xmm8 + vpsllq $63, %xmm12, %xmm9 + # rnd_0: 2 - 3 + xorq %r10, %rax + andq %r10, %rcx + rorq $4, %rax + xorq %r12, %rcx + vpsrlq $8, %xmm12, %xmm10 + vpsllq $56, %xmm12, %xmm11 + # rnd_0: 4 - 5 + xorq %r10, %rax + addq %rcx, %r13 + rorq $14, %rax + xorq %r15, %rdx + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_0: 6 - 7 + addq %rax, %r13 + movq %r14, %rcx + andq %rdx, %rbx + rorq $5, %rcx + vpsrlq $7, %xmm12, %xmm11 + vpxor %xmm10, %xmm8, %xmm8 + # rnd_0: 8 - 9 + xorq %r14, %rcx + xorq %r15, %rbx + rorq $6, %rcx + addq %r13, %r9 + vpxor %xmm11, %xmm8, %xmm8 + vpaddq %xmm1, %xmm13, %xmm1 + # rnd_0: 10 - 11 + xorq %r14, %rcx + addq %rbx, %r13 + rorq $28, %rcx + movq %r9, %rax + addq %rcx, %r13 + # rnd_1: 0 - 0 + rorq $23, %rax + vpaddq %xmm1, %xmm8, %xmm1 + # rnd_1: 1 - 1 + movq %r13, %rbx + movq %r10, %rcx + addq 24(%rsp), %r12 + xorq %r11, %rcx + vpsrlq $19, %xmm0, %xmm8 + vpsllq $45, %xmm0, %xmm9 + # rnd_1: 2 - 3 + xorq %r9, %rax + andq %r9, %rcx + rorq $4, %rax + xorq %r11, %rcx + vpsrlq $61, %xmm0, %xmm10 + vpsllq $3, %xmm0, %xmm11 + # rnd_1: 4 - 6 + xorq %r9, %rax + addq %rcx, %r12 + rorq $14, %rax + xorq %r14, %rbx + addq %rax, %r12 + movq %r13, %rcx + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_1: 7 - 8 + andq %rbx, %rdx + rorq $5, %rcx + xorq %r13, %rcx + xorq %r14, %rdx + vpxor %xmm10, %xmm8, %xmm8 + vpsrlq $6, %xmm0, %xmm11 + # rnd_1: 9 - 10 + rorq $6, %rcx + addq %r12, %r8 + xorq %r13, %rcx + addq %rdx, %r12 + vpxor %xmm11, %xmm8, %xmm8 + # rnd_1: 11 - 11 + rorq $28, %rcx + movq %r8, %rax + addq %rcx, %r12 + vpaddq %xmm1, %xmm8, %xmm1 + # msg_sched done: 2-5 + # msg_sched: 4-5 + # rnd_0: 0 - 0 + rorq $23, %rax + vpalignr $8, %xmm2, %xmm3, %xmm12 + vpalignr $8, %xmm6, %xmm7, %xmm13 + # rnd_0: 1 - 1 + movq %r12, %rdx + movq %r9, %rcx + addq 32(%rsp), %r11 + xorq %r10, %rcx + vpsrlq $0x01, %xmm12, %xmm8 + vpsllq $63, %xmm12, %xmm9 + # rnd_0: 2 - 3 + xorq %r8, %rax + andq %r8, %rcx + rorq $4, %rax + xorq %r10, %rcx + vpsrlq $8, %xmm12, %xmm10 + vpsllq $56, %xmm12, %xmm11 + # rnd_0: 4 - 5 + xorq %r8, %rax + addq %rcx, %r11 + rorq $14, %rax + xorq %r13, %rdx + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_0: 6 - 7 + addq %rax, %r11 + movq %r12, %rcx + andq %rdx, %rbx + rorq $5, %rcx + vpsrlq $7, %xmm12, %xmm11 + vpxor %xmm10, %xmm8, %xmm8 + # rnd_0: 8 - 9 + xorq %r12, %rcx + xorq %r13, %rbx + rorq $6, %rcx + addq %r11, %r15 + vpxor %xmm11, %xmm8, %xmm8 + vpaddq %xmm2, %xmm13, %xmm2 + # rnd_0: 10 - 11 + xorq %r12, %rcx + addq %rbx, %r11 + rorq $28, %rcx + movq %r15, %rax + addq %rcx, %r11 + # rnd_1: 0 - 0 + rorq $23, %rax + vpaddq %xmm2, %xmm8, %xmm2 + # rnd_1: 1 - 1 + movq %r11, %rbx + movq %r8, %rcx + addq 40(%rsp), %r10 + xorq %r9, %rcx + vpsrlq $19, %xmm1, %xmm8 + vpsllq $45, %xmm1, %xmm9 + # rnd_1: 2 - 3 + xorq %r15, %rax + andq %r15, %rcx + rorq $4, %rax + xorq %r9, %rcx + vpsrlq $61, %xmm1, %xmm10 + vpsllq $3, %xmm1, %xmm11 + # rnd_1: 4 - 6 + xorq %r15, %rax + addq %rcx, %r10 + rorq $14, %rax + xorq %r12, %rbx + addq %rax, %r10 + movq %r11, %rcx + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_1: 7 - 8 + andq %rbx, %rdx + rorq $5, %rcx + xorq %r11, %rcx + xorq %r12, %rdx + vpxor %xmm10, %xmm8, %xmm8 + vpsrlq $6, %xmm1, %xmm11 + # rnd_1: 9 - 10 + rorq $6, %rcx + addq %r10, %r14 + xorq %r11, %rcx + addq %rdx, %r10 + vpxor %xmm11, %xmm8, %xmm8 + # rnd_1: 11 - 11 + rorq $28, %rcx + movq %r14, %rax + addq %rcx, %r10 + vpaddq %xmm2, %xmm8, %xmm2 + # msg_sched done: 4-7 + # msg_sched: 6-7 + # rnd_0: 0 - 0 + rorq $23, %rax + vpalignr $8, %xmm3, %xmm4, %xmm12 + vpalignr $8, %xmm7, %xmm0, %xmm13 + # rnd_0: 1 - 1 + movq %r10, %rdx + movq %r15, %rcx + addq 48(%rsp), %r9 + xorq %r8, %rcx + vpsrlq $0x01, %xmm12, %xmm8 + vpsllq $63, %xmm12, %xmm9 + # rnd_0: 2 - 3 + xorq %r14, %rax + andq %r14, %rcx + rorq $4, %rax + xorq %r8, %rcx + vpsrlq $8, %xmm12, %xmm10 + vpsllq $56, %xmm12, %xmm11 + # rnd_0: 4 - 5 + xorq %r14, %rax + addq %rcx, %r9 + rorq $14, %rax + xorq %r11, %rdx + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_0: 6 - 7 + addq %rax, %r9 + movq %r10, %rcx + andq %rdx, %rbx + rorq $5, %rcx + vpsrlq $7, %xmm12, %xmm11 + vpxor %xmm10, %xmm8, %xmm8 + # rnd_0: 8 - 9 + xorq %r10, %rcx + xorq %r11, %rbx + rorq $6, %rcx + addq %r9, %r13 + vpxor %xmm11, %xmm8, %xmm8 + vpaddq %xmm3, %xmm13, %xmm3 + # rnd_0: 10 - 11 + xorq %r10, %rcx + addq %rbx, %r9 + rorq $28, %rcx + movq %r13, %rax + addq %rcx, %r9 + # rnd_1: 0 - 0 + rorq $23, %rax + vpaddq %xmm3, %xmm8, %xmm3 + # rnd_1: 1 - 1 + movq %r9, %rbx + movq %r14, %rcx + addq 56(%rsp), %r8 + xorq %r15, %rcx + vpsrlq $19, %xmm2, %xmm8 + vpsllq $45, %xmm2, %xmm9 + # rnd_1: 2 - 3 + xorq %r13, %rax + andq %r13, %rcx + rorq $4, %rax + xorq %r15, %rcx + vpsrlq $61, %xmm2, %xmm10 + vpsllq $3, %xmm2, %xmm11 + # rnd_1: 4 - 6 + xorq %r13, %rax + addq %rcx, %r8 + rorq $14, %rax + xorq %r10, %rbx + addq %rax, %r8 + movq %r9, %rcx + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_1: 7 - 8 + andq %rbx, %rdx + rorq $5, %rcx + xorq %r9, %rcx + xorq %r10, %rdx + vpxor %xmm10, %xmm8, %xmm8 + vpsrlq $6, %xmm2, %xmm11 + # rnd_1: 9 - 10 + rorq $6, %rcx + addq %r8, %r12 + xorq %r9, %rcx + addq %rdx, %r8 + vpxor %xmm11, %xmm8, %xmm8 + # rnd_1: 11 - 11 + rorq $28, %rcx + movq %r12, %rax + addq %rcx, %r8 + vpaddq %xmm3, %xmm8, %xmm3 + # msg_sched done: 6-9 + # msg_sched: 8-9 + # rnd_0: 0 - 0 + rorq $23, %rax + vpalignr $8, %xmm4, %xmm5, %xmm12 + vpalignr $8, %xmm0, %xmm1, %xmm13 + # rnd_0: 1 - 1 + movq %r8, %rdx + movq %r13, %rcx + addq 64(%rsp), %r15 + xorq %r14, %rcx + vpsrlq $0x01, %xmm12, %xmm8 + vpsllq $63, %xmm12, %xmm9 + # rnd_0: 2 - 3 + xorq %r12, %rax + andq %r12, %rcx + rorq $4, %rax + xorq %r14, %rcx + vpsrlq $8, %xmm12, %xmm10 + vpsllq $56, %xmm12, %xmm11 + # rnd_0: 4 - 5 + xorq %r12, %rax + addq %rcx, %r15 + rorq $14, %rax + xorq %r9, %rdx + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_0: 6 - 7 + addq %rax, %r15 + movq %r8, %rcx + andq %rdx, %rbx + rorq $5, %rcx + vpsrlq $7, %xmm12, %xmm11 + vpxor %xmm10, %xmm8, %xmm8 + # rnd_0: 8 - 9 + xorq %r8, %rcx + xorq %r9, %rbx + rorq $6, %rcx + addq %r15, %r11 + vpxor %xmm11, %xmm8, %xmm8 + vpaddq %xmm4, %xmm13, %xmm4 + # rnd_0: 10 - 11 + xorq %r8, %rcx + addq %rbx, %r15 + rorq $28, %rcx + movq %r11, %rax + addq %rcx, %r15 + # rnd_1: 0 - 0 + rorq $23, %rax + vpaddq %xmm4, %xmm8, %xmm4 + # rnd_1: 1 - 1 + movq %r15, %rbx + movq %r12, %rcx + addq 72(%rsp), %r14 + xorq %r13, %rcx + vpsrlq $19, %xmm3, %xmm8 + vpsllq $45, %xmm3, %xmm9 + # rnd_1: 2 - 3 + xorq %r11, %rax + andq %r11, %rcx + rorq $4, %rax + xorq %r13, %rcx + vpsrlq $61, %xmm3, %xmm10 + vpsllq $3, %xmm3, %xmm11 + # rnd_1: 4 - 6 + xorq %r11, %rax + addq %rcx, %r14 + rorq $14, %rax + xorq %r8, %rbx + addq %rax, %r14 + movq %r15, %rcx + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_1: 7 - 8 + andq %rbx, %rdx + rorq $5, %rcx + xorq %r15, %rcx + xorq %r8, %rdx + vpxor %xmm10, %xmm8, %xmm8 + vpsrlq $6, %xmm3, %xmm11 + # rnd_1: 9 - 10 + rorq $6, %rcx + addq %r14, %r10 + xorq %r15, %rcx + addq %rdx, %r14 + vpxor %xmm11, %xmm8, %xmm8 + # rnd_1: 11 - 11 + rorq $28, %rcx + movq %r10, %rax + addq %rcx, %r14 + vpaddq %xmm4, %xmm8, %xmm4 + # msg_sched done: 8-11 + # msg_sched: 10-11 + # rnd_0: 0 - 0 + rorq $23, %rax + vpalignr $8, %xmm5, %xmm6, %xmm12 + vpalignr $8, %xmm1, %xmm2, %xmm13 + # rnd_0: 1 - 1 + movq %r14, %rdx + movq %r11, %rcx + addq 80(%rsp), %r13 + xorq %r12, %rcx + vpsrlq $0x01, %xmm12, %xmm8 + vpsllq $63, %xmm12, %xmm9 + # rnd_0: 2 - 3 + xorq %r10, %rax + andq %r10, %rcx + rorq $4, %rax + xorq %r12, %rcx + vpsrlq $8, %xmm12, %xmm10 + vpsllq $56, %xmm12, %xmm11 + # rnd_0: 4 - 5 + xorq %r10, %rax + addq %rcx, %r13 + rorq $14, %rax + xorq %r15, %rdx + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_0: 6 - 7 + addq %rax, %r13 + movq %r14, %rcx + andq %rdx, %rbx + rorq $5, %rcx + vpsrlq $7, %xmm12, %xmm11 + vpxor %xmm10, %xmm8, %xmm8 + # rnd_0: 8 - 9 + xorq %r14, %rcx + xorq %r15, %rbx + rorq $6, %rcx + addq %r13, %r9 + vpxor %xmm11, %xmm8, %xmm8 + vpaddq %xmm5, %xmm13, %xmm5 + # rnd_0: 10 - 11 + xorq %r14, %rcx + addq %rbx, %r13 + rorq $28, %rcx + movq %r9, %rax + addq %rcx, %r13 + # rnd_1: 0 - 0 + rorq $23, %rax + vpaddq %xmm5, %xmm8, %xmm5 + # rnd_1: 1 - 1 + movq %r13, %rbx + movq %r10, %rcx + addq 88(%rsp), %r12 + xorq %r11, %rcx + vpsrlq $19, %xmm4, %xmm8 + vpsllq $45, %xmm4, %xmm9 + # rnd_1: 2 - 3 + xorq %r9, %rax + andq %r9, %rcx + rorq $4, %rax + xorq %r11, %rcx + vpsrlq $61, %xmm4, %xmm10 + vpsllq $3, %xmm4, %xmm11 + # rnd_1: 4 - 6 + xorq %r9, %rax + addq %rcx, %r12 + rorq $14, %rax + xorq %r14, %rbx + addq %rax, %r12 + movq %r13, %rcx + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_1: 7 - 8 + andq %rbx, %rdx + rorq $5, %rcx + xorq %r13, %rcx + xorq %r14, %rdx + vpxor %xmm10, %xmm8, %xmm8 + vpsrlq $6, %xmm4, %xmm11 + # rnd_1: 9 - 10 + rorq $6, %rcx + addq %r12, %r8 + xorq %r13, %rcx + addq %rdx, %r12 + vpxor %xmm11, %xmm8, %xmm8 + # rnd_1: 11 - 11 + rorq $28, %rcx + movq %r8, %rax + addq %rcx, %r12 + vpaddq %xmm5, %xmm8, %xmm5 + # msg_sched done: 10-13 + # msg_sched: 12-13 + # rnd_0: 0 - 0 + rorq $23, %rax + vpalignr $8, %xmm6, %xmm7, %xmm12 + vpalignr $8, %xmm2, %xmm3, %xmm13 + # rnd_0: 1 - 1 + movq %r12, %rdx + movq %r9, %rcx + addq 96(%rsp), %r11 + xorq %r10, %rcx + vpsrlq $0x01, %xmm12, %xmm8 + vpsllq $63, %xmm12, %xmm9 + # rnd_0: 2 - 3 + xorq %r8, %rax + andq %r8, %rcx + rorq $4, %rax + xorq %r10, %rcx + vpsrlq $8, %xmm12, %xmm10 + vpsllq $56, %xmm12, %xmm11 + # rnd_0: 4 - 5 + xorq %r8, %rax + addq %rcx, %r11 + rorq $14, %rax + xorq %r13, %rdx + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_0: 6 - 7 + addq %rax, %r11 + movq %r12, %rcx + andq %rdx, %rbx + rorq $5, %rcx + vpsrlq $7, %xmm12, %xmm11 + vpxor %xmm10, %xmm8, %xmm8 + # rnd_0: 8 - 9 + xorq %r12, %rcx + xorq %r13, %rbx + rorq $6, %rcx + addq %r11, %r15 + vpxor %xmm11, %xmm8, %xmm8 + vpaddq %xmm6, %xmm13, %xmm6 + # rnd_0: 10 - 11 + xorq %r12, %rcx + addq %rbx, %r11 + rorq $28, %rcx + movq %r15, %rax + addq %rcx, %r11 + # rnd_1: 0 - 0 + rorq $23, %rax + vpaddq %xmm6, %xmm8, %xmm6 + # rnd_1: 1 - 1 + movq %r11, %rbx + movq %r8, %rcx + addq 104(%rsp), %r10 + xorq %r9, %rcx + vpsrlq $19, %xmm5, %xmm8 + vpsllq $45, %xmm5, %xmm9 + # rnd_1: 2 - 3 + xorq %r15, %rax + andq %r15, %rcx + rorq $4, %rax + xorq %r9, %rcx + vpsrlq $61, %xmm5, %xmm10 + vpsllq $3, %xmm5, %xmm11 + # rnd_1: 4 - 6 + xorq %r15, %rax + addq %rcx, %r10 + rorq $14, %rax + xorq %r12, %rbx + addq %rax, %r10 + movq %r11, %rcx + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_1: 7 - 8 + andq %rbx, %rdx + rorq $5, %rcx + xorq %r11, %rcx + xorq %r12, %rdx + vpxor %xmm10, %xmm8, %xmm8 + vpsrlq $6, %xmm5, %xmm11 + # rnd_1: 9 - 10 + rorq $6, %rcx + addq %r10, %r14 + xorq %r11, %rcx + addq %rdx, %r10 + vpxor %xmm11, %xmm8, %xmm8 + # rnd_1: 11 - 11 + rorq $28, %rcx + movq %r14, %rax + addq %rcx, %r10 + vpaddq %xmm6, %xmm8, %xmm6 + # msg_sched done: 12-15 + # msg_sched: 14-15 + # rnd_0: 0 - 0 + rorq $23, %rax + vpalignr $8, %xmm7, %xmm0, %xmm12 + vpalignr $8, %xmm3, %xmm4, %xmm13 + # rnd_0: 1 - 1 + movq %r10, %rdx + movq %r15, %rcx + addq 112(%rsp), %r9 + xorq %r8, %rcx + vpsrlq $0x01, %xmm12, %xmm8 + vpsllq $63, %xmm12, %xmm9 + # rnd_0: 2 - 3 + xorq %r14, %rax + andq %r14, %rcx + rorq $4, %rax + xorq %r8, %rcx + vpsrlq $8, %xmm12, %xmm10 + vpsllq $56, %xmm12, %xmm11 + # rnd_0: 4 - 5 + xorq %r14, %rax + addq %rcx, %r9 + rorq $14, %rax + xorq %r11, %rdx + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_0: 6 - 7 + addq %rax, %r9 + movq %r10, %rcx + andq %rdx, %rbx + rorq $5, %rcx + vpsrlq $7, %xmm12, %xmm11 + vpxor %xmm10, %xmm8, %xmm8 + # rnd_0: 8 - 9 + xorq %r10, %rcx + xorq %r11, %rbx + rorq $6, %rcx + addq %r9, %r13 + vpxor %xmm11, %xmm8, %xmm8 + vpaddq %xmm7, %xmm13, %xmm7 + # rnd_0: 10 - 11 + xorq %r10, %rcx + addq %rbx, %r9 + rorq $28, %rcx + movq %r13, %rax + addq %rcx, %r9 + # rnd_1: 0 - 0 + rorq $23, %rax + vpaddq %xmm7, %xmm8, %xmm7 + # rnd_1: 1 - 1 + movq %r9, %rbx + movq %r14, %rcx + addq 120(%rsp), %r8 + xorq %r15, %rcx + vpsrlq $19, %xmm6, %xmm8 + vpsllq $45, %xmm6, %xmm9 + # rnd_1: 2 - 3 + xorq %r13, %rax + andq %r13, %rcx + rorq $4, %rax + xorq %r15, %rcx + vpsrlq $61, %xmm6, %xmm10 + vpsllq $3, %xmm6, %xmm11 + # rnd_1: 4 - 6 + xorq %r13, %rax + addq %rcx, %r8 + rorq $14, %rax + xorq %r10, %rbx + addq %rax, %r8 + movq %r9, %rcx + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_1: 7 - 8 + andq %rbx, %rdx + rorq $5, %rcx + xorq %r9, %rcx + xorq %r10, %rdx + vpxor %xmm10, %xmm8, %xmm8 + vpsrlq $6, %xmm6, %xmm11 + # rnd_1: 9 - 10 + rorq $6, %rcx + addq %r8, %r12 + xorq %r9, %rcx + addq %rdx, %r8 + vpxor %xmm11, %xmm8, %xmm8 + # rnd_1: 11 - 11 + rorq $28, %rcx + movq %r12, %rax + addq %rcx, %r8 + vpaddq %xmm7, %xmm8, %xmm7 + # msg_sched done: 14-17 + movq 136(%rsp), %rdx + vpaddq (%rdx), %xmm0, %xmm8 + vpaddq 16(%rdx), %xmm1, %xmm9 + vmovdqu %xmm8, (%rsp) + vmovdqu %xmm9, 16(%rsp) + vpaddq 32(%rdx), %xmm2, %xmm8 + vpaddq 48(%rdx), %xmm3, %xmm9 + vmovdqu %xmm8, 32(%rsp) + vmovdqu %xmm9, 48(%rsp) + vpaddq 64(%rdx), %xmm4, %xmm8 + vpaddq 80(%rdx), %xmm5, %xmm9 + vmovdqu %xmm8, 64(%rsp) + vmovdqu %xmm9, 80(%rsp) + vpaddq 96(%rdx), %xmm6, %xmm8 + vpaddq 112(%rdx), %xmm7, %xmm9 + vmovdqu %xmm8, 96(%rsp) + vmovdqu %xmm9, 112(%rsp) + subl $0x01, 128(%rsp) + jne L_sha512_len_avx1_start + # rnd_all_2: 0-1 + # rnd_0: 0 - 11 + rorq $23, %rax + movq %r8, %rdx + movq %r13, %rcx + addq (%rsp), %r15 + xorq %r14, %rcx + xorq %r12, %rax + andq %r12, %rcx + rorq $4, %rax + xorq %r14, %rcx + xorq %r12, %rax + addq %rcx, %r15 + rorq $14, %rax + xorq %r9, %rdx + addq %rax, %r15 + movq %r8, %rcx + andq %rdx, %rbx + rorq $5, %rcx + xorq %r8, %rcx + xorq %r9, %rbx + rorq $6, %rcx + addq %r15, %r11 + xorq %r8, %rcx + addq %rbx, %r15 + rorq $28, %rcx + movq %r11, %rax + addq %rcx, %r15 + # rnd_1: 0 - 11 + rorq $23, %rax + movq %r15, %rbx + movq %r12, %rcx + addq 8(%rsp), %r14 + xorq %r13, %rcx + xorq %r11, %rax + andq %r11, %rcx + rorq $4, %rax + xorq %r13, %rcx + xorq %r11, %rax + addq %rcx, %r14 + rorq $14, %rax + xorq %r8, %rbx + addq %rax, %r14 + movq %r15, %rcx + andq %rbx, %rdx + rorq $5, %rcx + xorq %r15, %rcx + xorq %r8, %rdx + rorq $6, %rcx + addq %r14, %r10 + xorq %r15, %rcx + addq %rdx, %r14 + rorq $28, %rcx + movq %r10, %rax + addq %rcx, %r14 + # rnd_all_2: 2-3 + # rnd_0: 0 - 11 + rorq $23, %rax + movq %r14, %rdx + movq %r11, %rcx + addq 16(%rsp), %r13 + xorq %r12, %rcx + xorq %r10, %rax + andq %r10, %rcx + rorq $4, %rax + xorq %r12, %rcx + xorq %r10, %rax + addq %rcx, %r13 + rorq $14, %rax + xorq %r15, %rdx + addq %rax, %r13 + movq %r14, %rcx + andq %rdx, %rbx + rorq $5, %rcx + xorq %r14, %rcx + xorq %r15, %rbx + rorq $6, %rcx + addq %r13, %r9 + xorq %r14, %rcx + addq %rbx, %r13 + rorq $28, %rcx + movq %r9, %rax + addq %rcx, %r13 + # rnd_1: 0 - 11 + rorq $23, %rax + movq %r13, %rbx + movq %r10, %rcx + addq 24(%rsp), %r12 + xorq %r11, %rcx + xorq %r9, %rax + andq %r9, %rcx + rorq $4, %rax + xorq %r11, %rcx + xorq %r9, %rax + addq %rcx, %r12 + rorq $14, %rax + xorq %r14, %rbx + addq %rax, %r12 + movq %r13, %rcx + andq %rbx, %rdx + rorq $5, %rcx + xorq %r13, %rcx + xorq %r14, %rdx + rorq $6, %rcx + addq %r12, %r8 + xorq %r13, %rcx + addq %rdx, %r12 + rorq $28, %rcx + movq %r8, %rax + addq %rcx, %r12 + # rnd_all_2: 4-5 + # rnd_0: 0 - 11 + rorq $23, %rax + movq %r12, %rdx + movq %r9, %rcx + addq 32(%rsp), %r11 + xorq %r10, %rcx + xorq %r8, %rax + andq %r8, %rcx + rorq $4, %rax + xorq %r10, %rcx + xorq %r8, %rax + addq %rcx, %r11 + rorq $14, %rax + xorq %r13, %rdx + addq %rax, %r11 + movq %r12, %rcx + andq %rdx, %rbx + rorq $5, %rcx + xorq %r12, %rcx + xorq %r13, %rbx + rorq $6, %rcx + addq %r11, %r15 + xorq %r12, %rcx + addq %rbx, %r11 + rorq $28, %rcx + movq %r15, %rax + addq %rcx, %r11 + # rnd_1: 0 - 11 + rorq $23, %rax + movq %r11, %rbx + movq %r8, %rcx + addq 40(%rsp), %r10 + xorq %r9, %rcx + xorq %r15, %rax + andq %r15, %rcx + rorq $4, %rax + xorq %r9, %rcx + xorq %r15, %rax + addq %rcx, %r10 + rorq $14, %rax + xorq %r12, %rbx + addq %rax, %r10 + movq %r11, %rcx + andq %rbx, %rdx + rorq $5, %rcx + xorq %r11, %rcx + xorq %r12, %rdx + rorq $6, %rcx + addq %r10, %r14 + xorq %r11, %rcx + addq %rdx, %r10 + rorq $28, %rcx + movq %r14, %rax + addq %rcx, %r10 + # rnd_all_2: 6-7 + # rnd_0: 0 - 11 + rorq $23, %rax + movq %r10, %rdx + movq %r15, %rcx + addq 48(%rsp), %r9 + xorq %r8, %rcx + xorq %r14, %rax + andq %r14, %rcx + rorq $4, %rax + xorq %r8, %rcx + xorq %r14, %rax + addq %rcx, %r9 + rorq $14, %rax + xorq %r11, %rdx + addq %rax, %r9 + movq %r10, %rcx + andq %rdx, %rbx + rorq $5, %rcx + xorq %r10, %rcx + xorq %r11, %rbx + rorq $6, %rcx + addq %r9, %r13 + xorq %r10, %rcx + addq %rbx, %r9 + rorq $28, %rcx + movq %r13, %rax + addq %rcx, %r9 + # rnd_1: 0 - 11 + rorq $23, %rax + movq %r9, %rbx + movq %r14, %rcx + addq 56(%rsp), %r8 + xorq %r15, %rcx + xorq %r13, %rax + andq %r13, %rcx + rorq $4, %rax + xorq %r15, %rcx + xorq %r13, %rax + addq %rcx, %r8 + rorq $14, %rax + xorq %r10, %rbx + addq %rax, %r8 + movq %r9, %rcx + andq %rbx, %rdx + rorq $5, %rcx + xorq %r9, %rcx + xorq %r10, %rdx + rorq $6, %rcx + addq %r8, %r12 + xorq %r9, %rcx + addq %rdx, %r8 + rorq $28, %rcx + movq %r12, %rax + addq %rcx, %r8 + # rnd_all_2: 8-9 + # rnd_0: 0 - 11 + rorq $23, %rax + movq %r8, %rdx + movq %r13, %rcx + addq 64(%rsp), %r15 + xorq %r14, %rcx + xorq %r12, %rax + andq %r12, %rcx + rorq $4, %rax + xorq %r14, %rcx + xorq %r12, %rax + addq %rcx, %r15 + rorq $14, %rax + xorq %r9, %rdx + addq %rax, %r15 + movq %r8, %rcx + andq %rdx, %rbx + rorq $5, %rcx + xorq %r8, %rcx + xorq %r9, %rbx + rorq $6, %rcx + addq %r15, %r11 + xorq %r8, %rcx + addq %rbx, %r15 + rorq $28, %rcx + movq %r11, %rax + addq %rcx, %r15 + # rnd_1: 0 - 11 + rorq $23, %rax + movq %r15, %rbx + movq %r12, %rcx + addq 72(%rsp), %r14 + xorq %r13, %rcx + xorq %r11, %rax + andq %r11, %rcx + rorq $4, %rax + xorq %r13, %rcx + xorq %r11, %rax + addq %rcx, %r14 + rorq $14, %rax + xorq %r8, %rbx + addq %rax, %r14 + movq %r15, %rcx + andq %rbx, %rdx + rorq $5, %rcx + xorq %r15, %rcx + xorq %r8, %rdx + rorq $6, %rcx + addq %r14, %r10 + xorq %r15, %rcx + addq %rdx, %r14 + rorq $28, %rcx + movq %r10, %rax + addq %rcx, %r14 + # rnd_all_2: 10-11 + # rnd_0: 0 - 11 + rorq $23, %rax + movq %r14, %rdx + movq %r11, %rcx + addq 80(%rsp), %r13 + xorq %r12, %rcx + xorq %r10, %rax + andq %r10, %rcx + rorq $4, %rax + xorq %r12, %rcx + xorq %r10, %rax + addq %rcx, %r13 + rorq $14, %rax + xorq %r15, %rdx + addq %rax, %r13 + movq %r14, %rcx + andq %rdx, %rbx + rorq $5, %rcx + xorq %r14, %rcx + xorq %r15, %rbx + rorq $6, %rcx + addq %r13, %r9 + xorq %r14, %rcx + addq %rbx, %r13 + rorq $28, %rcx + movq %r9, %rax + addq %rcx, %r13 + # rnd_1: 0 - 11 + rorq $23, %rax + movq %r13, %rbx + movq %r10, %rcx + addq 88(%rsp), %r12 + xorq %r11, %rcx + xorq %r9, %rax + andq %r9, %rcx + rorq $4, %rax + xorq %r11, %rcx + xorq %r9, %rax + addq %rcx, %r12 + rorq $14, %rax + xorq %r14, %rbx + addq %rax, %r12 + movq %r13, %rcx + andq %rbx, %rdx + rorq $5, %rcx + xorq %r13, %rcx + xorq %r14, %rdx + rorq $6, %rcx + addq %r12, %r8 + xorq %r13, %rcx + addq %rdx, %r12 + rorq $28, %rcx + movq %r8, %rax + addq %rcx, %r12 + # rnd_all_2: 12-13 + # rnd_0: 0 - 11 + rorq $23, %rax + movq %r12, %rdx + movq %r9, %rcx + addq 96(%rsp), %r11 + xorq %r10, %rcx + xorq %r8, %rax + andq %r8, %rcx + rorq $4, %rax + xorq %r10, %rcx + xorq %r8, %rax + addq %rcx, %r11 + rorq $14, %rax + xorq %r13, %rdx + addq %rax, %r11 + movq %r12, %rcx + andq %rdx, %rbx + rorq $5, %rcx + xorq %r12, %rcx + xorq %r13, %rbx + rorq $6, %rcx + addq %r11, %r15 + xorq %r12, %rcx + addq %rbx, %r11 + rorq $28, %rcx + movq %r15, %rax + addq %rcx, %r11 + # rnd_1: 0 - 11 + rorq $23, %rax + movq %r11, %rbx + movq %r8, %rcx + addq 104(%rsp), %r10 + xorq %r9, %rcx + xorq %r15, %rax + andq %r15, %rcx + rorq $4, %rax + xorq %r9, %rcx + xorq %r15, %rax + addq %rcx, %r10 + rorq $14, %rax + xorq %r12, %rbx + addq %rax, %r10 + movq %r11, %rcx + andq %rbx, %rdx + rorq $5, %rcx + xorq %r11, %rcx + xorq %r12, %rdx + rorq $6, %rcx + addq %r10, %r14 + xorq %r11, %rcx + addq %rdx, %r10 + rorq $28, %rcx + movq %r14, %rax + addq %rcx, %r10 + # rnd_all_2: 14-15 + # rnd_0: 0 - 11 + rorq $23, %rax + movq %r10, %rdx + movq %r15, %rcx + addq 112(%rsp), %r9 + xorq %r8, %rcx + xorq %r14, %rax + andq %r14, %rcx + rorq $4, %rax + xorq %r8, %rcx + xorq %r14, %rax + addq %rcx, %r9 + rorq $14, %rax + xorq %r11, %rdx + addq %rax, %r9 + movq %r10, %rcx + andq %rdx, %rbx + rorq $5, %rcx + xorq %r10, %rcx + xorq %r11, %rbx + rorq $6, %rcx + addq %r9, %r13 + xorq %r10, %rcx + addq %rbx, %r9 + rorq $28, %rcx + movq %r13, %rax + addq %rcx, %r9 + # rnd_1: 0 - 11 + rorq $23, %rax + movq %r9, %rbx + movq %r14, %rcx + addq 120(%rsp), %r8 + xorq %r15, %rcx + xorq %r13, %rax + andq %r13, %rcx + rorq $4, %rax + xorq %r15, %rcx + xorq %r13, %rax + addq %rcx, %r8 + rorq $14, %rax + xorq %r10, %rbx + addq %rax, %r8 + movq %r9, %rcx + andq %rbx, %rdx + rorq $5, %rcx + xorq %r9, %rcx + xorq %r10, %rdx + rorq $6, %rcx + addq %r8, %r12 + xorq %r9, %rcx + addq %rdx, %r8 + rorq $28, %rcx + movq %r12, %rax + addq %rcx, %r8 + addq (%rdi), %r8 + addq 8(%rdi), %r9 + addq 16(%rdi), %r10 + addq 24(%rdi), %r11 + addq 32(%rdi), %r12 + addq 40(%rdi), %r13 + addq 48(%rdi), %r14 + addq 56(%rdi), %r15 + leaq L_avx1_sha512_k(%rip), %rdx + addq $0x80, %rsi + subl $0x80, %ebp + movq %r8, (%rdi) + movq %r9, 8(%rdi) + movq %r10, 16(%rdi) + movq %r11, 24(%rdi) + movq %r12, 32(%rdi) + movq %r13, 40(%rdi) + movq %r14, 48(%rdi) + movq %r15, 56(%rdi) + jnz L_sha512_len_avx1_begin + xorq %rax, %rax + vzeroupper + addq $0x90, %rsp + popq %rbp + popq %r15 + popq %r14 + popq %r13 + popq %r12 + popq %rbx + repz retq +.size Transform_Sha512_AVX1_Len,.-Transform_Sha512_AVX1_Len +.align 16 +L_avx1_rorx_sha512_k: +.quad 0x428a2f98d728ae22,0x7137449123ef65cd +.quad 0xb5c0fbcfec4d3b2f,0xe9b5dba58189dbbc +.quad 0x3956c25bf348b538,0x59f111f1b605d019 +.quad 0x923f82a4af194f9b,0xab1c5ed5da6d8118 +.quad 0xd807aa98a3030242,0x12835b0145706fbe +.quad 0x243185be4ee4b28c,0x550c7dc3d5ffb4e2 +.quad 0x72be5d74f27b896f,0x80deb1fe3b1696b1 +.quad 0x9bdc06a725c71235,0xc19bf174cf692694 +.quad 0xe49b69c19ef14ad2,0xefbe4786384f25e3 +.quad 0xfc19dc68b8cd5b5,0x240ca1cc77ac9c65 +.quad 0x2de92c6f592b0275,0x4a7484aa6ea6e483 +.quad 0x5cb0a9dcbd41fbd4,0x76f988da831153b5 +.quad 0x983e5152ee66dfab,0xa831c66d2db43210 +.quad 0xb00327c898fb213f,0xbf597fc7beef0ee4 +.quad 0xc6e00bf33da88fc2,0xd5a79147930aa725 +.quad 0x6ca6351e003826f,0x142929670a0e6e70 +.quad 0x27b70a8546d22ffc,0x2e1b21385c26c926 +.quad 0x4d2c6dfc5ac42aed,0x53380d139d95b3df +.quad 0x650a73548baf63de,0x766a0abb3c77b2a8 +.quad 0x81c2c92e47edaee6,0x92722c851482353b +.quad 0xa2bfe8a14cf10364,0xa81a664bbc423001 +.quad 0xc24b8b70d0f89791,0xc76c51a30654be30 +.quad 0xd192e819d6ef5218,0xd69906245565a910 +.quad 0xf40e35855771202a,0x106aa07032bbd1b8 +.quad 0x19a4c116b8d2d0c8,0x1e376c085141ab53 +.quad 0x2748774cdf8eeb99,0x34b0bcb5e19b48a8 +.quad 0x391c0cb3c5c95a63,0x4ed8aa4ae3418acb +.quad 0x5b9cca4f7763e373,0x682e6ff3d6b2b8a3 +.quad 0x748f82ee5defb2fc,0x78a5636f43172f60 +.quad 0x84c87814a1f0ab72,0x8cc702081a6439ec +.quad 0x90befffa23631e28,0xa4506cebde82bde9 +.quad 0xbef9a3f7b2c67915,0xc67178f2e372532b +.quad 0xca273eceea26619c,0xd186b8c721c0c207 +.quad 0xeada7dd6cde0eb1e,0xf57d4f7fee6ed178 +.quad 0x6f067aa72176fba,0xa637dc5a2c898a6 +.quad 0x113f9804bef90dae,0x1b710b35131c471b +.quad 0x28db77f523047d84,0x32caab7b40c72493 +.quad 0x3c9ebe0a15c9bebc,0x431d67c49c100d4c +.quad 0x4cc5d4becb3e42b6,0x597f299cfc657e2a +.quad 0x5fcb6fab3ad6faec,0x6c44198c4a475817 +.align 16 +L_avx1_rorx_sha512_flip_mask: +.quad 0x1020304050607, 0x8090a0b0c0d0e0f +.globl Transform_Sha512_AVX1_RORX +.type Transform_Sha512_AVX1_RORX,@function +.align 4 +Transform_Sha512_AVX1_RORX: + pushq %rbx + pushq %r12 + pushq %r13 + pushq %r14 + pushq %r15 + subq $0x88, %rsp + leaq 64(%rdi), %rax + vmovdqa L_avx1_rorx_sha512_flip_mask(%rip), %xmm14 + movq (%rdi), %r8 + movq 8(%rdi), %r9 + movq 16(%rdi), %r10 + movq 24(%rdi), %r11 + movq 32(%rdi), %r12 + movq 40(%rdi), %r13 + movq 48(%rdi), %r14 + movq 56(%rdi), %r15 + vmovdqu (%rax), %xmm0 + vmovdqu 16(%rax), %xmm1 + vpshufb %xmm14, %xmm0, %xmm0 + vpshufb %xmm14, %xmm1, %xmm1 + vmovdqu 32(%rax), %xmm2 + vmovdqu 48(%rax), %xmm3 + vpshufb %xmm14, %xmm2, %xmm2 + vpshufb %xmm14, %xmm3, %xmm3 + vmovdqu 64(%rax), %xmm4 + vmovdqu 80(%rax), %xmm5 + vpshufb %xmm14, %xmm4, %xmm4 + vpshufb %xmm14, %xmm5, %xmm5 + vmovdqu 96(%rax), %xmm6 + vmovdqu 112(%rax), %xmm7 + vpshufb %xmm14, %xmm6, %xmm6 + vpshufb %xmm14, %xmm7, %xmm7 + movl $4, 128(%rsp) + leaq L_avx1_rorx_sha512_k(%rip), %rsi + movq %r9, %rbx + xorq %rdx, %rdx + xorq %r10, %rbx + vpaddq (%rsi), %xmm0, %xmm8 + vpaddq 16(%rsi), %xmm1, %xmm9 + vmovdqu %xmm8, (%rsp) + vmovdqu %xmm9, 16(%rsp) + vpaddq 32(%rsi), %xmm2, %xmm8 + vpaddq 48(%rsi), %xmm3, %xmm9 + vmovdqu %xmm8, 32(%rsp) + vmovdqu %xmm9, 48(%rsp) + vpaddq 64(%rsi), %xmm4, %xmm8 + vpaddq 80(%rsi), %xmm5, %xmm9 + vmovdqu %xmm8, 64(%rsp) + vmovdqu %xmm9, 80(%rsp) + vpaddq 96(%rsi), %xmm6, %xmm8 + vpaddq 112(%rsi), %xmm7, %xmm9 + vmovdqu %xmm8, 96(%rsp) + vmovdqu %xmm9, 112(%rsp) + # Start of 16 rounds +L_sha256_len_avx1_rorx_start: + addq $0x80, %rsi + # msg_sched: 0-1 + # rnd_0: 0 - 0 + rorxq $14, %r12, %rax + rorxq $18, %r12, %rcx + addq %rdx, %r8 + vpalignr $8, %xmm0, %xmm1, %xmm12 + vpalignr $8, %xmm4, %xmm5, %xmm13 + # rnd_0: 1 - 1 + addq (%rsp), %r15 + movq %r13, %rdx + xorq %rax, %rcx + vpsrlq $0x01, %xmm12, %xmm8 + vpsllq $63, %xmm12, %xmm9 + # rnd_0: 2 - 2 + xorq %r14, %rdx + rorxq $41, %r12, %rax + xorq %rcx, %rax + vpsrlq $8, %xmm12, %xmm10 + vpsllq $56, %xmm12, %xmm11 + # rnd_0: 3 - 3 + andq %r12, %rdx + addq %rax, %r15 + rorxq $28, %r8, %rax + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_0: 4 - 4 + rorxq $34, %r8, %rcx + xorq %r14, %rdx + xorq %rax, %rcx + vpsrlq $7, %xmm12, %xmm11 + vpxor %xmm10, %xmm8, %xmm8 + # rnd_0: 5 - 5 + rorxq $39, %r8, %rax + addq %rdx, %r15 + xorq %rcx, %rax + vpxor %xmm11, %xmm8, %xmm8 + vpaddq %xmm0, %xmm13, %xmm0 + # rnd_0: 6 - 7 + movq %r9, %rdx + addq %r15, %r11 + xorq %r8, %rdx + andq %rdx, %rbx + addq %rax, %r15 + xorq %r9, %rbx + vpaddq %xmm0, %xmm8, %xmm0 + # rnd_1: 0 - 0 + rorxq $14, %r11, %rax + rorxq $18, %r11, %rcx + addq %rbx, %r15 + vpsrlq $19, %xmm7, %xmm8 + vpsllq $45, %xmm7, %xmm9 + # rnd_1: 1 - 1 + addq 8(%rsp), %r14 + movq %r12, %rbx + xorq %rax, %rcx + vpsrlq $61, %xmm7, %xmm10 + vpsllq $3, %xmm7, %xmm11 + # rnd_1: 2 - 2 + xorq %r13, %rbx + rorxq $41, %r11, %rax + xorq %rcx, %rax + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_1: 3 - 4 + andq %r11, %rbx + addq %rax, %r14 + rorxq $28, %r15, %rax + rorxq $34, %r15, %rcx + xorq %r13, %rbx + xorq %rax, %rcx + vpxor %xmm10, %xmm8, %xmm8 + vpsrlq $6, %xmm7, %xmm11 + # rnd_1: 5 - 6 + rorxq $39, %r15, %rax + addq %rbx, %r14 + xorq %rcx, %rax + movq %r8, %rbx + addq %r14, %r10 + xorq %r15, %rbx + vpxor %xmm11, %xmm8, %xmm8 + # rnd_1: 7 - 7 + andq %rbx, %rdx + addq %rax, %r14 + xorq %r8, %rdx + vpaddq %xmm0, %xmm8, %xmm0 + # msg_sched done: 0-3 + # msg_sched: 2-3 + # rnd_0: 0 - 0 + rorxq $14, %r10, %rax + rorxq $18, %r10, %rcx + addq %rdx, %r14 + vpalignr $8, %xmm1, %xmm2, %xmm12 + vpalignr $8, %xmm5, %xmm6, %xmm13 + # rnd_0: 1 - 1 + addq 16(%rsp), %r13 + movq %r11, %rdx + xorq %rax, %rcx + vpsrlq $0x01, %xmm12, %xmm8 + vpsllq $63, %xmm12, %xmm9 + # rnd_0: 2 - 2 + xorq %r12, %rdx + rorxq $41, %r10, %rax + xorq %rcx, %rax + vpsrlq $8, %xmm12, %xmm10 + vpsllq $56, %xmm12, %xmm11 + # rnd_0: 3 - 3 + andq %r10, %rdx + addq %rax, %r13 + rorxq $28, %r14, %rax + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_0: 4 - 4 + rorxq $34, %r14, %rcx + xorq %r12, %rdx + xorq %rax, %rcx + vpsrlq $7, %xmm12, %xmm11 + vpxor %xmm10, %xmm8, %xmm8 + # rnd_0: 5 - 5 + rorxq $39, %r14, %rax + addq %rdx, %r13 + xorq %rcx, %rax + vpxor %xmm11, %xmm8, %xmm8 + vpaddq %xmm1, %xmm13, %xmm1 + # rnd_0: 6 - 7 + movq %r15, %rdx + addq %r13, %r9 + xorq %r14, %rdx + andq %rdx, %rbx + addq %rax, %r13 + xorq %r15, %rbx + vpaddq %xmm1, %xmm8, %xmm1 + # rnd_1: 0 - 0 + rorxq $14, %r9, %rax + rorxq $18, %r9, %rcx + addq %rbx, %r13 + vpsrlq $19, %xmm0, %xmm8 + vpsllq $45, %xmm0, %xmm9 + # rnd_1: 1 - 1 + addq 24(%rsp), %r12 + movq %r10, %rbx + xorq %rax, %rcx + vpsrlq $61, %xmm0, %xmm10 + vpsllq $3, %xmm0, %xmm11 + # rnd_1: 2 - 2 + xorq %r11, %rbx + rorxq $41, %r9, %rax + xorq %rcx, %rax + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_1: 3 - 4 + andq %r9, %rbx + addq %rax, %r12 + rorxq $28, %r13, %rax + rorxq $34, %r13, %rcx + xorq %r11, %rbx + xorq %rax, %rcx + vpxor %xmm10, %xmm8, %xmm8 + vpsrlq $6, %xmm0, %xmm11 + # rnd_1: 5 - 6 + rorxq $39, %r13, %rax + addq %rbx, %r12 + xorq %rcx, %rax + movq %r14, %rbx + addq %r12, %r8 + xorq %r13, %rbx + vpxor %xmm11, %xmm8, %xmm8 + # rnd_1: 7 - 7 + andq %rbx, %rdx + addq %rax, %r12 + xorq %r14, %rdx + vpaddq %xmm1, %xmm8, %xmm1 + # msg_sched done: 2-5 + # msg_sched: 4-5 + # rnd_0: 0 - 0 + rorxq $14, %r8, %rax + rorxq $18, %r8, %rcx + addq %rdx, %r12 + vpalignr $8, %xmm2, %xmm3, %xmm12 + vpalignr $8, %xmm6, %xmm7, %xmm13 + # rnd_0: 1 - 1 + addq 32(%rsp), %r11 + movq %r9, %rdx + xorq %rax, %rcx + vpsrlq $0x01, %xmm12, %xmm8 + vpsllq $63, %xmm12, %xmm9 + # rnd_0: 2 - 2 + xorq %r10, %rdx + rorxq $41, %r8, %rax + xorq %rcx, %rax + vpsrlq $8, %xmm12, %xmm10 + vpsllq $56, %xmm12, %xmm11 + # rnd_0: 3 - 3 + andq %r8, %rdx + addq %rax, %r11 + rorxq $28, %r12, %rax + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_0: 4 - 4 + rorxq $34, %r12, %rcx + xorq %r10, %rdx + xorq %rax, %rcx + vpsrlq $7, %xmm12, %xmm11 + vpxor %xmm10, %xmm8, %xmm8 + # rnd_0: 5 - 5 + rorxq $39, %r12, %rax + addq %rdx, %r11 + xorq %rcx, %rax + vpxor %xmm11, %xmm8, %xmm8 + vpaddq %xmm2, %xmm13, %xmm2 + # rnd_0: 6 - 7 + movq %r13, %rdx + addq %r11, %r15 + xorq %r12, %rdx + andq %rdx, %rbx + addq %rax, %r11 + xorq %r13, %rbx + vpaddq %xmm2, %xmm8, %xmm2 + # rnd_1: 0 - 0 + rorxq $14, %r15, %rax + rorxq $18, %r15, %rcx + addq %rbx, %r11 + vpsrlq $19, %xmm1, %xmm8 + vpsllq $45, %xmm1, %xmm9 + # rnd_1: 1 - 1 + addq 40(%rsp), %r10 + movq %r8, %rbx + xorq %rax, %rcx + vpsrlq $61, %xmm1, %xmm10 + vpsllq $3, %xmm1, %xmm11 + # rnd_1: 2 - 2 + xorq %r9, %rbx + rorxq $41, %r15, %rax + xorq %rcx, %rax + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_1: 3 - 4 + andq %r15, %rbx + addq %rax, %r10 + rorxq $28, %r11, %rax + rorxq $34, %r11, %rcx + xorq %r9, %rbx + xorq %rax, %rcx + vpxor %xmm10, %xmm8, %xmm8 + vpsrlq $6, %xmm1, %xmm11 + # rnd_1: 5 - 6 + rorxq $39, %r11, %rax + addq %rbx, %r10 + xorq %rcx, %rax + movq %r12, %rbx + addq %r10, %r14 + xorq %r11, %rbx + vpxor %xmm11, %xmm8, %xmm8 + # rnd_1: 7 - 7 + andq %rbx, %rdx + addq %rax, %r10 + xorq %r12, %rdx + vpaddq %xmm2, %xmm8, %xmm2 + # msg_sched done: 4-7 + # msg_sched: 6-7 + # rnd_0: 0 - 0 + rorxq $14, %r14, %rax + rorxq $18, %r14, %rcx + addq %rdx, %r10 + vpalignr $8, %xmm3, %xmm4, %xmm12 + vpalignr $8, %xmm7, %xmm0, %xmm13 + # rnd_0: 1 - 1 + addq 48(%rsp), %r9 + movq %r15, %rdx + xorq %rax, %rcx + vpsrlq $0x01, %xmm12, %xmm8 + vpsllq $63, %xmm12, %xmm9 + # rnd_0: 2 - 2 + xorq %r8, %rdx + rorxq $41, %r14, %rax + xorq %rcx, %rax + vpsrlq $8, %xmm12, %xmm10 + vpsllq $56, %xmm12, %xmm11 + # rnd_0: 3 - 3 + andq %r14, %rdx + addq %rax, %r9 + rorxq $28, %r10, %rax + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_0: 4 - 4 + rorxq $34, %r10, %rcx + xorq %r8, %rdx + xorq %rax, %rcx + vpsrlq $7, %xmm12, %xmm11 + vpxor %xmm10, %xmm8, %xmm8 + # rnd_0: 5 - 5 + rorxq $39, %r10, %rax + addq %rdx, %r9 + xorq %rcx, %rax + vpxor %xmm11, %xmm8, %xmm8 + vpaddq %xmm3, %xmm13, %xmm3 + # rnd_0: 6 - 7 + movq %r11, %rdx + addq %r9, %r13 + xorq %r10, %rdx + andq %rdx, %rbx + addq %rax, %r9 + xorq %r11, %rbx + vpaddq %xmm3, %xmm8, %xmm3 + # rnd_1: 0 - 0 + rorxq $14, %r13, %rax + rorxq $18, %r13, %rcx + addq %rbx, %r9 + vpsrlq $19, %xmm2, %xmm8 + vpsllq $45, %xmm2, %xmm9 + # rnd_1: 1 - 1 + addq 56(%rsp), %r8 + movq %r14, %rbx + xorq %rax, %rcx + vpsrlq $61, %xmm2, %xmm10 + vpsllq $3, %xmm2, %xmm11 + # rnd_1: 2 - 2 + xorq %r15, %rbx + rorxq $41, %r13, %rax + xorq %rcx, %rax + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_1: 3 - 4 + andq %r13, %rbx + addq %rax, %r8 + rorxq $28, %r9, %rax + rorxq $34, %r9, %rcx + xorq %r15, %rbx + xorq %rax, %rcx + vpxor %xmm10, %xmm8, %xmm8 + vpsrlq $6, %xmm2, %xmm11 + # rnd_1: 5 - 6 + rorxq $39, %r9, %rax + addq %rbx, %r8 + xorq %rcx, %rax + movq %r10, %rbx + addq %r8, %r12 + xorq %r9, %rbx + vpxor %xmm11, %xmm8, %xmm8 + # rnd_1: 7 - 7 + andq %rbx, %rdx + addq %rax, %r8 + xorq %r10, %rdx + vpaddq %xmm3, %xmm8, %xmm3 + # msg_sched done: 6-9 + # msg_sched: 8-9 + # rnd_0: 0 - 0 + rorxq $14, %r12, %rax + rorxq $18, %r12, %rcx + addq %rdx, %r8 + vpalignr $8, %xmm4, %xmm5, %xmm12 + vpalignr $8, %xmm0, %xmm1, %xmm13 + # rnd_0: 1 - 1 + addq 64(%rsp), %r15 + movq %r13, %rdx + xorq %rax, %rcx + vpsrlq $0x01, %xmm12, %xmm8 + vpsllq $63, %xmm12, %xmm9 + # rnd_0: 2 - 2 + xorq %r14, %rdx + rorxq $41, %r12, %rax + xorq %rcx, %rax + vpsrlq $8, %xmm12, %xmm10 + vpsllq $56, %xmm12, %xmm11 + # rnd_0: 3 - 3 + andq %r12, %rdx + addq %rax, %r15 + rorxq $28, %r8, %rax + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_0: 4 - 4 + rorxq $34, %r8, %rcx + xorq %r14, %rdx + xorq %rax, %rcx + vpsrlq $7, %xmm12, %xmm11 + vpxor %xmm10, %xmm8, %xmm8 + # rnd_0: 5 - 5 + rorxq $39, %r8, %rax + addq %rdx, %r15 + xorq %rcx, %rax + vpxor %xmm11, %xmm8, %xmm8 + vpaddq %xmm4, %xmm13, %xmm4 + # rnd_0: 6 - 7 + movq %r9, %rdx + addq %r15, %r11 + xorq %r8, %rdx + andq %rdx, %rbx + addq %rax, %r15 + xorq %r9, %rbx + vpaddq %xmm4, %xmm8, %xmm4 + # rnd_1: 0 - 0 + rorxq $14, %r11, %rax + rorxq $18, %r11, %rcx + addq %rbx, %r15 + vpsrlq $19, %xmm3, %xmm8 + vpsllq $45, %xmm3, %xmm9 + # rnd_1: 1 - 1 + addq 72(%rsp), %r14 + movq %r12, %rbx + xorq %rax, %rcx + vpsrlq $61, %xmm3, %xmm10 + vpsllq $3, %xmm3, %xmm11 + # rnd_1: 2 - 2 + xorq %r13, %rbx + rorxq $41, %r11, %rax + xorq %rcx, %rax + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_1: 3 - 4 + andq %r11, %rbx + addq %rax, %r14 + rorxq $28, %r15, %rax + rorxq $34, %r15, %rcx + xorq %r13, %rbx + xorq %rax, %rcx + vpxor %xmm10, %xmm8, %xmm8 + vpsrlq $6, %xmm3, %xmm11 + # rnd_1: 5 - 6 + rorxq $39, %r15, %rax + addq %rbx, %r14 + xorq %rcx, %rax + movq %r8, %rbx + addq %r14, %r10 + xorq %r15, %rbx + vpxor %xmm11, %xmm8, %xmm8 + # rnd_1: 7 - 7 + andq %rbx, %rdx + addq %rax, %r14 + xorq %r8, %rdx + vpaddq %xmm4, %xmm8, %xmm4 + # msg_sched done: 8-11 + # msg_sched: 10-11 + # rnd_0: 0 - 0 + rorxq $14, %r10, %rax + rorxq $18, %r10, %rcx + addq %rdx, %r14 + vpalignr $8, %xmm5, %xmm6, %xmm12 + vpalignr $8, %xmm1, %xmm2, %xmm13 + # rnd_0: 1 - 1 + addq 80(%rsp), %r13 + movq %r11, %rdx + xorq %rax, %rcx + vpsrlq $0x01, %xmm12, %xmm8 + vpsllq $63, %xmm12, %xmm9 + # rnd_0: 2 - 2 + xorq %r12, %rdx + rorxq $41, %r10, %rax + xorq %rcx, %rax + vpsrlq $8, %xmm12, %xmm10 + vpsllq $56, %xmm12, %xmm11 + # rnd_0: 3 - 3 + andq %r10, %rdx + addq %rax, %r13 + rorxq $28, %r14, %rax + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_0: 4 - 4 + rorxq $34, %r14, %rcx + xorq %r12, %rdx + xorq %rax, %rcx + vpsrlq $7, %xmm12, %xmm11 + vpxor %xmm10, %xmm8, %xmm8 + # rnd_0: 5 - 5 + rorxq $39, %r14, %rax + addq %rdx, %r13 + xorq %rcx, %rax + vpxor %xmm11, %xmm8, %xmm8 + vpaddq %xmm5, %xmm13, %xmm5 + # rnd_0: 6 - 7 + movq %r15, %rdx + addq %r13, %r9 + xorq %r14, %rdx + andq %rdx, %rbx + addq %rax, %r13 + xorq %r15, %rbx + vpaddq %xmm5, %xmm8, %xmm5 + # rnd_1: 0 - 0 + rorxq $14, %r9, %rax + rorxq $18, %r9, %rcx + addq %rbx, %r13 + vpsrlq $19, %xmm4, %xmm8 + vpsllq $45, %xmm4, %xmm9 + # rnd_1: 1 - 1 + addq 88(%rsp), %r12 + movq %r10, %rbx + xorq %rax, %rcx + vpsrlq $61, %xmm4, %xmm10 + vpsllq $3, %xmm4, %xmm11 + # rnd_1: 2 - 2 + xorq %r11, %rbx + rorxq $41, %r9, %rax + xorq %rcx, %rax + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_1: 3 - 4 + andq %r9, %rbx + addq %rax, %r12 + rorxq $28, %r13, %rax + rorxq $34, %r13, %rcx + xorq %r11, %rbx + xorq %rax, %rcx + vpxor %xmm10, %xmm8, %xmm8 + vpsrlq $6, %xmm4, %xmm11 + # rnd_1: 5 - 6 + rorxq $39, %r13, %rax + addq %rbx, %r12 + xorq %rcx, %rax + movq %r14, %rbx + addq %r12, %r8 + xorq %r13, %rbx + vpxor %xmm11, %xmm8, %xmm8 + # rnd_1: 7 - 7 + andq %rbx, %rdx + addq %rax, %r12 + xorq %r14, %rdx + vpaddq %xmm5, %xmm8, %xmm5 + # msg_sched done: 10-13 + # msg_sched: 12-13 + # rnd_0: 0 - 0 + rorxq $14, %r8, %rax + rorxq $18, %r8, %rcx + addq %rdx, %r12 + vpalignr $8, %xmm6, %xmm7, %xmm12 + vpalignr $8, %xmm2, %xmm3, %xmm13 + # rnd_0: 1 - 1 + addq 96(%rsp), %r11 + movq %r9, %rdx + xorq %rax, %rcx + vpsrlq $0x01, %xmm12, %xmm8 + vpsllq $63, %xmm12, %xmm9 + # rnd_0: 2 - 2 + xorq %r10, %rdx + rorxq $41, %r8, %rax + xorq %rcx, %rax + vpsrlq $8, %xmm12, %xmm10 + vpsllq $56, %xmm12, %xmm11 + # rnd_0: 3 - 3 + andq %r8, %rdx + addq %rax, %r11 + rorxq $28, %r12, %rax + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_0: 4 - 4 + rorxq $34, %r12, %rcx + xorq %r10, %rdx + xorq %rax, %rcx + vpsrlq $7, %xmm12, %xmm11 + vpxor %xmm10, %xmm8, %xmm8 + # rnd_0: 5 - 5 + rorxq $39, %r12, %rax + addq %rdx, %r11 + xorq %rcx, %rax + vpxor %xmm11, %xmm8, %xmm8 + vpaddq %xmm6, %xmm13, %xmm6 + # rnd_0: 6 - 7 + movq %r13, %rdx + addq %r11, %r15 + xorq %r12, %rdx + andq %rdx, %rbx + addq %rax, %r11 + xorq %r13, %rbx + vpaddq %xmm6, %xmm8, %xmm6 + # rnd_1: 0 - 0 + rorxq $14, %r15, %rax + rorxq $18, %r15, %rcx + addq %rbx, %r11 + vpsrlq $19, %xmm5, %xmm8 + vpsllq $45, %xmm5, %xmm9 + # rnd_1: 1 - 1 + addq 104(%rsp), %r10 + movq %r8, %rbx + xorq %rax, %rcx + vpsrlq $61, %xmm5, %xmm10 + vpsllq $3, %xmm5, %xmm11 + # rnd_1: 2 - 2 + xorq %r9, %rbx + rorxq $41, %r15, %rax + xorq %rcx, %rax + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_1: 3 - 4 + andq %r15, %rbx + addq %rax, %r10 + rorxq $28, %r11, %rax + rorxq $34, %r11, %rcx + xorq %r9, %rbx + xorq %rax, %rcx + vpxor %xmm10, %xmm8, %xmm8 + vpsrlq $6, %xmm5, %xmm11 + # rnd_1: 5 - 6 + rorxq $39, %r11, %rax + addq %rbx, %r10 + xorq %rcx, %rax + movq %r12, %rbx + addq %r10, %r14 + xorq %r11, %rbx + vpxor %xmm11, %xmm8, %xmm8 + # rnd_1: 7 - 7 + andq %rbx, %rdx + addq %rax, %r10 + xorq %r12, %rdx + vpaddq %xmm6, %xmm8, %xmm6 + # msg_sched done: 12-15 + # msg_sched: 14-15 + # rnd_0: 0 - 0 + rorxq $14, %r14, %rax + rorxq $18, %r14, %rcx + addq %rdx, %r10 + vpalignr $8, %xmm7, %xmm0, %xmm12 + vpalignr $8, %xmm3, %xmm4, %xmm13 + # rnd_0: 1 - 1 + addq 112(%rsp), %r9 + movq %r15, %rdx + xorq %rax, %rcx + vpsrlq $0x01, %xmm12, %xmm8 + vpsllq $63, %xmm12, %xmm9 + # rnd_0: 2 - 2 + xorq %r8, %rdx + rorxq $41, %r14, %rax + xorq %rcx, %rax + vpsrlq $8, %xmm12, %xmm10 + vpsllq $56, %xmm12, %xmm11 + # rnd_0: 3 - 3 + andq %r14, %rdx + addq %rax, %r9 + rorxq $28, %r10, %rax + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_0: 4 - 4 + rorxq $34, %r10, %rcx + xorq %r8, %rdx + xorq %rax, %rcx + vpsrlq $7, %xmm12, %xmm11 + vpxor %xmm10, %xmm8, %xmm8 + # rnd_0: 5 - 5 + rorxq $39, %r10, %rax + addq %rdx, %r9 + xorq %rcx, %rax + vpxor %xmm11, %xmm8, %xmm8 + vpaddq %xmm7, %xmm13, %xmm7 + # rnd_0: 6 - 7 + movq %r11, %rdx + addq %r9, %r13 + xorq %r10, %rdx + andq %rdx, %rbx + addq %rax, %r9 + xorq %r11, %rbx + vpaddq %xmm7, %xmm8, %xmm7 + # rnd_1: 0 - 0 + rorxq $14, %r13, %rax + rorxq $18, %r13, %rcx + addq %rbx, %r9 + vpsrlq $19, %xmm6, %xmm8 + vpsllq $45, %xmm6, %xmm9 + # rnd_1: 1 - 1 + addq 120(%rsp), %r8 + movq %r14, %rbx + xorq %rax, %rcx + vpsrlq $61, %xmm6, %xmm10 + vpsllq $3, %xmm6, %xmm11 + # rnd_1: 2 - 2 + xorq %r15, %rbx + rorxq $41, %r13, %rax + xorq %rcx, %rax + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_1: 3 - 4 + andq %r13, %rbx + addq %rax, %r8 + rorxq $28, %r9, %rax + rorxq $34, %r9, %rcx + xorq %r15, %rbx + xorq %rax, %rcx + vpxor %xmm10, %xmm8, %xmm8 + vpsrlq $6, %xmm6, %xmm11 + # rnd_1: 5 - 6 + rorxq $39, %r9, %rax + addq %rbx, %r8 + xorq %rcx, %rax + movq %r10, %rbx + addq %r8, %r12 + xorq %r9, %rbx + vpxor %xmm11, %xmm8, %xmm8 + # rnd_1: 7 - 7 + andq %rbx, %rdx + addq %rax, %r8 + xorq %r10, %rdx + vpaddq %xmm7, %xmm8, %xmm7 + # msg_sched done: 14-17 + vpaddq (%rsi), %xmm0, %xmm8 + vpaddq 16(%rsi), %xmm1, %xmm9 + vmovdqu %xmm8, (%rsp) + vmovdqu %xmm9, 16(%rsp) + vpaddq 32(%rsi), %xmm2, %xmm8 + vpaddq 48(%rsi), %xmm3, %xmm9 + vmovdqu %xmm8, 32(%rsp) + vmovdqu %xmm9, 48(%rsp) + vpaddq 64(%rsi), %xmm4, %xmm8 + vpaddq 80(%rsi), %xmm5, %xmm9 + vmovdqu %xmm8, 64(%rsp) + vmovdqu %xmm9, 80(%rsp) + vpaddq 96(%rsi), %xmm6, %xmm8 + vpaddq 112(%rsi), %xmm7, %xmm9 + vmovdqu %xmm8, 96(%rsp) + vmovdqu %xmm9, 112(%rsp) + subl $0x01, 128(%rsp) + jne L_sha256_len_avx1_rorx_start + # rnd_all_2: 0-1 + # rnd_0: 0 - 7 + rorxq $14, %r12, %rax + rorxq $18, %r12, %rcx + addq %rdx, %r8 + addq (%rsp), %r15 + movq %r13, %rdx + xorq %rax, %rcx + xorq %r14, %rdx + rorxq $41, %r12, %rax + xorq %rcx, %rax + andq %r12, %rdx + addq %rax, %r15 + rorxq $28, %r8, %rax + rorxq $34, %r8, %rcx + xorq %r14, %rdx + xorq %rax, %rcx + rorxq $39, %r8, %rax + addq %rdx, %r15 + xorq %rcx, %rax + movq %r9, %rdx + addq %r15, %r11 + xorq %r8, %rdx + andq %rdx, %rbx + addq %rax, %r15 + xorq %r9, %rbx + # rnd_1: 0 - 7 + rorxq $14, %r11, %rax + rorxq $18, %r11, %rcx + addq %rbx, %r15 + addq 8(%rsp), %r14 + movq %r12, %rbx + xorq %rax, %rcx + xorq %r13, %rbx + rorxq $41, %r11, %rax + xorq %rcx, %rax + andq %r11, %rbx + addq %rax, %r14 + rorxq $28, %r15, %rax + rorxq $34, %r15, %rcx + xorq %r13, %rbx + xorq %rax, %rcx + rorxq $39, %r15, %rax + addq %rbx, %r14 + xorq %rcx, %rax + movq %r8, %rbx + addq %r14, %r10 + xorq %r15, %rbx + andq %rbx, %rdx + addq %rax, %r14 + xorq %r8, %rdx + # rnd_all_2: 2-3 + # rnd_0: 0 - 7 + rorxq $14, %r10, %rax + rorxq $18, %r10, %rcx + addq %rdx, %r14 + addq 16(%rsp), %r13 + movq %r11, %rdx + xorq %rax, %rcx + xorq %r12, %rdx + rorxq $41, %r10, %rax + xorq %rcx, %rax + andq %r10, %rdx + addq %rax, %r13 + rorxq $28, %r14, %rax + rorxq $34, %r14, %rcx + xorq %r12, %rdx + xorq %rax, %rcx + rorxq $39, %r14, %rax + addq %rdx, %r13 + xorq %rcx, %rax + movq %r15, %rdx + addq %r13, %r9 + xorq %r14, %rdx + andq %rdx, %rbx + addq %rax, %r13 + xorq %r15, %rbx + # rnd_1: 0 - 7 + rorxq $14, %r9, %rax + rorxq $18, %r9, %rcx + addq %rbx, %r13 + addq 24(%rsp), %r12 + movq %r10, %rbx + xorq %rax, %rcx + xorq %r11, %rbx + rorxq $41, %r9, %rax + xorq %rcx, %rax + andq %r9, %rbx + addq %rax, %r12 + rorxq $28, %r13, %rax + rorxq $34, %r13, %rcx + xorq %r11, %rbx + xorq %rax, %rcx + rorxq $39, %r13, %rax + addq %rbx, %r12 + xorq %rcx, %rax + movq %r14, %rbx + addq %r12, %r8 + xorq %r13, %rbx + andq %rbx, %rdx + addq %rax, %r12 + xorq %r14, %rdx + # rnd_all_2: 4-5 + # rnd_0: 0 - 7 + rorxq $14, %r8, %rax + rorxq $18, %r8, %rcx + addq %rdx, %r12 + addq 32(%rsp), %r11 + movq %r9, %rdx + xorq %rax, %rcx + xorq %r10, %rdx + rorxq $41, %r8, %rax + xorq %rcx, %rax + andq %r8, %rdx + addq %rax, %r11 + rorxq $28, %r12, %rax + rorxq $34, %r12, %rcx + xorq %r10, %rdx + xorq %rax, %rcx + rorxq $39, %r12, %rax + addq %rdx, %r11 + xorq %rcx, %rax + movq %r13, %rdx + addq %r11, %r15 + xorq %r12, %rdx + andq %rdx, %rbx + addq %rax, %r11 + xorq %r13, %rbx + # rnd_1: 0 - 7 + rorxq $14, %r15, %rax + rorxq $18, %r15, %rcx + addq %rbx, %r11 + addq 40(%rsp), %r10 + movq %r8, %rbx + xorq %rax, %rcx + xorq %r9, %rbx + rorxq $41, %r15, %rax + xorq %rcx, %rax + andq %r15, %rbx + addq %rax, %r10 + rorxq $28, %r11, %rax + rorxq $34, %r11, %rcx + xorq %r9, %rbx + xorq %rax, %rcx + rorxq $39, %r11, %rax + addq %rbx, %r10 + xorq %rcx, %rax + movq %r12, %rbx + addq %r10, %r14 + xorq %r11, %rbx + andq %rbx, %rdx + addq %rax, %r10 + xorq %r12, %rdx + # rnd_all_2: 6-7 + # rnd_0: 0 - 7 + rorxq $14, %r14, %rax + rorxq $18, %r14, %rcx + addq %rdx, %r10 + addq 48(%rsp), %r9 + movq %r15, %rdx + xorq %rax, %rcx + xorq %r8, %rdx + rorxq $41, %r14, %rax + xorq %rcx, %rax + andq %r14, %rdx + addq %rax, %r9 + rorxq $28, %r10, %rax + rorxq $34, %r10, %rcx + xorq %r8, %rdx + xorq %rax, %rcx + rorxq $39, %r10, %rax + addq %rdx, %r9 + xorq %rcx, %rax + movq %r11, %rdx + addq %r9, %r13 + xorq %r10, %rdx + andq %rdx, %rbx + addq %rax, %r9 + xorq %r11, %rbx + # rnd_1: 0 - 7 + rorxq $14, %r13, %rax + rorxq $18, %r13, %rcx + addq %rbx, %r9 + addq 56(%rsp), %r8 + movq %r14, %rbx + xorq %rax, %rcx + xorq %r15, %rbx + rorxq $41, %r13, %rax + xorq %rcx, %rax + andq %r13, %rbx + addq %rax, %r8 + rorxq $28, %r9, %rax + rorxq $34, %r9, %rcx + xorq %r15, %rbx + xorq %rax, %rcx + rorxq $39, %r9, %rax + addq %rbx, %r8 + xorq %rcx, %rax + movq %r10, %rbx + addq %r8, %r12 + xorq %r9, %rbx + andq %rbx, %rdx + addq %rax, %r8 + xorq %r10, %rdx + # rnd_all_2: 8-9 + # rnd_0: 0 - 7 + rorxq $14, %r12, %rax + rorxq $18, %r12, %rcx + addq %rdx, %r8 + addq 64(%rsp), %r15 + movq %r13, %rdx + xorq %rax, %rcx + xorq %r14, %rdx + rorxq $41, %r12, %rax + xorq %rcx, %rax + andq %r12, %rdx + addq %rax, %r15 + rorxq $28, %r8, %rax + rorxq $34, %r8, %rcx + xorq %r14, %rdx + xorq %rax, %rcx + rorxq $39, %r8, %rax + addq %rdx, %r15 + xorq %rcx, %rax + movq %r9, %rdx + addq %r15, %r11 + xorq %r8, %rdx + andq %rdx, %rbx + addq %rax, %r15 + xorq %r9, %rbx + # rnd_1: 0 - 7 + rorxq $14, %r11, %rax + rorxq $18, %r11, %rcx + addq %rbx, %r15 + addq 72(%rsp), %r14 + movq %r12, %rbx + xorq %rax, %rcx + xorq %r13, %rbx + rorxq $41, %r11, %rax + xorq %rcx, %rax + andq %r11, %rbx + addq %rax, %r14 + rorxq $28, %r15, %rax + rorxq $34, %r15, %rcx + xorq %r13, %rbx + xorq %rax, %rcx + rorxq $39, %r15, %rax + addq %rbx, %r14 + xorq %rcx, %rax + movq %r8, %rbx + addq %r14, %r10 + xorq %r15, %rbx + andq %rbx, %rdx + addq %rax, %r14 + xorq %r8, %rdx + # rnd_all_2: 10-11 + # rnd_0: 0 - 7 + rorxq $14, %r10, %rax + rorxq $18, %r10, %rcx + addq %rdx, %r14 + addq 80(%rsp), %r13 + movq %r11, %rdx + xorq %rax, %rcx + xorq %r12, %rdx + rorxq $41, %r10, %rax + xorq %rcx, %rax + andq %r10, %rdx + addq %rax, %r13 + rorxq $28, %r14, %rax + rorxq $34, %r14, %rcx + xorq %r12, %rdx + xorq %rax, %rcx + rorxq $39, %r14, %rax + addq %rdx, %r13 + xorq %rcx, %rax + movq %r15, %rdx + addq %r13, %r9 + xorq %r14, %rdx + andq %rdx, %rbx + addq %rax, %r13 + xorq %r15, %rbx + # rnd_1: 0 - 7 + rorxq $14, %r9, %rax + rorxq $18, %r9, %rcx + addq %rbx, %r13 + addq 88(%rsp), %r12 + movq %r10, %rbx + xorq %rax, %rcx + xorq %r11, %rbx + rorxq $41, %r9, %rax + xorq %rcx, %rax + andq %r9, %rbx + addq %rax, %r12 + rorxq $28, %r13, %rax + rorxq $34, %r13, %rcx + xorq %r11, %rbx + xorq %rax, %rcx + rorxq $39, %r13, %rax + addq %rbx, %r12 + xorq %rcx, %rax + movq %r14, %rbx + addq %r12, %r8 + xorq %r13, %rbx + andq %rbx, %rdx + addq %rax, %r12 + xorq %r14, %rdx + # rnd_all_2: 12-13 + # rnd_0: 0 - 7 + rorxq $14, %r8, %rax + rorxq $18, %r8, %rcx + addq %rdx, %r12 + addq 96(%rsp), %r11 + movq %r9, %rdx + xorq %rax, %rcx + xorq %r10, %rdx + rorxq $41, %r8, %rax + xorq %rcx, %rax + andq %r8, %rdx + addq %rax, %r11 + rorxq $28, %r12, %rax + rorxq $34, %r12, %rcx + xorq %r10, %rdx + xorq %rax, %rcx + rorxq $39, %r12, %rax + addq %rdx, %r11 + xorq %rcx, %rax + movq %r13, %rdx + addq %r11, %r15 + xorq %r12, %rdx + andq %rdx, %rbx + addq %rax, %r11 + xorq %r13, %rbx + # rnd_1: 0 - 7 + rorxq $14, %r15, %rax + rorxq $18, %r15, %rcx + addq %rbx, %r11 + addq 104(%rsp), %r10 + movq %r8, %rbx + xorq %rax, %rcx + xorq %r9, %rbx + rorxq $41, %r15, %rax + xorq %rcx, %rax + andq %r15, %rbx + addq %rax, %r10 + rorxq $28, %r11, %rax + rorxq $34, %r11, %rcx + xorq %r9, %rbx + xorq %rax, %rcx + rorxq $39, %r11, %rax + addq %rbx, %r10 + xorq %rcx, %rax + movq %r12, %rbx + addq %r10, %r14 + xorq %r11, %rbx + andq %rbx, %rdx + addq %rax, %r10 + xorq %r12, %rdx + # rnd_all_2: 14-15 + # rnd_0: 0 - 7 + rorxq $14, %r14, %rax + rorxq $18, %r14, %rcx + addq %rdx, %r10 + addq 112(%rsp), %r9 + movq %r15, %rdx + xorq %rax, %rcx + xorq %r8, %rdx + rorxq $41, %r14, %rax + xorq %rcx, %rax + andq %r14, %rdx + addq %rax, %r9 + rorxq $28, %r10, %rax + rorxq $34, %r10, %rcx + xorq %r8, %rdx + xorq %rax, %rcx + rorxq $39, %r10, %rax + addq %rdx, %r9 + xorq %rcx, %rax + movq %r11, %rdx + addq %r9, %r13 + xorq %r10, %rdx + andq %rdx, %rbx + addq %rax, %r9 + xorq %r11, %rbx + # rnd_1: 0 - 7 + rorxq $14, %r13, %rax + rorxq $18, %r13, %rcx + addq %rbx, %r9 + addq 120(%rsp), %r8 + movq %r14, %rbx + xorq %rax, %rcx + xorq %r15, %rbx + rorxq $41, %r13, %rax + xorq %rcx, %rax + andq %r13, %rbx + addq %rax, %r8 + rorxq $28, %r9, %rax + rorxq $34, %r9, %rcx + xorq %r15, %rbx + xorq %rax, %rcx + rorxq $39, %r9, %rax + addq %rbx, %r8 + xorq %rcx, %rax + movq %r10, %rbx + addq %r8, %r12 + xorq %r9, %rbx + andq %rbx, %rdx + addq %rax, %r8 + xorq %r10, %rdx + addq %rdx, %r8 + addq %r8, (%rdi) + addq %r9, 8(%rdi) + addq %r10, 16(%rdi) + addq %r11, 24(%rdi) + addq %r12, 32(%rdi) + addq %r13, 40(%rdi) + addq %r14, 48(%rdi) + addq %r15, 56(%rdi) + xorq %rax, %rax + vzeroupper + addq $0x88, %rsp + popq %r15 + popq %r14 + popq %r13 + popq %r12 + popq %rbx + repz retq +.size Transform_Sha512_AVX1_RORX,.-Transform_Sha512_AVX1_RORX +.globl Transform_Sha512_AVX1_RORX_Len +.type Transform_Sha512_AVX1_RORX_Len,@function +.align 4 +Transform_Sha512_AVX1_RORX_Len: + pushq %rbx + pushq %r12 + pushq %r13 + pushq %r14 + pushq %r15 + pushq %rbp + movq %rsi, %rbp + subq $0x90, %rsp + movq 224(%rdi), %rsi + leaq L_avx1_rorx_sha512_k(%rip), %rcx + vmovdqa L_avx1_rorx_sha512_flip_mask(%rip), %xmm14 + movq (%rdi), %r8 + movq 8(%rdi), %r9 + movq 16(%rdi), %r10 + movq 24(%rdi), %r11 + movq 32(%rdi), %r12 + movq 40(%rdi), %r13 + movq 48(%rdi), %r14 + movq 56(%rdi), %r15 + # Start of loop processing a block +L_sha512_len_avx1_rorx_begin: + vmovdqu (%rsi), %xmm0 + vmovdqu 16(%rsi), %xmm1 + vpshufb %xmm14, %xmm0, %xmm0 + vpshufb %xmm14, %xmm1, %xmm1 + vmovdqu 32(%rsi), %xmm2 + vmovdqu 48(%rsi), %xmm3 + vpshufb %xmm14, %xmm2, %xmm2 + vpshufb %xmm14, %xmm3, %xmm3 + vmovdqu 64(%rsi), %xmm4 + vmovdqu 80(%rsi), %xmm5 + vpshufb %xmm14, %xmm4, %xmm4 + vpshufb %xmm14, %xmm5, %xmm5 + vmovdqu 96(%rsi), %xmm6 + vmovdqu 112(%rsi), %xmm7 + vpshufb %xmm14, %xmm6, %xmm6 + vpshufb %xmm14, %xmm7, %xmm7 + movl $4, 128(%rsp) + movq %r9, %rbx + xorq %rdx, %rdx + xorq %r10, %rbx + vpaddq (%rcx), %xmm0, %xmm8 + vpaddq 16(%rcx), %xmm1, %xmm9 + vmovdqu %xmm8, (%rsp) + vmovdqu %xmm9, 16(%rsp) + vpaddq 32(%rcx), %xmm2, %xmm8 + vpaddq 48(%rcx), %xmm3, %xmm9 + vmovdqu %xmm8, 32(%rsp) + vmovdqu %xmm9, 48(%rsp) + vpaddq 64(%rcx), %xmm4, %xmm8 + vpaddq 80(%rcx), %xmm5, %xmm9 + vmovdqu %xmm8, 64(%rsp) + vmovdqu %xmm9, 80(%rsp) + vpaddq 96(%rcx), %xmm6, %xmm8 + vpaddq 112(%rcx), %xmm7, %xmm9 + vmovdqu %xmm8, 96(%rsp) + vmovdqu %xmm9, 112(%rsp) + # Start of 16 rounds +L_sha512_len_avx1_rorx_start: + addq $0x80, %rcx + movq %rcx, 136(%rsp) + # msg_sched: 0-1 + # rnd_0: 0 - 0 + rorxq $14, %r12, %rax + rorxq $18, %r12, %rcx + addq %rdx, %r8 + vpalignr $8, %xmm0, %xmm1, %xmm12 + vpalignr $8, %xmm4, %xmm5, %xmm13 + # rnd_0: 1 - 1 + addq (%rsp), %r15 + movq %r13, %rdx + xorq %rax, %rcx + vpsrlq $0x01, %xmm12, %xmm8 + vpsllq $63, %xmm12, %xmm9 + # rnd_0: 2 - 2 + xorq %r14, %rdx + rorxq $41, %r12, %rax + xorq %rcx, %rax + vpsrlq $8, %xmm12, %xmm10 + vpsllq $56, %xmm12, %xmm11 + # rnd_0: 3 - 3 + andq %r12, %rdx + addq %rax, %r15 + rorxq $28, %r8, %rax + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_0: 4 - 4 + rorxq $34, %r8, %rcx + xorq %r14, %rdx + xorq %rax, %rcx + vpsrlq $7, %xmm12, %xmm11 + vpxor %xmm10, %xmm8, %xmm8 + # rnd_0: 5 - 5 + rorxq $39, %r8, %rax + addq %rdx, %r15 + xorq %rcx, %rax + vpxor %xmm11, %xmm8, %xmm8 + vpaddq %xmm0, %xmm13, %xmm0 + # rnd_0: 6 - 7 + movq %r9, %rdx + addq %r15, %r11 + xorq %r8, %rdx + andq %rdx, %rbx + addq %rax, %r15 + xorq %r9, %rbx + vpaddq %xmm0, %xmm8, %xmm0 + # rnd_1: 0 - 0 + rorxq $14, %r11, %rax + rorxq $18, %r11, %rcx + addq %rbx, %r15 + vpsrlq $19, %xmm7, %xmm8 + vpsllq $45, %xmm7, %xmm9 + # rnd_1: 1 - 1 + addq 8(%rsp), %r14 + movq %r12, %rbx + xorq %rax, %rcx + vpsrlq $61, %xmm7, %xmm10 + vpsllq $3, %xmm7, %xmm11 + # rnd_1: 2 - 2 + xorq %r13, %rbx + rorxq $41, %r11, %rax + xorq %rcx, %rax + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_1: 3 - 4 + andq %r11, %rbx + addq %rax, %r14 + rorxq $28, %r15, %rax + rorxq $34, %r15, %rcx + xorq %r13, %rbx + xorq %rax, %rcx + vpxor %xmm10, %xmm8, %xmm8 + vpsrlq $6, %xmm7, %xmm11 + # rnd_1: 5 - 6 + rorxq $39, %r15, %rax + addq %rbx, %r14 + xorq %rcx, %rax + movq %r8, %rbx + addq %r14, %r10 + xorq %r15, %rbx + vpxor %xmm11, %xmm8, %xmm8 + # rnd_1: 7 - 7 + andq %rbx, %rdx + addq %rax, %r14 + xorq %r8, %rdx + vpaddq %xmm0, %xmm8, %xmm0 + # msg_sched done: 0-3 + # msg_sched: 2-3 + # rnd_0: 0 - 0 + rorxq $14, %r10, %rax + rorxq $18, %r10, %rcx + addq %rdx, %r14 + vpalignr $8, %xmm1, %xmm2, %xmm12 + vpalignr $8, %xmm5, %xmm6, %xmm13 + # rnd_0: 1 - 1 + addq 16(%rsp), %r13 + movq %r11, %rdx + xorq %rax, %rcx + vpsrlq $0x01, %xmm12, %xmm8 + vpsllq $63, %xmm12, %xmm9 + # rnd_0: 2 - 2 + xorq %r12, %rdx + rorxq $41, %r10, %rax + xorq %rcx, %rax + vpsrlq $8, %xmm12, %xmm10 + vpsllq $56, %xmm12, %xmm11 + # rnd_0: 3 - 3 + andq %r10, %rdx + addq %rax, %r13 + rorxq $28, %r14, %rax + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_0: 4 - 4 + rorxq $34, %r14, %rcx + xorq %r12, %rdx + xorq %rax, %rcx + vpsrlq $7, %xmm12, %xmm11 + vpxor %xmm10, %xmm8, %xmm8 + # rnd_0: 5 - 5 + rorxq $39, %r14, %rax + addq %rdx, %r13 + xorq %rcx, %rax + vpxor %xmm11, %xmm8, %xmm8 + vpaddq %xmm1, %xmm13, %xmm1 + # rnd_0: 6 - 7 + movq %r15, %rdx + addq %r13, %r9 + xorq %r14, %rdx + andq %rdx, %rbx + addq %rax, %r13 + xorq %r15, %rbx + vpaddq %xmm1, %xmm8, %xmm1 + # rnd_1: 0 - 0 + rorxq $14, %r9, %rax + rorxq $18, %r9, %rcx + addq %rbx, %r13 + vpsrlq $19, %xmm0, %xmm8 + vpsllq $45, %xmm0, %xmm9 + # rnd_1: 1 - 1 + addq 24(%rsp), %r12 + movq %r10, %rbx + xorq %rax, %rcx + vpsrlq $61, %xmm0, %xmm10 + vpsllq $3, %xmm0, %xmm11 + # rnd_1: 2 - 2 + xorq %r11, %rbx + rorxq $41, %r9, %rax + xorq %rcx, %rax + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_1: 3 - 4 + andq %r9, %rbx + addq %rax, %r12 + rorxq $28, %r13, %rax + rorxq $34, %r13, %rcx + xorq %r11, %rbx + xorq %rax, %rcx + vpxor %xmm10, %xmm8, %xmm8 + vpsrlq $6, %xmm0, %xmm11 + # rnd_1: 5 - 6 + rorxq $39, %r13, %rax + addq %rbx, %r12 + xorq %rcx, %rax + movq %r14, %rbx + addq %r12, %r8 + xorq %r13, %rbx + vpxor %xmm11, %xmm8, %xmm8 + # rnd_1: 7 - 7 + andq %rbx, %rdx + addq %rax, %r12 + xorq %r14, %rdx + vpaddq %xmm1, %xmm8, %xmm1 + # msg_sched done: 2-5 + # msg_sched: 4-5 + # rnd_0: 0 - 0 + rorxq $14, %r8, %rax + rorxq $18, %r8, %rcx + addq %rdx, %r12 + vpalignr $8, %xmm2, %xmm3, %xmm12 + vpalignr $8, %xmm6, %xmm7, %xmm13 + # rnd_0: 1 - 1 + addq 32(%rsp), %r11 + movq %r9, %rdx + xorq %rax, %rcx + vpsrlq $0x01, %xmm12, %xmm8 + vpsllq $63, %xmm12, %xmm9 + # rnd_0: 2 - 2 + xorq %r10, %rdx + rorxq $41, %r8, %rax + xorq %rcx, %rax + vpsrlq $8, %xmm12, %xmm10 + vpsllq $56, %xmm12, %xmm11 + # rnd_0: 3 - 3 + andq %r8, %rdx + addq %rax, %r11 + rorxq $28, %r12, %rax + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_0: 4 - 4 + rorxq $34, %r12, %rcx + xorq %r10, %rdx + xorq %rax, %rcx + vpsrlq $7, %xmm12, %xmm11 + vpxor %xmm10, %xmm8, %xmm8 + # rnd_0: 5 - 5 + rorxq $39, %r12, %rax + addq %rdx, %r11 + xorq %rcx, %rax + vpxor %xmm11, %xmm8, %xmm8 + vpaddq %xmm2, %xmm13, %xmm2 + # rnd_0: 6 - 7 + movq %r13, %rdx + addq %r11, %r15 + xorq %r12, %rdx + andq %rdx, %rbx + addq %rax, %r11 + xorq %r13, %rbx + vpaddq %xmm2, %xmm8, %xmm2 + # rnd_1: 0 - 0 + rorxq $14, %r15, %rax + rorxq $18, %r15, %rcx + addq %rbx, %r11 + vpsrlq $19, %xmm1, %xmm8 + vpsllq $45, %xmm1, %xmm9 + # rnd_1: 1 - 1 + addq 40(%rsp), %r10 + movq %r8, %rbx + xorq %rax, %rcx + vpsrlq $61, %xmm1, %xmm10 + vpsllq $3, %xmm1, %xmm11 + # rnd_1: 2 - 2 + xorq %r9, %rbx + rorxq $41, %r15, %rax + xorq %rcx, %rax + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_1: 3 - 4 + andq %r15, %rbx + addq %rax, %r10 + rorxq $28, %r11, %rax + rorxq $34, %r11, %rcx + xorq %r9, %rbx + xorq %rax, %rcx + vpxor %xmm10, %xmm8, %xmm8 + vpsrlq $6, %xmm1, %xmm11 + # rnd_1: 5 - 6 + rorxq $39, %r11, %rax + addq %rbx, %r10 + xorq %rcx, %rax + movq %r12, %rbx + addq %r10, %r14 + xorq %r11, %rbx + vpxor %xmm11, %xmm8, %xmm8 + # rnd_1: 7 - 7 + andq %rbx, %rdx + addq %rax, %r10 + xorq %r12, %rdx + vpaddq %xmm2, %xmm8, %xmm2 + # msg_sched done: 4-7 + # msg_sched: 6-7 + # rnd_0: 0 - 0 + rorxq $14, %r14, %rax + rorxq $18, %r14, %rcx + addq %rdx, %r10 + vpalignr $8, %xmm3, %xmm4, %xmm12 + vpalignr $8, %xmm7, %xmm0, %xmm13 + # rnd_0: 1 - 1 + addq 48(%rsp), %r9 + movq %r15, %rdx + xorq %rax, %rcx + vpsrlq $0x01, %xmm12, %xmm8 + vpsllq $63, %xmm12, %xmm9 + # rnd_0: 2 - 2 + xorq %r8, %rdx + rorxq $41, %r14, %rax + xorq %rcx, %rax + vpsrlq $8, %xmm12, %xmm10 + vpsllq $56, %xmm12, %xmm11 + # rnd_0: 3 - 3 + andq %r14, %rdx + addq %rax, %r9 + rorxq $28, %r10, %rax + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_0: 4 - 4 + rorxq $34, %r10, %rcx + xorq %r8, %rdx + xorq %rax, %rcx + vpsrlq $7, %xmm12, %xmm11 + vpxor %xmm10, %xmm8, %xmm8 + # rnd_0: 5 - 5 + rorxq $39, %r10, %rax + addq %rdx, %r9 + xorq %rcx, %rax + vpxor %xmm11, %xmm8, %xmm8 + vpaddq %xmm3, %xmm13, %xmm3 + # rnd_0: 6 - 7 + movq %r11, %rdx + addq %r9, %r13 + xorq %r10, %rdx + andq %rdx, %rbx + addq %rax, %r9 + xorq %r11, %rbx + vpaddq %xmm3, %xmm8, %xmm3 + # rnd_1: 0 - 0 + rorxq $14, %r13, %rax + rorxq $18, %r13, %rcx + addq %rbx, %r9 + vpsrlq $19, %xmm2, %xmm8 + vpsllq $45, %xmm2, %xmm9 + # rnd_1: 1 - 1 + addq 56(%rsp), %r8 + movq %r14, %rbx + xorq %rax, %rcx + vpsrlq $61, %xmm2, %xmm10 + vpsllq $3, %xmm2, %xmm11 + # rnd_1: 2 - 2 + xorq %r15, %rbx + rorxq $41, %r13, %rax + xorq %rcx, %rax + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_1: 3 - 4 + andq %r13, %rbx + addq %rax, %r8 + rorxq $28, %r9, %rax + rorxq $34, %r9, %rcx + xorq %r15, %rbx + xorq %rax, %rcx + vpxor %xmm10, %xmm8, %xmm8 + vpsrlq $6, %xmm2, %xmm11 + # rnd_1: 5 - 6 + rorxq $39, %r9, %rax + addq %rbx, %r8 + xorq %rcx, %rax + movq %r10, %rbx + addq %r8, %r12 + xorq %r9, %rbx + vpxor %xmm11, %xmm8, %xmm8 + # rnd_1: 7 - 7 + andq %rbx, %rdx + addq %rax, %r8 + xorq %r10, %rdx + vpaddq %xmm3, %xmm8, %xmm3 + # msg_sched done: 6-9 + # msg_sched: 8-9 + # rnd_0: 0 - 0 + rorxq $14, %r12, %rax + rorxq $18, %r12, %rcx + addq %rdx, %r8 + vpalignr $8, %xmm4, %xmm5, %xmm12 + vpalignr $8, %xmm0, %xmm1, %xmm13 + # rnd_0: 1 - 1 + addq 64(%rsp), %r15 + movq %r13, %rdx + xorq %rax, %rcx + vpsrlq $0x01, %xmm12, %xmm8 + vpsllq $63, %xmm12, %xmm9 + # rnd_0: 2 - 2 + xorq %r14, %rdx + rorxq $41, %r12, %rax + xorq %rcx, %rax + vpsrlq $8, %xmm12, %xmm10 + vpsllq $56, %xmm12, %xmm11 + # rnd_0: 3 - 3 + andq %r12, %rdx + addq %rax, %r15 + rorxq $28, %r8, %rax + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_0: 4 - 4 + rorxq $34, %r8, %rcx + xorq %r14, %rdx + xorq %rax, %rcx + vpsrlq $7, %xmm12, %xmm11 + vpxor %xmm10, %xmm8, %xmm8 + # rnd_0: 5 - 5 + rorxq $39, %r8, %rax + addq %rdx, %r15 + xorq %rcx, %rax + vpxor %xmm11, %xmm8, %xmm8 + vpaddq %xmm4, %xmm13, %xmm4 + # rnd_0: 6 - 7 + movq %r9, %rdx + addq %r15, %r11 + xorq %r8, %rdx + andq %rdx, %rbx + addq %rax, %r15 + xorq %r9, %rbx + vpaddq %xmm4, %xmm8, %xmm4 + # rnd_1: 0 - 0 + rorxq $14, %r11, %rax + rorxq $18, %r11, %rcx + addq %rbx, %r15 + vpsrlq $19, %xmm3, %xmm8 + vpsllq $45, %xmm3, %xmm9 + # rnd_1: 1 - 1 + addq 72(%rsp), %r14 + movq %r12, %rbx + xorq %rax, %rcx + vpsrlq $61, %xmm3, %xmm10 + vpsllq $3, %xmm3, %xmm11 + # rnd_1: 2 - 2 + xorq %r13, %rbx + rorxq $41, %r11, %rax + xorq %rcx, %rax + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_1: 3 - 4 + andq %r11, %rbx + addq %rax, %r14 + rorxq $28, %r15, %rax + rorxq $34, %r15, %rcx + xorq %r13, %rbx + xorq %rax, %rcx + vpxor %xmm10, %xmm8, %xmm8 + vpsrlq $6, %xmm3, %xmm11 + # rnd_1: 5 - 6 + rorxq $39, %r15, %rax + addq %rbx, %r14 + xorq %rcx, %rax + movq %r8, %rbx + addq %r14, %r10 + xorq %r15, %rbx + vpxor %xmm11, %xmm8, %xmm8 + # rnd_1: 7 - 7 + andq %rbx, %rdx + addq %rax, %r14 + xorq %r8, %rdx + vpaddq %xmm4, %xmm8, %xmm4 + # msg_sched done: 8-11 + # msg_sched: 10-11 + # rnd_0: 0 - 0 + rorxq $14, %r10, %rax + rorxq $18, %r10, %rcx + addq %rdx, %r14 + vpalignr $8, %xmm5, %xmm6, %xmm12 + vpalignr $8, %xmm1, %xmm2, %xmm13 + # rnd_0: 1 - 1 + addq 80(%rsp), %r13 + movq %r11, %rdx + xorq %rax, %rcx + vpsrlq $0x01, %xmm12, %xmm8 + vpsllq $63, %xmm12, %xmm9 + # rnd_0: 2 - 2 + xorq %r12, %rdx + rorxq $41, %r10, %rax + xorq %rcx, %rax + vpsrlq $8, %xmm12, %xmm10 + vpsllq $56, %xmm12, %xmm11 + # rnd_0: 3 - 3 + andq %r10, %rdx + addq %rax, %r13 + rorxq $28, %r14, %rax + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_0: 4 - 4 + rorxq $34, %r14, %rcx + xorq %r12, %rdx + xorq %rax, %rcx + vpsrlq $7, %xmm12, %xmm11 + vpxor %xmm10, %xmm8, %xmm8 + # rnd_0: 5 - 5 + rorxq $39, %r14, %rax + addq %rdx, %r13 + xorq %rcx, %rax + vpxor %xmm11, %xmm8, %xmm8 + vpaddq %xmm5, %xmm13, %xmm5 + # rnd_0: 6 - 7 + movq %r15, %rdx + addq %r13, %r9 + xorq %r14, %rdx + andq %rdx, %rbx + addq %rax, %r13 + xorq %r15, %rbx + vpaddq %xmm5, %xmm8, %xmm5 + # rnd_1: 0 - 0 + rorxq $14, %r9, %rax + rorxq $18, %r9, %rcx + addq %rbx, %r13 + vpsrlq $19, %xmm4, %xmm8 + vpsllq $45, %xmm4, %xmm9 + # rnd_1: 1 - 1 + addq 88(%rsp), %r12 + movq %r10, %rbx + xorq %rax, %rcx + vpsrlq $61, %xmm4, %xmm10 + vpsllq $3, %xmm4, %xmm11 + # rnd_1: 2 - 2 + xorq %r11, %rbx + rorxq $41, %r9, %rax + xorq %rcx, %rax + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_1: 3 - 4 + andq %r9, %rbx + addq %rax, %r12 + rorxq $28, %r13, %rax + rorxq $34, %r13, %rcx + xorq %r11, %rbx + xorq %rax, %rcx + vpxor %xmm10, %xmm8, %xmm8 + vpsrlq $6, %xmm4, %xmm11 + # rnd_1: 5 - 6 + rorxq $39, %r13, %rax + addq %rbx, %r12 + xorq %rcx, %rax + movq %r14, %rbx + addq %r12, %r8 + xorq %r13, %rbx + vpxor %xmm11, %xmm8, %xmm8 + # rnd_1: 7 - 7 + andq %rbx, %rdx + addq %rax, %r12 + xorq %r14, %rdx + vpaddq %xmm5, %xmm8, %xmm5 + # msg_sched done: 10-13 + # msg_sched: 12-13 + # rnd_0: 0 - 0 + rorxq $14, %r8, %rax + rorxq $18, %r8, %rcx + addq %rdx, %r12 + vpalignr $8, %xmm6, %xmm7, %xmm12 + vpalignr $8, %xmm2, %xmm3, %xmm13 + # rnd_0: 1 - 1 + addq 96(%rsp), %r11 + movq %r9, %rdx + xorq %rax, %rcx + vpsrlq $0x01, %xmm12, %xmm8 + vpsllq $63, %xmm12, %xmm9 + # rnd_0: 2 - 2 + xorq %r10, %rdx + rorxq $41, %r8, %rax + xorq %rcx, %rax + vpsrlq $8, %xmm12, %xmm10 + vpsllq $56, %xmm12, %xmm11 + # rnd_0: 3 - 3 + andq %r8, %rdx + addq %rax, %r11 + rorxq $28, %r12, %rax + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_0: 4 - 4 + rorxq $34, %r12, %rcx + xorq %r10, %rdx + xorq %rax, %rcx + vpsrlq $7, %xmm12, %xmm11 + vpxor %xmm10, %xmm8, %xmm8 + # rnd_0: 5 - 5 + rorxq $39, %r12, %rax + addq %rdx, %r11 + xorq %rcx, %rax + vpxor %xmm11, %xmm8, %xmm8 + vpaddq %xmm6, %xmm13, %xmm6 + # rnd_0: 6 - 7 + movq %r13, %rdx + addq %r11, %r15 + xorq %r12, %rdx + andq %rdx, %rbx + addq %rax, %r11 + xorq %r13, %rbx + vpaddq %xmm6, %xmm8, %xmm6 + # rnd_1: 0 - 0 + rorxq $14, %r15, %rax + rorxq $18, %r15, %rcx + addq %rbx, %r11 + vpsrlq $19, %xmm5, %xmm8 + vpsllq $45, %xmm5, %xmm9 + # rnd_1: 1 - 1 + addq 104(%rsp), %r10 + movq %r8, %rbx + xorq %rax, %rcx + vpsrlq $61, %xmm5, %xmm10 + vpsllq $3, %xmm5, %xmm11 + # rnd_1: 2 - 2 + xorq %r9, %rbx + rorxq $41, %r15, %rax + xorq %rcx, %rax + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_1: 3 - 4 + andq %r15, %rbx + addq %rax, %r10 + rorxq $28, %r11, %rax + rorxq $34, %r11, %rcx + xorq %r9, %rbx + xorq %rax, %rcx + vpxor %xmm10, %xmm8, %xmm8 + vpsrlq $6, %xmm5, %xmm11 + # rnd_1: 5 - 6 + rorxq $39, %r11, %rax + addq %rbx, %r10 + xorq %rcx, %rax + movq %r12, %rbx + addq %r10, %r14 + xorq %r11, %rbx + vpxor %xmm11, %xmm8, %xmm8 + # rnd_1: 7 - 7 + andq %rbx, %rdx + addq %rax, %r10 + xorq %r12, %rdx + vpaddq %xmm6, %xmm8, %xmm6 + # msg_sched done: 12-15 + # msg_sched: 14-15 + # rnd_0: 0 - 0 + rorxq $14, %r14, %rax + rorxq $18, %r14, %rcx + addq %rdx, %r10 + vpalignr $8, %xmm7, %xmm0, %xmm12 + vpalignr $8, %xmm3, %xmm4, %xmm13 + # rnd_0: 1 - 1 + addq 112(%rsp), %r9 + movq %r15, %rdx + xorq %rax, %rcx + vpsrlq $0x01, %xmm12, %xmm8 + vpsllq $63, %xmm12, %xmm9 + # rnd_0: 2 - 2 + xorq %r8, %rdx + rorxq $41, %r14, %rax + xorq %rcx, %rax + vpsrlq $8, %xmm12, %xmm10 + vpsllq $56, %xmm12, %xmm11 + # rnd_0: 3 - 3 + andq %r14, %rdx + addq %rax, %r9 + rorxq $28, %r10, %rax + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_0: 4 - 4 + rorxq $34, %r10, %rcx + xorq %r8, %rdx + xorq %rax, %rcx + vpsrlq $7, %xmm12, %xmm11 + vpxor %xmm10, %xmm8, %xmm8 + # rnd_0: 5 - 5 + rorxq $39, %r10, %rax + addq %rdx, %r9 + xorq %rcx, %rax + vpxor %xmm11, %xmm8, %xmm8 + vpaddq %xmm7, %xmm13, %xmm7 + # rnd_0: 6 - 7 + movq %r11, %rdx + addq %r9, %r13 + xorq %r10, %rdx + andq %rdx, %rbx + addq %rax, %r9 + xorq %r11, %rbx + vpaddq %xmm7, %xmm8, %xmm7 + # rnd_1: 0 - 0 + rorxq $14, %r13, %rax + rorxq $18, %r13, %rcx + addq %rbx, %r9 + vpsrlq $19, %xmm6, %xmm8 + vpsllq $45, %xmm6, %xmm9 + # rnd_1: 1 - 1 + addq 120(%rsp), %r8 + movq %r14, %rbx + xorq %rax, %rcx + vpsrlq $61, %xmm6, %xmm10 + vpsllq $3, %xmm6, %xmm11 + # rnd_1: 2 - 2 + xorq %r15, %rbx + rorxq $41, %r13, %rax + xorq %rcx, %rax + vpor %xmm9, %xmm8, %xmm8 + vpor %xmm11, %xmm10, %xmm10 + # rnd_1: 3 - 4 + andq %r13, %rbx + addq %rax, %r8 + rorxq $28, %r9, %rax + rorxq $34, %r9, %rcx + xorq %r15, %rbx + xorq %rax, %rcx + vpxor %xmm10, %xmm8, %xmm8 + vpsrlq $6, %xmm6, %xmm11 + # rnd_1: 5 - 6 + rorxq $39, %r9, %rax + addq %rbx, %r8 + xorq %rcx, %rax + movq %r10, %rbx + addq %r8, %r12 + xorq %r9, %rbx + vpxor %xmm11, %xmm8, %xmm8 + # rnd_1: 7 - 7 + andq %rbx, %rdx + addq %rax, %r8 + xorq %r10, %rdx + vpaddq %xmm7, %xmm8, %xmm7 + # msg_sched done: 14-17 + movq 136(%rsp), %rcx + vpaddq (%rcx), %xmm0, %xmm8 + vpaddq 16(%rcx), %xmm1, %xmm9 + vmovdqu %xmm8, (%rsp) + vmovdqu %xmm9, 16(%rsp) + vpaddq 32(%rcx), %xmm2, %xmm8 + vpaddq 48(%rcx), %xmm3, %xmm9 + vmovdqu %xmm8, 32(%rsp) + vmovdqu %xmm9, 48(%rsp) + vpaddq 64(%rcx), %xmm4, %xmm8 + vpaddq 80(%rcx), %xmm5, %xmm9 + vmovdqu %xmm8, 64(%rsp) + vmovdqu %xmm9, 80(%rsp) + vpaddq 96(%rcx), %xmm6, %xmm8 + vpaddq 112(%rcx), %xmm7, %xmm9 + vmovdqu %xmm8, 96(%rsp) + vmovdqu %xmm9, 112(%rsp) + subl $0x01, 128(%rsp) + jne L_sha512_len_avx1_rorx_start + vpaddq (%rcx), %xmm0, %xmm8 + vpaddq 16(%rcx), %xmm1, %xmm9 + vmovdqu %xmm8, (%rsp) + vmovdqu %xmm9, 16(%rsp) + vpaddq 32(%rcx), %xmm2, %xmm8 + vpaddq 48(%rcx), %xmm3, %xmm9 + vmovdqu %xmm8, 32(%rsp) + vmovdqu %xmm9, 48(%rsp) + vpaddq 64(%rcx), %xmm4, %xmm8 + vpaddq 80(%rcx), %xmm5, %xmm9 + vmovdqu %xmm8, 64(%rsp) + vmovdqu %xmm9, 80(%rsp) + vpaddq 96(%rcx), %xmm6, %xmm8 + vpaddq 112(%rcx), %xmm7, %xmm9 + vmovdqu %xmm8, 96(%rsp) + vmovdqu %xmm9, 112(%rsp) + # rnd_all_2: 0-1 + # rnd_0: 0 - 7 + rorxq $14, %r12, %rax + rorxq $18, %r12, %rcx + addq %rdx, %r8 + addq (%rsp), %r15 + movq %r13, %rdx + xorq %rax, %rcx + xorq %r14, %rdx + rorxq $41, %r12, %rax + xorq %rcx, %rax + andq %r12, %rdx + addq %rax, %r15 + rorxq $28, %r8, %rax + rorxq $34, %r8, %rcx + xorq %r14, %rdx + xorq %rax, %rcx + rorxq $39, %r8, %rax + addq %rdx, %r15 + xorq %rcx, %rax + movq %r9, %rdx + addq %r15, %r11 + xorq %r8, %rdx + andq %rdx, %rbx + addq %rax, %r15 + xorq %r9, %rbx + # rnd_1: 0 - 7 + rorxq $14, %r11, %rax + rorxq $18, %r11, %rcx + addq %rbx, %r15 + addq 8(%rsp), %r14 + movq %r12, %rbx + xorq %rax, %rcx + xorq %r13, %rbx + rorxq $41, %r11, %rax + xorq %rcx, %rax + andq %r11, %rbx + addq %rax, %r14 + rorxq $28, %r15, %rax + rorxq $34, %r15, %rcx + xorq %r13, %rbx + xorq %rax, %rcx + rorxq $39, %r15, %rax + addq %rbx, %r14 + xorq %rcx, %rax + movq %r8, %rbx + addq %r14, %r10 + xorq %r15, %rbx + andq %rbx, %rdx + addq %rax, %r14 + xorq %r8, %rdx + # rnd_all_2: 2-3 + # rnd_0: 0 - 7 + rorxq $14, %r10, %rax + rorxq $18, %r10, %rcx + addq %rdx, %r14 + addq 16(%rsp), %r13 + movq %r11, %rdx + xorq %rax, %rcx + xorq %r12, %rdx + rorxq $41, %r10, %rax + xorq %rcx, %rax + andq %r10, %rdx + addq %rax, %r13 + rorxq $28, %r14, %rax + rorxq $34, %r14, %rcx + xorq %r12, %rdx + xorq %rax, %rcx + rorxq $39, %r14, %rax + addq %rdx, %r13 + xorq %rcx, %rax + movq %r15, %rdx + addq %r13, %r9 + xorq %r14, %rdx + andq %rdx, %rbx + addq %rax, %r13 + xorq %r15, %rbx + # rnd_1: 0 - 7 + rorxq $14, %r9, %rax + rorxq $18, %r9, %rcx + addq %rbx, %r13 + addq 24(%rsp), %r12 + movq %r10, %rbx + xorq %rax, %rcx + xorq %r11, %rbx + rorxq $41, %r9, %rax + xorq %rcx, %rax + andq %r9, %rbx + addq %rax, %r12 + rorxq $28, %r13, %rax + rorxq $34, %r13, %rcx + xorq %r11, %rbx + xorq %rax, %rcx + rorxq $39, %r13, %rax + addq %rbx, %r12 + xorq %rcx, %rax + movq %r14, %rbx + addq %r12, %r8 + xorq %r13, %rbx + andq %rbx, %rdx + addq %rax, %r12 + xorq %r14, %rdx + # rnd_all_2: 4-5 + # rnd_0: 0 - 7 + rorxq $14, %r8, %rax + rorxq $18, %r8, %rcx + addq %rdx, %r12 + addq 32(%rsp), %r11 + movq %r9, %rdx + xorq %rax, %rcx + xorq %r10, %rdx + rorxq $41, %r8, %rax + xorq %rcx, %rax + andq %r8, %rdx + addq %rax, %r11 + rorxq $28, %r12, %rax + rorxq $34, %r12, %rcx + xorq %r10, %rdx + xorq %rax, %rcx + rorxq $39, %r12, %rax + addq %rdx, %r11 + xorq %rcx, %rax + movq %r13, %rdx + addq %r11, %r15 + xorq %r12, %rdx + andq %rdx, %rbx + addq %rax, %r11 + xorq %r13, %rbx + # rnd_1: 0 - 7 + rorxq $14, %r15, %rax + rorxq $18, %r15, %rcx + addq %rbx, %r11 + addq 40(%rsp), %r10 + movq %r8, %rbx + xorq %rax, %rcx + xorq %r9, %rbx + rorxq $41, %r15, %rax + xorq %rcx, %rax + andq %r15, %rbx + addq %rax, %r10 + rorxq $28, %r11, %rax + rorxq $34, %r11, %rcx + xorq %r9, %rbx + xorq %rax, %rcx + rorxq $39, %r11, %rax + addq %rbx, %r10 + xorq %rcx, %rax + movq %r12, %rbx + addq %r10, %r14 + xorq %r11, %rbx + andq %rbx, %rdx + addq %rax, %r10 + xorq %r12, %rdx + # rnd_all_2: 6-7 + # rnd_0: 0 - 7 + rorxq $14, %r14, %rax + rorxq $18, %r14, %rcx + addq %rdx, %r10 + addq 48(%rsp), %r9 + movq %r15, %rdx + xorq %rax, %rcx + xorq %r8, %rdx + rorxq $41, %r14, %rax + xorq %rcx, %rax + andq %r14, %rdx + addq %rax, %r9 + rorxq $28, %r10, %rax + rorxq $34, %r10, %rcx + xorq %r8, %rdx + xorq %rax, %rcx + rorxq $39, %r10, %rax + addq %rdx, %r9 + xorq %rcx, %rax + movq %r11, %rdx + addq %r9, %r13 + xorq %r10, %rdx + andq %rdx, %rbx + addq %rax, %r9 + xorq %r11, %rbx + # rnd_1: 0 - 7 + rorxq $14, %r13, %rax + rorxq $18, %r13, %rcx + addq %rbx, %r9 + addq 56(%rsp), %r8 + movq %r14, %rbx + xorq %rax, %rcx + xorq %r15, %rbx + rorxq $41, %r13, %rax + xorq %rcx, %rax + andq %r13, %rbx + addq %rax, %r8 + rorxq $28, %r9, %rax + rorxq $34, %r9, %rcx + xorq %r15, %rbx + xorq %rax, %rcx + rorxq $39, %r9, %rax + addq %rbx, %r8 + xorq %rcx, %rax + movq %r10, %rbx + addq %r8, %r12 + xorq %r9, %rbx + andq %rbx, %rdx + addq %rax, %r8 + xorq %r10, %rdx + # rnd_all_2: 8-9 + # rnd_0: 0 - 7 + rorxq $14, %r12, %rax + rorxq $18, %r12, %rcx + addq %rdx, %r8 + addq 64(%rsp), %r15 + movq %r13, %rdx + xorq %rax, %rcx + xorq %r14, %rdx + rorxq $41, %r12, %rax + xorq %rcx, %rax + andq %r12, %rdx + addq %rax, %r15 + rorxq $28, %r8, %rax + rorxq $34, %r8, %rcx + xorq %r14, %rdx + xorq %rax, %rcx + rorxq $39, %r8, %rax + addq %rdx, %r15 + xorq %rcx, %rax + movq %r9, %rdx + addq %r15, %r11 + xorq %r8, %rdx + andq %rdx, %rbx + addq %rax, %r15 + xorq %r9, %rbx + # rnd_1: 0 - 7 + rorxq $14, %r11, %rax + rorxq $18, %r11, %rcx + addq %rbx, %r15 + addq 72(%rsp), %r14 + movq %r12, %rbx + xorq %rax, %rcx + xorq %r13, %rbx + rorxq $41, %r11, %rax + xorq %rcx, %rax + andq %r11, %rbx + addq %rax, %r14 + rorxq $28, %r15, %rax + rorxq $34, %r15, %rcx + xorq %r13, %rbx + xorq %rax, %rcx + rorxq $39, %r15, %rax + addq %rbx, %r14 + xorq %rcx, %rax + movq %r8, %rbx + addq %r14, %r10 + xorq %r15, %rbx + andq %rbx, %rdx + addq %rax, %r14 + xorq %r8, %rdx + # rnd_all_2: 10-11 + # rnd_0: 0 - 7 + rorxq $14, %r10, %rax + rorxq $18, %r10, %rcx + addq %rdx, %r14 + addq 80(%rsp), %r13 + movq %r11, %rdx + xorq %rax, %rcx + xorq %r12, %rdx + rorxq $41, %r10, %rax + xorq %rcx, %rax + andq %r10, %rdx + addq %rax, %r13 + rorxq $28, %r14, %rax + rorxq $34, %r14, %rcx + xorq %r12, %rdx + xorq %rax, %rcx + rorxq $39, %r14, %rax + addq %rdx, %r13 + xorq %rcx, %rax + movq %r15, %rdx + addq %r13, %r9 + xorq %r14, %rdx + andq %rdx, %rbx + addq %rax, %r13 + xorq %r15, %rbx + # rnd_1: 0 - 7 + rorxq $14, %r9, %rax + rorxq $18, %r9, %rcx + addq %rbx, %r13 + addq 88(%rsp), %r12 + movq %r10, %rbx + xorq %rax, %rcx + xorq %r11, %rbx + rorxq $41, %r9, %rax + xorq %rcx, %rax + andq %r9, %rbx + addq %rax, %r12 + rorxq $28, %r13, %rax + rorxq $34, %r13, %rcx + xorq %r11, %rbx + xorq %rax, %rcx + rorxq $39, %r13, %rax + addq %rbx, %r12 + xorq %rcx, %rax + movq %r14, %rbx + addq %r12, %r8 + xorq %r13, %rbx + andq %rbx, %rdx + addq %rax, %r12 + xorq %r14, %rdx + # rnd_all_2: 12-13 + # rnd_0: 0 - 7 + rorxq $14, %r8, %rax + rorxq $18, %r8, %rcx + addq %rdx, %r12 + addq 96(%rsp), %r11 + movq %r9, %rdx + xorq %rax, %rcx + xorq %r10, %rdx + rorxq $41, %r8, %rax + xorq %rcx, %rax + andq %r8, %rdx + addq %rax, %r11 + rorxq $28, %r12, %rax + rorxq $34, %r12, %rcx + xorq %r10, %rdx + xorq %rax, %rcx + rorxq $39, %r12, %rax + addq %rdx, %r11 + xorq %rcx, %rax + movq %r13, %rdx + addq %r11, %r15 + xorq %r12, %rdx + andq %rdx, %rbx + addq %rax, %r11 + xorq %r13, %rbx + # rnd_1: 0 - 7 + rorxq $14, %r15, %rax + rorxq $18, %r15, %rcx + addq %rbx, %r11 + addq 104(%rsp), %r10 + movq %r8, %rbx + xorq %rax, %rcx + xorq %r9, %rbx + rorxq $41, %r15, %rax + xorq %rcx, %rax + andq %r15, %rbx + addq %rax, %r10 + rorxq $28, %r11, %rax + rorxq $34, %r11, %rcx + xorq %r9, %rbx + xorq %rax, %rcx + rorxq $39, %r11, %rax + addq %rbx, %r10 + xorq %rcx, %rax + movq %r12, %rbx + addq %r10, %r14 + xorq %r11, %rbx + andq %rbx, %rdx + addq %rax, %r10 + xorq %r12, %rdx + # rnd_all_2: 14-15 + # rnd_0: 0 - 7 + rorxq $14, %r14, %rax + rorxq $18, %r14, %rcx + addq %rdx, %r10 + addq 112(%rsp), %r9 + movq %r15, %rdx + xorq %rax, %rcx + xorq %r8, %rdx + rorxq $41, %r14, %rax + xorq %rcx, %rax + andq %r14, %rdx + addq %rax, %r9 + rorxq $28, %r10, %rax + rorxq $34, %r10, %rcx + xorq %r8, %rdx + xorq %rax, %rcx + rorxq $39, %r10, %rax + addq %rdx, %r9 + xorq %rcx, %rax + movq %r11, %rdx + addq %r9, %r13 + xorq %r10, %rdx + andq %rdx, %rbx + addq %rax, %r9 + xorq %r11, %rbx + # rnd_1: 0 - 7 + rorxq $14, %r13, %rax + rorxq $18, %r13, %rcx + addq %rbx, %r9 + addq 120(%rsp), %r8 + movq %r14, %rbx + xorq %rax, %rcx + xorq %r15, %rbx + rorxq $41, %r13, %rax + xorq %rcx, %rax + andq %r13, %rbx + addq %rax, %r8 + rorxq $28, %r9, %rax + rorxq $34, %r9, %rcx + xorq %r15, %rbx + xorq %rax, %rcx + rorxq $39, %r9, %rax + addq %rbx, %r8 + xorq %rcx, %rax + movq %r10, %rbx + addq %r8, %r12 + xorq %r9, %rbx + andq %rbx, %rdx + addq %rax, %r8 + xorq %r10, %rdx + addq %rdx, %r8 + addq (%rdi), %r8 + addq 8(%rdi), %r9 + addq 16(%rdi), %r10 + addq 24(%rdi), %r11 + addq 32(%rdi), %r12 + addq 40(%rdi), %r13 + addq 48(%rdi), %r14 + addq 56(%rdi), %r15 + leaq L_avx1_rorx_sha512_k(%rip), %rcx + addq $0x80, %rsi + subl $0x80, %ebp + movq %r8, (%rdi) + movq %r9, 8(%rdi) + movq %r10, 16(%rdi) + movq %r11, 24(%rdi) + movq %r12, 32(%rdi) + movq %r13, 40(%rdi) + movq %r14, 48(%rdi) + movq %r15, 56(%rdi) + jnz L_sha512_len_avx1_rorx_begin + xorq %rax, %rax + vzeroupper + addq $0x90, %rsp + popq %rbp + popq %r15 + popq %r14 + popq %r13 + popq %r12 + popq %rbx + repz retq +.size Transform_Sha512_AVX1_RORX_Len,.-Transform_Sha512_AVX1_RORX_Len +#endif /* HAVE_INTEL_AVX1 */ +#ifdef HAVE_INTEL_AVX2 +.align 16 +L_avx2_sha512_k: +.quad 0x428a2f98d728ae22,0x7137449123ef65cd +.quad 0xb5c0fbcfec4d3b2f,0xe9b5dba58189dbbc +.quad 0x3956c25bf348b538,0x59f111f1b605d019 +.quad 0x923f82a4af194f9b,0xab1c5ed5da6d8118 +.quad 0xd807aa98a3030242,0x12835b0145706fbe +.quad 0x243185be4ee4b28c,0x550c7dc3d5ffb4e2 +.quad 0x72be5d74f27b896f,0x80deb1fe3b1696b1 +.quad 0x9bdc06a725c71235,0xc19bf174cf692694 +.quad 0xe49b69c19ef14ad2,0xefbe4786384f25e3 +.quad 0xfc19dc68b8cd5b5,0x240ca1cc77ac9c65 +.quad 0x2de92c6f592b0275,0x4a7484aa6ea6e483 +.quad 0x5cb0a9dcbd41fbd4,0x76f988da831153b5 +.quad 0x983e5152ee66dfab,0xa831c66d2db43210 +.quad 0xb00327c898fb213f,0xbf597fc7beef0ee4 +.quad 0xc6e00bf33da88fc2,0xd5a79147930aa725 +.quad 0x6ca6351e003826f,0x142929670a0e6e70 +.quad 0x27b70a8546d22ffc,0x2e1b21385c26c926 +.quad 0x4d2c6dfc5ac42aed,0x53380d139d95b3df +.quad 0x650a73548baf63de,0x766a0abb3c77b2a8 +.quad 0x81c2c92e47edaee6,0x92722c851482353b +.quad 0xa2bfe8a14cf10364,0xa81a664bbc423001 +.quad 0xc24b8b70d0f89791,0xc76c51a30654be30 +.quad 0xd192e819d6ef5218,0xd69906245565a910 +.quad 0xf40e35855771202a,0x106aa07032bbd1b8 +.quad 0x19a4c116b8d2d0c8,0x1e376c085141ab53 +.quad 0x2748774cdf8eeb99,0x34b0bcb5e19b48a8 +.quad 0x391c0cb3c5c95a63,0x4ed8aa4ae3418acb +.quad 0x5b9cca4f7763e373,0x682e6ff3d6b2b8a3 +.quad 0x748f82ee5defb2fc,0x78a5636f43172f60 +.quad 0x84c87814a1f0ab72,0x8cc702081a6439ec +.quad 0x90befffa23631e28,0xa4506cebde82bde9 +.quad 0xbef9a3f7b2c67915,0xc67178f2e372532b +.quad 0xca273eceea26619c,0xd186b8c721c0c207 +.quad 0xeada7dd6cde0eb1e,0xf57d4f7fee6ed178 +.quad 0x6f067aa72176fba,0xa637dc5a2c898a6 +.quad 0x113f9804bef90dae,0x1b710b35131c471b +.quad 0x28db77f523047d84,0x32caab7b40c72493 +.quad 0x3c9ebe0a15c9bebc,0x431d67c49c100d4c +.quad 0x4cc5d4becb3e42b6,0x597f299cfc657e2a +.quad 0x5fcb6fab3ad6faec,0x6c44198c4a475817 +.align 16 +L_avx2_sha512_k_2: +.quad 0x428a2f98d728ae22,0x7137449123ef65cd +.quad 0x428a2f98d728ae22,0x7137449123ef65cd +.quad 0xb5c0fbcfec4d3b2f,0xe9b5dba58189dbbc +.quad 0xb5c0fbcfec4d3b2f,0xe9b5dba58189dbbc +.quad 0x3956c25bf348b538,0x59f111f1b605d019 +.quad 0x3956c25bf348b538,0x59f111f1b605d019 +.quad 0x923f82a4af194f9b,0xab1c5ed5da6d8118 +.quad 0x923f82a4af194f9b,0xab1c5ed5da6d8118 +.quad 0xd807aa98a3030242,0x12835b0145706fbe +.quad 0xd807aa98a3030242,0x12835b0145706fbe +.quad 0x243185be4ee4b28c,0x550c7dc3d5ffb4e2 +.quad 0x243185be4ee4b28c,0x550c7dc3d5ffb4e2 +.quad 0x72be5d74f27b896f,0x80deb1fe3b1696b1 +.quad 0x72be5d74f27b896f,0x80deb1fe3b1696b1 +.quad 0x9bdc06a725c71235,0xc19bf174cf692694 +.quad 0x9bdc06a725c71235,0xc19bf174cf692694 +.quad 0xe49b69c19ef14ad2,0xefbe4786384f25e3 +.quad 0xe49b69c19ef14ad2,0xefbe4786384f25e3 +.quad 0xfc19dc68b8cd5b5,0x240ca1cc77ac9c65 +.quad 0xfc19dc68b8cd5b5,0x240ca1cc77ac9c65 +.quad 0x2de92c6f592b0275,0x4a7484aa6ea6e483 +.quad 0x2de92c6f592b0275,0x4a7484aa6ea6e483 +.quad 0x5cb0a9dcbd41fbd4,0x76f988da831153b5 +.quad 0x5cb0a9dcbd41fbd4,0x76f988da831153b5 +.quad 0x983e5152ee66dfab,0xa831c66d2db43210 +.quad 0x983e5152ee66dfab,0xa831c66d2db43210 +.quad 0xb00327c898fb213f,0xbf597fc7beef0ee4 +.quad 0xb00327c898fb213f,0xbf597fc7beef0ee4 +.quad 0xc6e00bf33da88fc2,0xd5a79147930aa725 +.quad 0xc6e00bf33da88fc2,0xd5a79147930aa725 +.quad 0x6ca6351e003826f,0x142929670a0e6e70 +.quad 0x6ca6351e003826f,0x142929670a0e6e70 +.quad 0x27b70a8546d22ffc,0x2e1b21385c26c926 +.quad 0x27b70a8546d22ffc,0x2e1b21385c26c926 +.quad 0x4d2c6dfc5ac42aed,0x53380d139d95b3df +.quad 0x4d2c6dfc5ac42aed,0x53380d139d95b3df +.quad 0x650a73548baf63de,0x766a0abb3c77b2a8 +.quad 0x650a73548baf63de,0x766a0abb3c77b2a8 +.quad 0x81c2c92e47edaee6,0x92722c851482353b +.quad 0x81c2c92e47edaee6,0x92722c851482353b +.quad 0xa2bfe8a14cf10364,0xa81a664bbc423001 +.quad 0xa2bfe8a14cf10364,0xa81a664bbc423001 +.quad 0xc24b8b70d0f89791,0xc76c51a30654be30 +.quad 0xc24b8b70d0f89791,0xc76c51a30654be30 +.quad 0xd192e819d6ef5218,0xd69906245565a910 +.quad 0xd192e819d6ef5218,0xd69906245565a910 +.quad 0xf40e35855771202a,0x106aa07032bbd1b8 +.quad 0xf40e35855771202a,0x106aa07032bbd1b8 +.quad 0x19a4c116b8d2d0c8,0x1e376c085141ab53 +.quad 0x19a4c116b8d2d0c8,0x1e376c085141ab53 +.quad 0x2748774cdf8eeb99,0x34b0bcb5e19b48a8 +.quad 0x2748774cdf8eeb99,0x34b0bcb5e19b48a8 +.quad 0x391c0cb3c5c95a63,0x4ed8aa4ae3418acb +.quad 0x391c0cb3c5c95a63,0x4ed8aa4ae3418acb +.quad 0x5b9cca4f7763e373,0x682e6ff3d6b2b8a3 +.quad 0x5b9cca4f7763e373,0x682e6ff3d6b2b8a3 +.quad 0x748f82ee5defb2fc,0x78a5636f43172f60 +.quad 0x748f82ee5defb2fc,0x78a5636f43172f60 +.quad 0x84c87814a1f0ab72,0x8cc702081a6439ec +.quad 0x84c87814a1f0ab72,0x8cc702081a6439ec +.quad 0x90befffa23631e28,0xa4506cebde82bde9 +.quad 0x90befffa23631e28,0xa4506cebde82bde9 +.quad 0xbef9a3f7b2c67915,0xc67178f2e372532b +.quad 0xbef9a3f7b2c67915,0xc67178f2e372532b +.quad 0xca273eceea26619c,0xd186b8c721c0c207 +.quad 0xca273eceea26619c,0xd186b8c721c0c207 +.quad 0xeada7dd6cde0eb1e,0xf57d4f7fee6ed178 +.quad 0xeada7dd6cde0eb1e,0xf57d4f7fee6ed178 +.quad 0x6f067aa72176fba,0xa637dc5a2c898a6 +.quad 0x6f067aa72176fba,0xa637dc5a2c898a6 +.quad 0x113f9804bef90dae,0x1b710b35131c471b +.quad 0x113f9804bef90dae,0x1b710b35131c471b +.quad 0x28db77f523047d84,0x32caab7b40c72493 +.quad 0x28db77f523047d84,0x32caab7b40c72493 +.quad 0x3c9ebe0a15c9bebc,0x431d67c49c100d4c +.quad 0x3c9ebe0a15c9bebc,0x431d67c49c100d4c +.quad 0x4cc5d4becb3e42b6,0x597f299cfc657e2a +.quad 0x4cc5d4becb3e42b6,0x597f299cfc657e2a +.quad 0x5fcb6fab3ad6faec,0x6c44198c4a475817 +.quad 0x5fcb6fab3ad6faec,0x6c44198c4a475817 +.align 8 +L_avx2_sha512_k_2_end: +.quad 1024+L_avx2_sha512_k_2 +.align 32 +L_avx2_sha512_flip_mask: +.quad 0x1020304050607, 0x8090a0b0c0d0e0f +.quad 0x1020304050607, 0x8090a0b0c0d0e0f +.globl Transform_Sha512_AVX2 +.type Transform_Sha512_AVX2,@function +.align 4 +Transform_Sha512_AVX2: + pushq %rbx + pushq %r12 + pushq %r13 + pushq %r14 + pushq %r15 + subq $0x88, %rsp + leaq 64(%rdi), %rax + vmovdqa L_avx2_sha512_flip_mask(%rip), %ymm15 + movq (%rdi), %r8 + movq 8(%rdi), %r9 + movq 16(%rdi), %r10 + movq 24(%rdi), %r11 + movq 32(%rdi), %r12 + movq 40(%rdi), %r13 + movq 48(%rdi), %r14 + movq 56(%rdi), %r15 + vmovdqu (%rax), %ymm0 + vmovdqu 32(%rax), %ymm1 + vpshufb %ymm15, %ymm0, %ymm0 + vpshufb %ymm15, %ymm1, %ymm1 + vmovdqu 64(%rax), %ymm2 + vmovdqu 96(%rax), %ymm3 + vpshufb %ymm15, %ymm2, %ymm2 + vpshufb %ymm15, %ymm3, %ymm3 + movl $4, 128(%rsp) + leaq L_avx2_sha512_k(%rip), %rsi + movq %r9, %rbx + movq %r12, %rax + xorq %r10, %rbx + vpaddq (%rsi), %ymm0, %ymm8 + vpaddq 32(%rsi), %ymm1, %ymm9 + vmovdqu %ymm8, (%rsp) + vmovdqu %ymm9, 32(%rsp) + vpaddq 64(%rsi), %ymm2, %ymm8 + vpaddq 96(%rsi), %ymm3, %ymm9 + vmovdqu %ymm8, 64(%rsp) + vmovdqu %ymm9, 96(%rsp) + # Start of 16 rounds +L_sha256_avx2_start: + addq $0x80, %rsi + rorq $23, %rax + vpblendd $3, %ymm1, %ymm0, %ymm12 + vpblendd $3, %ymm3, %ymm2, %ymm13 + movq %r8, %rdx + movq %r13, %rcx + addq (%rsp), %r15 + xorq %r14, %rcx + xorq %r12, %rax + andq %r12, %rcx + vpermq $57, %ymm12, %ymm12 + rorq $4, %rax + xorq %r14, %rcx + vpermq $57, %ymm13, %ymm13 + xorq %r12, %rax + addq %rcx, %r15 + rorq $14, %rax + xorq %r9, %rdx + vpsrlq $0x01, %ymm12, %ymm8 + addq %rax, %r15 + movq %r8, %rcx + vpsllq $63, %ymm12, %ymm9 + andq %rdx, %rbx + rorq $5, %rcx + vpsrlq $8, %ymm12, %ymm10 + xorq %r8, %rcx + xorq %r9, %rbx + vpsllq $56, %ymm12, %ymm11 + rorq $6, %rcx + addq %r15, %r11 + vpor %ymm9, %ymm8, %ymm8 + xorq %r8, %rcx + addq %rbx, %r15 + vpor %ymm11, %ymm10, %ymm10 + rorq $28, %rcx + movq %r11, %rax + addq %rcx, %r15 + rorq $23, %rax + vpsrlq $7, %ymm12, %ymm11 + movq %r15, %rbx + movq %r12, %rcx + vpxor %ymm10, %ymm8, %ymm8 + addq 8(%rsp), %r14 + xorq %r13, %rcx + vpxor %ymm11, %ymm8, %ymm8 + xorq %r11, %rax + andq %r11, %rcx + vpaddq %ymm0, %ymm13, %ymm0 + rorq $4, %rax + xorq %r13, %rcx + vpaddq %ymm0, %ymm8, %ymm0 + xorq %r11, %rax + addq %rcx, %r14 + vperm2I128 $0x81, %ymm3, %ymm3, %ymm14 + rorq $14, %rax + xorq %r8, %rbx + addq %rax, %r14 + movq %r15, %rcx + andq %rbx, %rdx + rorq $5, %rcx + vpsrlq $19, %ymm14, %ymm8 + xorq %r15, %rcx + xorq %r8, %rdx + vpsllq $45, %ymm14, %ymm9 + rorq $6, %rcx + addq %r14, %r10 + vpsrlq $61, %ymm14, %ymm10 + xorq %r15, %rcx + addq %rdx, %r14 + vpsllq $3, %ymm14, %ymm11 + rorq $28, %rcx + movq %r10, %rax + addq %rcx, %r14 + rorq $23, %rax + vpor %ymm9, %ymm8, %ymm8 + movq %r14, %rdx + movq %r11, %rcx + addq 16(%rsp), %r13 + xorq %r12, %rcx + vpor %ymm11, %ymm10, %ymm10 + xorq %r10, %rax + andq %r10, %rcx + vpxor %ymm10, %ymm8, %ymm8 + rorq $4, %rax + xorq %r12, %rcx + vpsrlq $6, %ymm14, %ymm11 + xorq %r10, %rax + addq %rcx, %r13 + vpxor %ymm11, %ymm8, %ymm8 + rorq $14, %rax + xorq %r15, %rdx + vpaddq %ymm0, %ymm8, %ymm0 + addq %rax, %r13 + movq %r14, %rcx + andq %rdx, %rbx + rorq $5, %rcx + vperm2I128 $8, %ymm0, %ymm0, %ymm14 + xorq %r14, %rcx + xorq %r15, %rbx + rorq $6, %rcx + addq %r13, %r9 + vpsrlq $19, %ymm14, %ymm8 + xorq %r14, %rcx + addq %rbx, %r13 + vpsllq $45, %ymm14, %ymm9 + rorq $28, %rcx + movq %r9, %rax + addq %rcx, %r13 + rorq $23, %rax + vpsrlq $61, %ymm14, %ymm10 + movq %r13, %rbx + movq %r10, %rcx + addq 24(%rsp), %r12 + xorq %r11, %rcx + vpsllq $3, %ymm14, %ymm11 + xorq %r9, %rax + andq %r9, %rcx + vpor %ymm9, %ymm8, %ymm8 + rorq $4, %rax + xorq %r11, %rcx + vpor %ymm11, %ymm10, %ymm10 + xorq %r9, %rax + addq %rcx, %r12 + vpxor %ymm10, %ymm8, %ymm8 + rorq $14, %rax + xorq %r14, %rbx + vpsrlq $6, %ymm14, %ymm11 + addq %rax, %r12 + movq %r13, %rcx + vpxor %ymm11, %ymm8, %ymm8 + andq %rbx, %rdx + rorq $5, %rcx + vpaddq %ymm0, %ymm8, %ymm0 + xorq %r13, %rcx + xorq %r14, %rdx + rorq $6, %rcx + addq %r12, %r8 + xorq %r13, %rcx + addq %rdx, %r12 + rorq $28, %rcx + movq %r8, %rax + addq %rcx, %r12 + rorq $23, %rax + vpblendd $3, %ymm2, %ymm1, %ymm12 + vpblendd $3, %ymm0, %ymm3, %ymm13 + movq %r12, %rdx + movq %r9, %rcx + addq 32(%rsp), %r11 + xorq %r10, %rcx + xorq %r8, %rax + andq %r8, %rcx + vpermq $57, %ymm12, %ymm12 + rorq $4, %rax + xorq %r10, %rcx + vpermq $57, %ymm13, %ymm13 + xorq %r8, %rax + addq %rcx, %r11 + rorq $14, %rax + xorq %r13, %rdx + vpsrlq $0x01, %ymm12, %ymm8 + addq %rax, %r11 + movq %r12, %rcx + vpsllq $63, %ymm12, %ymm9 + andq %rdx, %rbx + rorq $5, %rcx + vpsrlq $8, %ymm12, %ymm10 + xorq %r12, %rcx + xorq %r13, %rbx + vpsllq $56, %ymm12, %ymm11 + rorq $6, %rcx + addq %r11, %r15 + vpor %ymm9, %ymm8, %ymm8 + xorq %r12, %rcx + addq %rbx, %r11 + vpor %ymm11, %ymm10, %ymm10 + rorq $28, %rcx + movq %r15, %rax + addq %rcx, %r11 + rorq $23, %rax + vpsrlq $7, %ymm12, %ymm11 + movq %r11, %rbx + movq %r8, %rcx + vpxor %ymm10, %ymm8, %ymm8 + addq 40(%rsp), %r10 + xorq %r9, %rcx + vpxor %ymm11, %ymm8, %ymm8 + xorq %r15, %rax + andq %r15, %rcx + vpaddq %ymm1, %ymm13, %ymm1 + rorq $4, %rax + xorq %r9, %rcx + vpaddq %ymm1, %ymm8, %ymm1 + xorq %r15, %rax + addq %rcx, %r10 + vperm2I128 $0x81, %ymm0, %ymm0, %ymm14 + rorq $14, %rax + xorq %r12, %rbx + addq %rax, %r10 + movq %r11, %rcx + andq %rbx, %rdx + rorq $5, %rcx + vpsrlq $19, %ymm14, %ymm8 + xorq %r11, %rcx + xorq %r12, %rdx + vpsllq $45, %ymm14, %ymm9 + rorq $6, %rcx + addq %r10, %r14 + vpsrlq $61, %ymm14, %ymm10 + xorq %r11, %rcx + addq %rdx, %r10 + vpsllq $3, %ymm14, %ymm11 + rorq $28, %rcx + movq %r14, %rax + addq %rcx, %r10 + rorq $23, %rax + vpor %ymm9, %ymm8, %ymm8 + movq %r10, %rdx + movq %r15, %rcx + addq 48(%rsp), %r9 + xorq %r8, %rcx + vpor %ymm11, %ymm10, %ymm10 + xorq %r14, %rax + andq %r14, %rcx + vpxor %ymm10, %ymm8, %ymm8 + rorq $4, %rax + xorq %r8, %rcx + vpsrlq $6, %ymm14, %ymm11 + xorq %r14, %rax + addq %rcx, %r9 + vpxor %ymm11, %ymm8, %ymm8 + rorq $14, %rax + xorq %r11, %rdx + vpaddq %ymm1, %ymm8, %ymm1 + addq %rax, %r9 + movq %r10, %rcx + andq %rdx, %rbx + rorq $5, %rcx + vperm2I128 $8, %ymm1, %ymm1, %ymm14 + xorq %r10, %rcx + xorq %r11, %rbx + rorq $6, %rcx + addq %r9, %r13 + vpsrlq $19, %ymm14, %ymm8 + xorq %r10, %rcx + addq %rbx, %r9 + vpsllq $45, %ymm14, %ymm9 + rorq $28, %rcx + movq %r13, %rax + addq %rcx, %r9 + rorq $23, %rax + vpsrlq $61, %ymm14, %ymm10 + movq %r9, %rbx + movq %r14, %rcx + addq 56(%rsp), %r8 + xorq %r15, %rcx + vpsllq $3, %ymm14, %ymm11 + xorq %r13, %rax + andq %r13, %rcx + vpor %ymm9, %ymm8, %ymm8 + rorq $4, %rax + xorq %r15, %rcx + vpor %ymm11, %ymm10, %ymm10 + xorq %r13, %rax + addq %rcx, %r8 + vpxor %ymm10, %ymm8, %ymm8 + rorq $14, %rax + xorq %r10, %rbx + vpsrlq $6, %ymm14, %ymm11 + addq %rax, %r8 + movq %r9, %rcx + vpxor %ymm11, %ymm8, %ymm8 + andq %rbx, %rdx + rorq $5, %rcx + vpaddq %ymm1, %ymm8, %ymm1 + xorq %r9, %rcx + xorq %r10, %rdx + rorq $6, %rcx + addq %r8, %r12 + xorq %r9, %rcx + addq %rdx, %r8 + rorq $28, %rcx + movq %r12, %rax + addq %rcx, %r8 + rorq $23, %rax + vpblendd $3, %ymm3, %ymm2, %ymm12 + vpblendd $3, %ymm1, %ymm0, %ymm13 + movq %r8, %rdx + movq %r13, %rcx + addq 64(%rsp), %r15 + xorq %r14, %rcx + xorq %r12, %rax + andq %r12, %rcx + vpermq $57, %ymm12, %ymm12 + rorq $4, %rax + xorq %r14, %rcx + vpermq $57, %ymm13, %ymm13 + xorq %r12, %rax + addq %rcx, %r15 + rorq $14, %rax + xorq %r9, %rdx + vpsrlq $0x01, %ymm12, %ymm8 + addq %rax, %r15 + movq %r8, %rcx + vpsllq $63, %ymm12, %ymm9 + andq %rdx, %rbx + rorq $5, %rcx + vpsrlq $8, %ymm12, %ymm10 + xorq %r8, %rcx + xorq %r9, %rbx + vpsllq $56, %ymm12, %ymm11 + rorq $6, %rcx + addq %r15, %r11 + vpor %ymm9, %ymm8, %ymm8 + xorq %r8, %rcx + addq %rbx, %r15 + vpor %ymm11, %ymm10, %ymm10 + rorq $28, %rcx + movq %r11, %rax + addq %rcx, %r15 + rorq $23, %rax + vpsrlq $7, %ymm12, %ymm11 + movq %r15, %rbx + movq %r12, %rcx + vpxor %ymm10, %ymm8, %ymm8 + addq 72(%rsp), %r14 + xorq %r13, %rcx + vpxor %ymm11, %ymm8, %ymm8 + xorq %r11, %rax + andq %r11, %rcx + vpaddq %ymm2, %ymm13, %ymm2 + rorq $4, %rax + xorq %r13, %rcx + vpaddq %ymm2, %ymm8, %ymm2 + xorq %r11, %rax + addq %rcx, %r14 + vperm2I128 $0x81, %ymm1, %ymm1, %ymm14 + rorq $14, %rax + xorq %r8, %rbx + addq %rax, %r14 + movq %r15, %rcx + andq %rbx, %rdx + rorq $5, %rcx + vpsrlq $19, %ymm14, %ymm8 + xorq %r15, %rcx + xorq %r8, %rdx + vpsllq $45, %ymm14, %ymm9 + rorq $6, %rcx + addq %r14, %r10 + vpsrlq $61, %ymm14, %ymm10 + xorq %r15, %rcx + addq %rdx, %r14 + vpsllq $3, %ymm14, %ymm11 + rorq $28, %rcx + movq %r10, %rax + addq %rcx, %r14 + rorq $23, %rax + vpor %ymm9, %ymm8, %ymm8 + movq %r14, %rdx + movq %r11, %rcx + addq 80(%rsp), %r13 + xorq %r12, %rcx + vpor %ymm11, %ymm10, %ymm10 + xorq %r10, %rax + andq %r10, %rcx + vpxor %ymm10, %ymm8, %ymm8 + rorq $4, %rax + xorq %r12, %rcx + vpsrlq $6, %ymm14, %ymm11 + xorq %r10, %rax + addq %rcx, %r13 + vpxor %ymm11, %ymm8, %ymm8 + rorq $14, %rax + xorq %r15, %rdx + vpaddq %ymm2, %ymm8, %ymm2 + addq %rax, %r13 + movq %r14, %rcx + andq %rdx, %rbx + rorq $5, %rcx + vperm2I128 $8, %ymm2, %ymm2, %ymm14 + xorq %r14, %rcx + xorq %r15, %rbx + rorq $6, %rcx + addq %r13, %r9 + vpsrlq $19, %ymm14, %ymm8 + xorq %r14, %rcx + addq %rbx, %r13 + vpsllq $45, %ymm14, %ymm9 + rorq $28, %rcx + movq %r9, %rax + addq %rcx, %r13 + rorq $23, %rax + vpsrlq $61, %ymm14, %ymm10 + movq %r13, %rbx + movq %r10, %rcx + addq 88(%rsp), %r12 + xorq %r11, %rcx + vpsllq $3, %ymm14, %ymm11 + xorq %r9, %rax + andq %r9, %rcx + vpor %ymm9, %ymm8, %ymm8 + rorq $4, %rax + xorq %r11, %rcx + vpor %ymm11, %ymm10, %ymm10 + xorq %r9, %rax + addq %rcx, %r12 + vpxor %ymm10, %ymm8, %ymm8 + rorq $14, %rax + xorq %r14, %rbx + vpsrlq $6, %ymm14, %ymm11 + addq %rax, %r12 + movq %r13, %rcx + vpxor %ymm11, %ymm8, %ymm8 + andq %rbx, %rdx + rorq $5, %rcx + vpaddq %ymm2, %ymm8, %ymm2 + xorq %r13, %rcx + xorq %r14, %rdx + rorq $6, %rcx + addq %r12, %r8 + xorq %r13, %rcx + addq %rdx, %r12 + rorq $28, %rcx + movq %r8, %rax + addq %rcx, %r12 + rorq $23, %rax + vpblendd $3, %ymm0, %ymm3, %ymm12 + vpblendd $3, %ymm2, %ymm1, %ymm13 + movq %r12, %rdx + movq %r9, %rcx + addq 96(%rsp), %r11 + xorq %r10, %rcx + xorq %r8, %rax + andq %r8, %rcx + vpermq $57, %ymm12, %ymm12 + rorq $4, %rax + xorq %r10, %rcx + vpermq $57, %ymm13, %ymm13 + xorq %r8, %rax + addq %rcx, %r11 + rorq $14, %rax + xorq %r13, %rdx + vpsrlq $0x01, %ymm12, %ymm8 + addq %rax, %r11 + movq %r12, %rcx + vpsllq $63, %ymm12, %ymm9 + andq %rdx, %rbx + rorq $5, %rcx + vpsrlq $8, %ymm12, %ymm10 + xorq %r12, %rcx + xorq %r13, %rbx + vpsllq $56, %ymm12, %ymm11 + rorq $6, %rcx + addq %r11, %r15 + vpor %ymm9, %ymm8, %ymm8 + xorq %r12, %rcx + addq %rbx, %r11 + vpor %ymm11, %ymm10, %ymm10 + rorq $28, %rcx + movq %r15, %rax + addq %rcx, %r11 + rorq $23, %rax + vpsrlq $7, %ymm12, %ymm11 + movq %r11, %rbx + movq %r8, %rcx + vpxor %ymm10, %ymm8, %ymm8 + addq 104(%rsp), %r10 + xorq %r9, %rcx + vpxor %ymm11, %ymm8, %ymm8 + xorq %r15, %rax + andq %r15, %rcx + vpaddq %ymm3, %ymm13, %ymm3 + rorq $4, %rax + xorq %r9, %rcx + vpaddq %ymm3, %ymm8, %ymm3 + xorq %r15, %rax + addq %rcx, %r10 + vperm2I128 $0x81, %ymm2, %ymm2, %ymm14 + rorq $14, %rax + xorq %r12, %rbx + addq %rax, %r10 + movq %r11, %rcx + andq %rbx, %rdx + rorq $5, %rcx + vpsrlq $19, %ymm14, %ymm8 + xorq %r11, %rcx + xorq %r12, %rdx + vpsllq $45, %ymm14, %ymm9 + rorq $6, %rcx + addq %r10, %r14 + vpsrlq $61, %ymm14, %ymm10 + xorq %r11, %rcx + addq %rdx, %r10 + vpsllq $3, %ymm14, %ymm11 + rorq $28, %rcx + movq %r14, %rax + addq %rcx, %r10 + rorq $23, %rax + vpor %ymm9, %ymm8, %ymm8 + movq %r10, %rdx + movq %r15, %rcx + addq 112(%rsp), %r9 + xorq %r8, %rcx + vpor %ymm11, %ymm10, %ymm10 + xorq %r14, %rax + andq %r14, %rcx + vpxor %ymm10, %ymm8, %ymm8 + rorq $4, %rax + xorq %r8, %rcx + vpsrlq $6, %ymm14, %ymm11 + xorq %r14, %rax + addq %rcx, %r9 + vpxor %ymm11, %ymm8, %ymm8 + rorq $14, %rax + xorq %r11, %rdx + vpaddq %ymm3, %ymm8, %ymm3 + addq %rax, %r9 + movq %r10, %rcx + andq %rdx, %rbx + rorq $5, %rcx + vperm2I128 $8, %ymm3, %ymm3, %ymm14 + xorq %r10, %rcx + xorq %r11, %rbx + rorq $6, %rcx + addq %r9, %r13 + vpsrlq $19, %ymm14, %ymm8 + xorq %r10, %rcx + addq %rbx, %r9 + vpsllq $45, %ymm14, %ymm9 + rorq $28, %rcx + movq %r13, %rax + addq %rcx, %r9 + rorq $23, %rax + vpsrlq $61, %ymm14, %ymm10 + movq %r9, %rbx + movq %r14, %rcx + addq 120(%rsp), %r8 + xorq %r15, %rcx + vpsllq $3, %ymm14, %ymm11 + xorq %r13, %rax + andq %r13, %rcx + vpor %ymm9, %ymm8, %ymm8 + rorq $4, %rax + xorq %r15, %rcx + vpor %ymm11, %ymm10, %ymm10 + xorq %r13, %rax + addq %rcx, %r8 + vpxor %ymm10, %ymm8, %ymm8 + rorq $14, %rax + xorq %r10, %rbx + vpsrlq $6, %ymm14, %ymm11 + addq %rax, %r8 + movq %r9, %rcx + vpxor %ymm11, %ymm8, %ymm8 + andq %rbx, %rdx + rorq $5, %rcx + vpaddq %ymm3, %ymm8, %ymm3 + xorq %r9, %rcx + xorq %r10, %rdx + rorq $6, %rcx + addq %r8, %r12 + xorq %r9, %rcx + addq %rdx, %r8 + rorq $28, %rcx + movq %r12, %rax + addq %rcx, %r8 + vpaddq (%rsi), %ymm0, %ymm8 + vpaddq 32(%rsi), %ymm1, %ymm9 + vmovdqu %ymm8, (%rsp) + vmovdqu %ymm9, 32(%rsp) + vpaddq 64(%rsi), %ymm2, %ymm8 + vpaddq 96(%rsi), %ymm3, %ymm9 + vmovdqu %ymm8, 64(%rsp) + vmovdqu %ymm9, 96(%rsp) + subl $0x01, 128(%rsp) + jne L_sha256_avx2_start + rorq $23, %rax + movq %r8, %rdx + movq %r13, %rcx + addq (%rsp), %r15 + xorq %r14, %rcx + xorq %r12, %rax + andq %r12, %rcx + rorq $4, %rax + xorq %r14, %rcx + xorq %r12, %rax + addq %rcx, %r15 + rorq $14, %rax + xorq %r9, %rdx + addq %rax, %r15 + movq %r8, %rcx + andq %rdx, %rbx + rorq $5, %rcx + xorq %r8, %rcx + xorq %r9, %rbx + rorq $6, %rcx + addq %r15, %r11 + xorq %r8, %rcx + addq %rbx, %r15 + rorq $28, %rcx + movq %r11, %rax + addq %rcx, %r15 + rorq $23, %rax + movq %r15, %rbx + movq %r12, %rcx + addq 8(%rsp), %r14 + xorq %r13, %rcx + xorq %r11, %rax + andq %r11, %rcx + rorq $4, %rax + xorq %r13, %rcx + xorq %r11, %rax + addq %rcx, %r14 + rorq $14, %rax + xorq %r8, %rbx + addq %rax, %r14 + movq %r15, %rcx + andq %rbx, %rdx + rorq $5, %rcx + xorq %r15, %rcx + xorq %r8, %rdx + rorq $6, %rcx + addq %r14, %r10 + xorq %r15, %rcx + addq %rdx, %r14 + rorq $28, %rcx + movq %r10, %rax + addq %rcx, %r14 + rorq $23, %rax + movq %r14, %rdx + movq %r11, %rcx + addq 16(%rsp), %r13 + xorq %r12, %rcx + xorq %r10, %rax + andq %r10, %rcx + rorq $4, %rax + xorq %r12, %rcx + xorq %r10, %rax + addq %rcx, %r13 + rorq $14, %rax + xorq %r15, %rdx + addq %rax, %r13 + movq %r14, %rcx + andq %rdx, %rbx + rorq $5, %rcx + xorq %r14, %rcx + xorq %r15, %rbx + rorq $6, %rcx + addq %r13, %r9 + xorq %r14, %rcx + addq %rbx, %r13 + rorq $28, %rcx + movq %r9, %rax + addq %rcx, %r13 + rorq $23, %rax + movq %r13, %rbx + movq %r10, %rcx + addq 24(%rsp), %r12 + xorq %r11, %rcx + xorq %r9, %rax + andq %r9, %rcx + rorq $4, %rax + xorq %r11, %rcx + xorq %r9, %rax + addq %rcx, %r12 + rorq $14, %rax + xorq %r14, %rbx + addq %rax, %r12 + movq %r13, %rcx + andq %rbx, %rdx + rorq $5, %rcx + xorq %r13, %rcx + xorq %r14, %rdx + rorq $6, %rcx + addq %r12, %r8 + xorq %r13, %rcx + addq %rdx, %r12 + rorq $28, %rcx + movq %r8, %rax + addq %rcx, %r12 + rorq $23, %rax + movq %r12, %rdx + movq %r9, %rcx + addq 32(%rsp), %r11 + xorq %r10, %rcx + xorq %r8, %rax + andq %r8, %rcx + rorq $4, %rax + xorq %r10, %rcx + xorq %r8, %rax + addq %rcx, %r11 + rorq $14, %rax + xorq %r13, %rdx + addq %rax, %r11 + movq %r12, %rcx + andq %rdx, %rbx + rorq $5, %rcx + xorq %r12, %rcx + xorq %r13, %rbx + rorq $6, %rcx + addq %r11, %r15 + xorq %r12, %rcx + addq %rbx, %r11 + rorq $28, %rcx + movq %r15, %rax + addq %rcx, %r11 + rorq $23, %rax + movq %r11, %rbx + movq %r8, %rcx + addq 40(%rsp), %r10 + xorq %r9, %rcx + xorq %r15, %rax + andq %r15, %rcx + rorq $4, %rax + xorq %r9, %rcx + xorq %r15, %rax + addq %rcx, %r10 + rorq $14, %rax + xorq %r12, %rbx + addq %rax, %r10 + movq %r11, %rcx + andq %rbx, %rdx + rorq $5, %rcx + xorq %r11, %rcx + xorq %r12, %rdx + rorq $6, %rcx + addq %r10, %r14 + xorq %r11, %rcx + addq %rdx, %r10 + rorq $28, %rcx + movq %r14, %rax + addq %rcx, %r10 + rorq $23, %rax + movq %r10, %rdx + movq %r15, %rcx + addq 48(%rsp), %r9 + xorq %r8, %rcx + xorq %r14, %rax + andq %r14, %rcx + rorq $4, %rax + xorq %r8, %rcx + xorq %r14, %rax + addq %rcx, %r9 + rorq $14, %rax + xorq %r11, %rdx + addq %rax, %r9 + movq %r10, %rcx + andq %rdx, %rbx + rorq $5, %rcx + xorq %r10, %rcx + xorq %r11, %rbx + rorq $6, %rcx + addq %r9, %r13 + xorq %r10, %rcx + addq %rbx, %r9 + rorq $28, %rcx + movq %r13, %rax + addq %rcx, %r9 + rorq $23, %rax + movq %r9, %rbx + movq %r14, %rcx + addq 56(%rsp), %r8 + xorq %r15, %rcx + xorq %r13, %rax + andq %r13, %rcx + rorq $4, %rax + xorq %r15, %rcx + xorq %r13, %rax + addq %rcx, %r8 + rorq $14, %rax + xorq %r10, %rbx + addq %rax, %r8 + movq %r9, %rcx + andq %rbx, %rdx + rorq $5, %rcx + xorq %r9, %rcx + xorq %r10, %rdx + rorq $6, %rcx + addq %r8, %r12 + xorq %r9, %rcx + addq %rdx, %r8 + rorq $28, %rcx + movq %r12, %rax + addq %rcx, %r8 + rorq $23, %rax + movq %r8, %rdx + movq %r13, %rcx + addq 64(%rsp), %r15 + xorq %r14, %rcx + xorq %r12, %rax + andq %r12, %rcx + rorq $4, %rax + xorq %r14, %rcx + xorq %r12, %rax + addq %rcx, %r15 + rorq $14, %rax + xorq %r9, %rdx + addq %rax, %r15 + movq %r8, %rcx + andq %rdx, %rbx + rorq $5, %rcx + xorq %r8, %rcx + xorq %r9, %rbx + rorq $6, %rcx + addq %r15, %r11 + xorq %r8, %rcx + addq %rbx, %r15 + rorq $28, %rcx + movq %r11, %rax + addq %rcx, %r15 + rorq $23, %rax + movq %r15, %rbx + movq %r12, %rcx + addq 72(%rsp), %r14 + xorq %r13, %rcx + xorq %r11, %rax + andq %r11, %rcx + rorq $4, %rax + xorq %r13, %rcx + xorq %r11, %rax + addq %rcx, %r14 + rorq $14, %rax + xorq %r8, %rbx + addq %rax, %r14 + movq %r15, %rcx + andq %rbx, %rdx + rorq $5, %rcx + xorq %r15, %rcx + xorq %r8, %rdx + rorq $6, %rcx + addq %r14, %r10 + xorq %r15, %rcx + addq %rdx, %r14 + rorq $28, %rcx + movq %r10, %rax + addq %rcx, %r14 + rorq $23, %rax + movq %r14, %rdx + movq %r11, %rcx + addq 80(%rsp), %r13 + xorq %r12, %rcx + xorq %r10, %rax + andq %r10, %rcx + rorq $4, %rax + xorq %r12, %rcx + xorq %r10, %rax + addq %rcx, %r13 + rorq $14, %rax + xorq %r15, %rdx + addq %rax, %r13 + movq %r14, %rcx + andq %rdx, %rbx + rorq $5, %rcx + xorq %r14, %rcx + xorq %r15, %rbx + rorq $6, %rcx + addq %r13, %r9 + xorq %r14, %rcx + addq %rbx, %r13 + rorq $28, %rcx + movq %r9, %rax + addq %rcx, %r13 + rorq $23, %rax + movq %r13, %rbx + movq %r10, %rcx + addq 88(%rsp), %r12 + xorq %r11, %rcx + xorq %r9, %rax + andq %r9, %rcx + rorq $4, %rax + xorq %r11, %rcx + xorq %r9, %rax + addq %rcx, %r12 + rorq $14, %rax + xorq %r14, %rbx + addq %rax, %r12 + movq %r13, %rcx + andq %rbx, %rdx + rorq $5, %rcx + xorq %r13, %rcx + xorq %r14, %rdx + rorq $6, %rcx + addq %r12, %r8 + xorq %r13, %rcx + addq %rdx, %r12 + rorq $28, %rcx + movq %r8, %rax + addq %rcx, %r12 + rorq $23, %rax + movq %r12, %rdx + movq %r9, %rcx + addq 96(%rsp), %r11 + xorq %r10, %rcx + xorq %r8, %rax + andq %r8, %rcx + rorq $4, %rax + xorq %r10, %rcx + xorq %r8, %rax + addq %rcx, %r11 + rorq $14, %rax + xorq %r13, %rdx + addq %rax, %r11 + movq %r12, %rcx + andq %rdx, %rbx + rorq $5, %rcx + xorq %r12, %rcx + xorq %r13, %rbx + rorq $6, %rcx + addq %r11, %r15 + xorq %r12, %rcx + addq %rbx, %r11 + rorq $28, %rcx + movq %r15, %rax + addq %rcx, %r11 + rorq $23, %rax + movq %r11, %rbx + movq %r8, %rcx + addq 104(%rsp), %r10 + xorq %r9, %rcx + xorq %r15, %rax + andq %r15, %rcx + rorq $4, %rax + xorq %r9, %rcx + xorq %r15, %rax + addq %rcx, %r10 + rorq $14, %rax + xorq %r12, %rbx + addq %rax, %r10 + movq %r11, %rcx + andq %rbx, %rdx + rorq $5, %rcx + xorq %r11, %rcx + xorq %r12, %rdx + rorq $6, %rcx + addq %r10, %r14 + xorq %r11, %rcx + addq %rdx, %r10 + rorq $28, %rcx + movq %r14, %rax + addq %rcx, %r10 + rorq $23, %rax + movq %r10, %rdx + movq %r15, %rcx + addq 112(%rsp), %r9 + xorq %r8, %rcx + xorq %r14, %rax + andq %r14, %rcx + rorq $4, %rax + xorq %r8, %rcx + xorq %r14, %rax + addq %rcx, %r9 + rorq $14, %rax + xorq %r11, %rdx + addq %rax, %r9 + movq %r10, %rcx + andq %rdx, %rbx + rorq $5, %rcx + xorq %r10, %rcx + xorq %r11, %rbx + rorq $6, %rcx + addq %r9, %r13 + xorq %r10, %rcx + addq %rbx, %r9 + rorq $28, %rcx + movq %r13, %rax + addq %rcx, %r9 + rorq $23, %rax + movq %r9, %rbx + movq %r14, %rcx + addq 120(%rsp), %r8 + xorq %r15, %rcx + xorq %r13, %rax + andq %r13, %rcx + rorq $4, %rax + xorq %r15, %rcx + xorq %r13, %rax + addq %rcx, %r8 + rorq $14, %rax + xorq %r10, %rbx + addq %rax, %r8 + movq %r9, %rcx + andq %rbx, %rdx + rorq $5, %rcx + xorq %r9, %rcx + xorq %r10, %rdx + rorq $6, %rcx + addq %r8, %r12 + xorq %r9, %rcx + addq %rdx, %r8 + rorq $28, %rcx + movq %r12, %rax + addq %rcx, %r8 + addq %r8, (%rdi) + addq %r9, 8(%rdi) + addq %r10, 16(%rdi) + addq %r11, 24(%rdi) + addq %r12, 32(%rdi) + addq %r13, 40(%rdi) + addq %r14, 48(%rdi) + addq %r15, 56(%rdi) + xorq %rax, %rax + vzeroupper + addq $0x88, %rsp + popq %r15 + popq %r14 + popq %r13 + popq %r12 + popq %rbx + repz retq +.size Transform_Sha512_AVX2,.-Transform_Sha512_AVX2 +.globl Transform_Sha512_AVX2_Len +.type Transform_Sha512_AVX2_Len,@function +.align 4 +Transform_Sha512_AVX2_Len: + pushq %rbx + pushq %r12 + pushq %r13 + pushq %r14 + pushq %r15 + pushq %rbp + movq %rsi, %rbp + testb $0x80, %bpl + je L_sha512_len_avx2_block + movq 224(%rdi), %rcx + vmovdqu (%rcx), %ymm0 + vmovdqu 32(%rcx), %ymm1 + vmovdqu 64(%rcx), %ymm2 + vmovdqu 96(%rcx), %ymm3 + vmovups %ymm0, 64(%rdi) + vmovups %ymm1, 96(%rdi) + vmovups %ymm2, 128(%rdi) + vmovups %ymm3, 160(%rdi) + call Transform_Sha512_AVX2@plt + addq $0x80, 224(%rdi) + subl $0x80, %ebp + jz L_sha512_len_avx2_done +L_sha512_len_avx2_block: + movq 224(%rdi), %rcx + vmovdqa L_avx2_sha512_flip_mask(%rip), %ymm15 + movq (%rdi), %r8 + movq 8(%rdi), %r9 + movq 16(%rdi), %r10 + movq 24(%rdi), %r11 + movq 32(%rdi), %r12 + movq 40(%rdi), %r13 + movq 48(%rdi), %r14 + movq 56(%rdi), %r15 + # Start of loop processing two blocks +L_sha512_len_avx2_begin: + subq $0x540, %rsp + leaq L_avx2_sha512_k_2(%rip), %rsi + movq %r9, %rbx + movq %r12, %rax + vmovdqu (%rcx), %xmm0 + vmovdqu 16(%rcx), %xmm1 + vinserti128 $0x01, 128(%rcx), %ymm0, %ymm0 + vinserti128 $0x01, 144(%rcx), %ymm1, %ymm1 + vpshufb %ymm15, %ymm0, %ymm0 + vpshufb %ymm15, %ymm1, %ymm1 + vmovdqu 32(%rcx), %xmm2 + vmovdqu 48(%rcx), %xmm3 + vinserti128 $0x01, 160(%rcx), %ymm2, %ymm2 + vinserti128 $0x01, 176(%rcx), %ymm3, %ymm3 + vpshufb %ymm15, %ymm2, %ymm2 + vpshufb %ymm15, %ymm3, %ymm3 + vmovdqu 64(%rcx), %xmm4 + vmovdqu 80(%rcx), %xmm5 + vinserti128 $0x01, 192(%rcx), %ymm4, %ymm4 + vinserti128 $0x01, 208(%rcx), %ymm5, %ymm5 + vpshufb %ymm15, %ymm4, %ymm4 + vpshufb %ymm15, %ymm5, %ymm5 + vmovdqu 96(%rcx), %xmm6 + vmovdqu 112(%rcx), %xmm7 + vinserti128 $0x01, 224(%rcx), %ymm6, %ymm6 + vinserti128 $0x01, 240(%rcx), %ymm7, %ymm7 + vpshufb %ymm15, %ymm6, %ymm6 + vpshufb %ymm15, %ymm7, %ymm7 + xorq %r10, %rbx + # Start of 16 rounds +L_sha512_len_avx2_start: + vpaddq (%rsi), %ymm0, %ymm8 + vpaddq 32(%rsi), %ymm1, %ymm9 + vmovdqu %ymm8, (%rsp) + vmovdqu %ymm9, 32(%rsp) + vpaddq 64(%rsi), %ymm2, %ymm8 + vpaddq 96(%rsi), %ymm3, %ymm9 + vmovdqu %ymm8, 64(%rsp) + vmovdqu %ymm9, 96(%rsp) + vpaddq 128(%rsi), %ymm4, %ymm8 + vpaddq 160(%rsi), %ymm5, %ymm9 + vmovdqu %ymm8, 128(%rsp) + vmovdqu %ymm9, 160(%rsp) + vpaddq 192(%rsi), %ymm6, %ymm8 + vpaddq 224(%rsi), %ymm7, %ymm9 + vmovdqu %ymm8, 192(%rsp) + vmovdqu %ymm9, 224(%rsp) + # msg_sched: 0-1 + rorq $23, %rax + vpalignr $8, %ymm0, %ymm1, %ymm12 + vpalignr $8, %ymm4, %ymm5, %ymm13 + movq %r8, %rdx + movq %r13, %rcx + addq (%rsp), %r15 + xorq %r14, %rcx + vpsrlq $0x01, %ymm12, %ymm8 + vpsllq $63, %ymm12, %ymm9 + xorq %r12, %rax + andq %r12, %rcx + rorq $4, %rax + xorq %r14, %rcx + vpsrlq $8, %ymm12, %ymm10 + vpsllq $56, %ymm12, %ymm11 + xorq %r12, %rax + addq %rcx, %r15 + rorq $14, %rax + xorq %r9, %rdx + vpor %ymm9, %ymm8, %ymm8 + vpor %ymm11, %ymm10, %ymm10 + addq %rax, %r15 + movq %r8, %rcx + andq %rdx, %rbx + rorq $5, %rcx + vpsrlq $7, %ymm12, %ymm11 + vpxor %ymm10, %ymm8, %ymm8 + xorq %r8, %rcx + xorq %r9, %rbx + rorq $6, %rcx + addq %r15, %r11 + vpxor %ymm11, %ymm8, %ymm8 + vpaddq %ymm0, %ymm13, %ymm0 + xorq %r8, %rcx + addq %rbx, %r15 + rorq $28, %rcx + movq %r11, %rax + addq %rcx, %r15 + rorq $23, %rax + vpaddq %ymm0, %ymm8, %ymm0 + movq %r15, %rbx + movq %r12, %rcx + addq 8(%rsp), %r14 + xorq %r13, %rcx + vpsrlq $19, %ymm7, %ymm8 + vpsllq $45, %ymm7, %ymm9 + xorq %r11, %rax + andq %r11, %rcx + rorq $4, %rax + xorq %r13, %rcx + vpsrlq $61, %ymm7, %ymm10 + vpsllq $3, %ymm7, %ymm11 + xorq %r11, %rax + addq %rcx, %r14 + rorq $14, %rax + xorq %r8, %rbx + addq %rax, %r14 + movq %r15, %rcx + vpor %ymm9, %ymm8, %ymm8 + vpor %ymm11, %ymm10, %ymm10 + andq %rbx, %rdx + rorq $5, %rcx + xorq %r15, %rcx + xorq %r8, %rdx + vpxor %ymm10, %ymm8, %ymm8 + vpsrlq $6, %ymm7, %ymm11 + rorq $6, %rcx + addq %r14, %r10 + xorq %r15, %rcx + addq %rdx, %r14 + vpxor %ymm11, %ymm8, %ymm8 + rorq $28, %rcx + movq %r10, %rax + addq %rcx, %r14 + vpaddq %ymm0, %ymm8, %ymm0 + # msg_sched done: 0-3 + # msg_sched: 4-5 + rorq $23, %rax + vpalignr $8, %ymm1, %ymm2, %ymm12 + vpalignr $8, %ymm5, %ymm6, %ymm13 + movq %r14, %rdx + movq %r11, %rcx + addq 32(%rsp), %r13 + xorq %r12, %rcx + vpsrlq $0x01, %ymm12, %ymm8 + vpsllq $63, %ymm12, %ymm9 + xorq %r10, %rax + andq %r10, %rcx + rorq $4, %rax + xorq %r12, %rcx + vpsrlq $8, %ymm12, %ymm10 + vpsllq $56, %ymm12, %ymm11 + xorq %r10, %rax + addq %rcx, %r13 + rorq $14, %rax + xorq %r15, %rdx + vpor %ymm9, %ymm8, %ymm8 + vpor %ymm11, %ymm10, %ymm10 + addq %rax, %r13 + movq %r14, %rcx + andq %rdx, %rbx + rorq $5, %rcx + vpsrlq $7, %ymm12, %ymm11 + vpxor %ymm10, %ymm8, %ymm8 + xorq %r14, %rcx + xorq %r15, %rbx + rorq $6, %rcx + addq %r13, %r9 + vpxor %ymm11, %ymm8, %ymm8 + vpaddq %ymm1, %ymm13, %ymm1 + xorq %r14, %rcx + addq %rbx, %r13 + rorq $28, %rcx + movq %r9, %rax + addq %rcx, %r13 + rorq $23, %rax + vpaddq %ymm1, %ymm8, %ymm1 + movq %r13, %rbx + movq %r10, %rcx + addq 40(%rsp), %r12 + xorq %r11, %rcx + vpsrlq $19, %ymm0, %ymm8 + vpsllq $45, %ymm0, %ymm9 + xorq %r9, %rax + andq %r9, %rcx + rorq $4, %rax + xorq %r11, %rcx + vpsrlq $61, %ymm0, %ymm10 + vpsllq $3, %ymm0, %ymm11 + xorq %r9, %rax + addq %rcx, %r12 + rorq $14, %rax + xorq %r14, %rbx + addq %rax, %r12 + movq %r13, %rcx + vpor %ymm9, %ymm8, %ymm8 + vpor %ymm11, %ymm10, %ymm10 + andq %rbx, %rdx + rorq $5, %rcx + xorq %r13, %rcx + xorq %r14, %rdx + vpxor %ymm10, %ymm8, %ymm8 + vpsrlq $6, %ymm0, %ymm11 + rorq $6, %rcx + addq %r12, %r8 + xorq %r13, %rcx + addq %rdx, %r12 + vpxor %ymm11, %ymm8, %ymm8 + rorq $28, %rcx + movq %r8, %rax + addq %rcx, %r12 + vpaddq %ymm1, %ymm8, %ymm1 + # msg_sched done: 4-7 + # msg_sched: 8-9 + rorq $23, %rax + vpalignr $8, %ymm2, %ymm3, %ymm12 + vpalignr $8, %ymm6, %ymm7, %ymm13 + movq %r12, %rdx + movq %r9, %rcx + addq 64(%rsp), %r11 + xorq %r10, %rcx + vpsrlq $0x01, %ymm12, %ymm8 + vpsllq $63, %ymm12, %ymm9 + xorq %r8, %rax + andq %r8, %rcx + rorq $4, %rax + xorq %r10, %rcx + vpsrlq $8, %ymm12, %ymm10 + vpsllq $56, %ymm12, %ymm11 + xorq %r8, %rax + addq %rcx, %r11 + rorq $14, %rax + xorq %r13, %rdx + vpor %ymm9, %ymm8, %ymm8 + vpor %ymm11, %ymm10, %ymm10 + addq %rax, %r11 + movq %r12, %rcx + andq %rdx, %rbx + rorq $5, %rcx + vpsrlq $7, %ymm12, %ymm11 + vpxor %ymm10, %ymm8, %ymm8 + xorq %r12, %rcx + xorq %r13, %rbx + rorq $6, %rcx + addq %r11, %r15 + vpxor %ymm11, %ymm8, %ymm8 + vpaddq %ymm2, %ymm13, %ymm2 + xorq %r12, %rcx + addq %rbx, %r11 + rorq $28, %rcx + movq %r15, %rax + addq %rcx, %r11 + rorq $23, %rax + vpaddq %ymm2, %ymm8, %ymm2 + movq %r11, %rbx + movq %r8, %rcx + addq 72(%rsp), %r10 + xorq %r9, %rcx + vpsrlq $19, %ymm1, %ymm8 + vpsllq $45, %ymm1, %ymm9 + xorq %r15, %rax + andq %r15, %rcx + rorq $4, %rax + xorq %r9, %rcx + vpsrlq $61, %ymm1, %ymm10 + vpsllq $3, %ymm1, %ymm11 + xorq %r15, %rax + addq %rcx, %r10 + rorq $14, %rax + xorq %r12, %rbx + addq %rax, %r10 + movq %r11, %rcx + vpor %ymm9, %ymm8, %ymm8 + vpor %ymm11, %ymm10, %ymm10 + andq %rbx, %rdx + rorq $5, %rcx + xorq %r11, %rcx + xorq %r12, %rdx + vpxor %ymm10, %ymm8, %ymm8 + vpsrlq $6, %ymm1, %ymm11 + rorq $6, %rcx + addq %r10, %r14 + xorq %r11, %rcx + addq %rdx, %r10 + vpxor %ymm11, %ymm8, %ymm8 + rorq $28, %rcx + movq %r14, %rax + addq %rcx, %r10 + vpaddq %ymm2, %ymm8, %ymm2 + # msg_sched done: 8-11 + # msg_sched: 12-13 + rorq $23, %rax + vpalignr $8, %ymm3, %ymm4, %ymm12 + vpalignr $8, %ymm7, %ymm0, %ymm13 + movq %r10, %rdx + movq %r15, %rcx + addq 96(%rsp), %r9 + xorq %r8, %rcx + vpsrlq $0x01, %ymm12, %ymm8 + vpsllq $63, %ymm12, %ymm9 + xorq %r14, %rax + andq %r14, %rcx + rorq $4, %rax + xorq %r8, %rcx + vpsrlq $8, %ymm12, %ymm10 + vpsllq $56, %ymm12, %ymm11 + xorq %r14, %rax + addq %rcx, %r9 + rorq $14, %rax + xorq %r11, %rdx + vpor %ymm9, %ymm8, %ymm8 + vpor %ymm11, %ymm10, %ymm10 + addq %rax, %r9 + movq %r10, %rcx + andq %rdx, %rbx + rorq $5, %rcx + vpsrlq $7, %ymm12, %ymm11 + vpxor %ymm10, %ymm8, %ymm8 + xorq %r10, %rcx + xorq %r11, %rbx + rorq $6, %rcx + addq %r9, %r13 + vpxor %ymm11, %ymm8, %ymm8 + vpaddq %ymm3, %ymm13, %ymm3 + xorq %r10, %rcx + addq %rbx, %r9 + rorq $28, %rcx + movq %r13, %rax + addq %rcx, %r9 + rorq $23, %rax + vpaddq %ymm3, %ymm8, %ymm3 + movq %r9, %rbx + movq %r14, %rcx + addq 104(%rsp), %r8 + xorq %r15, %rcx + vpsrlq $19, %ymm2, %ymm8 + vpsllq $45, %ymm2, %ymm9 + xorq %r13, %rax + andq %r13, %rcx + rorq $4, %rax + xorq %r15, %rcx + vpsrlq $61, %ymm2, %ymm10 + vpsllq $3, %ymm2, %ymm11 + xorq %r13, %rax + addq %rcx, %r8 + rorq $14, %rax + xorq %r10, %rbx + addq %rax, %r8 + movq %r9, %rcx + vpor %ymm9, %ymm8, %ymm8 + vpor %ymm11, %ymm10, %ymm10 + andq %rbx, %rdx + rorq $5, %rcx + xorq %r9, %rcx + xorq %r10, %rdx + vpxor %ymm10, %ymm8, %ymm8 + vpsrlq $6, %ymm2, %ymm11 + rorq $6, %rcx + addq %r8, %r12 + xorq %r9, %rcx + addq %rdx, %r8 + vpxor %ymm11, %ymm8, %ymm8 + rorq $28, %rcx + movq %r12, %rax + addq %rcx, %r8 + vpaddq %ymm3, %ymm8, %ymm3 + # msg_sched done: 12-15 + # msg_sched: 16-17 + rorq $23, %rax + vpalignr $8, %ymm4, %ymm5, %ymm12 + vpalignr $8, %ymm0, %ymm1, %ymm13 + movq %r8, %rdx + movq %r13, %rcx + addq 128(%rsp), %r15 + xorq %r14, %rcx + vpsrlq $0x01, %ymm12, %ymm8 + vpsllq $63, %ymm12, %ymm9 + xorq %r12, %rax + andq %r12, %rcx + rorq $4, %rax + xorq %r14, %rcx + vpsrlq $8, %ymm12, %ymm10 + vpsllq $56, %ymm12, %ymm11 + xorq %r12, %rax + addq %rcx, %r15 + rorq $14, %rax + xorq %r9, %rdx + vpor %ymm9, %ymm8, %ymm8 + vpor %ymm11, %ymm10, %ymm10 + addq %rax, %r15 + movq %r8, %rcx + andq %rdx, %rbx + rorq $5, %rcx + vpsrlq $7, %ymm12, %ymm11 + vpxor %ymm10, %ymm8, %ymm8 + xorq %r8, %rcx + xorq %r9, %rbx + rorq $6, %rcx + addq %r15, %r11 + vpxor %ymm11, %ymm8, %ymm8 + vpaddq %ymm4, %ymm13, %ymm4 + xorq %r8, %rcx + addq %rbx, %r15 + rorq $28, %rcx + movq %r11, %rax + addq %rcx, %r15 + rorq $23, %rax + vpaddq %ymm4, %ymm8, %ymm4 + movq %r15, %rbx + movq %r12, %rcx + addq 136(%rsp), %r14 + xorq %r13, %rcx + vpsrlq $19, %ymm3, %ymm8 + vpsllq $45, %ymm3, %ymm9 + xorq %r11, %rax + andq %r11, %rcx + rorq $4, %rax + xorq %r13, %rcx + vpsrlq $61, %ymm3, %ymm10 + vpsllq $3, %ymm3, %ymm11 + xorq %r11, %rax + addq %rcx, %r14 + rorq $14, %rax + xorq %r8, %rbx + addq %rax, %r14 + movq %r15, %rcx + vpor %ymm9, %ymm8, %ymm8 + vpor %ymm11, %ymm10, %ymm10 + andq %rbx, %rdx + rorq $5, %rcx + xorq %r15, %rcx + xorq %r8, %rdx + vpxor %ymm10, %ymm8, %ymm8 + vpsrlq $6, %ymm3, %ymm11 + rorq $6, %rcx + addq %r14, %r10 + xorq %r15, %rcx + addq %rdx, %r14 + vpxor %ymm11, %ymm8, %ymm8 + rorq $28, %rcx + movq %r10, %rax + addq %rcx, %r14 + vpaddq %ymm4, %ymm8, %ymm4 + # msg_sched done: 16-19 + # msg_sched: 20-21 + rorq $23, %rax + vpalignr $8, %ymm5, %ymm6, %ymm12 + vpalignr $8, %ymm1, %ymm2, %ymm13 + movq %r14, %rdx + movq %r11, %rcx + addq 160(%rsp), %r13 + xorq %r12, %rcx + vpsrlq $0x01, %ymm12, %ymm8 + vpsllq $63, %ymm12, %ymm9 + xorq %r10, %rax + andq %r10, %rcx + rorq $4, %rax + xorq %r12, %rcx + vpsrlq $8, %ymm12, %ymm10 + vpsllq $56, %ymm12, %ymm11 + xorq %r10, %rax + addq %rcx, %r13 + rorq $14, %rax + xorq %r15, %rdx + vpor %ymm9, %ymm8, %ymm8 + vpor %ymm11, %ymm10, %ymm10 + addq %rax, %r13 + movq %r14, %rcx + andq %rdx, %rbx + rorq $5, %rcx + vpsrlq $7, %ymm12, %ymm11 + vpxor %ymm10, %ymm8, %ymm8 + xorq %r14, %rcx + xorq %r15, %rbx + rorq $6, %rcx + addq %r13, %r9 + vpxor %ymm11, %ymm8, %ymm8 + vpaddq %ymm5, %ymm13, %ymm5 + xorq %r14, %rcx + addq %rbx, %r13 + rorq $28, %rcx + movq %r9, %rax + addq %rcx, %r13 + rorq $23, %rax + vpaddq %ymm5, %ymm8, %ymm5 + movq %r13, %rbx + movq %r10, %rcx + addq 168(%rsp), %r12 + xorq %r11, %rcx + vpsrlq $19, %ymm4, %ymm8 + vpsllq $45, %ymm4, %ymm9 + xorq %r9, %rax + andq %r9, %rcx + rorq $4, %rax + xorq %r11, %rcx + vpsrlq $61, %ymm4, %ymm10 + vpsllq $3, %ymm4, %ymm11 + xorq %r9, %rax + addq %rcx, %r12 + rorq $14, %rax + xorq %r14, %rbx + addq %rax, %r12 + movq %r13, %rcx + vpor %ymm9, %ymm8, %ymm8 + vpor %ymm11, %ymm10, %ymm10 + andq %rbx, %rdx + rorq $5, %rcx + xorq %r13, %rcx + xorq %r14, %rdx + vpxor %ymm10, %ymm8, %ymm8 + vpsrlq $6, %ymm4, %ymm11 + rorq $6, %rcx + addq %r12, %r8 + xorq %r13, %rcx + addq %rdx, %r12 + vpxor %ymm11, %ymm8, %ymm8 + rorq $28, %rcx + movq %r8, %rax + addq %rcx, %r12 + vpaddq %ymm5, %ymm8, %ymm5 + # msg_sched done: 20-23 + # msg_sched: 24-25 + rorq $23, %rax + vpalignr $8, %ymm6, %ymm7, %ymm12 + vpalignr $8, %ymm2, %ymm3, %ymm13 + movq %r12, %rdx + movq %r9, %rcx + addq 192(%rsp), %r11 + xorq %r10, %rcx + vpsrlq $0x01, %ymm12, %ymm8 + vpsllq $63, %ymm12, %ymm9 + xorq %r8, %rax + andq %r8, %rcx + rorq $4, %rax + xorq %r10, %rcx + vpsrlq $8, %ymm12, %ymm10 + vpsllq $56, %ymm12, %ymm11 + xorq %r8, %rax + addq %rcx, %r11 + rorq $14, %rax + xorq %r13, %rdx + vpor %ymm9, %ymm8, %ymm8 + vpor %ymm11, %ymm10, %ymm10 + addq %rax, %r11 + movq %r12, %rcx + andq %rdx, %rbx + rorq $5, %rcx + vpsrlq $7, %ymm12, %ymm11 + vpxor %ymm10, %ymm8, %ymm8 + xorq %r12, %rcx + xorq %r13, %rbx + rorq $6, %rcx + addq %r11, %r15 + vpxor %ymm11, %ymm8, %ymm8 + vpaddq %ymm6, %ymm13, %ymm6 + xorq %r12, %rcx + addq %rbx, %r11 + rorq $28, %rcx + movq %r15, %rax + addq %rcx, %r11 + rorq $23, %rax + vpaddq %ymm6, %ymm8, %ymm6 + movq %r11, %rbx + movq %r8, %rcx + addq 200(%rsp), %r10 + xorq %r9, %rcx + vpsrlq $19, %ymm5, %ymm8 + vpsllq $45, %ymm5, %ymm9 + xorq %r15, %rax + andq %r15, %rcx + rorq $4, %rax + xorq %r9, %rcx + vpsrlq $61, %ymm5, %ymm10 + vpsllq $3, %ymm5, %ymm11 + xorq %r15, %rax + addq %rcx, %r10 + rorq $14, %rax + xorq %r12, %rbx + addq %rax, %r10 + movq %r11, %rcx + vpor %ymm9, %ymm8, %ymm8 + vpor %ymm11, %ymm10, %ymm10 + andq %rbx, %rdx + rorq $5, %rcx + xorq %r11, %rcx + xorq %r12, %rdx + vpxor %ymm10, %ymm8, %ymm8 + vpsrlq $6, %ymm5, %ymm11 + rorq $6, %rcx + addq %r10, %r14 + xorq %r11, %rcx + addq %rdx, %r10 + vpxor %ymm11, %ymm8, %ymm8 + rorq $28, %rcx + movq %r14, %rax + addq %rcx, %r10 + vpaddq %ymm6, %ymm8, %ymm6 + # msg_sched done: 24-27 + # msg_sched: 28-29 + rorq $23, %rax + vpalignr $8, %ymm7, %ymm0, %ymm12 + vpalignr $8, %ymm3, %ymm4, %ymm13 + movq %r10, %rdx + movq %r15, %rcx + addq 224(%rsp), %r9 + xorq %r8, %rcx + vpsrlq $0x01, %ymm12, %ymm8 + vpsllq $63, %ymm12, %ymm9 + xorq %r14, %rax + andq %r14, %rcx + rorq $4, %rax + xorq %r8, %rcx + vpsrlq $8, %ymm12, %ymm10 + vpsllq $56, %ymm12, %ymm11 + xorq %r14, %rax + addq %rcx, %r9 + rorq $14, %rax + xorq %r11, %rdx + vpor %ymm9, %ymm8, %ymm8 + vpor %ymm11, %ymm10, %ymm10 + addq %rax, %r9 + movq %r10, %rcx + andq %rdx, %rbx + rorq $5, %rcx + vpsrlq $7, %ymm12, %ymm11 + vpxor %ymm10, %ymm8, %ymm8 + xorq %r10, %rcx + xorq %r11, %rbx + rorq $6, %rcx + addq %r9, %r13 + vpxor %ymm11, %ymm8, %ymm8 + vpaddq %ymm7, %ymm13, %ymm7 + xorq %r10, %rcx + addq %rbx, %r9 + rorq $28, %rcx + movq %r13, %rax + addq %rcx, %r9 + rorq $23, %rax + vpaddq %ymm7, %ymm8, %ymm7 + movq %r9, %rbx + movq %r14, %rcx + addq 232(%rsp), %r8 + xorq %r15, %rcx + vpsrlq $19, %ymm6, %ymm8 + vpsllq $45, %ymm6, %ymm9 + xorq %r13, %rax + andq %r13, %rcx + rorq $4, %rax + xorq %r15, %rcx + vpsrlq $61, %ymm6, %ymm10 + vpsllq $3, %ymm6, %ymm11 + xorq %r13, %rax + addq %rcx, %r8 + rorq $14, %rax + xorq %r10, %rbx + addq %rax, %r8 + movq %r9, %rcx + vpor %ymm9, %ymm8, %ymm8 + vpor %ymm11, %ymm10, %ymm10 + andq %rbx, %rdx + rorq $5, %rcx + xorq %r9, %rcx + xorq %r10, %rdx + vpxor %ymm10, %ymm8, %ymm8 + vpsrlq $6, %ymm6, %ymm11 + rorq $6, %rcx + addq %r8, %r12 + xorq %r9, %rcx + addq %rdx, %r8 + vpxor %ymm11, %ymm8, %ymm8 + rorq $28, %rcx + movq %r12, %rax + addq %rcx, %r8 + vpaddq %ymm7, %ymm8, %ymm7 + # msg_sched done: 28-31 + addq $0x100, %rsi + addq $0x100, %rsp + cmpq L_avx2_sha512_k_2_end(%rip), %rsi + jne L_sha512_len_avx2_start + vpaddq (%rsi), %ymm0, %ymm8 + vpaddq 32(%rsi), %ymm1, %ymm9 + vmovdqu %ymm8, (%rsp) + vmovdqu %ymm9, 32(%rsp) + vpaddq 64(%rsi), %ymm2, %ymm8 + vpaddq 96(%rsi), %ymm3, %ymm9 + vmovdqu %ymm8, 64(%rsp) + vmovdqu %ymm9, 96(%rsp) + vpaddq 128(%rsi), %ymm4, %ymm8 + vpaddq 160(%rsi), %ymm5, %ymm9 + vmovdqu %ymm8, 128(%rsp) + vmovdqu %ymm9, 160(%rsp) + vpaddq 192(%rsi), %ymm6, %ymm8 + vpaddq 224(%rsi), %ymm7, %ymm9 + vmovdqu %ymm8, 192(%rsp) + vmovdqu %ymm9, 224(%rsp) + rorq $23, %rax + movq %r8, %rdx + movq %r13, %rcx + addq (%rsp), %r15 + xorq %r14, %rcx + xorq %r12, %rax + andq %r12, %rcx + rorq $4, %rax + xorq %r14, %rcx + xorq %r12, %rax + addq %rcx, %r15 + rorq $14, %rax + xorq %r9, %rdx + addq %rax, %r15 + movq %r8, %rcx + andq %rdx, %rbx + rorq $5, %rcx + xorq %r8, %rcx + xorq %r9, %rbx + rorq $6, %rcx + addq %r15, %r11 + xorq %r8, %rcx + addq %rbx, %r15 + rorq $28, %rcx + movq %r11, %rax + addq %rcx, %r15 + rorq $23, %rax + movq %r15, %rbx + movq %r12, %rcx + addq 8(%rsp), %r14 + xorq %r13, %rcx + xorq %r11, %rax + andq %r11, %rcx + rorq $4, %rax + xorq %r13, %rcx + xorq %r11, %rax + addq %rcx, %r14 + rorq $14, %rax + xorq %r8, %rbx + addq %rax, %r14 + movq %r15, %rcx + andq %rbx, %rdx + rorq $5, %rcx + xorq %r15, %rcx + xorq %r8, %rdx + rorq $6, %rcx + addq %r14, %r10 + xorq %r15, %rcx + addq %rdx, %r14 + rorq $28, %rcx + movq %r10, %rax + addq %rcx, %r14 + rorq $23, %rax + movq %r14, %rdx + movq %r11, %rcx + addq 32(%rsp), %r13 + xorq %r12, %rcx + xorq %r10, %rax + andq %r10, %rcx + rorq $4, %rax + xorq %r12, %rcx + xorq %r10, %rax + addq %rcx, %r13 + rorq $14, %rax + xorq %r15, %rdx + addq %rax, %r13 + movq %r14, %rcx + andq %rdx, %rbx + rorq $5, %rcx + xorq %r14, %rcx + xorq %r15, %rbx + rorq $6, %rcx + addq %r13, %r9 + xorq %r14, %rcx + addq %rbx, %r13 + rorq $28, %rcx + movq %r9, %rax + addq %rcx, %r13 + rorq $23, %rax + movq %r13, %rbx + movq %r10, %rcx + addq 40(%rsp), %r12 + xorq %r11, %rcx + xorq %r9, %rax + andq %r9, %rcx + rorq $4, %rax + xorq %r11, %rcx + xorq %r9, %rax + addq %rcx, %r12 + rorq $14, %rax + xorq %r14, %rbx + addq %rax, %r12 + movq %r13, %rcx + andq %rbx, %rdx + rorq $5, %rcx + xorq %r13, %rcx + xorq %r14, %rdx + rorq $6, %rcx + addq %r12, %r8 + xorq %r13, %rcx + addq %rdx, %r12 + rorq $28, %rcx + movq %r8, %rax + addq %rcx, %r12 + rorq $23, %rax + movq %r12, %rdx + movq %r9, %rcx + addq 64(%rsp), %r11 + xorq %r10, %rcx + xorq %r8, %rax + andq %r8, %rcx + rorq $4, %rax + xorq %r10, %rcx + xorq %r8, %rax + addq %rcx, %r11 + rorq $14, %rax + xorq %r13, %rdx + addq %rax, %r11 + movq %r12, %rcx + andq %rdx, %rbx + rorq $5, %rcx + xorq %r12, %rcx + xorq %r13, %rbx + rorq $6, %rcx + addq %r11, %r15 + xorq %r12, %rcx + addq %rbx, %r11 + rorq $28, %rcx + movq %r15, %rax + addq %rcx, %r11 + rorq $23, %rax + movq %r11, %rbx + movq %r8, %rcx + addq 72(%rsp), %r10 + xorq %r9, %rcx + xorq %r15, %rax + andq %r15, %rcx + rorq $4, %rax + xorq %r9, %rcx + xorq %r15, %rax + addq %rcx, %r10 + rorq $14, %rax + xorq %r12, %rbx + addq %rax, %r10 + movq %r11, %rcx + andq %rbx, %rdx + rorq $5, %rcx + xorq %r11, %rcx + xorq %r12, %rdx + rorq $6, %rcx + addq %r10, %r14 + xorq %r11, %rcx + addq %rdx, %r10 + rorq $28, %rcx + movq %r14, %rax + addq %rcx, %r10 + rorq $23, %rax + movq %r10, %rdx + movq %r15, %rcx + addq 96(%rsp), %r9 + xorq %r8, %rcx + xorq %r14, %rax + andq %r14, %rcx + rorq $4, %rax + xorq %r8, %rcx + xorq %r14, %rax + addq %rcx, %r9 + rorq $14, %rax + xorq %r11, %rdx + addq %rax, %r9 + movq %r10, %rcx + andq %rdx, %rbx + rorq $5, %rcx + xorq %r10, %rcx + xorq %r11, %rbx + rorq $6, %rcx + addq %r9, %r13 + xorq %r10, %rcx + addq %rbx, %r9 + rorq $28, %rcx + movq %r13, %rax + addq %rcx, %r9 + rorq $23, %rax + movq %r9, %rbx + movq %r14, %rcx + addq 104(%rsp), %r8 + xorq %r15, %rcx + xorq %r13, %rax + andq %r13, %rcx + rorq $4, %rax + xorq %r15, %rcx + xorq %r13, %rax + addq %rcx, %r8 + rorq $14, %rax + xorq %r10, %rbx + addq %rax, %r8 + movq %r9, %rcx + andq %rbx, %rdx + rorq $5, %rcx + xorq %r9, %rcx + xorq %r10, %rdx + rorq $6, %rcx + addq %r8, %r12 + xorq %r9, %rcx + addq %rdx, %r8 + rorq $28, %rcx + movq %r12, %rax + addq %rcx, %r8 + rorq $23, %rax + movq %r8, %rdx + movq %r13, %rcx + addq 128(%rsp), %r15 + xorq %r14, %rcx + xorq %r12, %rax + andq %r12, %rcx + rorq $4, %rax + xorq %r14, %rcx + xorq %r12, %rax + addq %rcx, %r15 + rorq $14, %rax + xorq %r9, %rdx + addq %rax, %r15 + movq %r8, %rcx + andq %rdx, %rbx + rorq $5, %rcx + xorq %r8, %rcx + xorq %r9, %rbx + rorq $6, %rcx + addq %r15, %r11 + xorq %r8, %rcx + addq %rbx, %r15 + rorq $28, %rcx + movq %r11, %rax + addq %rcx, %r15 + rorq $23, %rax + movq %r15, %rbx + movq %r12, %rcx + addq 136(%rsp), %r14 + xorq %r13, %rcx + xorq %r11, %rax + andq %r11, %rcx + rorq $4, %rax + xorq %r13, %rcx + xorq %r11, %rax + addq %rcx, %r14 + rorq $14, %rax + xorq %r8, %rbx + addq %rax, %r14 + movq %r15, %rcx + andq %rbx, %rdx + rorq $5, %rcx + xorq %r15, %rcx + xorq %r8, %rdx + rorq $6, %rcx + addq %r14, %r10 + xorq %r15, %rcx + addq %rdx, %r14 + rorq $28, %rcx + movq %r10, %rax + addq %rcx, %r14 + rorq $23, %rax + movq %r14, %rdx + movq %r11, %rcx + addq 160(%rsp), %r13 + xorq %r12, %rcx + xorq %r10, %rax + andq %r10, %rcx + rorq $4, %rax + xorq %r12, %rcx + xorq %r10, %rax + addq %rcx, %r13 + rorq $14, %rax + xorq %r15, %rdx + addq %rax, %r13 + movq %r14, %rcx + andq %rdx, %rbx + rorq $5, %rcx + xorq %r14, %rcx + xorq %r15, %rbx + rorq $6, %rcx + addq %r13, %r9 + xorq %r14, %rcx + addq %rbx, %r13 + rorq $28, %rcx + movq %r9, %rax + addq %rcx, %r13 + rorq $23, %rax + movq %r13, %rbx + movq %r10, %rcx + addq 168(%rsp), %r12 + xorq %r11, %rcx + xorq %r9, %rax + andq %r9, %rcx + rorq $4, %rax + xorq %r11, %rcx + xorq %r9, %rax + addq %rcx, %r12 + rorq $14, %rax + xorq %r14, %rbx + addq %rax, %r12 + movq %r13, %rcx + andq %rbx, %rdx + rorq $5, %rcx + xorq %r13, %rcx + xorq %r14, %rdx + rorq $6, %rcx + addq %r12, %r8 + xorq %r13, %rcx + addq %rdx, %r12 + rorq $28, %rcx + movq %r8, %rax + addq %rcx, %r12 + rorq $23, %rax + movq %r12, %rdx + movq %r9, %rcx + addq 192(%rsp), %r11 + xorq %r10, %rcx + xorq %r8, %rax + andq %r8, %rcx + rorq $4, %rax + xorq %r10, %rcx + xorq %r8, %rax + addq %rcx, %r11 + rorq $14, %rax + xorq %r13, %rdx + addq %rax, %r11 + movq %r12, %rcx + andq %rdx, %rbx + rorq $5, %rcx + xorq %r12, %rcx + xorq %r13, %rbx + rorq $6, %rcx + addq %r11, %r15 + xorq %r12, %rcx + addq %rbx, %r11 + rorq $28, %rcx + movq %r15, %rax + addq %rcx, %r11 + rorq $23, %rax + movq %r11, %rbx + movq %r8, %rcx + addq 200(%rsp), %r10 + xorq %r9, %rcx + xorq %r15, %rax + andq %r15, %rcx + rorq $4, %rax + xorq %r9, %rcx + xorq %r15, %rax + addq %rcx, %r10 + rorq $14, %rax + xorq %r12, %rbx + addq %rax, %r10 + movq %r11, %rcx + andq %rbx, %rdx + rorq $5, %rcx + xorq %r11, %rcx + xorq %r12, %rdx + rorq $6, %rcx + addq %r10, %r14 + xorq %r11, %rcx + addq %rdx, %r10 + rorq $28, %rcx + movq %r14, %rax + addq %rcx, %r10 + rorq $23, %rax + movq %r10, %rdx + movq %r15, %rcx + addq 224(%rsp), %r9 + xorq %r8, %rcx + xorq %r14, %rax + andq %r14, %rcx + rorq $4, %rax + xorq %r8, %rcx + xorq %r14, %rax + addq %rcx, %r9 + rorq $14, %rax + xorq %r11, %rdx + addq %rax, %r9 + movq %r10, %rcx + andq %rdx, %rbx + rorq $5, %rcx + xorq %r10, %rcx + xorq %r11, %rbx + rorq $6, %rcx + addq %r9, %r13 + xorq %r10, %rcx + addq %rbx, %r9 + rorq $28, %rcx + movq %r13, %rax + addq %rcx, %r9 + rorq $23, %rax + movq %r9, %rbx + movq %r14, %rcx + addq 232(%rsp), %r8 + xorq %r15, %rcx + xorq %r13, %rax + andq %r13, %rcx + rorq $4, %rax + xorq %r15, %rcx + xorq %r13, %rax + addq %rcx, %r8 + rorq $14, %rax + xorq %r10, %rbx + addq %rax, %r8 + movq %r9, %rcx + andq %rbx, %rdx + rorq $5, %rcx + xorq %r9, %rcx + xorq %r10, %rdx + rorq $6, %rcx + addq %r8, %r12 + xorq %r9, %rcx + addq %rdx, %r8 + rorq $28, %rcx + movq %r12, %rax + addq %rcx, %r8 + subq $0x400, %rsp + addq (%rdi), %r8 + addq 8(%rdi), %r9 + addq 16(%rdi), %r10 + addq 24(%rdi), %r11 + addq 32(%rdi), %r12 + addq 40(%rdi), %r13 + addq 48(%rdi), %r14 + addq 56(%rdi), %r15 + movq %r8, (%rdi) + movq %r9, 8(%rdi) + movq %r10, 16(%rdi) + movq %r11, 24(%rdi) + movq %r12, 32(%rdi) + movq %r13, 40(%rdi) + movq %r14, 48(%rdi) + movq %r15, 56(%rdi) + movq %r9, %rbx + movq %r12, %rax + xorq %r10, %rbx + movq $5, %rsi +L_sha512_len_avx2_tail: + rorq $23, %rax + movq %r8, %rdx + movq %r13, %rcx + addq 16(%rsp), %r15 + xorq %r14, %rcx + xorq %r12, %rax + andq %r12, %rcx + rorq $4, %rax + xorq %r14, %rcx + xorq %r12, %rax + addq %rcx, %r15 + rorq $14, %rax + xorq %r9, %rdx + addq %rax, %r15 + movq %r8, %rcx + andq %rdx, %rbx + rorq $5, %rcx + xorq %r8, %rcx + xorq %r9, %rbx + rorq $6, %rcx + addq %r15, %r11 + xorq %r8, %rcx + addq %rbx, %r15 + rorq $28, %rcx + movq %r11, %rax + addq %rcx, %r15 + rorq $23, %rax + movq %r15, %rbx + movq %r12, %rcx + addq 24(%rsp), %r14 + xorq %r13, %rcx + xorq %r11, %rax + andq %r11, %rcx + rorq $4, %rax + xorq %r13, %rcx + xorq %r11, %rax + addq %rcx, %r14 + rorq $14, %rax + xorq %r8, %rbx + addq %rax, %r14 + movq %r15, %rcx + andq %rbx, %rdx + rorq $5, %rcx + xorq %r15, %rcx + xorq %r8, %rdx + rorq $6, %rcx + addq %r14, %r10 + xorq %r15, %rcx + addq %rdx, %r14 + rorq $28, %rcx + movq %r10, %rax + addq %rcx, %r14 + rorq $23, %rax + movq %r14, %rdx + movq %r11, %rcx + addq 48(%rsp), %r13 + xorq %r12, %rcx + xorq %r10, %rax + andq %r10, %rcx + rorq $4, %rax + xorq %r12, %rcx + xorq %r10, %rax + addq %rcx, %r13 + rorq $14, %rax + xorq %r15, %rdx + addq %rax, %r13 + movq %r14, %rcx + andq %rdx, %rbx + rorq $5, %rcx + xorq %r14, %rcx + xorq %r15, %rbx + rorq $6, %rcx + addq %r13, %r9 + xorq %r14, %rcx + addq %rbx, %r13 + rorq $28, %rcx + movq %r9, %rax + addq %rcx, %r13 + rorq $23, %rax + movq %r13, %rbx + movq %r10, %rcx + addq 56(%rsp), %r12 + xorq %r11, %rcx + xorq %r9, %rax + andq %r9, %rcx + rorq $4, %rax + xorq %r11, %rcx + xorq %r9, %rax + addq %rcx, %r12 + rorq $14, %rax + xorq %r14, %rbx + addq %rax, %r12 + movq %r13, %rcx + andq %rbx, %rdx + rorq $5, %rcx + xorq %r13, %rcx + xorq %r14, %rdx + rorq $6, %rcx + addq %r12, %r8 + xorq %r13, %rcx + addq %rdx, %r12 + rorq $28, %rcx + movq %r8, %rax + addq %rcx, %r12 + rorq $23, %rax + movq %r12, %rdx + movq %r9, %rcx + addq 80(%rsp), %r11 + xorq %r10, %rcx + xorq %r8, %rax + andq %r8, %rcx + rorq $4, %rax + xorq %r10, %rcx + xorq %r8, %rax + addq %rcx, %r11 + rorq $14, %rax + xorq %r13, %rdx + addq %rax, %r11 + movq %r12, %rcx + andq %rdx, %rbx + rorq $5, %rcx + xorq %r12, %rcx + xorq %r13, %rbx + rorq $6, %rcx + addq %r11, %r15 + xorq %r12, %rcx + addq %rbx, %r11 + rorq $28, %rcx + movq %r15, %rax + addq %rcx, %r11 + rorq $23, %rax + movq %r11, %rbx + movq %r8, %rcx + addq 88(%rsp), %r10 + xorq %r9, %rcx + xorq %r15, %rax + andq %r15, %rcx + rorq $4, %rax + xorq %r9, %rcx + xorq %r15, %rax + addq %rcx, %r10 + rorq $14, %rax + xorq %r12, %rbx + addq %rax, %r10 + movq %r11, %rcx + andq %rbx, %rdx + rorq $5, %rcx + xorq %r11, %rcx + xorq %r12, %rdx + rorq $6, %rcx + addq %r10, %r14 + xorq %r11, %rcx + addq %rdx, %r10 + rorq $28, %rcx + movq %r14, %rax + addq %rcx, %r10 + rorq $23, %rax + movq %r10, %rdx + movq %r15, %rcx + addq 112(%rsp), %r9 + xorq %r8, %rcx + xorq %r14, %rax + andq %r14, %rcx + rorq $4, %rax + xorq %r8, %rcx + xorq %r14, %rax + addq %rcx, %r9 + rorq $14, %rax + xorq %r11, %rdx + addq %rax, %r9 + movq %r10, %rcx + andq %rdx, %rbx + rorq $5, %rcx + xorq %r10, %rcx + xorq %r11, %rbx + rorq $6, %rcx + addq %r9, %r13 + xorq %r10, %rcx + addq %rbx, %r9 + rorq $28, %rcx + movq %r13, %rax + addq %rcx, %r9 + rorq $23, %rax + movq %r9, %rbx + movq %r14, %rcx + addq 120(%rsp), %r8 + xorq %r15, %rcx + xorq %r13, %rax + andq %r13, %rcx + rorq $4, %rax + xorq %r15, %rcx + xorq %r13, %rax + addq %rcx, %r8 + rorq $14, %rax + xorq %r10, %rbx + addq %rax, %r8 + movq %r9, %rcx + andq %rbx, %rdx + rorq $5, %rcx + xorq %r9, %rcx + xorq %r10, %rdx + rorq $6, %rcx + addq %r8, %r12 + xorq %r9, %rcx + addq %rdx, %r8 + rorq $28, %rcx + movq %r12, %rax + addq %rcx, %r8 + rorq $23, %rax + movq %r8, %rdx + movq %r13, %rcx + addq 144(%rsp), %r15 + xorq %r14, %rcx + xorq %r12, %rax + andq %r12, %rcx + rorq $4, %rax + xorq %r14, %rcx + xorq %r12, %rax + addq %rcx, %r15 + rorq $14, %rax + xorq %r9, %rdx + addq %rax, %r15 + movq %r8, %rcx + andq %rdx, %rbx + rorq $5, %rcx + xorq %r8, %rcx + xorq %r9, %rbx + rorq $6, %rcx + addq %r15, %r11 + xorq %r8, %rcx + addq %rbx, %r15 + rorq $28, %rcx + movq %r11, %rax + addq %rcx, %r15 + rorq $23, %rax + movq %r15, %rbx + movq %r12, %rcx + addq 152(%rsp), %r14 + xorq %r13, %rcx + xorq %r11, %rax + andq %r11, %rcx + rorq $4, %rax + xorq %r13, %rcx + xorq %r11, %rax + addq %rcx, %r14 + rorq $14, %rax + xorq %r8, %rbx + addq %rax, %r14 + movq %r15, %rcx + andq %rbx, %rdx + rorq $5, %rcx + xorq %r15, %rcx + xorq %r8, %rdx + rorq $6, %rcx + addq %r14, %r10 + xorq %r15, %rcx + addq %rdx, %r14 + rorq $28, %rcx + movq %r10, %rax + addq %rcx, %r14 + rorq $23, %rax + movq %r14, %rdx + movq %r11, %rcx + addq 176(%rsp), %r13 + xorq %r12, %rcx + xorq %r10, %rax + andq %r10, %rcx + rorq $4, %rax + xorq %r12, %rcx + xorq %r10, %rax + addq %rcx, %r13 + rorq $14, %rax + xorq %r15, %rdx + addq %rax, %r13 + movq %r14, %rcx + andq %rdx, %rbx + rorq $5, %rcx + xorq %r14, %rcx + xorq %r15, %rbx + rorq $6, %rcx + addq %r13, %r9 + xorq %r14, %rcx + addq %rbx, %r13 + rorq $28, %rcx + movq %r9, %rax + addq %rcx, %r13 + rorq $23, %rax + movq %r13, %rbx + movq %r10, %rcx + addq 184(%rsp), %r12 + xorq %r11, %rcx + xorq %r9, %rax + andq %r9, %rcx + rorq $4, %rax + xorq %r11, %rcx + xorq %r9, %rax + addq %rcx, %r12 + rorq $14, %rax + xorq %r14, %rbx + addq %rax, %r12 + movq %r13, %rcx + andq %rbx, %rdx + rorq $5, %rcx + xorq %r13, %rcx + xorq %r14, %rdx + rorq $6, %rcx + addq %r12, %r8 + xorq %r13, %rcx + addq %rdx, %r12 + rorq $28, %rcx + movq %r8, %rax + addq %rcx, %r12 + rorq $23, %rax + movq %r12, %rdx + movq %r9, %rcx + addq 208(%rsp), %r11 + xorq %r10, %rcx + xorq %r8, %rax + andq %r8, %rcx + rorq $4, %rax + xorq %r10, %rcx + xorq %r8, %rax + addq %rcx, %r11 + rorq $14, %rax + xorq %r13, %rdx + addq %rax, %r11 + movq %r12, %rcx + andq %rdx, %rbx + rorq $5, %rcx + xorq %r12, %rcx + xorq %r13, %rbx + rorq $6, %rcx + addq %r11, %r15 + xorq %r12, %rcx + addq %rbx, %r11 + rorq $28, %rcx + movq %r15, %rax + addq %rcx, %r11 + rorq $23, %rax + movq %r11, %rbx + movq %r8, %rcx + addq 216(%rsp), %r10 + xorq %r9, %rcx + xorq %r15, %rax + andq %r15, %rcx + rorq $4, %rax + xorq %r9, %rcx + xorq %r15, %rax + addq %rcx, %r10 + rorq $14, %rax + xorq %r12, %rbx + addq %rax, %r10 + movq %r11, %rcx + andq %rbx, %rdx + rorq $5, %rcx + xorq %r11, %rcx + xorq %r12, %rdx + rorq $6, %rcx + addq %r10, %r14 + xorq %r11, %rcx + addq %rdx, %r10 + rorq $28, %rcx + movq %r14, %rax + addq %rcx, %r10 + rorq $23, %rax + movq %r10, %rdx + movq %r15, %rcx + addq 240(%rsp), %r9 + xorq %r8, %rcx + xorq %r14, %rax + andq %r14, %rcx + rorq $4, %rax + xorq %r8, %rcx + xorq %r14, %rax + addq %rcx, %r9 + rorq $14, %rax + xorq %r11, %rdx + addq %rax, %r9 + movq %r10, %rcx + andq %rdx, %rbx + rorq $5, %rcx + xorq %r10, %rcx + xorq %r11, %rbx + rorq $6, %rcx + addq %r9, %r13 + xorq %r10, %rcx + addq %rbx, %r9 + rorq $28, %rcx + movq %r13, %rax + addq %rcx, %r9 + rorq $23, %rax + movq %r9, %rbx + movq %r14, %rcx + addq 248(%rsp), %r8 + xorq %r15, %rcx + xorq %r13, %rax + andq %r13, %rcx + rorq $4, %rax + xorq %r15, %rcx + xorq %r13, %rax + addq %rcx, %r8 + rorq $14, %rax + xorq %r10, %rbx + addq %rax, %r8 + movq %r9, %rcx + andq %rbx, %rdx + rorq $5, %rcx + xorq %r9, %rcx + xorq %r10, %rdx + rorq $6, %rcx + addq %r8, %r12 + xorq %r9, %rcx + addq %rdx, %r8 + rorq $28, %rcx + movq %r12, %rax + addq %rcx, %r8 + addq $0x100, %rsp + subq $0x01, %rsi + jnz L_sha512_len_avx2_tail + addq (%rdi), %r8 + addq 8(%rdi), %r9 + addq 16(%rdi), %r10 + addq 24(%rdi), %r11 + addq 32(%rdi), %r12 + addq 40(%rdi), %r13 + addq 48(%rdi), %r14 + addq 56(%rdi), %r15 + movq 224(%rdi), %rcx + addq $0x40, %rsp + addq $0x100, %rcx + subl $0x100, %ebp + movq %rcx, 224(%rdi) + movq %r8, (%rdi) + movq %r9, 8(%rdi) + movq %r10, 16(%rdi) + movq %r11, 24(%rdi) + movq %r12, 32(%rdi) + movq %r13, 40(%rdi) + movq %r14, 48(%rdi) + movq %r15, 56(%rdi) + jnz L_sha512_len_avx2_begin +L_sha512_len_avx2_done: + xorq %rax, %rax + vzeroupper + popq %rbp + popq %r15 + popq %r14 + popq %r13 + popq %r12 + popq %rbx + repz retq +.size Transform_Sha512_AVX2_Len,.-Transform_Sha512_AVX2_Len +.align 16 +L_avx2_rorx_sha512_k: +.quad 0x428a2f98d728ae22,0x7137449123ef65cd +.quad 0xb5c0fbcfec4d3b2f,0xe9b5dba58189dbbc +.quad 0x3956c25bf348b538,0x59f111f1b605d019 +.quad 0x923f82a4af194f9b,0xab1c5ed5da6d8118 +.quad 0xd807aa98a3030242,0x12835b0145706fbe +.quad 0x243185be4ee4b28c,0x550c7dc3d5ffb4e2 +.quad 0x72be5d74f27b896f,0x80deb1fe3b1696b1 +.quad 0x9bdc06a725c71235,0xc19bf174cf692694 +.quad 0xe49b69c19ef14ad2,0xefbe4786384f25e3 +.quad 0xfc19dc68b8cd5b5,0x240ca1cc77ac9c65 +.quad 0x2de92c6f592b0275,0x4a7484aa6ea6e483 +.quad 0x5cb0a9dcbd41fbd4,0x76f988da831153b5 +.quad 0x983e5152ee66dfab,0xa831c66d2db43210 +.quad 0xb00327c898fb213f,0xbf597fc7beef0ee4 +.quad 0xc6e00bf33da88fc2,0xd5a79147930aa725 +.quad 0x6ca6351e003826f,0x142929670a0e6e70 +.quad 0x27b70a8546d22ffc,0x2e1b21385c26c926 +.quad 0x4d2c6dfc5ac42aed,0x53380d139d95b3df +.quad 0x650a73548baf63de,0x766a0abb3c77b2a8 +.quad 0x81c2c92e47edaee6,0x92722c851482353b +.quad 0xa2bfe8a14cf10364,0xa81a664bbc423001 +.quad 0xc24b8b70d0f89791,0xc76c51a30654be30 +.quad 0xd192e819d6ef5218,0xd69906245565a910 +.quad 0xf40e35855771202a,0x106aa07032bbd1b8 +.quad 0x19a4c116b8d2d0c8,0x1e376c085141ab53 +.quad 0x2748774cdf8eeb99,0x34b0bcb5e19b48a8 +.quad 0x391c0cb3c5c95a63,0x4ed8aa4ae3418acb +.quad 0x5b9cca4f7763e373,0x682e6ff3d6b2b8a3 +.quad 0x748f82ee5defb2fc,0x78a5636f43172f60 +.quad 0x84c87814a1f0ab72,0x8cc702081a6439ec +.quad 0x90befffa23631e28,0xa4506cebde82bde9 +.quad 0xbef9a3f7b2c67915,0xc67178f2e372532b +.quad 0xca273eceea26619c,0xd186b8c721c0c207 +.quad 0xeada7dd6cde0eb1e,0xf57d4f7fee6ed178 +.quad 0x6f067aa72176fba,0xa637dc5a2c898a6 +.quad 0x113f9804bef90dae,0x1b710b35131c471b +.quad 0x28db77f523047d84,0x32caab7b40c72493 +.quad 0x3c9ebe0a15c9bebc,0x431d67c49c100d4c +.quad 0x4cc5d4becb3e42b6,0x597f299cfc657e2a +.quad 0x5fcb6fab3ad6faec,0x6c44198c4a475817 +.align 16 +L_avx2_rorx_sha512_k_2: +.quad 0x428a2f98d728ae22,0x7137449123ef65cd +.quad 0x428a2f98d728ae22,0x7137449123ef65cd +.quad 0xb5c0fbcfec4d3b2f,0xe9b5dba58189dbbc +.quad 0xb5c0fbcfec4d3b2f,0xe9b5dba58189dbbc +.quad 0x3956c25bf348b538,0x59f111f1b605d019 +.quad 0x3956c25bf348b538,0x59f111f1b605d019 +.quad 0x923f82a4af194f9b,0xab1c5ed5da6d8118 +.quad 0x923f82a4af194f9b,0xab1c5ed5da6d8118 +.quad 0xd807aa98a3030242,0x12835b0145706fbe +.quad 0xd807aa98a3030242,0x12835b0145706fbe +.quad 0x243185be4ee4b28c,0x550c7dc3d5ffb4e2 +.quad 0x243185be4ee4b28c,0x550c7dc3d5ffb4e2 +.quad 0x72be5d74f27b896f,0x80deb1fe3b1696b1 +.quad 0x72be5d74f27b896f,0x80deb1fe3b1696b1 +.quad 0x9bdc06a725c71235,0xc19bf174cf692694 +.quad 0x9bdc06a725c71235,0xc19bf174cf692694 +.quad 0xe49b69c19ef14ad2,0xefbe4786384f25e3 +.quad 0xe49b69c19ef14ad2,0xefbe4786384f25e3 +.quad 0xfc19dc68b8cd5b5,0x240ca1cc77ac9c65 +.quad 0xfc19dc68b8cd5b5,0x240ca1cc77ac9c65 +.quad 0x2de92c6f592b0275,0x4a7484aa6ea6e483 +.quad 0x2de92c6f592b0275,0x4a7484aa6ea6e483 +.quad 0x5cb0a9dcbd41fbd4,0x76f988da831153b5 +.quad 0x5cb0a9dcbd41fbd4,0x76f988da831153b5 +.quad 0x983e5152ee66dfab,0xa831c66d2db43210 +.quad 0x983e5152ee66dfab,0xa831c66d2db43210 +.quad 0xb00327c898fb213f,0xbf597fc7beef0ee4 +.quad 0xb00327c898fb213f,0xbf597fc7beef0ee4 +.quad 0xc6e00bf33da88fc2,0xd5a79147930aa725 +.quad 0xc6e00bf33da88fc2,0xd5a79147930aa725 +.quad 0x6ca6351e003826f,0x142929670a0e6e70 +.quad 0x6ca6351e003826f,0x142929670a0e6e70 +.quad 0x27b70a8546d22ffc,0x2e1b21385c26c926 +.quad 0x27b70a8546d22ffc,0x2e1b21385c26c926 +.quad 0x4d2c6dfc5ac42aed,0x53380d139d95b3df +.quad 0x4d2c6dfc5ac42aed,0x53380d139d95b3df +.quad 0x650a73548baf63de,0x766a0abb3c77b2a8 +.quad 0x650a73548baf63de,0x766a0abb3c77b2a8 +.quad 0x81c2c92e47edaee6,0x92722c851482353b +.quad 0x81c2c92e47edaee6,0x92722c851482353b +.quad 0xa2bfe8a14cf10364,0xa81a664bbc423001 +.quad 0xa2bfe8a14cf10364,0xa81a664bbc423001 +.quad 0xc24b8b70d0f89791,0xc76c51a30654be30 +.quad 0xc24b8b70d0f89791,0xc76c51a30654be30 +.quad 0xd192e819d6ef5218,0xd69906245565a910 +.quad 0xd192e819d6ef5218,0xd69906245565a910 +.quad 0xf40e35855771202a,0x106aa07032bbd1b8 +.quad 0xf40e35855771202a,0x106aa07032bbd1b8 +.quad 0x19a4c116b8d2d0c8,0x1e376c085141ab53 +.quad 0x19a4c116b8d2d0c8,0x1e376c085141ab53 +.quad 0x2748774cdf8eeb99,0x34b0bcb5e19b48a8 +.quad 0x2748774cdf8eeb99,0x34b0bcb5e19b48a8 +.quad 0x391c0cb3c5c95a63,0x4ed8aa4ae3418acb +.quad 0x391c0cb3c5c95a63,0x4ed8aa4ae3418acb +.quad 0x5b9cca4f7763e373,0x682e6ff3d6b2b8a3 +.quad 0x5b9cca4f7763e373,0x682e6ff3d6b2b8a3 +.quad 0x748f82ee5defb2fc,0x78a5636f43172f60 +.quad 0x748f82ee5defb2fc,0x78a5636f43172f60 +.quad 0x84c87814a1f0ab72,0x8cc702081a6439ec +.quad 0x84c87814a1f0ab72,0x8cc702081a6439ec +.quad 0x90befffa23631e28,0xa4506cebde82bde9 +.quad 0x90befffa23631e28,0xa4506cebde82bde9 +.quad 0xbef9a3f7b2c67915,0xc67178f2e372532b +.quad 0xbef9a3f7b2c67915,0xc67178f2e372532b +.quad 0xca273eceea26619c,0xd186b8c721c0c207 +.quad 0xca273eceea26619c,0xd186b8c721c0c207 +.quad 0xeada7dd6cde0eb1e,0xf57d4f7fee6ed178 +.quad 0xeada7dd6cde0eb1e,0xf57d4f7fee6ed178 +.quad 0x6f067aa72176fba,0xa637dc5a2c898a6 +.quad 0x6f067aa72176fba,0xa637dc5a2c898a6 +.quad 0x113f9804bef90dae,0x1b710b35131c471b +.quad 0x113f9804bef90dae,0x1b710b35131c471b +.quad 0x28db77f523047d84,0x32caab7b40c72493 +.quad 0x28db77f523047d84,0x32caab7b40c72493 +.quad 0x3c9ebe0a15c9bebc,0x431d67c49c100d4c +.quad 0x3c9ebe0a15c9bebc,0x431d67c49c100d4c +.quad 0x4cc5d4becb3e42b6,0x597f299cfc657e2a +.quad 0x4cc5d4becb3e42b6,0x597f299cfc657e2a +.quad 0x5fcb6fab3ad6faec,0x6c44198c4a475817 +.quad 0x5fcb6fab3ad6faec,0x6c44198c4a475817 +.align 8 +L_avx2_rorx_sha512_k_2_end: +.quad 1024+L_avx2_rorx_sha512_k_2 +.align 32 +L_avx2_rorx_sha512_flip_mask: +.quad 0x1020304050607, 0x8090a0b0c0d0e0f +.quad 0x1020304050607, 0x8090a0b0c0d0e0f +.globl Transform_Sha512_AVX2_RORX +.type Transform_Sha512_AVX2_RORX,@function +.align 4 +Transform_Sha512_AVX2_RORX: + pushq %rbx + pushq %r12 + pushq %r13 + pushq %r14 + pushq %r15 + subq $0x88, %rsp + leaq 64(%rdi), %rcx + vmovdqa L_avx2_rorx_sha512_flip_mask(%rip), %ymm15 + movq (%rdi), %r8 + movq 8(%rdi), %r9 + movq 16(%rdi), %r10 + movq 24(%rdi), %r11 + movq 32(%rdi), %r12 + movq 40(%rdi), %r13 + movq 48(%rdi), %r14 + movq 56(%rdi), %r15 + vmovdqu (%rcx), %ymm0 + vmovdqu 32(%rcx), %ymm1 + vpshufb %ymm15, %ymm0, %ymm0 + vpshufb %ymm15, %ymm1, %ymm1 + vmovdqu 64(%rcx), %ymm2 + vmovdqu 96(%rcx), %ymm3 + vpshufb %ymm15, %ymm2, %ymm2 + vpshufb %ymm15, %ymm3, %ymm3 + movl $4, 128(%rsp) + leaq L_avx2_rorx_sha512_k(%rip), %rsi + movq %r9, %rbx + xorq %rdx, %rdx + xorq %r10, %rbx + # set_w_k: 0 + vpaddq (%rsi), %ymm0, %ymm8 + vpaddq 32(%rsi), %ymm1, %ymm9 + vmovdqu %ymm8, (%rsp) + vmovdqu %ymm9, 32(%rsp) + vpaddq 64(%rsi), %ymm2, %ymm8 + vpaddq 96(%rsi), %ymm3, %ymm9 + vmovdqu %ymm8, 64(%rsp) + vmovdqu %ymm9, 96(%rsp) + # Start of 16 rounds +L_sha256_len_avx2_rorx_start: + addq $0x80, %rsi + rorxq $14, %r12, %rax + rorxq $18, %r12, %rcx + addq %rdx, %r8 + vpblendd $3, %ymm1, %ymm0, %ymm12 + vpblendd $3, %ymm3, %ymm2, %ymm13 + addq (%rsp), %r15 + movq %r13, %rdx + xorq %rax, %rcx + vpermq $57, %ymm12, %ymm12 + xorq %r14, %rdx + rorxq $41, %r12, %rax + xorq %rcx, %rax + vpermq $57, %ymm13, %ymm13 + andq %r12, %rdx + addq %rax, %r15 + rorxq $28, %r8, %rax + vpsrlq $0x01, %ymm12, %ymm8 + vpsllq $63, %ymm12, %ymm9 + rorxq $34, %r8, %rcx + xorq %r14, %rdx + xorq %rax, %rcx + vpsrlq $8, %ymm12, %ymm10 + vpsllq $56, %ymm12, %ymm11 + vpor %ymm9, %ymm8, %ymm8 + vpor %ymm11, %ymm10, %ymm10 + rorxq $39, %r8, %rax + addq %rdx, %r15 + xorq %rcx, %rax + vpsrlq $7, %ymm12, %ymm11 + movq %r9, %rdx + addq %r15, %r11 + xorq %r8, %rdx + vperm2I128 $0x81, %ymm3, %ymm3, %ymm14 + andq %rdx, %rbx + addq %rax, %r15 + xorq %r9, %rbx + rorxq $14, %r11, %rax + rorxq $18, %r11, %rcx + addq %rbx, %r15 + vpxor %ymm10, %ymm8, %ymm8 + addq 8(%rsp), %r14 + movq %r12, %rbx + xorq %rax, %rcx + vpxor %ymm11, %ymm8, %ymm8 + xorq %r13, %rbx + rorxq $41, %r11, %rax + xorq %rcx, %rax + vpaddq %ymm0, %ymm13, %ymm0 + vpaddq %ymm0, %ymm8, %ymm0 + andq %r11, %rbx + addq %rax, %r14 + rorxq $28, %r15, %rax + vpsrlq $19, %ymm14, %ymm8 + vpsllq $45, %ymm14, %ymm9 + rorxq $34, %r15, %rcx + xorq %r13, %rbx + xorq %rax, %rcx + vpsrlq $61, %ymm14, %ymm10 + vpsllq $3, %ymm14, %ymm11 + vpor %ymm9, %ymm8, %ymm8 + rorxq $39, %r15, %rax + addq %rbx, %r14 + xorq %rcx, %rax + vpor %ymm11, %ymm10, %ymm10 + movq %r8, %rbx + leaq (%r10,%r14,1), %r10 + xorq %r15, %rbx + vpxor %ymm10, %ymm8, %ymm8 + andq %rbx, %rdx + addq %rax, %r14 + xorq %r8, %rdx + vpsrlq $6, %ymm14, %ymm11 + rorxq $14, %r10, %rax + rorxq $18, %r10, %rcx + addq %rdx, %r14 + vpxor %ymm11, %ymm8, %ymm8 + addq 16(%rsp), %r13 + movq %r11, %rdx + xorq %rax, %rcx + vpaddq %ymm0, %ymm8, %ymm0 + xorq %r12, %rdx + rorxq $41, %r10, %rax + xorq %rcx, %rax + vperm2I128 $8, %ymm0, %ymm0, %ymm14 + andq %r10, %rdx + addq %rax, %r13 + rorxq $28, %r14, %rax + rorxq $34, %r14, %rcx + xorq %r12, %rdx + xorq %rax, %rcx + vpsrlq $19, %ymm14, %ymm8 + vpsllq $45, %ymm14, %ymm9 + rorxq $39, %r14, %rax + addq %rdx, %r13 + xorq %rcx, %rax + vpsrlq $61, %ymm14, %ymm10 + vpsllq $3, %ymm14, %ymm11 + vpor %ymm9, %ymm8, %ymm8 + movq %r15, %rdx + addq %r13, %r9 + xorq %r14, %rdx + vpor %ymm11, %ymm10, %ymm10 + andq %rdx, %rbx + addq %rax, %r13 + xorq %r15, %rbx + vpxor %ymm10, %ymm8, %ymm8 + rorxq $14, %r9, %rax + rorxq $18, %r9, %rcx + addq %rbx, %r13 + vpsrlq $6, %ymm14, %ymm11 + addq 24(%rsp), %r12 + movq %r10, %rbx + xorq %rax, %rcx + xorq %r11, %rbx + rorxq $41, %r9, %rax + xorq %rcx, %rax + vpxor %ymm11, %ymm8, %ymm8 + andq %r9, %rbx + addq %rax, %r12 + rorxq $28, %r13, %rax + rorxq $34, %r13, %rcx + xorq %r11, %rbx + xorq %rax, %rcx + vpaddq %ymm0, %ymm8, %ymm0 + rorxq $39, %r13, %rax + addq %rbx, %r12 + xorq %rcx, %rax + vpaddq (%rsi), %ymm0, %ymm8 + movq %r14, %rbx + leaq (%r8,%r12,1), %r8 + xorq %r13, %rbx + andq %rbx, %rdx + addq %rax, %r12 + xorq %r14, %rdx + vmovdqu %ymm8, (%rsp) + rorxq $14, %r8, %rax + rorxq $18, %r8, %rcx + addq %rdx, %r12 + vpblendd $3, %ymm2, %ymm1, %ymm12 + vpblendd $3, %ymm0, %ymm3, %ymm13 + addq 32(%rsp), %r11 + movq %r9, %rdx + xorq %rax, %rcx + vpermq $57, %ymm12, %ymm12 + xorq %r10, %rdx + rorxq $41, %r8, %rax + xorq %rcx, %rax + vpermq $57, %ymm13, %ymm13 + andq %r8, %rdx + addq %rax, %r11 + rorxq $28, %r12, %rax + vpsrlq $0x01, %ymm12, %ymm8 + vpsllq $63, %ymm12, %ymm9 + rorxq $34, %r12, %rcx + xorq %r10, %rdx + xorq %rax, %rcx + vpsrlq $8, %ymm12, %ymm10 + vpsllq $56, %ymm12, %ymm11 + vpor %ymm9, %ymm8, %ymm8 + vpor %ymm11, %ymm10, %ymm10 + rorxq $39, %r12, %rax + addq %rdx, %r11 + xorq %rcx, %rax + vpsrlq $7, %ymm12, %ymm11 + movq %r13, %rdx + addq %r11, %r15 + xorq %r12, %rdx + vperm2I128 $0x81, %ymm0, %ymm0, %ymm14 + andq %rdx, %rbx + addq %rax, %r11 + xorq %r13, %rbx + rorxq $14, %r15, %rax + rorxq $18, %r15, %rcx + addq %rbx, %r11 + vpxor %ymm10, %ymm8, %ymm8 + addq 40(%rsp), %r10 + movq %r8, %rbx + xorq %rax, %rcx + vpxor %ymm11, %ymm8, %ymm8 + xorq %r9, %rbx + rorxq $41, %r15, %rax + xorq %rcx, %rax + vpaddq %ymm1, %ymm13, %ymm1 + vpaddq %ymm1, %ymm8, %ymm1 + andq %r15, %rbx + addq %rax, %r10 + rorxq $28, %r11, %rax + vpsrlq $19, %ymm14, %ymm8 + vpsllq $45, %ymm14, %ymm9 + rorxq $34, %r11, %rcx + xorq %r9, %rbx + xorq %rax, %rcx + vpsrlq $61, %ymm14, %ymm10 + vpsllq $3, %ymm14, %ymm11 + vpor %ymm9, %ymm8, %ymm8 + rorxq $39, %r11, %rax + addq %rbx, %r10 + xorq %rcx, %rax + vpor %ymm11, %ymm10, %ymm10 + movq %r12, %rbx + leaq (%r14,%r10,1), %r14 + xorq %r11, %rbx + vpxor %ymm10, %ymm8, %ymm8 + andq %rbx, %rdx + addq %rax, %r10 + xorq %r12, %rdx + vpsrlq $6, %ymm14, %ymm11 + rorxq $14, %r14, %rax + rorxq $18, %r14, %rcx + addq %rdx, %r10 + vpxor %ymm11, %ymm8, %ymm8 + addq 48(%rsp), %r9 + movq %r15, %rdx + xorq %rax, %rcx + vpaddq %ymm1, %ymm8, %ymm1 + xorq %r8, %rdx + rorxq $41, %r14, %rax + xorq %rcx, %rax + vperm2I128 $8, %ymm1, %ymm1, %ymm14 + andq %r14, %rdx + addq %rax, %r9 + rorxq $28, %r10, %rax + rorxq $34, %r10, %rcx + xorq %r8, %rdx + xorq %rax, %rcx + vpsrlq $19, %ymm14, %ymm8 + vpsllq $45, %ymm14, %ymm9 + rorxq $39, %r10, %rax + addq %rdx, %r9 + xorq %rcx, %rax + vpsrlq $61, %ymm14, %ymm10 + vpsllq $3, %ymm14, %ymm11 + vpor %ymm9, %ymm8, %ymm8 + movq %r11, %rdx + addq %r9, %r13 + xorq %r10, %rdx + vpor %ymm11, %ymm10, %ymm10 + andq %rdx, %rbx + addq %rax, %r9 + xorq %r11, %rbx + vpxor %ymm10, %ymm8, %ymm8 + rorxq $14, %r13, %rax + rorxq $18, %r13, %rcx + addq %rbx, %r9 + vpsrlq $6, %ymm14, %ymm11 + addq 56(%rsp), %r8 + movq %r14, %rbx + xorq %rax, %rcx + xorq %r15, %rbx + rorxq $41, %r13, %rax + xorq %rcx, %rax + vpxor %ymm11, %ymm8, %ymm8 + andq %r13, %rbx + addq %rax, %r8 + rorxq $28, %r9, %rax + rorxq $34, %r9, %rcx + xorq %r15, %rbx + xorq %rax, %rcx + vpaddq %ymm1, %ymm8, %ymm1 + rorxq $39, %r9, %rax + addq %rbx, %r8 + xorq %rcx, %rax + vpaddq 32(%rsi), %ymm1, %ymm8 + movq %r10, %rbx + leaq (%r12,%r8,1), %r12 + xorq %r9, %rbx + andq %rbx, %rdx + addq %rax, %r8 + xorq %r10, %rdx + vmovdqu %ymm8, 32(%rsp) + rorxq $14, %r12, %rax + rorxq $18, %r12, %rcx + addq %rdx, %r8 + vpblendd $3, %ymm3, %ymm2, %ymm12 + vpblendd $3, %ymm1, %ymm0, %ymm13 + addq 64(%rsp), %r15 + movq %r13, %rdx + xorq %rax, %rcx + vpermq $57, %ymm12, %ymm12 + xorq %r14, %rdx + rorxq $41, %r12, %rax + xorq %rcx, %rax + vpermq $57, %ymm13, %ymm13 + andq %r12, %rdx + addq %rax, %r15 + rorxq $28, %r8, %rax + vpsrlq $0x01, %ymm12, %ymm8 + vpsllq $63, %ymm12, %ymm9 + rorxq $34, %r8, %rcx + xorq %r14, %rdx + xorq %rax, %rcx + vpsrlq $8, %ymm12, %ymm10 + vpsllq $56, %ymm12, %ymm11 + vpor %ymm9, %ymm8, %ymm8 + vpor %ymm11, %ymm10, %ymm10 + rorxq $39, %r8, %rax + addq %rdx, %r15 + xorq %rcx, %rax + vpsrlq $7, %ymm12, %ymm11 + movq %r9, %rdx + addq %r15, %r11 + xorq %r8, %rdx + vperm2I128 $0x81, %ymm1, %ymm1, %ymm14 + andq %rdx, %rbx + addq %rax, %r15 + xorq %r9, %rbx + rorxq $14, %r11, %rax + rorxq $18, %r11, %rcx + addq %rbx, %r15 + vpxor %ymm10, %ymm8, %ymm8 + addq 72(%rsp), %r14 + movq %r12, %rbx + xorq %rax, %rcx + vpxor %ymm11, %ymm8, %ymm8 + xorq %r13, %rbx + rorxq $41, %r11, %rax + xorq %rcx, %rax + vpaddq %ymm2, %ymm13, %ymm2 + vpaddq %ymm2, %ymm8, %ymm2 + andq %r11, %rbx + addq %rax, %r14 + rorxq $28, %r15, %rax + vpsrlq $19, %ymm14, %ymm8 + vpsllq $45, %ymm14, %ymm9 + rorxq $34, %r15, %rcx + xorq %r13, %rbx + xorq %rax, %rcx + vpsrlq $61, %ymm14, %ymm10 + vpsllq $3, %ymm14, %ymm11 + vpor %ymm9, %ymm8, %ymm8 + rorxq $39, %r15, %rax + addq %rbx, %r14 + xorq %rcx, %rax + vpor %ymm11, %ymm10, %ymm10 + movq %r8, %rbx + leaq (%r10,%r14,1), %r10 + xorq %r15, %rbx + vpxor %ymm10, %ymm8, %ymm8 + andq %rbx, %rdx + addq %rax, %r14 + xorq %r8, %rdx + vpsrlq $6, %ymm14, %ymm11 + rorxq $14, %r10, %rax + rorxq $18, %r10, %rcx + addq %rdx, %r14 + vpxor %ymm11, %ymm8, %ymm8 + addq 80(%rsp), %r13 + movq %r11, %rdx + xorq %rax, %rcx + vpaddq %ymm2, %ymm8, %ymm2 + xorq %r12, %rdx + rorxq $41, %r10, %rax + xorq %rcx, %rax + vperm2I128 $8, %ymm2, %ymm2, %ymm14 + andq %r10, %rdx + addq %rax, %r13 + rorxq $28, %r14, %rax + rorxq $34, %r14, %rcx + xorq %r12, %rdx + xorq %rax, %rcx + vpsrlq $19, %ymm14, %ymm8 + vpsllq $45, %ymm14, %ymm9 + rorxq $39, %r14, %rax + addq %rdx, %r13 + xorq %rcx, %rax + vpsrlq $61, %ymm14, %ymm10 + vpsllq $3, %ymm14, %ymm11 + vpor %ymm9, %ymm8, %ymm8 + movq %r15, %rdx + addq %r13, %r9 + xorq %r14, %rdx + vpor %ymm11, %ymm10, %ymm10 + andq %rdx, %rbx + addq %rax, %r13 + xorq %r15, %rbx + vpxor %ymm10, %ymm8, %ymm8 + rorxq $14, %r9, %rax + rorxq $18, %r9, %rcx + addq %rbx, %r13 + vpsrlq $6, %ymm14, %ymm11 + addq 88(%rsp), %r12 + movq %r10, %rbx + xorq %rax, %rcx + xorq %r11, %rbx + rorxq $41, %r9, %rax + xorq %rcx, %rax + vpxor %ymm11, %ymm8, %ymm8 + andq %r9, %rbx + addq %rax, %r12 + rorxq $28, %r13, %rax + rorxq $34, %r13, %rcx + xorq %r11, %rbx + xorq %rax, %rcx + vpaddq %ymm2, %ymm8, %ymm2 + rorxq $39, %r13, %rax + addq %rbx, %r12 + xorq %rcx, %rax + vpaddq 64(%rsi), %ymm2, %ymm8 + movq %r14, %rbx + leaq (%r8,%r12,1), %r8 + xorq %r13, %rbx + andq %rbx, %rdx + addq %rax, %r12 + xorq %r14, %rdx + vmovdqu %ymm8, 64(%rsp) + rorxq $14, %r8, %rax + rorxq $18, %r8, %rcx + addq %rdx, %r12 + vpblendd $3, %ymm0, %ymm3, %ymm12 + vpblendd $3, %ymm2, %ymm1, %ymm13 + addq 96(%rsp), %r11 + movq %r9, %rdx + xorq %rax, %rcx + vpermq $57, %ymm12, %ymm12 + xorq %r10, %rdx + rorxq $41, %r8, %rax + xorq %rcx, %rax + vpermq $57, %ymm13, %ymm13 + andq %r8, %rdx + addq %rax, %r11 + rorxq $28, %r12, %rax + vpsrlq $0x01, %ymm12, %ymm8 + vpsllq $63, %ymm12, %ymm9 + rorxq $34, %r12, %rcx + xorq %r10, %rdx + xorq %rax, %rcx + vpsrlq $8, %ymm12, %ymm10 + vpsllq $56, %ymm12, %ymm11 + vpor %ymm9, %ymm8, %ymm8 + vpor %ymm11, %ymm10, %ymm10 + rorxq $39, %r12, %rax + addq %rdx, %r11 + xorq %rcx, %rax + vpsrlq $7, %ymm12, %ymm11 + movq %r13, %rdx + addq %r11, %r15 + xorq %r12, %rdx + vperm2I128 $0x81, %ymm2, %ymm2, %ymm14 + andq %rdx, %rbx + addq %rax, %r11 + xorq %r13, %rbx + rorxq $14, %r15, %rax + rorxq $18, %r15, %rcx + addq %rbx, %r11 + vpxor %ymm10, %ymm8, %ymm8 + addq 104(%rsp), %r10 + movq %r8, %rbx + xorq %rax, %rcx + vpxor %ymm11, %ymm8, %ymm8 + xorq %r9, %rbx + rorxq $41, %r15, %rax + xorq %rcx, %rax + vpaddq %ymm3, %ymm13, %ymm3 + vpaddq %ymm3, %ymm8, %ymm3 + andq %r15, %rbx + addq %rax, %r10 + rorxq $28, %r11, %rax + vpsrlq $19, %ymm14, %ymm8 + vpsllq $45, %ymm14, %ymm9 + rorxq $34, %r11, %rcx + xorq %r9, %rbx + xorq %rax, %rcx + vpsrlq $61, %ymm14, %ymm10 + vpsllq $3, %ymm14, %ymm11 + vpor %ymm9, %ymm8, %ymm8 + rorxq $39, %r11, %rax + addq %rbx, %r10 + xorq %rcx, %rax + vpor %ymm11, %ymm10, %ymm10 + movq %r12, %rbx + leaq (%r14,%r10,1), %r14 + xorq %r11, %rbx + vpxor %ymm10, %ymm8, %ymm8 + andq %rbx, %rdx + addq %rax, %r10 + xorq %r12, %rdx + vpsrlq $6, %ymm14, %ymm11 + rorxq $14, %r14, %rax + rorxq $18, %r14, %rcx + addq %rdx, %r10 + vpxor %ymm11, %ymm8, %ymm8 + addq 112(%rsp), %r9 + movq %r15, %rdx + xorq %rax, %rcx + vpaddq %ymm3, %ymm8, %ymm3 + xorq %r8, %rdx + rorxq $41, %r14, %rax + xorq %rcx, %rax + vperm2I128 $8, %ymm3, %ymm3, %ymm14 + andq %r14, %rdx + addq %rax, %r9 + rorxq $28, %r10, %rax + rorxq $34, %r10, %rcx + xorq %r8, %rdx + xorq %rax, %rcx + vpsrlq $19, %ymm14, %ymm8 + vpsllq $45, %ymm14, %ymm9 + rorxq $39, %r10, %rax + addq %rdx, %r9 + xorq %rcx, %rax + vpsrlq $61, %ymm14, %ymm10 + vpsllq $3, %ymm14, %ymm11 + vpor %ymm9, %ymm8, %ymm8 + movq %r11, %rdx + addq %r9, %r13 + xorq %r10, %rdx + vpor %ymm11, %ymm10, %ymm10 + andq %rdx, %rbx + addq %rax, %r9 + xorq %r11, %rbx + vpxor %ymm10, %ymm8, %ymm8 + rorxq $14, %r13, %rax + rorxq $18, %r13, %rcx + addq %rbx, %r9 + vpsrlq $6, %ymm14, %ymm11 + addq 120(%rsp), %r8 + movq %r14, %rbx + xorq %rax, %rcx + xorq %r15, %rbx + rorxq $41, %r13, %rax + xorq %rcx, %rax + vpxor %ymm11, %ymm8, %ymm8 + andq %r13, %rbx + addq %rax, %r8 + rorxq $28, %r9, %rax + rorxq $34, %r9, %rcx + xorq %r15, %rbx + xorq %rax, %rcx + vpaddq %ymm3, %ymm8, %ymm3 + rorxq $39, %r9, %rax + addq %rbx, %r8 + xorq %rcx, %rax + vpaddq 96(%rsi), %ymm3, %ymm8 + movq %r10, %rbx + leaq (%r12,%r8,1), %r12 + xorq %r9, %rbx + andq %rbx, %rdx + addq %rax, %r8 + xorq %r10, %rdx + vmovdqu %ymm8, 96(%rsp) + subl $0x01, 128(%rsp) + jne L_sha256_len_avx2_rorx_start + # rnd_all_4: 0-3 + rorxq $14, %r12, %rax + rorxq $18, %r12, %rcx + addq %rdx, %r8 + addq (%rsp), %r15 + movq %r13, %rdx + xorq %rax, %rcx + xorq %r14, %rdx + rorxq $41, %r12, %rax + xorq %rcx, %rax + andq %r12, %rdx + addq %rax, %r15 + rorxq $28, %r8, %rax + rorxq $34, %r8, %rcx + xorq %r14, %rdx + xorq %rax, %rcx + rorxq $39, %r8, %rax + addq %rdx, %r15 + xorq %rcx, %rax + movq %r9, %rdx + addq %r15, %r11 + xorq %r8, %rdx + andq %rdx, %rbx + addq %rax, %r15 + xorq %r9, %rbx + rorxq $14, %r11, %rax + rorxq $18, %r11, %rcx + addq %rbx, %r15 + addq 8(%rsp), %r14 + movq %r12, %rbx + xorq %rax, %rcx + xorq %r13, %rbx + rorxq $41, %r11, %rax + xorq %rcx, %rax + andq %r11, %rbx + addq %rax, %r14 + rorxq $28, %r15, %rax + rorxq $34, %r15, %rcx + xorq %r13, %rbx + xorq %rax, %rcx + rorxq $39, %r15, %rax + addq %rbx, %r14 + xorq %rcx, %rax + movq %r8, %rbx + leaq (%r10,%r14,1), %r10 + xorq %r15, %rbx + andq %rbx, %rdx + addq %rax, %r14 + xorq %r8, %rdx + rorxq $14, %r10, %rax + rorxq $18, %r10, %rcx + addq %rdx, %r14 + addq 16(%rsp), %r13 + movq %r11, %rdx + xorq %rax, %rcx + xorq %r12, %rdx + rorxq $41, %r10, %rax + xorq %rcx, %rax + andq %r10, %rdx + addq %rax, %r13 + rorxq $28, %r14, %rax + rorxq $34, %r14, %rcx + xorq %r12, %rdx + xorq %rax, %rcx + rorxq $39, %r14, %rax + addq %rdx, %r13 + xorq %rcx, %rax + movq %r15, %rdx + addq %r13, %r9 + xorq %r14, %rdx + andq %rdx, %rbx + addq %rax, %r13 + xorq %r15, %rbx + rorxq $14, %r9, %rax + rorxq $18, %r9, %rcx + addq %rbx, %r13 + addq 24(%rsp), %r12 + movq %r10, %rbx + xorq %rax, %rcx + xorq %r11, %rbx + rorxq $41, %r9, %rax + xorq %rcx, %rax + andq %r9, %rbx + addq %rax, %r12 + rorxq $28, %r13, %rax + rorxq $34, %r13, %rcx + xorq %r11, %rbx + xorq %rax, %rcx + rorxq $39, %r13, %rax + addq %rbx, %r12 + xorq %rcx, %rax + movq %r14, %rbx + leaq (%r8,%r12,1), %r8 + xorq %r13, %rbx + andq %rbx, %rdx + addq %rax, %r12 + xorq %r14, %rdx + # rnd_all_4: 4-7 + rorxq $14, %r8, %rax + rorxq $18, %r8, %rcx + addq %rdx, %r12 + addq 32(%rsp), %r11 + movq %r9, %rdx + xorq %rax, %rcx + xorq %r10, %rdx + rorxq $41, %r8, %rax + xorq %rcx, %rax + andq %r8, %rdx + addq %rax, %r11 + rorxq $28, %r12, %rax + rorxq $34, %r12, %rcx + xorq %r10, %rdx + xorq %rax, %rcx + rorxq $39, %r12, %rax + addq %rdx, %r11 + xorq %rcx, %rax + movq %r13, %rdx + addq %r11, %r15 + xorq %r12, %rdx + andq %rdx, %rbx + addq %rax, %r11 + xorq %r13, %rbx + rorxq $14, %r15, %rax + rorxq $18, %r15, %rcx + addq %rbx, %r11 + addq 40(%rsp), %r10 + movq %r8, %rbx + xorq %rax, %rcx + xorq %r9, %rbx + rorxq $41, %r15, %rax + xorq %rcx, %rax + andq %r15, %rbx + addq %rax, %r10 + rorxq $28, %r11, %rax + rorxq $34, %r11, %rcx + xorq %r9, %rbx + xorq %rax, %rcx + rorxq $39, %r11, %rax + addq %rbx, %r10 + xorq %rcx, %rax + movq %r12, %rbx + leaq (%r14,%r10,1), %r14 + xorq %r11, %rbx + andq %rbx, %rdx + addq %rax, %r10 + xorq %r12, %rdx + rorxq $14, %r14, %rax + rorxq $18, %r14, %rcx + addq %rdx, %r10 + addq 48(%rsp), %r9 + movq %r15, %rdx + xorq %rax, %rcx + xorq %r8, %rdx + rorxq $41, %r14, %rax + xorq %rcx, %rax + andq %r14, %rdx + addq %rax, %r9 + rorxq $28, %r10, %rax + rorxq $34, %r10, %rcx + xorq %r8, %rdx + xorq %rax, %rcx + rorxq $39, %r10, %rax + addq %rdx, %r9 + xorq %rcx, %rax + movq %r11, %rdx + addq %r9, %r13 + xorq %r10, %rdx + andq %rdx, %rbx + addq %rax, %r9 + xorq %r11, %rbx + rorxq $14, %r13, %rax + rorxq $18, %r13, %rcx + addq %rbx, %r9 + addq 56(%rsp), %r8 + movq %r14, %rbx + xorq %rax, %rcx + xorq %r15, %rbx + rorxq $41, %r13, %rax + xorq %rcx, %rax + andq %r13, %rbx + addq %rax, %r8 + rorxq $28, %r9, %rax + rorxq $34, %r9, %rcx + xorq %r15, %rbx + xorq %rax, %rcx + rorxq $39, %r9, %rax + addq %rbx, %r8 + xorq %rcx, %rax + movq %r10, %rbx + leaq (%r12,%r8,1), %r12 + xorq %r9, %rbx + andq %rbx, %rdx + addq %rax, %r8 + xorq %r10, %rdx + # rnd_all_4: 8-11 + rorxq $14, %r12, %rax + rorxq $18, %r12, %rcx + addq %rdx, %r8 + addq 64(%rsp), %r15 + movq %r13, %rdx + xorq %rax, %rcx + xorq %r14, %rdx + rorxq $41, %r12, %rax + xorq %rcx, %rax + andq %r12, %rdx + addq %rax, %r15 + rorxq $28, %r8, %rax + rorxq $34, %r8, %rcx + xorq %r14, %rdx + xorq %rax, %rcx + rorxq $39, %r8, %rax + addq %rdx, %r15 + xorq %rcx, %rax + movq %r9, %rdx + addq %r15, %r11 + xorq %r8, %rdx + andq %rdx, %rbx + addq %rax, %r15 + xorq %r9, %rbx + rorxq $14, %r11, %rax + rorxq $18, %r11, %rcx + addq %rbx, %r15 + addq 72(%rsp), %r14 + movq %r12, %rbx + xorq %rax, %rcx + xorq %r13, %rbx + rorxq $41, %r11, %rax + xorq %rcx, %rax + andq %r11, %rbx + addq %rax, %r14 + rorxq $28, %r15, %rax + rorxq $34, %r15, %rcx + xorq %r13, %rbx + xorq %rax, %rcx + rorxq $39, %r15, %rax + addq %rbx, %r14 + xorq %rcx, %rax + movq %r8, %rbx + leaq (%r10,%r14,1), %r10 + xorq %r15, %rbx + andq %rbx, %rdx + addq %rax, %r14 + xorq %r8, %rdx + rorxq $14, %r10, %rax + rorxq $18, %r10, %rcx + addq %rdx, %r14 + addq 80(%rsp), %r13 + movq %r11, %rdx + xorq %rax, %rcx + xorq %r12, %rdx + rorxq $41, %r10, %rax + xorq %rcx, %rax + andq %r10, %rdx + addq %rax, %r13 + rorxq $28, %r14, %rax + rorxq $34, %r14, %rcx + xorq %r12, %rdx + xorq %rax, %rcx + rorxq $39, %r14, %rax + addq %rdx, %r13 + xorq %rcx, %rax + movq %r15, %rdx + addq %r13, %r9 + xorq %r14, %rdx + andq %rdx, %rbx + addq %rax, %r13 + xorq %r15, %rbx + rorxq $14, %r9, %rax + rorxq $18, %r9, %rcx + addq %rbx, %r13 + addq 88(%rsp), %r12 + movq %r10, %rbx + xorq %rax, %rcx + xorq %r11, %rbx + rorxq $41, %r9, %rax + xorq %rcx, %rax + andq %r9, %rbx + addq %rax, %r12 + rorxq $28, %r13, %rax + rorxq $34, %r13, %rcx + xorq %r11, %rbx + xorq %rax, %rcx + rorxq $39, %r13, %rax + addq %rbx, %r12 + xorq %rcx, %rax + movq %r14, %rbx + leaq (%r8,%r12,1), %r8 + xorq %r13, %rbx + andq %rbx, %rdx + addq %rax, %r12 + xorq %r14, %rdx + # rnd_all_4: 12-15 + rorxq $14, %r8, %rax + rorxq $18, %r8, %rcx + addq %rdx, %r12 + addq 96(%rsp), %r11 + movq %r9, %rdx + xorq %rax, %rcx + xorq %r10, %rdx + rorxq $41, %r8, %rax + xorq %rcx, %rax + andq %r8, %rdx + addq %rax, %r11 + rorxq $28, %r12, %rax + rorxq $34, %r12, %rcx + xorq %r10, %rdx + xorq %rax, %rcx + rorxq $39, %r12, %rax + addq %rdx, %r11 + xorq %rcx, %rax + movq %r13, %rdx + addq %r11, %r15 + xorq %r12, %rdx + andq %rdx, %rbx + addq %rax, %r11 + xorq %r13, %rbx + rorxq $14, %r15, %rax + rorxq $18, %r15, %rcx + addq %rbx, %r11 + addq 104(%rsp), %r10 + movq %r8, %rbx + xorq %rax, %rcx + xorq %r9, %rbx + rorxq $41, %r15, %rax + xorq %rcx, %rax + andq %r15, %rbx + addq %rax, %r10 + rorxq $28, %r11, %rax + rorxq $34, %r11, %rcx + xorq %r9, %rbx + xorq %rax, %rcx + rorxq $39, %r11, %rax + addq %rbx, %r10 + xorq %rcx, %rax + movq %r12, %rbx + leaq (%r14,%r10,1), %r14 + xorq %r11, %rbx + andq %rbx, %rdx + addq %rax, %r10 + xorq %r12, %rdx + rorxq $14, %r14, %rax + rorxq $18, %r14, %rcx + addq %rdx, %r10 + addq 112(%rsp), %r9 + movq %r15, %rdx + xorq %rax, %rcx + xorq %r8, %rdx + rorxq $41, %r14, %rax + xorq %rcx, %rax + andq %r14, %rdx + addq %rax, %r9 + rorxq $28, %r10, %rax + rorxq $34, %r10, %rcx + xorq %r8, %rdx + xorq %rax, %rcx + rorxq $39, %r10, %rax + addq %rdx, %r9 + xorq %rcx, %rax + movq %r11, %rdx + addq %r9, %r13 + xorq %r10, %rdx + andq %rdx, %rbx + addq %rax, %r9 + xorq %r11, %rbx + rorxq $14, %r13, %rax + rorxq $18, %r13, %rcx + addq %rbx, %r9 + addq 120(%rsp), %r8 + movq %r14, %rbx + xorq %rax, %rcx + xorq %r15, %rbx + rorxq $41, %r13, %rax + xorq %rcx, %rax + andq %r13, %rbx + addq %rax, %r8 + rorxq $28, %r9, %rax + rorxq $34, %r9, %rcx + xorq %r15, %rbx + xorq %rax, %rcx + rorxq $39, %r9, %rax + addq %rbx, %r8 + xorq %rcx, %rax + movq %r10, %rbx + leaq (%r12,%r8,1), %r12 + xorq %r9, %rbx + andq %rbx, %rdx + addq %rax, %r8 + xorq %r10, %rdx + addq %rdx, %r8 + addq %r8, (%rdi) + addq %r9, 8(%rdi) + addq %r10, 16(%rdi) + addq %r11, 24(%rdi) + addq %r12, 32(%rdi) + addq %r13, 40(%rdi) + addq %r14, 48(%rdi) + addq %r15, 56(%rdi) + xorq %rax, %rax + vzeroupper + addq $0x88, %rsp + popq %r15 + popq %r14 + popq %r13 + popq %r12 + popq %rbx + repz retq +.size Transform_Sha512_AVX2_RORX,.-Transform_Sha512_AVX2_RORX +.globl Transform_Sha512_AVX2_RORX_Len +.type Transform_Sha512_AVX2_RORX_Len,@function +.align 4 +Transform_Sha512_AVX2_RORX_Len: + pushq %rbx + pushq %r12 + pushq %r13 + pushq %r14 + pushq %r15 + pushq %rbp + testb $0x80, %sil + je L_sha512_len_avx2_rorx_block + movq 224(%rdi), %rax + push %rsi + vmovdqu (%rax), %ymm0 + vmovdqu 32(%rax), %ymm1 + vmovdqu 64(%rax), %ymm2 + vmovdqu 96(%rax), %ymm3 + vmovups %ymm0, 64(%rdi) + vmovups %ymm1, 96(%rdi) + vmovups %ymm2, 128(%rdi) + vmovups %ymm3, 160(%rdi) + call Transform_Sha512_AVX2_RORX@plt + pop %rsi + addq $0x80, 224(%rdi) + subl $0x80, %esi + jz L_sha512_len_avx2_rorx_done +L_sha512_len_avx2_rorx_block: + movq 224(%rdi), %rax + vmovdqa L_avx2_rorx_sha512_flip_mask(%rip), %ymm15 + movq (%rdi), %r8 + movq 8(%rdi), %r9 + movq 16(%rdi), %r10 + movq 24(%rdi), %r11 + movq 32(%rdi), %r12 + movq 40(%rdi), %r13 + movq 48(%rdi), %r14 + movq 56(%rdi), %r15 + # Start of loop processing two blocks +L_sha512_len_avx2_rorx_begin: + subq $0x540, %rsp + leaq L_avx2_rorx_sha512_k_2(%rip), %rbp + movq %r9, %rbx + xorq %rdx, %rdx + vmovdqu (%rax), %xmm0 + vmovdqu 16(%rax), %xmm1 + vinserti128 $0x01, 128(%rax), %ymm0, %ymm0 + vinserti128 $0x01, 144(%rax), %ymm1, %ymm1 + vpshufb %ymm15, %ymm0, %ymm0 + vpshufb %ymm15, %ymm1, %ymm1 + vmovdqu 32(%rax), %xmm2 + vmovdqu 48(%rax), %xmm3 + vinserti128 $0x01, 160(%rax), %ymm2, %ymm2 + vinserti128 $0x01, 176(%rax), %ymm3, %ymm3 + vpshufb %ymm15, %ymm2, %ymm2 + vpshufb %ymm15, %ymm3, %ymm3 + vmovdqu 64(%rax), %xmm4 + vmovdqu 80(%rax), %xmm5 + vinserti128 $0x01, 192(%rax), %ymm4, %ymm4 + vinserti128 $0x01, 208(%rax), %ymm5, %ymm5 + vpshufb %ymm15, %ymm4, %ymm4 + vpshufb %ymm15, %ymm5, %ymm5 + vmovdqu 96(%rax), %xmm6 + vmovdqu 112(%rax), %xmm7 + vinserti128 $0x01, 224(%rax), %ymm6, %ymm6 + vinserti128 $0x01, 240(%rax), %ymm7, %ymm7 + vpshufb %ymm15, %ymm6, %ymm6 + vpshufb %ymm15, %ymm7, %ymm7 + xorq %r10, %rbx + # Start of 16 rounds +L_sha512_len_avx2_rorx_start: + vpaddq (%rbp), %ymm0, %ymm8 + vpaddq 32(%rbp), %ymm1, %ymm9 + vmovdqu %ymm8, (%rsp) + vmovdqu %ymm9, 32(%rsp) + vpaddq 64(%rbp), %ymm2, %ymm8 + vpaddq 96(%rbp), %ymm3, %ymm9 + vmovdqu %ymm8, 64(%rsp) + vmovdqu %ymm9, 96(%rsp) + vpaddq 128(%rbp), %ymm4, %ymm8 + vpaddq 160(%rbp), %ymm5, %ymm9 + vmovdqu %ymm8, 128(%rsp) + vmovdqu %ymm9, 160(%rsp) + vpaddq 192(%rbp), %ymm6, %ymm8 + vpaddq 224(%rbp), %ymm7, %ymm9 + vmovdqu %ymm8, 192(%rsp) + vmovdqu %ymm9, 224(%rsp) + # msg_sched: 0-1 + rorxq $14, %r12, %rax + rorxq $18, %r12, %rcx + addq %rdx, %r8 + vpalignr $8, %ymm0, %ymm1, %ymm12 + addq (%rsp), %r15 + movq %r13, %rdx + xorq %rax, %rcx + vpalignr $8, %ymm4, %ymm5, %ymm13 + xorq %r14, %rdx + rorxq $41, %r12, %rax + xorq %rcx, %rax + vpsrlq $0x01, %ymm12, %ymm8 + vpsllq $63, %ymm12, %ymm9 + andq %r12, %rdx + addq %rax, %r15 + rorxq $28, %r8, %rax + vpsrlq $8, %ymm12, %ymm10 + vpsllq $56, %ymm12, %ymm11 + rorxq $34, %r8, %rcx + xorq %r14, %rdx + xorq %rax, %rcx + vpor %ymm9, %ymm8, %ymm8 + vpor %ymm11, %ymm10, %ymm10 + rorxq $39, %r8, %rax + addq %rdx, %r15 + xorq %rcx, %rax + vpsrlq $7, %ymm12, %ymm11 + vpxor %ymm10, %ymm8, %ymm8 + movq %r9, %rdx + addq %r15, %r11 + xorq %r8, %rdx + vpxor %ymm11, %ymm8, %ymm8 + vpaddq %ymm0, %ymm13, %ymm0 + andq %rdx, %rbx + addq %rax, %r15 + xorq %r9, %rbx + vpaddq %ymm0, %ymm8, %ymm0 + rorxq $14, %r11, %rax + rorxq $18, %r11, %rcx + addq %rbx, %r15 + vpsrlq $19, %ymm7, %ymm8 + vpsllq $45, %ymm7, %ymm9 + addq 8(%rsp), %r14 + movq %r12, %rbx + xorq %rax, %rcx + vpsrlq $61, %ymm7, %ymm10 + vpsllq $3, %ymm7, %ymm11 + xorq %r13, %rbx + rorxq $41, %r11, %rax + xorq %rcx, %rax + vpor %ymm9, %ymm8, %ymm8 + vpor %ymm11, %ymm10, %ymm10 + andq %r11, %rbx + addq %rax, %r14 + rorxq $28, %r15, %rax + rorxq $34, %r15, %rcx + xorq %r13, %rbx + xorq %rax, %rcx + vpxor %ymm10, %ymm8, %ymm8 + vpsrlq $6, %ymm7, %ymm11 + rorxq $39, %r15, %rax + addq %rbx, %r14 + xorq %rcx, %rax + movq %r8, %rbx + leaq (%r10,%r14,1), %r10 + xorq %r15, %rbx + vpxor %ymm11, %ymm8, %ymm8 + andq %rbx, %rdx + addq %rax, %r14 + xorq %r8, %rdx + vpaddq %ymm0, %ymm8, %ymm0 + # msg_sched done: 0-3 + # msg_sched: 4-5 + rorxq $14, %r10, %rax + rorxq $18, %r10, %rcx + addq %rdx, %r14 + vpalignr $8, %ymm1, %ymm2, %ymm12 + addq 32(%rsp), %r13 + movq %r11, %rdx + xorq %rax, %rcx + vpalignr $8, %ymm5, %ymm6, %ymm13 + xorq %r12, %rdx + rorxq $41, %r10, %rax + xorq %rcx, %rax + vpsrlq $0x01, %ymm12, %ymm8 + vpsllq $63, %ymm12, %ymm9 + andq %r10, %rdx + addq %rax, %r13 + rorxq $28, %r14, %rax + vpsrlq $8, %ymm12, %ymm10 + vpsllq $56, %ymm12, %ymm11 + rorxq $34, %r14, %rcx + xorq %r12, %rdx + xorq %rax, %rcx + vpor %ymm9, %ymm8, %ymm8 + vpor %ymm11, %ymm10, %ymm10 + rorxq $39, %r14, %rax + addq %rdx, %r13 + xorq %rcx, %rax + vpsrlq $7, %ymm12, %ymm11 + vpxor %ymm10, %ymm8, %ymm8 + movq %r15, %rdx + addq %r13, %r9 + xorq %r14, %rdx + vpxor %ymm11, %ymm8, %ymm8 + vpaddq %ymm1, %ymm13, %ymm1 + andq %rdx, %rbx + addq %rax, %r13 + xorq %r15, %rbx + vpaddq %ymm1, %ymm8, %ymm1 + rorxq $14, %r9, %rax + rorxq $18, %r9, %rcx + addq %rbx, %r13 + vpsrlq $19, %ymm0, %ymm8 + vpsllq $45, %ymm0, %ymm9 + addq 40(%rsp), %r12 + movq %r10, %rbx + xorq %rax, %rcx + vpsrlq $61, %ymm0, %ymm10 + vpsllq $3, %ymm0, %ymm11 + xorq %r11, %rbx + rorxq $41, %r9, %rax + xorq %rcx, %rax + vpor %ymm9, %ymm8, %ymm8 + vpor %ymm11, %ymm10, %ymm10 + andq %r9, %rbx + addq %rax, %r12 + rorxq $28, %r13, %rax + rorxq $34, %r13, %rcx + xorq %r11, %rbx + xorq %rax, %rcx + vpxor %ymm10, %ymm8, %ymm8 + vpsrlq $6, %ymm0, %ymm11 + rorxq $39, %r13, %rax + addq %rbx, %r12 + xorq %rcx, %rax + movq %r14, %rbx + leaq (%r8,%r12,1), %r8 + xorq %r13, %rbx + vpxor %ymm11, %ymm8, %ymm8 + andq %rbx, %rdx + addq %rax, %r12 + xorq %r14, %rdx + vpaddq %ymm1, %ymm8, %ymm1 + # msg_sched done: 4-7 + # msg_sched: 8-9 + rorxq $14, %r8, %rax + rorxq $18, %r8, %rcx + addq %rdx, %r12 + vpalignr $8, %ymm2, %ymm3, %ymm12 + addq 64(%rsp), %r11 + movq %r9, %rdx + xorq %rax, %rcx + vpalignr $8, %ymm6, %ymm7, %ymm13 + xorq %r10, %rdx + rorxq $41, %r8, %rax + xorq %rcx, %rax + vpsrlq $0x01, %ymm12, %ymm8 + vpsllq $63, %ymm12, %ymm9 + andq %r8, %rdx + addq %rax, %r11 + rorxq $28, %r12, %rax + vpsrlq $8, %ymm12, %ymm10 + vpsllq $56, %ymm12, %ymm11 + rorxq $34, %r12, %rcx + xorq %r10, %rdx + xorq %rax, %rcx + vpor %ymm9, %ymm8, %ymm8 + vpor %ymm11, %ymm10, %ymm10 + rorxq $39, %r12, %rax + addq %rdx, %r11 + xorq %rcx, %rax + vpsrlq $7, %ymm12, %ymm11 + vpxor %ymm10, %ymm8, %ymm8 + movq %r13, %rdx + addq %r11, %r15 + xorq %r12, %rdx + vpxor %ymm11, %ymm8, %ymm8 + vpaddq %ymm2, %ymm13, %ymm2 + andq %rdx, %rbx + addq %rax, %r11 + xorq %r13, %rbx + vpaddq %ymm2, %ymm8, %ymm2 + rorxq $14, %r15, %rax + rorxq $18, %r15, %rcx + addq %rbx, %r11 + vpsrlq $19, %ymm1, %ymm8 + vpsllq $45, %ymm1, %ymm9 + addq 72(%rsp), %r10 + movq %r8, %rbx + xorq %rax, %rcx + vpsrlq $61, %ymm1, %ymm10 + vpsllq $3, %ymm1, %ymm11 + xorq %r9, %rbx + rorxq $41, %r15, %rax + xorq %rcx, %rax + vpor %ymm9, %ymm8, %ymm8 + vpor %ymm11, %ymm10, %ymm10 + andq %r15, %rbx + addq %rax, %r10 + rorxq $28, %r11, %rax + rorxq $34, %r11, %rcx + xorq %r9, %rbx + xorq %rax, %rcx + vpxor %ymm10, %ymm8, %ymm8 + vpsrlq $6, %ymm1, %ymm11 + rorxq $39, %r11, %rax + addq %rbx, %r10 + xorq %rcx, %rax + movq %r12, %rbx + leaq (%r14,%r10,1), %r14 + xorq %r11, %rbx + vpxor %ymm11, %ymm8, %ymm8 + andq %rbx, %rdx + addq %rax, %r10 + xorq %r12, %rdx + vpaddq %ymm2, %ymm8, %ymm2 + # msg_sched done: 8-11 + # msg_sched: 12-13 + rorxq $14, %r14, %rax + rorxq $18, %r14, %rcx + addq %rdx, %r10 + vpalignr $8, %ymm3, %ymm4, %ymm12 + addq 96(%rsp), %r9 + movq %r15, %rdx + xorq %rax, %rcx + vpalignr $8, %ymm7, %ymm0, %ymm13 + xorq %r8, %rdx + rorxq $41, %r14, %rax + xorq %rcx, %rax + vpsrlq $0x01, %ymm12, %ymm8 + vpsllq $63, %ymm12, %ymm9 + andq %r14, %rdx + addq %rax, %r9 + rorxq $28, %r10, %rax + vpsrlq $8, %ymm12, %ymm10 + vpsllq $56, %ymm12, %ymm11 + rorxq $34, %r10, %rcx + xorq %r8, %rdx + xorq %rax, %rcx + vpor %ymm9, %ymm8, %ymm8 + vpor %ymm11, %ymm10, %ymm10 + rorxq $39, %r10, %rax + addq %rdx, %r9 + xorq %rcx, %rax + vpsrlq $7, %ymm12, %ymm11 + vpxor %ymm10, %ymm8, %ymm8 + movq %r11, %rdx + addq %r9, %r13 + xorq %r10, %rdx + vpxor %ymm11, %ymm8, %ymm8 + vpaddq %ymm3, %ymm13, %ymm3 + andq %rdx, %rbx + addq %rax, %r9 + xorq %r11, %rbx + vpaddq %ymm3, %ymm8, %ymm3 + rorxq $14, %r13, %rax + rorxq $18, %r13, %rcx + addq %rbx, %r9 + vpsrlq $19, %ymm2, %ymm8 + vpsllq $45, %ymm2, %ymm9 + addq 104(%rsp), %r8 + movq %r14, %rbx + xorq %rax, %rcx + vpsrlq $61, %ymm2, %ymm10 + vpsllq $3, %ymm2, %ymm11 + xorq %r15, %rbx + rorxq $41, %r13, %rax + xorq %rcx, %rax + vpor %ymm9, %ymm8, %ymm8 + vpor %ymm11, %ymm10, %ymm10 + andq %r13, %rbx + addq %rax, %r8 + rorxq $28, %r9, %rax + rorxq $34, %r9, %rcx + xorq %r15, %rbx + xorq %rax, %rcx + vpxor %ymm10, %ymm8, %ymm8 + vpsrlq $6, %ymm2, %ymm11 + rorxq $39, %r9, %rax + addq %rbx, %r8 + xorq %rcx, %rax + movq %r10, %rbx + leaq (%r12,%r8,1), %r12 + xorq %r9, %rbx + vpxor %ymm11, %ymm8, %ymm8 + andq %rbx, %rdx + addq %rax, %r8 + xorq %r10, %rdx + vpaddq %ymm3, %ymm8, %ymm3 + # msg_sched done: 12-15 + # msg_sched: 16-17 + rorxq $14, %r12, %rax + rorxq $18, %r12, %rcx + addq %rdx, %r8 + vpalignr $8, %ymm4, %ymm5, %ymm12 + addq 128(%rsp), %r15 + movq %r13, %rdx + xorq %rax, %rcx + vpalignr $8, %ymm0, %ymm1, %ymm13 + xorq %r14, %rdx + rorxq $41, %r12, %rax + xorq %rcx, %rax + vpsrlq $0x01, %ymm12, %ymm8 + vpsllq $63, %ymm12, %ymm9 + andq %r12, %rdx + addq %rax, %r15 + rorxq $28, %r8, %rax + vpsrlq $8, %ymm12, %ymm10 + vpsllq $56, %ymm12, %ymm11 + rorxq $34, %r8, %rcx + xorq %r14, %rdx + xorq %rax, %rcx + vpor %ymm9, %ymm8, %ymm8 + vpor %ymm11, %ymm10, %ymm10 + rorxq $39, %r8, %rax + addq %rdx, %r15 + xorq %rcx, %rax + vpsrlq $7, %ymm12, %ymm11 + vpxor %ymm10, %ymm8, %ymm8 + movq %r9, %rdx + addq %r15, %r11 + xorq %r8, %rdx + vpxor %ymm11, %ymm8, %ymm8 + vpaddq %ymm4, %ymm13, %ymm4 + andq %rdx, %rbx + addq %rax, %r15 + xorq %r9, %rbx + vpaddq %ymm4, %ymm8, %ymm4 + rorxq $14, %r11, %rax + rorxq $18, %r11, %rcx + addq %rbx, %r15 + vpsrlq $19, %ymm3, %ymm8 + vpsllq $45, %ymm3, %ymm9 + addq 136(%rsp), %r14 + movq %r12, %rbx + xorq %rax, %rcx + vpsrlq $61, %ymm3, %ymm10 + vpsllq $3, %ymm3, %ymm11 + xorq %r13, %rbx + rorxq $41, %r11, %rax + xorq %rcx, %rax + vpor %ymm9, %ymm8, %ymm8 + vpor %ymm11, %ymm10, %ymm10 + andq %r11, %rbx + addq %rax, %r14 + rorxq $28, %r15, %rax + rorxq $34, %r15, %rcx + xorq %r13, %rbx + xorq %rax, %rcx + vpxor %ymm10, %ymm8, %ymm8 + vpsrlq $6, %ymm3, %ymm11 + rorxq $39, %r15, %rax + addq %rbx, %r14 + xorq %rcx, %rax + movq %r8, %rbx + leaq (%r10,%r14,1), %r10 + xorq %r15, %rbx + vpxor %ymm11, %ymm8, %ymm8 + andq %rbx, %rdx + addq %rax, %r14 + xorq %r8, %rdx + vpaddq %ymm4, %ymm8, %ymm4 + # msg_sched done: 16-19 + # msg_sched: 20-21 + rorxq $14, %r10, %rax + rorxq $18, %r10, %rcx + addq %rdx, %r14 + vpalignr $8, %ymm5, %ymm6, %ymm12 + addq 160(%rsp), %r13 + movq %r11, %rdx + xorq %rax, %rcx + vpalignr $8, %ymm1, %ymm2, %ymm13 + xorq %r12, %rdx + rorxq $41, %r10, %rax + xorq %rcx, %rax + vpsrlq $0x01, %ymm12, %ymm8 + vpsllq $63, %ymm12, %ymm9 + andq %r10, %rdx + addq %rax, %r13 + rorxq $28, %r14, %rax + vpsrlq $8, %ymm12, %ymm10 + vpsllq $56, %ymm12, %ymm11 + rorxq $34, %r14, %rcx + xorq %r12, %rdx + xorq %rax, %rcx + vpor %ymm9, %ymm8, %ymm8 + vpor %ymm11, %ymm10, %ymm10 + rorxq $39, %r14, %rax + addq %rdx, %r13 + xorq %rcx, %rax + vpsrlq $7, %ymm12, %ymm11 + vpxor %ymm10, %ymm8, %ymm8 + movq %r15, %rdx + addq %r13, %r9 + xorq %r14, %rdx + vpxor %ymm11, %ymm8, %ymm8 + vpaddq %ymm5, %ymm13, %ymm5 + andq %rdx, %rbx + addq %rax, %r13 + xorq %r15, %rbx + vpaddq %ymm5, %ymm8, %ymm5 + rorxq $14, %r9, %rax + rorxq $18, %r9, %rcx + addq %rbx, %r13 + vpsrlq $19, %ymm4, %ymm8 + vpsllq $45, %ymm4, %ymm9 + addq 168(%rsp), %r12 + movq %r10, %rbx + xorq %rax, %rcx + vpsrlq $61, %ymm4, %ymm10 + vpsllq $3, %ymm4, %ymm11 + xorq %r11, %rbx + rorxq $41, %r9, %rax + xorq %rcx, %rax + vpor %ymm9, %ymm8, %ymm8 + vpor %ymm11, %ymm10, %ymm10 + andq %r9, %rbx + addq %rax, %r12 + rorxq $28, %r13, %rax + rorxq $34, %r13, %rcx + xorq %r11, %rbx + xorq %rax, %rcx + vpxor %ymm10, %ymm8, %ymm8 + vpsrlq $6, %ymm4, %ymm11 + rorxq $39, %r13, %rax + addq %rbx, %r12 + xorq %rcx, %rax + movq %r14, %rbx + leaq (%r8,%r12,1), %r8 + xorq %r13, %rbx + vpxor %ymm11, %ymm8, %ymm8 + andq %rbx, %rdx + addq %rax, %r12 + xorq %r14, %rdx + vpaddq %ymm5, %ymm8, %ymm5 + # msg_sched done: 20-23 + # msg_sched: 24-25 + rorxq $14, %r8, %rax + rorxq $18, %r8, %rcx + addq %rdx, %r12 + vpalignr $8, %ymm6, %ymm7, %ymm12 + addq 192(%rsp), %r11 + movq %r9, %rdx + xorq %rax, %rcx + vpalignr $8, %ymm2, %ymm3, %ymm13 + xorq %r10, %rdx + rorxq $41, %r8, %rax + xorq %rcx, %rax + vpsrlq $0x01, %ymm12, %ymm8 + vpsllq $63, %ymm12, %ymm9 + andq %r8, %rdx + addq %rax, %r11 + rorxq $28, %r12, %rax + vpsrlq $8, %ymm12, %ymm10 + vpsllq $56, %ymm12, %ymm11 + rorxq $34, %r12, %rcx + xorq %r10, %rdx + xorq %rax, %rcx + vpor %ymm9, %ymm8, %ymm8 + vpor %ymm11, %ymm10, %ymm10 + rorxq $39, %r12, %rax + addq %rdx, %r11 + xorq %rcx, %rax + vpsrlq $7, %ymm12, %ymm11 + vpxor %ymm10, %ymm8, %ymm8 + movq %r13, %rdx + addq %r11, %r15 + xorq %r12, %rdx + vpxor %ymm11, %ymm8, %ymm8 + vpaddq %ymm6, %ymm13, %ymm6 + andq %rdx, %rbx + addq %rax, %r11 + xorq %r13, %rbx + vpaddq %ymm6, %ymm8, %ymm6 + rorxq $14, %r15, %rax + rorxq $18, %r15, %rcx + addq %rbx, %r11 + vpsrlq $19, %ymm5, %ymm8 + vpsllq $45, %ymm5, %ymm9 + addq 200(%rsp), %r10 + movq %r8, %rbx + xorq %rax, %rcx + vpsrlq $61, %ymm5, %ymm10 + vpsllq $3, %ymm5, %ymm11 + xorq %r9, %rbx + rorxq $41, %r15, %rax + xorq %rcx, %rax + vpor %ymm9, %ymm8, %ymm8 + vpor %ymm11, %ymm10, %ymm10 + andq %r15, %rbx + addq %rax, %r10 + rorxq $28, %r11, %rax + rorxq $34, %r11, %rcx + xorq %r9, %rbx + xorq %rax, %rcx + vpxor %ymm10, %ymm8, %ymm8 + vpsrlq $6, %ymm5, %ymm11 + rorxq $39, %r11, %rax + addq %rbx, %r10 + xorq %rcx, %rax + movq %r12, %rbx + leaq (%r14,%r10,1), %r14 + xorq %r11, %rbx + vpxor %ymm11, %ymm8, %ymm8 + andq %rbx, %rdx + addq %rax, %r10 + xorq %r12, %rdx + vpaddq %ymm6, %ymm8, %ymm6 + # msg_sched done: 24-27 + # msg_sched: 28-29 + rorxq $14, %r14, %rax + rorxq $18, %r14, %rcx + addq %rdx, %r10 + vpalignr $8, %ymm7, %ymm0, %ymm12 + addq 224(%rsp), %r9 + movq %r15, %rdx + xorq %rax, %rcx + vpalignr $8, %ymm3, %ymm4, %ymm13 + xorq %r8, %rdx + rorxq $41, %r14, %rax + xorq %rcx, %rax + vpsrlq $0x01, %ymm12, %ymm8 + vpsllq $63, %ymm12, %ymm9 + andq %r14, %rdx + addq %rax, %r9 + rorxq $28, %r10, %rax + vpsrlq $8, %ymm12, %ymm10 + vpsllq $56, %ymm12, %ymm11 + rorxq $34, %r10, %rcx + xorq %r8, %rdx + xorq %rax, %rcx + vpor %ymm9, %ymm8, %ymm8 + vpor %ymm11, %ymm10, %ymm10 + rorxq $39, %r10, %rax + addq %rdx, %r9 + xorq %rcx, %rax + vpsrlq $7, %ymm12, %ymm11 + vpxor %ymm10, %ymm8, %ymm8 + movq %r11, %rdx + addq %r9, %r13 + xorq %r10, %rdx + vpxor %ymm11, %ymm8, %ymm8 + vpaddq %ymm7, %ymm13, %ymm7 + andq %rdx, %rbx + addq %rax, %r9 + xorq %r11, %rbx + vpaddq %ymm7, %ymm8, %ymm7 + rorxq $14, %r13, %rax + rorxq $18, %r13, %rcx + addq %rbx, %r9 + vpsrlq $19, %ymm6, %ymm8 + vpsllq $45, %ymm6, %ymm9 + addq 232(%rsp), %r8 + movq %r14, %rbx + xorq %rax, %rcx + vpsrlq $61, %ymm6, %ymm10 + vpsllq $3, %ymm6, %ymm11 + xorq %r15, %rbx + rorxq $41, %r13, %rax + xorq %rcx, %rax + vpor %ymm9, %ymm8, %ymm8 + vpor %ymm11, %ymm10, %ymm10 + andq %r13, %rbx + addq %rax, %r8 + rorxq $28, %r9, %rax + rorxq $34, %r9, %rcx + xorq %r15, %rbx + xorq %rax, %rcx + vpxor %ymm10, %ymm8, %ymm8 + vpsrlq $6, %ymm6, %ymm11 + rorxq $39, %r9, %rax + addq %rbx, %r8 + xorq %rcx, %rax + movq %r10, %rbx + leaq (%r12,%r8,1), %r12 + xorq %r9, %rbx + vpxor %ymm11, %ymm8, %ymm8 + andq %rbx, %rdx + addq %rax, %r8 + xorq %r10, %rdx + vpaddq %ymm7, %ymm8, %ymm7 + # msg_sched done: 28-31 + addq $0x100, %rbp + addq $0x100, %rsp + cmpq L_avx2_rorx_sha512_k_2_end(%rip), %rbp + jne L_sha512_len_avx2_rorx_start + vpaddq (%rbp), %ymm0, %ymm8 + vpaddq 32(%rbp), %ymm1, %ymm9 + vmovdqu %ymm8, (%rsp) + vmovdqu %ymm9, 32(%rsp) + vpaddq 64(%rbp), %ymm2, %ymm8 + vpaddq 96(%rbp), %ymm3, %ymm9 + vmovdqu %ymm8, 64(%rsp) + vmovdqu %ymm9, 96(%rsp) + vpaddq 128(%rbp), %ymm4, %ymm8 + vpaddq 160(%rbp), %ymm5, %ymm9 + vmovdqu %ymm8, 128(%rsp) + vmovdqu %ymm9, 160(%rsp) + vpaddq 192(%rbp), %ymm6, %ymm8 + vpaddq 224(%rbp), %ymm7, %ymm9 + vmovdqu %ymm8, 192(%rsp) + vmovdqu %ymm9, 224(%rsp) + # rnd_all_2: 0-1 + rorxq $14, %r12, %rax + rorxq $18, %r12, %rcx + addq %rdx, %r8 + addq (%rsp), %r15 + movq %r13, %rdx + xorq %rax, %rcx + xorq %r14, %rdx + rorxq $41, %r12, %rax + xorq %rcx, %rax + andq %r12, %rdx + addq %rax, %r15 + rorxq $28, %r8, %rax + rorxq $34, %r8, %rcx + xorq %r14, %rdx + xorq %rax, %rcx + rorxq $39, %r8, %rax + addq %rdx, %r15 + xorq %rcx, %rax + movq %r9, %rdx + addq %r15, %r11 + xorq %r8, %rdx + andq %rdx, %rbx + addq %rax, %r15 + xorq %r9, %rbx + rorxq $14, %r11, %rax + rorxq $18, %r11, %rcx + addq %rbx, %r15 + addq 8(%rsp), %r14 + movq %r12, %rbx + xorq %rax, %rcx + xorq %r13, %rbx + rorxq $41, %r11, %rax + xorq %rcx, %rax + andq %r11, %rbx + addq %rax, %r14 + rorxq $28, %r15, %rax + rorxq $34, %r15, %rcx + xorq %r13, %rbx + xorq %rax, %rcx + rorxq $39, %r15, %rax + addq %rbx, %r14 + xorq %rcx, %rax + movq %r8, %rbx + leaq (%r10,%r14,1), %r10 + xorq %r15, %rbx + andq %rbx, %rdx + addq %rax, %r14 + xorq %r8, %rdx + # rnd_all_2: 4-5 + rorxq $14, %r10, %rax + rorxq $18, %r10, %rcx + addq %rdx, %r14 + addq 32(%rsp), %r13 + movq %r11, %rdx + xorq %rax, %rcx + xorq %r12, %rdx + rorxq $41, %r10, %rax + xorq %rcx, %rax + andq %r10, %rdx + addq %rax, %r13 + rorxq $28, %r14, %rax + rorxq $34, %r14, %rcx + xorq %r12, %rdx + xorq %rax, %rcx + rorxq $39, %r14, %rax + addq %rdx, %r13 + xorq %rcx, %rax + movq %r15, %rdx + addq %r13, %r9 + xorq %r14, %rdx + andq %rdx, %rbx + addq %rax, %r13 + xorq %r15, %rbx + rorxq $14, %r9, %rax + rorxq $18, %r9, %rcx + addq %rbx, %r13 + addq 40(%rsp), %r12 + movq %r10, %rbx + xorq %rax, %rcx + xorq %r11, %rbx + rorxq $41, %r9, %rax + xorq %rcx, %rax + andq %r9, %rbx + addq %rax, %r12 + rorxq $28, %r13, %rax + rorxq $34, %r13, %rcx + xorq %r11, %rbx + xorq %rax, %rcx + rorxq $39, %r13, %rax + addq %rbx, %r12 + xorq %rcx, %rax + movq %r14, %rbx + leaq (%r8,%r12,1), %r8 + xorq %r13, %rbx + andq %rbx, %rdx + addq %rax, %r12 + xorq %r14, %rdx + # rnd_all_2: 8-9 + rorxq $14, %r8, %rax + rorxq $18, %r8, %rcx + addq %rdx, %r12 + addq 64(%rsp), %r11 + movq %r9, %rdx + xorq %rax, %rcx + xorq %r10, %rdx + rorxq $41, %r8, %rax + xorq %rcx, %rax + andq %r8, %rdx + addq %rax, %r11 + rorxq $28, %r12, %rax + rorxq $34, %r12, %rcx + xorq %r10, %rdx + xorq %rax, %rcx + rorxq $39, %r12, %rax + addq %rdx, %r11 + xorq %rcx, %rax + movq %r13, %rdx + addq %r11, %r15 + xorq %r12, %rdx + andq %rdx, %rbx + addq %rax, %r11 + xorq %r13, %rbx + rorxq $14, %r15, %rax + rorxq $18, %r15, %rcx + addq %rbx, %r11 + addq 72(%rsp), %r10 + movq %r8, %rbx + xorq %rax, %rcx + xorq %r9, %rbx + rorxq $41, %r15, %rax + xorq %rcx, %rax + andq %r15, %rbx + addq %rax, %r10 + rorxq $28, %r11, %rax + rorxq $34, %r11, %rcx + xorq %r9, %rbx + xorq %rax, %rcx + rorxq $39, %r11, %rax + addq %rbx, %r10 + xorq %rcx, %rax + movq %r12, %rbx + leaq (%r14,%r10,1), %r14 + xorq %r11, %rbx + andq %rbx, %rdx + addq %rax, %r10 + xorq %r12, %rdx + # rnd_all_2: 12-13 + rorxq $14, %r14, %rax + rorxq $18, %r14, %rcx + addq %rdx, %r10 + addq 96(%rsp), %r9 + movq %r15, %rdx + xorq %rax, %rcx + xorq %r8, %rdx + rorxq $41, %r14, %rax + xorq %rcx, %rax + andq %r14, %rdx + addq %rax, %r9 + rorxq $28, %r10, %rax + rorxq $34, %r10, %rcx + xorq %r8, %rdx + xorq %rax, %rcx + rorxq $39, %r10, %rax + addq %rdx, %r9 + xorq %rcx, %rax + movq %r11, %rdx + addq %r9, %r13 + xorq %r10, %rdx + andq %rdx, %rbx + addq %rax, %r9 + xorq %r11, %rbx + rorxq $14, %r13, %rax + rorxq $18, %r13, %rcx + addq %rbx, %r9 + addq 104(%rsp), %r8 + movq %r14, %rbx + xorq %rax, %rcx + xorq %r15, %rbx + rorxq $41, %r13, %rax + xorq %rcx, %rax + andq %r13, %rbx + addq %rax, %r8 + rorxq $28, %r9, %rax + rorxq $34, %r9, %rcx + xorq %r15, %rbx + xorq %rax, %rcx + rorxq $39, %r9, %rax + addq %rbx, %r8 + xorq %rcx, %rax + movq %r10, %rbx + leaq (%r12,%r8,1), %r12 + xorq %r9, %rbx + andq %rbx, %rdx + addq %rax, %r8 + xorq %r10, %rdx + # rnd_all_2: 16-17 + rorxq $14, %r12, %rax + rorxq $18, %r12, %rcx + addq %rdx, %r8 + addq 128(%rsp), %r15 + movq %r13, %rdx + xorq %rax, %rcx + xorq %r14, %rdx + rorxq $41, %r12, %rax + xorq %rcx, %rax + andq %r12, %rdx + addq %rax, %r15 + rorxq $28, %r8, %rax + rorxq $34, %r8, %rcx + xorq %r14, %rdx + xorq %rax, %rcx + rorxq $39, %r8, %rax + addq %rdx, %r15 + xorq %rcx, %rax + movq %r9, %rdx + addq %r15, %r11 + xorq %r8, %rdx + andq %rdx, %rbx + addq %rax, %r15 + xorq %r9, %rbx + rorxq $14, %r11, %rax + rorxq $18, %r11, %rcx + addq %rbx, %r15 + addq 136(%rsp), %r14 + movq %r12, %rbx + xorq %rax, %rcx + xorq %r13, %rbx + rorxq $41, %r11, %rax + xorq %rcx, %rax + andq %r11, %rbx + addq %rax, %r14 + rorxq $28, %r15, %rax + rorxq $34, %r15, %rcx + xorq %r13, %rbx + xorq %rax, %rcx + rorxq $39, %r15, %rax + addq %rbx, %r14 + xorq %rcx, %rax + movq %r8, %rbx + leaq (%r10,%r14,1), %r10 + xorq %r15, %rbx + andq %rbx, %rdx + addq %rax, %r14 + xorq %r8, %rdx + # rnd_all_2: 20-21 + rorxq $14, %r10, %rax + rorxq $18, %r10, %rcx + addq %rdx, %r14 + addq 160(%rsp), %r13 + movq %r11, %rdx + xorq %rax, %rcx + xorq %r12, %rdx + rorxq $41, %r10, %rax + xorq %rcx, %rax + andq %r10, %rdx + addq %rax, %r13 + rorxq $28, %r14, %rax + rorxq $34, %r14, %rcx + xorq %r12, %rdx + xorq %rax, %rcx + rorxq $39, %r14, %rax + addq %rdx, %r13 + xorq %rcx, %rax + movq %r15, %rdx + addq %r13, %r9 + xorq %r14, %rdx + andq %rdx, %rbx + addq %rax, %r13 + xorq %r15, %rbx + rorxq $14, %r9, %rax + rorxq $18, %r9, %rcx + addq %rbx, %r13 + addq 168(%rsp), %r12 + movq %r10, %rbx + xorq %rax, %rcx + xorq %r11, %rbx + rorxq $41, %r9, %rax + xorq %rcx, %rax + andq %r9, %rbx + addq %rax, %r12 + rorxq $28, %r13, %rax + rorxq $34, %r13, %rcx + xorq %r11, %rbx + xorq %rax, %rcx + rorxq $39, %r13, %rax + addq %rbx, %r12 + xorq %rcx, %rax + movq %r14, %rbx + leaq (%r8,%r12,1), %r8 + xorq %r13, %rbx + andq %rbx, %rdx + addq %rax, %r12 + xorq %r14, %rdx + # rnd_all_2: 24-25 + rorxq $14, %r8, %rax + rorxq $18, %r8, %rcx + addq %rdx, %r12 + addq 192(%rsp), %r11 + movq %r9, %rdx + xorq %rax, %rcx + xorq %r10, %rdx + rorxq $41, %r8, %rax + xorq %rcx, %rax + andq %r8, %rdx + addq %rax, %r11 + rorxq $28, %r12, %rax + rorxq $34, %r12, %rcx + xorq %r10, %rdx + xorq %rax, %rcx + rorxq $39, %r12, %rax + addq %rdx, %r11 + xorq %rcx, %rax + movq %r13, %rdx + addq %r11, %r15 + xorq %r12, %rdx + andq %rdx, %rbx + addq %rax, %r11 + xorq %r13, %rbx + rorxq $14, %r15, %rax + rorxq $18, %r15, %rcx + addq %rbx, %r11 + addq 200(%rsp), %r10 + movq %r8, %rbx + xorq %rax, %rcx + xorq %r9, %rbx + rorxq $41, %r15, %rax + xorq %rcx, %rax + andq %r15, %rbx + addq %rax, %r10 + rorxq $28, %r11, %rax + rorxq $34, %r11, %rcx + xorq %r9, %rbx + xorq %rax, %rcx + rorxq $39, %r11, %rax + addq %rbx, %r10 + xorq %rcx, %rax + movq %r12, %rbx + leaq (%r14,%r10,1), %r14 + xorq %r11, %rbx + andq %rbx, %rdx + addq %rax, %r10 + xorq %r12, %rdx + # rnd_all_2: 28-29 + rorxq $14, %r14, %rax + rorxq $18, %r14, %rcx + addq %rdx, %r10 + addq 224(%rsp), %r9 + movq %r15, %rdx + xorq %rax, %rcx + xorq %r8, %rdx + rorxq $41, %r14, %rax + xorq %rcx, %rax + andq %r14, %rdx + addq %rax, %r9 + rorxq $28, %r10, %rax + rorxq $34, %r10, %rcx + xorq %r8, %rdx + xorq %rax, %rcx + rorxq $39, %r10, %rax + addq %rdx, %r9 + xorq %rcx, %rax + movq %r11, %rdx + addq %r9, %r13 + xorq %r10, %rdx + andq %rdx, %rbx + addq %rax, %r9 + xorq %r11, %rbx + rorxq $14, %r13, %rax + rorxq $18, %r13, %rcx + addq %rbx, %r9 + addq 232(%rsp), %r8 + movq %r14, %rbx + xorq %rax, %rcx + xorq %r15, %rbx + rorxq $41, %r13, %rax + xorq %rcx, %rax + andq %r13, %rbx + addq %rax, %r8 + rorxq $28, %r9, %rax + rorxq $34, %r9, %rcx + xorq %r15, %rbx + xorq %rax, %rcx + rorxq $39, %r9, %rax + addq %rbx, %r8 + xorq %rcx, %rax + movq %r10, %rbx + leaq (%r12,%r8,1), %r12 + xorq %r9, %rbx + andq %rbx, %rdx + addq %rax, %r8 + xorq %r10, %rdx + addq %rdx, %r8 + subq $0x400, %rsp + addq (%rdi), %r8 + addq 8(%rdi), %r9 + addq 16(%rdi), %r10 + addq 24(%rdi), %r11 + addq 32(%rdi), %r12 + addq 40(%rdi), %r13 + addq 48(%rdi), %r14 + addq 56(%rdi), %r15 + movq %r8, (%rdi) + movq %r9, 8(%rdi) + movq %r10, 16(%rdi) + movq %r11, 24(%rdi) + movq %r12, 32(%rdi) + movq %r13, 40(%rdi) + movq %r14, 48(%rdi) + movq %r15, 56(%rdi) + movq %r9, %rbx + xorq %rdx, %rdx + xorq %r10, %rbx + movq $5, %rbp +L_sha512_len_avx2_rorx_tail: + # rnd_all_2: 2-3 + rorxq $14, %r12, %rax + rorxq $18, %r12, %rcx + addq %rdx, %r8 + addq 16(%rsp), %r15 + movq %r13, %rdx + xorq %rax, %rcx + xorq %r14, %rdx + rorxq $41, %r12, %rax + xorq %rcx, %rax + andq %r12, %rdx + addq %rax, %r15 + rorxq $28, %r8, %rax + rorxq $34, %r8, %rcx + xorq %r14, %rdx + xorq %rax, %rcx + rorxq $39, %r8, %rax + addq %rdx, %r15 + xorq %rcx, %rax + movq %r9, %rdx + addq %r15, %r11 + xorq %r8, %rdx + andq %rdx, %rbx + addq %rax, %r15 + xorq %r9, %rbx + rorxq $14, %r11, %rax + rorxq $18, %r11, %rcx + addq %rbx, %r15 + addq 24(%rsp), %r14 + movq %r12, %rbx + xorq %rax, %rcx + xorq %r13, %rbx + rorxq $41, %r11, %rax + xorq %rcx, %rax + andq %r11, %rbx + addq %rax, %r14 + rorxq $28, %r15, %rax + rorxq $34, %r15, %rcx + xorq %r13, %rbx + xorq %rax, %rcx + rorxq $39, %r15, %rax + addq %rbx, %r14 + xorq %rcx, %rax + movq %r8, %rbx + leaq (%r10,%r14,1), %r10 + xorq %r15, %rbx + andq %rbx, %rdx + addq %rax, %r14 + xorq %r8, %rdx + # rnd_all_2: 6-7 + rorxq $14, %r10, %rax + rorxq $18, %r10, %rcx + addq %rdx, %r14 + addq 48(%rsp), %r13 + movq %r11, %rdx + xorq %rax, %rcx + xorq %r12, %rdx + rorxq $41, %r10, %rax + xorq %rcx, %rax + andq %r10, %rdx + addq %rax, %r13 + rorxq $28, %r14, %rax + rorxq $34, %r14, %rcx + xorq %r12, %rdx + xorq %rax, %rcx + rorxq $39, %r14, %rax + addq %rdx, %r13 + xorq %rcx, %rax + movq %r15, %rdx + addq %r13, %r9 + xorq %r14, %rdx + andq %rdx, %rbx + addq %rax, %r13 + xorq %r15, %rbx + rorxq $14, %r9, %rax + rorxq $18, %r9, %rcx + addq %rbx, %r13 + addq 56(%rsp), %r12 + movq %r10, %rbx + xorq %rax, %rcx + xorq %r11, %rbx + rorxq $41, %r9, %rax + xorq %rcx, %rax + andq %r9, %rbx + addq %rax, %r12 + rorxq $28, %r13, %rax + rorxq $34, %r13, %rcx + xorq %r11, %rbx + xorq %rax, %rcx + rorxq $39, %r13, %rax + addq %rbx, %r12 + xorq %rcx, %rax + movq %r14, %rbx + leaq (%r8,%r12,1), %r8 + xorq %r13, %rbx + andq %rbx, %rdx + addq %rax, %r12 + xorq %r14, %rdx + # rnd_all_2: 10-11 + rorxq $14, %r8, %rax + rorxq $18, %r8, %rcx + addq %rdx, %r12 + addq 80(%rsp), %r11 + movq %r9, %rdx + xorq %rax, %rcx + xorq %r10, %rdx + rorxq $41, %r8, %rax + xorq %rcx, %rax + andq %r8, %rdx + addq %rax, %r11 + rorxq $28, %r12, %rax + rorxq $34, %r12, %rcx + xorq %r10, %rdx + xorq %rax, %rcx + rorxq $39, %r12, %rax + addq %rdx, %r11 + xorq %rcx, %rax + movq %r13, %rdx + addq %r11, %r15 + xorq %r12, %rdx + andq %rdx, %rbx + addq %rax, %r11 + xorq %r13, %rbx + rorxq $14, %r15, %rax + rorxq $18, %r15, %rcx + addq %rbx, %r11 + addq 88(%rsp), %r10 + movq %r8, %rbx + xorq %rax, %rcx + xorq %r9, %rbx + rorxq $41, %r15, %rax + xorq %rcx, %rax + andq %r15, %rbx + addq %rax, %r10 + rorxq $28, %r11, %rax + rorxq $34, %r11, %rcx + xorq %r9, %rbx + xorq %rax, %rcx + rorxq $39, %r11, %rax + addq %rbx, %r10 + xorq %rcx, %rax + movq %r12, %rbx + leaq (%r14,%r10,1), %r14 + xorq %r11, %rbx + andq %rbx, %rdx + addq %rax, %r10 + xorq %r12, %rdx + # rnd_all_2: 14-15 + rorxq $14, %r14, %rax + rorxq $18, %r14, %rcx + addq %rdx, %r10 + addq 112(%rsp), %r9 + movq %r15, %rdx + xorq %rax, %rcx + xorq %r8, %rdx + rorxq $41, %r14, %rax + xorq %rcx, %rax + andq %r14, %rdx + addq %rax, %r9 + rorxq $28, %r10, %rax + rorxq $34, %r10, %rcx + xorq %r8, %rdx + xorq %rax, %rcx + rorxq $39, %r10, %rax + addq %rdx, %r9 + xorq %rcx, %rax + movq %r11, %rdx + addq %r9, %r13 + xorq %r10, %rdx + andq %rdx, %rbx + addq %rax, %r9 + xorq %r11, %rbx + rorxq $14, %r13, %rax + rorxq $18, %r13, %rcx + addq %rbx, %r9 + addq 120(%rsp), %r8 + movq %r14, %rbx + xorq %rax, %rcx + xorq %r15, %rbx + rorxq $41, %r13, %rax + xorq %rcx, %rax + andq %r13, %rbx + addq %rax, %r8 + rorxq $28, %r9, %rax + rorxq $34, %r9, %rcx + xorq %r15, %rbx + xorq %rax, %rcx + rorxq $39, %r9, %rax + addq %rbx, %r8 + xorq %rcx, %rax + movq %r10, %rbx + leaq (%r12,%r8,1), %r12 + xorq %r9, %rbx + andq %rbx, %rdx + addq %rax, %r8 + xorq %r10, %rdx + # rnd_all_2: 18-19 + rorxq $14, %r12, %rax + rorxq $18, %r12, %rcx + addq %rdx, %r8 + addq 144(%rsp), %r15 + movq %r13, %rdx + xorq %rax, %rcx + xorq %r14, %rdx + rorxq $41, %r12, %rax + xorq %rcx, %rax + andq %r12, %rdx + addq %rax, %r15 + rorxq $28, %r8, %rax + rorxq $34, %r8, %rcx + xorq %r14, %rdx + xorq %rax, %rcx + rorxq $39, %r8, %rax + addq %rdx, %r15 + xorq %rcx, %rax + movq %r9, %rdx + addq %r15, %r11 + xorq %r8, %rdx + andq %rdx, %rbx + addq %rax, %r15 + xorq %r9, %rbx + rorxq $14, %r11, %rax + rorxq $18, %r11, %rcx + addq %rbx, %r15 + addq 152(%rsp), %r14 + movq %r12, %rbx + xorq %rax, %rcx + xorq %r13, %rbx + rorxq $41, %r11, %rax + xorq %rcx, %rax + andq %r11, %rbx + addq %rax, %r14 + rorxq $28, %r15, %rax + rorxq $34, %r15, %rcx + xorq %r13, %rbx + xorq %rax, %rcx + rorxq $39, %r15, %rax + addq %rbx, %r14 + xorq %rcx, %rax + movq %r8, %rbx + leaq (%r10,%r14,1), %r10 + xorq %r15, %rbx + andq %rbx, %rdx + addq %rax, %r14 + xorq %r8, %rdx + # rnd_all_2: 22-23 + rorxq $14, %r10, %rax + rorxq $18, %r10, %rcx + addq %rdx, %r14 + addq 176(%rsp), %r13 + movq %r11, %rdx + xorq %rax, %rcx + xorq %r12, %rdx + rorxq $41, %r10, %rax + xorq %rcx, %rax + andq %r10, %rdx + addq %rax, %r13 + rorxq $28, %r14, %rax + rorxq $34, %r14, %rcx + xorq %r12, %rdx + xorq %rax, %rcx + rorxq $39, %r14, %rax + addq %rdx, %r13 + xorq %rcx, %rax + movq %r15, %rdx + addq %r13, %r9 + xorq %r14, %rdx + andq %rdx, %rbx + addq %rax, %r13 + xorq %r15, %rbx + rorxq $14, %r9, %rax + rorxq $18, %r9, %rcx + addq %rbx, %r13 + addq 184(%rsp), %r12 + movq %r10, %rbx + xorq %rax, %rcx + xorq %r11, %rbx + rorxq $41, %r9, %rax + xorq %rcx, %rax + andq %r9, %rbx + addq %rax, %r12 + rorxq $28, %r13, %rax + rorxq $34, %r13, %rcx + xorq %r11, %rbx + xorq %rax, %rcx + rorxq $39, %r13, %rax + addq %rbx, %r12 + xorq %rcx, %rax + movq %r14, %rbx + leaq (%r8,%r12,1), %r8 + xorq %r13, %rbx + andq %rbx, %rdx + addq %rax, %r12 + xorq %r14, %rdx + # rnd_all_2: 26-27 + rorxq $14, %r8, %rax + rorxq $18, %r8, %rcx + addq %rdx, %r12 + addq 208(%rsp), %r11 + movq %r9, %rdx + xorq %rax, %rcx + xorq %r10, %rdx + rorxq $41, %r8, %rax + xorq %rcx, %rax + andq %r8, %rdx + addq %rax, %r11 + rorxq $28, %r12, %rax + rorxq $34, %r12, %rcx + xorq %r10, %rdx + xorq %rax, %rcx + rorxq $39, %r12, %rax + addq %rdx, %r11 + xorq %rcx, %rax + movq %r13, %rdx + addq %r11, %r15 + xorq %r12, %rdx + andq %rdx, %rbx + addq %rax, %r11 + xorq %r13, %rbx + rorxq $14, %r15, %rax + rorxq $18, %r15, %rcx + addq %rbx, %r11 + addq 216(%rsp), %r10 + movq %r8, %rbx + xorq %rax, %rcx + xorq %r9, %rbx + rorxq $41, %r15, %rax + xorq %rcx, %rax + andq %r15, %rbx + addq %rax, %r10 + rorxq $28, %r11, %rax + rorxq $34, %r11, %rcx + xorq %r9, %rbx + xorq %rax, %rcx + rorxq $39, %r11, %rax + addq %rbx, %r10 + xorq %rcx, %rax + movq %r12, %rbx + leaq (%r14,%r10,1), %r14 + xorq %r11, %rbx + andq %rbx, %rdx + addq %rax, %r10 + xorq %r12, %rdx + # rnd_all_2: 30-31 + rorxq $14, %r14, %rax + rorxq $18, %r14, %rcx + addq %rdx, %r10 + addq 240(%rsp), %r9 + movq %r15, %rdx + xorq %rax, %rcx + xorq %r8, %rdx + rorxq $41, %r14, %rax + xorq %rcx, %rax + andq %r14, %rdx + addq %rax, %r9 + rorxq $28, %r10, %rax + rorxq $34, %r10, %rcx + xorq %r8, %rdx + xorq %rax, %rcx + rorxq $39, %r10, %rax + addq %rdx, %r9 + xorq %rcx, %rax + movq %r11, %rdx + addq %r9, %r13 + xorq %r10, %rdx + andq %rdx, %rbx + addq %rax, %r9 + xorq %r11, %rbx + rorxq $14, %r13, %rax + rorxq $18, %r13, %rcx + addq %rbx, %r9 + addq 248(%rsp), %r8 + movq %r14, %rbx + xorq %rax, %rcx + xorq %r15, %rbx + rorxq $41, %r13, %rax + xorq %rcx, %rax + andq %r13, %rbx + addq %rax, %r8 + rorxq $28, %r9, %rax + rorxq $34, %r9, %rcx + xorq %r15, %rbx + xorq %rax, %rcx + rorxq $39, %r9, %rax + addq %rbx, %r8 + xorq %rcx, %rax + movq %r10, %rbx + leaq (%r12,%r8,1), %r12 + xorq %r9, %rbx + andq %rbx, %rdx + addq %rax, %r8 + xorq %r10, %rdx + addq $0x100, %rsp + subq $0x01, %rbp + jnz L_sha512_len_avx2_rorx_tail + addq %rdx, %r8 + addq (%rdi), %r8 + addq 8(%rdi), %r9 + addq 16(%rdi), %r10 + addq 24(%rdi), %r11 + addq 32(%rdi), %r12 + addq 40(%rdi), %r13 + addq 48(%rdi), %r14 + addq 56(%rdi), %r15 + movq 224(%rdi), %rax + addq $0x40, %rsp + addq $0x100, %rax + subl $0x100, %esi + movq %rax, 224(%rdi) + movq %r8, (%rdi) + movq %r9, 8(%rdi) + movq %r10, 16(%rdi) + movq %r11, 24(%rdi) + movq %r12, 32(%rdi) + movq %r13, 40(%rdi) + movq %r14, 48(%rdi) + movq %r15, 56(%rdi) + jnz L_sha512_len_avx2_rorx_begin +L_sha512_len_avx2_rorx_done: + xorq %rax, %rax + vzeroupper + popq %rbp + popq %r15 + popq %r14 + popq %r13 + popq %r12 + popq %rbx + repz retq +.size Transform_Sha512_AVX2_RORX_Len,.-Transform_Sha512_AVX2_RORX_Len +#endif /* HAVE_INTEL_AVX2 */ diff --git a/wolfcrypt/src/sp_x86_64.c b/wolfcrypt/src/sp_x86_64.c index 8b93add104..5dc48c7b0e 100644 --- a/wolfcrypt/src/sp_x86_64.c +++ b/wolfcrypt/src/sp_x86_64.c @@ -39,7 +39,9 @@ defined(WOLFSSL_HAVE_SP_ECC) #ifdef RSA_LOW_MEM +#ifndef SP_RSA_PRIVATE_EXP_D #define SP_RSA_PRIVATE_EXP_D +#endif #ifndef WOLFSSL_SP_SMALL #define WOLFSSL_SP_SMALL @@ -179,5663 +181,13 @@ static void sp_2048_to_bin(sp_digit* r, byte* a) } } -/* Multiply a and b into r. (r = a * b) - * - * r A single precision integer. - * a A single precision integer. - * b A single precision integer. - */ -SP_NOINLINE static void sp_2048_mul_16(sp_digit* r, const sp_digit* a, - const sp_digit* b) -{ - sp_digit tmp[16]; - - __asm__ __volatile__ ( - "# A[0] * B[0]\n\t" - "movq 0(%[b]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "movq %%rax, (%[tmp])\n\t" - "movq %%rdx, %%rcx\n\t" - "# A[0] * B[1]\n\t" - "movq 8(%[b]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%rbx, %%rbx\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[1] * B[0]\n\t" - "movq 0(%[b]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "movq %%rcx, 8(%[tmp])\n\t" - "# A[0] * B[2]\n\t" - "movq 16(%[b]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%rcx, %%rcx\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[1] * B[1]\n\t" - "movq 8(%[b]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[2] * B[0]\n\t" - "movq 0(%[b]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "movq %%r8, 16(%[tmp])\n\t" - "# A[0] * B[3]\n\t" - "movq 24(%[b]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[1] * B[2]\n\t" - "movq 16(%[b]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[2] * B[1]\n\t" - "movq 8(%[b]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[3] * B[0]\n\t" - "movq 0(%[b]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "movq %%rbx, 24(%[tmp])\n\t" - "# A[0] * B[4]\n\t" - "movq 32(%[b]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%rbx, %%rbx\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[1] * B[3]\n\t" - "movq 24(%[b]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[2] * B[2]\n\t" - "movq 16(%[b]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[3] * B[1]\n\t" - "movq 8(%[b]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[4] * B[0]\n\t" - "movq 0(%[b]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "movq %%rcx, 32(%[tmp])\n\t" - "# A[0] * B[5]\n\t" - "movq 40(%[b]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%rcx, %%rcx\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[1] * B[4]\n\t" - "movq 32(%[b]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[2] * B[3]\n\t" - "movq 24(%[b]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[3] * B[2]\n\t" - "movq 16(%[b]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[4] * B[1]\n\t" - "movq 8(%[b]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[5] * B[0]\n\t" - "movq 0(%[b]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "movq %%r8, 40(%[tmp])\n\t" - "# A[0] * B[6]\n\t" - "movq 48(%[b]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[1] * B[5]\n\t" - "movq 40(%[b]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[2] * B[4]\n\t" - "movq 32(%[b]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[3] * B[3]\n\t" - "movq 24(%[b]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[4] * B[2]\n\t" - "movq 16(%[b]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[5] * B[1]\n\t" - "movq 8(%[b]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[6] * B[0]\n\t" - "movq 0(%[b]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "movq %%rbx, 48(%[tmp])\n\t" - "# A[0] * B[7]\n\t" - "movq 56(%[b]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%rbx, %%rbx\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[1] * B[6]\n\t" - "movq 48(%[b]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[2] * B[5]\n\t" - "movq 40(%[b]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[3] * B[4]\n\t" - "movq 32(%[b]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[4] * B[3]\n\t" - "movq 24(%[b]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[5] * B[2]\n\t" - "movq 16(%[b]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[6] * B[1]\n\t" - "movq 8(%[b]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[7] * B[0]\n\t" - "movq 0(%[b]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "movq %%rcx, 56(%[tmp])\n\t" - "# A[0] * B[8]\n\t" - "movq 64(%[b]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%rcx, %%rcx\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[1] * B[7]\n\t" - "movq 56(%[b]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[2] * B[6]\n\t" - "movq 48(%[b]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[3] * B[5]\n\t" - "movq 40(%[b]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[4] * B[4]\n\t" - "movq 32(%[b]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[5] * B[3]\n\t" - "movq 24(%[b]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[6] * B[2]\n\t" - "movq 16(%[b]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[7] * B[1]\n\t" - "movq 8(%[b]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[8] * B[0]\n\t" - "movq 0(%[b]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "movq %%r8, 64(%[tmp])\n\t" - "# A[0] * B[9]\n\t" - "movq 72(%[b]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[1] * B[8]\n\t" - "movq 64(%[b]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[2] * B[7]\n\t" - "movq 56(%[b]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[3] * B[6]\n\t" - "movq 48(%[b]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[4] * B[5]\n\t" - "movq 40(%[b]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[5] * B[4]\n\t" - "movq 32(%[b]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[6] * B[3]\n\t" - "movq 24(%[b]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[7] * B[2]\n\t" - "movq 16(%[b]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[8] * B[1]\n\t" - "movq 8(%[b]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[9] * B[0]\n\t" - "movq 0(%[b]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "movq %%rbx, 72(%[tmp])\n\t" - "# A[0] * B[10]\n\t" - "movq 80(%[b]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%rbx, %%rbx\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[1] * B[9]\n\t" - "movq 72(%[b]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[2] * B[8]\n\t" - "movq 64(%[b]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[3] * B[7]\n\t" - "movq 56(%[b]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[4] * B[6]\n\t" - "movq 48(%[b]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[5] * B[5]\n\t" - "movq 40(%[b]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[6] * B[4]\n\t" - "movq 32(%[b]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[7] * B[3]\n\t" - "movq 24(%[b]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[8] * B[2]\n\t" - "movq 16(%[b]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[9] * B[1]\n\t" - "movq 8(%[b]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[10] * B[0]\n\t" - "movq 0(%[b]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "movq %%rcx, 80(%[tmp])\n\t" - "# A[0] * B[11]\n\t" - "movq 88(%[b]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%rcx, %%rcx\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[1] * B[10]\n\t" - "movq 80(%[b]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[2] * B[9]\n\t" - "movq 72(%[b]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[3] * B[8]\n\t" - "movq 64(%[b]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[4] * B[7]\n\t" - "movq 56(%[b]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[5] * B[6]\n\t" - "movq 48(%[b]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[6] * B[5]\n\t" - "movq 40(%[b]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[7] * B[4]\n\t" - "movq 32(%[b]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[8] * B[3]\n\t" - "movq 24(%[b]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[9] * B[2]\n\t" - "movq 16(%[b]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[10] * B[1]\n\t" - "movq 8(%[b]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[11] * B[0]\n\t" - "movq 0(%[b]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "movq %%r8, 88(%[tmp])\n\t" - "# A[0] * B[12]\n\t" - "movq 96(%[b]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[1] * B[11]\n\t" - "movq 88(%[b]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[2] * B[10]\n\t" - "movq 80(%[b]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[3] * B[9]\n\t" - "movq 72(%[b]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[4] * B[8]\n\t" - "movq 64(%[b]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[5] * B[7]\n\t" - "movq 56(%[b]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[6] * B[6]\n\t" - "movq 48(%[b]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[7] * B[5]\n\t" - "movq 40(%[b]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[8] * B[4]\n\t" - "movq 32(%[b]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[9] * B[3]\n\t" - "movq 24(%[b]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[10] * B[2]\n\t" - "movq 16(%[b]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[11] * B[1]\n\t" - "movq 8(%[b]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[12] * B[0]\n\t" - "movq 0(%[b]), %%rax\n\t" - "mulq 96(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "movq %%rbx, 96(%[tmp])\n\t" - "# A[0] * B[13]\n\t" - "movq 104(%[b]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%rbx, %%rbx\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[1] * B[12]\n\t" - "movq 96(%[b]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[2] * B[11]\n\t" - "movq 88(%[b]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[3] * B[10]\n\t" - "movq 80(%[b]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[4] * B[9]\n\t" - "movq 72(%[b]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[5] * B[8]\n\t" - "movq 64(%[b]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[6] * B[7]\n\t" - "movq 56(%[b]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[7] * B[6]\n\t" - "movq 48(%[b]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[8] * B[5]\n\t" - "movq 40(%[b]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[9] * B[4]\n\t" - "movq 32(%[b]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[10] * B[3]\n\t" - "movq 24(%[b]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[11] * B[2]\n\t" - "movq 16(%[b]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[12] * B[1]\n\t" - "movq 8(%[b]), %%rax\n\t" - "mulq 96(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[13] * B[0]\n\t" - "movq 0(%[b]), %%rax\n\t" - "mulq 104(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "movq %%rcx, 104(%[tmp])\n\t" - "# A[0] * B[14]\n\t" - "movq 112(%[b]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%rcx, %%rcx\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[1] * B[13]\n\t" - "movq 104(%[b]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[2] * B[12]\n\t" - "movq 96(%[b]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[3] * B[11]\n\t" - "movq 88(%[b]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[4] * B[10]\n\t" - "movq 80(%[b]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[5] * B[9]\n\t" - "movq 72(%[b]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[6] * B[8]\n\t" - "movq 64(%[b]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[7] * B[7]\n\t" - "movq 56(%[b]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[8] * B[6]\n\t" - "movq 48(%[b]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[9] * B[5]\n\t" - "movq 40(%[b]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[10] * B[4]\n\t" - "movq 32(%[b]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[11] * B[3]\n\t" - "movq 24(%[b]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[12] * B[2]\n\t" - "movq 16(%[b]), %%rax\n\t" - "mulq 96(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[13] * B[1]\n\t" - "movq 8(%[b]), %%rax\n\t" - "mulq 104(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[14] * B[0]\n\t" - "movq 0(%[b]), %%rax\n\t" - "mulq 112(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "movq %%r8, 112(%[tmp])\n\t" - "# A[0] * B[15]\n\t" - "movq 120(%[b]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[1] * B[14]\n\t" - "movq 112(%[b]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[2] * B[13]\n\t" - "movq 104(%[b]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[3] * B[12]\n\t" - "movq 96(%[b]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[4] * B[11]\n\t" - "movq 88(%[b]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[5] * B[10]\n\t" - "movq 80(%[b]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[6] * B[9]\n\t" - "movq 72(%[b]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[7] * B[8]\n\t" - "movq 64(%[b]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[8] * B[7]\n\t" - "movq 56(%[b]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[9] * B[6]\n\t" - "movq 48(%[b]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[10] * B[5]\n\t" - "movq 40(%[b]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[11] * B[4]\n\t" - "movq 32(%[b]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[12] * B[3]\n\t" - "movq 24(%[b]), %%rax\n\t" - "mulq 96(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[13] * B[2]\n\t" - "movq 16(%[b]), %%rax\n\t" - "mulq 104(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[14] * B[1]\n\t" - "movq 8(%[b]), %%rax\n\t" - "mulq 112(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[15] * B[0]\n\t" - "movq 0(%[b]), %%rax\n\t" - "mulq 120(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "movq %%rbx, 120(%[tmp])\n\t" - "# A[1] * B[15]\n\t" - "movq 120(%[b]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "xorq %%rbx, %%rbx\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[2] * B[14]\n\t" - "movq 112(%[b]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[3] * B[13]\n\t" - "movq 104(%[b]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[4] * B[12]\n\t" - "movq 96(%[b]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[5] * B[11]\n\t" - "movq 88(%[b]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[6] * B[10]\n\t" - "movq 80(%[b]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[7] * B[9]\n\t" - "movq 72(%[b]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[8] * B[8]\n\t" - "movq 64(%[b]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[9] * B[7]\n\t" - "movq 56(%[b]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[10] * B[6]\n\t" - "movq 48(%[b]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[11] * B[5]\n\t" - "movq 40(%[b]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[12] * B[4]\n\t" - "movq 32(%[b]), %%rax\n\t" - "mulq 96(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[13] * B[3]\n\t" - "movq 24(%[b]), %%rax\n\t" - "mulq 104(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[14] * B[2]\n\t" - "movq 16(%[b]), %%rax\n\t" - "mulq 112(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[15] * B[1]\n\t" - "movq 8(%[b]), %%rax\n\t" - "mulq 120(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "movq %%rcx, 128(%[r])\n\t" - "# A[2] * B[15]\n\t" - "movq 120(%[b]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "xorq %%rcx, %%rcx\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[3] * B[14]\n\t" - "movq 112(%[b]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[4] * B[13]\n\t" - "movq 104(%[b]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[5] * B[12]\n\t" - "movq 96(%[b]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[6] * B[11]\n\t" - "movq 88(%[b]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[7] * B[10]\n\t" - "movq 80(%[b]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[8] * B[9]\n\t" - "movq 72(%[b]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[9] * B[8]\n\t" - "movq 64(%[b]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[10] * B[7]\n\t" - "movq 56(%[b]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[11] * B[6]\n\t" - "movq 48(%[b]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[12] * B[5]\n\t" - "movq 40(%[b]), %%rax\n\t" - "mulq 96(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[13] * B[4]\n\t" - "movq 32(%[b]), %%rax\n\t" - "mulq 104(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[14] * B[3]\n\t" - "movq 24(%[b]), %%rax\n\t" - "mulq 112(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[15] * B[2]\n\t" - "movq 16(%[b]), %%rax\n\t" - "mulq 120(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "movq %%r8, 136(%[r])\n\t" - "# A[3] * B[15]\n\t" - "movq 120(%[b]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[4] * B[14]\n\t" - "movq 112(%[b]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[5] * B[13]\n\t" - "movq 104(%[b]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[6] * B[12]\n\t" - "movq 96(%[b]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[7] * B[11]\n\t" - "movq 88(%[b]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[8] * B[10]\n\t" - "movq 80(%[b]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[9] * B[9]\n\t" - "movq 72(%[b]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[10] * B[8]\n\t" - "movq 64(%[b]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[11] * B[7]\n\t" - "movq 56(%[b]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[12] * B[6]\n\t" - "movq 48(%[b]), %%rax\n\t" - "mulq 96(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[13] * B[5]\n\t" - "movq 40(%[b]), %%rax\n\t" - "mulq 104(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[14] * B[4]\n\t" - "movq 32(%[b]), %%rax\n\t" - "mulq 112(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[15] * B[3]\n\t" - "movq 24(%[b]), %%rax\n\t" - "mulq 120(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "movq %%rbx, 144(%[r])\n\t" - "# A[4] * B[15]\n\t" - "movq 120(%[b]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "xorq %%rbx, %%rbx\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[5] * B[14]\n\t" - "movq 112(%[b]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[6] * B[13]\n\t" - "movq 104(%[b]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[7] * B[12]\n\t" - "movq 96(%[b]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[8] * B[11]\n\t" - "movq 88(%[b]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[9] * B[10]\n\t" - "movq 80(%[b]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[10] * B[9]\n\t" - "movq 72(%[b]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[11] * B[8]\n\t" - "movq 64(%[b]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[12] * B[7]\n\t" - "movq 56(%[b]), %%rax\n\t" - "mulq 96(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[13] * B[6]\n\t" - "movq 48(%[b]), %%rax\n\t" - "mulq 104(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[14] * B[5]\n\t" - "movq 40(%[b]), %%rax\n\t" - "mulq 112(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[15] * B[4]\n\t" - "movq 32(%[b]), %%rax\n\t" - "mulq 120(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "movq %%rcx, 152(%[r])\n\t" - "# A[5] * B[15]\n\t" - "movq 120(%[b]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "xorq %%rcx, %%rcx\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[6] * B[14]\n\t" - "movq 112(%[b]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[7] * B[13]\n\t" - "movq 104(%[b]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[8] * B[12]\n\t" - "movq 96(%[b]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[9] * B[11]\n\t" - "movq 88(%[b]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[10] * B[10]\n\t" - "movq 80(%[b]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[11] * B[9]\n\t" - "movq 72(%[b]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[12] * B[8]\n\t" - "movq 64(%[b]), %%rax\n\t" - "mulq 96(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[13] * B[7]\n\t" - "movq 56(%[b]), %%rax\n\t" - "mulq 104(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[14] * B[6]\n\t" - "movq 48(%[b]), %%rax\n\t" - "mulq 112(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[15] * B[5]\n\t" - "movq 40(%[b]), %%rax\n\t" - "mulq 120(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "movq %%r8, 160(%[r])\n\t" - "# A[6] * B[15]\n\t" - "movq 120(%[b]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[7] * B[14]\n\t" - "movq 112(%[b]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[8] * B[13]\n\t" - "movq 104(%[b]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[9] * B[12]\n\t" - "movq 96(%[b]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[10] * B[11]\n\t" - "movq 88(%[b]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[11] * B[10]\n\t" - "movq 80(%[b]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[12] * B[9]\n\t" - "movq 72(%[b]), %%rax\n\t" - "mulq 96(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[13] * B[8]\n\t" - "movq 64(%[b]), %%rax\n\t" - "mulq 104(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[14] * B[7]\n\t" - "movq 56(%[b]), %%rax\n\t" - "mulq 112(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[15] * B[6]\n\t" - "movq 48(%[b]), %%rax\n\t" - "mulq 120(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "movq %%rbx, 168(%[r])\n\t" - "# A[7] * B[15]\n\t" - "movq 120(%[b]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "xorq %%rbx, %%rbx\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[8] * B[14]\n\t" - "movq 112(%[b]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[9] * B[13]\n\t" - "movq 104(%[b]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[10] * B[12]\n\t" - "movq 96(%[b]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[11] * B[11]\n\t" - "movq 88(%[b]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[12] * B[10]\n\t" - "movq 80(%[b]), %%rax\n\t" - "mulq 96(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[13] * B[9]\n\t" - "movq 72(%[b]), %%rax\n\t" - "mulq 104(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[14] * B[8]\n\t" - "movq 64(%[b]), %%rax\n\t" - "mulq 112(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[15] * B[7]\n\t" - "movq 56(%[b]), %%rax\n\t" - "mulq 120(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "movq %%rcx, 176(%[r])\n\t" - "# A[8] * B[15]\n\t" - "movq 120(%[b]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "xorq %%rcx, %%rcx\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[9] * B[14]\n\t" - "movq 112(%[b]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[10] * B[13]\n\t" - "movq 104(%[b]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[11] * B[12]\n\t" - "movq 96(%[b]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[12] * B[11]\n\t" - "movq 88(%[b]), %%rax\n\t" - "mulq 96(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[13] * B[10]\n\t" - "movq 80(%[b]), %%rax\n\t" - "mulq 104(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[14] * B[9]\n\t" - "movq 72(%[b]), %%rax\n\t" - "mulq 112(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[15] * B[8]\n\t" - "movq 64(%[b]), %%rax\n\t" - "mulq 120(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "movq %%r8, 184(%[r])\n\t" - "# A[9] * B[15]\n\t" - "movq 120(%[b]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[10] * B[14]\n\t" - "movq 112(%[b]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[11] * B[13]\n\t" - "movq 104(%[b]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[12] * B[12]\n\t" - "movq 96(%[b]), %%rax\n\t" - "mulq 96(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[13] * B[11]\n\t" - "movq 88(%[b]), %%rax\n\t" - "mulq 104(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[14] * B[10]\n\t" - "movq 80(%[b]), %%rax\n\t" - "mulq 112(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[15] * B[9]\n\t" - "movq 72(%[b]), %%rax\n\t" - "mulq 120(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "movq %%rbx, 192(%[r])\n\t" - "# A[10] * B[15]\n\t" - "movq 120(%[b]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "xorq %%rbx, %%rbx\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[11] * B[14]\n\t" - "movq 112(%[b]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[12] * B[13]\n\t" - "movq 104(%[b]), %%rax\n\t" - "mulq 96(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[13] * B[12]\n\t" - "movq 96(%[b]), %%rax\n\t" - "mulq 104(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[14] * B[11]\n\t" - "movq 88(%[b]), %%rax\n\t" - "mulq 112(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[15] * B[10]\n\t" - "movq 80(%[b]), %%rax\n\t" - "mulq 120(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "movq %%rcx, 200(%[r])\n\t" - "# A[11] * B[15]\n\t" - "movq 120(%[b]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "xorq %%rcx, %%rcx\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[12] * B[14]\n\t" - "movq 112(%[b]), %%rax\n\t" - "mulq 96(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[13] * B[13]\n\t" - "movq 104(%[b]), %%rax\n\t" - "mulq 104(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[14] * B[12]\n\t" - "movq 96(%[b]), %%rax\n\t" - "mulq 112(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[15] * B[11]\n\t" - "movq 88(%[b]), %%rax\n\t" - "mulq 120(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "movq %%r8, 208(%[r])\n\t" - "# A[12] * B[15]\n\t" - "movq 120(%[b]), %%rax\n\t" - "mulq 96(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[13] * B[14]\n\t" - "movq 112(%[b]), %%rax\n\t" - "mulq 104(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[14] * B[13]\n\t" - "movq 104(%[b]), %%rax\n\t" - "mulq 112(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[15] * B[12]\n\t" - "movq 96(%[b]), %%rax\n\t" - "mulq 120(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "movq %%rbx, 216(%[r])\n\t" - "# A[13] * B[15]\n\t" - "movq 120(%[b]), %%rax\n\t" - "mulq 104(%[a])\n\t" - "xorq %%rbx, %%rbx\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[14] * B[14]\n\t" - "movq 112(%[b]), %%rax\n\t" - "mulq 112(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[15] * B[13]\n\t" - "movq 104(%[b]), %%rax\n\t" - "mulq 120(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "movq %%rcx, 224(%[r])\n\t" - "# A[14] * B[15]\n\t" - "movq 120(%[b]), %%rax\n\t" - "mulq 112(%[a])\n\t" - "xorq %%rcx, %%rcx\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[15] * B[14]\n\t" - "movq 112(%[b]), %%rax\n\t" - "mulq 120(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "movq %%r8, 232(%[r])\n\t" - "# A[15] * B[15]\n\t" - "movq 120(%[b]), %%rax\n\t" - "mulq 120(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "movq %%rbx, 240(%[r])\n\t" - "movq %%rcx, 248(%[r])\n\t" - : - : [r] "r" (r), [a] "r" (a), [b] "r" (b), [tmp] "r" (tmp) - : "memory", "rax", "rdx", "rbx", "rcx", "r8" - ); - - XMEMCPY(r, tmp, sizeof(tmp)); -} - -/* Square a and put result in r. (r = a * a) - * - * r A single precision integer. - * a A single precision integer. - */ -SP_NOINLINE static void sp_2048_sqr_16(sp_digit* r, const sp_digit* a) -{ - sp_digit tmp[16]; - - __asm__ __volatile__ ( - "# A[0] * A[0]\n\t" - "movq 0(%[a]), %%rax\n\t" - "mulq %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "movq %%rax, (%[tmp])\n\t" - "movq %%rdx, %%r8\n\t" - "# A[0] * A[1]\n\t" - "movq 8(%[a]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%rcx, %%rcx\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%r9\n\t" - "adcq $0, %%rcx\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%r9\n\t" - "adcq $0, %%rcx\n\t" - "movq %%r8, 8(%[tmp])\n\t" - "# A[0] * A[2]\n\t" - "movq 16(%[a]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "addq %%rax, %%r9\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "addq %%rax, %%r9\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[1] * A[1]\n\t" - "movq 8(%[a]), %%rax\n\t" - "mulq %%rax\n\t" - "addq %%rax, %%r9\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "movq %%r9, 16(%[tmp])\n\t" - "# A[0] * A[3]\n\t" - "movq 24(%[a]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%r9, %%r9\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%r9\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%r9\n\t" - "# A[1] * A[2]\n\t" - "movq 16(%[a]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%r9\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%r9\n\t" - "movq %%rcx, 24(%[tmp])\n\t" - "# A[0] * A[4]\n\t" - "movq 32(%[a]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%rcx, %%rcx\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%r9\n\t" - "adcq $0, %%rcx\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%r9\n\t" - "adcq $0, %%rcx\n\t" - "# A[1] * A[3]\n\t" - "movq 24(%[a]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%r9\n\t" - "adcq $0, %%rcx\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%r9\n\t" - "adcq $0, %%rcx\n\t" - "# A[2] * A[2]\n\t" - "movq 16(%[a]), %%rax\n\t" - "mulq %%rax\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%r9\n\t" - "adcq $0, %%rcx\n\t" - "movq %%r8, 32(%[tmp])\n\t" - "# A[0] * A[5]\n\t" - "movq 40(%[a]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[1] * A[4]\n\t" - "movq 32(%[a]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[2] * A[3]\n\t" - "movq 24(%[a]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%r10, %%r9\n\t" - "adcq %%r11, %%rcx\n\t" - "adcq %%r12, %%r8\n\t" - "movq %%r9, 40(%[tmp])\n\t" - "# A[0] * A[6]\n\t" - "movq 48(%[a]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%r9, %%r9\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[1] * A[5]\n\t" - "movq 40(%[a]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[2] * A[4]\n\t" - "movq 32(%[a]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[3] * A[3]\n\t" - "movq 24(%[a]), %%rax\n\t" - "mulq %%rax\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%rcx\n\t" - "adcq %%r11, %%r8\n\t" - "adcq %%r12, %%r9\n\t" - "movq %%rcx, 48(%[tmp])\n\t" - "# A[0] * A[7]\n\t" - "movq 56(%[a]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%rcx, %%rcx\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[1] * A[6]\n\t" - "movq 48(%[a]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[2] * A[5]\n\t" - "movq 40(%[a]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[3] * A[4]\n\t" - "movq 32(%[a]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%r10, %%r8\n\t" - "adcq %%r11, %%r9\n\t" - "adcq %%r12, %%rcx\n\t" - "movq %%r8, 56(%[tmp])\n\t" - "# A[0] * A[8]\n\t" - "movq 64(%[a]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[1] * A[7]\n\t" - "movq 56(%[a]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[2] * A[6]\n\t" - "movq 48(%[a]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[3] * A[5]\n\t" - "movq 40(%[a]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[4] * A[4]\n\t" - "movq 32(%[a]), %%rax\n\t" - "mulq %%rax\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%r9\n\t" - "adcq %%r11, %%rcx\n\t" - "adcq %%r12, %%r8\n\t" - "movq %%r9, 64(%[tmp])\n\t" - "# A[0] * A[9]\n\t" - "movq 72(%[a]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%r9, %%r9\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[1] * A[8]\n\t" - "movq 64(%[a]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[2] * A[7]\n\t" - "movq 56(%[a]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[3] * A[6]\n\t" - "movq 48(%[a]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[4] * A[5]\n\t" - "movq 40(%[a]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%r10, %%rcx\n\t" - "adcq %%r11, %%r8\n\t" - "adcq %%r12, %%r9\n\t" - "movq %%rcx, 72(%[tmp])\n\t" - "# A[0] * A[10]\n\t" - "movq 80(%[a]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%rcx, %%rcx\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[1] * A[9]\n\t" - "movq 72(%[a]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[2] * A[8]\n\t" - "movq 64(%[a]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[3] * A[7]\n\t" - "movq 56(%[a]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[4] * A[6]\n\t" - "movq 48(%[a]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[5] * A[5]\n\t" - "movq 40(%[a]), %%rax\n\t" - "mulq %%rax\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%r8\n\t" - "adcq %%r11, %%r9\n\t" - "adcq %%r12, %%rcx\n\t" - "movq %%r8, 80(%[tmp])\n\t" - "# A[0] * A[11]\n\t" - "movq 88(%[a]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[1] * A[10]\n\t" - "movq 80(%[a]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[2] * A[9]\n\t" - "movq 72(%[a]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[3] * A[8]\n\t" - "movq 64(%[a]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[4] * A[7]\n\t" - "movq 56(%[a]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[5] * A[6]\n\t" - "movq 48(%[a]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%r10, %%r9\n\t" - "adcq %%r11, %%rcx\n\t" - "adcq %%r12, %%r8\n\t" - "movq %%r9, 88(%[tmp])\n\t" - "# A[0] * A[12]\n\t" - "movq 96(%[a]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%r9, %%r9\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[1] * A[11]\n\t" - "movq 88(%[a]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[2] * A[10]\n\t" - "movq 80(%[a]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[3] * A[9]\n\t" - "movq 72(%[a]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[4] * A[8]\n\t" - "movq 64(%[a]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[5] * A[7]\n\t" - "movq 56(%[a]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[6] * A[6]\n\t" - "movq 48(%[a]), %%rax\n\t" - "mulq %%rax\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%rcx\n\t" - "adcq %%r11, %%r8\n\t" - "adcq %%r12, %%r9\n\t" - "movq %%rcx, 96(%[tmp])\n\t" - "# A[0] * A[13]\n\t" - "movq 104(%[a]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%rcx, %%rcx\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[1] * A[12]\n\t" - "movq 96(%[a]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[2] * A[11]\n\t" - "movq 88(%[a]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[3] * A[10]\n\t" - "movq 80(%[a]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[4] * A[9]\n\t" - "movq 72(%[a]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[5] * A[8]\n\t" - "movq 64(%[a]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[6] * A[7]\n\t" - "movq 56(%[a]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%r10, %%r8\n\t" - "adcq %%r11, %%r9\n\t" - "adcq %%r12, %%rcx\n\t" - "movq %%r8, 104(%[tmp])\n\t" - "# A[0] * A[14]\n\t" - "movq 112(%[a]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[1] * A[13]\n\t" - "movq 104(%[a]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[2] * A[12]\n\t" - "movq 96(%[a]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[3] * A[11]\n\t" - "movq 88(%[a]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[4] * A[10]\n\t" - "movq 80(%[a]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[5] * A[9]\n\t" - "movq 72(%[a]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[6] * A[8]\n\t" - "movq 64(%[a]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[7] * A[7]\n\t" - "movq 56(%[a]), %%rax\n\t" - "mulq %%rax\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%r9\n\t" - "adcq %%r11, %%rcx\n\t" - "adcq %%r12, %%r8\n\t" - "movq %%r9, 112(%[tmp])\n\t" - "# A[0] * A[15]\n\t" - "movq 120(%[a]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%r9, %%r9\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[1] * A[14]\n\t" - "movq 112(%[a]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[2] * A[13]\n\t" - "movq 104(%[a]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[3] * A[12]\n\t" - "movq 96(%[a]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[4] * A[11]\n\t" - "movq 88(%[a]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[5] * A[10]\n\t" - "movq 80(%[a]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[6] * A[9]\n\t" - "movq 72(%[a]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[7] * A[8]\n\t" - "movq 64(%[a]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%r10, %%rcx\n\t" - "adcq %%r11, %%r8\n\t" - "adcq %%r12, %%r9\n\t" - "movq %%rcx, 120(%[tmp])\n\t" - "# A[1] * A[15]\n\t" - "movq 120(%[a]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "xorq %%rcx, %%rcx\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[2] * A[14]\n\t" - "movq 112(%[a]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[3] * A[13]\n\t" - "movq 104(%[a]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[4] * A[12]\n\t" - "movq 96(%[a]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[5] * A[11]\n\t" - "movq 88(%[a]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[6] * A[10]\n\t" - "movq 80(%[a]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[7] * A[9]\n\t" - "movq 72(%[a]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[8] * A[8]\n\t" - "movq 64(%[a]), %%rax\n\t" - "mulq %%rax\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%r8\n\t" - "adcq %%r11, %%r9\n\t" - "adcq %%r12, %%rcx\n\t" - "movq %%r8, 128(%[r])\n\t" - "# A[2] * A[15]\n\t" - "movq 120(%[a]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[3] * A[14]\n\t" - "movq 112(%[a]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[4] * A[13]\n\t" - "movq 104(%[a]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[5] * A[12]\n\t" - "movq 96(%[a]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[6] * A[11]\n\t" - "movq 88(%[a]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[7] * A[10]\n\t" - "movq 80(%[a]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[8] * A[9]\n\t" - "movq 72(%[a]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%r10, %%r9\n\t" - "adcq %%r11, %%rcx\n\t" - "adcq %%r12, %%r8\n\t" - "movq %%r9, 136(%[r])\n\t" - "# A[3] * A[15]\n\t" - "movq 120(%[a]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "xorq %%r9, %%r9\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[4] * A[14]\n\t" - "movq 112(%[a]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[5] * A[13]\n\t" - "movq 104(%[a]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[6] * A[12]\n\t" - "movq 96(%[a]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[7] * A[11]\n\t" - "movq 88(%[a]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[8] * A[10]\n\t" - "movq 80(%[a]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[9] * A[9]\n\t" - "movq 72(%[a]), %%rax\n\t" - "mulq %%rax\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%rcx\n\t" - "adcq %%r11, %%r8\n\t" - "adcq %%r12, %%r9\n\t" - "movq %%rcx, 144(%[r])\n\t" - "# A[4] * A[15]\n\t" - "movq 120(%[a]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "xorq %%rcx, %%rcx\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[5] * A[14]\n\t" - "movq 112(%[a]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[6] * A[13]\n\t" - "movq 104(%[a]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[7] * A[12]\n\t" - "movq 96(%[a]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[8] * A[11]\n\t" - "movq 88(%[a]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[9] * A[10]\n\t" - "movq 80(%[a]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%r10, %%r8\n\t" - "adcq %%r11, %%r9\n\t" - "adcq %%r12, %%rcx\n\t" - "movq %%r8, 152(%[r])\n\t" - "# A[5] * A[15]\n\t" - "movq 120(%[a]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[6] * A[14]\n\t" - "movq 112(%[a]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[7] * A[13]\n\t" - "movq 104(%[a]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[8] * A[12]\n\t" - "movq 96(%[a]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[9] * A[11]\n\t" - "movq 88(%[a]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[10] * A[10]\n\t" - "movq 80(%[a]), %%rax\n\t" - "mulq %%rax\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%r9\n\t" - "adcq %%r11, %%rcx\n\t" - "adcq %%r12, %%r8\n\t" - "movq %%r9, 160(%[r])\n\t" - "# A[6] * A[15]\n\t" - "movq 120(%[a]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "xorq %%r9, %%r9\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[7] * A[14]\n\t" - "movq 112(%[a]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[8] * A[13]\n\t" - "movq 104(%[a]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[9] * A[12]\n\t" - "movq 96(%[a]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[10] * A[11]\n\t" - "movq 88(%[a]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%r10, %%rcx\n\t" - "adcq %%r11, %%r8\n\t" - "adcq %%r12, %%r9\n\t" - "movq %%rcx, 168(%[r])\n\t" - "# A[7] * A[15]\n\t" - "movq 120(%[a]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "xorq %%rcx, %%rcx\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[8] * A[14]\n\t" - "movq 112(%[a]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[9] * A[13]\n\t" - "movq 104(%[a]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[10] * A[12]\n\t" - "movq 96(%[a]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[11] * A[11]\n\t" - "movq 88(%[a]), %%rax\n\t" - "mulq %%rax\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%r8\n\t" - "adcq %%r11, %%r9\n\t" - "adcq %%r12, %%rcx\n\t" - "movq %%r8, 176(%[r])\n\t" - "# A[8] * A[15]\n\t" - "movq 120(%[a]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[9] * A[14]\n\t" - "movq 112(%[a]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[10] * A[13]\n\t" - "movq 104(%[a]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[11] * A[12]\n\t" - "movq 96(%[a]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%r10, %%r9\n\t" - "adcq %%r11, %%rcx\n\t" - "adcq %%r12, %%r8\n\t" - "movq %%r9, 184(%[r])\n\t" - "# A[9] * A[15]\n\t" - "movq 120(%[a]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "xorq %%r9, %%r9\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[10] * A[14]\n\t" - "movq 112(%[a]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[11] * A[13]\n\t" - "movq 104(%[a]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[12] * A[12]\n\t" - "movq 96(%[a]), %%rax\n\t" - "mulq %%rax\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%rcx\n\t" - "adcq %%r11, %%r8\n\t" - "adcq %%r12, %%r9\n\t" - "movq %%rcx, 192(%[r])\n\t" - "# A[10] * A[15]\n\t" - "movq 120(%[a]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "xorq %%rcx, %%rcx\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[11] * A[14]\n\t" - "movq 112(%[a]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[12] * A[13]\n\t" - "movq 104(%[a]), %%rax\n\t" - "mulq 96(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%r10, %%r8\n\t" - "adcq %%r11, %%r9\n\t" - "adcq %%r12, %%rcx\n\t" - "movq %%r8, 200(%[r])\n\t" - "# A[11] * A[15]\n\t" - "movq 120(%[a]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "addq %%rax, %%r9\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "addq %%rax, %%r9\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[12] * A[14]\n\t" - "movq 112(%[a]), %%rax\n\t" - "mulq 96(%[a])\n\t" - "addq %%rax, %%r9\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "addq %%rax, %%r9\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[13] * A[13]\n\t" - "movq 104(%[a]), %%rax\n\t" - "mulq %%rax\n\t" - "addq %%rax, %%r9\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "movq %%r9, 208(%[r])\n\t" - "# A[12] * A[15]\n\t" - "movq 120(%[a]), %%rax\n\t" - "mulq 96(%[a])\n\t" - "xorq %%r9, %%r9\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%r9\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%r9\n\t" - "# A[13] * A[14]\n\t" - "movq 112(%[a]), %%rax\n\t" - "mulq 104(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%r9\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%r9\n\t" - "movq %%rcx, 216(%[r])\n\t" - "# A[13] * A[15]\n\t" - "movq 120(%[a]), %%rax\n\t" - "mulq 104(%[a])\n\t" - "xorq %%rcx, %%rcx\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%r9\n\t" - "adcq $0, %%rcx\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%r9\n\t" - "adcq $0, %%rcx\n\t" - "# A[14] * A[14]\n\t" - "movq 112(%[a]), %%rax\n\t" - "mulq %%rax\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%r9\n\t" - "adcq $0, %%rcx\n\t" - "movq %%r8, 224(%[r])\n\t" - "# A[14] * A[15]\n\t" - "movq 120(%[a]), %%rax\n\t" - "mulq 112(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "addq %%rax, %%r9\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "addq %%rax, %%r9\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "movq %%r9, 232(%[r])\n\t" - "# A[15] * A[15]\n\t" - "movq 120(%[a]), %%rax\n\t" - "mulq %%rax\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "movq %%rcx, 240(%[r])\n\t" - "movq %%r8, 248(%[r])\n\t" - : - : [r] "r" (r), [a] "r" (a), [tmp] "r" (tmp) - : "memory", "rax", "rdx", "rcx", "r8", "r9", "r10", "r11", "r12" - ); - - XMEMCPY(r, tmp, sizeof(tmp)); -} - -#ifdef HAVE_INTEL_AVX2 -/* Multiply a and b into r. (r = a * b) - * - * r Result of multiplication. - * a First number to multiply. - * b Second number to multiply. - */ -SP_NOINLINE static void sp_2048_mul_avx2_16(sp_digit* r, const sp_digit* a, - const sp_digit* b) -{ - sp_digit tmp[2*16]; - - __asm__ __volatile__ ( - "movq 0(%[a]), %%rdx\n\t" - "xorq %%r15, %%r15\n\t" - "# A[0] * B[0]\n\t" - "mulx 0(%[b]), %%r10, %%r11\n\t" - "# A[0] * B[1]\n\t" - "mulx 8(%[b]), %%r8, %%r12\n\t" - "adcxq %%r8, %%r11\n\t" - "# A[0] * B[2]\n\t" - "mulx 16(%[b]), %%r8, %%r13\n\t" - "adcxq %%r8, %%r12\n\t" - "# A[0] * B[3]\n\t" - "mulx 24(%[b]), %%r8, %%r14\n\t" - "adcxq %%r8, %%r13\n\t" - "movq %%r10, 0(%[t])\n\t" - "movq %%r11, 8(%[t])\n\t" - "movq %%r12, 16(%[t])\n\t" - "movq %%r13, 24(%[t])\n\t" - "# A[0] * B[4]\n\t" - "mulx 32(%[b]), %%r8, %%rax\n\t" - "adcxq %%r8, %%r14\n\t" - "# A[0] * B[5]\n\t" - "mulx 40(%[b]), %%r8, %%r10\n\t" - "adcxq %%r8, %%rax\n\t" - "# A[0] * B[6]\n\t" - "mulx 48(%[b]), %%r8, %%r11\n\t" - "adcxq %%r8, %%r10\n\t" - "# A[0] * B[7]\n\t" - "mulx 56(%[b]), %%r8, %%r12\n\t" - "adcxq %%r8, %%r11\n\t" - "movq %%r14, 32(%[t])\n\t" - "movq %%rax, 40(%[t])\n\t" - "movq %%r10, 48(%[t])\n\t" - "movq %%r11, 56(%[t])\n\t" - "# A[0] * B[8]\n\t" - "mulx 64(%[b]), %%r8, %%r13\n\t" - "adcxq %%r8, %%r12\n\t" - "# A[0] * B[9]\n\t" - "mulx 72(%[b]), %%r8, %%r14\n\t" - "adcxq %%r8, %%r13\n\t" - "# A[0] * B[10]\n\t" - "mulx 80(%[b]), %%r8, %%rax\n\t" - "adcxq %%r8, %%r14\n\t" - "# A[0] * B[11]\n\t" - "mulx 88(%[b]), %%r8, %%r10\n\t" - "adcxq %%r8, %%rax\n\t" - "movq %%r12, 64(%[t])\n\t" - "movq %%r13, 72(%[t])\n\t" - "movq %%r14, 80(%[t])\n\t" - "movq %%rax, 88(%[t])\n\t" - "# A[0] * B[12]\n\t" - "mulx 96(%[b]), %%r8, %%r11\n\t" - "adcxq %%r8, %%r10\n\t" - "# A[0] * B[13]\n\t" - "mulx 104(%[b]), %%r8, %%r12\n\t" - "adcxq %%r8, %%r11\n\t" - "# A[0] * B[14]\n\t" - "mulx 112(%[b]), %%r8, %%r13\n\t" - "adcxq %%r8, %%r12\n\t" - "# A[0] * B[15]\n\t" - "mulx 120(%[b]), %%r8, %%r14\n\t" - "adcxq %%r8, %%r13\n\t" - "adcxq %%r15, %%r14\n\t" - "movq %%r15, %%rcx\n\t" - "adcxq %%r15, %%rcx\n\t" - "movq %%r10, 96(%[t])\n\t" - "movq %%r11, 104(%[t])\n\t" - "movq %%r12, 112(%[t])\n\t" - "movq %%r13, 120(%[t])\n\t" - "movq %%r14, 128(%[t])\n\t" - "movq 8(%[a]), %%rdx\n\t" - "xorq %%r15, %%r15\n\t" - "movq 8(%[t]), %%r11\n\t" - "movq 16(%[t]), %%r12\n\t" - "movq 24(%[t]), %%r13\n\t" - "movq 32(%[t]), %%r14\n\t" - "movq 40(%[t]), %%rax\n\t" - "# A[1] * B[0]\n\t" - "mulx 0(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[1] * B[1]\n\t" - "mulx 8(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[1] * B[2]\n\t" - "mulx 16(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[1] * B[3]\n\t" - "mulx 24(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "movq %%r11, 8(%[t])\n\t" - "movq %%r12, 16(%[t])\n\t" - "movq %%r13, 24(%[t])\n\t" - "movq %%r14, 32(%[t])\n\t" - "movq 48(%[t]), %%r10\n\t" - "movq 56(%[t]), %%r11\n\t" - "movq 64(%[t]), %%r12\n\t" - "movq 72(%[t]), %%r13\n\t" - "# A[1] * B[4]\n\t" - "mulx 32(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[1] * B[5]\n\t" - "mulx 40(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[1] * B[6]\n\t" - "mulx 48(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[1] * B[7]\n\t" - "mulx 56(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "movq %%rax, 40(%[t])\n\t" - "movq %%r10, 48(%[t])\n\t" - "movq %%r11, 56(%[t])\n\t" - "movq %%r12, 64(%[t])\n\t" - "movq 80(%[t]), %%r14\n\t" - "movq 88(%[t]), %%rax\n\t" - "movq 96(%[t]), %%r10\n\t" - "movq 104(%[t]), %%r11\n\t" - "# A[1] * B[8]\n\t" - "mulx 64(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[1] * B[9]\n\t" - "mulx 72(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[1] * B[10]\n\t" - "mulx 80(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[1] * B[11]\n\t" - "mulx 88(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "movq %%r13, 72(%[t])\n\t" - "movq %%r14, 80(%[t])\n\t" - "movq %%rax, 88(%[t])\n\t" - "movq %%r10, 96(%[t])\n\t" - "movq 112(%[t]), %%r12\n\t" - "movq 120(%[t]), %%r13\n\t" - "movq 128(%[t]), %%r14\n\t" - "# A[1] * B[12]\n\t" - "mulx 96(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[1] * B[13]\n\t" - "mulx 104(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[1] * B[14]\n\t" - "mulx 112(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[1] * B[15]\n\t" - "mulx 120(%[b]), %%r8, %%r9\n\t" - "movq %%r15, %%rax\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "adcxq %%rcx, %%rax\n\t" - "movq %%r15, %%rcx\n\t" - "adoxq %%r15, %%rcx\n\t" - "adcxq %%r15, %%rcx\n\t" - "movq %%r11, 104(%[t])\n\t" - "movq %%r12, 112(%[t])\n\t" - "movq %%r13, 120(%[t])\n\t" - "movq %%r14, 128(%[t])\n\t" - "movq %%rax, 136(%[t])\n\t" - "movq 16(%[a]), %%rdx\n\t" - "xorq %%r15, %%r15\n\t" - "movq 16(%[t]), %%r12\n\t" - "movq 24(%[t]), %%r13\n\t" - "movq 32(%[t]), %%r14\n\t" - "movq 40(%[t]), %%rax\n\t" - "movq 48(%[t]), %%r10\n\t" - "# A[2] * B[0]\n\t" - "mulx 0(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[2] * B[1]\n\t" - "mulx 8(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[2] * B[2]\n\t" - "mulx 16(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[2] * B[3]\n\t" - "mulx 24(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "movq %%r12, 16(%[t])\n\t" - "movq %%r13, 24(%[t])\n\t" - "movq %%r14, 32(%[t])\n\t" - "movq %%rax, 40(%[t])\n\t" - "movq 56(%[t]), %%r11\n\t" - "movq 64(%[t]), %%r12\n\t" - "movq 72(%[t]), %%r13\n\t" - "movq 80(%[t]), %%r14\n\t" - "# A[2] * B[4]\n\t" - "mulx 32(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[2] * B[5]\n\t" - "mulx 40(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[2] * B[6]\n\t" - "mulx 48(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[2] * B[7]\n\t" - "mulx 56(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "movq %%r10, 48(%[t])\n\t" - "movq %%r11, 56(%[t])\n\t" - "movq %%r12, 64(%[t])\n\t" - "movq %%r13, 72(%[t])\n\t" - "movq 88(%[t]), %%rax\n\t" - "movq 96(%[t]), %%r10\n\t" - "movq 104(%[t]), %%r11\n\t" - "movq 112(%[t]), %%r12\n\t" - "# A[2] * B[8]\n\t" - "mulx 64(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[2] * B[9]\n\t" - "mulx 72(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[2] * B[10]\n\t" - "mulx 80(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[2] * B[11]\n\t" - "mulx 88(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "movq %%r14, 80(%[t])\n\t" - "movq %%rax, 88(%[t])\n\t" - "movq %%r10, 96(%[t])\n\t" - "movq %%r11, 104(%[t])\n\t" - "movq 120(%[t]), %%r13\n\t" - "movq 128(%[t]), %%r14\n\t" - "movq 136(%[t]), %%rax\n\t" - "# A[2] * B[12]\n\t" - "mulx 96(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[2] * B[13]\n\t" - "mulx 104(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[2] * B[14]\n\t" - "mulx 112(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[2] * B[15]\n\t" - "mulx 120(%[b]), %%r8, %%r9\n\t" - "movq %%r15, %%r10\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "adcxq %%rcx, %%r10\n\t" - "movq %%r15, %%rcx\n\t" - "adoxq %%r15, %%rcx\n\t" - "adcxq %%r15, %%rcx\n\t" - "movq %%r12, 112(%[t])\n\t" - "movq %%r13, 120(%[t])\n\t" - "movq %%r14, 128(%[t])\n\t" - "movq %%rax, 136(%[t])\n\t" - "movq %%r10, 144(%[t])\n\t" - "movq 24(%[a]), %%rdx\n\t" - "xorq %%r15, %%r15\n\t" - "movq 24(%[t]), %%r13\n\t" - "movq 32(%[t]), %%r14\n\t" - "movq 40(%[t]), %%rax\n\t" - "movq 48(%[t]), %%r10\n\t" - "movq 56(%[t]), %%r11\n\t" - "# A[3] * B[0]\n\t" - "mulx 0(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[3] * B[1]\n\t" - "mulx 8(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[3] * B[2]\n\t" - "mulx 16(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[3] * B[3]\n\t" - "mulx 24(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "movq %%r13, 24(%[t])\n\t" - "movq %%r14, 32(%[t])\n\t" - "movq %%rax, 40(%[t])\n\t" - "movq %%r10, 48(%[t])\n\t" - "movq 64(%[t]), %%r12\n\t" - "movq 72(%[t]), %%r13\n\t" - "movq 80(%[t]), %%r14\n\t" - "movq 88(%[t]), %%rax\n\t" - "# A[3] * B[4]\n\t" - "mulx 32(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[3] * B[5]\n\t" - "mulx 40(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[3] * B[6]\n\t" - "mulx 48(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[3] * B[7]\n\t" - "mulx 56(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "movq %%r11, 56(%[t])\n\t" - "movq %%r12, 64(%[t])\n\t" - "movq %%r13, 72(%[t])\n\t" - "movq %%r14, 80(%[t])\n\t" - "movq 96(%[t]), %%r10\n\t" - "movq 104(%[t]), %%r11\n\t" - "movq 112(%[t]), %%r12\n\t" - "movq 120(%[t]), %%r13\n\t" - "# A[3] * B[8]\n\t" - "mulx 64(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[3] * B[9]\n\t" - "mulx 72(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[3] * B[10]\n\t" - "mulx 80(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[3] * B[11]\n\t" - "mulx 88(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "movq %%rax, 88(%[t])\n\t" - "movq %%r10, 96(%[t])\n\t" - "movq %%r11, 104(%[t])\n\t" - "movq %%r12, 112(%[t])\n\t" - "movq 128(%[t]), %%r14\n\t" - "movq 136(%[t]), %%rax\n\t" - "movq 144(%[t]), %%r10\n\t" - "# A[3] * B[12]\n\t" - "mulx 96(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[3] * B[13]\n\t" - "mulx 104(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[3] * B[14]\n\t" - "mulx 112(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[3] * B[15]\n\t" - "mulx 120(%[b]), %%r8, %%r9\n\t" - "movq %%r15, %%r11\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "adcxq %%rcx, %%r11\n\t" - "movq %%r15, %%rcx\n\t" - "adoxq %%r15, %%rcx\n\t" - "adcxq %%r15, %%rcx\n\t" - "movq %%r13, 120(%[t])\n\t" - "movq %%r14, 128(%[t])\n\t" - "movq %%rax, 136(%[t])\n\t" - "movq %%r10, 144(%[t])\n\t" - "movq %%r11, 152(%[t])\n\t" - "movq 32(%[a]), %%rdx\n\t" - "xorq %%r15, %%r15\n\t" - "movq 32(%[t]), %%r14\n\t" - "movq 40(%[t]), %%rax\n\t" - "movq 48(%[t]), %%r10\n\t" - "movq 56(%[t]), %%r11\n\t" - "movq 64(%[t]), %%r12\n\t" - "# A[4] * B[0]\n\t" - "mulx 0(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[4] * B[1]\n\t" - "mulx 8(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[4] * B[2]\n\t" - "mulx 16(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[4] * B[3]\n\t" - "mulx 24(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "movq %%r14, 32(%[t])\n\t" - "movq %%rax, 40(%[t])\n\t" - "movq %%r10, 48(%[t])\n\t" - "movq %%r11, 56(%[t])\n\t" - "movq 72(%[t]), %%r13\n\t" - "movq 80(%[t]), %%r14\n\t" - "movq 88(%[t]), %%rax\n\t" - "movq 96(%[t]), %%r10\n\t" - "# A[4] * B[4]\n\t" - "mulx 32(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[4] * B[5]\n\t" - "mulx 40(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[4] * B[6]\n\t" - "mulx 48(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[4] * B[7]\n\t" - "mulx 56(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "movq %%r12, 64(%[t])\n\t" - "movq %%r13, 72(%[t])\n\t" - "movq %%r14, 80(%[t])\n\t" - "movq %%rax, 88(%[t])\n\t" - "movq 104(%[t]), %%r11\n\t" - "movq 112(%[t]), %%r12\n\t" - "movq 120(%[t]), %%r13\n\t" - "movq 128(%[t]), %%r14\n\t" - "# A[4] * B[8]\n\t" - "mulx 64(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[4] * B[9]\n\t" - "mulx 72(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[4] * B[10]\n\t" - "mulx 80(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[4] * B[11]\n\t" - "mulx 88(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "movq %%r10, 96(%[t])\n\t" - "movq %%r11, 104(%[t])\n\t" - "movq %%r12, 112(%[t])\n\t" - "movq %%r13, 120(%[t])\n\t" - "movq 136(%[t]), %%rax\n\t" - "movq 144(%[t]), %%r10\n\t" - "movq 152(%[t]), %%r11\n\t" - "# A[4] * B[12]\n\t" - "mulx 96(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[4] * B[13]\n\t" - "mulx 104(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[4] * B[14]\n\t" - "mulx 112(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[4] * B[15]\n\t" - "mulx 120(%[b]), %%r8, %%r9\n\t" - "movq %%r15, %%r12\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "adcxq %%rcx, %%r12\n\t" - "movq %%r15, %%rcx\n\t" - "adoxq %%r15, %%rcx\n\t" - "adcxq %%r15, %%rcx\n\t" - "movq %%r14, 128(%[t])\n\t" - "movq %%rax, 136(%[t])\n\t" - "movq %%r10, 144(%[t])\n\t" - "movq %%r11, 152(%[t])\n\t" - "movq %%r12, 160(%[t])\n\t" - "movq 40(%[a]), %%rdx\n\t" - "xorq %%r15, %%r15\n\t" - "movq 40(%[t]), %%rax\n\t" - "movq 48(%[t]), %%r10\n\t" - "movq 56(%[t]), %%r11\n\t" - "movq 64(%[t]), %%r12\n\t" - "movq 72(%[t]), %%r13\n\t" - "# A[5] * B[0]\n\t" - "mulx 0(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[5] * B[1]\n\t" - "mulx 8(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[5] * B[2]\n\t" - "mulx 16(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[5] * B[3]\n\t" - "mulx 24(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "movq %%rax, 40(%[t])\n\t" - "movq %%r10, 48(%[t])\n\t" - "movq %%r11, 56(%[t])\n\t" - "movq %%r12, 64(%[t])\n\t" - "movq 80(%[t]), %%r14\n\t" - "movq 88(%[t]), %%rax\n\t" - "movq 96(%[t]), %%r10\n\t" - "movq 104(%[t]), %%r11\n\t" - "# A[5] * B[4]\n\t" - "mulx 32(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[5] * B[5]\n\t" - "mulx 40(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[5] * B[6]\n\t" - "mulx 48(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[5] * B[7]\n\t" - "mulx 56(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "movq %%r13, 72(%[t])\n\t" - "movq %%r14, 80(%[t])\n\t" - "movq %%rax, 88(%[t])\n\t" - "movq %%r10, 96(%[t])\n\t" - "movq 112(%[t]), %%r12\n\t" - "movq 120(%[t]), %%r13\n\t" - "movq 128(%[t]), %%r14\n\t" - "movq 136(%[t]), %%rax\n\t" - "# A[5] * B[8]\n\t" - "mulx 64(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[5] * B[9]\n\t" - "mulx 72(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[5] * B[10]\n\t" - "mulx 80(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[5] * B[11]\n\t" - "mulx 88(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "movq %%r11, 104(%[t])\n\t" - "movq %%r12, 112(%[t])\n\t" - "movq %%r13, 120(%[t])\n\t" - "movq %%r14, 128(%[t])\n\t" - "movq 144(%[t]), %%r10\n\t" - "movq 152(%[t]), %%r11\n\t" - "movq 160(%[t]), %%r12\n\t" - "# A[5] * B[12]\n\t" - "mulx 96(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[5] * B[13]\n\t" - "mulx 104(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[5] * B[14]\n\t" - "mulx 112(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[5] * B[15]\n\t" - "mulx 120(%[b]), %%r8, %%r9\n\t" - "movq %%r15, %%r13\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "adcxq %%rcx, %%r13\n\t" - "movq %%r15, %%rcx\n\t" - "adoxq %%r15, %%rcx\n\t" - "adcxq %%r15, %%rcx\n\t" - "movq %%rax, 136(%[t])\n\t" - "movq %%r10, 144(%[t])\n\t" - "movq %%r11, 152(%[t])\n\t" - "movq %%r12, 160(%[t])\n\t" - "movq %%r13, 168(%[t])\n\t" - "movq 48(%[a]), %%rdx\n\t" - "xorq %%r15, %%r15\n\t" - "movq 48(%[t]), %%r10\n\t" - "movq 56(%[t]), %%r11\n\t" - "movq 64(%[t]), %%r12\n\t" - "movq 72(%[t]), %%r13\n\t" - "movq 80(%[t]), %%r14\n\t" - "# A[6] * B[0]\n\t" - "mulx 0(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[6] * B[1]\n\t" - "mulx 8(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[6] * B[2]\n\t" - "mulx 16(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[6] * B[3]\n\t" - "mulx 24(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "movq %%r10, 48(%[t])\n\t" - "movq %%r11, 56(%[t])\n\t" - "movq %%r12, 64(%[t])\n\t" - "movq %%r13, 72(%[t])\n\t" - "movq 88(%[t]), %%rax\n\t" - "movq 96(%[t]), %%r10\n\t" - "movq 104(%[t]), %%r11\n\t" - "movq 112(%[t]), %%r12\n\t" - "# A[6] * B[4]\n\t" - "mulx 32(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[6] * B[5]\n\t" - "mulx 40(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[6] * B[6]\n\t" - "mulx 48(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[6] * B[7]\n\t" - "mulx 56(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "movq %%r14, 80(%[t])\n\t" - "movq %%rax, 88(%[t])\n\t" - "movq %%r10, 96(%[t])\n\t" - "movq %%r11, 104(%[t])\n\t" - "movq 120(%[t]), %%r13\n\t" - "movq 128(%[t]), %%r14\n\t" - "movq 136(%[t]), %%rax\n\t" - "movq 144(%[t]), %%r10\n\t" - "# A[6] * B[8]\n\t" - "mulx 64(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[6] * B[9]\n\t" - "mulx 72(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[6] * B[10]\n\t" - "mulx 80(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[6] * B[11]\n\t" - "mulx 88(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "movq %%r12, 112(%[t])\n\t" - "movq %%r13, 120(%[t])\n\t" - "movq %%r14, 128(%[t])\n\t" - "movq %%rax, 136(%[t])\n\t" - "movq 152(%[t]), %%r11\n\t" - "movq 160(%[t]), %%r12\n\t" - "movq 168(%[t]), %%r13\n\t" - "# A[6] * B[12]\n\t" - "mulx 96(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[6] * B[13]\n\t" - "mulx 104(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[6] * B[14]\n\t" - "mulx 112(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[6] * B[15]\n\t" - "mulx 120(%[b]), %%r8, %%r9\n\t" - "movq %%r15, %%r14\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "adcxq %%rcx, %%r14\n\t" - "movq %%r15, %%rcx\n\t" - "adoxq %%r15, %%rcx\n\t" - "adcxq %%r15, %%rcx\n\t" - "movq %%r10, 144(%[t])\n\t" - "movq %%r11, 152(%[t])\n\t" - "movq %%r12, 160(%[t])\n\t" - "movq %%r13, 168(%[t])\n\t" - "movq %%r14, 176(%[t])\n\t" - "movq 56(%[a]), %%rdx\n\t" - "xorq %%r15, %%r15\n\t" - "movq 56(%[t]), %%r11\n\t" - "movq 64(%[t]), %%r12\n\t" - "movq 72(%[t]), %%r13\n\t" - "movq 80(%[t]), %%r14\n\t" - "movq 88(%[t]), %%rax\n\t" - "# A[7] * B[0]\n\t" - "mulx 0(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[7] * B[1]\n\t" - "mulx 8(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[7] * B[2]\n\t" - "mulx 16(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[7] * B[3]\n\t" - "mulx 24(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "movq %%r11, 56(%[t])\n\t" - "movq %%r12, 64(%[t])\n\t" - "movq %%r13, 72(%[t])\n\t" - "movq %%r14, 80(%[t])\n\t" - "movq 96(%[t]), %%r10\n\t" - "movq 104(%[t]), %%r11\n\t" - "movq 112(%[t]), %%r12\n\t" - "movq 120(%[t]), %%r13\n\t" - "# A[7] * B[4]\n\t" - "mulx 32(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[7] * B[5]\n\t" - "mulx 40(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[7] * B[6]\n\t" - "mulx 48(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[7] * B[7]\n\t" - "mulx 56(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "movq %%rax, 88(%[t])\n\t" - "movq %%r10, 96(%[t])\n\t" - "movq %%r11, 104(%[t])\n\t" - "movq %%r12, 112(%[t])\n\t" - "movq 128(%[t]), %%r14\n\t" - "movq 136(%[t]), %%rax\n\t" - "movq 144(%[t]), %%r10\n\t" - "movq 152(%[t]), %%r11\n\t" - "# A[7] * B[8]\n\t" - "mulx 64(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[7] * B[9]\n\t" - "mulx 72(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[7] * B[10]\n\t" - "mulx 80(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[7] * B[11]\n\t" - "mulx 88(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "movq %%r13, 120(%[t])\n\t" - "movq %%r14, 128(%[t])\n\t" - "movq %%rax, 136(%[t])\n\t" - "movq %%r10, 144(%[t])\n\t" - "movq 160(%[t]), %%r12\n\t" - "movq 168(%[t]), %%r13\n\t" - "movq 176(%[t]), %%r14\n\t" - "# A[7] * B[12]\n\t" - "mulx 96(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[7] * B[13]\n\t" - "mulx 104(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[7] * B[14]\n\t" - "mulx 112(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[7] * B[15]\n\t" - "mulx 120(%[b]), %%r8, %%r9\n\t" - "movq %%r15, %%rax\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "adcxq %%rcx, %%rax\n\t" - "movq %%r15, %%rcx\n\t" - "adoxq %%r15, %%rcx\n\t" - "adcxq %%r15, %%rcx\n\t" - "movq %%r11, 152(%[t])\n\t" - "movq %%r12, 160(%[t])\n\t" - "movq %%r13, 168(%[t])\n\t" - "movq %%r14, 176(%[t])\n\t" - "movq %%rax, 184(%[t])\n\t" - "movq 64(%[a]), %%rdx\n\t" - "xorq %%r15, %%r15\n\t" - "movq 64(%[t]), %%r12\n\t" - "movq 72(%[t]), %%r13\n\t" - "movq 80(%[t]), %%r14\n\t" - "movq 88(%[t]), %%rax\n\t" - "movq 96(%[t]), %%r10\n\t" - "# A[8] * B[0]\n\t" - "mulx 0(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[8] * B[1]\n\t" - "mulx 8(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[8] * B[2]\n\t" - "mulx 16(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[8] * B[3]\n\t" - "mulx 24(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "movq %%r12, 64(%[t])\n\t" - "movq %%r13, 72(%[t])\n\t" - "movq %%r14, 80(%[t])\n\t" - "movq %%rax, 88(%[t])\n\t" - "movq 104(%[t]), %%r11\n\t" - "movq 112(%[t]), %%r12\n\t" - "movq 120(%[t]), %%r13\n\t" - "movq 128(%[t]), %%r14\n\t" - "# A[8] * B[4]\n\t" - "mulx 32(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[8] * B[5]\n\t" - "mulx 40(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[8] * B[6]\n\t" - "mulx 48(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[8] * B[7]\n\t" - "mulx 56(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "movq %%r10, 96(%[t])\n\t" - "movq %%r11, 104(%[t])\n\t" - "movq %%r12, 112(%[t])\n\t" - "movq %%r13, 120(%[t])\n\t" - "movq 136(%[t]), %%rax\n\t" - "movq 144(%[t]), %%r10\n\t" - "movq 152(%[t]), %%r11\n\t" - "movq 160(%[t]), %%r12\n\t" - "# A[8] * B[8]\n\t" - "mulx 64(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[8] * B[9]\n\t" - "mulx 72(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[8] * B[10]\n\t" - "mulx 80(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[8] * B[11]\n\t" - "mulx 88(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "movq %%r14, 128(%[t])\n\t" - "movq %%rax, 136(%[t])\n\t" - "movq %%r10, 144(%[t])\n\t" - "movq %%r11, 152(%[t])\n\t" - "movq 168(%[t]), %%r13\n\t" - "movq 176(%[t]), %%r14\n\t" - "movq 184(%[t]), %%rax\n\t" - "# A[8] * B[12]\n\t" - "mulx 96(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[8] * B[13]\n\t" - "mulx 104(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[8] * B[14]\n\t" - "mulx 112(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[8] * B[15]\n\t" - "mulx 120(%[b]), %%r8, %%r9\n\t" - "movq %%r15, %%r10\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "adcxq %%rcx, %%r10\n\t" - "movq %%r15, %%rcx\n\t" - "adoxq %%r15, %%rcx\n\t" - "adcxq %%r15, %%rcx\n\t" - "movq %%r12, 160(%[t])\n\t" - "movq %%r13, 168(%[t])\n\t" - "movq %%r14, 176(%[t])\n\t" - "movq %%rax, 184(%[t])\n\t" - "movq %%r10, 192(%[t])\n\t" - "movq 72(%[a]), %%rdx\n\t" - "xorq %%r15, %%r15\n\t" - "movq 72(%[t]), %%r13\n\t" - "movq 80(%[t]), %%r14\n\t" - "movq 88(%[t]), %%rax\n\t" - "movq 96(%[t]), %%r10\n\t" - "movq 104(%[t]), %%r11\n\t" - "# A[9] * B[0]\n\t" - "mulx 0(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[9] * B[1]\n\t" - "mulx 8(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[9] * B[2]\n\t" - "mulx 16(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[9] * B[3]\n\t" - "mulx 24(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "movq %%r13, 72(%[t])\n\t" - "movq %%r14, 80(%[t])\n\t" - "movq %%rax, 88(%[t])\n\t" - "movq %%r10, 96(%[t])\n\t" - "movq 112(%[t]), %%r12\n\t" - "movq 120(%[t]), %%r13\n\t" - "movq 128(%[t]), %%r14\n\t" - "movq 136(%[t]), %%rax\n\t" - "# A[9] * B[4]\n\t" - "mulx 32(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[9] * B[5]\n\t" - "mulx 40(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[9] * B[6]\n\t" - "mulx 48(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[9] * B[7]\n\t" - "mulx 56(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "movq %%r11, 104(%[t])\n\t" - "movq %%r12, 112(%[t])\n\t" - "movq %%r13, 120(%[t])\n\t" - "movq %%r14, 128(%[t])\n\t" - "movq 144(%[t]), %%r10\n\t" - "movq 152(%[t]), %%r11\n\t" - "movq 160(%[t]), %%r12\n\t" - "movq 168(%[t]), %%r13\n\t" - "# A[9] * B[8]\n\t" - "mulx 64(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[9] * B[9]\n\t" - "mulx 72(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[9] * B[10]\n\t" - "mulx 80(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[9] * B[11]\n\t" - "mulx 88(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "movq %%rax, 136(%[t])\n\t" - "movq %%r10, 144(%[t])\n\t" - "movq %%r11, 152(%[t])\n\t" - "movq %%r12, 160(%[t])\n\t" - "movq 176(%[t]), %%r14\n\t" - "movq 184(%[t]), %%rax\n\t" - "movq 192(%[t]), %%r10\n\t" - "# A[9] * B[12]\n\t" - "mulx 96(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[9] * B[13]\n\t" - "mulx 104(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[9] * B[14]\n\t" - "mulx 112(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[9] * B[15]\n\t" - "mulx 120(%[b]), %%r8, %%r9\n\t" - "movq %%r15, %%r11\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "adcxq %%rcx, %%r11\n\t" - "movq %%r15, %%rcx\n\t" - "adoxq %%r15, %%rcx\n\t" - "adcxq %%r15, %%rcx\n\t" - "movq %%r13, 168(%[t])\n\t" - "movq %%r14, 176(%[t])\n\t" - "movq %%rax, 184(%[t])\n\t" - "movq %%r10, 192(%[t])\n\t" - "movq %%r11, 200(%[t])\n\t" - "movq 80(%[a]), %%rdx\n\t" - "xorq %%r15, %%r15\n\t" - "movq 80(%[t]), %%r14\n\t" - "movq 88(%[t]), %%rax\n\t" - "movq 96(%[t]), %%r10\n\t" - "movq 104(%[t]), %%r11\n\t" - "movq 112(%[t]), %%r12\n\t" - "# A[10] * B[0]\n\t" - "mulx 0(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[10] * B[1]\n\t" - "mulx 8(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[10] * B[2]\n\t" - "mulx 16(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[10] * B[3]\n\t" - "mulx 24(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "movq %%r14, 80(%[t])\n\t" - "movq %%rax, 88(%[t])\n\t" - "movq %%r10, 96(%[t])\n\t" - "movq %%r11, 104(%[t])\n\t" - "movq 120(%[t]), %%r13\n\t" - "movq 128(%[t]), %%r14\n\t" - "movq 136(%[t]), %%rax\n\t" - "movq 144(%[t]), %%r10\n\t" - "# A[10] * B[4]\n\t" - "mulx 32(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[10] * B[5]\n\t" - "mulx 40(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[10] * B[6]\n\t" - "mulx 48(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[10] * B[7]\n\t" - "mulx 56(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "movq %%r12, 112(%[t])\n\t" - "movq %%r13, 120(%[t])\n\t" - "movq %%r14, 128(%[t])\n\t" - "movq %%rax, 136(%[t])\n\t" - "movq 152(%[t]), %%r11\n\t" - "movq 160(%[t]), %%r12\n\t" - "movq 168(%[t]), %%r13\n\t" - "movq 176(%[t]), %%r14\n\t" - "# A[10] * B[8]\n\t" - "mulx 64(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[10] * B[9]\n\t" - "mulx 72(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[10] * B[10]\n\t" - "mulx 80(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[10] * B[11]\n\t" - "mulx 88(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "movq %%r10, 144(%[t])\n\t" - "movq %%r11, 152(%[t])\n\t" - "movq %%r12, 160(%[t])\n\t" - "movq %%r13, 168(%[t])\n\t" - "movq 184(%[t]), %%rax\n\t" - "movq 192(%[t]), %%r10\n\t" - "movq 200(%[t]), %%r11\n\t" - "# A[10] * B[12]\n\t" - "mulx 96(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[10] * B[13]\n\t" - "mulx 104(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[10] * B[14]\n\t" - "mulx 112(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[10] * B[15]\n\t" - "mulx 120(%[b]), %%r8, %%r9\n\t" - "movq %%r15, %%r12\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "adcxq %%rcx, %%r12\n\t" - "movq %%r15, %%rcx\n\t" - "adoxq %%r15, %%rcx\n\t" - "adcxq %%r15, %%rcx\n\t" - "movq %%r14, 176(%[t])\n\t" - "movq %%rax, 184(%[t])\n\t" - "movq %%r10, 192(%[t])\n\t" - "movq %%r11, 200(%[t])\n\t" - "movq %%r12, 208(%[t])\n\t" - "movq 88(%[a]), %%rdx\n\t" - "xorq %%r15, %%r15\n\t" - "movq 88(%[t]), %%rax\n\t" - "movq 96(%[t]), %%r10\n\t" - "movq 104(%[t]), %%r11\n\t" - "movq 112(%[t]), %%r12\n\t" - "movq 120(%[t]), %%r13\n\t" - "# A[11] * B[0]\n\t" - "mulx 0(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[11] * B[1]\n\t" - "mulx 8(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[11] * B[2]\n\t" - "mulx 16(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[11] * B[3]\n\t" - "mulx 24(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "movq %%rax, 88(%[t])\n\t" - "movq %%r10, 96(%[t])\n\t" - "movq %%r11, 104(%[t])\n\t" - "movq %%r12, 112(%[t])\n\t" - "movq 128(%[t]), %%r14\n\t" - "movq 136(%[t]), %%rax\n\t" - "movq 144(%[t]), %%r10\n\t" - "movq 152(%[t]), %%r11\n\t" - "# A[11] * B[4]\n\t" - "mulx 32(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[11] * B[5]\n\t" - "mulx 40(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[11] * B[6]\n\t" - "mulx 48(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[11] * B[7]\n\t" - "mulx 56(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "movq %%r13, 120(%[t])\n\t" - "movq %%r14, 128(%[t])\n\t" - "movq %%rax, 136(%[t])\n\t" - "movq %%r10, 144(%[t])\n\t" - "movq 160(%[t]), %%r12\n\t" - "movq 168(%[t]), %%r13\n\t" - "movq 176(%[t]), %%r14\n\t" - "movq 184(%[t]), %%rax\n\t" - "# A[11] * B[8]\n\t" - "mulx 64(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[11] * B[9]\n\t" - "mulx 72(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[11] * B[10]\n\t" - "mulx 80(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[11] * B[11]\n\t" - "mulx 88(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "movq %%r11, 152(%[t])\n\t" - "movq %%r12, 160(%[t])\n\t" - "movq %%r13, 168(%[t])\n\t" - "movq %%r14, 176(%[t])\n\t" - "movq 192(%[t]), %%r10\n\t" - "movq 200(%[t]), %%r11\n\t" - "movq 208(%[t]), %%r12\n\t" - "# A[11] * B[12]\n\t" - "mulx 96(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[11] * B[13]\n\t" - "mulx 104(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[11] * B[14]\n\t" - "mulx 112(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[11] * B[15]\n\t" - "mulx 120(%[b]), %%r8, %%r9\n\t" - "movq %%r15, %%r13\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "adcxq %%rcx, %%r13\n\t" - "movq %%r15, %%rcx\n\t" - "adoxq %%r15, %%rcx\n\t" - "adcxq %%r15, %%rcx\n\t" - "movq %%rax, 184(%[t])\n\t" - "movq %%r10, 192(%[t])\n\t" - "movq %%r11, 200(%[t])\n\t" - "movq %%r12, 208(%[t])\n\t" - "movq %%r13, 216(%[t])\n\t" - "movq 96(%[a]), %%rdx\n\t" - "xorq %%r15, %%r15\n\t" - "movq 96(%[t]), %%r10\n\t" - "movq 104(%[t]), %%r11\n\t" - "movq 112(%[t]), %%r12\n\t" - "movq 120(%[t]), %%r13\n\t" - "movq 128(%[t]), %%r14\n\t" - "# A[12] * B[0]\n\t" - "mulx 0(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[12] * B[1]\n\t" - "mulx 8(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[12] * B[2]\n\t" - "mulx 16(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[12] * B[3]\n\t" - "mulx 24(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "movq %%r10, 96(%[t])\n\t" - "movq %%r11, 104(%[t])\n\t" - "movq %%r12, 112(%[t])\n\t" - "movq %%r13, 120(%[t])\n\t" - "movq 136(%[t]), %%rax\n\t" - "movq 144(%[t]), %%r10\n\t" - "movq 152(%[t]), %%r11\n\t" - "movq 160(%[t]), %%r12\n\t" - "# A[12] * B[4]\n\t" - "mulx 32(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[12] * B[5]\n\t" - "mulx 40(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[12] * B[6]\n\t" - "mulx 48(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[12] * B[7]\n\t" - "mulx 56(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "movq %%r14, 128(%[t])\n\t" - "movq %%rax, 136(%[t])\n\t" - "movq %%r10, 144(%[t])\n\t" - "movq %%r11, 152(%[t])\n\t" - "movq 168(%[t]), %%r13\n\t" - "movq 176(%[t]), %%r14\n\t" - "movq 184(%[t]), %%rax\n\t" - "movq 192(%[t]), %%r10\n\t" - "# A[12] * B[8]\n\t" - "mulx 64(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[12] * B[9]\n\t" - "mulx 72(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[12] * B[10]\n\t" - "mulx 80(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[12] * B[11]\n\t" - "mulx 88(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "movq %%r12, 160(%[t])\n\t" - "movq %%r13, 168(%[t])\n\t" - "movq %%r14, 176(%[t])\n\t" - "movq %%rax, 184(%[t])\n\t" - "movq 200(%[t]), %%r11\n\t" - "movq 208(%[t]), %%r12\n\t" - "movq 216(%[t]), %%r13\n\t" - "# A[12] * B[12]\n\t" - "mulx 96(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[12] * B[13]\n\t" - "mulx 104(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[12] * B[14]\n\t" - "mulx 112(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[12] * B[15]\n\t" - "mulx 120(%[b]), %%r8, %%r9\n\t" - "movq %%r15, %%r14\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "adcxq %%rcx, %%r14\n\t" - "movq %%r15, %%rcx\n\t" - "adoxq %%r15, %%rcx\n\t" - "adcxq %%r15, %%rcx\n\t" - "movq %%r10, 192(%[t])\n\t" - "movq %%r11, 200(%[t])\n\t" - "movq %%r12, 208(%[t])\n\t" - "movq %%r13, 216(%[t])\n\t" - "movq %%r14, 224(%[t])\n\t" - "movq 104(%[a]), %%rdx\n\t" - "xorq %%r15, %%r15\n\t" - "movq 104(%[t]), %%r11\n\t" - "movq 112(%[t]), %%r12\n\t" - "movq 120(%[t]), %%r13\n\t" - "movq 128(%[t]), %%r14\n\t" - "movq 136(%[t]), %%rax\n\t" - "# A[13] * B[0]\n\t" - "mulx 0(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[13] * B[1]\n\t" - "mulx 8(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[13] * B[2]\n\t" - "mulx 16(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[13] * B[3]\n\t" - "mulx 24(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "movq %%r11, 104(%[t])\n\t" - "movq %%r12, 112(%[t])\n\t" - "movq %%r13, 120(%[t])\n\t" - "movq %%r14, 128(%[t])\n\t" - "movq 144(%[t]), %%r10\n\t" - "movq 152(%[t]), %%r11\n\t" - "movq 160(%[t]), %%r12\n\t" - "movq 168(%[t]), %%r13\n\t" - "# A[13] * B[4]\n\t" - "mulx 32(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[13] * B[5]\n\t" - "mulx 40(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[13] * B[6]\n\t" - "mulx 48(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[13] * B[7]\n\t" - "mulx 56(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "movq %%rax, 136(%[t])\n\t" - "movq %%r10, 144(%[t])\n\t" - "movq %%r11, 152(%[t])\n\t" - "movq %%r12, 160(%[t])\n\t" - "movq 176(%[t]), %%r14\n\t" - "movq 184(%[t]), %%rax\n\t" - "movq 192(%[t]), %%r10\n\t" - "movq 200(%[t]), %%r11\n\t" - "# A[13] * B[8]\n\t" - "mulx 64(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[13] * B[9]\n\t" - "mulx 72(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[13] * B[10]\n\t" - "mulx 80(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[13] * B[11]\n\t" - "mulx 88(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "movq %%r13, 168(%[t])\n\t" - "movq %%r14, 176(%[t])\n\t" - "movq %%rax, 184(%[t])\n\t" - "movq %%r10, 192(%[t])\n\t" - "movq 208(%[t]), %%r12\n\t" - "movq 216(%[t]), %%r13\n\t" - "movq 224(%[t]), %%r14\n\t" - "# A[13] * B[12]\n\t" - "mulx 96(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[13] * B[13]\n\t" - "mulx 104(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[13] * B[14]\n\t" - "mulx 112(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[13] * B[15]\n\t" - "mulx 120(%[b]), %%r8, %%r9\n\t" - "movq %%r15, %%rax\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "adcxq %%rcx, %%rax\n\t" - "movq %%r15, %%rcx\n\t" - "adoxq %%r15, %%rcx\n\t" - "adcxq %%r15, %%rcx\n\t" - "movq %%r11, 200(%[t])\n\t" - "movq %%r12, 208(%[t])\n\t" - "movq %%r13, 216(%[t])\n\t" - "movq %%r14, 224(%[t])\n\t" - "movq %%rax, 232(%[t])\n\t" - "movq 112(%[a]), %%rdx\n\t" - "xorq %%r15, %%r15\n\t" - "movq 112(%[t]), %%r12\n\t" - "movq 120(%[t]), %%r13\n\t" - "movq 128(%[t]), %%r14\n\t" - "movq 136(%[t]), %%rax\n\t" - "movq 144(%[t]), %%r10\n\t" - "# A[14] * B[0]\n\t" - "mulx 0(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[14] * B[1]\n\t" - "mulx 8(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[14] * B[2]\n\t" - "mulx 16(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[14] * B[3]\n\t" - "mulx 24(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "movq %%r12, 112(%[t])\n\t" - "movq %%r13, 120(%[t])\n\t" - "movq %%r14, 128(%[t])\n\t" - "movq %%rax, 136(%[t])\n\t" - "movq 152(%[t]), %%r11\n\t" - "movq 160(%[t]), %%r12\n\t" - "movq 168(%[t]), %%r13\n\t" - "movq 176(%[t]), %%r14\n\t" - "# A[14] * B[4]\n\t" - "mulx 32(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[14] * B[5]\n\t" - "mulx 40(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[14] * B[6]\n\t" - "mulx 48(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[14] * B[7]\n\t" - "mulx 56(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "movq %%r10, 144(%[t])\n\t" - "movq %%r11, 152(%[t])\n\t" - "movq %%r12, 160(%[t])\n\t" - "movq %%r13, 168(%[t])\n\t" - "movq 184(%[t]), %%rax\n\t" - "movq 192(%[t]), %%r10\n\t" - "movq 200(%[t]), %%r11\n\t" - "movq 208(%[t]), %%r12\n\t" - "# A[14] * B[8]\n\t" - "mulx 64(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[14] * B[9]\n\t" - "mulx 72(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[14] * B[10]\n\t" - "mulx 80(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[14] * B[11]\n\t" - "mulx 88(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "movq %%r14, 176(%[t])\n\t" - "movq %%rax, 184(%[t])\n\t" - "movq %%r10, 192(%[t])\n\t" - "movq %%r11, 200(%[t])\n\t" - "movq 216(%[t]), %%r13\n\t" - "movq 224(%[t]), %%r14\n\t" - "movq 232(%[t]), %%rax\n\t" - "# A[14] * B[12]\n\t" - "mulx 96(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[14] * B[13]\n\t" - "mulx 104(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[14] * B[14]\n\t" - "mulx 112(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[14] * B[15]\n\t" - "mulx 120(%[b]), %%r8, %%r9\n\t" - "movq %%r15, %%r10\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "adcxq %%rcx, %%r10\n\t" - "movq %%r15, %%rcx\n\t" - "adoxq %%r15, %%rcx\n\t" - "adcxq %%r15, %%rcx\n\t" - "movq %%r12, 208(%[t])\n\t" - "movq %%r13, 216(%[t])\n\t" - "movq %%r14, 224(%[t])\n\t" - "movq %%rax, 232(%[t])\n\t" - "movq %%r10, 240(%[t])\n\t" - "movq 120(%[a]), %%rdx\n\t" - "xorq %%r15, %%r15\n\t" - "movq 120(%[t]), %%r13\n\t" - "movq 128(%[t]), %%r14\n\t" - "movq 136(%[t]), %%rax\n\t" - "movq 144(%[t]), %%r10\n\t" - "movq 152(%[t]), %%r11\n\t" - "# A[15] * B[0]\n\t" - "mulx 0(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[15] * B[1]\n\t" - "mulx 8(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[15] * B[2]\n\t" - "mulx 16(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[15] * B[3]\n\t" - "mulx 24(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "movq %%r13, 120(%[t])\n\t" - "movq %%r14, 128(%[t])\n\t" - "movq %%rax, 136(%[t])\n\t" - "movq %%r10, 144(%[t])\n\t" - "movq 160(%[t]), %%r12\n\t" - "movq 168(%[t]), %%r13\n\t" - "movq 176(%[t]), %%r14\n\t" - "movq 184(%[t]), %%rax\n\t" - "# A[15] * B[4]\n\t" - "mulx 32(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[15] * B[5]\n\t" - "mulx 40(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[15] * B[6]\n\t" - "mulx 48(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[15] * B[7]\n\t" - "mulx 56(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "movq %%r11, 152(%[t])\n\t" - "movq %%r12, 160(%[t])\n\t" - "movq %%r13, 168(%[t])\n\t" - "movq %%r14, 176(%[t])\n\t" - "movq 192(%[t]), %%r10\n\t" - "movq 200(%[t]), %%r11\n\t" - "movq 208(%[t]), %%r12\n\t" - "movq 216(%[t]), %%r13\n\t" - "# A[15] * B[8]\n\t" - "mulx 64(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[15] * B[9]\n\t" - "mulx 72(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[15] * B[10]\n\t" - "mulx 80(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[15] * B[11]\n\t" - "mulx 88(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "movq %%rax, 184(%[t])\n\t" - "movq %%r10, 192(%[t])\n\t" - "movq %%r11, 200(%[t])\n\t" - "movq %%r12, 208(%[t])\n\t" - "movq 224(%[t]), %%r14\n\t" - "movq 232(%[t]), %%rax\n\t" - "movq 240(%[t]), %%r10\n\t" - "# A[15] * B[12]\n\t" - "mulx 96(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[15] * B[13]\n\t" - "mulx 104(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[15] * B[14]\n\t" - "mulx 112(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[15] * B[15]\n\t" - "mulx 120(%[b]), %%r8, %%r9\n\t" - "movq %%r15, %%r11\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "adcxq %%rcx, %%r11\n\t" - "movq %%r13, 216(%[t])\n\t" - "movq %%r14, 224(%[t])\n\t" - "movq %%rax, 232(%[t])\n\t" - "movq %%r10, 240(%[t])\n\t" - "movq %%r11, 248(%[t])\n\t" - : - : [a] "r" (a), [b] "r" (b), [t] "r" (tmp) - : "memory", "rax", "rdx", "rcx", - "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15" - ); - - XMEMCPY(r, tmp, sizeof(tmp)); -} -#endif /* HAVE_INTEL_AVX2 */ - -#ifdef HAVE_INTEL_AVX2 -/* Square a and put result in r. (r = a * a) - * - * r A single precision integer. - * a A single precision integer. - */ -SP_NOINLINE static void sp_2048_sqr_avx2_16(sp_digit* r, const sp_digit* a) -{ - sp_digit tmp[32]; - - __asm__ __volatile__ ( - "xorq %%r8, %%r8\n\t" - "# Diagonal 1\n\t" - "xorq %%r10, %%r10\n\t" - "xorq %%r11, %%r11\n\t" - "xorq %%r12, %%r12\n\t" - "xorq %%r13, %%r13\n\t" - "xorq %%r14, %%r14\n\t" - "xorq %%r15, %%r15\n\t" - "# A[1] x A[0]\n\t" - "movq 0(%[a]), %%rdx\n\t" - "mulxq 8(%[a]), %%r10, %%r11\n\t" - "# A[2] x A[0]\n\t" - "mulxq 16(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "# A[3] x A[0]\n\t" - "mulxq 24(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "# A[4] x A[0]\n\t" - "mulxq 32(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "# A[5] x A[0]\n\t" - "mulxq 40(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "movq %%r10, 8(%[tmp])\n\t" - "movq %%r11, 16(%[tmp])\n\t" - "movq %%r12, 24(%[tmp])\n\t" - "movq %%r13, 32(%[tmp])\n\t" - "movq %%r14, 40(%[tmp])\n\t" - "movq %%r8, %%r10\n\t" - "movq %%r8, %%r11\n\t" - "movq %%r8, %%r12\n\t" - "movq %%r8, %%r13\n\t" - "movq %%r8, %%r14\n\t" - "# A[6] x A[0]\n\t" - "mulxq 48(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "# A[7] x A[0]\n\t" - "mulxq 56(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "# A[8] x A[0]\n\t" - "mulxq 64(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "# A[9] x A[0]\n\t" - "mulxq 72(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "# A[10] x A[0]\n\t" - "mulxq 80(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "movq %%r15, 48(%[tmp])\n\t" - "movq %%r10, 56(%[tmp])\n\t" - "movq %%r11, 64(%[tmp])\n\t" - "movq %%r12, 72(%[tmp])\n\t" - "movq %%r13, 80(%[tmp])\n\t" - "movq %%r8, %%r15\n\t" - "movq %%r8, %%r10\n\t" - "movq %%r8, %%r11\n\t" - "movq %%r8, %%r12\n\t" - "movq %%r8, %%r13\n\t" - "# A[11] x A[0]\n\t" - "mulxq 88(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "# A[12] x A[0]\n\t" - "mulxq 96(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "# A[13] x A[0]\n\t" - "mulxq 104(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "# A[14] x A[0]\n\t" - "mulxq 112(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "# A[15] x A[0]\n\t" - "mulxq 120(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "movq %%r14, 88(%[tmp])\n\t" - "movq %%r15, 96(%[tmp])\n\t" - "movq %%r10, 104(%[tmp])\n\t" - "movq %%r11, 112(%[tmp])\n\t" - "movq %%r12, 120(%[tmp])\n\t" - "# Carry\n\t" - "adcxq %%r8, %%r13\n\t" - "movq %%r8, %%r9\n\t" - "adcxq %%r8, %%r9\n\t" - "adoxq %%r8, %%r9\n\t" - "movq %%r13, 128(%[tmp])\n\t" - "xorq %%r8, %%r8\n\t" - "# Diagonal 2\n\t" - "movq 24(%[tmp]), %%r13\n\t" - "movq 32(%[tmp]), %%r14\n\t" - "movq 40(%[tmp]), %%r15\n\t" - "movq 48(%[tmp]), %%r10\n\t" - "movq 56(%[tmp]), %%r11\n\t" - "movq 64(%[tmp]), %%r12\n\t" - "# A[2] x A[1]\n\t" - "movq 8(%[a]), %%rdx\n\t" - "mulxq 16(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "# A[3] x A[1]\n\t" - "mulxq 24(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "# A[4] x A[1]\n\t" - "mulxq 32(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "# A[5] x A[1]\n\t" - "mulxq 40(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "# A[6] x A[1]\n\t" - "mulxq 48(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "movq %%r13, 24(%[tmp])\n\t" - "movq %%r14, 32(%[tmp])\n\t" - "movq %%r15, 40(%[tmp])\n\t" - "movq %%r10, 48(%[tmp])\n\t" - "movq %%r11, 56(%[tmp])\n\t" - "movq 72(%[tmp]), %%r13\n\t" - "movq 80(%[tmp]), %%r14\n\t" - "movq 88(%[tmp]), %%r15\n\t" - "movq 96(%[tmp]), %%r10\n\t" - "movq 104(%[tmp]), %%r11\n\t" - "# A[7] x A[1]\n\t" - "mulxq 56(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "# A[8] x A[1]\n\t" - "mulxq 64(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "# A[9] x A[1]\n\t" - "mulxq 72(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "# A[10] x A[1]\n\t" - "mulxq 80(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "# A[11] x A[1]\n\t" - "mulxq 88(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "movq %%r12, 64(%[tmp])\n\t" - "movq %%r13, 72(%[tmp])\n\t" - "movq %%r14, 80(%[tmp])\n\t" - "movq %%r15, 88(%[tmp])\n\t" - "movq %%r10, 96(%[tmp])\n\t" - "movq 112(%[tmp]), %%r12\n\t" - "movq 120(%[tmp]), %%r13\n\t" - "movq 128(%[tmp]), %%r14\n\t" - "movq %%r8, %%r15\n\t" - "movq %%r8, %%r10\n\t" - "# A[12] x A[1]\n\t" - "mulxq 96(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "# A[13] x A[1]\n\t" - "mulxq 104(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "# A[14] x A[1]\n\t" - "mulxq 112(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "# A[15] x A[1]\n\t" - "mulxq 120(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "# A[15] x A[2]\n\t" - "movq 16(%[a]), %%rdx\n\t" - "mulxq 120(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "movq %%r11, 104(%[tmp])\n\t" - "movq %%r12, 112(%[tmp])\n\t" - "movq %%r13, 120(%[tmp])\n\t" - "movq %%r14, 128(%[tmp])\n\t" - "movq %%r15, 136(%[tmp])\n\t" - "# Carry\n\t" - "adcxq %%r9, %%r10\n\t" - "movq %%r8, %%r9\n\t" - "adcxq %%r8, %%r9\n\t" - "adoxq %%r8, %%r9\n\t" - "movq %%r10, 144(%[tmp])\n\t" - "xorq %%r8, %%r8\n\t" - "# Diagonal 3\n\t" - "movq 40(%[tmp]), %%r10\n\t" - "movq 48(%[tmp]), %%r11\n\t" - "movq 56(%[tmp]), %%r12\n\t" - "movq 64(%[tmp]), %%r13\n\t" - "movq 72(%[tmp]), %%r14\n\t" - "movq 80(%[tmp]), %%r15\n\t" - "# A[3] x A[2]\n\t" - "mulxq 24(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "# A[4] x A[2]\n\t" - "mulxq 32(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "# A[5] x A[2]\n\t" - "mulxq 40(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "# A[6] x A[2]\n\t" - "mulxq 48(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "# A[7] x A[2]\n\t" - "mulxq 56(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "movq %%r10, 40(%[tmp])\n\t" - "movq %%r11, 48(%[tmp])\n\t" - "movq %%r12, 56(%[tmp])\n\t" - "movq %%r13, 64(%[tmp])\n\t" - "movq %%r14, 72(%[tmp])\n\t" - "movq 88(%[tmp]), %%r10\n\t" - "movq 96(%[tmp]), %%r11\n\t" - "movq 104(%[tmp]), %%r12\n\t" - "movq 112(%[tmp]), %%r13\n\t" - "movq 120(%[tmp]), %%r14\n\t" - "# A[8] x A[2]\n\t" - "mulxq 64(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "# A[9] x A[2]\n\t" - "mulxq 72(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "# A[10] x A[2]\n\t" - "mulxq 80(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "# A[11] x A[2]\n\t" - "mulxq 88(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "# A[12] x A[2]\n\t" - "mulxq 96(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "movq %%r15, 80(%[tmp])\n\t" - "movq %%r10, 88(%[tmp])\n\t" - "movq %%r11, 96(%[tmp])\n\t" - "movq %%r12, 104(%[tmp])\n\t" - "movq %%r13, 112(%[tmp])\n\t" - "movq 128(%[tmp]), %%r15\n\t" - "movq 136(%[tmp]), %%r10\n\t" - "movq 144(%[tmp]), %%r11\n\t" - "movq %%r8, %%r12\n\t" - "movq %%r8, %%r13\n\t" - "# A[13] x A[2]\n\t" - "mulxq 104(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "# A[14] x A[2]\n\t" - "mulxq 112(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "# A[14] x A[3]\n\t" - "movq 112(%[a]), %%rdx\n\t" - "mulxq 24(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "# A[14] x A[4]\n\t" - "mulxq 32(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "# A[14] x A[5]\n\t" - "mulxq 40(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "movq %%r14, 120(%[tmp])\n\t" - "movq %%r15, 128(%[tmp])\n\t" - "movq %%r10, 136(%[tmp])\n\t" - "movq %%r11, 144(%[tmp])\n\t" - "movq %%r12, 152(%[tmp])\n\t" - "# Carry\n\t" - "adcxq %%r9, %%r13\n\t" - "movq %%r8, %%r9\n\t" - "adcxq %%r8, %%r9\n\t" - "adoxq %%r8, %%r9\n\t" - "movq %%r13, 160(%[tmp])\n\t" - "xorq %%r8, %%r8\n\t" - "# Diagonal 4\n\t" - "movq 56(%[tmp]), %%r13\n\t" - "movq 64(%[tmp]), %%r14\n\t" - "movq 72(%[tmp]), %%r15\n\t" - "movq 80(%[tmp]), %%r10\n\t" - "movq 88(%[tmp]), %%r11\n\t" - "movq 96(%[tmp]), %%r12\n\t" - "# A[4] x A[3]\n\t" - "movq 24(%[a]), %%rdx\n\t" - "mulxq 32(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "# A[5] x A[3]\n\t" - "mulxq 40(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "# A[6] x A[3]\n\t" - "mulxq 48(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "# A[7] x A[3]\n\t" - "mulxq 56(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "# A[8] x A[3]\n\t" - "mulxq 64(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "movq %%r13, 56(%[tmp])\n\t" - "movq %%r14, 64(%[tmp])\n\t" - "movq %%r15, 72(%[tmp])\n\t" - "movq %%r10, 80(%[tmp])\n\t" - "movq %%r11, 88(%[tmp])\n\t" - "movq 104(%[tmp]), %%r13\n\t" - "movq 112(%[tmp]), %%r14\n\t" - "movq 120(%[tmp]), %%r15\n\t" - "movq 128(%[tmp]), %%r10\n\t" - "movq 136(%[tmp]), %%r11\n\t" - "# A[9] x A[3]\n\t" - "mulxq 72(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "# A[10] x A[3]\n\t" - "mulxq 80(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "# A[11] x A[3]\n\t" - "mulxq 88(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "# A[12] x A[3]\n\t" - "mulxq 96(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "# A[13] x A[3]\n\t" - "mulxq 104(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "movq %%r12, 96(%[tmp])\n\t" - "movq %%r13, 104(%[tmp])\n\t" - "movq %%r14, 112(%[tmp])\n\t" - "movq %%r15, 120(%[tmp])\n\t" - "movq %%r10, 128(%[tmp])\n\t" - "movq 144(%[tmp]), %%r12\n\t" - "movq 152(%[tmp]), %%r13\n\t" - "movq 160(%[tmp]), %%r14\n\t" - "movq %%r8, %%r15\n\t" - "movq %%r8, %%r10\n\t" - "# A[13] x A[4]\n\t" - "movq 104(%[a]), %%rdx\n\t" - "mulxq 32(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "# A[13] x A[5]\n\t" - "mulxq 40(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "# A[13] x A[6]\n\t" - "mulxq 48(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "# A[13] x A[7]\n\t" - "mulxq 56(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "# A[13] x A[8]\n\t" - "mulxq 64(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "movq %%r11, 136(%[tmp])\n\t" - "movq %%r12, 144(%[tmp])\n\t" - "movq %%r13, 152(%[tmp])\n\t" - "movq %%r14, 160(%[tmp])\n\t" - "movq %%r15, 168(%[tmp])\n\t" - "# Carry\n\t" - "adcxq %%r9, %%r10\n\t" - "movq %%r8, %%r9\n\t" - "adcxq %%r8, %%r9\n\t" - "adoxq %%r8, %%r9\n\t" - "movq %%r10, 176(%[tmp])\n\t" - "xorq %%r8, %%r8\n\t" - "# Diagonal 5\n\t" - "movq 72(%[tmp]), %%r10\n\t" - "movq 80(%[tmp]), %%r11\n\t" - "movq 88(%[tmp]), %%r12\n\t" - "movq 96(%[tmp]), %%r13\n\t" - "movq 104(%[tmp]), %%r14\n\t" - "movq 112(%[tmp]), %%r15\n\t" - "# A[5] x A[4]\n\t" - "movq 32(%[a]), %%rdx\n\t" - "mulxq 40(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "# A[6] x A[4]\n\t" - "mulxq 48(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "# A[7] x A[4]\n\t" - "mulxq 56(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "# A[8] x A[4]\n\t" - "mulxq 64(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "# A[9] x A[4]\n\t" - "mulxq 72(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "movq %%r10, 72(%[tmp])\n\t" - "movq %%r11, 80(%[tmp])\n\t" - "movq %%r12, 88(%[tmp])\n\t" - "movq %%r13, 96(%[tmp])\n\t" - "movq %%r14, 104(%[tmp])\n\t" - "movq 120(%[tmp]), %%r10\n\t" - "movq 128(%[tmp]), %%r11\n\t" - "movq 136(%[tmp]), %%r12\n\t" - "movq 144(%[tmp]), %%r13\n\t" - "movq 152(%[tmp]), %%r14\n\t" - "# A[10] x A[4]\n\t" - "mulxq 80(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "# A[11] x A[4]\n\t" - "mulxq 88(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "# A[12] x A[4]\n\t" - "mulxq 96(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "# A[12] x A[5]\n\t" - "movq 96(%[a]), %%rdx\n\t" - "mulxq 40(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "# A[12] x A[6]\n\t" - "mulxq 48(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "movq %%r15, 112(%[tmp])\n\t" - "movq %%r10, 120(%[tmp])\n\t" - "movq %%r11, 128(%[tmp])\n\t" - "movq %%r12, 136(%[tmp])\n\t" - "movq %%r13, 144(%[tmp])\n\t" - "movq 160(%[tmp]), %%r15\n\t" - "movq 168(%[tmp]), %%r10\n\t" - "movq 176(%[tmp]), %%r11\n\t" - "movq %%r8, %%r12\n\t" - "movq %%r8, %%r13\n\t" - "# A[12] x A[7]\n\t" - "mulxq 56(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "# A[12] x A[8]\n\t" - "mulxq 64(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "# A[12] x A[9]\n\t" - "mulxq 72(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "# A[12] x A[10]\n\t" - "mulxq 80(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "# A[12] x A[11]\n\t" - "mulxq 88(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "movq %%r14, 152(%[tmp])\n\t" - "movq %%r15, 160(%[tmp])\n\t" - "movq %%r10, 168(%[tmp])\n\t" - "movq %%r11, 176(%[tmp])\n\t" - "movq %%r12, 184(%[tmp])\n\t" - "# Carry\n\t" - "adcxq %%r9, %%r13\n\t" - "movq %%r8, %%r9\n\t" - "adcxq %%r8, %%r9\n\t" - "adoxq %%r8, %%r9\n\t" - "movq %%r13, 192(%[tmp])\n\t" - "xorq %%r8, %%r8\n\t" - "# Diagonal 6\n\t" - "movq 88(%[tmp]), %%r13\n\t" - "movq 96(%[tmp]), %%r14\n\t" - "movq 104(%[tmp]), %%r15\n\t" - "movq 112(%[tmp]), %%r10\n\t" - "movq 120(%[tmp]), %%r11\n\t" - "movq 128(%[tmp]), %%r12\n\t" - "# A[6] x A[5]\n\t" - "movq 40(%[a]), %%rdx\n\t" - "mulxq 48(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "# A[7] x A[5]\n\t" - "mulxq 56(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "# A[8] x A[5]\n\t" - "mulxq 64(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "# A[9] x A[5]\n\t" - "mulxq 72(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "# A[10] x A[5]\n\t" - "mulxq 80(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "movq %%r13, 88(%[tmp])\n\t" - "movq %%r14, 96(%[tmp])\n\t" - "movq %%r15, 104(%[tmp])\n\t" - "movq %%r10, 112(%[tmp])\n\t" - "movq %%r11, 120(%[tmp])\n\t" - "movq 136(%[tmp]), %%r13\n\t" - "movq 144(%[tmp]), %%r14\n\t" - "movq 152(%[tmp]), %%r15\n\t" - "movq 160(%[tmp]), %%r10\n\t" - "movq 168(%[tmp]), %%r11\n\t" - "# A[11] x A[5]\n\t" - "mulxq 88(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "# A[11] x A[6]\n\t" - "movq 88(%[a]), %%rdx\n\t" - "mulxq 48(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "# A[11] x A[7]\n\t" - "mulxq 56(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "# A[11] x A[8]\n\t" - "mulxq 64(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "# A[11] x A[9]\n\t" - "mulxq 72(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "movq %%r12, 128(%[tmp])\n\t" - "movq %%r13, 136(%[tmp])\n\t" - "movq %%r14, 144(%[tmp])\n\t" - "movq %%r15, 152(%[tmp])\n\t" - "movq %%r10, 160(%[tmp])\n\t" - "movq 176(%[tmp]), %%r12\n\t" - "movq 184(%[tmp]), %%r13\n\t" - "movq 192(%[tmp]), %%r14\n\t" - "movq %%r8, %%r15\n\t" - "movq %%r8, %%r10\n\t" - "# A[11] x A[10]\n\t" - "mulxq 80(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "# A[13] x A[9]\n\t" - "movq 104(%[a]), %%rdx\n\t" - "mulxq 72(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "# A[13] x A[10]\n\t" - "mulxq 80(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "# A[13] x A[11]\n\t" - "mulxq 88(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "# A[13] x A[12]\n\t" - "mulxq 96(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "movq %%r11, 168(%[tmp])\n\t" - "movq %%r12, 176(%[tmp])\n\t" - "movq %%r13, 184(%[tmp])\n\t" - "movq %%r14, 192(%[tmp])\n\t" - "movq %%r15, 200(%[tmp])\n\t" - "# Carry\n\t" - "adcxq %%r9, %%r10\n\t" - "movq %%r8, %%r9\n\t" - "adcxq %%r8, %%r9\n\t" - "adoxq %%r8, %%r9\n\t" - "movq %%r10, 208(%[tmp])\n\t" - "xorq %%r8, %%r8\n\t" - "# Diagonal 7\n\t" - "movq 104(%[tmp]), %%r10\n\t" - "movq 112(%[tmp]), %%r11\n\t" - "movq 120(%[tmp]), %%r12\n\t" - "movq 128(%[tmp]), %%r13\n\t" - "movq 136(%[tmp]), %%r14\n\t" - "movq 144(%[tmp]), %%r15\n\t" - "# A[7] x A[6]\n\t" - "movq 48(%[a]), %%rdx\n\t" - "mulxq 56(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "# A[8] x A[6]\n\t" - "mulxq 64(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "# A[9] x A[6]\n\t" - "mulxq 72(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "# A[10] x A[6]\n\t" - "mulxq 80(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "# A[10] x A[7]\n\t" - "movq 80(%[a]), %%rdx\n\t" - "mulxq 56(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "movq %%r10, 104(%[tmp])\n\t" - "movq %%r11, 112(%[tmp])\n\t" - "movq %%r12, 120(%[tmp])\n\t" - "movq %%r13, 128(%[tmp])\n\t" - "movq %%r14, 136(%[tmp])\n\t" - "movq 152(%[tmp]), %%r10\n\t" - "movq 160(%[tmp]), %%r11\n\t" - "movq 168(%[tmp]), %%r12\n\t" - "movq 176(%[tmp]), %%r13\n\t" - "movq 184(%[tmp]), %%r14\n\t" - "# A[10] x A[8]\n\t" - "mulxq 64(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "# A[10] x A[9]\n\t" - "mulxq 72(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "# A[14] x A[6]\n\t" - "movq 112(%[a]), %%rdx\n\t" - "mulxq 48(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "# A[14] x A[7]\n\t" - "mulxq 56(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "# A[14] x A[8]\n\t" - "mulxq 64(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "movq %%r15, 144(%[tmp])\n\t" - "movq %%r10, 152(%[tmp])\n\t" - "movq %%r11, 160(%[tmp])\n\t" - "movq %%r12, 168(%[tmp])\n\t" - "movq %%r13, 176(%[tmp])\n\t" - "movq 192(%[tmp]), %%r15\n\t" - "movq 200(%[tmp]), %%r10\n\t" - "movq 208(%[tmp]), %%r11\n\t" - "movq %%r8, %%r12\n\t" - "movq %%r8, %%r13\n\t" - "# A[14] x A[9]\n\t" - "mulxq 72(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "# A[14] x A[10]\n\t" - "mulxq 80(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "# A[14] x A[11]\n\t" - "mulxq 88(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "# A[14] x A[12]\n\t" - "mulxq 96(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "# A[14] x A[13]\n\t" - "mulxq 104(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "movq %%r14, 184(%[tmp])\n\t" - "movq %%r15, 192(%[tmp])\n\t" - "movq %%r10, 200(%[tmp])\n\t" - "movq %%r11, 208(%[tmp])\n\t" - "movq %%r12, 216(%[tmp])\n\t" - "# Carry\n\t" - "adcxq %%r9, %%r13\n\t" - "movq %%r8, %%r9\n\t" - "adcxq %%r8, %%r9\n\t" - "adoxq %%r8, %%r9\n\t" - "movq %%r13, 224(%[tmp])\n\t" - "xorq %%r8, %%r8\n\t" - "# Diagonal 8\n\t" - "movq 120(%[tmp]), %%r13\n\t" - "movq 128(%[tmp]), %%r14\n\t" - "movq 136(%[tmp]), %%r15\n\t" - "movq 144(%[tmp]), %%r10\n\t" - "movq 152(%[tmp]), %%r11\n\t" - "movq 160(%[tmp]), %%r12\n\t" - "# A[8] x A[7]\n\t" - "movq 56(%[a]), %%rdx\n\t" - "mulxq 64(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "# A[9] x A[7]\n\t" - "mulxq 72(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "# A[9] x A[8]\n\t" - "movq 64(%[a]), %%rdx\n\t" - "mulxq 72(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "# A[15] x A[3]\n\t" - "movq 120(%[a]), %%rdx\n\t" - "mulxq 24(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "# A[15] x A[4]\n\t" - "mulxq 32(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "movq %%r13, 120(%[tmp])\n\t" - "movq %%r14, 128(%[tmp])\n\t" - "movq %%r15, 136(%[tmp])\n\t" - "movq %%r10, 144(%[tmp])\n\t" - "movq %%r11, 152(%[tmp])\n\t" - "movq 168(%[tmp]), %%r13\n\t" - "movq 176(%[tmp]), %%r14\n\t" - "movq 184(%[tmp]), %%r15\n\t" - "movq 192(%[tmp]), %%r10\n\t" - "movq 200(%[tmp]), %%r11\n\t" - "# A[15] x A[5]\n\t" - "mulxq 40(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "# A[15] x A[6]\n\t" - "mulxq 48(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "# A[15] x A[7]\n\t" - "mulxq 56(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "# A[15] x A[8]\n\t" - "mulxq 64(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "# A[15] x A[9]\n\t" - "mulxq 72(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "movq %%r12, 160(%[tmp])\n\t" - "movq %%r13, 168(%[tmp])\n\t" - "movq %%r14, 176(%[tmp])\n\t" - "movq %%r15, 184(%[tmp])\n\t" - "movq %%r10, 192(%[tmp])\n\t" - "movq 208(%[tmp]), %%r12\n\t" - "movq 216(%[tmp]), %%r13\n\t" - "movq 224(%[tmp]), %%r14\n\t" - "movq %%r8, %%r15\n\t" - "movq %%r8, %%r10\n\t" - "# A[15] x A[10]\n\t" - "mulxq 80(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "# A[15] x A[11]\n\t" - "mulxq 88(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "# A[15] x A[12]\n\t" - "mulxq 96(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "# A[15] x A[13]\n\t" - "mulxq 104(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "# A[15] x A[14]\n\t" - "mulxq 112(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "movq %%r11, 200(%[tmp])\n\t" - "movq %%r12, 208(%[tmp])\n\t" - "movq %%r13, 216(%[tmp])\n\t" - "movq %%r14, 224(%[tmp])\n\t" - "movq %%r15, 232(%[tmp])\n\t" - "# Carry\n\t" - "adcxq %%r9, %%r10\n\t" - "movq %%r8, %%r9\n\t" - "adcxq %%r8, %%r9\n\t" - "adoxq %%r8, %%r9\n\t" - "movq %%r10, 240(%[tmp])\n\t" - "movq %%r9, 248(%[tmp])\n\t" - "xorq %%r8, %%r8\n\t" - "# Double and Add in A[i] x A[i]\n\t" - "movq 8(%[tmp]), %%r11\n\t" - "# A[0] x A[0]\n\t" - "movq 0(%[a]), %%rdx\n\t" - "mulxq %%rdx, %%rax, %%rcx\n\t" - "movq %%rax, 0(%[tmp])\n\t" - "adoxq %%r11, %%r11\n\t" - "adcxq %%rcx, %%r11\n\t" - "movq %%r11, 8(%[tmp])\n\t" - "movq 16(%[tmp]), %%r10\n\t" - "movq 24(%[tmp]), %%r11\n\t" - "# A[1] x A[1]\n\t" - "movq 8(%[a]), %%rdx\n\t" - "mulxq %%rdx, %%rax, %%rcx\n\t" - "adoxq %%r10, %%r10\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r11, %%r11\n\t" - "adcxq %%rcx, %%r11\n\t" - "movq %%r10, 16(%[tmp])\n\t" - "movq %%r11, 24(%[tmp])\n\t" - "movq 32(%[tmp]), %%r10\n\t" - "movq 40(%[tmp]), %%r11\n\t" - "# A[2] x A[2]\n\t" - "movq 16(%[a]), %%rdx\n\t" - "mulxq %%rdx, %%rax, %%rcx\n\t" - "adoxq %%r10, %%r10\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r11, %%r11\n\t" - "adcxq %%rcx, %%r11\n\t" - "movq %%r10, 32(%[tmp])\n\t" - "movq %%r11, 40(%[tmp])\n\t" - "movq 48(%[tmp]), %%r10\n\t" - "movq 56(%[tmp]), %%r11\n\t" - "# A[3] x A[3]\n\t" - "movq 24(%[a]), %%rdx\n\t" - "mulxq %%rdx, %%rax, %%rcx\n\t" - "adoxq %%r10, %%r10\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r11, %%r11\n\t" - "adcxq %%rcx, %%r11\n\t" - "movq %%r10, 48(%[tmp])\n\t" - "movq %%r11, 56(%[tmp])\n\t" - "movq 64(%[tmp]), %%r10\n\t" - "movq 72(%[tmp]), %%r11\n\t" - "# A[4] x A[4]\n\t" - "movq 32(%[a]), %%rdx\n\t" - "mulxq %%rdx, %%rax, %%rcx\n\t" - "adoxq %%r10, %%r10\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r11, %%r11\n\t" - "adcxq %%rcx, %%r11\n\t" - "movq %%r10, 64(%[tmp])\n\t" - "movq %%r11, 72(%[tmp])\n\t" - "movq 80(%[tmp]), %%r10\n\t" - "movq 88(%[tmp]), %%r11\n\t" - "# A[5] x A[5]\n\t" - "movq 40(%[a]), %%rdx\n\t" - "mulxq %%rdx, %%rax, %%rcx\n\t" - "adoxq %%r10, %%r10\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r11, %%r11\n\t" - "adcxq %%rcx, %%r11\n\t" - "movq %%r10, 80(%[tmp])\n\t" - "movq %%r11, 88(%[tmp])\n\t" - "movq 96(%[tmp]), %%r10\n\t" - "movq 104(%[tmp]), %%r11\n\t" - "# A[6] x A[6]\n\t" - "movq 48(%[a]), %%rdx\n\t" - "mulxq %%rdx, %%rax, %%rcx\n\t" - "adoxq %%r10, %%r10\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r11, %%r11\n\t" - "adcxq %%rcx, %%r11\n\t" - "movq %%r10, 96(%[tmp])\n\t" - "movq %%r11, 104(%[tmp])\n\t" - "movq 112(%[tmp]), %%r10\n\t" - "movq 120(%[tmp]), %%r11\n\t" - "# A[7] x A[7]\n\t" - "movq 56(%[a]), %%rdx\n\t" - "mulxq %%rdx, %%rax, %%rcx\n\t" - "adoxq %%r10, %%r10\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r11, %%r11\n\t" - "adcxq %%rcx, %%r11\n\t" - "movq %%r10, 112(%[tmp])\n\t" - "movq %%r11, 120(%[tmp])\n\t" - "movq 128(%[tmp]), %%r10\n\t" - "movq 136(%[tmp]), %%r11\n\t" - "# A[8] x A[8]\n\t" - "movq 64(%[a]), %%rdx\n\t" - "mulxq %%rdx, %%rax, %%rcx\n\t" - "adoxq %%r10, %%r10\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r11, %%r11\n\t" - "adcxq %%rcx, %%r11\n\t" - "movq %%r10, 128(%[r])\n\t" - "movq %%r11, 136(%[r])\n\t" - "movq 144(%[tmp]), %%r10\n\t" - "movq 152(%[tmp]), %%r11\n\t" - "# A[9] x A[9]\n\t" - "movq 72(%[a]), %%rdx\n\t" - "mulxq %%rdx, %%rax, %%rcx\n\t" - "adoxq %%r10, %%r10\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r11, %%r11\n\t" - "adcxq %%rcx, %%r11\n\t" - "movq %%r10, 144(%[r])\n\t" - "movq %%r11, 152(%[r])\n\t" - "movq 160(%[tmp]), %%r10\n\t" - "movq 168(%[tmp]), %%r11\n\t" - "# A[10] x A[10]\n\t" - "movq 80(%[a]), %%rdx\n\t" - "mulxq %%rdx, %%rax, %%rcx\n\t" - "adoxq %%r10, %%r10\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r11, %%r11\n\t" - "adcxq %%rcx, %%r11\n\t" - "movq %%r10, 160(%[r])\n\t" - "movq %%r11, 168(%[r])\n\t" - "movq 176(%[tmp]), %%r10\n\t" - "movq 184(%[tmp]), %%r11\n\t" - "# A[11] x A[11]\n\t" - "movq 88(%[a]), %%rdx\n\t" - "mulxq %%rdx, %%rax, %%rcx\n\t" - "adoxq %%r10, %%r10\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r11, %%r11\n\t" - "adcxq %%rcx, %%r11\n\t" - "movq %%r10, 176(%[r])\n\t" - "movq %%r11, 184(%[r])\n\t" - "movq 192(%[tmp]), %%r10\n\t" - "movq 200(%[tmp]), %%r11\n\t" - "# A[12] x A[12]\n\t" - "movq 96(%[a]), %%rdx\n\t" - "mulxq %%rdx, %%rax, %%rcx\n\t" - "adoxq %%r10, %%r10\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r11, %%r11\n\t" - "adcxq %%rcx, %%r11\n\t" - "movq %%r10, 192(%[r])\n\t" - "movq %%r11, 200(%[r])\n\t" - "movq 208(%[tmp]), %%r10\n\t" - "movq 216(%[tmp]), %%r11\n\t" - "# A[13] x A[13]\n\t" - "movq 104(%[a]), %%rdx\n\t" - "mulxq %%rdx, %%rax, %%rcx\n\t" - "adoxq %%r10, %%r10\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r11, %%r11\n\t" - "adcxq %%rcx, %%r11\n\t" - "movq %%r10, 208(%[r])\n\t" - "movq %%r11, 216(%[r])\n\t" - "movq 224(%[tmp]), %%r10\n\t" - "movq 232(%[tmp]), %%r11\n\t" - "# A[14] x A[14]\n\t" - "movq 112(%[a]), %%rdx\n\t" - "mulxq %%rdx, %%rax, %%rcx\n\t" - "adoxq %%r10, %%r10\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r11, %%r11\n\t" - "adcxq %%rcx, %%r11\n\t" - "movq %%r10, 224(%[r])\n\t" - "movq %%r11, 232(%[r])\n\t" - "movq 240(%[tmp]), %%r10\n\t" - "movq 248(%[tmp]), %%r11\n\t" - "# A[15] x A[15]\n\t" - "movq 120(%[a]), %%rdx\n\t" - "mulxq %%rdx, %%rax, %%rcx\n\t" - "adoxq %%r10, %%r10\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r11, %%r11\n\t" - "adcxq %%rcx, %%r11\n\t" - "movq %%r10, 240(%[r])\n\t" - "movq %%r11, 248(%[r])\n\t" - : - : [r] "r" (r), [a] "r" (a), [tmp] "r" (tmp) - : "memory", "rax", "rdx", "rcx", "r8", "r9", "r10", "r11", - "r12", "r13", "r14", "r15" - ); - - XMEMCPY(r, tmp, sizeof(tmp)/2); -} -#endif /* HAVE_INTEL_AVX2 */ - -/* Add b to a into r. (r = a + b) - * - * r A single precision integer. - * a A single precision integer. - * b A single precision integer. - */ -SP_NOINLINE static sp_digit sp_2048_add_16(sp_digit* r, const sp_digit* a, - const sp_digit* b) -{ - sp_digit c = 0; - - __asm__ __volatile__ ( - "movq (%[a]), %%rax\n\t" - "addq (%[b]), %%rax\n\t" - "movq %%rax, (%[r])\n\t" - "movq 8(%[a]), %%rax\n\t" - "adcq 8(%[b]), %%rax\n\t" - "movq %%rax, 8(%[r])\n\t" - "movq 16(%[a]), %%rax\n\t" - "adcq 16(%[b]), %%rax\n\t" - "movq %%rax, 16(%[r])\n\t" - "movq 24(%[a]), %%rax\n\t" - "adcq 24(%[b]), %%rax\n\t" - "movq %%rax, 24(%[r])\n\t" - "movq 32(%[a]), %%rax\n\t" - "adcq 32(%[b]), %%rax\n\t" - "movq %%rax, 32(%[r])\n\t" - "movq 40(%[a]), %%rax\n\t" - "adcq 40(%[b]), %%rax\n\t" - "movq %%rax, 40(%[r])\n\t" - "movq 48(%[a]), %%rax\n\t" - "adcq 48(%[b]), %%rax\n\t" - "movq %%rax, 48(%[r])\n\t" - "movq 56(%[a]), %%rax\n\t" - "adcq 56(%[b]), %%rax\n\t" - "movq %%rax, 56(%[r])\n\t" - "movq 64(%[a]), %%rax\n\t" - "adcq 64(%[b]), %%rax\n\t" - "movq %%rax, 64(%[r])\n\t" - "movq 72(%[a]), %%rax\n\t" - "adcq 72(%[b]), %%rax\n\t" - "movq %%rax, 72(%[r])\n\t" - "movq 80(%[a]), %%rax\n\t" - "adcq 80(%[b]), %%rax\n\t" - "movq %%rax, 80(%[r])\n\t" - "movq 88(%[a]), %%rax\n\t" - "adcq 88(%[b]), %%rax\n\t" - "movq %%rax, 88(%[r])\n\t" - "movq 96(%[a]), %%rax\n\t" - "adcq 96(%[b]), %%rax\n\t" - "movq %%rax, 96(%[r])\n\t" - "movq 104(%[a]), %%rax\n\t" - "adcq 104(%[b]), %%rax\n\t" - "movq %%rax, 104(%[r])\n\t" - "movq 112(%[a]), %%rax\n\t" - "adcq 112(%[b]), %%rax\n\t" - "movq %%rax, 112(%[r])\n\t" - "movq 120(%[a]), %%rax\n\t" - "adcq 120(%[b]), %%rax\n\t" - "movq %%rax, 120(%[r])\n\t" - "adcq $0, %[c]\n\t" - : [c] "+r" (c) - : [r] "r" (r), [a] "r" (a), [b] "r" (b) - : "memory", "rax" - ); - - return c; -} - -/* Sub b from a into a. (a -= b) - * - * a A single precision integer and result. - * b A single precision integer. - */ -SP_NOINLINE static sp_digit sp_2048_sub_in_place_32(sp_digit* a, - const sp_digit* b) -{ - sp_digit c = 0; - - __asm__ __volatile__ ( - "movq 0(%[a]), %%r8\n\t" - "movq 8(%[a]), %%r9\n\t" - "movq 0(%[b]), %%rdx\n\t" - "movq 8(%[b]), %%rcx\n\t" - "subq %%rdx, %%r8\n\t" - "movq 16(%[b]), %%rdx\n\t" - "movq %%r8, 0(%[a])\n\t" - "movq 16(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 24(%[b]), %%rcx\n\t" - "movq %%r9, 8(%[a])\n\t" - "movq 24(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq 32(%[b]), %%rdx\n\t" - "movq %%r8, 16(%[a])\n\t" - "movq 32(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 40(%[b]), %%rcx\n\t" - "movq %%r9, 24(%[a])\n\t" - "movq 40(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq 48(%[b]), %%rdx\n\t" - "movq %%r8, 32(%[a])\n\t" - "movq 48(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 56(%[b]), %%rcx\n\t" - "movq %%r9, 40(%[a])\n\t" - "movq 56(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq 64(%[b]), %%rdx\n\t" - "movq %%r8, 48(%[a])\n\t" - "movq 64(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 72(%[b]), %%rcx\n\t" - "movq %%r9, 56(%[a])\n\t" - "movq 72(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq 80(%[b]), %%rdx\n\t" - "movq %%r8, 64(%[a])\n\t" - "movq 80(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 88(%[b]), %%rcx\n\t" - "movq %%r9, 72(%[a])\n\t" - "movq 88(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq 96(%[b]), %%rdx\n\t" - "movq %%r8, 80(%[a])\n\t" - "movq 96(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 104(%[b]), %%rcx\n\t" - "movq %%r9, 88(%[a])\n\t" - "movq 104(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq 112(%[b]), %%rdx\n\t" - "movq %%r8, 96(%[a])\n\t" - "movq 112(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 120(%[b]), %%rcx\n\t" - "movq %%r9, 104(%[a])\n\t" - "movq 120(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq 128(%[b]), %%rdx\n\t" - "movq %%r8, 112(%[a])\n\t" - "movq 128(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 136(%[b]), %%rcx\n\t" - "movq %%r9, 120(%[a])\n\t" - "movq 136(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq 144(%[b]), %%rdx\n\t" - "movq %%r8, 128(%[a])\n\t" - "movq 144(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 152(%[b]), %%rcx\n\t" - "movq %%r9, 136(%[a])\n\t" - "movq 152(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq 160(%[b]), %%rdx\n\t" - "movq %%r8, 144(%[a])\n\t" - "movq 160(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 168(%[b]), %%rcx\n\t" - "movq %%r9, 152(%[a])\n\t" - "movq 168(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq 176(%[b]), %%rdx\n\t" - "movq %%r8, 160(%[a])\n\t" - "movq 176(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 184(%[b]), %%rcx\n\t" - "movq %%r9, 168(%[a])\n\t" - "movq 184(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq 192(%[b]), %%rdx\n\t" - "movq %%r8, 176(%[a])\n\t" - "movq 192(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 200(%[b]), %%rcx\n\t" - "movq %%r9, 184(%[a])\n\t" - "movq 200(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq 208(%[b]), %%rdx\n\t" - "movq %%r8, 192(%[a])\n\t" - "movq 208(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 216(%[b]), %%rcx\n\t" - "movq %%r9, 200(%[a])\n\t" - "movq 216(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq 224(%[b]), %%rdx\n\t" - "movq %%r8, 208(%[a])\n\t" - "movq 224(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 232(%[b]), %%rcx\n\t" - "movq %%r9, 216(%[a])\n\t" - "movq 232(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq 240(%[b]), %%rdx\n\t" - "movq %%r8, 224(%[a])\n\t" - "movq 240(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 248(%[b]), %%rcx\n\t" - "movq %%r9, 232(%[a])\n\t" - "movq 248(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq %%r8, 240(%[a])\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq %%r9, 248(%[a])\n\t" - "sbbq $0, %[c]\n\t" - : [c] "+r" (c) - : [a] "r" (a), [b] "r" (b) - : "memory", "rdx", "rcx", "r8", "r9" - ); - - return c; -} - -/* Add b to a into r. (r = a + b) - * - * r A single precision integer. - * a A single precision integer. - * b A single precision integer. - */ -SP_NOINLINE static sp_digit sp_2048_add_32(sp_digit* r, const sp_digit* a, - const sp_digit* b) -{ - sp_digit c = 0; - - __asm__ __volatile__ ( - "movq (%[a]), %%rax\n\t" - "addq (%[b]), %%rax\n\t" - "movq %%rax, (%[r])\n\t" - "movq 8(%[a]), %%rax\n\t" - "adcq 8(%[b]), %%rax\n\t" - "movq %%rax, 8(%[r])\n\t" - "movq 16(%[a]), %%rax\n\t" - "adcq 16(%[b]), %%rax\n\t" - "movq %%rax, 16(%[r])\n\t" - "movq 24(%[a]), %%rax\n\t" - "adcq 24(%[b]), %%rax\n\t" - "movq %%rax, 24(%[r])\n\t" - "movq 32(%[a]), %%rax\n\t" - "adcq 32(%[b]), %%rax\n\t" - "movq %%rax, 32(%[r])\n\t" - "movq 40(%[a]), %%rax\n\t" - "adcq 40(%[b]), %%rax\n\t" - "movq %%rax, 40(%[r])\n\t" - "movq 48(%[a]), %%rax\n\t" - "adcq 48(%[b]), %%rax\n\t" - "movq %%rax, 48(%[r])\n\t" - "movq 56(%[a]), %%rax\n\t" - "adcq 56(%[b]), %%rax\n\t" - "movq %%rax, 56(%[r])\n\t" - "movq 64(%[a]), %%rax\n\t" - "adcq 64(%[b]), %%rax\n\t" - "movq %%rax, 64(%[r])\n\t" - "movq 72(%[a]), %%rax\n\t" - "adcq 72(%[b]), %%rax\n\t" - "movq %%rax, 72(%[r])\n\t" - "movq 80(%[a]), %%rax\n\t" - "adcq 80(%[b]), %%rax\n\t" - "movq %%rax, 80(%[r])\n\t" - "movq 88(%[a]), %%rax\n\t" - "adcq 88(%[b]), %%rax\n\t" - "movq %%rax, 88(%[r])\n\t" - "movq 96(%[a]), %%rax\n\t" - "adcq 96(%[b]), %%rax\n\t" - "movq %%rax, 96(%[r])\n\t" - "movq 104(%[a]), %%rax\n\t" - "adcq 104(%[b]), %%rax\n\t" - "movq %%rax, 104(%[r])\n\t" - "movq 112(%[a]), %%rax\n\t" - "adcq 112(%[b]), %%rax\n\t" - "movq %%rax, 112(%[r])\n\t" - "movq 120(%[a]), %%rax\n\t" - "adcq 120(%[b]), %%rax\n\t" - "movq %%rax, 120(%[r])\n\t" - "movq 128(%[a]), %%rax\n\t" - "adcq 128(%[b]), %%rax\n\t" - "movq %%rax, 128(%[r])\n\t" - "movq 136(%[a]), %%rax\n\t" - "adcq 136(%[b]), %%rax\n\t" - "movq %%rax, 136(%[r])\n\t" - "movq 144(%[a]), %%rax\n\t" - "adcq 144(%[b]), %%rax\n\t" - "movq %%rax, 144(%[r])\n\t" - "movq 152(%[a]), %%rax\n\t" - "adcq 152(%[b]), %%rax\n\t" - "movq %%rax, 152(%[r])\n\t" - "movq 160(%[a]), %%rax\n\t" - "adcq 160(%[b]), %%rax\n\t" - "movq %%rax, 160(%[r])\n\t" - "movq 168(%[a]), %%rax\n\t" - "adcq 168(%[b]), %%rax\n\t" - "movq %%rax, 168(%[r])\n\t" - "movq 176(%[a]), %%rax\n\t" - "adcq 176(%[b]), %%rax\n\t" - "movq %%rax, 176(%[r])\n\t" - "movq 184(%[a]), %%rax\n\t" - "adcq 184(%[b]), %%rax\n\t" - "movq %%rax, 184(%[r])\n\t" - "movq 192(%[a]), %%rax\n\t" - "adcq 192(%[b]), %%rax\n\t" - "movq %%rax, 192(%[r])\n\t" - "movq 200(%[a]), %%rax\n\t" - "adcq 200(%[b]), %%rax\n\t" - "movq %%rax, 200(%[r])\n\t" - "movq 208(%[a]), %%rax\n\t" - "adcq 208(%[b]), %%rax\n\t" - "movq %%rax, 208(%[r])\n\t" - "movq 216(%[a]), %%rax\n\t" - "adcq 216(%[b]), %%rax\n\t" - "movq %%rax, 216(%[r])\n\t" - "movq 224(%[a]), %%rax\n\t" - "adcq 224(%[b]), %%rax\n\t" - "movq %%rax, 224(%[r])\n\t" - "movq 232(%[a]), %%rax\n\t" - "adcq 232(%[b]), %%rax\n\t" - "movq %%rax, 232(%[r])\n\t" - "movq 240(%[a]), %%rax\n\t" - "adcq 240(%[b]), %%rax\n\t" - "movq %%rax, 240(%[r])\n\t" - "movq 248(%[a]), %%rax\n\t" - "adcq 248(%[b]), %%rax\n\t" - "movq %%rax, 248(%[r])\n\t" - "adcq $0, %[c]\n\t" - : [c] "+r" (c) - : [r] "r" (r), [a] "r" (a), [b] "r" (b) - : "memory", "rax" - ); - - return c; -} - +extern void sp_2048_mul_16(sp_digit* r, const sp_digit* a, const sp_digit* b); +extern void sp_2048_sqr_16(sp_digit* r, const sp_digit* a); +extern void sp_2048_mul_avx2_16(sp_digit* r, const sp_digit* a, const sp_digit* b); +extern void sp_2048_sqr_avx2_16(sp_digit* r, const sp_digit* a); +extern sp_digit sp_2048_add_16(sp_digit* r, const sp_digit* a, const sp_digit* b); +extern sp_digit sp_2048_sub_in_place_32(sp_digit* a, const sp_digit* b); +extern sp_digit sp_2048_add_32(sp_digit* r, const sp_digit* a, const sp_digit* b); /* AND m into each word of a and store in r. * * r A single precision integer. @@ -6013,90 +365,7 @@ static void sp_2048_mont_setup(sp_digit* a, sp_digit* rho) } #if !defined(SP_RSA_PRIVATE_EXP_D) && defined(WOLFSSL_HAVE_SP_RSA) -/* Sub b from a into a. (a -= b) - * - * a A single precision integer and result. - * b A single precision integer. - */ -SP_NOINLINE static sp_digit sp_2048_sub_in_place_16(sp_digit* a, - const sp_digit* b) -{ - sp_digit c = 0; - - __asm__ __volatile__ ( - "movq 0(%[a]), %%r8\n\t" - "movq 8(%[a]), %%r9\n\t" - "movq 0(%[b]), %%rdx\n\t" - "movq 8(%[b]), %%rcx\n\t" - "subq %%rdx, %%r8\n\t" - "movq 16(%[b]), %%rdx\n\t" - "movq %%r8, 0(%[a])\n\t" - "movq 16(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 24(%[b]), %%rcx\n\t" - "movq %%r9, 8(%[a])\n\t" - "movq 24(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq 32(%[b]), %%rdx\n\t" - "movq %%r8, 16(%[a])\n\t" - "movq 32(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 40(%[b]), %%rcx\n\t" - "movq %%r9, 24(%[a])\n\t" - "movq 40(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq 48(%[b]), %%rdx\n\t" - "movq %%r8, 32(%[a])\n\t" - "movq 48(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 56(%[b]), %%rcx\n\t" - "movq %%r9, 40(%[a])\n\t" - "movq 56(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq 64(%[b]), %%rdx\n\t" - "movq %%r8, 48(%[a])\n\t" - "movq 64(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 72(%[b]), %%rcx\n\t" - "movq %%r9, 56(%[a])\n\t" - "movq 72(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq 80(%[b]), %%rdx\n\t" - "movq %%r8, 64(%[a])\n\t" - "movq 80(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 88(%[b]), %%rcx\n\t" - "movq %%r9, 72(%[a])\n\t" - "movq 88(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq 96(%[b]), %%rdx\n\t" - "movq %%r8, 80(%[a])\n\t" - "movq 96(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 104(%[b]), %%rcx\n\t" - "movq %%r9, 88(%[a])\n\t" - "movq 104(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq 112(%[b]), %%rdx\n\t" - "movq %%r8, 96(%[a])\n\t" - "movq 112(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 120(%[b]), %%rcx\n\t" - "movq %%r9, 104(%[a])\n\t" - "movq 120(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq %%r8, 112(%[a])\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq %%r9, 120(%[a])\n\t" - "sbbq $0, %[c]\n\t" - : [c] "+r" (c) - : [a] "r" (a), [b] "r" (b) - : "memory", "rdx", "rcx", "r8", "r9" - ); - - return c; -} - +extern sp_digit sp_2048_sub_in_place_16(sp_digit* a, const sp_digit* b); /* r = 2^n mod m where n is the number of bits to reduce by. * Given m must be 2048 bits, just need to subtract. * @@ -6111,334 +380,8 @@ static void sp_2048_mont_norm_16(sp_digit* r, sp_digit* m) sp_2048_sub_in_place_16(r, m); } -/* Conditionally subtract b from a using the mask m. - * m is -1 to subtract and 0 when not copying. - * - * r A single precision number representing condition subtract result. - * a A single precision number to subtract from. - * b A single precision number to subtract. - * m Mask value to apply. - */ -static sp_digit sp_2048_cond_sub_16(sp_digit* r, sp_digit* a, sp_digit* b, - sp_digit m) -{ - sp_digit t[16]; - sp_digit c = 0; - - __asm__ __volatile__ ( - "movq 0(%[b]), %%rax\n\t" - "movq 8(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 0(%[t])\n\t" - "movq %%rcx, 8(%[t])\n\t" - "movq 16(%[b]), %%rax\n\t" - "movq 24(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 16(%[t])\n\t" - "movq %%rcx, 24(%[t])\n\t" - "movq 32(%[b]), %%rax\n\t" - "movq 40(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 32(%[t])\n\t" - "movq %%rcx, 40(%[t])\n\t" - "movq 48(%[b]), %%rax\n\t" - "movq 56(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 48(%[t])\n\t" - "movq %%rcx, 56(%[t])\n\t" - "movq 64(%[b]), %%rax\n\t" - "movq 72(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 64(%[t])\n\t" - "movq %%rcx, 72(%[t])\n\t" - "movq 80(%[b]), %%rax\n\t" - "movq 88(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 80(%[t])\n\t" - "movq %%rcx, 88(%[t])\n\t" - "movq 96(%[b]), %%rax\n\t" - "movq 104(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 96(%[t])\n\t" - "movq %%rcx, 104(%[t])\n\t" - "movq 112(%[b]), %%rax\n\t" - "movq 120(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 112(%[t])\n\t" - "movq %%rcx, 120(%[t])\n\t" - "movq (%[a]), %%rax\n\t" - "movq (%[t]), %%rdx\n\t" - "subq %%rdx,%%rax\n\t" - "movq 8(%[a]), %%rcx\n\t" - "movq 8(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 0(%[r])\n\t" - "movq 16(%[a]), %%rax\n\t" - "movq 16(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 8(%[r])\n\t" - "movq 24(%[a]), %%rcx\n\t" - "movq 24(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 16(%[r])\n\t" - "movq 32(%[a]), %%rax\n\t" - "movq 32(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 24(%[r])\n\t" - "movq 40(%[a]), %%rcx\n\t" - "movq 40(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 32(%[r])\n\t" - "movq 48(%[a]), %%rax\n\t" - "movq 48(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 40(%[r])\n\t" - "movq 56(%[a]), %%rcx\n\t" - "movq 56(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 48(%[r])\n\t" - "movq 64(%[a]), %%rax\n\t" - "movq 64(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 56(%[r])\n\t" - "movq 72(%[a]), %%rcx\n\t" - "movq 72(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 64(%[r])\n\t" - "movq 80(%[a]), %%rax\n\t" - "movq 80(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 72(%[r])\n\t" - "movq 88(%[a]), %%rcx\n\t" - "movq 88(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 80(%[r])\n\t" - "movq 96(%[a]), %%rax\n\t" - "movq 96(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 88(%[r])\n\t" - "movq 104(%[a]), %%rcx\n\t" - "movq 104(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 96(%[r])\n\t" - "movq 112(%[a]), %%rax\n\t" - "movq 112(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 104(%[r])\n\t" - "movq 120(%[a]), %%rcx\n\t" - "movq 120(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 112(%[r])\n\t" - "movq %%rcx, 120(%[r])\n\t" - "sbbq $0, %[c]\n\t" - : [c] "+r" (c) - : [r] "r" (r), [a] "r" (a), [b] "r" (b), [m] "r" (m), [t] "r" (t) - : "memory", "rax", "rcx", "rdx" - ); - - return c; -} - -/* Reduce the number back to 2048 bits using Montgomery reduction. - * - * a A single precision number to reduce in place. - * m The single precision number representing the modulus. - * mp The digit representing the negative inverse of m mod 2^n. - */ -SP_NOINLINE static void sp_2048_mont_reduce_16(sp_digit* a, sp_digit* m, - sp_digit mp) -{ - sp_digit ca = 0; - - __asm__ __volatile__ ( - "# i = 0\n\t" - "xorq %%rcx, %%rcx\n\t" - "movq 0(%[a]), %%r12\n\t" - "movq 8(%[a]), %%r13\n\t" - "\nL_mont_loop_16:\n\t" - "# mu = a[i] * mp\n\t" - "movq %%r12, %%r10\n\t" - "imulq %[mp], %%r10\n\t" - "# a[i+0] += m[0] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 0(%[m])\n\t" - "addq %%rax, %%r12\n\t" - "adcq %%rdx, %%r9\n\t" - "# a[i+1] += m[1] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 8(%[m])\n\t" - "movq %%r13, %%r12\n\t" - "addq %%rax, %%r12\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r12\n\t" - "adcq $0, %%r8\n\t" - "# a[i+2] += m[2] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 16(%[m])\n\t" - "movq 16(%[a]), %%r13\n\t" - "addq %%rax, %%r13\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r13\n\t" - "adcq $0, %%r9\n\t" - "# a[i+3] += m[3] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 24(%[m])\n\t" - "movq 24(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 24(%[a])\n\t" - "adcq $0, %%r8\n\t" - "# a[i+4] += m[4] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 32(%[m])\n\t" - "movq 32(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r11\n\t" - "movq %%r11, 32(%[a])\n\t" - "adcq $0, %%r9\n\t" - "# a[i+5] += m[5] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 40(%[m])\n\t" - "movq 40(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 40(%[a])\n\t" - "adcq $0, %%r8\n\t" - "# a[i+6] += m[6] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 48(%[m])\n\t" - "movq 48(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r11\n\t" - "movq %%r11, 48(%[a])\n\t" - "adcq $0, %%r9\n\t" - "# a[i+7] += m[7] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 56(%[m])\n\t" - "movq 56(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 56(%[a])\n\t" - "adcq $0, %%r8\n\t" - "# a[i+8] += m[8] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 64(%[m])\n\t" - "movq 64(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r11\n\t" - "movq %%r11, 64(%[a])\n\t" - "adcq $0, %%r9\n\t" - "# a[i+9] += m[9] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 72(%[m])\n\t" - "movq 72(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 72(%[a])\n\t" - "adcq $0, %%r8\n\t" - "# a[i+10] += m[10] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 80(%[m])\n\t" - "movq 80(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r11\n\t" - "movq %%r11, 80(%[a])\n\t" - "adcq $0, %%r9\n\t" - "# a[i+11] += m[11] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 88(%[m])\n\t" - "movq 88(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 88(%[a])\n\t" - "adcq $0, %%r8\n\t" - "# a[i+12] += m[12] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 96(%[m])\n\t" - "movq 96(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r11\n\t" - "movq %%r11, 96(%[a])\n\t" - "adcq $0, %%r9\n\t" - "# a[i+13] += m[13] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 104(%[m])\n\t" - "movq 104(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 104(%[a])\n\t" - "adcq $0, %%r8\n\t" - "# a[i+14] += m[14] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 112(%[m])\n\t" - "movq 112(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r11\n\t" - "movq %%r11, 112(%[a])\n\t" - "adcq $0, %%r9\n\t" - "# a[i+15] += m[15] * mu\n\t" - "movq %%r10, %%rax\n\t" - "mulq 120(%[m])\n\t" - "movq 120(%[a]), %%r11\n\t" - "addq %%rax, %%r9\n\t" - "adcq %[ca], %%rdx\n\t" - "movq $0, %[ca]\n\t" - "adcq $0, %[ca]\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 120(%[a])\n\t" - "adcq %%rdx, 128(%[a])\n\t" - "adcq $0, %[ca]\n\t" - "# i += 1\n\t" - "addq $8, %[a]\n\t" - "addq $8, %%rcx\n\t" - "cmpq $128, %%rcx\n\t" - "jl L_mont_loop_16\n\t" - "movq %%r12, 0(%[a])\n\t" - "movq %%r13, 8(%[a])\n\t" - : [ca] "+r" (ca), [a] "+r" (a) - : [m] "r" (m), [mp] "r" (mp) - : "memory", "rax", "rdx", "rcx", "r8", "r9", "r10", "r11", - "r12", "r13" - ); - - sp_2048_cond_sub_16(a - 16, a, m, (sp_digit)0 - ca); -} - +extern sp_digit sp_2048_cond_sub_16(sp_digit* r, sp_digit* a, sp_digit* b, sp_digit m); +extern void sp_2048_mont_reduce_16(sp_digit* a, sp_digit* m, sp_digit mp); /* Multiply two Montogmery form numbers mod the modulus (prime). * (r = a * b mod m) * @@ -6469,263 +412,8 @@ static void sp_2048_mont_sqr_16(sp_digit* r, sp_digit* a, sp_digit* m, sp_2048_mont_reduce_16(r, m, mp); } -/* Mul a by digit b into r. (r = a * b) - * - * r A single precision integer. - * a A single precision integer. - * b A single precision digit. - */ -SP_NOINLINE static void sp_2048_mul_d_16(sp_digit* r, const sp_digit* a, - const sp_digit b) -{ - __asm__ __volatile__ ( - "# A[0] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq (%[a])\n\t" - "movq %%rax, %%rbx\n\t" - "movq %%rdx, %%rcx\n\t" - "movq %%rbx, 0(%[r])\n\t" - "# A[1] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rbx, %%rbx\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "movq %%rcx, 8(%[r])\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[2] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rcx, %%rcx\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "movq %%r8, 16(%[r])\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[3] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "movq %%rbx, 24(%[r])\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[4] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rbx, %%rbx\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "movq %%rcx, 32(%[r])\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[5] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rcx, %%rcx\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "movq %%r8, 40(%[r])\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[6] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "movq %%rbx, 48(%[r])\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[7] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rbx, %%rbx\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "movq %%rcx, 56(%[r])\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[8] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rcx, %%rcx\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "movq %%r8, 64(%[r])\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[9] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "movq %%rbx, 72(%[r])\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[10] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rbx, %%rbx\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "movq %%rcx, 80(%[r])\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[11] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rcx, %%rcx\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "movq %%r8, 88(%[r])\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[12] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 96(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "movq %%rbx, 96(%[r])\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[13] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rbx, %%rbx\n\t" - "mulq 104(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "movq %%rcx, 104(%[r])\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[14] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rcx, %%rcx\n\t" - "mulq 112(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "movq %%r8, 112(%[r])\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[15] * B\n\t" - "movq %[b], %%rax\n\t" - "mulq 120(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "movq %%rbx, 120(%[r])\n\t" - "movq %%rcx, 128(%[r])\n\t" - : - : [r] "r" (r), [a] "r" (a), [b] "r" (b) - : "memory", "rax", "rdx", "rbx", "rcx", "r8" - ); -} - -#ifdef HAVE_INTEL_AVX2 -/* Mul a by digit b into r. (r = a * b) - * - * r A single precision integer. - * a A single precision integer. - * b A single precision digit. - */ -SP_NOINLINE static void sp_2048_mul_d_avx2_16(sp_digit* r, const sp_digit* a, - const sp_digit b) -{ - __asm__ __volatile__ ( - "# A[0] * B\n\t" - "movq %[b], %%rdx\n\t" - "xorq %%r10, %%r10\n\t" - "mulxq (%[a]), %%r8, %%r9\n\t" - "movq %%r8, 0(%[r])\n\t" - "# A[1] * B\n\t" - "mulxq 8(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 8(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[2] * B\n\t" - "mulxq 16(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 16(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[3] * B\n\t" - "mulxq 24(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 24(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[4] * B\n\t" - "mulxq 32(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 32(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[5] * B\n\t" - "mulxq 40(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 40(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[6] * B\n\t" - "mulxq 48(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 48(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[7] * B\n\t" - "mulxq 56(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 56(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[8] * B\n\t" - "mulxq 64(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 64(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[9] * B\n\t" - "mulxq 72(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 72(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[10] * B\n\t" - "mulxq 80(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 80(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[11] * B\n\t" - "mulxq 88(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 88(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[12] * B\n\t" - "mulxq 96(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 96(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[13] * B\n\t" - "mulxq 104(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 104(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[14] * B\n\t" - "mulxq 112(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 112(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[15] * B\n\t" - "mulxq 120(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "adoxq %%rcx, %%r8\n\t" - "adcxq %%r10, %%r8\n\t" - "movq %%r9, 120(%[r])\n\t" - "movq %%r8, 128(%[r])\n\t" - : - : [r] "r" (r), [a] "r" (a), [b] "r" (b) - : "memory", "rax", "rdx", "rcx", "r8", "r9", "r10" - ); -} -#endif /* HAVE_INTEL_AVX2 */ - +extern void sp_2048_mul_d_16(sp_digit* r, const sp_digit* a, const sp_digit b); +extern void sp_2048_mul_d_avx2_16(sp_digit* r, const sp_digit* a, const sp_digit b); /* Divide the double width number (d1|d0) by the dividend. (d1|d0 / div) * * d1 The high order half of the number to divide. @@ -6733,175 +421,19 @@ SP_NOINLINE static void sp_2048_mul_d_avx2_16(sp_digit* r, const sp_digit* a, * div The dividend. * returns the result of the division. */ -static sp_digit div_2048_word_16(sp_digit d1, sp_digit d0, sp_digit div) +static WC_INLINE sp_digit div_2048_word_16(sp_digit d1, sp_digit d0, + sp_digit div) { - sp_digit r; - + register sp_digit r asm("rax"); __asm__ __volatile__ ( - "movq %[d0], %%rax\n\t" - "movq %[d1], %%rdx\n\t" - "divq %[div]\n\t" - "movq %%rax, %[r]\n\t" - : [r] "=r" (r) - : [d1] "r" (d1), [d0] "r" (d0), [div] "r" (div) - : "rax", "rdx" + "divq %3" + : "=a" (r) + : "d" (d1), "a" (d0), "r" (div) + : ); - return r; } - -/* Compare a with b in constant time. - * - * a A single precision integer. - * b A single precision integer. - * return -ve, 0 or +ve if a is less than, equal to or greater than b - * respectively. - */ -static int64_t sp_2048_cmp_16(sp_digit* a, sp_digit* b) -{ - sp_digit r = -1; - sp_digit one = 1; - - __asm__ __volatile__ ( - "xorq %%rcx, %%rcx\n\t" - "movq $-1, %%rdx\n\t" - "movq 120(%[a]), %%rbx\n\t" - "movq 120(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 112(%[a]), %%rbx\n\t" - "movq 112(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 104(%[a]), %%rbx\n\t" - "movq 104(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 96(%[a]), %%rbx\n\t" - "movq 96(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 88(%[a]), %%rbx\n\t" - "movq 88(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 80(%[a]), %%rbx\n\t" - "movq 80(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 72(%[a]), %%rbx\n\t" - "movq 72(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 64(%[a]), %%rbx\n\t" - "movq 64(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 56(%[a]), %%rbx\n\t" - "movq 56(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 48(%[a]), %%rbx\n\t" - "movq 48(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 40(%[a]), %%rbx\n\t" - "movq 40(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 32(%[a]), %%rbx\n\t" - "movq 32(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 24(%[a]), %%rbx\n\t" - "movq 24(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 16(%[a]), %%rbx\n\t" - "movq 16(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 8(%[a]), %%rbx\n\t" - "movq 8(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 0(%[a]), %%rbx\n\t" - "movq 0(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "xorq %%rdx, %[r]\n\t" - : [r] "+r" (r) - : [a] "r" (a), [b] "r" (b), [one] "r" (one) - : "rax", "rdx", "rcx", "rbx", "r8" - ); - - return r; -} - +extern int64_t sp_2048_cmp_16(sp_digit* a, sp_digit* b); /* Divide d in a and put remainder into r (m*d + r = a) * m is not calculated as it is not needed at this time. * @@ -7104,142 +636,7 @@ static int sp_2048_mod_exp_16(sp_digit* r, sp_digit* a, sp_digit* e, return err; } -#ifdef HAVE_INTEL_AVX2 -/* Reduce the number back to 2048 bits using Montgomery reduction. - * - * a A single precision number to reduce in place. - * m The single precision number representing the modulus. - * mp The digit representing the negative inverse of m mod 2^n. - */ -SP_NOINLINE static void sp_2048_mont_reduce_avx2_16(sp_digit* a, sp_digit* m, - sp_digit mp) -{ - sp_digit ca = 0; - - __asm__ __volatile__ ( - "# i = 0\n\t" - "movq 0(%[a]), %%r12\n\t" - "xorq %%rcx, %%rcx\n\t" - "\nL_mont_loop_avx2_16:\n\t" - "xorq %%r9, %%r9\n\t" - "movq %%r12, %%r10\n\t" - "# mu = a[i] * mp\n\t" - "movq %%r12, %%rdx\n\t" - "mulxq %[mp], %%rdx, %%r8\n\t" - "# a[i+0] += m[0] * mu\n\t" - "mulxq 0(%[m]), %%rax, %%r8\n\t" - "movq 8(%[a]), %%r12\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r12\n\t" - "# a[i+1] += m[1] * mu\n\t" - "mulxq 8(%[m]), %%rax, %%r8\n\t" - "movq 16(%[a]), %%r10\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%r8, %%r10\n\t" - "# a[i+2] += m[2] * mu\n\t" - "mulxq 16(%[m]), %%rax, %%r8\n\t" - "movq 24(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 16(%[a])\n\t" - "# a[i+3] += m[3] * mu\n\t" - "mulxq 24(%[m]), %%rax, %%r8\n\t" - "movq 32(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 24(%[a])\n\t" - "# a[i+4] += m[4] * mu\n\t" - "mulxq 32(%[m]), %%rax, %%r8\n\t" - "movq 40(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 32(%[a])\n\t" - "# a[i+5] += m[5] * mu\n\t" - "mulxq 40(%[m]), %%rax, %%r8\n\t" - "movq 48(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 40(%[a])\n\t" - "# a[i+6] += m[6] * mu\n\t" - "mulxq 48(%[m]), %%rax, %%r8\n\t" - "movq 56(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 48(%[a])\n\t" - "# a[i+7] += m[7] * mu\n\t" - "mulxq 56(%[m]), %%rax, %%r8\n\t" - "movq 64(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 56(%[a])\n\t" - "# a[i+8] += m[8] * mu\n\t" - "mulxq 64(%[m]), %%rax, %%r8\n\t" - "movq 72(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 64(%[a])\n\t" - "# a[i+9] += m[9] * mu\n\t" - "mulxq 72(%[m]), %%rax, %%r8\n\t" - "movq 80(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 72(%[a])\n\t" - "# a[i+10] += m[10] * mu\n\t" - "mulxq 80(%[m]), %%rax, %%r8\n\t" - "movq 88(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 80(%[a])\n\t" - "# a[i+11] += m[11] * mu\n\t" - "mulxq 88(%[m]), %%rax, %%r8\n\t" - "movq 96(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 88(%[a])\n\t" - "# a[i+12] += m[12] * mu\n\t" - "mulxq 96(%[m]), %%rax, %%r8\n\t" - "movq 104(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 96(%[a])\n\t" - "# a[i+13] += m[13] * mu\n\t" - "mulxq 104(%[m]), %%rax, %%r8\n\t" - "movq 112(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 104(%[a])\n\t" - "# a[i+14] += m[14] * mu\n\t" - "mulxq 112(%[m]), %%rax, %%r8\n\t" - "movq 120(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 112(%[a])\n\t" - "# a[i+15] += m[15] * mu\n\t" - "mulxq 120(%[m]), %%rax, %%r8\n\t" - "movq 128(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 120(%[a])\n\t" - "adcxq %[ca], %%r10\n\t" - "movq %%r9, %[ca]\n\t" - "adoxq %%r9, %[ca]\n\t" - "adcxq %%r9, %[ca]\n\t" - "movq %%r10, 128(%[a])\n\t" - "# i += 1\n\t" - "addq $8, %[a]\n\t" - "addq $1, %%rcx\n\t" - "cmpq $16, %%rcx\n\t" - "jl L_mont_loop_avx2_16\n\t" - "movq %%r12, 0(%[a])\n\t" - : [ca] "+r" (ca), [a] "+r" (a) - : [m] "r" (m), [mp] "r" (mp) - : "memory", "rax", "rdx", "rcx", "r8", "r9", "r10", "r11", "r12" - ); - - sp_2048_cond_sub_16(a - 16, a, m, (sp_digit)0 - ca); -} -#endif /* HAVE_INTEL_AVX2 */ - +extern void sp_2048_mont_reduce_avx2_16(sp_digit* a, sp_digit* m, sp_digit mp); #ifdef HAVE_INTEL_AVX2 /* Multiply two Montogmery form numbers mod the modulus (prime). * (r = a * b mod m) @@ -7438,606 +835,8 @@ static void sp_2048_mont_norm_32(sp_digit* r, sp_digit* m) } #endif /* WOLFSSL_HAVE_SP_DH */ -/* Conditionally subtract b from a using the mask m. - * m is -1 to subtract and 0 when not copying. - * - * r A single precision number representing condition subtract result. - * a A single precision number to subtract from. - * b A single precision number to subtract. - * m Mask value to apply. - */ -static sp_digit sp_2048_cond_sub_32(sp_digit* r, sp_digit* a, sp_digit* b, - sp_digit m) -{ - sp_digit t[32]; - sp_digit c = 0; - - __asm__ __volatile__ ( - "movq 0(%[b]), %%rax\n\t" - "movq 8(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 0(%[t])\n\t" - "movq %%rcx, 8(%[t])\n\t" - "movq 16(%[b]), %%rax\n\t" - "movq 24(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 16(%[t])\n\t" - "movq %%rcx, 24(%[t])\n\t" - "movq 32(%[b]), %%rax\n\t" - "movq 40(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 32(%[t])\n\t" - "movq %%rcx, 40(%[t])\n\t" - "movq 48(%[b]), %%rax\n\t" - "movq 56(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 48(%[t])\n\t" - "movq %%rcx, 56(%[t])\n\t" - "movq 64(%[b]), %%rax\n\t" - "movq 72(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 64(%[t])\n\t" - "movq %%rcx, 72(%[t])\n\t" - "movq 80(%[b]), %%rax\n\t" - "movq 88(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 80(%[t])\n\t" - "movq %%rcx, 88(%[t])\n\t" - "movq 96(%[b]), %%rax\n\t" - "movq 104(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 96(%[t])\n\t" - "movq %%rcx, 104(%[t])\n\t" - "movq 112(%[b]), %%rax\n\t" - "movq 120(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 112(%[t])\n\t" - "movq %%rcx, 120(%[t])\n\t" - "movq 128(%[b]), %%rax\n\t" - "movq 136(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 128(%[t])\n\t" - "movq %%rcx, 136(%[t])\n\t" - "movq 144(%[b]), %%rax\n\t" - "movq 152(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 144(%[t])\n\t" - "movq %%rcx, 152(%[t])\n\t" - "movq 160(%[b]), %%rax\n\t" - "movq 168(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 160(%[t])\n\t" - "movq %%rcx, 168(%[t])\n\t" - "movq 176(%[b]), %%rax\n\t" - "movq 184(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 176(%[t])\n\t" - "movq %%rcx, 184(%[t])\n\t" - "movq 192(%[b]), %%rax\n\t" - "movq 200(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 192(%[t])\n\t" - "movq %%rcx, 200(%[t])\n\t" - "movq 208(%[b]), %%rax\n\t" - "movq 216(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 208(%[t])\n\t" - "movq %%rcx, 216(%[t])\n\t" - "movq 224(%[b]), %%rax\n\t" - "movq 232(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 224(%[t])\n\t" - "movq %%rcx, 232(%[t])\n\t" - "movq 240(%[b]), %%rax\n\t" - "movq 248(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 240(%[t])\n\t" - "movq %%rcx, 248(%[t])\n\t" - "movq (%[a]), %%rax\n\t" - "movq (%[t]), %%rdx\n\t" - "subq %%rdx,%%rax\n\t" - "movq 8(%[a]), %%rcx\n\t" - "movq 8(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 0(%[r])\n\t" - "movq 16(%[a]), %%rax\n\t" - "movq 16(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 8(%[r])\n\t" - "movq 24(%[a]), %%rcx\n\t" - "movq 24(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 16(%[r])\n\t" - "movq 32(%[a]), %%rax\n\t" - "movq 32(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 24(%[r])\n\t" - "movq 40(%[a]), %%rcx\n\t" - "movq 40(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 32(%[r])\n\t" - "movq 48(%[a]), %%rax\n\t" - "movq 48(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 40(%[r])\n\t" - "movq 56(%[a]), %%rcx\n\t" - "movq 56(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 48(%[r])\n\t" - "movq 64(%[a]), %%rax\n\t" - "movq 64(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 56(%[r])\n\t" - "movq 72(%[a]), %%rcx\n\t" - "movq 72(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 64(%[r])\n\t" - "movq 80(%[a]), %%rax\n\t" - "movq 80(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 72(%[r])\n\t" - "movq 88(%[a]), %%rcx\n\t" - "movq 88(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 80(%[r])\n\t" - "movq 96(%[a]), %%rax\n\t" - "movq 96(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 88(%[r])\n\t" - "movq 104(%[a]), %%rcx\n\t" - "movq 104(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 96(%[r])\n\t" - "movq 112(%[a]), %%rax\n\t" - "movq 112(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 104(%[r])\n\t" - "movq 120(%[a]), %%rcx\n\t" - "movq 120(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 112(%[r])\n\t" - "movq 128(%[a]), %%rax\n\t" - "movq 128(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 120(%[r])\n\t" - "movq 136(%[a]), %%rcx\n\t" - "movq 136(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 128(%[r])\n\t" - "movq 144(%[a]), %%rax\n\t" - "movq 144(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 136(%[r])\n\t" - "movq 152(%[a]), %%rcx\n\t" - "movq 152(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 144(%[r])\n\t" - "movq 160(%[a]), %%rax\n\t" - "movq 160(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 152(%[r])\n\t" - "movq 168(%[a]), %%rcx\n\t" - "movq 168(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 160(%[r])\n\t" - "movq 176(%[a]), %%rax\n\t" - "movq 176(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 168(%[r])\n\t" - "movq 184(%[a]), %%rcx\n\t" - "movq 184(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 176(%[r])\n\t" - "movq 192(%[a]), %%rax\n\t" - "movq 192(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 184(%[r])\n\t" - "movq 200(%[a]), %%rcx\n\t" - "movq 200(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 192(%[r])\n\t" - "movq 208(%[a]), %%rax\n\t" - "movq 208(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 200(%[r])\n\t" - "movq 216(%[a]), %%rcx\n\t" - "movq 216(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 208(%[r])\n\t" - "movq 224(%[a]), %%rax\n\t" - "movq 224(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 216(%[r])\n\t" - "movq 232(%[a]), %%rcx\n\t" - "movq 232(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 224(%[r])\n\t" - "movq 240(%[a]), %%rax\n\t" - "movq 240(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 232(%[r])\n\t" - "movq 248(%[a]), %%rcx\n\t" - "movq 248(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 240(%[r])\n\t" - "movq %%rcx, 248(%[r])\n\t" - "sbbq $0, %[c]\n\t" - : [c] "+r" (c) - : [r] "r" (r), [a] "r" (a), [b] "r" (b), [m] "r" (m), [t] "r" (t) - : "memory", "rax", "rcx", "rdx" - ); - - return c; -} - -/* Reduce the number back to 2048 bits using Montgomery reduction. - * - * a A single precision number to reduce in place. - * m The single precision number representing the modulus. - * mp The digit representing the negative inverse of m mod 2^n. - */ -SP_NOINLINE static void sp_2048_mont_reduce_32(sp_digit* a, sp_digit* m, - sp_digit mp) -{ - sp_digit ca = 0; - - __asm__ __volatile__ ( - "# i = 0\n\t" - "xorq %%rcx, %%rcx\n\t" - "movq 0(%[a]), %%r12\n\t" - "movq 8(%[a]), %%r13\n\t" - "\nL_mont_loop_32:\n\t" - "# mu = a[i] * mp\n\t" - "movq %%r12, %%r10\n\t" - "imulq %[mp], %%r10\n\t" - "# a[i+0] += m[0] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 0(%[m])\n\t" - "addq %%rax, %%r12\n\t" - "adcq %%rdx, %%r9\n\t" - "# a[i+1] += m[1] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 8(%[m])\n\t" - "movq %%r13, %%r12\n\t" - "addq %%rax, %%r12\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r12\n\t" - "adcq $0, %%r8\n\t" - "# a[i+2] += m[2] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 16(%[m])\n\t" - "movq 16(%[a]), %%r13\n\t" - "addq %%rax, %%r13\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r13\n\t" - "adcq $0, %%r9\n\t" - "# a[i+3] += m[3] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 24(%[m])\n\t" - "movq 24(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 24(%[a])\n\t" - "adcq $0, %%r8\n\t" - "# a[i+4] += m[4] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 32(%[m])\n\t" - "movq 32(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r11\n\t" - "movq %%r11, 32(%[a])\n\t" - "adcq $0, %%r9\n\t" - "# a[i+5] += m[5] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 40(%[m])\n\t" - "movq 40(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 40(%[a])\n\t" - "adcq $0, %%r8\n\t" - "# a[i+6] += m[6] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 48(%[m])\n\t" - "movq 48(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r11\n\t" - "movq %%r11, 48(%[a])\n\t" - "adcq $0, %%r9\n\t" - "# a[i+7] += m[7] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 56(%[m])\n\t" - "movq 56(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 56(%[a])\n\t" - "adcq $0, %%r8\n\t" - "# a[i+8] += m[8] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 64(%[m])\n\t" - "movq 64(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r11\n\t" - "movq %%r11, 64(%[a])\n\t" - "adcq $0, %%r9\n\t" - "# a[i+9] += m[9] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 72(%[m])\n\t" - "movq 72(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 72(%[a])\n\t" - "adcq $0, %%r8\n\t" - "# a[i+10] += m[10] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 80(%[m])\n\t" - "movq 80(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r11\n\t" - "movq %%r11, 80(%[a])\n\t" - "adcq $0, %%r9\n\t" - "# a[i+11] += m[11] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 88(%[m])\n\t" - "movq 88(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 88(%[a])\n\t" - "adcq $0, %%r8\n\t" - "# a[i+12] += m[12] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 96(%[m])\n\t" - "movq 96(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r11\n\t" - "movq %%r11, 96(%[a])\n\t" - "adcq $0, %%r9\n\t" - "# a[i+13] += m[13] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 104(%[m])\n\t" - "movq 104(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 104(%[a])\n\t" - "adcq $0, %%r8\n\t" - "# a[i+14] += m[14] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 112(%[m])\n\t" - "movq 112(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r11\n\t" - "movq %%r11, 112(%[a])\n\t" - "adcq $0, %%r9\n\t" - "# a[i+15] += m[15] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 120(%[m])\n\t" - "movq 120(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 120(%[a])\n\t" - "adcq $0, %%r8\n\t" - "# a[i+16] += m[16] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 128(%[m])\n\t" - "movq 128(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r11\n\t" - "movq %%r11, 128(%[a])\n\t" - "adcq $0, %%r9\n\t" - "# a[i+17] += m[17] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 136(%[m])\n\t" - "movq 136(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 136(%[a])\n\t" - "adcq $0, %%r8\n\t" - "# a[i+18] += m[18] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 144(%[m])\n\t" - "movq 144(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r11\n\t" - "movq %%r11, 144(%[a])\n\t" - "adcq $0, %%r9\n\t" - "# a[i+19] += m[19] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 152(%[m])\n\t" - "movq 152(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 152(%[a])\n\t" - "adcq $0, %%r8\n\t" - "# a[i+20] += m[20] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 160(%[m])\n\t" - "movq 160(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r11\n\t" - "movq %%r11, 160(%[a])\n\t" - "adcq $0, %%r9\n\t" - "# a[i+21] += m[21] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 168(%[m])\n\t" - "movq 168(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 168(%[a])\n\t" - "adcq $0, %%r8\n\t" - "# a[i+22] += m[22] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 176(%[m])\n\t" - "movq 176(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r11\n\t" - "movq %%r11, 176(%[a])\n\t" - "adcq $0, %%r9\n\t" - "# a[i+23] += m[23] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 184(%[m])\n\t" - "movq 184(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 184(%[a])\n\t" - "adcq $0, %%r8\n\t" - "# a[i+24] += m[24] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 192(%[m])\n\t" - "movq 192(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r11\n\t" - "movq %%r11, 192(%[a])\n\t" - "adcq $0, %%r9\n\t" - "# a[i+25] += m[25] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 200(%[m])\n\t" - "movq 200(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 200(%[a])\n\t" - "adcq $0, %%r8\n\t" - "# a[i+26] += m[26] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 208(%[m])\n\t" - "movq 208(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r11\n\t" - "movq %%r11, 208(%[a])\n\t" - "adcq $0, %%r9\n\t" - "# a[i+27] += m[27] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 216(%[m])\n\t" - "movq 216(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 216(%[a])\n\t" - "adcq $0, %%r8\n\t" - "# a[i+28] += m[28] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 224(%[m])\n\t" - "movq 224(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r11\n\t" - "movq %%r11, 224(%[a])\n\t" - "adcq $0, %%r9\n\t" - "# a[i+29] += m[29] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 232(%[m])\n\t" - "movq 232(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 232(%[a])\n\t" - "adcq $0, %%r8\n\t" - "# a[i+30] += m[30] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 240(%[m])\n\t" - "movq 240(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r11\n\t" - "movq %%r11, 240(%[a])\n\t" - "adcq $0, %%r9\n\t" - "# a[i+31] += m[31] * mu\n\t" - "movq %%r10, %%rax\n\t" - "mulq 248(%[m])\n\t" - "movq 248(%[a]), %%r11\n\t" - "addq %%rax, %%r9\n\t" - "adcq %[ca], %%rdx\n\t" - "movq $0, %[ca]\n\t" - "adcq $0, %[ca]\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 248(%[a])\n\t" - "adcq %%rdx, 256(%[a])\n\t" - "adcq $0, %[ca]\n\t" - "# i += 1\n\t" - "addq $8, %[a]\n\t" - "addq $8, %%rcx\n\t" - "cmpq $256, %%rcx\n\t" - "jl L_mont_loop_32\n\t" - "movq %%r12, 0(%[a])\n\t" - "movq %%r13, 8(%[a])\n\t" - : [ca] "+r" (ca), [a] "+r" (a) - : [m] "r" (m), [mp] "r" (mp) - : "memory", "rax", "rdx", "rcx", "r8", "r9", "r10", "r11", - "r12", "r13" - ); - - sp_2048_cond_sub_32(a - 32, a, m, (sp_digit)0 - ca); -} - +extern sp_digit sp_2048_cond_sub_32(sp_digit* r, sp_digit* a, sp_digit* b, sp_digit m); +extern void sp_2048_mont_reduce_32(sp_digit* a, sp_digit* m, sp_digit mp); /* Multiply two Montogmery form numbers mod the modulus (prime). * (r = a * b mod m) * @@ -8068,487 +867,8 @@ static void sp_2048_mont_sqr_32(sp_digit* r, sp_digit* a, sp_digit* m, sp_2048_mont_reduce_32(r, m, mp); } -/* Mul a by digit b into r. (r = a * b) - * - * r A single precision integer. - * a A single precision integer. - * b A single precision digit. - */ -SP_NOINLINE static void sp_2048_mul_d_32(sp_digit* r, const sp_digit* a, - const sp_digit b) -{ - __asm__ __volatile__ ( - "# A[0] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq (%[a])\n\t" - "movq %%rax, %%rbx\n\t" - "movq %%rdx, %%rcx\n\t" - "movq %%rbx, 0(%[r])\n\t" - "# A[1] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rbx, %%rbx\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "movq %%rcx, 8(%[r])\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[2] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rcx, %%rcx\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "movq %%r8, 16(%[r])\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[3] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "movq %%rbx, 24(%[r])\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[4] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rbx, %%rbx\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "movq %%rcx, 32(%[r])\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[5] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rcx, %%rcx\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "movq %%r8, 40(%[r])\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[6] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "movq %%rbx, 48(%[r])\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[7] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rbx, %%rbx\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "movq %%rcx, 56(%[r])\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[8] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rcx, %%rcx\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "movq %%r8, 64(%[r])\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[9] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "movq %%rbx, 72(%[r])\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[10] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rbx, %%rbx\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "movq %%rcx, 80(%[r])\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[11] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rcx, %%rcx\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "movq %%r8, 88(%[r])\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[12] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 96(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "movq %%rbx, 96(%[r])\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[13] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rbx, %%rbx\n\t" - "mulq 104(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "movq %%rcx, 104(%[r])\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[14] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rcx, %%rcx\n\t" - "mulq 112(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "movq %%r8, 112(%[r])\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[15] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 120(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "movq %%rbx, 120(%[r])\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[16] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rbx, %%rbx\n\t" - "mulq 128(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "movq %%rcx, 128(%[r])\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[17] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rcx, %%rcx\n\t" - "mulq 136(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "movq %%r8, 136(%[r])\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[18] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 144(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "movq %%rbx, 144(%[r])\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[19] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rbx, %%rbx\n\t" - "mulq 152(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "movq %%rcx, 152(%[r])\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[20] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rcx, %%rcx\n\t" - "mulq 160(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "movq %%r8, 160(%[r])\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[21] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 168(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "movq %%rbx, 168(%[r])\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[22] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rbx, %%rbx\n\t" - "mulq 176(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "movq %%rcx, 176(%[r])\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[23] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rcx, %%rcx\n\t" - "mulq 184(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "movq %%r8, 184(%[r])\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[24] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 192(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "movq %%rbx, 192(%[r])\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[25] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rbx, %%rbx\n\t" - "mulq 200(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "movq %%rcx, 200(%[r])\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[26] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rcx, %%rcx\n\t" - "mulq 208(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "movq %%r8, 208(%[r])\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[27] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 216(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "movq %%rbx, 216(%[r])\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[28] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rbx, %%rbx\n\t" - "mulq 224(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "movq %%rcx, 224(%[r])\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[29] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rcx, %%rcx\n\t" - "mulq 232(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "movq %%r8, 232(%[r])\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[30] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 240(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "movq %%rbx, 240(%[r])\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[31] * B\n\t" - "movq %[b], %%rax\n\t" - "mulq 248(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "movq %%rcx, 248(%[r])\n\t" - "movq %%r8, 256(%[r])\n\t" - : - : [r] "r" (r), [a] "r" (a), [b] "r" (b) - : "memory", "rax", "rdx", "rbx", "rcx", "r8" - ); -} - -#ifdef HAVE_INTEL_AVX2 -/* Mul a by digit b into r. (r = a * b) - * - * r A single precision integer. - * a A single precision integer. - * b A single precision digit. - */ -SP_NOINLINE static void sp_2048_mul_d_avx2_32(sp_digit* r, const sp_digit* a, - const sp_digit b) -{ - __asm__ __volatile__ ( - "# A[0] * B\n\t" - "movq %[b], %%rdx\n\t" - "xorq %%r10, %%r10\n\t" - "mulxq (%[a]), %%r8, %%r9\n\t" - "movq %%r8, 0(%[r])\n\t" - "# A[1] * B\n\t" - "mulxq 8(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 8(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[2] * B\n\t" - "mulxq 16(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 16(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[3] * B\n\t" - "mulxq 24(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 24(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[4] * B\n\t" - "mulxq 32(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 32(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[5] * B\n\t" - "mulxq 40(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 40(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[6] * B\n\t" - "mulxq 48(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 48(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[7] * B\n\t" - "mulxq 56(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 56(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[8] * B\n\t" - "mulxq 64(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 64(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[9] * B\n\t" - "mulxq 72(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 72(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[10] * B\n\t" - "mulxq 80(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 80(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[11] * B\n\t" - "mulxq 88(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 88(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[12] * B\n\t" - "mulxq 96(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 96(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[13] * B\n\t" - "mulxq 104(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 104(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[14] * B\n\t" - "mulxq 112(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 112(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[15] * B\n\t" - "mulxq 120(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 120(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[16] * B\n\t" - "mulxq 128(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 128(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[17] * B\n\t" - "mulxq 136(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 136(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[18] * B\n\t" - "mulxq 144(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 144(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[19] * B\n\t" - "mulxq 152(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 152(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[20] * B\n\t" - "mulxq 160(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 160(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[21] * B\n\t" - "mulxq 168(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 168(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[22] * B\n\t" - "mulxq 176(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 176(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[23] * B\n\t" - "mulxq 184(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 184(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[24] * B\n\t" - "mulxq 192(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 192(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[25] * B\n\t" - "mulxq 200(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 200(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[26] * B\n\t" - "mulxq 208(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 208(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[27] * B\n\t" - "mulxq 216(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 216(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[28] * B\n\t" - "mulxq 224(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 224(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[29] * B\n\t" - "mulxq 232(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 232(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[30] * B\n\t" - "mulxq 240(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 240(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[31] * B\n\t" - "mulxq 248(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "adoxq %%rcx, %%r8\n\t" - "adcxq %%r10, %%r8\n\t" - "movq %%r9, 248(%[r])\n\t" - "movq %%r8, 256(%[r])\n\t" - : - : [r] "r" (r), [a] "r" (a), [b] "r" (b) - : "memory", "rax", "rdx", "rcx", "r8", "r9", "r10" - ); -} -#endif /* HAVE_INTEL_AVX2 */ - +extern void sp_2048_mul_d_32(sp_digit* r, const sp_digit* a, const sp_digit b); +extern void sp_2048_mul_d_avx2_32(sp_digit* r, const sp_digit* a, const sp_digit b); /* Divide the double width number (d1|d0) by the dividend. (d1|d0 / div) * * d1 The high order half of the number to divide. @@ -8556,23 +876,18 @@ SP_NOINLINE static void sp_2048_mul_d_avx2_32(sp_digit* r, const sp_digit* a, * div The dividend. * returns the result of the division. */ -static sp_digit div_2048_word_32(sp_digit d1, sp_digit d0, sp_digit div) +static WC_INLINE sp_digit div_2048_word_32(sp_digit d1, sp_digit d0, + sp_digit div) { - sp_digit r; - + register sp_digit r asm("rax"); __asm__ __volatile__ ( - "movq %[d0], %%rax\n\t" - "movq %[d1], %%rdx\n\t" - "divq %[div]\n\t" - "movq %%rax, %[r]\n\t" - : [r] "=r" (r) - : [d1] "r" (d1), [d0] "r" (d0), [div] "r" (div) - : "rax", "rdx" + "divq %3" + : "=a" (r) + : "d" (d1), "a" (d0), "r" (div) + : ); - return r; } - /* AND m into each word of a and store in r. * * r A single precision integer. @@ -8602,286 +917,7 @@ static void sp_2048_mask_32(sp_digit* r, sp_digit* a, sp_digit m) #endif } -/* Compare a with b in constant time. - * - * a A single precision integer. - * b A single precision integer. - * return -ve, 0 or +ve if a is less than, equal to or greater than b - * respectively. - */ -static int64_t sp_2048_cmp_32(sp_digit* a, sp_digit* b) -{ - sp_digit r = -1; - sp_digit one = 1; - - __asm__ __volatile__ ( - "xorq %%rcx, %%rcx\n\t" - "movq $-1, %%rdx\n\t" - "movq 248(%[a]), %%rbx\n\t" - "movq 248(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 240(%[a]), %%rbx\n\t" - "movq 240(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 232(%[a]), %%rbx\n\t" - "movq 232(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 224(%[a]), %%rbx\n\t" - "movq 224(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 216(%[a]), %%rbx\n\t" - "movq 216(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 208(%[a]), %%rbx\n\t" - "movq 208(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 200(%[a]), %%rbx\n\t" - "movq 200(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 192(%[a]), %%rbx\n\t" - "movq 192(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 184(%[a]), %%rbx\n\t" - "movq 184(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 176(%[a]), %%rbx\n\t" - "movq 176(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 168(%[a]), %%rbx\n\t" - "movq 168(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 160(%[a]), %%rbx\n\t" - "movq 160(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 152(%[a]), %%rbx\n\t" - "movq 152(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 144(%[a]), %%rbx\n\t" - "movq 144(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 136(%[a]), %%rbx\n\t" - "movq 136(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 128(%[a]), %%rbx\n\t" - "movq 128(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 120(%[a]), %%rbx\n\t" - "movq 120(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 112(%[a]), %%rbx\n\t" - "movq 112(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 104(%[a]), %%rbx\n\t" - "movq 104(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 96(%[a]), %%rbx\n\t" - "movq 96(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 88(%[a]), %%rbx\n\t" - "movq 88(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 80(%[a]), %%rbx\n\t" - "movq 80(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 72(%[a]), %%rbx\n\t" - "movq 72(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 64(%[a]), %%rbx\n\t" - "movq 64(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 56(%[a]), %%rbx\n\t" - "movq 56(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 48(%[a]), %%rbx\n\t" - "movq 48(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 40(%[a]), %%rbx\n\t" - "movq 40(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 32(%[a]), %%rbx\n\t" - "movq 32(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 24(%[a]), %%rbx\n\t" - "movq 24(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 16(%[a]), %%rbx\n\t" - "movq 16(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 8(%[a]), %%rbx\n\t" - "movq 8(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 0(%[a]), %%rbx\n\t" - "movq 0(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "xorq %%rdx, %[r]\n\t" - : [r] "+r" (r) - : [a] "r" (a), [b] "r" (b), [one] "r" (one) - : "rax", "rdx", "rcx", "rbx", "r8" - ); - - return r; -} - +extern int64_t sp_2048_cmp_32(sp_digit* a, sp_digit* b); /* Divide d in a and put remainder into r (m*d + r = a) * m is not calculated as it is not needed at this time. * @@ -9145,238 +1181,7 @@ static int sp_2048_mod_exp_32(sp_digit* r, sp_digit* a, sp_digit* e, } #endif /* SP_RSA_PRIVATE_EXP_D || WOLFSSL_HAVE_SP_DH */ -#ifdef HAVE_INTEL_AVX2 -/* Reduce the number back to 2048 bits using Montgomery reduction. - * - * a A single precision number to reduce in place. - * m The single precision number representing the modulus. - * mp The digit representing the negative inverse of m mod 2^n. - */ -SP_NOINLINE static void sp_2048_mont_reduce_avx2_32(sp_digit* a, sp_digit* m, - sp_digit mp) -{ - sp_digit ca = 0; - - __asm__ __volatile__ ( - "# i = 0\n\t" - "movq 0(%[a]), %%r12\n\t" - "xorq %%rcx, %%rcx\n\t" - "\nL_mont_loop_avx2_32:\n\t" - "xorq %%r9, %%r9\n\t" - "movq %%r12, %%r10\n\t" - "# mu = a[i] * mp\n\t" - "movq %%r12, %%rdx\n\t" - "mulxq %[mp], %%rdx, %%r8\n\t" - "# a[i+0] += m[0] * mu\n\t" - "mulxq 0(%[m]), %%rax, %%r8\n\t" - "movq 8(%[a]), %%r12\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r12\n\t" - "# a[i+1] += m[1] * mu\n\t" - "mulxq 8(%[m]), %%rax, %%r8\n\t" - "movq 16(%[a]), %%r10\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%r8, %%r10\n\t" - "# a[i+2] += m[2] * mu\n\t" - "mulxq 16(%[m]), %%rax, %%r8\n\t" - "movq 24(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 16(%[a])\n\t" - "# a[i+3] += m[3] * mu\n\t" - "mulxq 24(%[m]), %%rax, %%r8\n\t" - "movq 32(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 24(%[a])\n\t" - "# a[i+4] += m[4] * mu\n\t" - "mulxq 32(%[m]), %%rax, %%r8\n\t" - "movq 40(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 32(%[a])\n\t" - "# a[i+5] += m[5] * mu\n\t" - "mulxq 40(%[m]), %%rax, %%r8\n\t" - "movq 48(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 40(%[a])\n\t" - "# a[i+6] += m[6] * mu\n\t" - "mulxq 48(%[m]), %%rax, %%r8\n\t" - "movq 56(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 48(%[a])\n\t" - "# a[i+7] += m[7] * mu\n\t" - "mulxq 56(%[m]), %%rax, %%r8\n\t" - "movq 64(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 56(%[a])\n\t" - "# a[i+8] += m[8] * mu\n\t" - "mulxq 64(%[m]), %%rax, %%r8\n\t" - "movq 72(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 64(%[a])\n\t" - "# a[i+9] += m[9] * mu\n\t" - "mulxq 72(%[m]), %%rax, %%r8\n\t" - "movq 80(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 72(%[a])\n\t" - "# a[i+10] += m[10] * mu\n\t" - "mulxq 80(%[m]), %%rax, %%r8\n\t" - "movq 88(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 80(%[a])\n\t" - "# a[i+11] += m[11] * mu\n\t" - "mulxq 88(%[m]), %%rax, %%r8\n\t" - "movq 96(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 88(%[a])\n\t" - "# a[i+12] += m[12] * mu\n\t" - "mulxq 96(%[m]), %%rax, %%r8\n\t" - "movq 104(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 96(%[a])\n\t" - "# a[i+13] += m[13] * mu\n\t" - "mulxq 104(%[m]), %%rax, %%r8\n\t" - "movq 112(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 104(%[a])\n\t" - "# a[i+14] += m[14] * mu\n\t" - "mulxq 112(%[m]), %%rax, %%r8\n\t" - "movq 120(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 112(%[a])\n\t" - "# a[i+15] += m[15] * mu\n\t" - "mulxq 120(%[m]), %%rax, %%r8\n\t" - "movq 128(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 120(%[a])\n\t" - "# a[i+16] += m[16] * mu\n\t" - "mulxq 128(%[m]), %%rax, %%r8\n\t" - "movq 136(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 128(%[a])\n\t" - "# a[i+17] += m[17] * mu\n\t" - "mulxq 136(%[m]), %%rax, %%r8\n\t" - "movq 144(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 136(%[a])\n\t" - "# a[i+18] += m[18] * mu\n\t" - "mulxq 144(%[m]), %%rax, %%r8\n\t" - "movq 152(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 144(%[a])\n\t" - "# a[i+19] += m[19] * mu\n\t" - "mulxq 152(%[m]), %%rax, %%r8\n\t" - "movq 160(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 152(%[a])\n\t" - "# a[i+20] += m[20] * mu\n\t" - "mulxq 160(%[m]), %%rax, %%r8\n\t" - "movq 168(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 160(%[a])\n\t" - "# a[i+21] += m[21] * mu\n\t" - "mulxq 168(%[m]), %%rax, %%r8\n\t" - "movq 176(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 168(%[a])\n\t" - "# a[i+22] += m[22] * mu\n\t" - "mulxq 176(%[m]), %%rax, %%r8\n\t" - "movq 184(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 176(%[a])\n\t" - "# a[i+23] += m[23] * mu\n\t" - "mulxq 184(%[m]), %%rax, %%r8\n\t" - "movq 192(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 184(%[a])\n\t" - "# a[i+24] += m[24] * mu\n\t" - "mulxq 192(%[m]), %%rax, %%r8\n\t" - "movq 200(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 192(%[a])\n\t" - "# a[i+25] += m[25] * mu\n\t" - "mulxq 200(%[m]), %%rax, %%r8\n\t" - "movq 208(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 200(%[a])\n\t" - "# a[i+26] += m[26] * mu\n\t" - "mulxq 208(%[m]), %%rax, %%r8\n\t" - "movq 216(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 208(%[a])\n\t" - "# a[i+27] += m[27] * mu\n\t" - "mulxq 216(%[m]), %%rax, %%r8\n\t" - "movq 224(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 216(%[a])\n\t" - "# a[i+28] += m[28] * mu\n\t" - "mulxq 224(%[m]), %%rax, %%r8\n\t" - "movq 232(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 224(%[a])\n\t" - "# a[i+29] += m[29] * mu\n\t" - "mulxq 232(%[m]), %%rax, %%r8\n\t" - "movq 240(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 232(%[a])\n\t" - "# a[i+30] += m[30] * mu\n\t" - "mulxq 240(%[m]), %%rax, %%r8\n\t" - "movq 248(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 240(%[a])\n\t" - "# a[i+31] += m[31] * mu\n\t" - "mulxq 248(%[m]), %%rax, %%r8\n\t" - "movq 256(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 248(%[a])\n\t" - "adcxq %[ca], %%r10\n\t" - "movq %%r9, %[ca]\n\t" - "adoxq %%r9, %[ca]\n\t" - "adcxq %%r9, %[ca]\n\t" - "movq %%r10, 256(%[a])\n\t" - "# i += 1\n\t" - "addq $8, %[a]\n\t" - "addq $1, %%rcx\n\t" - "cmpq $32, %%rcx\n\t" - "jl L_mont_loop_avx2_32\n\t" - "movq %%r12, 0(%[a])\n\t" - : [ca] "+r" (ca), [a] "+r" (a) - : [m] "r" (m), [mp] "r" (mp) - : "memory", "rax", "rdx", "rcx", "r8", "r9", "r10", "r11", "r12" - ); - - sp_2048_cond_sub_32(a - 32, a, m, (sp_digit)0 - ca); -} -#endif /* HAVE_INTEL_AVX2 */ - +extern void sp_2048_mont_reduce_avx2_32(sp_digit* a, sp_digit* m, sp_digit mp); #ifdef HAVE_INTEL_AVX2 /* Multiply two Montogmery form numbers mod the modulus (prime). * (r = a * b mod m) @@ -10027,7 +1832,7 @@ int sp_DhExp_2048(mp_int* base, const byte* exp, word32 expLen, } #endif /* WOLFSSL_HAVE_SP_DH */ -#endif /* WOLFSSL_SP_NO_2048 */ +#endif /* !WOLFSSL_SP_NO_2048 */ #ifndef WOLFSSL_SP_NO_3072 /* Read big endian unsigned byte aray into r. @@ -10158,11901 +1963,13 @@ static void sp_3072_to_bin(sp_digit* r, byte* a) } } -/* Multiply a and b into r. (r = a * b) - * - * r A single precision integer. - * a A single precision integer. - * b A single precision integer. - */ -SP_NOINLINE static void sp_3072_mul_24(sp_digit* r, const sp_digit* a, - const sp_digit* b) -{ - sp_digit tmp[24]; - - __asm__ __volatile__ ( - "# A[0] * B[0]\n\t" - "movq 0(%[b]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "movq %%rax, (%[tmp])\n\t" - "movq %%rdx, %%rcx\n\t" - "# A[0] * B[1]\n\t" - "movq 8(%[b]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%rbx, %%rbx\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[1] * B[0]\n\t" - "movq 0(%[b]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "movq %%rcx, 8(%[tmp])\n\t" - "# A[0] * B[2]\n\t" - "movq 16(%[b]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%rcx, %%rcx\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[1] * B[1]\n\t" - "movq 8(%[b]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[2] * B[0]\n\t" - "movq 0(%[b]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "movq %%r8, 16(%[tmp])\n\t" - "# A[0] * B[3]\n\t" - "movq 24(%[b]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[1] * B[2]\n\t" - "movq 16(%[b]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[2] * B[1]\n\t" - "movq 8(%[b]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[3] * B[0]\n\t" - "movq 0(%[b]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "movq %%rbx, 24(%[tmp])\n\t" - "# A[0] * B[4]\n\t" - "movq 32(%[b]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%rbx, %%rbx\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[1] * B[3]\n\t" - "movq 24(%[b]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[2] * B[2]\n\t" - "movq 16(%[b]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[3] * B[1]\n\t" - "movq 8(%[b]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[4] * B[0]\n\t" - "movq 0(%[b]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "movq %%rcx, 32(%[tmp])\n\t" - "# A[0] * B[5]\n\t" - "movq 40(%[b]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%rcx, %%rcx\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[1] * B[4]\n\t" - "movq 32(%[b]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[2] * B[3]\n\t" - "movq 24(%[b]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[3] * B[2]\n\t" - "movq 16(%[b]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[4] * B[1]\n\t" - "movq 8(%[b]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[5] * B[0]\n\t" - "movq 0(%[b]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "movq %%r8, 40(%[tmp])\n\t" - "# A[0] * B[6]\n\t" - "movq 48(%[b]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[1] * B[5]\n\t" - "movq 40(%[b]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[2] * B[4]\n\t" - "movq 32(%[b]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[3] * B[3]\n\t" - "movq 24(%[b]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[4] * B[2]\n\t" - "movq 16(%[b]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[5] * B[1]\n\t" - "movq 8(%[b]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[6] * B[0]\n\t" - "movq 0(%[b]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "movq %%rbx, 48(%[tmp])\n\t" - "# A[0] * B[7]\n\t" - "movq 56(%[b]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%rbx, %%rbx\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[1] * B[6]\n\t" - "movq 48(%[b]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[2] * B[5]\n\t" - "movq 40(%[b]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[3] * B[4]\n\t" - "movq 32(%[b]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[4] * B[3]\n\t" - "movq 24(%[b]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[5] * B[2]\n\t" - "movq 16(%[b]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[6] * B[1]\n\t" - "movq 8(%[b]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[7] * B[0]\n\t" - "movq 0(%[b]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "movq %%rcx, 56(%[tmp])\n\t" - "# A[0] * B[8]\n\t" - "movq 64(%[b]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%rcx, %%rcx\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[1] * B[7]\n\t" - "movq 56(%[b]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[2] * B[6]\n\t" - "movq 48(%[b]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[3] * B[5]\n\t" - "movq 40(%[b]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[4] * B[4]\n\t" - "movq 32(%[b]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[5] * B[3]\n\t" - "movq 24(%[b]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[6] * B[2]\n\t" - "movq 16(%[b]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[7] * B[1]\n\t" - "movq 8(%[b]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[8] * B[0]\n\t" - "movq 0(%[b]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "movq %%r8, 64(%[tmp])\n\t" - "# A[0] * B[9]\n\t" - "movq 72(%[b]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[1] * B[8]\n\t" - "movq 64(%[b]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[2] * B[7]\n\t" - "movq 56(%[b]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[3] * B[6]\n\t" - "movq 48(%[b]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[4] * B[5]\n\t" - "movq 40(%[b]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[5] * B[4]\n\t" - "movq 32(%[b]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[6] * B[3]\n\t" - "movq 24(%[b]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[7] * B[2]\n\t" - "movq 16(%[b]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[8] * B[1]\n\t" - "movq 8(%[b]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[9] * B[0]\n\t" - "movq 0(%[b]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "movq %%rbx, 72(%[tmp])\n\t" - "# A[0] * B[10]\n\t" - "movq 80(%[b]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%rbx, %%rbx\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[1] * B[9]\n\t" - "movq 72(%[b]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[2] * B[8]\n\t" - "movq 64(%[b]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[3] * B[7]\n\t" - "movq 56(%[b]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[4] * B[6]\n\t" - "movq 48(%[b]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[5] * B[5]\n\t" - "movq 40(%[b]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[6] * B[4]\n\t" - "movq 32(%[b]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[7] * B[3]\n\t" - "movq 24(%[b]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[8] * B[2]\n\t" - "movq 16(%[b]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[9] * B[1]\n\t" - "movq 8(%[b]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[10] * B[0]\n\t" - "movq 0(%[b]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "movq %%rcx, 80(%[tmp])\n\t" - "# A[0] * B[11]\n\t" - "movq 88(%[b]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%rcx, %%rcx\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[1] * B[10]\n\t" - "movq 80(%[b]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[2] * B[9]\n\t" - "movq 72(%[b]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[3] * B[8]\n\t" - "movq 64(%[b]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[4] * B[7]\n\t" - "movq 56(%[b]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[5] * B[6]\n\t" - "movq 48(%[b]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[6] * B[5]\n\t" - "movq 40(%[b]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[7] * B[4]\n\t" - "movq 32(%[b]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[8] * B[3]\n\t" - "movq 24(%[b]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[9] * B[2]\n\t" - "movq 16(%[b]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[10] * B[1]\n\t" - "movq 8(%[b]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[11] * B[0]\n\t" - "movq 0(%[b]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "movq %%r8, 88(%[tmp])\n\t" - "# A[0] * B[12]\n\t" - "movq 96(%[b]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[1] * B[11]\n\t" - "movq 88(%[b]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[2] * B[10]\n\t" - "movq 80(%[b]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[3] * B[9]\n\t" - "movq 72(%[b]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[4] * B[8]\n\t" - "movq 64(%[b]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[5] * B[7]\n\t" - "movq 56(%[b]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[6] * B[6]\n\t" - "movq 48(%[b]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[7] * B[5]\n\t" - "movq 40(%[b]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[8] * B[4]\n\t" - "movq 32(%[b]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[9] * B[3]\n\t" - "movq 24(%[b]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[10] * B[2]\n\t" - "movq 16(%[b]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[11] * B[1]\n\t" - "movq 8(%[b]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[12] * B[0]\n\t" - "movq 0(%[b]), %%rax\n\t" - "mulq 96(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "movq %%rbx, 96(%[tmp])\n\t" - "# A[0] * B[13]\n\t" - "movq 104(%[b]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%rbx, %%rbx\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[1] * B[12]\n\t" - "movq 96(%[b]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[2] * B[11]\n\t" - "movq 88(%[b]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[3] * B[10]\n\t" - "movq 80(%[b]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[4] * B[9]\n\t" - "movq 72(%[b]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[5] * B[8]\n\t" - "movq 64(%[b]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[6] * B[7]\n\t" - "movq 56(%[b]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[7] * B[6]\n\t" - "movq 48(%[b]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[8] * B[5]\n\t" - "movq 40(%[b]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[9] * B[4]\n\t" - "movq 32(%[b]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[10] * B[3]\n\t" - "movq 24(%[b]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[11] * B[2]\n\t" - "movq 16(%[b]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[12] * B[1]\n\t" - "movq 8(%[b]), %%rax\n\t" - "mulq 96(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[13] * B[0]\n\t" - "movq 0(%[b]), %%rax\n\t" - "mulq 104(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "movq %%rcx, 104(%[tmp])\n\t" - "# A[0] * B[14]\n\t" - "movq 112(%[b]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%rcx, %%rcx\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[1] * B[13]\n\t" - "movq 104(%[b]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[2] * B[12]\n\t" - "movq 96(%[b]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[3] * B[11]\n\t" - "movq 88(%[b]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[4] * B[10]\n\t" - "movq 80(%[b]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[5] * B[9]\n\t" - "movq 72(%[b]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[6] * B[8]\n\t" - "movq 64(%[b]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[7] * B[7]\n\t" - "movq 56(%[b]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[8] * B[6]\n\t" - "movq 48(%[b]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[9] * B[5]\n\t" - "movq 40(%[b]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[10] * B[4]\n\t" - "movq 32(%[b]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[11] * B[3]\n\t" - "movq 24(%[b]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[12] * B[2]\n\t" - "movq 16(%[b]), %%rax\n\t" - "mulq 96(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[13] * B[1]\n\t" - "movq 8(%[b]), %%rax\n\t" - "mulq 104(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[14] * B[0]\n\t" - "movq 0(%[b]), %%rax\n\t" - "mulq 112(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "movq %%r8, 112(%[tmp])\n\t" - "# A[0] * B[15]\n\t" - "movq 120(%[b]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[1] * B[14]\n\t" - "movq 112(%[b]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[2] * B[13]\n\t" - "movq 104(%[b]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[3] * B[12]\n\t" - "movq 96(%[b]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[4] * B[11]\n\t" - "movq 88(%[b]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[5] * B[10]\n\t" - "movq 80(%[b]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[6] * B[9]\n\t" - "movq 72(%[b]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[7] * B[8]\n\t" - "movq 64(%[b]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[8] * B[7]\n\t" - "movq 56(%[b]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[9] * B[6]\n\t" - "movq 48(%[b]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[10] * B[5]\n\t" - "movq 40(%[b]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[11] * B[4]\n\t" - "movq 32(%[b]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[12] * B[3]\n\t" - "movq 24(%[b]), %%rax\n\t" - "mulq 96(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[13] * B[2]\n\t" - "movq 16(%[b]), %%rax\n\t" - "mulq 104(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[14] * B[1]\n\t" - "movq 8(%[b]), %%rax\n\t" - "mulq 112(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[15] * B[0]\n\t" - "movq 0(%[b]), %%rax\n\t" - "mulq 120(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "movq %%rbx, 120(%[tmp])\n\t" - "# A[0] * B[16]\n\t" - "movq 128(%[b]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%rbx, %%rbx\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[1] * B[15]\n\t" - "movq 120(%[b]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[2] * B[14]\n\t" - "movq 112(%[b]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[3] * B[13]\n\t" - "movq 104(%[b]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[4] * B[12]\n\t" - "movq 96(%[b]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[5] * B[11]\n\t" - "movq 88(%[b]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[6] * B[10]\n\t" - "movq 80(%[b]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[7] * B[9]\n\t" - "movq 72(%[b]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[8] * B[8]\n\t" - "movq 64(%[b]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[9] * B[7]\n\t" - "movq 56(%[b]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[10] * B[6]\n\t" - "movq 48(%[b]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[11] * B[5]\n\t" - "movq 40(%[b]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[12] * B[4]\n\t" - "movq 32(%[b]), %%rax\n\t" - "mulq 96(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[13] * B[3]\n\t" - "movq 24(%[b]), %%rax\n\t" - "mulq 104(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[14] * B[2]\n\t" - "movq 16(%[b]), %%rax\n\t" - "mulq 112(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[15] * B[1]\n\t" - "movq 8(%[b]), %%rax\n\t" - "mulq 120(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[16] * B[0]\n\t" - "movq 0(%[b]), %%rax\n\t" - "mulq 128(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "movq %%rcx, 128(%[tmp])\n\t" - "# A[0] * B[17]\n\t" - "movq 136(%[b]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%rcx, %%rcx\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[1] * B[16]\n\t" - "movq 128(%[b]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[2] * B[15]\n\t" - "movq 120(%[b]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[3] * B[14]\n\t" - "movq 112(%[b]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[4] * B[13]\n\t" - "movq 104(%[b]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[5] * B[12]\n\t" - "movq 96(%[b]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[6] * B[11]\n\t" - "movq 88(%[b]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[7] * B[10]\n\t" - "movq 80(%[b]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[8] * B[9]\n\t" - "movq 72(%[b]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[9] * B[8]\n\t" - "movq 64(%[b]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[10] * B[7]\n\t" - "movq 56(%[b]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[11] * B[6]\n\t" - "movq 48(%[b]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[12] * B[5]\n\t" - "movq 40(%[b]), %%rax\n\t" - "mulq 96(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[13] * B[4]\n\t" - "movq 32(%[b]), %%rax\n\t" - "mulq 104(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[14] * B[3]\n\t" - "movq 24(%[b]), %%rax\n\t" - "mulq 112(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[15] * B[2]\n\t" - "movq 16(%[b]), %%rax\n\t" - "mulq 120(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[16] * B[1]\n\t" - "movq 8(%[b]), %%rax\n\t" - "mulq 128(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[17] * B[0]\n\t" - "movq 0(%[b]), %%rax\n\t" - "mulq 136(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "movq %%r8, 136(%[tmp])\n\t" - "# A[0] * B[18]\n\t" - "movq 144(%[b]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[1] * B[17]\n\t" - "movq 136(%[b]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[2] * B[16]\n\t" - "movq 128(%[b]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[3] * B[15]\n\t" - "movq 120(%[b]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[4] * B[14]\n\t" - "movq 112(%[b]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[5] * B[13]\n\t" - "movq 104(%[b]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[6] * B[12]\n\t" - "movq 96(%[b]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[7] * B[11]\n\t" - "movq 88(%[b]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[8] * B[10]\n\t" - "movq 80(%[b]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[9] * B[9]\n\t" - "movq 72(%[b]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[10] * B[8]\n\t" - "movq 64(%[b]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[11] * B[7]\n\t" - "movq 56(%[b]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[12] * B[6]\n\t" - "movq 48(%[b]), %%rax\n\t" - "mulq 96(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[13] * B[5]\n\t" - "movq 40(%[b]), %%rax\n\t" - "mulq 104(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[14] * B[4]\n\t" - "movq 32(%[b]), %%rax\n\t" - "mulq 112(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[15] * B[3]\n\t" - "movq 24(%[b]), %%rax\n\t" - "mulq 120(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[16] * B[2]\n\t" - "movq 16(%[b]), %%rax\n\t" - "mulq 128(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[17] * B[1]\n\t" - "movq 8(%[b]), %%rax\n\t" - "mulq 136(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[18] * B[0]\n\t" - "movq 0(%[b]), %%rax\n\t" - "mulq 144(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "movq %%rbx, 144(%[tmp])\n\t" - "# A[0] * B[19]\n\t" - "movq 152(%[b]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%rbx, %%rbx\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[1] * B[18]\n\t" - "movq 144(%[b]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[2] * B[17]\n\t" - "movq 136(%[b]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[3] * B[16]\n\t" - "movq 128(%[b]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[4] * B[15]\n\t" - "movq 120(%[b]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[5] * B[14]\n\t" - "movq 112(%[b]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[6] * B[13]\n\t" - "movq 104(%[b]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[7] * B[12]\n\t" - "movq 96(%[b]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[8] * B[11]\n\t" - "movq 88(%[b]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[9] * B[10]\n\t" - "movq 80(%[b]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[10] * B[9]\n\t" - "movq 72(%[b]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[11] * B[8]\n\t" - "movq 64(%[b]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[12] * B[7]\n\t" - "movq 56(%[b]), %%rax\n\t" - "mulq 96(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[13] * B[6]\n\t" - "movq 48(%[b]), %%rax\n\t" - "mulq 104(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[14] * B[5]\n\t" - "movq 40(%[b]), %%rax\n\t" - "mulq 112(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[15] * B[4]\n\t" - "movq 32(%[b]), %%rax\n\t" - "mulq 120(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[16] * B[3]\n\t" - "movq 24(%[b]), %%rax\n\t" - "mulq 128(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[17] * B[2]\n\t" - "movq 16(%[b]), %%rax\n\t" - "mulq 136(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[18] * B[1]\n\t" - "movq 8(%[b]), %%rax\n\t" - "mulq 144(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[19] * B[0]\n\t" - "movq 0(%[b]), %%rax\n\t" - "mulq 152(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "movq %%rcx, 152(%[tmp])\n\t" - "# A[0] * B[20]\n\t" - "movq 160(%[b]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%rcx, %%rcx\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[1] * B[19]\n\t" - "movq 152(%[b]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[2] * B[18]\n\t" - "movq 144(%[b]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[3] * B[17]\n\t" - "movq 136(%[b]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[4] * B[16]\n\t" - "movq 128(%[b]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[5] * B[15]\n\t" - "movq 120(%[b]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[6] * B[14]\n\t" - "movq 112(%[b]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[7] * B[13]\n\t" - "movq 104(%[b]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[8] * B[12]\n\t" - "movq 96(%[b]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[9] * B[11]\n\t" - "movq 88(%[b]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[10] * B[10]\n\t" - "movq 80(%[b]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[11] * B[9]\n\t" - "movq 72(%[b]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[12] * B[8]\n\t" - "movq 64(%[b]), %%rax\n\t" - "mulq 96(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[13] * B[7]\n\t" - "movq 56(%[b]), %%rax\n\t" - "mulq 104(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[14] * B[6]\n\t" - "movq 48(%[b]), %%rax\n\t" - "mulq 112(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[15] * B[5]\n\t" - "movq 40(%[b]), %%rax\n\t" - "mulq 120(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[16] * B[4]\n\t" - "movq 32(%[b]), %%rax\n\t" - "mulq 128(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[17] * B[3]\n\t" - "movq 24(%[b]), %%rax\n\t" - "mulq 136(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[18] * B[2]\n\t" - "movq 16(%[b]), %%rax\n\t" - "mulq 144(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[19] * B[1]\n\t" - "movq 8(%[b]), %%rax\n\t" - "mulq 152(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[20] * B[0]\n\t" - "movq 0(%[b]), %%rax\n\t" - "mulq 160(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "movq %%r8, 160(%[tmp])\n\t" - "# A[0] * B[21]\n\t" - "movq 168(%[b]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[1] * B[20]\n\t" - "movq 160(%[b]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[2] * B[19]\n\t" - "movq 152(%[b]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[3] * B[18]\n\t" - "movq 144(%[b]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[4] * B[17]\n\t" - "movq 136(%[b]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[5] * B[16]\n\t" - "movq 128(%[b]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[6] * B[15]\n\t" - "movq 120(%[b]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[7] * B[14]\n\t" - "movq 112(%[b]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[8] * B[13]\n\t" - "movq 104(%[b]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[9] * B[12]\n\t" - "movq 96(%[b]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[10] * B[11]\n\t" - "movq 88(%[b]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[11] * B[10]\n\t" - "movq 80(%[b]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[12] * B[9]\n\t" - "movq 72(%[b]), %%rax\n\t" - "mulq 96(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[13] * B[8]\n\t" - "movq 64(%[b]), %%rax\n\t" - "mulq 104(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[14] * B[7]\n\t" - "movq 56(%[b]), %%rax\n\t" - "mulq 112(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[15] * B[6]\n\t" - "movq 48(%[b]), %%rax\n\t" - "mulq 120(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[16] * B[5]\n\t" - "movq 40(%[b]), %%rax\n\t" - "mulq 128(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[17] * B[4]\n\t" - "movq 32(%[b]), %%rax\n\t" - "mulq 136(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[18] * B[3]\n\t" - "movq 24(%[b]), %%rax\n\t" - "mulq 144(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[19] * B[2]\n\t" - "movq 16(%[b]), %%rax\n\t" - "mulq 152(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[20] * B[1]\n\t" - "movq 8(%[b]), %%rax\n\t" - "mulq 160(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[21] * B[0]\n\t" - "movq 0(%[b]), %%rax\n\t" - "mulq 168(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "movq %%rbx, 168(%[tmp])\n\t" - "# A[0] * B[22]\n\t" - "movq 176(%[b]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%rbx, %%rbx\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[1] * B[21]\n\t" - "movq 168(%[b]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[2] * B[20]\n\t" - "movq 160(%[b]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[3] * B[19]\n\t" - "movq 152(%[b]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[4] * B[18]\n\t" - "movq 144(%[b]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[5] * B[17]\n\t" - "movq 136(%[b]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[6] * B[16]\n\t" - "movq 128(%[b]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[7] * B[15]\n\t" - "movq 120(%[b]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[8] * B[14]\n\t" - "movq 112(%[b]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[9] * B[13]\n\t" - "movq 104(%[b]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[10] * B[12]\n\t" - "movq 96(%[b]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[11] * B[11]\n\t" - "movq 88(%[b]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[12] * B[10]\n\t" - "movq 80(%[b]), %%rax\n\t" - "mulq 96(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[13] * B[9]\n\t" - "movq 72(%[b]), %%rax\n\t" - "mulq 104(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[14] * B[8]\n\t" - "movq 64(%[b]), %%rax\n\t" - "mulq 112(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[15] * B[7]\n\t" - "movq 56(%[b]), %%rax\n\t" - "mulq 120(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[16] * B[6]\n\t" - "movq 48(%[b]), %%rax\n\t" - "mulq 128(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[17] * B[5]\n\t" - "movq 40(%[b]), %%rax\n\t" - "mulq 136(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[18] * B[4]\n\t" - "movq 32(%[b]), %%rax\n\t" - "mulq 144(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[19] * B[3]\n\t" - "movq 24(%[b]), %%rax\n\t" - "mulq 152(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[20] * B[2]\n\t" - "movq 16(%[b]), %%rax\n\t" - "mulq 160(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[21] * B[1]\n\t" - "movq 8(%[b]), %%rax\n\t" - "mulq 168(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[22] * B[0]\n\t" - "movq 0(%[b]), %%rax\n\t" - "mulq 176(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "movq %%rcx, 176(%[tmp])\n\t" - "# A[0] * B[23]\n\t" - "movq 184(%[b]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%rcx, %%rcx\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[1] * B[22]\n\t" - "movq 176(%[b]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[2] * B[21]\n\t" - "movq 168(%[b]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[3] * B[20]\n\t" - "movq 160(%[b]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[4] * B[19]\n\t" - "movq 152(%[b]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[5] * B[18]\n\t" - "movq 144(%[b]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[6] * B[17]\n\t" - "movq 136(%[b]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[7] * B[16]\n\t" - "movq 128(%[b]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[8] * B[15]\n\t" - "movq 120(%[b]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[9] * B[14]\n\t" - "movq 112(%[b]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[10] * B[13]\n\t" - "movq 104(%[b]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[11] * B[12]\n\t" - "movq 96(%[b]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[12] * B[11]\n\t" - "movq 88(%[b]), %%rax\n\t" - "mulq 96(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[13] * B[10]\n\t" - "movq 80(%[b]), %%rax\n\t" - "mulq 104(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[14] * B[9]\n\t" - "movq 72(%[b]), %%rax\n\t" - "mulq 112(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[15] * B[8]\n\t" - "movq 64(%[b]), %%rax\n\t" - "mulq 120(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[16] * B[7]\n\t" - "movq 56(%[b]), %%rax\n\t" - "mulq 128(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[17] * B[6]\n\t" - "movq 48(%[b]), %%rax\n\t" - "mulq 136(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[18] * B[5]\n\t" - "movq 40(%[b]), %%rax\n\t" - "mulq 144(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[19] * B[4]\n\t" - "movq 32(%[b]), %%rax\n\t" - "mulq 152(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[20] * B[3]\n\t" - "movq 24(%[b]), %%rax\n\t" - "mulq 160(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[21] * B[2]\n\t" - "movq 16(%[b]), %%rax\n\t" - "mulq 168(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[22] * B[1]\n\t" - "movq 8(%[b]), %%rax\n\t" - "mulq 176(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[23] * B[0]\n\t" - "movq 0(%[b]), %%rax\n\t" - "mulq 184(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "movq %%r8, 184(%[tmp])\n\t" - "# A[1] * B[23]\n\t" - "movq 184(%[b]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[2] * B[22]\n\t" - "movq 176(%[b]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[3] * B[21]\n\t" - "movq 168(%[b]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[4] * B[20]\n\t" - "movq 160(%[b]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[5] * B[19]\n\t" - "movq 152(%[b]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[6] * B[18]\n\t" - "movq 144(%[b]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[7] * B[17]\n\t" - "movq 136(%[b]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[8] * B[16]\n\t" - "movq 128(%[b]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[9] * B[15]\n\t" - "movq 120(%[b]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[10] * B[14]\n\t" - "movq 112(%[b]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[11] * B[13]\n\t" - "movq 104(%[b]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[12] * B[12]\n\t" - "movq 96(%[b]), %%rax\n\t" - "mulq 96(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[13] * B[11]\n\t" - "movq 88(%[b]), %%rax\n\t" - "mulq 104(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[14] * B[10]\n\t" - "movq 80(%[b]), %%rax\n\t" - "mulq 112(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[15] * B[9]\n\t" - "movq 72(%[b]), %%rax\n\t" - "mulq 120(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[16] * B[8]\n\t" - "movq 64(%[b]), %%rax\n\t" - "mulq 128(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[17] * B[7]\n\t" - "movq 56(%[b]), %%rax\n\t" - "mulq 136(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[18] * B[6]\n\t" - "movq 48(%[b]), %%rax\n\t" - "mulq 144(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[19] * B[5]\n\t" - "movq 40(%[b]), %%rax\n\t" - "mulq 152(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[20] * B[4]\n\t" - "movq 32(%[b]), %%rax\n\t" - "mulq 160(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[21] * B[3]\n\t" - "movq 24(%[b]), %%rax\n\t" - "mulq 168(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[22] * B[2]\n\t" - "movq 16(%[b]), %%rax\n\t" - "mulq 176(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[23] * B[1]\n\t" - "movq 8(%[b]), %%rax\n\t" - "mulq 184(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "movq %%rbx, 192(%[r])\n\t" - "# A[2] * B[23]\n\t" - "movq 184(%[b]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "xorq %%rbx, %%rbx\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[3] * B[22]\n\t" - "movq 176(%[b]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[4] * B[21]\n\t" - "movq 168(%[b]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[5] * B[20]\n\t" - "movq 160(%[b]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[6] * B[19]\n\t" - "movq 152(%[b]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[7] * B[18]\n\t" - "movq 144(%[b]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[8] * B[17]\n\t" - "movq 136(%[b]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[9] * B[16]\n\t" - "movq 128(%[b]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[10] * B[15]\n\t" - "movq 120(%[b]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[11] * B[14]\n\t" - "movq 112(%[b]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[12] * B[13]\n\t" - "movq 104(%[b]), %%rax\n\t" - "mulq 96(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[13] * B[12]\n\t" - "movq 96(%[b]), %%rax\n\t" - "mulq 104(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[14] * B[11]\n\t" - "movq 88(%[b]), %%rax\n\t" - "mulq 112(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[15] * B[10]\n\t" - "movq 80(%[b]), %%rax\n\t" - "mulq 120(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[16] * B[9]\n\t" - "movq 72(%[b]), %%rax\n\t" - "mulq 128(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[17] * B[8]\n\t" - "movq 64(%[b]), %%rax\n\t" - "mulq 136(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[18] * B[7]\n\t" - "movq 56(%[b]), %%rax\n\t" - "mulq 144(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[19] * B[6]\n\t" - "movq 48(%[b]), %%rax\n\t" - "mulq 152(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[20] * B[5]\n\t" - "movq 40(%[b]), %%rax\n\t" - "mulq 160(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[21] * B[4]\n\t" - "movq 32(%[b]), %%rax\n\t" - "mulq 168(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[22] * B[3]\n\t" - "movq 24(%[b]), %%rax\n\t" - "mulq 176(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[23] * B[2]\n\t" - "movq 16(%[b]), %%rax\n\t" - "mulq 184(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "movq %%rcx, 200(%[r])\n\t" - "# A[3] * B[23]\n\t" - "movq 184(%[b]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "xorq %%rcx, %%rcx\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[4] * B[22]\n\t" - "movq 176(%[b]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[5] * B[21]\n\t" - "movq 168(%[b]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[6] * B[20]\n\t" - "movq 160(%[b]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[7] * B[19]\n\t" - "movq 152(%[b]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[8] * B[18]\n\t" - "movq 144(%[b]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[9] * B[17]\n\t" - "movq 136(%[b]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[10] * B[16]\n\t" - "movq 128(%[b]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[11] * B[15]\n\t" - "movq 120(%[b]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[12] * B[14]\n\t" - "movq 112(%[b]), %%rax\n\t" - "mulq 96(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[13] * B[13]\n\t" - "movq 104(%[b]), %%rax\n\t" - "mulq 104(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[14] * B[12]\n\t" - "movq 96(%[b]), %%rax\n\t" - "mulq 112(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[15] * B[11]\n\t" - "movq 88(%[b]), %%rax\n\t" - "mulq 120(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[16] * B[10]\n\t" - "movq 80(%[b]), %%rax\n\t" - "mulq 128(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[17] * B[9]\n\t" - "movq 72(%[b]), %%rax\n\t" - "mulq 136(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[18] * B[8]\n\t" - "movq 64(%[b]), %%rax\n\t" - "mulq 144(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[19] * B[7]\n\t" - "movq 56(%[b]), %%rax\n\t" - "mulq 152(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[20] * B[6]\n\t" - "movq 48(%[b]), %%rax\n\t" - "mulq 160(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[21] * B[5]\n\t" - "movq 40(%[b]), %%rax\n\t" - "mulq 168(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[22] * B[4]\n\t" - "movq 32(%[b]), %%rax\n\t" - "mulq 176(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[23] * B[3]\n\t" - "movq 24(%[b]), %%rax\n\t" - "mulq 184(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "movq %%r8, 208(%[r])\n\t" - "# A[4] * B[23]\n\t" - "movq 184(%[b]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[5] * B[22]\n\t" - "movq 176(%[b]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[6] * B[21]\n\t" - "movq 168(%[b]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[7] * B[20]\n\t" - "movq 160(%[b]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[8] * B[19]\n\t" - "movq 152(%[b]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[9] * B[18]\n\t" - "movq 144(%[b]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[10] * B[17]\n\t" - "movq 136(%[b]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[11] * B[16]\n\t" - "movq 128(%[b]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[12] * B[15]\n\t" - "movq 120(%[b]), %%rax\n\t" - "mulq 96(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[13] * B[14]\n\t" - "movq 112(%[b]), %%rax\n\t" - "mulq 104(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[14] * B[13]\n\t" - "movq 104(%[b]), %%rax\n\t" - "mulq 112(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[15] * B[12]\n\t" - "movq 96(%[b]), %%rax\n\t" - "mulq 120(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[16] * B[11]\n\t" - "movq 88(%[b]), %%rax\n\t" - "mulq 128(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[17] * B[10]\n\t" - "movq 80(%[b]), %%rax\n\t" - "mulq 136(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[18] * B[9]\n\t" - "movq 72(%[b]), %%rax\n\t" - "mulq 144(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[19] * B[8]\n\t" - "movq 64(%[b]), %%rax\n\t" - "mulq 152(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[20] * B[7]\n\t" - "movq 56(%[b]), %%rax\n\t" - "mulq 160(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[21] * B[6]\n\t" - "movq 48(%[b]), %%rax\n\t" - "mulq 168(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[22] * B[5]\n\t" - "movq 40(%[b]), %%rax\n\t" - "mulq 176(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[23] * B[4]\n\t" - "movq 32(%[b]), %%rax\n\t" - "mulq 184(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "movq %%rbx, 216(%[r])\n\t" - "# A[5] * B[23]\n\t" - "movq 184(%[b]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "xorq %%rbx, %%rbx\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[6] * B[22]\n\t" - "movq 176(%[b]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[7] * B[21]\n\t" - "movq 168(%[b]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[8] * B[20]\n\t" - "movq 160(%[b]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[9] * B[19]\n\t" - "movq 152(%[b]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[10] * B[18]\n\t" - "movq 144(%[b]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[11] * B[17]\n\t" - "movq 136(%[b]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[12] * B[16]\n\t" - "movq 128(%[b]), %%rax\n\t" - "mulq 96(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[13] * B[15]\n\t" - "movq 120(%[b]), %%rax\n\t" - "mulq 104(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[14] * B[14]\n\t" - "movq 112(%[b]), %%rax\n\t" - "mulq 112(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[15] * B[13]\n\t" - "movq 104(%[b]), %%rax\n\t" - "mulq 120(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[16] * B[12]\n\t" - "movq 96(%[b]), %%rax\n\t" - "mulq 128(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[17] * B[11]\n\t" - "movq 88(%[b]), %%rax\n\t" - "mulq 136(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[18] * B[10]\n\t" - "movq 80(%[b]), %%rax\n\t" - "mulq 144(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[19] * B[9]\n\t" - "movq 72(%[b]), %%rax\n\t" - "mulq 152(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[20] * B[8]\n\t" - "movq 64(%[b]), %%rax\n\t" - "mulq 160(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[21] * B[7]\n\t" - "movq 56(%[b]), %%rax\n\t" - "mulq 168(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[22] * B[6]\n\t" - "movq 48(%[b]), %%rax\n\t" - "mulq 176(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[23] * B[5]\n\t" - "movq 40(%[b]), %%rax\n\t" - "mulq 184(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "movq %%rcx, 224(%[r])\n\t" - "# A[6] * B[23]\n\t" - "movq 184(%[b]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "xorq %%rcx, %%rcx\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[7] * B[22]\n\t" - "movq 176(%[b]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[8] * B[21]\n\t" - "movq 168(%[b]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[9] * B[20]\n\t" - "movq 160(%[b]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[10] * B[19]\n\t" - "movq 152(%[b]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[11] * B[18]\n\t" - "movq 144(%[b]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[12] * B[17]\n\t" - "movq 136(%[b]), %%rax\n\t" - "mulq 96(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[13] * B[16]\n\t" - "movq 128(%[b]), %%rax\n\t" - "mulq 104(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[14] * B[15]\n\t" - "movq 120(%[b]), %%rax\n\t" - "mulq 112(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[15] * B[14]\n\t" - "movq 112(%[b]), %%rax\n\t" - "mulq 120(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[16] * B[13]\n\t" - "movq 104(%[b]), %%rax\n\t" - "mulq 128(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[17] * B[12]\n\t" - "movq 96(%[b]), %%rax\n\t" - "mulq 136(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[18] * B[11]\n\t" - "movq 88(%[b]), %%rax\n\t" - "mulq 144(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[19] * B[10]\n\t" - "movq 80(%[b]), %%rax\n\t" - "mulq 152(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[20] * B[9]\n\t" - "movq 72(%[b]), %%rax\n\t" - "mulq 160(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[21] * B[8]\n\t" - "movq 64(%[b]), %%rax\n\t" - "mulq 168(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[22] * B[7]\n\t" - "movq 56(%[b]), %%rax\n\t" - "mulq 176(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[23] * B[6]\n\t" - "movq 48(%[b]), %%rax\n\t" - "mulq 184(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "movq %%r8, 232(%[r])\n\t" - "# A[7] * B[23]\n\t" - "movq 184(%[b]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[8] * B[22]\n\t" - "movq 176(%[b]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[9] * B[21]\n\t" - "movq 168(%[b]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[10] * B[20]\n\t" - "movq 160(%[b]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[11] * B[19]\n\t" - "movq 152(%[b]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[12] * B[18]\n\t" - "movq 144(%[b]), %%rax\n\t" - "mulq 96(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[13] * B[17]\n\t" - "movq 136(%[b]), %%rax\n\t" - "mulq 104(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[14] * B[16]\n\t" - "movq 128(%[b]), %%rax\n\t" - "mulq 112(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[15] * B[15]\n\t" - "movq 120(%[b]), %%rax\n\t" - "mulq 120(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[16] * B[14]\n\t" - "movq 112(%[b]), %%rax\n\t" - "mulq 128(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[17] * B[13]\n\t" - "movq 104(%[b]), %%rax\n\t" - "mulq 136(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[18] * B[12]\n\t" - "movq 96(%[b]), %%rax\n\t" - "mulq 144(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[19] * B[11]\n\t" - "movq 88(%[b]), %%rax\n\t" - "mulq 152(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[20] * B[10]\n\t" - "movq 80(%[b]), %%rax\n\t" - "mulq 160(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[21] * B[9]\n\t" - "movq 72(%[b]), %%rax\n\t" - "mulq 168(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[22] * B[8]\n\t" - "movq 64(%[b]), %%rax\n\t" - "mulq 176(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[23] * B[7]\n\t" - "movq 56(%[b]), %%rax\n\t" - "mulq 184(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "movq %%rbx, 240(%[r])\n\t" - "# A[8] * B[23]\n\t" - "movq 184(%[b]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "xorq %%rbx, %%rbx\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[9] * B[22]\n\t" - "movq 176(%[b]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[10] * B[21]\n\t" - "movq 168(%[b]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[11] * B[20]\n\t" - "movq 160(%[b]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[12] * B[19]\n\t" - "movq 152(%[b]), %%rax\n\t" - "mulq 96(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[13] * B[18]\n\t" - "movq 144(%[b]), %%rax\n\t" - "mulq 104(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[14] * B[17]\n\t" - "movq 136(%[b]), %%rax\n\t" - "mulq 112(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[15] * B[16]\n\t" - "movq 128(%[b]), %%rax\n\t" - "mulq 120(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[16] * B[15]\n\t" - "movq 120(%[b]), %%rax\n\t" - "mulq 128(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[17] * B[14]\n\t" - "movq 112(%[b]), %%rax\n\t" - "mulq 136(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[18] * B[13]\n\t" - "movq 104(%[b]), %%rax\n\t" - "mulq 144(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[19] * B[12]\n\t" - "movq 96(%[b]), %%rax\n\t" - "mulq 152(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[20] * B[11]\n\t" - "movq 88(%[b]), %%rax\n\t" - "mulq 160(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[21] * B[10]\n\t" - "movq 80(%[b]), %%rax\n\t" - "mulq 168(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[22] * B[9]\n\t" - "movq 72(%[b]), %%rax\n\t" - "mulq 176(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[23] * B[8]\n\t" - "movq 64(%[b]), %%rax\n\t" - "mulq 184(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "movq %%rcx, 248(%[r])\n\t" - "# A[9] * B[23]\n\t" - "movq 184(%[b]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "xorq %%rcx, %%rcx\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[10] * B[22]\n\t" - "movq 176(%[b]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[11] * B[21]\n\t" - "movq 168(%[b]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[12] * B[20]\n\t" - "movq 160(%[b]), %%rax\n\t" - "mulq 96(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[13] * B[19]\n\t" - "movq 152(%[b]), %%rax\n\t" - "mulq 104(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[14] * B[18]\n\t" - "movq 144(%[b]), %%rax\n\t" - "mulq 112(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[15] * B[17]\n\t" - "movq 136(%[b]), %%rax\n\t" - "mulq 120(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[16] * B[16]\n\t" - "movq 128(%[b]), %%rax\n\t" - "mulq 128(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[17] * B[15]\n\t" - "movq 120(%[b]), %%rax\n\t" - "mulq 136(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[18] * B[14]\n\t" - "movq 112(%[b]), %%rax\n\t" - "mulq 144(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[19] * B[13]\n\t" - "movq 104(%[b]), %%rax\n\t" - "mulq 152(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[20] * B[12]\n\t" - "movq 96(%[b]), %%rax\n\t" - "mulq 160(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[21] * B[11]\n\t" - "movq 88(%[b]), %%rax\n\t" - "mulq 168(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[22] * B[10]\n\t" - "movq 80(%[b]), %%rax\n\t" - "mulq 176(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[23] * B[9]\n\t" - "movq 72(%[b]), %%rax\n\t" - "mulq 184(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "movq %%r8, 256(%[r])\n\t" - "# A[10] * B[23]\n\t" - "movq 184(%[b]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[11] * B[22]\n\t" - "movq 176(%[b]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[12] * B[21]\n\t" - "movq 168(%[b]), %%rax\n\t" - "mulq 96(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[13] * B[20]\n\t" - "movq 160(%[b]), %%rax\n\t" - "mulq 104(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[14] * B[19]\n\t" - "movq 152(%[b]), %%rax\n\t" - "mulq 112(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[15] * B[18]\n\t" - "movq 144(%[b]), %%rax\n\t" - "mulq 120(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[16] * B[17]\n\t" - "movq 136(%[b]), %%rax\n\t" - "mulq 128(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[17] * B[16]\n\t" - "movq 128(%[b]), %%rax\n\t" - "mulq 136(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[18] * B[15]\n\t" - "movq 120(%[b]), %%rax\n\t" - "mulq 144(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[19] * B[14]\n\t" - "movq 112(%[b]), %%rax\n\t" - "mulq 152(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[20] * B[13]\n\t" - "movq 104(%[b]), %%rax\n\t" - "mulq 160(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[21] * B[12]\n\t" - "movq 96(%[b]), %%rax\n\t" - "mulq 168(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[22] * B[11]\n\t" - "movq 88(%[b]), %%rax\n\t" - "mulq 176(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[23] * B[10]\n\t" - "movq 80(%[b]), %%rax\n\t" - "mulq 184(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "movq %%rbx, 264(%[r])\n\t" - "# A[11] * B[23]\n\t" - "movq 184(%[b]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "xorq %%rbx, %%rbx\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[12] * B[22]\n\t" - "movq 176(%[b]), %%rax\n\t" - "mulq 96(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[13] * B[21]\n\t" - "movq 168(%[b]), %%rax\n\t" - "mulq 104(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[14] * B[20]\n\t" - "movq 160(%[b]), %%rax\n\t" - "mulq 112(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[15] * B[19]\n\t" - "movq 152(%[b]), %%rax\n\t" - "mulq 120(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[16] * B[18]\n\t" - "movq 144(%[b]), %%rax\n\t" - "mulq 128(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[17] * B[17]\n\t" - "movq 136(%[b]), %%rax\n\t" - "mulq 136(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[18] * B[16]\n\t" - "movq 128(%[b]), %%rax\n\t" - "mulq 144(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[19] * B[15]\n\t" - "movq 120(%[b]), %%rax\n\t" - "mulq 152(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[20] * B[14]\n\t" - "movq 112(%[b]), %%rax\n\t" - "mulq 160(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[21] * B[13]\n\t" - "movq 104(%[b]), %%rax\n\t" - "mulq 168(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[22] * B[12]\n\t" - "movq 96(%[b]), %%rax\n\t" - "mulq 176(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[23] * B[11]\n\t" - "movq 88(%[b]), %%rax\n\t" - "mulq 184(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "movq %%rcx, 272(%[r])\n\t" - "# A[12] * B[23]\n\t" - "movq 184(%[b]), %%rax\n\t" - "mulq 96(%[a])\n\t" - "xorq %%rcx, %%rcx\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[13] * B[22]\n\t" - "movq 176(%[b]), %%rax\n\t" - "mulq 104(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[14] * B[21]\n\t" - "movq 168(%[b]), %%rax\n\t" - "mulq 112(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[15] * B[20]\n\t" - "movq 160(%[b]), %%rax\n\t" - "mulq 120(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[16] * B[19]\n\t" - "movq 152(%[b]), %%rax\n\t" - "mulq 128(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[17] * B[18]\n\t" - "movq 144(%[b]), %%rax\n\t" - "mulq 136(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[18] * B[17]\n\t" - "movq 136(%[b]), %%rax\n\t" - "mulq 144(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[19] * B[16]\n\t" - "movq 128(%[b]), %%rax\n\t" - "mulq 152(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[20] * B[15]\n\t" - "movq 120(%[b]), %%rax\n\t" - "mulq 160(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[21] * B[14]\n\t" - "movq 112(%[b]), %%rax\n\t" - "mulq 168(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[22] * B[13]\n\t" - "movq 104(%[b]), %%rax\n\t" - "mulq 176(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[23] * B[12]\n\t" - "movq 96(%[b]), %%rax\n\t" - "mulq 184(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "movq %%r8, 280(%[r])\n\t" - "# A[13] * B[23]\n\t" - "movq 184(%[b]), %%rax\n\t" - "mulq 104(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[14] * B[22]\n\t" - "movq 176(%[b]), %%rax\n\t" - "mulq 112(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[15] * B[21]\n\t" - "movq 168(%[b]), %%rax\n\t" - "mulq 120(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[16] * B[20]\n\t" - "movq 160(%[b]), %%rax\n\t" - "mulq 128(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[17] * B[19]\n\t" - "movq 152(%[b]), %%rax\n\t" - "mulq 136(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[18] * B[18]\n\t" - "movq 144(%[b]), %%rax\n\t" - "mulq 144(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[19] * B[17]\n\t" - "movq 136(%[b]), %%rax\n\t" - "mulq 152(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[20] * B[16]\n\t" - "movq 128(%[b]), %%rax\n\t" - "mulq 160(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[21] * B[15]\n\t" - "movq 120(%[b]), %%rax\n\t" - "mulq 168(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[22] * B[14]\n\t" - "movq 112(%[b]), %%rax\n\t" - "mulq 176(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[23] * B[13]\n\t" - "movq 104(%[b]), %%rax\n\t" - "mulq 184(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "movq %%rbx, 288(%[r])\n\t" - "# A[14] * B[23]\n\t" - "movq 184(%[b]), %%rax\n\t" - "mulq 112(%[a])\n\t" - "xorq %%rbx, %%rbx\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[15] * B[22]\n\t" - "movq 176(%[b]), %%rax\n\t" - "mulq 120(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[16] * B[21]\n\t" - "movq 168(%[b]), %%rax\n\t" - "mulq 128(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[17] * B[20]\n\t" - "movq 160(%[b]), %%rax\n\t" - "mulq 136(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[18] * B[19]\n\t" - "movq 152(%[b]), %%rax\n\t" - "mulq 144(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[19] * B[18]\n\t" - "movq 144(%[b]), %%rax\n\t" - "mulq 152(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[20] * B[17]\n\t" - "movq 136(%[b]), %%rax\n\t" - "mulq 160(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[21] * B[16]\n\t" - "movq 128(%[b]), %%rax\n\t" - "mulq 168(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[22] * B[15]\n\t" - "movq 120(%[b]), %%rax\n\t" - "mulq 176(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[23] * B[14]\n\t" - "movq 112(%[b]), %%rax\n\t" - "mulq 184(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "movq %%rcx, 296(%[r])\n\t" - "# A[15] * B[23]\n\t" - "movq 184(%[b]), %%rax\n\t" - "mulq 120(%[a])\n\t" - "xorq %%rcx, %%rcx\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[16] * B[22]\n\t" - "movq 176(%[b]), %%rax\n\t" - "mulq 128(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[17] * B[21]\n\t" - "movq 168(%[b]), %%rax\n\t" - "mulq 136(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[18] * B[20]\n\t" - "movq 160(%[b]), %%rax\n\t" - "mulq 144(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[19] * B[19]\n\t" - "movq 152(%[b]), %%rax\n\t" - "mulq 152(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[20] * B[18]\n\t" - "movq 144(%[b]), %%rax\n\t" - "mulq 160(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[21] * B[17]\n\t" - "movq 136(%[b]), %%rax\n\t" - "mulq 168(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[22] * B[16]\n\t" - "movq 128(%[b]), %%rax\n\t" - "mulq 176(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[23] * B[15]\n\t" - "movq 120(%[b]), %%rax\n\t" - "mulq 184(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "movq %%r8, 304(%[r])\n\t" - "# A[16] * B[23]\n\t" - "movq 184(%[b]), %%rax\n\t" - "mulq 128(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[17] * B[22]\n\t" - "movq 176(%[b]), %%rax\n\t" - "mulq 136(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[18] * B[21]\n\t" - "movq 168(%[b]), %%rax\n\t" - "mulq 144(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[19] * B[20]\n\t" - "movq 160(%[b]), %%rax\n\t" - "mulq 152(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[20] * B[19]\n\t" - "movq 152(%[b]), %%rax\n\t" - "mulq 160(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[21] * B[18]\n\t" - "movq 144(%[b]), %%rax\n\t" - "mulq 168(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[22] * B[17]\n\t" - "movq 136(%[b]), %%rax\n\t" - "mulq 176(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[23] * B[16]\n\t" - "movq 128(%[b]), %%rax\n\t" - "mulq 184(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "movq %%rbx, 312(%[r])\n\t" - "# A[17] * B[23]\n\t" - "movq 184(%[b]), %%rax\n\t" - "mulq 136(%[a])\n\t" - "xorq %%rbx, %%rbx\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[18] * B[22]\n\t" - "movq 176(%[b]), %%rax\n\t" - "mulq 144(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[19] * B[21]\n\t" - "movq 168(%[b]), %%rax\n\t" - "mulq 152(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[20] * B[20]\n\t" - "movq 160(%[b]), %%rax\n\t" - "mulq 160(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[21] * B[19]\n\t" - "movq 152(%[b]), %%rax\n\t" - "mulq 168(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[22] * B[18]\n\t" - "movq 144(%[b]), %%rax\n\t" - "mulq 176(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[23] * B[17]\n\t" - "movq 136(%[b]), %%rax\n\t" - "mulq 184(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "movq %%rcx, 320(%[r])\n\t" - "# A[18] * B[23]\n\t" - "movq 184(%[b]), %%rax\n\t" - "mulq 144(%[a])\n\t" - "xorq %%rcx, %%rcx\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[19] * B[22]\n\t" - "movq 176(%[b]), %%rax\n\t" - "mulq 152(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[20] * B[21]\n\t" - "movq 168(%[b]), %%rax\n\t" - "mulq 160(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[21] * B[20]\n\t" - "movq 160(%[b]), %%rax\n\t" - "mulq 168(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[22] * B[19]\n\t" - "movq 152(%[b]), %%rax\n\t" - "mulq 176(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[23] * B[18]\n\t" - "movq 144(%[b]), %%rax\n\t" - "mulq 184(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "movq %%r8, 328(%[r])\n\t" - "# A[19] * B[23]\n\t" - "movq 184(%[b]), %%rax\n\t" - "mulq 152(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[20] * B[22]\n\t" - "movq 176(%[b]), %%rax\n\t" - "mulq 160(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[21] * B[21]\n\t" - "movq 168(%[b]), %%rax\n\t" - "mulq 168(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[22] * B[20]\n\t" - "movq 160(%[b]), %%rax\n\t" - "mulq 176(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[23] * B[19]\n\t" - "movq 152(%[b]), %%rax\n\t" - "mulq 184(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "movq %%rbx, 336(%[r])\n\t" - "# A[20] * B[23]\n\t" - "movq 184(%[b]), %%rax\n\t" - "mulq 160(%[a])\n\t" - "xorq %%rbx, %%rbx\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[21] * B[22]\n\t" - "movq 176(%[b]), %%rax\n\t" - "mulq 168(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[22] * B[21]\n\t" - "movq 168(%[b]), %%rax\n\t" - "mulq 176(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[23] * B[20]\n\t" - "movq 160(%[b]), %%rax\n\t" - "mulq 184(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "movq %%rcx, 344(%[r])\n\t" - "# A[21] * B[23]\n\t" - "movq 184(%[b]), %%rax\n\t" - "mulq 168(%[a])\n\t" - "xorq %%rcx, %%rcx\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[22] * B[22]\n\t" - "movq 176(%[b]), %%rax\n\t" - "mulq 176(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[23] * B[21]\n\t" - "movq 168(%[b]), %%rax\n\t" - "mulq 184(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "movq %%r8, 352(%[r])\n\t" - "# A[22] * B[23]\n\t" - "movq 184(%[b]), %%rax\n\t" - "mulq 176(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[23] * B[22]\n\t" - "movq 176(%[b]), %%rax\n\t" - "mulq 184(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "movq %%rbx, 360(%[r])\n\t" - "# A[23] * B[23]\n\t" - "movq 184(%[b]), %%rax\n\t" - "mulq 184(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "movq %%rcx, 368(%[r])\n\t" - "movq %%r8, 376(%[r])\n\t" - : - : [r] "r" (r), [a] "r" (a), [b] "r" (b), [tmp] "r" (tmp) - : "memory", "rax", "rdx", "rbx", "rcx", "r8" - ); - - XMEMCPY(r, tmp, sizeof(tmp)); -} - -/* Square a and put result in r. (r = a * a) - * - * r A single precision integer. - * a A single precision integer. - */ -SP_NOINLINE static void sp_3072_sqr_24(sp_digit* r, const sp_digit* a) -{ - sp_digit tmp[24]; - - __asm__ __volatile__ ( - "# A[0] * A[0]\n\t" - "movq 0(%[a]), %%rax\n\t" - "mulq %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "movq %%rax, (%[tmp])\n\t" - "movq %%rdx, %%r8\n\t" - "# A[0] * A[1]\n\t" - "movq 8(%[a]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%rcx, %%rcx\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%r9\n\t" - "adcq $0, %%rcx\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%r9\n\t" - "adcq $0, %%rcx\n\t" - "movq %%r8, 8(%[tmp])\n\t" - "# A[0] * A[2]\n\t" - "movq 16(%[a]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "addq %%rax, %%r9\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "addq %%rax, %%r9\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[1] * A[1]\n\t" - "movq 8(%[a]), %%rax\n\t" - "mulq %%rax\n\t" - "addq %%rax, %%r9\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "movq %%r9, 16(%[tmp])\n\t" - "# A[0] * A[3]\n\t" - "movq 24(%[a]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%r9, %%r9\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%r9\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%r9\n\t" - "# A[1] * A[2]\n\t" - "movq 16(%[a]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%r9\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%r9\n\t" - "movq %%rcx, 24(%[tmp])\n\t" - "# A[0] * A[4]\n\t" - "movq 32(%[a]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%rcx, %%rcx\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%r9\n\t" - "adcq $0, %%rcx\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%r9\n\t" - "adcq $0, %%rcx\n\t" - "# A[1] * A[3]\n\t" - "movq 24(%[a]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%r9\n\t" - "adcq $0, %%rcx\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%r9\n\t" - "adcq $0, %%rcx\n\t" - "# A[2] * A[2]\n\t" - "movq 16(%[a]), %%rax\n\t" - "mulq %%rax\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%r9\n\t" - "adcq $0, %%rcx\n\t" - "movq %%r8, 32(%[tmp])\n\t" - "# A[0] * A[5]\n\t" - "movq 40(%[a]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[1] * A[4]\n\t" - "movq 32(%[a]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[2] * A[3]\n\t" - "movq 24(%[a]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%r10, %%r9\n\t" - "adcq %%r11, %%rcx\n\t" - "adcq %%r12, %%r8\n\t" - "movq %%r9, 40(%[tmp])\n\t" - "# A[0] * A[6]\n\t" - "movq 48(%[a]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%r9, %%r9\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[1] * A[5]\n\t" - "movq 40(%[a]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[2] * A[4]\n\t" - "movq 32(%[a]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[3] * A[3]\n\t" - "movq 24(%[a]), %%rax\n\t" - "mulq %%rax\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%rcx\n\t" - "adcq %%r11, %%r8\n\t" - "adcq %%r12, %%r9\n\t" - "movq %%rcx, 48(%[tmp])\n\t" - "# A[0] * A[7]\n\t" - "movq 56(%[a]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%rcx, %%rcx\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[1] * A[6]\n\t" - "movq 48(%[a]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[2] * A[5]\n\t" - "movq 40(%[a]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[3] * A[4]\n\t" - "movq 32(%[a]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%r10, %%r8\n\t" - "adcq %%r11, %%r9\n\t" - "adcq %%r12, %%rcx\n\t" - "movq %%r8, 56(%[tmp])\n\t" - "# A[0] * A[8]\n\t" - "movq 64(%[a]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[1] * A[7]\n\t" - "movq 56(%[a]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[2] * A[6]\n\t" - "movq 48(%[a]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[3] * A[5]\n\t" - "movq 40(%[a]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[4] * A[4]\n\t" - "movq 32(%[a]), %%rax\n\t" - "mulq %%rax\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%r9\n\t" - "adcq %%r11, %%rcx\n\t" - "adcq %%r12, %%r8\n\t" - "movq %%r9, 64(%[tmp])\n\t" - "# A[0] * A[9]\n\t" - "movq 72(%[a]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%r9, %%r9\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[1] * A[8]\n\t" - "movq 64(%[a]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[2] * A[7]\n\t" - "movq 56(%[a]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[3] * A[6]\n\t" - "movq 48(%[a]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[4] * A[5]\n\t" - "movq 40(%[a]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%r10, %%rcx\n\t" - "adcq %%r11, %%r8\n\t" - "adcq %%r12, %%r9\n\t" - "movq %%rcx, 72(%[tmp])\n\t" - "# A[0] * A[10]\n\t" - "movq 80(%[a]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%rcx, %%rcx\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[1] * A[9]\n\t" - "movq 72(%[a]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[2] * A[8]\n\t" - "movq 64(%[a]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[3] * A[7]\n\t" - "movq 56(%[a]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[4] * A[6]\n\t" - "movq 48(%[a]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[5] * A[5]\n\t" - "movq 40(%[a]), %%rax\n\t" - "mulq %%rax\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%r8\n\t" - "adcq %%r11, %%r9\n\t" - "adcq %%r12, %%rcx\n\t" - "movq %%r8, 80(%[tmp])\n\t" - "# A[0] * A[11]\n\t" - "movq 88(%[a]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[1] * A[10]\n\t" - "movq 80(%[a]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[2] * A[9]\n\t" - "movq 72(%[a]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[3] * A[8]\n\t" - "movq 64(%[a]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[4] * A[7]\n\t" - "movq 56(%[a]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[5] * A[6]\n\t" - "movq 48(%[a]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%r10, %%r9\n\t" - "adcq %%r11, %%rcx\n\t" - "adcq %%r12, %%r8\n\t" - "movq %%r9, 88(%[tmp])\n\t" - "# A[0] * A[12]\n\t" - "movq 96(%[a]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%r9, %%r9\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[1] * A[11]\n\t" - "movq 88(%[a]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[2] * A[10]\n\t" - "movq 80(%[a]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[3] * A[9]\n\t" - "movq 72(%[a]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[4] * A[8]\n\t" - "movq 64(%[a]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[5] * A[7]\n\t" - "movq 56(%[a]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[6] * A[6]\n\t" - "movq 48(%[a]), %%rax\n\t" - "mulq %%rax\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%rcx\n\t" - "adcq %%r11, %%r8\n\t" - "adcq %%r12, %%r9\n\t" - "movq %%rcx, 96(%[tmp])\n\t" - "# A[0] * A[13]\n\t" - "movq 104(%[a]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%rcx, %%rcx\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[1] * A[12]\n\t" - "movq 96(%[a]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[2] * A[11]\n\t" - "movq 88(%[a]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[3] * A[10]\n\t" - "movq 80(%[a]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[4] * A[9]\n\t" - "movq 72(%[a]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[5] * A[8]\n\t" - "movq 64(%[a]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[6] * A[7]\n\t" - "movq 56(%[a]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%r10, %%r8\n\t" - "adcq %%r11, %%r9\n\t" - "adcq %%r12, %%rcx\n\t" - "movq %%r8, 104(%[tmp])\n\t" - "# A[0] * A[14]\n\t" - "movq 112(%[a]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[1] * A[13]\n\t" - "movq 104(%[a]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[2] * A[12]\n\t" - "movq 96(%[a]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[3] * A[11]\n\t" - "movq 88(%[a]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[4] * A[10]\n\t" - "movq 80(%[a]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[5] * A[9]\n\t" - "movq 72(%[a]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[6] * A[8]\n\t" - "movq 64(%[a]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[7] * A[7]\n\t" - "movq 56(%[a]), %%rax\n\t" - "mulq %%rax\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%r9\n\t" - "adcq %%r11, %%rcx\n\t" - "adcq %%r12, %%r8\n\t" - "movq %%r9, 112(%[tmp])\n\t" - "# A[0] * A[15]\n\t" - "movq 120(%[a]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%r9, %%r9\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[1] * A[14]\n\t" - "movq 112(%[a]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[2] * A[13]\n\t" - "movq 104(%[a]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[3] * A[12]\n\t" - "movq 96(%[a]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[4] * A[11]\n\t" - "movq 88(%[a]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[5] * A[10]\n\t" - "movq 80(%[a]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[6] * A[9]\n\t" - "movq 72(%[a]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[7] * A[8]\n\t" - "movq 64(%[a]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%r10, %%rcx\n\t" - "adcq %%r11, %%r8\n\t" - "adcq %%r12, %%r9\n\t" - "movq %%rcx, 120(%[tmp])\n\t" - "# A[0] * A[16]\n\t" - "movq 128(%[a]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%rcx, %%rcx\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[1] * A[15]\n\t" - "movq 120(%[a]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[2] * A[14]\n\t" - "movq 112(%[a]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[3] * A[13]\n\t" - "movq 104(%[a]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[4] * A[12]\n\t" - "movq 96(%[a]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[5] * A[11]\n\t" - "movq 88(%[a]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[6] * A[10]\n\t" - "movq 80(%[a]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[7] * A[9]\n\t" - "movq 72(%[a]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[8] * A[8]\n\t" - "movq 64(%[a]), %%rax\n\t" - "mulq %%rax\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%r8\n\t" - "adcq %%r11, %%r9\n\t" - "adcq %%r12, %%rcx\n\t" - "movq %%r8, 128(%[tmp])\n\t" - "# A[0] * A[17]\n\t" - "movq 136(%[a]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[1] * A[16]\n\t" - "movq 128(%[a]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[2] * A[15]\n\t" - "movq 120(%[a]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[3] * A[14]\n\t" - "movq 112(%[a]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[4] * A[13]\n\t" - "movq 104(%[a]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[5] * A[12]\n\t" - "movq 96(%[a]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[6] * A[11]\n\t" - "movq 88(%[a]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[7] * A[10]\n\t" - "movq 80(%[a]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[8] * A[9]\n\t" - "movq 72(%[a]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%r10, %%r9\n\t" - "adcq %%r11, %%rcx\n\t" - "adcq %%r12, %%r8\n\t" - "movq %%r9, 136(%[tmp])\n\t" - "# A[0] * A[18]\n\t" - "movq 144(%[a]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%r9, %%r9\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[1] * A[17]\n\t" - "movq 136(%[a]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[2] * A[16]\n\t" - "movq 128(%[a]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[3] * A[15]\n\t" - "movq 120(%[a]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[4] * A[14]\n\t" - "movq 112(%[a]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[5] * A[13]\n\t" - "movq 104(%[a]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[6] * A[12]\n\t" - "movq 96(%[a]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[7] * A[11]\n\t" - "movq 88(%[a]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[8] * A[10]\n\t" - "movq 80(%[a]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[9] * A[9]\n\t" - "movq 72(%[a]), %%rax\n\t" - "mulq %%rax\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%rcx\n\t" - "adcq %%r11, %%r8\n\t" - "adcq %%r12, %%r9\n\t" - "movq %%rcx, 144(%[tmp])\n\t" - "# A[0] * A[19]\n\t" - "movq 152(%[a]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%rcx, %%rcx\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[1] * A[18]\n\t" - "movq 144(%[a]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[2] * A[17]\n\t" - "movq 136(%[a]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[3] * A[16]\n\t" - "movq 128(%[a]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[4] * A[15]\n\t" - "movq 120(%[a]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[5] * A[14]\n\t" - "movq 112(%[a]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[6] * A[13]\n\t" - "movq 104(%[a]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[7] * A[12]\n\t" - "movq 96(%[a]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[8] * A[11]\n\t" - "movq 88(%[a]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[9] * A[10]\n\t" - "movq 80(%[a]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%r10, %%r8\n\t" - "adcq %%r11, %%r9\n\t" - "adcq %%r12, %%rcx\n\t" - "movq %%r8, 152(%[tmp])\n\t" - "# A[0] * A[20]\n\t" - "movq 160(%[a]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[1] * A[19]\n\t" - "movq 152(%[a]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[2] * A[18]\n\t" - "movq 144(%[a]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[3] * A[17]\n\t" - "movq 136(%[a]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[4] * A[16]\n\t" - "movq 128(%[a]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[5] * A[15]\n\t" - "movq 120(%[a]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[6] * A[14]\n\t" - "movq 112(%[a]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[7] * A[13]\n\t" - "movq 104(%[a]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[8] * A[12]\n\t" - "movq 96(%[a]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[9] * A[11]\n\t" - "movq 88(%[a]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[10] * A[10]\n\t" - "movq 80(%[a]), %%rax\n\t" - "mulq %%rax\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%r9\n\t" - "adcq %%r11, %%rcx\n\t" - "adcq %%r12, %%r8\n\t" - "movq %%r9, 160(%[tmp])\n\t" - "# A[0] * A[21]\n\t" - "movq 168(%[a]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%r9, %%r9\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[1] * A[20]\n\t" - "movq 160(%[a]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[2] * A[19]\n\t" - "movq 152(%[a]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[3] * A[18]\n\t" - "movq 144(%[a]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[4] * A[17]\n\t" - "movq 136(%[a]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[5] * A[16]\n\t" - "movq 128(%[a]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[6] * A[15]\n\t" - "movq 120(%[a]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[7] * A[14]\n\t" - "movq 112(%[a]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[8] * A[13]\n\t" - "movq 104(%[a]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[9] * A[12]\n\t" - "movq 96(%[a]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[10] * A[11]\n\t" - "movq 88(%[a]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%r10, %%rcx\n\t" - "adcq %%r11, %%r8\n\t" - "adcq %%r12, %%r9\n\t" - "movq %%rcx, 168(%[tmp])\n\t" - "# A[0] * A[22]\n\t" - "movq 176(%[a]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%rcx, %%rcx\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[1] * A[21]\n\t" - "movq 168(%[a]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[2] * A[20]\n\t" - "movq 160(%[a]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[3] * A[19]\n\t" - "movq 152(%[a]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[4] * A[18]\n\t" - "movq 144(%[a]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[5] * A[17]\n\t" - "movq 136(%[a]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[6] * A[16]\n\t" - "movq 128(%[a]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[7] * A[15]\n\t" - "movq 120(%[a]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[8] * A[14]\n\t" - "movq 112(%[a]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[9] * A[13]\n\t" - "movq 104(%[a]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[10] * A[12]\n\t" - "movq 96(%[a]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[11] * A[11]\n\t" - "movq 88(%[a]), %%rax\n\t" - "mulq %%rax\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%r8\n\t" - "adcq %%r11, %%r9\n\t" - "adcq %%r12, %%rcx\n\t" - "movq %%r8, 176(%[tmp])\n\t" - "# A[0] * A[23]\n\t" - "movq 184(%[a]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[1] * A[22]\n\t" - "movq 176(%[a]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[2] * A[21]\n\t" - "movq 168(%[a]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[3] * A[20]\n\t" - "movq 160(%[a]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[4] * A[19]\n\t" - "movq 152(%[a]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[5] * A[18]\n\t" - "movq 144(%[a]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[6] * A[17]\n\t" - "movq 136(%[a]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[7] * A[16]\n\t" - "movq 128(%[a]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[8] * A[15]\n\t" - "movq 120(%[a]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[9] * A[14]\n\t" - "movq 112(%[a]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[10] * A[13]\n\t" - "movq 104(%[a]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[11] * A[12]\n\t" - "movq 96(%[a]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%r10, %%r9\n\t" - "adcq %%r11, %%rcx\n\t" - "adcq %%r12, %%r8\n\t" - "movq %%r9, 184(%[tmp])\n\t" - "# A[1] * A[23]\n\t" - "movq 184(%[a]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "xorq %%r9, %%r9\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[2] * A[22]\n\t" - "movq 176(%[a]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[3] * A[21]\n\t" - "movq 168(%[a]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[4] * A[20]\n\t" - "movq 160(%[a]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[5] * A[19]\n\t" - "movq 152(%[a]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[6] * A[18]\n\t" - "movq 144(%[a]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[7] * A[17]\n\t" - "movq 136(%[a]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[8] * A[16]\n\t" - "movq 128(%[a]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[9] * A[15]\n\t" - "movq 120(%[a]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[10] * A[14]\n\t" - "movq 112(%[a]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[11] * A[13]\n\t" - "movq 104(%[a]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[12] * A[12]\n\t" - "movq 96(%[a]), %%rax\n\t" - "mulq %%rax\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%rcx\n\t" - "adcq %%r11, %%r8\n\t" - "adcq %%r12, %%r9\n\t" - "movq %%rcx, 192(%[r])\n\t" - "# A[2] * A[23]\n\t" - "movq 184(%[a]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "xorq %%rcx, %%rcx\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[3] * A[22]\n\t" - "movq 176(%[a]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[4] * A[21]\n\t" - "movq 168(%[a]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[5] * A[20]\n\t" - "movq 160(%[a]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[6] * A[19]\n\t" - "movq 152(%[a]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[7] * A[18]\n\t" - "movq 144(%[a]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[8] * A[17]\n\t" - "movq 136(%[a]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[9] * A[16]\n\t" - "movq 128(%[a]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[10] * A[15]\n\t" - "movq 120(%[a]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[11] * A[14]\n\t" - "movq 112(%[a]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[12] * A[13]\n\t" - "movq 104(%[a]), %%rax\n\t" - "mulq 96(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%r10, %%r8\n\t" - "adcq %%r11, %%r9\n\t" - "adcq %%r12, %%rcx\n\t" - "movq %%r8, 200(%[r])\n\t" - "# A[3] * A[23]\n\t" - "movq 184(%[a]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[4] * A[22]\n\t" - "movq 176(%[a]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[5] * A[21]\n\t" - "movq 168(%[a]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[6] * A[20]\n\t" - "movq 160(%[a]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[7] * A[19]\n\t" - "movq 152(%[a]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[8] * A[18]\n\t" - "movq 144(%[a]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[9] * A[17]\n\t" - "movq 136(%[a]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[10] * A[16]\n\t" - "movq 128(%[a]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[11] * A[15]\n\t" - "movq 120(%[a]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[12] * A[14]\n\t" - "movq 112(%[a]), %%rax\n\t" - "mulq 96(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[13] * A[13]\n\t" - "movq 104(%[a]), %%rax\n\t" - "mulq %%rax\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%r9\n\t" - "adcq %%r11, %%rcx\n\t" - "adcq %%r12, %%r8\n\t" - "movq %%r9, 208(%[r])\n\t" - "# A[4] * A[23]\n\t" - "movq 184(%[a]), %%rax\n\t" - "mulq 32(%[a])\n\t" - "xorq %%r9, %%r9\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[5] * A[22]\n\t" - "movq 176(%[a]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[6] * A[21]\n\t" - "movq 168(%[a]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[7] * A[20]\n\t" - "movq 160(%[a]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[8] * A[19]\n\t" - "movq 152(%[a]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[9] * A[18]\n\t" - "movq 144(%[a]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[10] * A[17]\n\t" - "movq 136(%[a]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[11] * A[16]\n\t" - "movq 128(%[a]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[12] * A[15]\n\t" - "movq 120(%[a]), %%rax\n\t" - "mulq 96(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[13] * A[14]\n\t" - "movq 112(%[a]), %%rax\n\t" - "mulq 104(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%r10, %%rcx\n\t" - "adcq %%r11, %%r8\n\t" - "adcq %%r12, %%r9\n\t" - "movq %%rcx, 216(%[r])\n\t" - "# A[5] * A[23]\n\t" - "movq 184(%[a]), %%rax\n\t" - "mulq 40(%[a])\n\t" - "xorq %%rcx, %%rcx\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[6] * A[22]\n\t" - "movq 176(%[a]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[7] * A[21]\n\t" - "movq 168(%[a]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[8] * A[20]\n\t" - "movq 160(%[a]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[9] * A[19]\n\t" - "movq 152(%[a]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[10] * A[18]\n\t" - "movq 144(%[a]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[11] * A[17]\n\t" - "movq 136(%[a]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[12] * A[16]\n\t" - "movq 128(%[a]), %%rax\n\t" - "mulq 96(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[13] * A[15]\n\t" - "movq 120(%[a]), %%rax\n\t" - "mulq 104(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[14] * A[14]\n\t" - "movq 112(%[a]), %%rax\n\t" - "mulq %%rax\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%r8\n\t" - "adcq %%r11, %%r9\n\t" - "adcq %%r12, %%rcx\n\t" - "movq %%r8, 224(%[r])\n\t" - "# A[6] * A[23]\n\t" - "movq 184(%[a]), %%rax\n\t" - "mulq 48(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[7] * A[22]\n\t" - "movq 176(%[a]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[8] * A[21]\n\t" - "movq 168(%[a]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[9] * A[20]\n\t" - "movq 160(%[a]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[10] * A[19]\n\t" - "movq 152(%[a]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[11] * A[18]\n\t" - "movq 144(%[a]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[12] * A[17]\n\t" - "movq 136(%[a]), %%rax\n\t" - "mulq 96(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[13] * A[16]\n\t" - "movq 128(%[a]), %%rax\n\t" - "mulq 104(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[14] * A[15]\n\t" - "movq 120(%[a]), %%rax\n\t" - "mulq 112(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%r10, %%r9\n\t" - "adcq %%r11, %%rcx\n\t" - "adcq %%r12, %%r8\n\t" - "movq %%r9, 232(%[r])\n\t" - "# A[7] * A[23]\n\t" - "movq 184(%[a]), %%rax\n\t" - "mulq 56(%[a])\n\t" - "xorq %%r9, %%r9\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[8] * A[22]\n\t" - "movq 176(%[a]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[9] * A[21]\n\t" - "movq 168(%[a]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[10] * A[20]\n\t" - "movq 160(%[a]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[11] * A[19]\n\t" - "movq 152(%[a]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[12] * A[18]\n\t" - "movq 144(%[a]), %%rax\n\t" - "mulq 96(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[13] * A[17]\n\t" - "movq 136(%[a]), %%rax\n\t" - "mulq 104(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[14] * A[16]\n\t" - "movq 128(%[a]), %%rax\n\t" - "mulq 112(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[15] * A[15]\n\t" - "movq 120(%[a]), %%rax\n\t" - "mulq %%rax\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%rcx\n\t" - "adcq %%r11, %%r8\n\t" - "adcq %%r12, %%r9\n\t" - "movq %%rcx, 240(%[r])\n\t" - "# A[8] * A[23]\n\t" - "movq 184(%[a]), %%rax\n\t" - "mulq 64(%[a])\n\t" - "xorq %%rcx, %%rcx\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[9] * A[22]\n\t" - "movq 176(%[a]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[10] * A[21]\n\t" - "movq 168(%[a]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[11] * A[20]\n\t" - "movq 160(%[a]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[12] * A[19]\n\t" - "movq 152(%[a]), %%rax\n\t" - "mulq 96(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[13] * A[18]\n\t" - "movq 144(%[a]), %%rax\n\t" - "mulq 104(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[14] * A[17]\n\t" - "movq 136(%[a]), %%rax\n\t" - "mulq 112(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[15] * A[16]\n\t" - "movq 128(%[a]), %%rax\n\t" - "mulq 120(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%r10, %%r8\n\t" - "adcq %%r11, %%r9\n\t" - "adcq %%r12, %%rcx\n\t" - "movq %%r8, 248(%[r])\n\t" - "# A[9] * A[23]\n\t" - "movq 184(%[a]), %%rax\n\t" - "mulq 72(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[10] * A[22]\n\t" - "movq 176(%[a]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[11] * A[21]\n\t" - "movq 168(%[a]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[12] * A[20]\n\t" - "movq 160(%[a]), %%rax\n\t" - "mulq 96(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[13] * A[19]\n\t" - "movq 152(%[a]), %%rax\n\t" - "mulq 104(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[14] * A[18]\n\t" - "movq 144(%[a]), %%rax\n\t" - "mulq 112(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[15] * A[17]\n\t" - "movq 136(%[a]), %%rax\n\t" - "mulq 120(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[16] * A[16]\n\t" - "movq 128(%[a]), %%rax\n\t" - "mulq %%rax\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%r9\n\t" - "adcq %%r11, %%rcx\n\t" - "adcq %%r12, %%r8\n\t" - "movq %%r9, 256(%[r])\n\t" - "# A[10] * A[23]\n\t" - "movq 184(%[a]), %%rax\n\t" - "mulq 80(%[a])\n\t" - "xorq %%r9, %%r9\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[11] * A[22]\n\t" - "movq 176(%[a]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[12] * A[21]\n\t" - "movq 168(%[a]), %%rax\n\t" - "mulq 96(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[13] * A[20]\n\t" - "movq 160(%[a]), %%rax\n\t" - "mulq 104(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[14] * A[19]\n\t" - "movq 152(%[a]), %%rax\n\t" - "mulq 112(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[15] * A[18]\n\t" - "movq 144(%[a]), %%rax\n\t" - "mulq 120(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[16] * A[17]\n\t" - "movq 136(%[a]), %%rax\n\t" - "mulq 128(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%r10, %%rcx\n\t" - "adcq %%r11, %%r8\n\t" - "adcq %%r12, %%r9\n\t" - "movq %%rcx, 264(%[r])\n\t" - "# A[11] * A[23]\n\t" - "movq 184(%[a]), %%rax\n\t" - "mulq 88(%[a])\n\t" - "xorq %%rcx, %%rcx\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[12] * A[22]\n\t" - "movq 176(%[a]), %%rax\n\t" - "mulq 96(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[13] * A[21]\n\t" - "movq 168(%[a]), %%rax\n\t" - "mulq 104(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[14] * A[20]\n\t" - "movq 160(%[a]), %%rax\n\t" - "mulq 112(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[15] * A[19]\n\t" - "movq 152(%[a]), %%rax\n\t" - "mulq 120(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[16] * A[18]\n\t" - "movq 144(%[a]), %%rax\n\t" - "mulq 128(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[17] * A[17]\n\t" - "movq 136(%[a]), %%rax\n\t" - "mulq %%rax\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%r8\n\t" - "adcq %%r11, %%r9\n\t" - "adcq %%r12, %%rcx\n\t" - "movq %%r8, 272(%[r])\n\t" - "# A[12] * A[23]\n\t" - "movq 184(%[a]), %%rax\n\t" - "mulq 96(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[13] * A[22]\n\t" - "movq 176(%[a]), %%rax\n\t" - "mulq 104(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[14] * A[21]\n\t" - "movq 168(%[a]), %%rax\n\t" - "mulq 112(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[15] * A[20]\n\t" - "movq 160(%[a]), %%rax\n\t" - "mulq 120(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[16] * A[19]\n\t" - "movq 152(%[a]), %%rax\n\t" - "mulq 128(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[17] * A[18]\n\t" - "movq 144(%[a]), %%rax\n\t" - "mulq 136(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%r10, %%r9\n\t" - "adcq %%r11, %%rcx\n\t" - "adcq %%r12, %%r8\n\t" - "movq %%r9, 280(%[r])\n\t" - "# A[13] * A[23]\n\t" - "movq 184(%[a]), %%rax\n\t" - "mulq 104(%[a])\n\t" - "xorq %%r9, %%r9\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[14] * A[22]\n\t" - "movq 176(%[a]), %%rax\n\t" - "mulq 112(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[15] * A[21]\n\t" - "movq 168(%[a]), %%rax\n\t" - "mulq 120(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[16] * A[20]\n\t" - "movq 160(%[a]), %%rax\n\t" - "mulq 128(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[17] * A[19]\n\t" - "movq 152(%[a]), %%rax\n\t" - "mulq 136(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[18] * A[18]\n\t" - "movq 144(%[a]), %%rax\n\t" - "mulq %%rax\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%rcx\n\t" - "adcq %%r11, %%r8\n\t" - "adcq %%r12, %%r9\n\t" - "movq %%rcx, 288(%[r])\n\t" - "# A[14] * A[23]\n\t" - "movq 184(%[a]), %%rax\n\t" - "mulq 112(%[a])\n\t" - "xorq %%rcx, %%rcx\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[15] * A[22]\n\t" - "movq 176(%[a]), %%rax\n\t" - "mulq 120(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[16] * A[21]\n\t" - "movq 168(%[a]), %%rax\n\t" - "mulq 128(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[17] * A[20]\n\t" - "movq 160(%[a]), %%rax\n\t" - "mulq 136(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[18] * A[19]\n\t" - "movq 152(%[a]), %%rax\n\t" - "mulq 144(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%r10, %%r8\n\t" - "adcq %%r11, %%r9\n\t" - "adcq %%r12, %%rcx\n\t" - "movq %%r8, 296(%[r])\n\t" - "# A[15] * A[23]\n\t" - "movq 184(%[a]), %%rax\n\t" - "mulq 120(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[16] * A[22]\n\t" - "movq 176(%[a]), %%rax\n\t" - "mulq 128(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[17] * A[21]\n\t" - "movq 168(%[a]), %%rax\n\t" - "mulq 136(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[18] * A[20]\n\t" - "movq 160(%[a]), %%rax\n\t" - "mulq 144(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[19] * A[19]\n\t" - "movq 152(%[a]), %%rax\n\t" - "mulq %%rax\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%r9\n\t" - "adcq %%r11, %%rcx\n\t" - "adcq %%r12, %%r8\n\t" - "movq %%r9, 304(%[r])\n\t" - "# A[16] * A[23]\n\t" - "movq 184(%[a]), %%rax\n\t" - "mulq 128(%[a])\n\t" - "xorq %%r9, %%r9\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[17] * A[22]\n\t" - "movq 176(%[a]), %%rax\n\t" - "mulq 136(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[18] * A[21]\n\t" - "movq 168(%[a]), %%rax\n\t" - "mulq 144(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[19] * A[20]\n\t" - "movq 160(%[a]), %%rax\n\t" - "mulq 152(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%r10, %%rcx\n\t" - "adcq %%r11, %%r8\n\t" - "adcq %%r12, %%r9\n\t" - "movq %%rcx, 312(%[r])\n\t" - "# A[17] * A[23]\n\t" - "movq 184(%[a]), %%rax\n\t" - "mulq 136(%[a])\n\t" - "xorq %%rcx, %%rcx\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[18] * A[22]\n\t" - "movq 176(%[a]), %%rax\n\t" - "mulq 144(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[19] * A[21]\n\t" - "movq 168(%[a]), %%rax\n\t" - "mulq 152(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[20] * A[20]\n\t" - "movq 160(%[a]), %%rax\n\t" - "mulq %%rax\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%r8\n\t" - "adcq %%r11, %%r9\n\t" - "adcq %%r12, %%rcx\n\t" - "movq %%r8, 320(%[r])\n\t" - "# A[18] * A[23]\n\t" - "movq 184(%[a]), %%rax\n\t" - "mulq 144(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r10\n\t" - "movq %%rdx, %%r11\n\t" - "# A[19] * A[22]\n\t" - "movq 176(%[a]), %%rax\n\t" - "mulq 152(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[20] * A[21]\n\t" - "movq 168(%[a]), %%rax\n\t" - "mulq 160(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "addq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "addq %%r10, %%r9\n\t" - "adcq %%r11, %%rcx\n\t" - "adcq %%r12, %%r8\n\t" - "movq %%r9, 328(%[r])\n\t" - "# A[19] * A[23]\n\t" - "movq 184(%[a]), %%rax\n\t" - "mulq 152(%[a])\n\t" - "xorq %%r9, %%r9\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%r9\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%r9\n\t" - "# A[20] * A[22]\n\t" - "movq 176(%[a]), %%rax\n\t" - "mulq 160(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%r9\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%r9\n\t" - "# A[21] * A[21]\n\t" - "movq 168(%[a]), %%rax\n\t" - "mulq %%rax\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%r9\n\t" - "movq %%rcx, 336(%[r])\n\t" - "# A[20] * A[23]\n\t" - "movq 184(%[a]), %%rax\n\t" - "mulq 160(%[a])\n\t" - "xorq %%rcx, %%rcx\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%r9\n\t" - "adcq $0, %%rcx\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%r9\n\t" - "adcq $0, %%rcx\n\t" - "# A[21] * A[22]\n\t" - "movq 176(%[a]), %%rax\n\t" - "mulq 168(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%r9\n\t" - "adcq $0, %%rcx\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%r9\n\t" - "adcq $0, %%rcx\n\t" - "movq %%r8, 344(%[r])\n\t" - "# A[21] * A[23]\n\t" - "movq 184(%[a]), %%rax\n\t" - "mulq 168(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "addq %%rax, %%r9\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "addq %%rax, %%r9\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[22] * A[22]\n\t" - "movq 176(%[a]), %%rax\n\t" - "mulq %%rax\n\t" - "addq %%rax, %%r9\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "movq %%r9, 352(%[r])\n\t" - "# A[22] * A[23]\n\t" - "movq 184(%[a]), %%rax\n\t" - "mulq 176(%[a])\n\t" - "xorq %%r9, %%r9\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%r9\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%r9\n\t" - "movq %%rcx, 360(%[r])\n\t" - "# A[23] * A[23]\n\t" - "movq 184(%[a]), %%rax\n\t" - "mulq %%rax\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%r9\n\t" - "movq %%r8, 368(%[r])\n\t" - "movq %%r9, 376(%[r])\n\t" - : - : [r] "r" (r), [a] "r" (a), [tmp] "r" (tmp) - : "memory", "rax", "rdx", "rcx", "r8", "r9", "r10", "r11", "r12" - ); - - XMEMCPY(r, tmp, sizeof(tmp)); -} - -#ifdef HAVE_INTEL_AVX2 -/* Multiply a and b into r. (r = a * b) - * - * r Result of multiplication. - * a First number to multiply. - * b Second number to multiply. - */ -SP_NOINLINE static void sp_3072_mul_avx2_24(sp_digit* r, const sp_digit* a, - const sp_digit* b) -{ - sp_digit tmp[2*24]; - - __asm__ __volatile__ ( - "movq 0(%[a]), %%rdx\n\t" - "xorq %%r15, %%r15\n\t" - "# A[0] * B[0]\n\t" - "mulx 0(%[b]), %%r10, %%r11\n\t" - "# A[0] * B[1]\n\t" - "mulx 8(%[b]), %%r8, %%r12\n\t" - "adcxq %%r8, %%r11\n\t" - "# A[0] * B[2]\n\t" - "mulx 16(%[b]), %%r8, %%r13\n\t" - "adcxq %%r8, %%r12\n\t" - "# A[0] * B[3]\n\t" - "mulx 24(%[b]), %%r8, %%r14\n\t" - "adcxq %%r8, %%r13\n\t" - "movq %%r10, 0(%[t])\n\t" - "movq %%r11, 8(%[t])\n\t" - "movq %%r12, 16(%[t])\n\t" - "movq %%r13, 24(%[t])\n\t" - "# A[0] * B[4]\n\t" - "mulx 32(%[b]), %%r8, %%rax\n\t" - "adcxq %%r8, %%r14\n\t" - "# A[0] * B[5]\n\t" - "mulx 40(%[b]), %%r8, %%r10\n\t" - "adcxq %%r8, %%rax\n\t" - "# A[0] * B[6]\n\t" - "mulx 48(%[b]), %%r8, %%r11\n\t" - "adcxq %%r8, %%r10\n\t" - "# A[0] * B[7]\n\t" - "mulx 56(%[b]), %%r8, %%r12\n\t" - "adcxq %%r8, %%r11\n\t" - "movq %%r14, 32(%[t])\n\t" - "movq %%rax, 40(%[t])\n\t" - "movq %%r10, 48(%[t])\n\t" - "movq %%r11, 56(%[t])\n\t" - "# A[0] * B[8]\n\t" - "mulx 64(%[b]), %%r8, %%r13\n\t" - "adcxq %%r8, %%r12\n\t" - "# A[0] * B[9]\n\t" - "mulx 72(%[b]), %%r8, %%r14\n\t" - "adcxq %%r8, %%r13\n\t" - "# A[0] * B[10]\n\t" - "mulx 80(%[b]), %%r8, %%rax\n\t" - "adcxq %%r8, %%r14\n\t" - "# A[0] * B[11]\n\t" - "mulx 88(%[b]), %%r8, %%r10\n\t" - "adcxq %%r8, %%rax\n\t" - "movq %%r12, 64(%[t])\n\t" - "movq %%r13, 72(%[t])\n\t" - "movq %%r14, 80(%[t])\n\t" - "movq %%rax, 88(%[t])\n\t" - "# A[0] * B[12]\n\t" - "mulx 96(%[b]), %%r8, %%r11\n\t" - "adcxq %%r8, %%r10\n\t" - "# A[0] * B[13]\n\t" - "mulx 104(%[b]), %%r8, %%r12\n\t" - "adcxq %%r8, %%r11\n\t" - "# A[0] * B[14]\n\t" - "mulx 112(%[b]), %%r8, %%r13\n\t" - "adcxq %%r8, %%r12\n\t" - "# A[0] * B[15]\n\t" - "mulx 120(%[b]), %%r8, %%r14\n\t" - "adcxq %%r8, %%r13\n\t" - "movq %%r10, 96(%[t])\n\t" - "movq %%r11, 104(%[t])\n\t" - "movq %%r12, 112(%[t])\n\t" - "movq %%r13, 120(%[t])\n\t" - "# A[0] * B[16]\n\t" - "mulx 128(%[b]), %%r8, %%rax\n\t" - "adcxq %%r8, %%r14\n\t" - "# A[0] * B[17]\n\t" - "mulx 136(%[b]), %%r8, %%r10\n\t" - "adcxq %%r8, %%rax\n\t" - "# A[0] * B[18]\n\t" - "mulx 144(%[b]), %%r8, %%r11\n\t" - "adcxq %%r8, %%r10\n\t" - "# A[0] * B[19]\n\t" - "mulx 152(%[b]), %%r8, %%r12\n\t" - "adcxq %%r8, %%r11\n\t" - "movq %%r14, 128(%[t])\n\t" - "movq %%rax, 136(%[t])\n\t" - "movq %%r10, 144(%[t])\n\t" - "movq %%r11, 152(%[t])\n\t" - "# A[0] * B[20]\n\t" - "mulx 160(%[b]), %%r8, %%r13\n\t" - "adcxq %%r8, %%r12\n\t" - "# A[0] * B[21]\n\t" - "mulx 168(%[b]), %%r8, %%r14\n\t" - "adcxq %%r8, %%r13\n\t" - "# A[0] * B[22]\n\t" - "mulx 176(%[b]), %%r8, %%rax\n\t" - "adcxq %%r8, %%r14\n\t" - "# A[0] * B[23]\n\t" - "mulx 184(%[b]), %%r8, %%r10\n\t" - "adcxq %%r8, %%rax\n\t" - "adcxq %%r15, %%r10\n\t" - "movq %%r15, %%rcx\n\t" - "adcxq %%r15, %%rcx\n\t" - "movq %%r12, 160(%[t])\n\t" - "movq %%r13, 168(%[t])\n\t" - "movq %%r14, 176(%[t])\n\t" - "movq %%rax, 184(%[t])\n\t" - "movq %%r10, 192(%[t])\n\t" - "movq 8(%[a]), %%rdx\n\t" - "xorq %%r15, %%r15\n\t" - "movq 8(%[t]), %%r11\n\t" - "movq 16(%[t]), %%r12\n\t" - "movq 24(%[t]), %%r13\n\t" - "movq 32(%[t]), %%r14\n\t" - "movq 40(%[t]), %%rax\n\t" - "# A[1] * B[0]\n\t" - "mulx 0(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[1] * B[1]\n\t" - "mulx 8(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[1] * B[2]\n\t" - "mulx 16(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[1] * B[3]\n\t" - "mulx 24(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "movq %%r11, 8(%[t])\n\t" - "movq %%r12, 16(%[t])\n\t" - "movq %%r13, 24(%[t])\n\t" - "movq %%r14, 32(%[t])\n\t" - "movq 48(%[t]), %%r10\n\t" - "movq 56(%[t]), %%r11\n\t" - "movq 64(%[t]), %%r12\n\t" - "movq 72(%[t]), %%r13\n\t" - "# A[1] * B[4]\n\t" - "mulx 32(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[1] * B[5]\n\t" - "mulx 40(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[1] * B[6]\n\t" - "mulx 48(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[1] * B[7]\n\t" - "mulx 56(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "movq %%rax, 40(%[t])\n\t" - "movq %%r10, 48(%[t])\n\t" - "movq %%r11, 56(%[t])\n\t" - "movq %%r12, 64(%[t])\n\t" - "movq 80(%[t]), %%r14\n\t" - "movq 88(%[t]), %%rax\n\t" - "movq 96(%[t]), %%r10\n\t" - "movq 104(%[t]), %%r11\n\t" - "# A[1] * B[8]\n\t" - "mulx 64(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[1] * B[9]\n\t" - "mulx 72(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[1] * B[10]\n\t" - "mulx 80(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[1] * B[11]\n\t" - "mulx 88(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "movq %%r13, 72(%[t])\n\t" - "movq %%r14, 80(%[t])\n\t" - "movq %%rax, 88(%[t])\n\t" - "movq %%r10, 96(%[t])\n\t" - "movq 112(%[t]), %%r12\n\t" - "movq 120(%[t]), %%r13\n\t" - "movq 128(%[t]), %%r14\n\t" - "movq 136(%[t]), %%rax\n\t" - "# A[1] * B[12]\n\t" - "mulx 96(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[1] * B[13]\n\t" - "mulx 104(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[1] * B[14]\n\t" - "mulx 112(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[1] * B[15]\n\t" - "mulx 120(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "movq %%r11, 104(%[t])\n\t" - "movq %%r12, 112(%[t])\n\t" - "movq %%r13, 120(%[t])\n\t" - "movq %%r14, 128(%[t])\n\t" - "movq 144(%[t]), %%r10\n\t" - "movq 152(%[t]), %%r11\n\t" - "movq 160(%[t]), %%r12\n\t" - "movq 168(%[t]), %%r13\n\t" - "# A[1] * B[16]\n\t" - "mulx 128(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[1] * B[17]\n\t" - "mulx 136(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[1] * B[18]\n\t" - "mulx 144(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[1] * B[19]\n\t" - "mulx 152(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "movq %%rax, 136(%[t])\n\t" - "movq %%r10, 144(%[t])\n\t" - "movq %%r11, 152(%[t])\n\t" - "movq %%r12, 160(%[t])\n\t" - "movq 176(%[t]), %%r14\n\t" - "movq 184(%[t]), %%rax\n\t" - "movq 192(%[t]), %%r10\n\t" - "# A[1] * B[20]\n\t" - "mulx 160(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[1] * B[21]\n\t" - "mulx 168(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[1] * B[22]\n\t" - "mulx 176(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[1] * B[23]\n\t" - "mulx 184(%[b]), %%r8, %%r9\n\t" - "movq %%r15, %%r11\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "adcxq %%rcx, %%r11\n\t" - "movq %%r15, %%rcx\n\t" - "adoxq %%r15, %%rcx\n\t" - "adcxq %%r15, %%rcx\n\t" - "movq %%r13, 168(%[t])\n\t" - "movq %%r14, 176(%[t])\n\t" - "movq %%rax, 184(%[t])\n\t" - "movq %%r10, 192(%[t])\n\t" - "movq %%r11, 200(%[t])\n\t" - "movq 16(%[a]), %%rdx\n\t" - "xorq %%r15, %%r15\n\t" - "movq 16(%[t]), %%r12\n\t" - "movq 24(%[t]), %%r13\n\t" - "movq 32(%[t]), %%r14\n\t" - "movq 40(%[t]), %%rax\n\t" - "movq 48(%[t]), %%r10\n\t" - "# A[2] * B[0]\n\t" - "mulx 0(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[2] * B[1]\n\t" - "mulx 8(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[2] * B[2]\n\t" - "mulx 16(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[2] * B[3]\n\t" - "mulx 24(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "movq %%r12, 16(%[t])\n\t" - "movq %%r13, 24(%[t])\n\t" - "movq %%r14, 32(%[t])\n\t" - "movq %%rax, 40(%[t])\n\t" - "movq 56(%[t]), %%r11\n\t" - "movq 64(%[t]), %%r12\n\t" - "movq 72(%[t]), %%r13\n\t" - "movq 80(%[t]), %%r14\n\t" - "# A[2] * B[4]\n\t" - "mulx 32(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[2] * B[5]\n\t" - "mulx 40(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[2] * B[6]\n\t" - "mulx 48(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[2] * B[7]\n\t" - "mulx 56(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "movq %%r10, 48(%[t])\n\t" - "movq %%r11, 56(%[t])\n\t" - "movq %%r12, 64(%[t])\n\t" - "movq %%r13, 72(%[t])\n\t" - "movq 88(%[t]), %%rax\n\t" - "movq 96(%[t]), %%r10\n\t" - "movq 104(%[t]), %%r11\n\t" - "movq 112(%[t]), %%r12\n\t" - "# A[2] * B[8]\n\t" - "mulx 64(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[2] * B[9]\n\t" - "mulx 72(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[2] * B[10]\n\t" - "mulx 80(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[2] * B[11]\n\t" - "mulx 88(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "movq %%r14, 80(%[t])\n\t" - "movq %%rax, 88(%[t])\n\t" - "movq %%r10, 96(%[t])\n\t" - "movq %%r11, 104(%[t])\n\t" - "movq 120(%[t]), %%r13\n\t" - "movq 128(%[t]), %%r14\n\t" - "movq 136(%[t]), %%rax\n\t" - "movq 144(%[t]), %%r10\n\t" - "# A[2] * B[12]\n\t" - "mulx 96(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[2] * B[13]\n\t" - "mulx 104(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[2] * B[14]\n\t" - "mulx 112(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[2] * B[15]\n\t" - "mulx 120(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "movq %%r12, 112(%[t])\n\t" - "movq %%r13, 120(%[t])\n\t" - "movq %%r14, 128(%[t])\n\t" - "movq %%rax, 136(%[t])\n\t" - "movq 152(%[t]), %%r11\n\t" - "movq 160(%[t]), %%r12\n\t" - "movq 168(%[t]), %%r13\n\t" - "movq 176(%[t]), %%r14\n\t" - "# A[2] * B[16]\n\t" - "mulx 128(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[2] * B[17]\n\t" - "mulx 136(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[2] * B[18]\n\t" - "mulx 144(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[2] * B[19]\n\t" - "mulx 152(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "movq %%r10, 144(%[t])\n\t" - "movq %%r11, 152(%[t])\n\t" - "movq %%r12, 160(%[t])\n\t" - "movq %%r13, 168(%[t])\n\t" - "movq 184(%[t]), %%rax\n\t" - "movq 192(%[t]), %%r10\n\t" - "movq 200(%[t]), %%r11\n\t" - "# A[2] * B[20]\n\t" - "mulx 160(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[2] * B[21]\n\t" - "mulx 168(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[2] * B[22]\n\t" - "mulx 176(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[2] * B[23]\n\t" - "mulx 184(%[b]), %%r8, %%r9\n\t" - "movq %%r15, %%r12\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "adcxq %%rcx, %%r12\n\t" - "movq %%r15, %%rcx\n\t" - "adoxq %%r15, %%rcx\n\t" - "adcxq %%r15, %%rcx\n\t" - "movq %%r14, 176(%[t])\n\t" - "movq %%rax, 184(%[t])\n\t" - "movq %%r10, 192(%[t])\n\t" - "movq %%r11, 200(%[t])\n\t" - "movq %%r12, 208(%[t])\n\t" - "movq 24(%[a]), %%rdx\n\t" - "xorq %%r15, %%r15\n\t" - "movq 24(%[t]), %%r13\n\t" - "movq 32(%[t]), %%r14\n\t" - "movq 40(%[t]), %%rax\n\t" - "movq 48(%[t]), %%r10\n\t" - "movq 56(%[t]), %%r11\n\t" - "# A[3] * B[0]\n\t" - "mulx 0(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[3] * B[1]\n\t" - "mulx 8(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[3] * B[2]\n\t" - "mulx 16(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[3] * B[3]\n\t" - "mulx 24(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "movq %%r13, 24(%[t])\n\t" - "movq %%r14, 32(%[t])\n\t" - "movq %%rax, 40(%[t])\n\t" - "movq %%r10, 48(%[t])\n\t" - "movq 64(%[t]), %%r12\n\t" - "movq 72(%[t]), %%r13\n\t" - "movq 80(%[t]), %%r14\n\t" - "movq 88(%[t]), %%rax\n\t" - "# A[3] * B[4]\n\t" - "mulx 32(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[3] * B[5]\n\t" - "mulx 40(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[3] * B[6]\n\t" - "mulx 48(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[3] * B[7]\n\t" - "mulx 56(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "movq %%r11, 56(%[t])\n\t" - "movq %%r12, 64(%[t])\n\t" - "movq %%r13, 72(%[t])\n\t" - "movq %%r14, 80(%[t])\n\t" - "movq 96(%[t]), %%r10\n\t" - "movq 104(%[t]), %%r11\n\t" - "movq 112(%[t]), %%r12\n\t" - "movq 120(%[t]), %%r13\n\t" - "# A[3] * B[8]\n\t" - "mulx 64(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[3] * B[9]\n\t" - "mulx 72(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[3] * B[10]\n\t" - "mulx 80(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[3] * B[11]\n\t" - "mulx 88(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "movq %%rax, 88(%[t])\n\t" - "movq %%r10, 96(%[t])\n\t" - "movq %%r11, 104(%[t])\n\t" - "movq %%r12, 112(%[t])\n\t" - "movq 128(%[t]), %%r14\n\t" - "movq 136(%[t]), %%rax\n\t" - "movq 144(%[t]), %%r10\n\t" - "movq 152(%[t]), %%r11\n\t" - "# A[3] * B[12]\n\t" - "mulx 96(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[3] * B[13]\n\t" - "mulx 104(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[3] * B[14]\n\t" - "mulx 112(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[3] * B[15]\n\t" - "mulx 120(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "movq %%r13, 120(%[t])\n\t" - "movq %%r14, 128(%[t])\n\t" - "movq %%rax, 136(%[t])\n\t" - "movq %%r10, 144(%[t])\n\t" - "movq 160(%[t]), %%r12\n\t" - "movq 168(%[t]), %%r13\n\t" - "movq 176(%[t]), %%r14\n\t" - "movq 184(%[t]), %%rax\n\t" - "# A[3] * B[16]\n\t" - "mulx 128(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[3] * B[17]\n\t" - "mulx 136(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[3] * B[18]\n\t" - "mulx 144(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[3] * B[19]\n\t" - "mulx 152(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "movq %%r11, 152(%[t])\n\t" - "movq %%r12, 160(%[t])\n\t" - "movq %%r13, 168(%[t])\n\t" - "movq %%r14, 176(%[t])\n\t" - "movq 192(%[t]), %%r10\n\t" - "movq 200(%[t]), %%r11\n\t" - "movq 208(%[t]), %%r12\n\t" - "# A[3] * B[20]\n\t" - "mulx 160(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[3] * B[21]\n\t" - "mulx 168(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[3] * B[22]\n\t" - "mulx 176(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[3] * B[23]\n\t" - "mulx 184(%[b]), %%r8, %%r9\n\t" - "movq %%r15, %%r13\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "adcxq %%rcx, %%r13\n\t" - "movq %%r15, %%rcx\n\t" - "adoxq %%r15, %%rcx\n\t" - "adcxq %%r15, %%rcx\n\t" - "movq %%rax, 184(%[t])\n\t" - "movq %%r10, 192(%[t])\n\t" - "movq %%r11, 200(%[t])\n\t" - "movq %%r12, 208(%[t])\n\t" - "movq %%r13, 216(%[t])\n\t" - "movq 32(%[a]), %%rdx\n\t" - "xorq %%r15, %%r15\n\t" - "movq 32(%[t]), %%r14\n\t" - "movq 40(%[t]), %%rax\n\t" - "movq 48(%[t]), %%r10\n\t" - "movq 56(%[t]), %%r11\n\t" - "movq 64(%[t]), %%r12\n\t" - "# A[4] * B[0]\n\t" - "mulx 0(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[4] * B[1]\n\t" - "mulx 8(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[4] * B[2]\n\t" - "mulx 16(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[4] * B[3]\n\t" - "mulx 24(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "movq %%r14, 32(%[t])\n\t" - "movq %%rax, 40(%[t])\n\t" - "movq %%r10, 48(%[t])\n\t" - "movq %%r11, 56(%[t])\n\t" - "movq 72(%[t]), %%r13\n\t" - "movq 80(%[t]), %%r14\n\t" - "movq 88(%[t]), %%rax\n\t" - "movq 96(%[t]), %%r10\n\t" - "# A[4] * B[4]\n\t" - "mulx 32(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[4] * B[5]\n\t" - "mulx 40(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[4] * B[6]\n\t" - "mulx 48(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[4] * B[7]\n\t" - "mulx 56(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "movq %%r12, 64(%[t])\n\t" - "movq %%r13, 72(%[t])\n\t" - "movq %%r14, 80(%[t])\n\t" - "movq %%rax, 88(%[t])\n\t" - "movq 104(%[t]), %%r11\n\t" - "movq 112(%[t]), %%r12\n\t" - "movq 120(%[t]), %%r13\n\t" - "movq 128(%[t]), %%r14\n\t" - "# A[4] * B[8]\n\t" - "mulx 64(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[4] * B[9]\n\t" - "mulx 72(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[4] * B[10]\n\t" - "mulx 80(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[4] * B[11]\n\t" - "mulx 88(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "movq %%r10, 96(%[t])\n\t" - "movq %%r11, 104(%[t])\n\t" - "movq %%r12, 112(%[t])\n\t" - "movq %%r13, 120(%[t])\n\t" - "movq 136(%[t]), %%rax\n\t" - "movq 144(%[t]), %%r10\n\t" - "movq 152(%[t]), %%r11\n\t" - "movq 160(%[t]), %%r12\n\t" - "# A[4] * B[12]\n\t" - "mulx 96(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[4] * B[13]\n\t" - "mulx 104(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[4] * B[14]\n\t" - "mulx 112(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[4] * B[15]\n\t" - "mulx 120(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "movq %%r14, 128(%[t])\n\t" - "movq %%rax, 136(%[t])\n\t" - "movq %%r10, 144(%[t])\n\t" - "movq %%r11, 152(%[t])\n\t" - "movq 168(%[t]), %%r13\n\t" - "movq 176(%[t]), %%r14\n\t" - "movq 184(%[t]), %%rax\n\t" - "movq 192(%[t]), %%r10\n\t" - "# A[4] * B[16]\n\t" - "mulx 128(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[4] * B[17]\n\t" - "mulx 136(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[4] * B[18]\n\t" - "mulx 144(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[4] * B[19]\n\t" - "mulx 152(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "movq %%r12, 160(%[t])\n\t" - "movq %%r13, 168(%[t])\n\t" - "movq %%r14, 176(%[t])\n\t" - "movq %%rax, 184(%[t])\n\t" - "movq 200(%[t]), %%r11\n\t" - "movq 208(%[t]), %%r12\n\t" - "movq 216(%[t]), %%r13\n\t" - "# A[4] * B[20]\n\t" - "mulx 160(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[4] * B[21]\n\t" - "mulx 168(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[4] * B[22]\n\t" - "mulx 176(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[4] * B[23]\n\t" - "mulx 184(%[b]), %%r8, %%r9\n\t" - "movq %%r15, %%r14\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "adcxq %%rcx, %%r14\n\t" - "movq %%r15, %%rcx\n\t" - "adoxq %%r15, %%rcx\n\t" - "adcxq %%r15, %%rcx\n\t" - "movq %%r10, 192(%[t])\n\t" - "movq %%r11, 200(%[t])\n\t" - "movq %%r12, 208(%[t])\n\t" - "movq %%r13, 216(%[t])\n\t" - "movq %%r14, 224(%[t])\n\t" - "movq 40(%[a]), %%rdx\n\t" - "xorq %%r15, %%r15\n\t" - "movq 40(%[t]), %%rax\n\t" - "movq 48(%[t]), %%r10\n\t" - "movq 56(%[t]), %%r11\n\t" - "movq 64(%[t]), %%r12\n\t" - "movq 72(%[t]), %%r13\n\t" - "# A[5] * B[0]\n\t" - "mulx 0(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[5] * B[1]\n\t" - "mulx 8(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[5] * B[2]\n\t" - "mulx 16(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[5] * B[3]\n\t" - "mulx 24(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "movq %%rax, 40(%[t])\n\t" - "movq %%r10, 48(%[t])\n\t" - "movq %%r11, 56(%[t])\n\t" - "movq %%r12, 64(%[t])\n\t" - "movq 80(%[t]), %%r14\n\t" - "movq 88(%[t]), %%rax\n\t" - "movq 96(%[t]), %%r10\n\t" - "movq 104(%[t]), %%r11\n\t" - "# A[5] * B[4]\n\t" - "mulx 32(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[5] * B[5]\n\t" - "mulx 40(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[5] * B[6]\n\t" - "mulx 48(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[5] * B[7]\n\t" - "mulx 56(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "movq %%r13, 72(%[t])\n\t" - "movq %%r14, 80(%[t])\n\t" - "movq %%rax, 88(%[t])\n\t" - "movq %%r10, 96(%[t])\n\t" - "movq 112(%[t]), %%r12\n\t" - "movq 120(%[t]), %%r13\n\t" - "movq 128(%[t]), %%r14\n\t" - "movq 136(%[t]), %%rax\n\t" - "# A[5] * B[8]\n\t" - "mulx 64(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[5] * B[9]\n\t" - "mulx 72(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[5] * B[10]\n\t" - "mulx 80(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[5] * B[11]\n\t" - "mulx 88(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "movq %%r11, 104(%[t])\n\t" - "movq %%r12, 112(%[t])\n\t" - "movq %%r13, 120(%[t])\n\t" - "movq %%r14, 128(%[t])\n\t" - "movq 144(%[t]), %%r10\n\t" - "movq 152(%[t]), %%r11\n\t" - "movq 160(%[t]), %%r12\n\t" - "movq 168(%[t]), %%r13\n\t" - "# A[5] * B[12]\n\t" - "mulx 96(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[5] * B[13]\n\t" - "mulx 104(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[5] * B[14]\n\t" - "mulx 112(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[5] * B[15]\n\t" - "mulx 120(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "movq %%rax, 136(%[t])\n\t" - "movq %%r10, 144(%[t])\n\t" - "movq %%r11, 152(%[t])\n\t" - "movq %%r12, 160(%[t])\n\t" - "movq 176(%[t]), %%r14\n\t" - "movq 184(%[t]), %%rax\n\t" - "movq 192(%[t]), %%r10\n\t" - "movq 200(%[t]), %%r11\n\t" - "# A[5] * B[16]\n\t" - "mulx 128(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[5] * B[17]\n\t" - "mulx 136(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[5] * B[18]\n\t" - "mulx 144(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[5] * B[19]\n\t" - "mulx 152(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "movq %%r13, 168(%[t])\n\t" - "movq %%r14, 176(%[t])\n\t" - "movq %%rax, 184(%[t])\n\t" - "movq %%r10, 192(%[t])\n\t" - "movq 208(%[t]), %%r12\n\t" - "movq 216(%[t]), %%r13\n\t" - "movq 224(%[t]), %%r14\n\t" - "# A[5] * B[20]\n\t" - "mulx 160(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[5] * B[21]\n\t" - "mulx 168(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[5] * B[22]\n\t" - "mulx 176(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[5] * B[23]\n\t" - "mulx 184(%[b]), %%r8, %%r9\n\t" - "movq %%r15, %%rax\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "adcxq %%rcx, %%rax\n\t" - "movq %%r15, %%rcx\n\t" - "adoxq %%r15, %%rcx\n\t" - "adcxq %%r15, %%rcx\n\t" - "movq %%r11, 200(%[t])\n\t" - "movq %%r12, 208(%[t])\n\t" - "movq %%r13, 216(%[t])\n\t" - "movq %%r14, 224(%[t])\n\t" - "movq %%rax, 232(%[t])\n\t" - "movq 48(%[a]), %%rdx\n\t" - "xorq %%r15, %%r15\n\t" - "movq 48(%[t]), %%r10\n\t" - "movq 56(%[t]), %%r11\n\t" - "movq 64(%[t]), %%r12\n\t" - "movq 72(%[t]), %%r13\n\t" - "movq 80(%[t]), %%r14\n\t" - "# A[6] * B[0]\n\t" - "mulx 0(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[6] * B[1]\n\t" - "mulx 8(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[6] * B[2]\n\t" - "mulx 16(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[6] * B[3]\n\t" - "mulx 24(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "movq %%r10, 48(%[t])\n\t" - "movq %%r11, 56(%[t])\n\t" - "movq %%r12, 64(%[t])\n\t" - "movq %%r13, 72(%[t])\n\t" - "movq 88(%[t]), %%rax\n\t" - "movq 96(%[t]), %%r10\n\t" - "movq 104(%[t]), %%r11\n\t" - "movq 112(%[t]), %%r12\n\t" - "# A[6] * B[4]\n\t" - "mulx 32(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[6] * B[5]\n\t" - "mulx 40(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[6] * B[6]\n\t" - "mulx 48(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[6] * B[7]\n\t" - "mulx 56(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "movq %%r14, 80(%[t])\n\t" - "movq %%rax, 88(%[t])\n\t" - "movq %%r10, 96(%[t])\n\t" - "movq %%r11, 104(%[t])\n\t" - "movq 120(%[t]), %%r13\n\t" - "movq 128(%[t]), %%r14\n\t" - "movq 136(%[t]), %%rax\n\t" - "movq 144(%[t]), %%r10\n\t" - "# A[6] * B[8]\n\t" - "mulx 64(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[6] * B[9]\n\t" - "mulx 72(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[6] * B[10]\n\t" - "mulx 80(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[6] * B[11]\n\t" - "mulx 88(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "movq %%r12, 112(%[t])\n\t" - "movq %%r13, 120(%[t])\n\t" - "movq %%r14, 128(%[t])\n\t" - "movq %%rax, 136(%[t])\n\t" - "movq 152(%[t]), %%r11\n\t" - "movq 160(%[t]), %%r12\n\t" - "movq 168(%[t]), %%r13\n\t" - "movq 176(%[t]), %%r14\n\t" - "# A[6] * B[12]\n\t" - "mulx 96(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[6] * B[13]\n\t" - "mulx 104(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[6] * B[14]\n\t" - "mulx 112(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[6] * B[15]\n\t" - "mulx 120(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "movq %%r10, 144(%[t])\n\t" - "movq %%r11, 152(%[t])\n\t" - "movq %%r12, 160(%[t])\n\t" - "movq %%r13, 168(%[t])\n\t" - "movq 184(%[t]), %%rax\n\t" - "movq 192(%[t]), %%r10\n\t" - "movq 200(%[t]), %%r11\n\t" - "movq 208(%[t]), %%r12\n\t" - "# A[6] * B[16]\n\t" - "mulx 128(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[6] * B[17]\n\t" - "mulx 136(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[6] * B[18]\n\t" - "mulx 144(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[6] * B[19]\n\t" - "mulx 152(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "movq %%r14, 176(%[t])\n\t" - "movq %%rax, 184(%[t])\n\t" - "movq %%r10, 192(%[t])\n\t" - "movq %%r11, 200(%[t])\n\t" - "movq 216(%[t]), %%r13\n\t" - "movq 224(%[t]), %%r14\n\t" - "movq 232(%[t]), %%rax\n\t" - "# A[6] * B[20]\n\t" - "mulx 160(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[6] * B[21]\n\t" - "mulx 168(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[6] * B[22]\n\t" - "mulx 176(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[6] * B[23]\n\t" - "mulx 184(%[b]), %%r8, %%r9\n\t" - "movq %%r15, %%r10\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "adcxq %%rcx, %%r10\n\t" - "movq %%r15, %%rcx\n\t" - "adoxq %%r15, %%rcx\n\t" - "adcxq %%r15, %%rcx\n\t" - "movq %%r12, 208(%[t])\n\t" - "movq %%r13, 216(%[t])\n\t" - "movq %%r14, 224(%[t])\n\t" - "movq %%rax, 232(%[t])\n\t" - "movq %%r10, 240(%[t])\n\t" - "movq 56(%[a]), %%rdx\n\t" - "xorq %%r15, %%r15\n\t" - "movq 56(%[t]), %%r11\n\t" - "movq 64(%[t]), %%r12\n\t" - "movq 72(%[t]), %%r13\n\t" - "movq 80(%[t]), %%r14\n\t" - "movq 88(%[t]), %%rax\n\t" - "# A[7] * B[0]\n\t" - "mulx 0(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[7] * B[1]\n\t" - "mulx 8(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[7] * B[2]\n\t" - "mulx 16(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[7] * B[3]\n\t" - "mulx 24(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "movq %%r11, 56(%[t])\n\t" - "movq %%r12, 64(%[t])\n\t" - "movq %%r13, 72(%[t])\n\t" - "movq %%r14, 80(%[t])\n\t" - "movq 96(%[t]), %%r10\n\t" - "movq 104(%[t]), %%r11\n\t" - "movq 112(%[t]), %%r12\n\t" - "movq 120(%[t]), %%r13\n\t" - "# A[7] * B[4]\n\t" - "mulx 32(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[7] * B[5]\n\t" - "mulx 40(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[7] * B[6]\n\t" - "mulx 48(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[7] * B[7]\n\t" - "mulx 56(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "movq %%rax, 88(%[t])\n\t" - "movq %%r10, 96(%[t])\n\t" - "movq %%r11, 104(%[t])\n\t" - "movq %%r12, 112(%[t])\n\t" - "movq 128(%[t]), %%r14\n\t" - "movq 136(%[t]), %%rax\n\t" - "movq 144(%[t]), %%r10\n\t" - "movq 152(%[t]), %%r11\n\t" - "# A[7] * B[8]\n\t" - "mulx 64(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[7] * B[9]\n\t" - "mulx 72(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[7] * B[10]\n\t" - "mulx 80(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[7] * B[11]\n\t" - "mulx 88(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "movq %%r13, 120(%[t])\n\t" - "movq %%r14, 128(%[t])\n\t" - "movq %%rax, 136(%[t])\n\t" - "movq %%r10, 144(%[t])\n\t" - "movq 160(%[t]), %%r12\n\t" - "movq 168(%[t]), %%r13\n\t" - "movq 176(%[t]), %%r14\n\t" - "movq 184(%[t]), %%rax\n\t" - "# A[7] * B[12]\n\t" - "mulx 96(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[7] * B[13]\n\t" - "mulx 104(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[7] * B[14]\n\t" - "mulx 112(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[7] * B[15]\n\t" - "mulx 120(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "movq %%r11, 152(%[t])\n\t" - "movq %%r12, 160(%[t])\n\t" - "movq %%r13, 168(%[t])\n\t" - "movq %%r14, 176(%[t])\n\t" - "movq 192(%[t]), %%r10\n\t" - "movq 200(%[t]), %%r11\n\t" - "movq 208(%[t]), %%r12\n\t" - "movq 216(%[t]), %%r13\n\t" - "# A[7] * B[16]\n\t" - "mulx 128(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[7] * B[17]\n\t" - "mulx 136(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[7] * B[18]\n\t" - "mulx 144(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[7] * B[19]\n\t" - "mulx 152(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "movq %%rax, 184(%[t])\n\t" - "movq %%r10, 192(%[t])\n\t" - "movq %%r11, 200(%[t])\n\t" - "movq %%r12, 208(%[t])\n\t" - "movq 224(%[t]), %%r14\n\t" - "movq 232(%[t]), %%rax\n\t" - "movq 240(%[t]), %%r10\n\t" - "# A[7] * B[20]\n\t" - "mulx 160(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[7] * B[21]\n\t" - "mulx 168(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[7] * B[22]\n\t" - "mulx 176(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[7] * B[23]\n\t" - "mulx 184(%[b]), %%r8, %%r9\n\t" - "movq %%r15, %%r11\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "adcxq %%rcx, %%r11\n\t" - "movq %%r15, %%rcx\n\t" - "adoxq %%r15, %%rcx\n\t" - "adcxq %%r15, %%rcx\n\t" - "movq %%r13, 216(%[t])\n\t" - "movq %%r14, 224(%[t])\n\t" - "movq %%rax, 232(%[t])\n\t" - "movq %%r10, 240(%[t])\n\t" - "movq %%r11, 248(%[t])\n\t" - "movq 64(%[a]), %%rdx\n\t" - "xorq %%r15, %%r15\n\t" - "movq 64(%[t]), %%r12\n\t" - "movq 72(%[t]), %%r13\n\t" - "movq 80(%[t]), %%r14\n\t" - "movq 88(%[t]), %%rax\n\t" - "movq 96(%[t]), %%r10\n\t" - "# A[8] * B[0]\n\t" - "mulx 0(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[8] * B[1]\n\t" - "mulx 8(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[8] * B[2]\n\t" - "mulx 16(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[8] * B[3]\n\t" - "mulx 24(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "movq %%r12, 64(%[t])\n\t" - "movq %%r13, 72(%[t])\n\t" - "movq %%r14, 80(%[t])\n\t" - "movq %%rax, 88(%[t])\n\t" - "movq 104(%[t]), %%r11\n\t" - "movq 112(%[t]), %%r12\n\t" - "movq 120(%[t]), %%r13\n\t" - "movq 128(%[t]), %%r14\n\t" - "# A[8] * B[4]\n\t" - "mulx 32(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[8] * B[5]\n\t" - "mulx 40(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[8] * B[6]\n\t" - "mulx 48(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[8] * B[7]\n\t" - "mulx 56(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "movq %%r10, 96(%[t])\n\t" - "movq %%r11, 104(%[t])\n\t" - "movq %%r12, 112(%[t])\n\t" - "movq %%r13, 120(%[t])\n\t" - "movq 136(%[t]), %%rax\n\t" - "movq 144(%[t]), %%r10\n\t" - "movq 152(%[t]), %%r11\n\t" - "movq 160(%[t]), %%r12\n\t" - "# A[8] * B[8]\n\t" - "mulx 64(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[8] * B[9]\n\t" - "mulx 72(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[8] * B[10]\n\t" - "mulx 80(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[8] * B[11]\n\t" - "mulx 88(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "movq %%r14, 128(%[t])\n\t" - "movq %%rax, 136(%[t])\n\t" - "movq %%r10, 144(%[t])\n\t" - "movq %%r11, 152(%[t])\n\t" - "movq 168(%[t]), %%r13\n\t" - "movq 176(%[t]), %%r14\n\t" - "movq 184(%[t]), %%rax\n\t" - "movq 192(%[t]), %%r10\n\t" - "# A[8] * B[12]\n\t" - "mulx 96(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[8] * B[13]\n\t" - "mulx 104(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[8] * B[14]\n\t" - "mulx 112(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[8] * B[15]\n\t" - "mulx 120(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "movq %%r12, 160(%[t])\n\t" - "movq %%r13, 168(%[t])\n\t" - "movq %%r14, 176(%[t])\n\t" - "movq %%rax, 184(%[t])\n\t" - "movq 200(%[t]), %%r11\n\t" - "movq 208(%[t]), %%r12\n\t" - "movq 216(%[t]), %%r13\n\t" - "movq 224(%[t]), %%r14\n\t" - "# A[8] * B[16]\n\t" - "mulx 128(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[8] * B[17]\n\t" - "mulx 136(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[8] * B[18]\n\t" - "mulx 144(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[8] * B[19]\n\t" - "mulx 152(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "movq %%r10, 192(%[t])\n\t" - "movq %%r11, 200(%[t])\n\t" - "movq %%r12, 208(%[t])\n\t" - "movq %%r13, 216(%[t])\n\t" - "movq 232(%[t]), %%rax\n\t" - "movq 240(%[t]), %%r10\n\t" - "movq 248(%[t]), %%r11\n\t" - "# A[8] * B[20]\n\t" - "mulx 160(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[8] * B[21]\n\t" - "mulx 168(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[8] * B[22]\n\t" - "mulx 176(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[8] * B[23]\n\t" - "mulx 184(%[b]), %%r8, %%r9\n\t" - "movq %%r15, %%r12\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "adcxq %%rcx, %%r12\n\t" - "movq %%r15, %%rcx\n\t" - "adoxq %%r15, %%rcx\n\t" - "adcxq %%r15, %%rcx\n\t" - "movq %%r14, 224(%[t])\n\t" - "movq %%rax, 232(%[t])\n\t" - "movq %%r10, 240(%[t])\n\t" - "movq %%r11, 248(%[t])\n\t" - "movq %%r12, 256(%[t])\n\t" - "movq 72(%[a]), %%rdx\n\t" - "xorq %%r15, %%r15\n\t" - "movq 72(%[t]), %%r13\n\t" - "movq 80(%[t]), %%r14\n\t" - "movq 88(%[t]), %%rax\n\t" - "movq 96(%[t]), %%r10\n\t" - "movq 104(%[t]), %%r11\n\t" - "# A[9] * B[0]\n\t" - "mulx 0(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[9] * B[1]\n\t" - "mulx 8(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[9] * B[2]\n\t" - "mulx 16(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[9] * B[3]\n\t" - "mulx 24(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "movq %%r13, 72(%[t])\n\t" - "movq %%r14, 80(%[t])\n\t" - "movq %%rax, 88(%[t])\n\t" - "movq %%r10, 96(%[t])\n\t" - "movq 112(%[t]), %%r12\n\t" - "movq 120(%[t]), %%r13\n\t" - "movq 128(%[t]), %%r14\n\t" - "movq 136(%[t]), %%rax\n\t" - "# A[9] * B[4]\n\t" - "mulx 32(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[9] * B[5]\n\t" - "mulx 40(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[9] * B[6]\n\t" - "mulx 48(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[9] * B[7]\n\t" - "mulx 56(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "movq %%r11, 104(%[t])\n\t" - "movq %%r12, 112(%[t])\n\t" - "movq %%r13, 120(%[t])\n\t" - "movq %%r14, 128(%[t])\n\t" - "movq 144(%[t]), %%r10\n\t" - "movq 152(%[t]), %%r11\n\t" - "movq 160(%[t]), %%r12\n\t" - "movq 168(%[t]), %%r13\n\t" - "# A[9] * B[8]\n\t" - "mulx 64(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[9] * B[9]\n\t" - "mulx 72(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[9] * B[10]\n\t" - "mulx 80(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[9] * B[11]\n\t" - "mulx 88(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "movq %%rax, 136(%[t])\n\t" - "movq %%r10, 144(%[t])\n\t" - "movq %%r11, 152(%[t])\n\t" - "movq %%r12, 160(%[t])\n\t" - "movq 176(%[t]), %%r14\n\t" - "movq 184(%[t]), %%rax\n\t" - "movq 192(%[t]), %%r10\n\t" - "movq 200(%[t]), %%r11\n\t" - "# A[9] * B[12]\n\t" - "mulx 96(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[9] * B[13]\n\t" - "mulx 104(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[9] * B[14]\n\t" - "mulx 112(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[9] * B[15]\n\t" - "mulx 120(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "movq %%r13, 168(%[t])\n\t" - "movq %%r14, 176(%[t])\n\t" - "movq %%rax, 184(%[t])\n\t" - "movq %%r10, 192(%[t])\n\t" - "movq 208(%[t]), %%r12\n\t" - "movq 216(%[t]), %%r13\n\t" - "movq 224(%[t]), %%r14\n\t" - "movq 232(%[t]), %%rax\n\t" - "# A[9] * B[16]\n\t" - "mulx 128(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[9] * B[17]\n\t" - "mulx 136(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[9] * B[18]\n\t" - "mulx 144(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[9] * B[19]\n\t" - "mulx 152(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "movq %%r11, 200(%[t])\n\t" - "movq %%r12, 208(%[t])\n\t" - "movq %%r13, 216(%[t])\n\t" - "movq %%r14, 224(%[t])\n\t" - "movq 240(%[t]), %%r10\n\t" - "movq 248(%[t]), %%r11\n\t" - "movq 256(%[t]), %%r12\n\t" - "# A[9] * B[20]\n\t" - "mulx 160(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[9] * B[21]\n\t" - "mulx 168(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[9] * B[22]\n\t" - "mulx 176(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[9] * B[23]\n\t" - "mulx 184(%[b]), %%r8, %%r9\n\t" - "movq %%r15, %%r13\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "adcxq %%rcx, %%r13\n\t" - "movq %%r15, %%rcx\n\t" - "adoxq %%r15, %%rcx\n\t" - "adcxq %%r15, %%rcx\n\t" - "movq %%rax, 232(%[t])\n\t" - "movq %%r10, 240(%[t])\n\t" - "movq %%r11, 248(%[t])\n\t" - "movq %%r12, 256(%[t])\n\t" - "movq %%r13, 264(%[t])\n\t" - "movq 80(%[a]), %%rdx\n\t" - "xorq %%r15, %%r15\n\t" - "movq 80(%[t]), %%r14\n\t" - "movq 88(%[t]), %%rax\n\t" - "movq 96(%[t]), %%r10\n\t" - "movq 104(%[t]), %%r11\n\t" - "movq 112(%[t]), %%r12\n\t" - "# A[10] * B[0]\n\t" - "mulx 0(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[10] * B[1]\n\t" - "mulx 8(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[10] * B[2]\n\t" - "mulx 16(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[10] * B[3]\n\t" - "mulx 24(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "movq %%r14, 80(%[t])\n\t" - "movq %%rax, 88(%[t])\n\t" - "movq %%r10, 96(%[t])\n\t" - "movq %%r11, 104(%[t])\n\t" - "movq 120(%[t]), %%r13\n\t" - "movq 128(%[t]), %%r14\n\t" - "movq 136(%[t]), %%rax\n\t" - "movq 144(%[t]), %%r10\n\t" - "# A[10] * B[4]\n\t" - "mulx 32(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[10] * B[5]\n\t" - "mulx 40(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[10] * B[6]\n\t" - "mulx 48(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[10] * B[7]\n\t" - "mulx 56(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "movq %%r12, 112(%[t])\n\t" - "movq %%r13, 120(%[t])\n\t" - "movq %%r14, 128(%[t])\n\t" - "movq %%rax, 136(%[t])\n\t" - "movq 152(%[t]), %%r11\n\t" - "movq 160(%[t]), %%r12\n\t" - "movq 168(%[t]), %%r13\n\t" - "movq 176(%[t]), %%r14\n\t" - "# A[10] * B[8]\n\t" - "mulx 64(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[10] * B[9]\n\t" - "mulx 72(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[10] * B[10]\n\t" - "mulx 80(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[10] * B[11]\n\t" - "mulx 88(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "movq %%r10, 144(%[t])\n\t" - "movq %%r11, 152(%[t])\n\t" - "movq %%r12, 160(%[t])\n\t" - "movq %%r13, 168(%[t])\n\t" - "movq 184(%[t]), %%rax\n\t" - "movq 192(%[t]), %%r10\n\t" - "movq 200(%[t]), %%r11\n\t" - "movq 208(%[t]), %%r12\n\t" - "# A[10] * B[12]\n\t" - "mulx 96(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[10] * B[13]\n\t" - "mulx 104(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[10] * B[14]\n\t" - "mulx 112(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[10] * B[15]\n\t" - "mulx 120(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "movq %%r14, 176(%[t])\n\t" - "movq %%rax, 184(%[t])\n\t" - "movq %%r10, 192(%[t])\n\t" - "movq %%r11, 200(%[t])\n\t" - "movq 216(%[t]), %%r13\n\t" - "movq 224(%[t]), %%r14\n\t" - "movq 232(%[t]), %%rax\n\t" - "movq 240(%[t]), %%r10\n\t" - "# A[10] * B[16]\n\t" - "mulx 128(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[10] * B[17]\n\t" - "mulx 136(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[10] * B[18]\n\t" - "mulx 144(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[10] * B[19]\n\t" - "mulx 152(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "movq %%r12, 208(%[t])\n\t" - "movq %%r13, 216(%[t])\n\t" - "movq %%r14, 224(%[t])\n\t" - "movq %%rax, 232(%[t])\n\t" - "movq 248(%[t]), %%r11\n\t" - "movq 256(%[t]), %%r12\n\t" - "movq 264(%[t]), %%r13\n\t" - "# A[10] * B[20]\n\t" - "mulx 160(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[10] * B[21]\n\t" - "mulx 168(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[10] * B[22]\n\t" - "mulx 176(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[10] * B[23]\n\t" - "mulx 184(%[b]), %%r8, %%r9\n\t" - "movq %%r15, %%r14\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "adcxq %%rcx, %%r14\n\t" - "movq %%r15, %%rcx\n\t" - "adoxq %%r15, %%rcx\n\t" - "adcxq %%r15, %%rcx\n\t" - "movq %%r10, 240(%[t])\n\t" - "movq %%r11, 248(%[t])\n\t" - "movq %%r12, 256(%[t])\n\t" - "movq %%r13, 264(%[t])\n\t" - "movq %%r14, 272(%[t])\n\t" - "movq 88(%[a]), %%rdx\n\t" - "xorq %%r15, %%r15\n\t" - "movq 88(%[t]), %%rax\n\t" - "movq 96(%[t]), %%r10\n\t" - "movq 104(%[t]), %%r11\n\t" - "movq 112(%[t]), %%r12\n\t" - "movq 120(%[t]), %%r13\n\t" - "# A[11] * B[0]\n\t" - "mulx 0(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[11] * B[1]\n\t" - "mulx 8(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[11] * B[2]\n\t" - "mulx 16(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[11] * B[3]\n\t" - "mulx 24(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "movq %%rax, 88(%[t])\n\t" - "movq %%r10, 96(%[t])\n\t" - "movq %%r11, 104(%[t])\n\t" - "movq %%r12, 112(%[t])\n\t" - "movq 128(%[t]), %%r14\n\t" - "movq 136(%[t]), %%rax\n\t" - "movq 144(%[t]), %%r10\n\t" - "movq 152(%[t]), %%r11\n\t" - "# A[11] * B[4]\n\t" - "mulx 32(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[11] * B[5]\n\t" - "mulx 40(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[11] * B[6]\n\t" - "mulx 48(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[11] * B[7]\n\t" - "mulx 56(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "movq %%r13, 120(%[t])\n\t" - "movq %%r14, 128(%[t])\n\t" - "movq %%rax, 136(%[t])\n\t" - "movq %%r10, 144(%[t])\n\t" - "movq 160(%[t]), %%r12\n\t" - "movq 168(%[t]), %%r13\n\t" - "movq 176(%[t]), %%r14\n\t" - "movq 184(%[t]), %%rax\n\t" - "# A[11] * B[8]\n\t" - "mulx 64(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[11] * B[9]\n\t" - "mulx 72(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[11] * B[10]\n\t" - "mulx 80(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[11] * B[11]\n\t" - "mulx 88(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "movq %%r11, 152(%[t])\n\t" - "movq %%r12, 160(%[t])\n\t" - "movq %%r13, 168(%[t])\n\t" - "movq %%r14, 176(%[t])\n\t" - "movq 192(%[t]), %%r10\n\t" - "movq 200(%[t]), %%r11\n\t" - "movq 208(%[t]), %%r12\n\t" - "movq 216(%[t]), %%r13\n\t" - "# A[11] * B[12]\n\t" - "mulx 96(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[11] * B[13]\n\t" - "mulx 104(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[11] * B[14]\n\t" - "mulx 112(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[11] * B[15]\n\t" - "mulx 120(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "movq %%rax, 184(%[t])\n\t" - "movq %%r10, 192(%[t])\n\t" - "movq %%r11, 200(%[t])\n\t" - "movq %%r12, 208(%[t])\n\t" - "movq 224(%[t]), %%r14\n\t" - "movq 232(%[t]), %%rax\n\t" - "movq 240(%[t]), %%r10\n\t" - "movq 248(%[t]), %%r11\n\t" - "# A[11] * B[16]\n\t" - "mulx 128(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[11] * B[17]\n\t" - "mulx 136(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[11] * B[18]\n\t" - "mulx 144(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[11] * B[19]\n\t" - "mulx 152(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "movq %%r13, 216(%[t])\n\t" - "movq %%r14, 224(%[t])\n\t" - "movq %%rax, 232(%[t])\n\t" - "movq %%r10, 240(%[t])\n\t" - "movq 256(%[t]), %%r12\n\t" - "movq 264(%[t]), %%r13\n\t" - "movq 272(%[t]), %%r14\n\t" - "# A[11] * B[20]\n\t" - "mulx 160(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[11] * B[21]\n\t" - "mulx 168(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[11] * B[22]\n\t" - "mulx 176(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[11] * B[23]\n\t" - "mulx 184(%[b]), %%r8, %%r9\n\t" - "movq %%r15, %%rax\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "adcxq %%rcx, %%rax\n\t" - "movq %%r15, %%rcx\n\t" - "adoxq %%r15, %%rcx\n\t" - "adcxq %%r15, %%rcx\n\t" - "movq %%r11, 248(%[t])\n\t" - "movq %%r12, 256(%[t])\n\t" - "movq %%r13, 264(%[t])\n\t" - "movq %%r14, 272(%[t])\n\t" - "movq %%rax, 280(%[t])\n\t" - "movq 96(%[a]), %%rdx\n\t" - "xorq %%r15, %%r15\n\t" - "movq 96(%[t]), %%r10\n\t" - "movq 104(%[t]), %%r11\n\t" - "movq 112(%[t]), %%r12\n\t" - "movq 120(%[t]), %%r13\n\t" - "movq 128(%[t]), %%r14\n\t" - "# A[12] * B[0]\n\t" - "mulx 0(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[12] * B[1]\n\t" - "mulx 8(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[12] * B[2]\n\t" - "mulx 16(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[12] * B[3]\n\t" - "mulx 24(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "movq %%r10, 96(%[t])\n\t" - "movq %%r11, 104(%[t])\n\t" - "movq %%r12, 112(%[t])\n\t" - "movq %%r13, 120(%[t])\n\t" - "movq 136(%[t]), %%rax\n\t" - "movq 144(%[t]), %%r10\n\t" - "movq 152(%[t]), %%r11\n\t" - "movq 160(%[t]), %%r12\n\t" - "# A[12] * B[4]\n\t" - "mulx 32(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[12] * B[5]\n\t" - "mulx 40(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[12] * B[6]\n\t" - "mulx 48(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[12] * B[7]\n\t" - "mulx 56(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "movq %%r14, 128(%[t])\n\t" - "movq %%rax, 136(%[t])\n\t" - "movq %%r10, 144(%[t])\n\t" - "movq %%r11, 152(%[t])\n\t" - "movq 168(%[t]), %%r13\n\t" - "movq 176(%[t]), %%r14\n\t" - "movq 184(%[t]), %%rax\n\t" - "movq 192(%[t]), %%r10\n\t" - "# A[12] * B[8]\n\t" - "mulx 64(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[12] * B[9]\n\t" - "mulx 72(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[12] * B[10]\n\t" - "mulx 80(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[12] * B[11]\n\t" - "mulx 88(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "movq %%r12, 160(%[t])\n\t" - "movq %%r13, 168(%[t])\n\t" - "movq %%r14, 176(%[t])\n\t" - "movq %%rax, 184(%[t])\n\t" - "movq 200(%[t]), %%r11\n\t" - "movq 208(%[t]), %%r12\n\t" - "movq 216(%[t]), %%r13\n\t" - "movq 224(%[t]), %%r14\n\t" - "# A[12] * B[12]\n\t" - "mulx 96(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[12] * B[13]\n\t" - "mulx 104(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[12] * B[14]\n\t" - "mulx 112(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[12] * B[15]\n\t" - "mulx 120(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "movq %%r10, 192(%[t])\n\t" - "movq %%r11, 200(%[t])\n\t" - "movq %%r12, 208(%[t])\n\t" - "movq %%r13, 216(%[t])\n\t" - "movq 232(%[t]), %%rax\n\t" - "movq 240(%[t]), %%r10\n\t" - "movq 248(%[t]), %%r11\n\t" - "movq 256(%[t]), %%r12\n\t" - "# A[12] * B[16]\n\t" - "mulx 128(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[12] * B[17]\n\t" - "mulx 136(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[12] * B[18]\n\t" - "mulx 144(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[12] * B[19]\n\t" - "mulx 152(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "movq %%r14, 224(%[t])\n\t" - "movq %%rax, 232(%[t])\n\t" - "movq %%r10, 240(%[t])\n\t" - "movq %%r11, 248(%[t])\n\t" - "movq 264(%[t]), %%r13\n\t" - "movq 272(%[t]), %%r14\n\t" - "movq 280(%[t]), %%rax\n\t" - "# A[12] * B[20]\n\t" - "mulx 160(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[12] * B[21]\n\t" - "mulx 168(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[12] * B[22]\n\t" - "mulx 176(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[12] * B[23]\n\t" - "mulx 184(%[b]), %%r8, %%r9\n\t" - "movq %%r15, %%r10\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "adcxq %%rcx, %%r10\n\t" - "movq %%r15, %%rcx\n\t" - "adoxq %%r15, %%rcx\n\t" - "adcxq %%r15, %%rcx\n\t" - "movq %%r12, 256(%[t])\n\t" - "movq %%r13, 264(%[t])\n\t" - "movq %%r14, 272(%[t])\n\t" - "movq %%rax, 280(%[t])\n\t" - "movq %%r10, 288(%[t])\n\t" - "movq 104(%[a]), %%rdx\n\t" - "xorq %%r15, %%r15\n\t" - "movq 104(%[t]), %%r11\n\t" - "movq 112(%[t]), %%r12\n\t" - "movq 120(%[t]), %%r13\n\t" - "movq 128(%[t]), %%r14\n\t" - "movq 136(%[t]), %%rax\n\t" - "# A[13] * B[0]\n\t" - "mulx 0(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[13] * B[1]\n\t" - "mulx 8(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[13] * B[2]\n\t" - "mulx 16(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[13] * B[3]\n\t" - "mulx 24(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "movq %%r11, 104(%[t])\n\t" - "movq %%r12, 112(%[t])\n\t" - "movq %%r13, 120(%[t])\n\t" - "movq %%r14, 128(%[t])\n\t" - "movq 144(%[t]), %%r10\n\t" - "movq 152(%[t]), %%r11\n\t" - "movq 160(%[t]), %%r12\n\t" - "movq 168(%[t]), %%r13\n\t" - "# A[13] * B[4]\n\t" - "mulx 32(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[13] * B[5]\n\t" - "mulx 40(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[13] * B[6]\n\t" - "mulx 48(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[13] * B[7]\n\t" - "mulx 56(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "movq %%rax, 136(%[t])\n\t" - "movq %%r10, 144(%[t])\n\t" - "movq %%r11, 152(%[t])\n\t" - "movq %%r12, 160(%[t])\n\t" - "movq 176(%[t]), %%r14\n\t" - "movq 184(%[t]), %%rax\n\t" - "movq 192(%[t]), %%r10\n\t" - "movq 200(%[t]), %%r11\n\t" - "# A[13] * B[8]\n\t" - "mulx 64(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[13] * B[9]\n\t" - "mulx 72(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[13] * B[10]\n\t" - "mulx 80(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[13] * B[11]\n\t" - "mulx 88(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "movq %%r13, 168(%[t])\n\t" - "movq %%r14, 176(%[t])\n\t" - "movq %%rax, 184(%[t])\n\t" - "movq %%r10, 192(%[t])\n\t" - "movq 208(%[t]), %%r12\n\t" - "movq 216(%[t]), %%r13\n\t" - "movq 224(%[t]), %%r14\n\t" - "movq 232(%[t]), %%rax\n\t" - "# A[13] * B[12]\n\t" - "mulx 96(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[13] * B[13]\n\t" - "mulx 104(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[13] * B[14]\n\t" - "mulx 112(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[13] * B[15]\n\t" - "mulx 120(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "movq %%r11, 200(%[t])\n\t" - "movq %%r12, 208(%[t])\n\t" - "movq %%r13, 216(%[t])\n\t" - "movq %%r14, 224(%[t])\n\t" - "movq 240(%[t]), %%r10\n\t" - "movq 248(%[t]), %%r11\n\t" - "movq 256(%[t]), %%r12\n\t" - "movq 264(%[t]), %%r13\n\t" - "# A[13] * B[16]\n\t" - "mulx 128(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[13] * B[17]\n\t" - "mulx 136(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[13] * B[18]\n\t" - "mulx 144(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[13] * B[19]\n\t" - "mulx 152(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "movq %%rax, 232(%[t])\n\t" - "movq %%r10, 240(%[t])\n\t" - "movq %%r11, 248(%[t])\n\t" - "movq %%r12, 256(%[t])\n\t" - "movq 272(%[t]), %%r14\n\t" - "movq 280(%[t]), %%rax\n\t" - "movq 288(%[t]), %%r10\n\t" - "# A[13] * B[20]\n\t" - "mulx 160(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[13] * B[21]\n\t" - "mulx 168(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[13] * B[22]\n\t" - "mulx 176(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[13] * B[23]\n\t" - "mulx 184(%[b]), %%r8, %%r9\n\t" - "movq %%r15, %%r11\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "adcxq %%rcx, %%r11\n\t" - "movq %%r15, %%rcx\n\t" - "adoxq %%r15, %%rcx\n\t" - "adcxq %%r15, %%rcx\n\t" - "movq %%r13, 264(%[t])\n\t" - "movq %%r14, 272(%[t])\n\t" - "movq %%rax, 280(%[t])\n\t" - "movq %%r10, 288(%[t])\n\t" - "movq %%r11, 296(%[t])\n\t" - "movq 112(%[a]), %%rdx\n\t" - "xorq %%r15, %%r15\n\t" - "movq 112(%[t]), %%r12\n\t" - "movq 120(%[t]), %%r13\n\t" - "movq 128(%[t]), %%r14\n\t" - "movq 136(%[t]), %%rax\n\t" - "movq 144(%[t]), %%r10\n\t" - "# A[14] * B[0]\n\t" - "mulx 0(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[14] * B[1]\n\t" - "mulx 8(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[14] * B[2]\n\t" - "mulx 16(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[14] * B[3]\n\t" - "mulx 24(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "movq %%r12, 112(%[t])\n\t" - "movq %%r13, 120(%[t])\n\t" - "movq %%r14, 128(%[t])\n\t" - "movq %%rax, 136(%[t])\n\t" - "movq 152(%[t]), %%r11\n\t" - "movq 160(%[t]), %%r12\n\t" - "movq 168(%[t]), %%r13\n\t" - "movq 176(%[t]), %%r14\n\t" - "# A[14] * B[4]\n\t" - "mulx 32(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[14] * B[5]\n\t" - "mulx 40(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[14] * B[6]\n\t" - "mulx 48(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[14] * B[7]\n\t" - "mulx 56(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "movq %%r10, 144(%[t])\n\t" - "movq %%r11, 152(%[t])\n\t" - "movq %%r12, 160(%[t])\n\t" - "movq %%r13, 168(%[t])\n\t" - "movq 184(%[t]), %%rax\n\t" - "movq 192(%[t]), %%r10\n\t" - "movq 200(%[t]), %%r11\n\t" - "movq 208(%[t]), %%r12\n\t" - "# A[14] * B[8]\n\t" - "mulx 64(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[14] * B[9]\n\t" - "mulx 72(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[14] * B[10]\n\t" - "mulx 80(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[14] * B[11]\n\t" - "mulx 88(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "movq %%r14, 176(%[t])\n\t" - "movq %%rax, 184(%[t])\n\t" - "movq %%r10, 192(%[t])\n\t" - "movq %%r11, 200(%[t])\n\t" - "movq 216(%[t]), %%r13\n\t" - "movq 224(%[t]), %%r14\n\t" - "movq 232(%[t]), %%rax\n\t" - "movq 240(%[t]), %%r10\n\t" - "# A[14] * B[12]\n\t" - "mulx 96(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[14] * B[13]\n\t" - "mulx 104(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[14] * B[14]\n\t" - "mulx 112(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[14] * B[15]\n\t" - "mulx 120(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "movq %%r12, 208(%[t])\n\t" - "movq %%r13, 216(%[t])\n\t" - "movq %%r14, 224(%[t])\n\t" - "movq %%rax, 232(%[t])\n\t" - "movq 248(%[t]), %%r11\n\t" - "movq 256(%[t]), %%r12\n\t" - "movq 264(%[t]), %%r13\n\t" - "movq 272(%[t]), %%r14\n\t" - "# A[14] * B[16]\n\t" - "mulx 128(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[14] * B[17]\n\t" - "mulx 136(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[14] * B[18]\n\t" - "mulx 144(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[14] * B[19]\n\t" - "mulx 152(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "movq %%r10, 240(%[t])\n\t" - "movq %%r11, 248(%[t])\n\t" - "movq %%r12, 256(%[t])\n\t" - "movq %%r13, 264(%[t])\n\t" - "movq 280(%[t]), %%rax\n\t" - "movq 288(%[t]), %%r10\n\t" - "movq 296(%[t]), %%r11\n\t" - "# A[14] * B[20]\n\t" - "mulx 160(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[14] * B[21]\n\t" - "mulx 168(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[14] * B[22]\n\t" - "mulx 176(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[14] * B[23]\n\t" - "mulx 184(%[b]), %%r8, %%r9\n\t" - "movq %%r15, %%r12\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "adcxq %%rcx, %%r12\n\t" - "movq %%r15, %%rcx\n\t" - "adoxq %%r15, %%rcx\n\t" - "adcxq %%r15, %%rcx\n\t" - "movq %%r14, 272(%[t])\n\t" - "movq %%rax, 280(%[t])\n\t" - "movq %%r10, 288(%[t])\n\t" - "movq %%r11, 296(%[t])\n\t" - "movq %%r12, 304(%[t])\n\t" - "movq 120(%[a]), %%rdx\n\t" - "xorq %%r15, %%r15\n\t" - "movq 120(%[t]), %%r13\n\t" - "movq 128(%[t]), %%r14\n\t" - "movq 136(%[t]), %%rax\n\t" - "movq 144(%[t]), %%r10\n\t" - "movq 152(%[t]), %%r11\n\t" - "# A[15] * B[0]\n\t" - "mulx 0(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[15] * B[1]\n\t" - "mulx 8(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[15] * B[2]\n\t" - "mulx 16(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[15] * B[3]\n\t" - "mulx 24(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "movq %%r13, 120(%[t])\n\t" - "movq %%r14, 128(%[t])\n\t" - "movq %%rax, 136(%[t])\n\t" - "movq %%r10, 144(%[t])\n\t" - "movq 160(%[t]), %%r12\n\t" - "movq 168(%[t]), %%r13\n\t" - "movq 176(%[t]), %%r14\n\t" - "movq 184(%[t]), %%rax\n\t" - "# A[15] * B[4]\n\t" - "mulx 32(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[15] * B[5]\n\t" - "mulx 40(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[15] * B[6]\n\t" - "mulx 48(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[15] * B[7]\n\t" - "mulx 56(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "movq %%r11, 152(%[t])\n\t" - "movq %%r12, 160(%[t])\n\t" - "movq %%r13, 168(%[t])\n\t" - "movq %%r14, 176(%[t])\n\t" - "movq 192(%[t]), %%r10\n\t" - "movq 200(%[t]), %%r11\n\t" - "movq 208(%[t]), %%r12\n\t" - "movq 216(%[t]), %%r13\n\t" - "# A[15] * B[8]\n\t" - "mulx 64(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[15] * B[9]\n\t" - "mulx 72(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[15] * B[10]\n\t" - "mulx 80(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[15] * B[11]\n\t" - "mulx 88(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "movq %%rax, 184(%[t])\n\t" - "movq %%r10, 192(%[t])\n\t" - "movq %%r11, 200(%[t])\n\t" - "movq %%r12, 208(%[t])\n\t" - "movq 224(%[t]), %%r14\n\t" - "movq 232(%[t]), %%rax\n\t" - "movq 240(%[t]), %%r10\n\t" - "movq 248(%[t]), %%r11\n\t" - "# A[15] * B[12]\n\t" - "mulx 96(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[15] * B[13]\n\t" - "mulx 104(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[15] * B[14]\n\t" - "mulx 112(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[15] * B[15]\n\t" - "mulx 120(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "movq %%r13, 216(%[t])\n\t" - "movq %%r14, 224(%[t])\n\t" - "movq %%rax, 232(%[t])\n\t" - "movq %%r10, 240(%[t])\n\t" - "movq 256(%[t]), %%r12\n\t" - "movq 264(%[t]), %%r13\n\t" - "movq 272(%[t]), %%r14\n\t" - "movq 280(%[t]), %%rax\n\t" - "# A[15] * B[16]\n\t" - "mulx 128(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[15] * B[17]\n\t" - "mulx 136(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[15] * B[18]\n\t" - "mulx 144(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[15] * B[19]\n\t" - "mulx 152(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "movq %%r11, 248(%[t])\n\t" - "movq %%r12, 256(%[t])\n\t" - "movq %%r13, 264(%[t])\n\t" - "movq %%r14, 272(%[t])\n\t" - "movq 288(%[t]), %%r10\n\t" - "movq 296(%[t]), %%r11\n\t" - "movq 304(%[t]), %%r12\n\t" - "# A[15] * B[20]\n\t" - "mulx 160(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[15] * B[21]\n\t" - "mulx 168(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[15] * B[22]\n\t" - "mulx 176(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[15] * B[23]\n\t" - "mulx 184(%[b]), %%r8, %%r9\n\t" - "movq %%r15, %%r13\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "adcxq %%rcx, %%r13\n\t" - "movq %%r15, %%rcx\n\t" - "adoxq %%r15, %%rcx\n\t" - "adcxq %%r15, %%rcx\n\t" - "movq %%rax, 280(%[t])\n\t" - "movq %%r10, 288(%[t])\n\t" - "movq %%r11, 296(%[t])\n\t" - "movq %%r12, 304(%[t])\n\t" - "movq %%r13, 312(%[t])\n\t" - "movq 128(%[a]), %%rdx\n\t" - "xorq %%r15, %%r15\n\t" - "movq 128(%[t]), %%r14\n\t" - "movq 136(%[t]), %%rax\n\t" - "movq 144(%[t]), %%r10\n\t" - "movq 152(%[t]), %%r11\n\t" - "movq 160(%[t]), %%r12\n\t" - "# A[16] * B[0]\n\t" - "mulx 0(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[16] * B[1]\n\t" - "mulx 8(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[16] * B[2]\n\t" - "mulx 16(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[16] * B[3]\n\t" - "mulx 24(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "movq %%r14, 128(%[t])\n\t" - "movq %%rax, 136(%[t])\n\t" - "movq %%r10, 144(%[t])\n\t" - "movq %%r11, 152(%[t])\n\t" - "movq 168(%[t]), %%r13\n\t" - "movq 176(%[t]), %%r14\n\t" - "movq 184(%[t]), %%rax\n\t" - "movq 192(%[t]), %%r10\n\t" - "# A[16] * B[4]\n\t" - "mulx 32(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[16] * B[5]\n\t" - "mulx 40(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[16] * B[6]\n\t" - "mulx 48(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[16] * B[7]\n\t" - "mulx 56(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "movq %%r12, 160(%[t])\n\t" - "movq %%r13, 168(%[t])\n\t" - "movq %%r14, 176(%[t])\n\t" - "movq %%rax, 184(%[t])\n\t" - "movq 200(%[t]), %%r11\n\t" - "movq 208(%[t]), %%r12\n\t" - "movq 216(%[t]), %%r13\n\t" - "movq 224(%[t]), %%r14\n\t" - "# A[16] * B[8]\n\t" - "mulx 64(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[16] * B[9]\n\t" - "mulx 72(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[16] * B[10]\n\t" - "mulx 80(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[16] * B[11]\n\t" - "mulx 88(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "movq %%r10, 192(%[t])\n\t" - "movq %%r11, 200(%[t])\n\t" - "movq %%r12, 208(%[t])\n\t" - "movq %%r13, 216(%[t])\n\t" - "movq 232(%[t]), %%rax\n\t" - "movq 240(%[t]), %%r10\n\t" - "movq 248(%[t]), %%r11\n\t" - "movq 256(%[t]), %%r12\n\t" - "# A[16] * B[12]\n\t" - "mulx 96(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[16] * B[13]\n\t" - "mulx 104(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[16] * B[14]\n\t" - "mulx 112(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[16] * B[15]\n\t" - "mulx 120(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "movq %%r14, 224(%[t])\n\t" - "movq %%rax, 232(%[t])\n\t" - "movq %%r10, 240(%[t])\n\t" - "movq %%r11, 248(%[t])\n\t" - "movq 264(%[t]), %%r13\n\t" - "movq 272(%[t]), %%r14\n\t" - "movq 280(%[t]), %%rax\n\t" - "movq 288(%[t]), %%r10\n\t" - "# A[16] * B[16]\n\t" - "mulx 128(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[16] * B[17]\n\t" - "mulx 136(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[16] * B[18]\n\t" - "mulx 144(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[16] * B[19]\n\t" - "mulx 152(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "movq %%r12, 256(%[t])\n\t" - "movq %%r13, 264(%[t])\n\t" - "movq %%r14, 272(%[t])\n\t" - "movq %%rax, 280(%[t])\n\t" - "movq 296(%[t]), %%r11\n\t" - "movq 304(%[t]), %%r12\n\t" - "movq 312(%[t]), %%r13\n\t" - "# A[16] * B[20]\n\t" - "mulx 160(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[16] * B[21]\n\t" - "mulx 168(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[16] * B[22]\n\t" - "mulx 176(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[16] * B[23]\n\t" - "mulx 184(%[b]), %%r8, %%r9\n\t" - "movq %%r15, %%r14\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "adcxq %%rcx, %%r14\n\t" - "movq %%r15, %%rcx\n\t" - "adoxq %%r15, %%rcx\n\t" - "adcxq %%r15, %%rcx\n\t" - "movq %%r10, 288(%[t])\n\t" - "movq %%r11, 296(%[t])\n\t" - "movq %%r12, 304(%[t])\n\t" - "movq %%r13, 312(%[t])\n\t" - "movq %%r14, 320(%[t])\n\t" - "movq 136(%[a]), %%rdx\n\t" - "xorq %%r15, %%r15\n\t" - "movq 136(%[t]), %%rax\n\t" - "movq 144(%[t]), %%r10\n\t" - "movq 152(%[t]), %%r11\n\t" - "movq 160(%[t]), %%r12\n\t" - "movq 168(%[t]), %%r13\n\t" - "# A[17] * B[0]\n\t" - "mulx 0(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[17] * B[1]\n\t" - "mulx 8(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[17] * B[2]\n\t" - "mulx 16(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[17] * B[3]\n\t" - "mulx 24(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "movq %%rax, 136(%[t])\n\t" - "movq %%r10, 144(%[t])\n\t" - "movq %%r11, 152(%[t])\n\t" - "movq %%r12, 160(%[t])\n\t" - "movq 176(%[t]), %%r14\n\t" - "movq 184(%[t]), %%rax\n\t" - "movq 192(%[t]), %%r10\n\t" - "movq 200(%[t]), %%r11\n\t" - "# A[17] * B[4]\n\t" - "mulx 32(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[17] * B[5]\n\t" - "mulx 40(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[17] * B[6]\n\t" - "mulx 48(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[17] * B[7]\n\t" - "mulx 56(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "movq %%r13, 168(%[t])\n\t" - "movq %%r14, 176(%[t])\n\t" - "movq %%rax, 184(%[t])\n\t" - "movq %%r10, 192(%[t])\n\t" - "movq 208(%[t]), %%r12\n\t" - "movq 216(%[t]), %%r13\n\t" - "movq 224(%[t]), %%r14\n\t" - "movq 232(%[t]), %%rax\n\t" - "# A[17] * B[8]\n\t" - "mulx 64(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[17] * B[9]\n\t" - "mulx 72(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[17] * B[10]\n\t" - "mulx 80(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[17] * B[11]\n\t" - "mulx 88(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "movq %%r11, 200(%[t])\n\t" - "movq %%r12, 208(%[t])\n\t" - "movq %%r13, 216(%[t])\n\t" - "movq %%r14, 224(%[t])\n\t" - "movq 240(%[t]), %%r10\n\t" - "movq 248(%[t]), %%r11\n\t" - "movq 256(%[t]), %%r12\n\t" - "movq 264(%[t]), %%r13\n\t" - "# A[17] * B[12]\n\t" - "mulx 96(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[17] * B[13]\n\t" - "mulx 104(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[17] * B[14]\n\t" - "mulx 112(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[17] * B[15]\n\t" - "mulx 120(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "movq %%rax, 232(%[t])\n\t" - "movq %%r10, 240(%[t])\n\t" - "movq %%r11, 248(%[t])\n\t" - "movq %%r12, 256(%[t])\n\t" - "movq 272(%[t]), %%r14\n\t" - "movq 280(%[t]), %%rax\n\t" - "movq 288(%[t]), %%r10\n\t" - "movq 296(%[t]), %%r11\n\t" - "# A[17] * B[16]\n\t" - "mulx 128(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[17] * B[17]\n\t" - "mulx 136(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[17] * B[18]\n\t" - "mulx 144(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[17] * B[19]\n\t" - "mulx 152(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "movq %%r13, 264(%[t])\n\t" - "movq %%r14, 272(%[t])\n\t" - "movq %%rax, 280(%[t])\n\t" - "movq %%r10, 288(%[t])\n\t" - "movq 304(%[t]), %%r12\n\t" - "movq 312(%[t]), %%r13\n\t" - "movq 320(%[t]), %%r14\n\t" - "# A[17] * B[20]\n\t" - "mulx 160(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[17] * B[21]\n\t" - "mulx 168(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[17] * B[22]\n\t" - "mulx 176(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[17] * B[23]\n\t" - "mulx 184(%[b]), %%r8, %%r9\n\t" - "movq %%r15, %%rax\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "adcxq %%rcx, %%rax\n\t" - "movq %%r15, %%rcx\n\t" - "adoxq %%r15, %%rcx\n\t" - "adcxq %%r15, %%rcx\n\t" - "movq %%r11, 296(%[t])\n\t" - "movq %%r12, 304(%[t])\n\t" - "movq %%r13, 312(%[t])\n\t" - "movq %%r14, 320(%[t])\n\t" - "movq %%rax, 328(%[t])\n\t" - "movq 144(%[a]), %%rdx\n\t" - "xorq %%r15, %%r15\n\t" - "movq 144(%[t]), %%r10\n\t" - "movq 152(%[t]), %%r11\n\t" - "movq 160(%[t]), %%r12\n\t" - "movq 168(%[t]), %%r13\n\t" - "movq 176(%[t]), %%r14\n\t" - "# A[18] * B[0]\n\t" - "mulx 0(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[18] * B[1]\n\t" - "mulx 8(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[18] * B[2]\n\t" - "mulx 16(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[18] * B[3]\n\t" - "mulx 24(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "movq %%r10, 144(%[t])\n\t" - "movq %%r11, 152(%[t])\n\t" - "movq %%r12, 160(%[t])\n\t" - "movq %%r13, 168(%[t])\n\t" - "movq 184(%[t]), %%rax\n\t" - "movq 192(%[t]), %%r10\n\t" - "movq 200(%[t]), %%r11\n\t" - "movq 208(%[t]), %%r12\n\t" - "# A[18] * B[4]\n\t" - "mulx 32(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[18] * B[5]\n\t" - "mulx 40(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[18] * B[6]\n\t" - "mulx 48(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[18] * B[7]\n\t" - "mulx 56(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "movq %%r14, 176(%[t])\n\t" - "movq %%rax, 184(%[t])\n\t" - "movq %%r10, 192(%[t])\n\t" - "movq %%r11, 200(%[t])\n\t" - "movq 216(%[t]), %%r13\n\t" - "movq 224(%[t]), %%r14\n\t" - "movq 232(%[t]), %%rax\n\t" - "movq 240(%[t]), %%r10\n\t" - "# A[18] * B[8]\n\t" - "mulx 64(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[18] * B[9]\n\t" - "mulx 72(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[18] * B[10]\n\t" - "mulx 80(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[18] * B[11]\n\t" - "mulx 88(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "movq %%r12, 208(%[t])\n\t" - "movq %%r13, 216(%[t])\n\t" - "movq %%r14, 224(%[t])\n\t" - "movq %%rax, 232(%[t])\n\t" - "movq 248(%[t]), %%r11\n\t" - "movq 256(%[t]), %%r12\n\t" - "movq 264(%[t]), %%r13\n\t" - "movq 272(%[t]), %%r14\n\t" - "# A[18] * B[12]\n\t" - "mulx 96(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[18] * B[13]\n\t" - "mulx 104(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[18] * B[14]\n\t" - "mulx 112(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[18] * B[15]\n\t" - "mulx 120(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "movq %%r10, 240(%[t])\n\t" - "movq %%r11, 248(%[t])\n\t" - "movq %%r12, 256(%[t])\n\t" - "movq %%r13, 264(%[t])\n\t" - "movq 280(%[t]), %%rax\n\t" - "movq 288(%[t]), %%r10\n\t" - "movq 296(%[t]), %%r11\n\t" - "movq 304(%[t]), %%r12\n\t" - "# A[18] * B[16]\n\t" - "mulx 128(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[18] * B[17]\n\t" - "mulx 136(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[18] * B[18]\n\t" - "mulx 144(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[18] * B[19]\n\t" - "mulx 152(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "movq %%r14, 272(%[t])\n\t" - "movq %%rax, 280(%[t])\n\t" - "movq %%r10, 288(%[t])\n\t" - "movq %%r11, 296(%[t])\n\t" - "movq 312(%[t]), %%r13\n\t" - "movq 320(%[t]), %%r14\n\t" - "movq 328(%[t]), %%rax\n\t" - "# A[18] * B[20]\n\t" - "mulx 160(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[18] * B[21]\n\t" - "mulx 168(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[18] * B[22]\n\t" - "mulx 176(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[18] * B[23]\n\t" - "mulx 184(%[b]), %%r8, %%r9\n\t" - "movq %%r15, %%r10\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "adcxq %%rcx, %%r10\n\t" - "movq %%r15, %%rcx\n\t" - "adoxq %%r15, %%rcx\n\t" - "adcxq %%r15, %%rcx\n\t" - "movq %%r12, 304(%[t])\n\t" - "movq %%r13, 312(%[t])\n\t" - "movq %%r14, 320(%[t])\n\t" - "movq %%rax, 328(%[t])\n\t" - "movq %%r10, 336(%[t])\n\t" - "movq 152(%[a]), %%rdx\n\t" - "xorq %%r15, %%r15\n\t" - "movq 152(%[t]), %%r11\n\t" - "movq 160(%[t]), %%r12\n\t" - "movq 168(%[t]), %%r13\n\t" - "movq 176(%[t]), %%r14\n\t" - "movq 184(%[t]), %%rax\n\t" - "# A[19] * B[0]\n\t" - "mulx 0(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[19] * B[1]\n\t" - "mulx 8(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[19] * B[2]\n\t" - "mulx 16(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[19] * B[3]\n\t" - "mulx 24(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "movq %%r11, 152(%[t])\n\t" - "movq %%r12, 160(%[t])\n\t" - "movq %%r13, 168(%[t])\n\t" - "movq %%r14, 176(%[t])\n\t" - "movq 192(%[t]), %%r10\n\t" - "movq 200(%[t]), %%r11\n\t" - "movq 208(%[t]), %%r12\n\t" - "movq 216(%[t]), %%r13\n\t" - "# A[19] * B[4]\n\t" - "mulx 32(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[19] * B[5]\n\t" - "mulx 40(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[19] * B[6]\n\t" - "mulx 48(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[19] * B[7]\n\t" - "mulx 56(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "movq %%rax, 184(%[t])\n\t" - "movq %%r10, 192(%[t])\n\t" - "movq %%r11, 200(%[t])\n\t" - "movq %%r12, 208(%[t])\n\t" - "movq 224(%[t]), %%r14\n\t" - "movq 232(%[t]), %%rax\n\t" - "movq 240(%[t]), %%r10\n\t" - "movq 248(%[t]), %%r11\n\t" - "# A[19] * B[8]\n\t" - "mulx 64(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[19] * B[9]\n\t" - "mulx 72(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[19] * B[10]\n\t" - "mulx 80(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[19] * B[11]\n\t" - "mulx 88(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "movq %%r13, 216(%[t])\n\t" - "movq %%r14, 224(%[t])\n\t" - "movq %%rax, 232(%[t])\n\t" - "movq %%r10, 240(%[t])\n\t" - "movq 256(%[t]), %%r12\n\t" - "movq 264(%[t]), %%r13\n\t" - "movq 272(%[t]), %%r14\n\t" - "movq 280(%[t]), %%rax\n\t" - "# A[19] * B[12]\n\t" - "mulx 96(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[19] * B[13]\n\t" - "mulx 104(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[19] * B[14]\n\t" - "mulx 112(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[19] * B[15]\n\t" - "mulx 120(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "movq %%r11, 248(%[t])\n\t" - "movq %%r12, 256(%[t])\n\t" - "movq %%r13, 264(%[t])\n\t" - "movq %%r14, 272(%[t])\n\t" - "movq 288(%[t]), %%r10\n\t" - "movq 296(%[t]), %%r11\n\t" - "movq 304(%[t]), %%r12\n\t" - "movq 312(%[t]), %%r13\n\t" - "# A[19] * B[16]\n\t" - "mulx 128(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[19] * B[17]\n\t" - "mulx 136(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[19] * B[18]\n\t" - "mulx 144(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[19] * B[19]\n\t" - "mulx 152(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "movq %%rax, 280(%[t])\n\t" - "movq %%r10, 288(%[t])\n\t" - "movq %%r11, 296(%[t])\n\t" - "movq %%r12, 304(%[t])\n\t" - "movq 320(%[t]), %%r14\n\t" - "movq 328(%[t]), %%rax\n\t" - "movq 336(%[t]), %%r10\n\t" - "# A[19] * B[20]\n\t" - "mulx 160(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[19] * B[21]\n\t" - "mulx 168(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[19] * B[22]\n\t" - "mulx 176(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[19] * B[23]\n\t" - "mulx 184(%[b]), %%r8, %%r9\n\t" - "movq %%r15, %%r11\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "adcxq %%rcx, %%r11\n\t" - "movq %%r15, %%rcx\n\t" - "adoxq %%r15, %%rcx\n\t" - "adcxq %%r15, %%rcx\n\t" - "movq %%r13, 312(%[t])\n\t" - "movq %%r14, 320(%[t])\n\t" - "movq %%rax, 328(%[t])\n\t" - "movq %%r10, 336(%[t])\n\t" - "movq %%r11, 344(%[t])\n\t" - "movq 160(%[a]), %%rdx\n\t" - "xorq %%r15, %%r15\n\t" - "movq 160(%[t]), %%r12\n\t" - "movq 168(%[t]), %%r13\n\t" - "movq 176(%[t]), %%r14\n\t" - "movq 184(%[t]), %%rax\n\t" - "movq 192(%[t]), %%r10\n\t" - "# A[20] * B[0]\n\t" - "mulx 0(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[20] * B[1]\n\t" - "mulx 8(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[20] * B[2]\n\t" - "mulx 16(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[20] * B[3]\n\t" - "mulx 24(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "movq %%r12, 160(%[t])\n\t" - "movq %%r13, 168(%[t])\n\t" - "movq %%r14, 176(%[t])\n\t" - "movq %%rax, 184(%[t])\n\t" - "movq 200(%[t]), %%r11\n\t" - "movq 208(%[t]), %%r12\n\t" - "movq 216(%[t]), %%r13\n\t" - "movq 224(%[t]), %%r14\n\t" - "# A[20] * B[4]\n\t" - "mulx 32(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[20] * B[5]\n\t" - "mulx 40(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[20] * B[6]\n\t" - "mulx 48(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[20] * B[7]\n\t" - "mulx 56(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "movq %%r10, 192(%[t])\n\t" - "movq %%r11, 200(%[t])\n\t" - "movq %%r12, 208(%[t])\n\t" - "movq %%r13, 216(%[t])\n\t" - "movq 232(%[t]), %%rax\n\t" - "movq 240(%[t]), %%r10\n\t" - "movq 248(%[t]), %%r11\n\t" - "movq 256(%[t]), %%r12\n\t" - "# A[20] * B[8]\n\t" - "mulx 64(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[20] * B[9]\n\t" - "mulx 72(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[20] * B[10]\n\t" - "mulx 80(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[20] * B[11]\n\t" - "mulx 88(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "movq %%r14, 224(%[t])\n\t" - "movq %%rax, 232(%[t])\n\t" - "movq %%r10, 240(%[t])\n\t" - "movq %%r11, 248(%[t])\n\t" - "movq 264(%[t]), %%r13\n\t" - "movq 272(%[t]), %%r14\n\t" - "movq 280(%[t]), %%rax\n\t" - "movq 288(%[t]), %%r10\n\t" - "# A[20] * B[12]\n\t" - "mulx 96(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[20] * B[13]\n\t" - "mulx 104(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[20] * B[14]\n\t" - "mulx 112(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[20] * B[15]\n\t" - "mulx 120(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "movq %%r12, 256(%[t])\n\t" - "movq %%r13, 264(%[t])\n\t" - "movq %%r14, 272(%[t])\n\t" - "movq %%rax, 280(%[t])\n\t" - "movq 296(%[t]), %%r11\n\t" - "movq 304(%[t]), %%r12\n\t" - "movq 312(%[t]), %%r13\n\t" - "movq 320(%[t]), %%r14\n\t" - "# A[20] * B[16]\n\t" - "mulx 128(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[20] * B[17]\n\t" - "mulx 136(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[20] * B[18]\n\t" - "mulx 144(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[20] * B[19]\n\t" - "mulx 152(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "movq %%r10, 288(%[t])\n\t" - "movq %%r11, 296(%[t])\n\t" - "movq %%r12, 304(%[t])\n\t" - "movq %%r13, 312(%[t])\n\t" - "movq 328(%[t]), %%rax\n\t" - "movq 336(%[t]), %%r10\n\t" - "movq 344(%[t]), %%r11\n\t" - "# A[20] * B[20]\n\t" - "mulx 160(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[20] * B[21]\n\t" - "mulx 168(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[20] * B[22]\n\t" - "mulx 176(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[20] * B[23]\n\t" - "mulx 184(%[b]), %%r8, %%r9\n\t" - "movq %%r15, %%r12\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "adcxq %%rcx, %%r12\n\t" - "movq %%r15, %%rcx\n\t" - "adoxq %%r15, %%rcx\n\t" - "adcxq %%r15, %%rcx\n\t" - "movq %%r14, 320(%[t])\n\t" - "movq %%rax, 328(%[t])\n\t" - "movq %%r10, 336(%[t])\n\t" - "movq %%r11, 344(%[t])\n\t" - "movq %%r12, 352(%[t])\n\t" - "movq 168(%[a]), %%rdx\n\t" - "xorq %%r15, %%r15\n\t" - "movq 168(%[t]), %%r13\n\t" - "movq 176(%[t]), %%r14\n\t" - "movq 184(%[t]), %%rax\n\t" - "movq 192(%[t]), %%r10\n\t" - "movq 200(%[t]), %%r11\n\t" - "# A[21] * B[0]\n\t" - "mulx 0(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[21] * B[1]\n\t" - "mulx 8(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[21] * B[2]\n\t" - "mulx 16(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[21] * B[3]\n\t" - "mulx 24(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "movq %%r13, 168(%[t])\n\t" - "movq %%r14, 176(%[t])\n\t" - "movq %%rax, 184(%[t])\n\t" - "movq %%r10, 192(%[t])\n\t" - "movq 208(%[t]), %%r12\n\t" - "movq 216(%[t]), %%r13\n\t" - "movq 224(%[t]), %%r14\n\t" - "movq 232(%[t]), %%rax\n\t" - "# A[21] * B[4]\n\t" - "mulx 32(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[21] * B[5]\n\t" - "mulx 40(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[21] * B[6]\n\t" - "mulx 48(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[21] * B[7]\n\t" - "mulx 56(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "movq %%r11, 200(%[t])\n\t" - "movq %%r12, 208(%[t])\n\t" - "movq %%r13, 216(%[t])\n\t" - "movq %%r14, 224(%[t])\n\t" - "movq 240(%[t]), %%r10\n\t" - "movq 248(%[t]), %%r11\n\t" - "movq 256(%[t]), %%r12\n\t" - "movq 264(%[t]), %%r13\n\t" - "# A[21] * B[8]\n\t" - "mulx 64(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[21] * B[9]\n\t" - "mulx 72(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[21] * B[10]\n\t" - "mulx 80(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[21] * B[11]\n\t" - "mulx 88(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "movq %%rax, 232(%[t])\n\t" - "movq %%r10, 240(%[t])\n\t" - "movq %%r11, 248(%[t])\n\t" - "movq %%r12, 256(%[t])\n\t" - "movq 272(%[t]), %%r14\n\t" - "movq 280(%[t]), %%rax\n\t" - "movq 288(%[t]), %%r10\n\t" - "movq 296(%[t]), %%r11\n\t" - "# A[21] * B[12]\n\t" - "mulx 96(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[21] * B[13]\n\t" - "mulx 104(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[21] * B[14]\n\t" - "mulx 112(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[21] * B[15]\n\t" - "mulx 120(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "movq %%r13, 264(%[t])\n\t" - "movq %%r14, 272(%[t])\n\t" - "movq %%rax, 280(%[t])\n\t" - "movq %%r10, 288(%[t])\n\t" - "movq 304(%[t]), %%r12\n\t" - "movq 312(%[t]), %%r13\n\t" - "movq 320(%[t]), %%r14\n\t" - "movq 328(%[t]), %%rax\n\t" - "# A[21] * B[16]\n\t" - "mulx 128(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[21] * B[17]\n\t" - "mulx 136(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[21] * B[18]\n\t" - "mulx 144(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[21] * B[19]\n\t" - "mulx 152(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "movq %%r11, 296(%[t])\n\t" - "movq %%r12, 304(%[t])\n\t" - "movq %%r13, 312(%[t])\n\t" - "movq %%r14, 320(%[t])\n\t" - "movq 336(%[t]), %%r10\n\t" - "movq 344(%[t]), %%r11\n\t" - "movq 352(%[t]), %%r12\n\t" - "# A[21] * B[20]\n\t" - "mulx 160(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[21] * B[21]\n\t" - "mulx 168(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[21] * B[22]\n\t" - "mulx 176(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[21] * B[23]\n\t" - "mulx 184(%[b]), %%r8, %%r9\n\t" - "movq %%r15, %%r13\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "adcxq %%rcx, %%r13\n\t" - "movq %%r15, %%rcx\n\t" - "adoxq %%r15, %%rcx\n\t" - "adcxq %%r15, %%rcx\n\t" - "movq %%rax, 328(%[t])\n\t" - "movq %%r10, 336(%[t])\n\t" - "movq %%r11, 344(%[t])\n\t" - "movq %%r12, 352(%[t])\n\t" - "movq %%r13, 360(%[t])\n\t" - "movq 176(%[a]), %%rdx\n\t" - "xorq %%r15, %%r15\n\t" - "movq 176(%[t]), %%r14\n\t" - "movq 184(%[t]), %%rax\n\t" - "movq 192(%[t]), %%r10\n\t" - "movq 200(%[t]), %%r11\n\t" - "movq 208(%[t]), %%r12\n\t" - "# A[22] * B[0]\n\t" - "mulx 0(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[22] * B[1]\n\t" - "mulx 8(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[22] * B[2]\n\t" - "mulx 16(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[22] * B[3]\n\t" - "mulx 24(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "movq %%r14, 176(%[t])\n\t" - "movq %%rax, 184(%[t])\n\t" - "movq %%r10, 192(%[t])\n\t" - "movq %%r11, 200(%[t])\n\t" - "movq 216(%[t]), %%r13\n\t" - "movq 224(%[t]), %%r14\n\t" - "movq 232(%[t]), %%rax\n\t" - "movq 240(%[t]), %%r10\n\t" - "# A[22] * B[4]\n\t" - "mulx 32(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[22] * B[5]\n\t" - "mulx 40(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[22] * B[6]\n\t" - "mulx 48(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[22] * B[7]\n\t" - "mulx 56(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "movq %%r12, 208(%[t])\n\t" - "movq %%r13, 216(%[t])\n\t" - "movq %%r14, 224(%[t])\n\t" - "movq %%rax, 232(%[t])\n\t" - "movq 248(%[t]), %%r11\n\t" - "movq 256(%[t]), %%r12\n\t" - "movq 264(%[t]), %%r13\n\t" - "movq 272(%[t]), %%r14\n\t" - "# A[22] * B[8]\n\t" - "mulx 64(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[22] * B[9]\n\t" - "mulx 72(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[22] * B[10]\n\t" - "mulx 80(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[22] * B[11]\n\t" - "mulx 88(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "movq %%r10, 240(%[t])\n\t" - "movq %%r11, 248(%[t])\n\t" - "movq %%r12, 256(%[t])\n\t" - "movq %%r13, 264(%[t])\n\t" - "movq 280(%[t]), %%rax\n\t" - "movq 288(%[t]), %%r10\n\t" - "movq 296(%[t]), %%r11\n\t" - "movq 304(%[t]), %%r12\n\t" - "# A[22] * B[12]\n\t" - "mulx 96(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[22] * B[13]\n\t" - "mulx 104(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[22] * B[14]\n\t" - "mulx 112(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[22] * B[15]\n\t" - "mulx 120(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "movq %%r14, 272(%[t])\n\t" - "movq %%rax, 280(%[t])\n\t" - "movq %%r10, 288(%[t])\n\t" - "movq %%r11, 296(%[t])\n\t" - "movq 312(%[t]), %%r13\n\t" - "movq 320(%[t]), %%r14\n\t" - "movq 328(%[t]), %%rax\n\t" - "movq 336(%[t]), %%r10\n\t" - "# A[22] * B[16]\n\t" - "mulx 128(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[22] * B[17]\n\t" - "mulx 136(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[22] * B[18]\n\t" - "mulx 144(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[22] * B[19]\n\t" - "mulx 152(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "movq %%r12, 304(%[t])\n\t" - "movq %%r13, 312(%[t])\n\t" - "movq %%r14, 320(%[t])\n\t" - "movq %%rax, 328(%[t])\n\t" - "movq 344(%[t]), %%r11\n\t" - "movq 352(%[t]), %%r12\n\t" - "movq 360(%[t]), %%r13\n\t" - "# A[22] * B[20]\n\t" - "mulx 160(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[22] * B[21]\n\t" - "mulx 168(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[22] * B[22]\n\t" - "mulx 176(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[22] * B[23]\n\t" - "mulx 184(%[b]), %%r8, %%r9\n\t" - "movq %%r15, %%r14\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "adcxq %%rcx, %%r14\n\t" - "movq %%r15, %%rcx\n\t" - "adoxq %%r15, %%rcx\n\t" - "adcxq %%r15, %%rcx\n\t" - "movq %%r10, 336(%[t])\n\t" - "movq %%r11, 344(%[t])\n\t" - "movq %%r12, 352(%[t])\n\t" - "movq %%r13, 360(%[t])\n\t" - "movq %%r14, 368(%[t])\n\t" - "movq 184(%[a]), %%rdx\n\t" - "xorq %%r15, %%r15\n\t" - "movq 184(%[t]), %%rax\n\t" - "movq 192(%[t]), %%r10\n\t" - "movq 200(%[t]), %%r11\n\t" - "movq 208(%[t]), %%r12\n\t" - "movq 216(%[t]), %%r13\n\t" - "# A[23] * B[0]\n\t" - "mulx 0(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[23] * B[1]\n\t" - "mulx 8(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[23] * B[2]\n\t" - "mulx 16(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[23] * B[3]\n\t" - "mulx 24(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "movq %%rax, 184(%[t])\n\t" - "movq %%r10, 192(%[t])\n\t" - "movq %%r11, 200(%[t])\n\t" - "movq %%r12, 208(%[t])\n\t" - "movq 224(%[t]), %%r14\n\t" - "movq 232(%[t]), %%rax\n\t" - "movq 240(%[t]), %%r10\n\t" - "movq 248(%[t]), %%r11\n\t" - "# A[23] * B[4]\n\t" - "mulx 32(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[23] * B[5]\n\t" - "mulx 40(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[23] * B[6]\n\t" - "mulx 48(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[23] * B[7]\n\t" - "mulx 56(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "movq %%r13, 216(%[t])\n\t" - "movq %%r14, 224(%[t])\n\t" - "movq %%rax, 232(%[t])\n\t" - "movq %%r10, 240(%[t])\n\t" - "movq 256(%[t]), %%r12\n\t" - "movq 264(%[t]), %%r13\n\t" - "movq 272(%[t]), %%r14\n\t" - "movq 280(%[t]), %%rax\n\t" - "# A[23] * B[8]\n\t" - "mulx 64(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[23] * B[9]\n\t" - "mulx 72(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[23] * B[10]\n\t" - "mulx 80(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[23] * B[11]\n\t" - "mulx 88(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "movq %%r11, 248(%[t])\n\t" - "movq %%r12, 256(%[t])\n\t" - "movq %%r13, 264(%[t])\n\t" - "movq %%r14, 272(%[t])\n\t" - "movq 288(%[t]), %%r10\n\t" - "movq 296(%[t]), %%r11\n\t" - "movq 304(%[t]), %%r12\n\t" - "movq 312(%[t]), %%r13\n\t" - "# A[23] * B[12]\n\t" - "mulx 96(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[23] * B[13]\n\t" - "mulx 104(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "# A[23] * B[14]\n\t" - "mulx 112(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[23] * B[15]\n\t" - "mulx 120(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "movq %%rax, 280(%[t])\n\t" - "movq %%r10, 288(%[t])\n\t" - "movq %%r11, 296(%[t])\n\t" - "movq %%r12, 304(%[t])\n\t" - "movq 320(%[t]), %%r14\n\t" - "movq 328(%[t]), %%rax\n\t" - "movq 336(%[t]), %%r10\n\t" - "movq 344(%[t]), %%r11\n\t" - "# A[23] * B[16]\n\t" - "mulx 128(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[23] * B[17]\n\t" - "mulx 136(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "# A[23] * B[18]\n\t" - "mulx 144(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%r9, %%r10\n\t" - "# A[23] * B[19]\n\t" - "mulx 152(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r10\n\t" - "adoxq %%r9, %%r11\n\t" - "movq %%r13, 312(%[t])\n\t" - "movq %%r14, 320(%[t])\n\t" - "movq %%rax, 328(%[t])\n\t" - "movq %%r10, 336(%[t])\n\t" - "movq 352(%[t]), %%r12\n\t" - "movq 360(%[t]), %%r13\n\t" - "movq 368(%[t]), %%r14\n\t" - "# A[23] * B[20]\n\t" - "mulx 160(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r11\n\t" - "adoxq %%r9, %%r12\n\t" - "# A[23] * B[21]\n\t" - "mulx 168(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%r9, %%r13\n\t" - "# A[23] * B[22]\n\t" - "mulx 176(%[b]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%r9, %%r14\n\t" - "# A[23] * B[23]\n\t" - "mulx 184(%[b]), %%r8, %%r9\n\t" - "movq %%r15, %%rax\n\t" - "adcxq %%r8, %%r14\n\t" - "adoxq %%r9, %%rax\n\t" - "adcxq %%rcx, %%rax\n\t" - "movq %%r11, 344(%[t])\n\t" - "movq %%r12, 352(%[t])\n\t" - "movq %%r13, 360(%[t])\n\t" - "movq %%r14, 368(%[t])\n\t" - "movq %%rax, 376(%[t])\n\t" - : - : [a] "r" (a), [b] "r" (b), [t] "r" (tmp) - : "memory", "rax", "rdx", "rcx", - "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15" - ); - - XMEMCPY(r, tmp, sizeof(tmp)); -} -#endif /* HAVE_INTEL_AVX2 */ - -#ifdef HAVE_INTEL_AVX2 -/* Square a and put result in r. (r = a * a) - * - * r A single precision integer. - * a A single precision integer. - */ -SP_NOINLINE static void sp_3072_sqr_avx2_24(sp_digit* r, const sp_digit* a) -{ - sp_digit tmp[48]; - - __asm__ __volatile__ ( - "xorq %%r8, %%r8\n\t" - "# Diagonal 1\n\t" - "xorq %%r10, %%r10\n\t" - "xorq %%r11, %%r11\n\t" - "xorq %%r12, %%r12\n\t" - "xorq %%r13, %%r13\n\t" - "xorq %%r14, %%r14\n\t" - "xorq %%r15, %%r15\n\t" - "# A[1] x A[0]\n\t" - "movq 0(%[a]), %%rdx\n\t" - "mulxq 8(%[a]), %%r10, %%r11\n\t" - "# A[2] x A[0]\n\t" - "mulxq 16(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "# A[3] x A[0]\n\t" - "mulxq 24(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "# A[4] x A[0]\n\t" - "mulxq 32(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "# A[5] x A[0]\n\t" - "mulxq 40(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "movq %%r10, 8(%[tmp])\n\t" - "movq %%r11, 16(%[tmp])\n\t" - "movq %%r12, 24(%[tmp])\n\t" - "movq %%r13, 32(%[tmp])\n\t" - "movq %%r14, 40(%[tmp])\n\t" - "movq %%r8, %%r10\n\t" - "movq %%r8, %%r11\n\t" - "movq %%r8, %%r12\n\t" - "movq %%r8, %%r13\n\t" - "movq %%r8, %%r14\n\t" - "# A[6] x A[0]\n\t" - "mulxq 48(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "# A[7] x A[0]\n\t" - "mulxq 56(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "# A[8] x A[0]\n\t" - "mulxq 64(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "# A[9] x A[0]\n\t" - "mulxq 72(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "# A[10] x A[0]\n\t" - "mulxq 80(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "movq %%r15, 48(%[tmp])\n\t" - "movq %%r10, 56(%[tmp])\n\t" - "movq %%r11, 64(%[tmp])\n\t" - "movq %%r12, 72(%[tmp])\n\t" - "movq %%r13, 80(%[tmp])\n\t" - "movq %%r8, %%r15\n\t" - "movq %%r8, %%r10\n\t" - "movq %%r8, %%r11\n\t" - "movq %%r8, %%r12\n\t" - "movq %%r8, %%r13\n\t" - "# A[11] x A[0]\n\t" - "mulxq 88(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "# A[12] x A[0]\n\t" - "mulxq 96(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "# A[13] x A[0]\n\t" - "mulxq 104(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "# A[14] x A[0]\n\t" - "mulxq 112(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "# A[15] x A[0]\n\t" - "mulxq 120(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "movq %%r14, 88(%[tmp])\n\t" - "movq %%r15, 96(%[tmp])\n\t" - "movq %%r10, 104(%[tmp])\n\t" - "movq %%r11, 112(%[tmp])\n\t" - "movq %%r12, 120(%[tmp])\n\t" - "movq %%r8, %%r14\n\t" - "movq %%r8, %%r15\n\t" - "movq %%r8, %%r10\n\t" - "movq %%r8, %%r11\n\t" - "movq %%r8, %%r12\n\t" - "# A[16] x A[0]\n\t" - "mulxq 128(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "# A[17] x A[0]\n\t" - "mulxq 136(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "# A[18] x A[0]\n\t" - "mulxq 144(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "# A[19] x A[0]\n\t" - "mulxq 152(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "# A[20] x A[0]\n\t" - "mulxq 160(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "movq %%r13, 128(%[tmp])\n\t" - "movq %%r14, 136(%[tmp])\n\t" - "movq %%r15, 144(%[tmp])\n\t" - "movq %%r10, 152(%[tmp])\n\t" - "movq %%r11, 160(%[tmp])\n\t" - "movq %%r8, %%r13\n\t" - "movq %%r8, %%r14\n\t" - "movq %%r8, %%r15\n\t" - "# A[21] x A[0]\n\t" - "mulxq 168(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "# A[22] x A[0]\n\t" - "mulxq 176(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "# A[23] x A[0]\n\t" - "mulxq 184(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "movq %%r12, 168(%[tmp])\n\t" - "movq %%r13, 176(%[tmp])\n\t" - "movq %%r14, 184(%[tmp])\n\t" - "# Carry\n\t" - "adcxq %%r8, %%r15\n\t" - "movq %%r8, %%r9\n\t" - "adcxq %%r8, %%r9\n\t" - "adoxq %%r8, %%r9\n\t" - "movq %%r15, 192(%[tmp])\n\t" - "xorq %%r8, %%r8\n\t" - "# Diagonal 2\n\t" - "movq 24(%[tmp]), %%r15\n\t" - "movq 32(%[tmp]), %%r10\n\t" - "movq 40(%[tmp]), %%r11\n\t" - "movq 48(%[tmp]), %%r12\n\t" - "movq 56(%[tmp]), %%r13\n\t" - "movq 64(%[tmp]), %%r14\n\t" - "# A[2] x A[1]\n\t" - "movq 8(%[a]), %%rdx\n\t" - "mulxq 16(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "# A[3] x A[1]\n\t" - "mulxq 24(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "# A[4] x A[1]\n\t" - "mulxq 32(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "# A[5] x A[1]\n\t" - "mulxq 40(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "# A[6] x A[1]\n\t" - "mulxq 48(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "movq %%r15, 24(%[tmp])\n\t" - "movq %%r10, 32(%[tmp])\n\t" - "movq %%r11, 40(%[tmp])\n\t" - "movq %%r12, 48(%[tmp])\n\t" - "movq %%r13, 56(%[tmp])\n\t" - "movq 72(%[tmp]), %%r15\n\t" - "movq 80(%[tmp]), %%r10\n\t" - "movq 88(%[tmp]), %%r11\n\t" - "movq 96(%[tmp]), %%r12\n\t" - "movq 104(%[tmp]), %%r13\n\t" - "# A[7] x A[1]\n\t" - "mulxq 56(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "# A[8] x A[1]\n\t" - "mulxq 64(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "# A[9] x A[1]\n\t" - "mulxq 72(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "# A[10] x A[1]\n\t" - "mulxq 80(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "# A[11] x A[1]\n\t" - "mulxq 88(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "movq %%r14, 64(%[tmp])\n\t" - "movq %%r15, 72(%[tmp])\n\t" - "movq %%r10, 80(%[tmp])\n\t" - "movq %%r11, 88(%[tmp])\n\t" - "movq %%r12, 96(%[tmp])\n\t" - "movq 112(%[tmp]), %%r14\n\t" - "movq 120(%[tmp]), %%r15\n\t" - "movq 128(%[tmp]), %%r10\n\t" - "movq 136(%[tmp]), %%r11\n\t" - "movq 144(%[tmp]), %%r12\n\t" - "# A[12] x A[1]\n\t" - "mulxq 96(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "# A[13] x A[1]\n\t" - "mulxq 104(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "# A[14] x A[1]\n\t" - "mulxq 112(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "# A[15] x A[1]\n\t" - "mulxq 120(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "# A[16] x A[1]\n\t" - "mulxq 128(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "movq %%r13, 104(%[tmp])\n\t" - "movq %%r14, 112(%[tmp])\n\t" - "movq %%r15, 120(%[tmp])\n\t" - "movq %%r10, 128(%[tmp])\n\t" - "movq %%r11, 136(%[tmp])\n\t" - "movq 152(%[tmp]), %%r13\n\t" - "movq 160(%[tmp]), %%r14\n\t" - "movq 168(%[tmp]), %%r15\n\t" - "movq 176(%[tmp]), %%r10\n\t" - "movq 184(%[tmp]), %%r11\n\t" - "# A[17] x A[1]\n\t" - "mulxq 136(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "# A[18] x A[1]\n\t" - "mulxq 144(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "# A[19] x A[1]\n\t" - "mulxq 152(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "# A[20] x A[1]\n\t" - "mulxq 160(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "# A[21] x A[1]\n\t" - "mulxq 168(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "movq %%r12, 144(%[tmp])\n\t" - "movq %%r13, 152(%[tmp])\n\t" - "movq %%r14, 160(%[tmp])\n\t" - "movq %%r15, 168(%[tmp])\n\t" - "movq %%r10, 176(%[tmp])\n\t" - "movq 192(%[tmp]), %%r12\n\t" - "movq %%r8, %%r13\n\t" - "movq %%r8, %%r14\n\t" - "# A[22] x A[1]\n\t" - "mulxq 176(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "# A[23] x A[1]\n\t" - "mulxq 184(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "# A[23] x A[2]\n\t" - "movq 16(%[a]), %%rdx\n\t" - "mulxq 184(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "movq %%r11, 184(%[tmp])\n\t" - "movq %%r12, 192(%[tmp])\n\t" - "movq %%r13, 200(%[tmp])\n\t" - "# Carry\n\t" - "adcxq %%r9, %%r14\n\t" - "movq %%r8, %%r9\n\t" - "adcxq %%r8, %%r9\n\t" - "adoxq %%r8, %%r9\n\t" - "movq %%r14, 208(%[tmp])\n\t" - "xorq %%r8, %%r8\n\t" - "# Diagonal 3\n\t" - "movq 40(%[tmp]), %%r14\n\t" - "movq 48(%[tmp]), %%r15\n\t" - "movq 56(%[tmp]), %%r10\n\t" - "movq 64(%[tmp]), %%r11\n\t" - "movq 72(%[tmp]), %%r12\n\t" - "movq 80(%[tmp]), %%r13\n\t" - "# A[3] x A[2]\n\t" - "mulxq 24(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "# A[4] x A[2]\n\t" - "mulxq 32(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "# A[5] x A[2]\n\t" - "mulxq 40(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "# A[6] x A[2]\n\t" - "mulxq 48(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "# A[7] x A[2]\n\t" - "mulxq 56(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "movq %%r14, 40(%[tmp])\n\t" - "movq %%r15, 48(%[tmp])\n\t" - "movq %%r10, 56(%[tmp])\n\t" - "movq %%r11, 64(%[tmp])\n\t" - "movq %%r12, 72(%[tmp])\n\t" - "movq 88(%[tmp]), %%r14\n\t" - "movq 96(%[tmp]), %%r15\n\t" - "movq 104(%[tmp]), %%r10\n\t" - "movq 112(%[tmp]), %%r11\n\t" - "movq 120(%[tmp]), %%r12\n\t" - "# A[8] x A[2]\n\t" - "mulxq 64(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "# A[9] x A[2]\n\t" - "mulxq 72(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "# A[10] x A[2]\n\t" - "mulxq 80(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "# A[11] x A[2]\n\t" - "mulxq 88(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "# A[12] x A[2]\n\t" - "mulxq 96(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "movq %%r13, 80(%[tmp])\n\t" - "movq %%r14, 88(%[tmp])\n\t" - "movq %%r15, 96(%[tmp])\n\t" - "movq %%r10, 104(%[tmp])\n\t" - "movq %%r11, 112(%[tmp])\n\t" - "movq 128(%[tmp]), %%r13\n\t" - "movq 136(%[tmp]), %%r14\n\t" - "movq 144(%[tmp]), %%r15\n\t" - "movq 152(%[tmp]), %%r10\n\t" - "movq 160(%[tmp]), %%r11\n\t" - "# A[13] x A[2]\n\t" - "mulxq 104(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "# A[14] x A[2]\n\t" - "mulxq 112(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "# A[15] x A[2]\n\t" - "mulxq 120(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "# A[16] x A[2]\n\t" - "mulxq 128(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "# A[17] x A[2]\n\t" - "mulxq 136(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "movq %%r12, 120(%[tmp])\n\t" - "movq %%r13, 128(%[tmp])\n\t" - "movq %%r14, 136(%[tmp])\n\t" - "movq %%r15, 144(%[tmp])\n\t" - "movq %%r10, 152(%[tmp])\n\t" - "movq 168(%[tmp]), %%r12\n\t" - "movq 176(%[tmp]), %%r13\n\t" - "movq 184(%[tmp]), %%r14\n\t" - "movq 192(%[tmp]), %%r15\n\t" - "movq 200(%[tmp]), %%r10\n\t" - "# A[18] x A[2]\n\t" - "mulxq 144(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "# A[19] x A[2]\n\t" - "mulxq 152(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "# A[20] x A[2]\n\t" - "mulxq 160(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "# A[21] x A[2]\n\t" - "mulxq 168(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "# A[22] x A[2]\n\t" - "mulxq 176(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "movq %%r11, 160(%[tmp])\n\t" - "movq %%r12, 168(%[tmp])\n\t" - "movq %%r13, 176(%[tmp])\n\t" - "movq %%r14, 184(%[tmp])\n\t" - "movq %%r15, 192(%[tmp])\n\t" - "movq 208(%[tmp]), %%r11\n\t" - "movq %%r8, %%r12\n\t" - "movq %%r8, %%r13\n\t" - "# A[22] x A[3]\n\t" - "movq 176(%[a]), %%rdx\n\t" - "mulxq 24(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "# A[22] x A[4]\n\t" - "mulxq 32(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "# A[22] x A[5]\n\t" - "mulxq 40(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "movq %%r10, 200(%[tmp])\n\t" - "movq %%r11, 208(%[tmp])\n\t" - "movq %%r12, 216(%[tmp])\n\t" - "# Carry\n\t" - "adcxq %%r9, %%r13\n\t" - "movq %%r8, %%r9\n\t" - "adcxq %%r8, %%r9\n\t" - "adoxq %%r8, %%r9\n\t" - "movq %%r13, 224(%[tmp])\n\t" - "xorq %%r8, %%r8\n\t" - "# Diagonal 4\n\t" - "movq 56(%[tmp]), %%r13\n\t" - "movq 64(%[tmp]), %%r14\n\t" - "movq 72(%[tmp]), %%r15\n\t" - "movq 80(%[tmp]), %%r10\n\t" - "movq 88(%[tmp]), %%r11\n\t" - "movq 96(%[tmp]), %%r12\n\t" - "# A[4] x A[3]\n\t" - "movq 24(%[a]), %%rdx\n\t" - "mulxq 32(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "# A[5] x A[3]\n\t" - "mulxq 40(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "# A[6] x A[3]\n\t" - "mulxq 48(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "# A[7] x A[3]\n\t" - "mulxq 56(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "# A[8] x A[3]\n\t" - "mulxq 64(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "movq %%r13, 56(%[tmp])\n\t" - "movq %%r14, 64(%[tmp])\n\t" - "movq %%r15, 72(%[tmp])\n\t" - "movq %%r10, 80(%[tmp])\n\t" - "movq %%r11, 88(%[tmp])\n\t" - "movq 104(%[tmp]), %%r13\n\t" - "movq 112(%[tmp]), %%r14\n\t" - "movq 120(%[tmp]), %%r15\n\t" - "movq 128(%[tmp]), %%r10\n\t" - "movq 136(%[tmp]), %%r11\n\t" - "# A[9] x A[3]\n\t" - "mulxq 72(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "# A[10] x A[3]\n\t" - "mulxq 80(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "# A[11] x A[3]\n\t" - "mulxq 88(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "# A[12] x A[3]\n\t" - "mulxq 96(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "# A[13] x A[3]\n\t" - "mulxq 104(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "movq %%r12, 96(%[tmp])\n\t" - "movq %%r13, 104(%[tmp])\n\t" - "movq %%r14, 112(%[tmp])\n\t" - "movq %%r15, 120(%[tmp])\n\t" - "movq %%r10, 128(%[tmp])\n\t" - "movq 144(%[tmp]), %%r12\n\t" - "movq 152(%[tmp]), %%r13\n\t" - "movq 160(%[tmp]), %%r14\n\t" - "movq 168(%[tmp]), %%r15\n\t" - "movq 176(%[tmp]), %%r10\n\t" - "# A[14] x A[3]\n\t" - "mulxq 112(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "# A[15] x A[3]\n\t" - "mulxq 120(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "# A[16] x A[3]\n\t" - "mulxq 128(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "# A[17] x A[3]\n\t" - "mulxq 136(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "# A[18] x A[3]\n\t" - "mulxq 144(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "movq %%r11, 136(%[tmp])\n\t" - "movq %%r12, 144(%[tmp])\n\t" - "movq %%r13, 152(%[tmp])\n\t" - "movq %%r14, 160(%[tmp])\n\t" - "movq %%r15, 168(%[tmp])\n\t" - "movq 184(%[tmp]), %%r11\n\t" - "movq 192(%[tmp]), %%r12\n\t" - "movq 200(%[tmp]), %%r13\n\t" - "movq 208(%[tmp]), %%r14\n\t" - "movq 216(%[tmp]), %%r15\n\t" - "# A[19] x A[3]\n\t" - "mulxq 152(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "# A[20] x A[3]\n\t" - "mulxq 160(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "# A[21] x A[3]\n\t" - "mulxq 168(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "# A[21] x A[4]\n\t" - "movq 168(%[a]), %%rdx\n\t" - "mulxq 32(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "# A[21] x A[5]\n\t" - "mulxq 40(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "movq %%r10, 176(%[tmp])\n\t" - "movq %%r11, 184(%[tmp])\n\t" - "movq %%r12, 192(%[tmp])\n\t" - "movq %%r13, 200(%[tmp])\n\t" - "movq %%r14, 208(%[tmp])\n\t" - "movq 224(%[tmp]), %%r10\n\t" - "movq %%r8, %%r11\n\t" - "movq %%r8, %%r12\n\t" - "# A[21] x A[6]\n\t" - "mulxq 48(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "# A[21] x A[7]\n\t" - "mulxq 56(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "# A[21] x A[8]\n\t" - "mulxq 64(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "movq %%r15, 216(%[tmp])\n\t" - "movq %%r10, 224(%[tmp])\n\t" - "movq %%r11, 232(%[tmp])\n\t" - "# Carry\n\t" - "adcxq %%r9, %%r12\n\t" - "movq %%r8, %%r9\n\t" - "adcxq %%r8, %%r9\n\t" - "adoxq %%r8, %%r9\n\t" - "movq %%r12, 240(%[tmp])\n\t" - "xorq %%r8, %%r8\n\t" - "# Diagonal 5\n\t" - "movq 72(%[tmp]), %%r12\n\t" - "movq 80(%[tmp]), %%r13\n\t" - "movq 88(%[tmp]), %%r14\n\t" - "movq 96(%[tmp]), %%r15\n\t" - "movq 104(%[tmp]), %%r10\n\t" - "movq 112(%[tmp]), %%r11\n\t" - "# A[5] x A[4]\n\t" - "movq 32(%[a]), %%rdx\n\t" - "mulxq 40(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "# A[6] x A[4]\n\t" - "mulxq 48(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "# A[7] x A[4]\n\t" - "mulxq 56(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "# A[8] x A[4]\n\t" - "mulxq 64(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "# A[9] x A[4]\n\t" - "mulxq 72(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "movq %%r12, 72(%[tmp])\n\t" - "movq %%r13, 80(%[tmp])\n\t" - "movq %%r14, 88(%[tmp])\n\t" - "movq %%r15, 96(%[tmp])\n\t" - "movq %%r10, 104(%[tmp])\n\t" - "movq 120(%[tmp]), %%r12\n\t" - "movq 128(%[tmp]), %%r13\n\t" - "movq 136(%[tmp]), %%r14\n\t" - "movq 144(%[tmp]), %%r15\n\t" - "movq 152(%[tmp]), %%r10\n\t" - "# A[10] x A[4]\n\t" - "mulxq 80(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "# A[11] x A[4]\n\t" - "mulxq 88(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "# A[12] x A[4]\n\t" - "mulxq 96(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "# A[13] x A[4]\n\t" - "mulxq 104(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "# A[14] x A[4]\n\t" - "mulxq 112(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "movq %%r11, 112(%[tmp])\n\t" - "movq %%r12, 120(%[tmp])\n\t" - "movq %%r13, 128(%[tmp])\n\t" - "movq %%r14, 136(%[tmp])\n\t" - "movq %%r15, 144(%[tmp])\n\t" - "movq 160(%[tmp]), %%r11\n\t" - "movq 168(%[tmp]), %%r12\n\t" - "movq 176(%[tmp]), %%r13\n\t" - "movq 184(%[tmp]), %%r14\n\t" - "movq 192(%[tmp]), %%r15\n\t" - "# A[15] x A[4]\n\t" - "mulxq 120(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "# A[16] x A[4]\n\t" - "mulxq 128(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "# A[17] x A[4]\n\t" - "mulxq 136(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "# A[18] x A[4]\n\t" - "mulxq 144(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "# A[19] x A[4]\n\t" - "mulxq 152(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "movq %%r10, 152(%[tmp])\n\t" - "movq %%r11, 160(%[tmp])\n\t" - "movq %%r12, 168(%[tmp])\n\t" - "movq %%r13, 176(%[tmp])\n\t" - "movq %%r14, 184(%[tmp])\n\t" - "movq 200(%[tmp]), %%r10\n\t" - "movq 208(%[tmp]), %%r11\n\t" - "movq 216(%[tmp]), %%r12\n\t" - "movq 224(%[tmp]), %%r13\n\t" - "movq 232(%[tmp]), %%r14\n\t" - "# A[20] x A[4]\n\t" - "mulxq 160(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "# A[20] x A[5]\n\t" - "movq 160(%[a]), %%rdx\n\t" - "mulxq 40(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "# A[20] x A[6]\n\t" - "mulxq 48(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "# A[20] x A[7]\n\t" - "mulxq 56(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "# A[20] x A[8]\n\t" - "mulxq 64(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "movq %%r15, 192(%[tmp])\n\t" - "movq %%r10, 200(%[tmp])\n\t" - "movq %%r11, 208(%[tmp])\n\t" - "movq %%r12, 216(%[tmp])\n\t" - "movq %%r13, 224(%[tmp])\n\t" - "movq 240(%[tmp]), %%r15\n\t" - "movq %%r8, %%r10\n\t" - "movq %%r8, %%r11\n\t" - "# A[20] x A[9]\n\t" - "mulxq 72(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "# A[20] x A[10]\n\t" - "mulxq 80(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "# A[20] x A[11]\n\t" - "mulxq 88(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "movq %%r14, 232(%[tmp])\n\t" - "movq %%r15, 240(%[tmp])\n\t" - "movq %%r10, 248(%[tmp])\n\t" - "# Carry\n\t" - "adcxq %%r9, %%r11\n\t" - "movq %%r8, %%r9\n\t" - "adcxq %%r8, %%r9\n\t" - "adoxq %%r8, %%r9\n\t" - "movq %%r11, 256(%[tmp])\n\t" - "xorq %%r8, %%r8\n\t" - "# Diagonal 6\n\t" - "movq 88(%[tmp]), %%r11\n\t" - "movq 96(%[tmp]), %%r12\n\t" - "movq 104(%[tmp]), %%r13\n\t" - "movq 112(%[tmp]), %%r14\n\t" - "movq 120(%[tmp]), %%r15\n\t" - "movq 128(%[tmp]), %%r10\n\t" - "# A[6] x A[5]\n\t" - "movq 40(%[a]), %%rdx\n\t" - "mulxq 48(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "# A[7] x A[5]\n\t" - "mulxq 56(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "# A[8] x A[5]\n\t" - "mulxq 64(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "# A[9] x A[5]\n\t" - "mulxq 72(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "# A[10] x A[5]\n\t" - "mulxq 80(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "movq %%r11, 88(%[tmp])\n\t" - "movq %%r12, 96(%[tmp])\n\t" - "movq %%r13, 104(%[tmp])\n\t" - "movq %%r14, 112(%[tmp])\n\t" - "movq %%r15, 120(%[tmp])\n\t" - "movq 136(%[tmp]), %%r11\n\t" - "movq 144(%[tmp]), %%r12\n\t" - "movq 152(%[tmp]), %%r13\n\t" - "movq 160(%[tmp]), %%r14\n\t" - "movq 168(%[tmp]), %%r15\n\t" - "# A[11] x A[5]\n\t" - "mulxq 88(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "# A[12] x A[5]\n\t" - "mulxq 96(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "# A[13] x A[5]\n\t" - "mulxq 104(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "# A[14] x A[5]\n\t" - "mulxq 112(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "# A[15] x A[5]\n\t" - "mulxq 120(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "movq %%r10, 128(%[tmp])\n\t" - "movq %%r11, 136(%[tmp])\n\t" - "movq %%r12, 144(%[tmp])\n\t" - "movq %%r13, 152(%[tmp])\n\t" - "movq %%r14, 160(%[tmp])\n\t" - "movq 176(%[tmp]), %%r10\n\t" - "movq 184(%[tmp]), %%r11\n\t" - "movq 192(%[tmp]), %%r12\n\t" - "movq 200(%[tmp]), %%r13\n\t" - "movq 208(%[tmp]), %%r14\n\t" - "# A[16] x A[5]\n\t" - "mulxq 128(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "# A[17] x A[5]\n\t" - "mulxq 136(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "# A[18] x A[5]\n\t" - "mulxq 144(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "# A[19] x A[5]\n\t" - "mulxq 152(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "# A[19] x A[6]\n\t" - "movq 152(%[a]), %%rdx\n\t" - "mulxq 48(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "movq %%r15, 168(%[tmp])\n\t" - "movq %%r10, 176(%[tmp])\n\t" - "movq %%r11, 184(%[tmp])\n\t" - "movq %%r12, 192(%[tmp])\n\t" - "movq %%r13, 200(%[tmp])\n\t" - "movq 216(%[tmp]), %%r15\n\t" - "movq 224(%[tmp]), %%r10\n\t" - "movq 232(%[tmp]), %%r11\n\t" - "movq 240(%[tmp]), %%r12\n\t" - "movq 248(%[tmp]), %%r13\n\t" - "# A[19] x A[7]\n\t" - "mulxq 56(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "# A[19] x A[8]\n\t" - "mulxq 64(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "# A[19] x A[9]\n\t" - "mulxq 72(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "# A[19] x A[10]\n\t" - "mulxq 80(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "# A[19] x A[11]\n\t" - "mulxq 88(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "movq %%r14, 208(%[tmp])\n\t" - "movq %%r15, 216(%[tmp])\n\t" - "movq %%r10, 224(%[tmp])\n\t" - "movq %%r11, 232(%[tmp])\n\t" - "movq %%r12, 240(%[tmp])\n\t" - "movq 256(%[tmp]), %%r14\n\t" - "movq %%r8, %%r15\n\t" - "movq %%r8, %%r10\n\t" - "# A[19] x A[12]\n\t" - "mulxq 96(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "# A[19] x A[13]\n\t" - "mulxq 104(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "# A[19] x A[14]\n\t" - "mulxq 112(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "movq %%r13, 248(%[tmp])\n\t" - "movq %%r14, 256(%[tmp])\n\t" - "movq %%r15, 264(%[tmp])\n\t" - "# Carry\n\t" - "adcxq %%r9, %%r10\n\t" - "movq %%r8, %%r9\n\t" - "adcxq %%r8, %%r9\n\t" - "adoxq %%r8, %%r9\n\t" - "movq %%r10, 272(%[tmp])\n\t" - "xorq %%r8, %%r8\n\t" - "# Diagonal 7\n\t" - "movq 104(%[tmp]), %%r10\n\t" - "movq 112(%[tmp]), %%r11\n\t" - "movq 120(%[tmp]), %%r12\n\t" - "movq 128(%[tmp]), %%r13\n\t" - "movq 136(%[tmp]), %%r14\n\t" - "movq 144(%[tmp]), %%r15\n\t" - "# A[7] x A[6]\n\t" - "movq 48(%[a]), %%rdx\n\t" - "mulxq 56(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "# A[8] x A[6]\n\t" - "mulxq 64(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "# A[9] x A[6]\n\t" - "mulxq 72(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "# A[10] x A[6]\n\t" - "mulxq 80(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "# A[11] x A[6]\n\t" - "mulxq 88(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "movq %%r10, 104(%[tmp])\n\t" - "movq %%r11, 112(%[tmp])\n\t" - "movq %%r12, 120(%[tmp])\n\t" - "movq %%r13, 128(%[tmp])\n\t" - "movq %%r14, 136(%[tmp])\n\t" - "movq 152(%[tmp]), %%r10\n\t" - "movq 160(%[tmp]), %%r11\n\t" - "movq 168(%[tmp]), %%r12\n\t" - "movq 176(%[tmp]), %%r13\n\t" - "movq 184(%[tmp]), %%r14\n\t" - "# A[12] x A[6]\n\t" - "mulxq 96(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "# A[13] x A[6]\n\t" - "mulxq 104(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "# A[14] x A[6]\n\t" - "mulxq 112(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "# A[15] x A[6]\n\t" - "mulxq 120(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "# A[16] x A[6]\n\t" - "mulxq 128(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "movq %%r15, 144(%[tmp])\n\t" - "movq %%r10, 152(%[tmp])\n\t" - "movq %%r11, 160(%[tmp])\n\t" - "movq %%r12, 168(%[tmp])\n\t" - "movq %%r13, 176(%[tmp])\n\t" - "movq 192(%[tmp]), %%r15\n\t" - "movq 200(%[tmp]), %%r10\n\t" - "movq 208(%[tmp]), %%r11\n\t" - "movq 216(%[tmp]), %%r12\n\t" - "movq 224(%[tmp]), %%r13\n\t" - "# A[17] x A[6]\n\t" - "mulxq 136(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "# A[18] x A[6]\n\t" - "mulxq 144(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "# A[18] x A[7]\n\t" - "movq 144(%[a]), %%rdx\n\t" - "mulxq 56(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "# A[18] x A[8]\n\t" - "mulxq 64(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "# A[18] x A[9]\n\t" - "mulxq 72(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "movq %%r14, 184(%[tmp])\n\t" - "movq %%r15, 192(%[tmp])\n\t" - "movq %%r10, 200(%[tmp])\n\t" - "movq %%r11, 208(%[tmp])\n\t" - "movq %%r12, 216(%[tmp])\n\t" - "movq 232(%[tmp]), %%r14\n\t" - "movq 240(%[tmp]), %%r15\n\t" - "movq 248(%[tmp]), %%r10\n\t" - "movq 256(%[tmp]), %%r11\n\t" - "movq 264(%[tmp]), %%r12\n\t" - "# A[18] x A[10]\n\t" - "mulxq 80(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "# A[18] x A[11]\n\t" - "mulxq 88(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "# A[18] x A[12]\n\t" - "mulxq 96(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "# A[18] x A[13]\n\t" - "mulxq 104(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "# A[18] x A[14]\n\t" - "mulxq 112(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "movq %%r13, 224(%[tmp])\n\t" - "movq %%r14, 232(%[tmp])\n\t" - "movq %%r15, 240(%[tmp])\n\t" - "movq %%r10, 248(%[tmp])\n\t" - "movq %%r11, 256(%[tmp])\n\t" - "movq 272(%[tmp]), %%r13\n\t" - "movq %%r8, %%r14\n\t" - "movq %%r8, %%r15\n\t" - "# A[18] x A[15]\n\t" - "mulxq 120(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "# A[18] x A[16]\n\t" - "mulxq 128(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "# A[18] x A[17]\n\t" - "mulxq 136(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "movq %%r12, 264(%[tmp])\n\t" - "movq %%r13, 272(%[tmp])\n\t" - "movq %%r14, 280(%[tmp])\n\t" - "# Carry\n\t" - "adcxq %%r9, %%r15\n\t" - "movq %%r8, %%r9\n\t" - "adcxq %%r8, %%r9\n\t" - "adoxq %%r8, %%r9\n\t" - "movq %%r15, 288(%[tmp])\n\t" - "xorq %%r8, %%r8\n\t" - "# Diagonal 8\n\t" - "movq 120(%[tmp]), %%r15\n\t" - "movq 128(%[tmp]), %%r10\n\t" - "movq 136(%[tmp]), %%r11\n\t" - "movq 144(%[tmp]), %%r12\n\t" - "movq 152(%[tmp]), %%r13\n\t" - "movq 160(%[tmp]), %%r14\n\t" - "# A[8] x A[7]\n\t" - "movq 56(%[a]), %%rdx\n\t" - "mulxq 64(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "# A[9] x A[7]\n\t" - "mulxq 72(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "# A[10] x A[7]\n\t" - "mulxq 80(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "# A[11] x A[7]\n\t" - "mulxq 88(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "# A[12] x A[7]\n\t" - "mulxq 96(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "movq %%r15, 120(%[tmp])\n\t" - "movq %%r10, 128(%[tmp])\n\t" - "movq %%r11, 136(%[tmp])\n\t" - "movq %%r12, 144(%[tmp])\n\t" - "movq %%r13, 152(%[tmp])\n\t" - "movq 168(%[tmp]), %%r15\n\t" - "movq 176(%[tmp]), %%r10\n\t" - "movq 184(%[tmp]), %%r11\n\t" - "movq 192(%[tmp]), %%r12\n\t" - "movq 200(%[tmp]), %%r13\n\t" - "# A[13] x A[7]\n\t" - "mulxq 104(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "# A[14] x A[7]\n\t" - "mulxq 112(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "# A[15] x A[7]\n\t" - "mulxq 120(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "# A[16] x A[7]\n\t" - "mulxq 128(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "# A[17] x A[7]\n\t" - "mulxq 136(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "movq %%r14, 160(%[tmp])\n\t" - "movq %%r15, 168(%[tmp])\n\t" - "movq %%r10, 176(%[tmp])\n\t" - "movq %%r11, 184(%[tmp])\n\t" - "movq %%r12, 192(%[tmp])\n\t" - "movq 208(%[tmp]), %%r14\n\t" - "movq 216(%[tmp]), %%r15\n\t" - "movq 224(%[tmp]), %%r10\n\t" - "movq 232(%[tmp]), %%r11\n\t" - "movq 240(%[tmp]), %%r12\n\t" - "# A[17] x A[8]\n\t" - "movq 136(%[a]), %%rdx\n\t" - "mulxq 64(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "# A[17] x A[9]\n\t" - "mulxq 72(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "# A[17] x A[10]\n\t" - "mulxq 80(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "# A[17] x A[11]\n\t" - "mulxq 88(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "# A[17] x A[12]\n\t" - "mulxq 96(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "movq %%r13, 200(%[tmp])\n\t" - "movq %%r14, 208(%[tmp])\n\t" - "movq %%r15, 216(%[tmp])\n\t" - "movq %%r10, 224(%[tmp])\n\t" - "movq %%r11, 232(%[tmp])\n\t" - "movq 248(%[tmp]), %%r13\n\t" - "movq 256(%[tmp]), %%r14\n\t" - "movq 264(%[tmp]), %%r15\n\t" - "movq 272(%[tmp]), %%r10\n\t" - "movq 280(%[tmp]), %%r11\n\t" - "# A[17] x A[13]\n\t" - "mulxq 104(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "# A[17] x A[14]\n\t" - "mulxq 112(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "# A[17] x A[15]\n\t" - "mulxq 120(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "# A[17] x A[16]\n\t" - "mulxq 128(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "# A[19] x A[15]\n\t" - "movq 152(%[a]), %%rdx\n\t" - "mulxq 120(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "movq %%r12, 240(%[tmp])\n\t" - "movq %%r13, 248(%[tmp])\n\t" - "movq %%r14, 256(%[tmp])\n\t" - "movq %%r15, 264(%[tmp])\n\t" - "movq %%r10, 272(%[tmp])\n\t" - "movq 288(%[tmp]), %%r12\n\t" - "movq %%r8, %%r13\n\t" - "movq %%r8, %%r14\n\t" - "# A[19] x A[16]\n\t" - "mulxq 128(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "# A[19] x A[17]\n\t" - "mulxq 136(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "# A[19] x A[18]\n\t" - "mulxq 144(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "movq %%r11, 280(%[tmp])\n\t" - "movq %%r12, 288(%[tmp])\n\t" - "movq %%r13, 296(%[tmp])\n\t" - "# Carry\n\t" - "adcxq %%r9, %%r14\n\t" - "movq %%r8, %%r9\n\t" - "adcxq %%r8, %%r9\n\t" - "adoxq %%r8, %%r9\n\t" - "movq %%r14, 304(%[tmp])\n\t" - "xorq %%r8, %%r8\n\t" - "# Diagonal 9\n\t" - "movq 136(%[tmp]), %%r14\n\t" - "movq 144(%[tmp]), %%r15\n\t" - "movq 152(%[tmp]), %%r10\n\t" - "movq 160(%[tmp]), %%r11\n\t" - "movq 168(%[tmp]), %%r12\n\t" - "movq 176(%[tmp]), %%r13\n\t" - "# A[9] x A[8]\n\t" - "movq 64(%[a]), %%rdx\n\t" - "mulxq 72(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "# A[10] x A[8]\n\t" - "mulxq 80(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "# A[11] x A[8]\n\t" - "mulxq 88(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "# A[12] x A[8]\n\t" - "mulxq 96(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "# A[13] x A[8]\n\t" - "mulxq 104(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "movq %%r14, 136(%[tmp])\n\t" - "movq %%r15, 144(%[tmp])\n\t" - "movq %%r10, 152(%[tmp])\n\t" - "movq %%r11, 160(%[tmp])\n\t" - "movq %%r12, 168(%[tmp])\n\t" - "movq 184(%[tmp]), %%r14\n\t" - "movq 192(%[tmp]), %%r15\n\t" - "movq 200(%[tmp]), %%r10\n\t" - "movq 208(%[tmp]), %%r11\n\t" - "movq 216(%[tmp]), %%r12\n\t" - "# A[14] x A[8]\n\t" - "mulxq 112(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "# A[15] x A[8]\n\t" - "mulxq 120(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "# A[16] x A[8]\n\t" - "mulxq 128(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "# A[16] x A[9]\n\t" - "movq 128(%[a]), %%rdx\n\t" - "mulxq 72(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "# A[16] x A[10]\n\t" - "mulxq 80(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "movq %%r13, 176(%[tmp])\n\t" - "movq %%r14, 184(%[tmp])\n\t" - "movq %%r15, 192(%[tmp])\n\t" - "movq %%r10, 200(%[tmp])\n\t" - "movq %%r11, 208(%[tmp])\n\t" - "movq 224(%[tmp]), %%r13\n\t" - "movq 232(%[tmp]), %%r14\n\t" - "movq 240(%[tmp]), %%r15\n\t" - "movq 248(%[tmp]), %%r10\n\t" - "movq 256(%[tmp]), %%r11\n\t" - "# A[16] x A[11]\n\t" - "mulxq 88(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "# A[16] x A[12]\n\t" - "mulxq 96(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "# A[16] x A[13]\n\t" - "mulxq 104(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "# A[16] x A[14]\n\t" - "mulxq 112(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "# A[16] x A[15]\n\t" - "mulxq 120(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "movq %%r12, 216(%[tmp])\n\t" - "movq %%r13, 224(%[tmp])\n\t" - "movq %%r14, 232(%[tmp])\n\t" - "movq %%r15, 240(%[tmp])\n\t" - "movq %%r10, 248(%[tmp])\n\t" - "movq 264(%[tmp]), %%r12\n\t" - "movq 272(%[tmp]), %%r13\n\t" - "movq 280(%[tmp]), %%r14\n\t" - "movq 288(%[tmp]), %%r15\n\t" - "movq 296(%[tmp]), %%r10\n\t" - "# A[20] x A[12]\n\t" - "movq 160(%[a]), %%rdx\n\t" - "mulxq 96(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "# A[20] x A[13]\n\t" - "mulxq 104(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "# A[20] x A[14]\n\t" - "mulxq 112(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "# A[20] x A[15]\n\t" - "mulxq 120(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "# A[20] x A[16]\n\t" - "mulxq 128(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "movq %%r11, 256(%[tmp])\n\t" - "movq %%r12, 264(%[tmp])\n\t" - "movq %%r13, 272(%[tmp])\n\t" - "movq %%r14, 280(%[tmp])\n\t" - "movq %%r15, 288(%[tmp])\n\t" - "movq 304(%[tmp]), %%r11\n\t" - "movq %%r8, %%r12\n\t" - "movq %%r8, %%r13\n\t" - "# A[20] x A[17]\n\t" - "mulxq 136(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "# A[20] x A[18]\n\t" - "mulxq 144(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "# A[20] x A[19]\n\t" - "mulxq 152(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "movq %%r10, 296(%[tmp])\n\t" - "movq %%r11, 304(%[tmp])\n\t" - "movq %%r12, 312(%[tmp])\n\t" - "# Carry\n\t" - "adcxq %%r9, %%r13\n\t" - "movq %%r8, %%r9\n\t" - "adcxq %%r8, %%r9\n\t" - "adoxq %%r8, %%r9\n\t" - "movq %%r13, 320(%[tmp])\n\t" - "xorq %%r8, %%r8\n\t" - "# Diagonal 10\n\t" - "movq 152(%[tmp]), %%r13\n\t" - "movq 160(%[tmp]), %%r14\n\t" - "movq 168(%[tmp]), %%r15\n\t" - "movq 176(%[tmp]), %%r10\n\t" - "movq 184(%[tmp]), %%r11\n\t" - "movq 192(%[tmp]), %%r12\n\t" - "# A[10] x A[9]\n\t" - "movq 72(%[a]), %%rdx\n\t" - "mulxq 80(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "# A[11] x A[9]\n\t" - "mulxq 88(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "# A[12] x A[9]\n\t" - "mulxq 96(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "# A[13] x A[9]\n\t" - "mulxq 104(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "# A[14] x A[9]\n\t" - "mulxq 112(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "movq %%r13, 152(%[tmp])\n\t" - "movq %%r14, 160(%[tmp])\n\t" - "movq %%r15, 168(%[tmp])\n\t" - "movq %%r10, 176(%[tmp])\n\t" - "movq %%r11, 184(%[tmp])\n\t" - "movq 200(%[tmp]), %%r13\n\t" - "movq 208(%[tmp]), %%r14\n\t" - "movq 216(%[tmp]), %%r15\n\t" - "movq 224(%[tmp]), %%r10\n\t" - "movq 232(%[tmp]), %%r11\n\t" - "# A[15] x A[9]\n\t" - "mulxq 120(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "# A[15] x A[10]\n\t" - "movq 120(%[a]), %%rdx\n\t" - "mulxq 80(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "# A[15] x A[11]\n\t" - "mulxq 88(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "# A[15] x A[12]\n\t" - "mulxq 96(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "# A[15] x A[13]\n\t" - "mulxq 104(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "movq %%r12, 192(%[tmp])\n\t" - "movq %%r13, 200(%[tmp])\n\t" - "movq %%r14, 208(%[tmp])\n\t" - "movq %%r15, 216(%[tmp])\n\t" - "movq %%r10, 224(%[tmp])\n\t" - "movq 240(%[tmp]), %%r12\n\t" - "movq 248(%[tmp]), %%r13\n\t" - "movq 256(%[tmp]), %%r14\n\t" - "movq 264(%[tmp]), %%r15\n\t" - "movq 272(%[tmp]), %%r10\n\t" - "# A[15] x A[14]\n\t" - "mulxq 112(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "# A[21] x A[9]\n\t" - "movq 168(%[a]), %%rdx\n\t" - "mulxq 72(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "# A[21] x A[10]\n\t" - "mulxq 80(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "# A[21] x A[11]\n\t" - "mulxq 88(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "# A[21] x A[12]\n\t" - "mulxq 96(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "movq %%r11, 232(%[tmp])\n\t" - "movq %%r12, 240(%[tmp])\n\t" - "movq %%r13, 248(%[tmp])\n\t" - "movq %%r14, 256(%[tmp])\n\t" - "movq %%r15, 264(%[tmp])\n\t" - "movq 280(%[tmp]), %%r11\n\t" - "movq 288(%[tmp]), %%r12\n\t" - "movq 296(%[tmp]), %%r13\n\t" - "movq 304(%[tmp]), %%r14\n\t" - "movq 312(%[tmp]), %%r15\n\t" - "# A[21] x A[13]\n\t" - "mulxq 104(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "# A[21] x A[14]\n\t" - "mulxq 112(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "# A[21] x A[15]\n\t" - "mulxq 120(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "# A[21] x A[16]\n\t" - "mulxq 128(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "# A[21] x A[17]\n\t" - "mulxq 136(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "movq %%r10, 272(%[tmp])\n\t" - "movq %%r11, 280(%[tmp])\n\t" - "movq %%r12, 288(%[tmp])\n\t" - "movq %%r13, 296(%[tmp])\n\t" - "movq %%r14, 304(%[tmp])\n\t" - "movq 320(%[tmp]), %%r10\n\t" - "movq %%r8, %%r11\n\t" - "movq %%r8, %%r12\n\t" - "# A[21] x A[18]\n\t" - "mulxq 144(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "# A[21] x A[19]\n\t" - "mulxq 152(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "# A[21] x A[20]\n\t" - "mulxq 160(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "movq %%r15, 312(%[tmp])\n\t" - "movq %%r10, 320(%[tmp])\n\t" - "movq %%r11, 328(%[tmp])\n\t" - "# Carry\n\t" - "adcxq %%r9, %%r12\n\t" - "movq %%r8, %%r9\n\t" - "adcxq %%r8, %%r9\n\t" - "adoxq %%r8, %%r9\n\t" - "movq %%r12, 336(%[tmp])\n\t" - "xorq %%r8, %%r8\n\t" - "# Diagonal 11\n\t" - "movq 168(%[tmp]), %%r12\n\t" - "movq 176(%[tmp]), %%r13\n\t" - "movq 184(%[tmp]), %%r14\n\t" - "movq 192(%[tmp]), %%r15\n\t" - "movq 200(%[tmp]), %%r10\n\t" - "movq 208(%[tmp]), %%r11\n\t" - "# A[11] x A[10]\n\t" - "movq 80(%[a]), %%rdx\n\t" - "mulxq 88(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "# A[12] x A[10]\n\t" - "mulxq 96(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "# A[13] x A[10]\n\t" - "mulxq 104(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "# A[14] x A[10]\n\t" - "mulxq 112(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "# A[14] x A[11]\n\t" - "movq 112(%[a]), %%rdx\n\t" - "mulxq 88(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "movq %%r12, 168(%[tmp])\n\t" - "movq %%r13, 176(%[tmp])\n\t" - "movq %%r14, 184(%[tmp])\n\t" - "movq %%r15, 192(%[tmp])\n\t" - "movq %%r10, 200(%[tmp])\n\t" - "movq 216(%[tmp]), %%r12\n\t" - "movq 224(%[tmp]), %%r13\n\t" - "movq 232(%[tmp]), %%r14\n\t" - "movq 240(%[tmp]), %%r15\n\t" - "movq 248(%[tmp]), %%r10\n\t" - "# A[14] x A[12]\n\t" - "mulxq 96(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "# A[14] x A[13]\n\t" - "mulxq 104(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "# A[22] x A[6]\n\t" - "movq 176(%[a]), %%rdx\n\t" - "mulxq 48(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "# A[22] x A[7]\n\t" - "mulxq 56(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "# A[22] x A[8]\n\t" - "mulxq 64(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "movq %%r11, 208(%[tmp])\n\t" - "movq %%r12, 216(%[tmp])\n\t" - "movq %%r13, 224(%[tmp])\n\t" - "movq %%r14, 232(%[tmp])\n\t" - "movq %%r15, 240(%[tmp])\n\t" - "movq 256(%[tmp]), %%r11\n\t" - "movq 264(%[tmp]), %%r12\n\t" - "movq 272(%[tmp]), %%r13\n\t" - "movq 280(%[tmp]), %%r14\n\t" - "movq 288(%[tmp]), %%r15\n\t" - "# A[22] x A[9]\n\t" - "mulxq 72(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "# A[22] x A[10]\n\t" - "mulxq 80(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "# A[22] x A[11]\n\t" - "mulxq 88(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "# A[22] x A[12]\n\t" - "mulxq 96(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "# A[22] x A[13]\n\t" - "mulxq 104(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "movq %%r10, 248(%[tmp])\n\t" - "movq %%r11, 256(%[tmp])\n\t" - "movq %%r12, 264(%[tmp])\n\t" - "movq %%r13, 272(%[tmp])\n\t" - "movq %%r14, 280(%[tmp])\n\t" - "movq 296(%[tmp]), %%r10\n\t" - "movq 304(%[tmp]), %%r11\n\t" - "movq 312(%[tmp]), %%r12\n\t" - "movq 320(%[tmp]), %%r13\n\t" - "movq 328(%[tmp]), %%r14\n\t" - "# A[22] x A[14]\n\t" - "mulxq 112(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "# A[22] x A[15]\n\t" - "mulxq 120(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "# A[22] x A[16]\n\t" - "mulxq 128(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "# A[22] x A[17]\n\t" - "mulxq 136(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "# A[22] x A[18]\n\t" - "mulxq 144(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "movq %%r15, 288(%[tmp])\n\t" - "movq %%r10, 296(%[tmp])\n\t" - "movq %%r11, 304(%[tmp])\n\t" - "movq %%r12, 312(%[tmp])\n\t" - "movq %%r13, 320(%[tmp])\n\t" - "movq 336(%[tmp]), %%r15\n\t" - "movq %%r8, %%r10\n\t" - "movq %%r8, %%r11\n\t" - "# A[22] x A[19]\n\t" - "mulxq 152(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "# A[22] x A[20]\n\t" - "mulxq 160(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "# A[22] x A[21]\n\t" - "mulxq 168(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "movq %%r14, 328(%[tmp])\n\t" - "movq %%r15, 336(%[tmp])\n\t" - "movq %%r10, 344(%[tmp])\n\t" - "# Carry\n\t" - "adcxq %%r9, %%r11\n\t" - "movq %%r8, %%r9\n\t" - "adcxq %%r8, %%r9\n\t" - "adoxq %%r8, %%r9\n\t" - "movq %%r11, 352(%[tmp])\n\t" - "xorq %%r8, %%r8\n\t" - "# Diagonal 12\n\t" - "movq 184(%[tmp]), %%r11\n\t" - "movq 192(%[tmp]), %%r12\n\t" - "movq 200(%[tmp]), %%r13\n\t" - "movq 208(%[tmp]), %%r14\n\t" - "movq 216(%[tmp]), %%r15\n\t" - "movq 224(%[tmp]), %%r10\n\t" - "# A[12] x A[11]\n\t" - "movq 88(%[a]), %%rdx\n\t" - "mulxq 96(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "# A[13] x A[11]\n\t" - "mulxq 104(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "# A[13] x A[12]\n\t" - "movq 96(%[a]), %%rdx\n\t" - "mulxq 104(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "# A[23] x A[3]\n\t" - "movq 184(%[a]), %%rdx\n\t" - "mulxq 24(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "# A[23] x A[4]\n\t" - "mulxq 32(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "movq %%r11, 184(%[tmp])\n\t" - "movq %%r12, 192(%[tmp])\n\t" - "movq %%r13, 200(%[tmp])\n\t" - "movq %%r14, 208(%[tmp])\n\t" - "movq %%r15, 216(%[tmp])\n\t" - "movq 232(%[tmp]), %%r11\n\t" - "movq 240(%[tmp]), %%r12\n\t" - "movq 248(%[tmp]), %%r13\n\t" - "movq 256(%[tmp]), %%r14\n\t" - "movq 264(%[tmp]), %%r15\n\t" - "# A[23] x A[5]\n\t" - "mulxq 40(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "# A[23] x A[6]\n\t" - "mulxq 48(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "# A[23] x A[7]\n\t" - "mulxq 56(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "# A[23] x A[8]\n\t" - "mulxq 64(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "# A[23] x A[9]\n\t" - "mulxq 72(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "movq %%r10, 224(%[tmp])\n\t" - "movq %%r11, 232(%[tmp])\n\t" - "movq %%r12, 240(%[tmp])\n\t" - "movq %%r13, 248(%[tmp])\n\t" - "movq %%r14, 256(%[tmp])\n\t" - "movq 272(%[tmp]), %%r10\n\t" - "movq 280(%[tmp]), %%r11\n\t" - "movq 288(%[tmp]), %%r12\n\t" - "movq 296(%[tmp]), %%r13\n\t" - "movq 304(%[tmp]), %%r14\n\t" - "# A[23] x A[10]\n\t" - "mulxq 80(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "# A[23] x A[11]\n\t" - "mulxq 88(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "# A[23] x A[12]\n\t" - "mulxq 96(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "# A[23] x A[13]\n\t" - "mulxq 104(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "# A[23] x A[14]\n\t" - "mulxq 112(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "movq %%r15, 264(%[tmp])\n\t" - "movq %%r10, 272(%[tmp])\n\t" - "movq %%r11, 280(%[tmp])\n\t" - "movq %%r12, 288(%[tmp])\n\t" - "movq %%r13, 296(%[tmp])\n\t" - "movq 312(%[tmp]), %%r15\n\t" - "movq 320(%[tmp]), %%r10\n\t" - "movq 328(%[tmp]), %%r11\n\t" - "movq 336(%[tmp]), %%r12\n\t" - "movq 344(%[tmp]), %%r13\n\t" - "# A[23] x A[15]\n\t" - "mulxq 120(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "# A[23] x A[16]\n\t" - "mulxq 128(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "# A[23] x A[17]\n\t" - "mulxq 136(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%rcx, %%r11\n\t" - "# A[23] x A[18]\n\t" - "mulxq 144(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%rcx, %%r12\n\t" - "# A[23] x A[19]\n\t" - "mulxq 152(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "movq %%r14, 304(%[tmp])\n\t" - "movq %%r15, 312(%[tmp])\n\t" - "movq %%r10, 320(%[tmp])\n\t" - "movq %%r11, 328(%[tmp])\n\t" - "movq %%r12, 336(%[tmp])\n\t" - "movq 352(%[tmp]), %%r14\n\t" - "movq %%r8, %%r15\n\t" - "movq %%r8, %%r10\n\t" - "# A[23] x A[20]\n\t" - "mulxq 160(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "# A[23] x A[21]\n\t" - "mulxq 168(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "# A[23] x A[22]\n\t" - "mulxq 176(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r15\n\t" - "adoxq %%rcx, %%r10\n\t" - "movq %%r13, 344(%[tmp])\n\t" - "movq %%r14, 352(%[tmp])\n\t" - "movq %%r15, 360(%[tmp])\n\t" - "# Carry\n\t" - "adcxq %%r9, %%r10\n\t" - "movq %%r8, %%r9\n\t" - "adcxq %%r8, %%r9\n\t" - "adoxq %%r8, %%r9\n\t" - "movq %%r10, 368(%[tmp])\n\t" - "movq %%r9, 376(%[tmp])\n\t" - "xorq %%r8, %%r8\n\t" - "# Double and Add in A[i] x A[i]\n\t" - "movq 8(%[tmp]), %%r11\n\t" - "# A[0] x A[0]\n\t" - "movq 0(%[a]), %%rdx\n\t" - "mulxq %%rdx, %%rax, %%rcx\n\t" - "movq %%rax, 0(%[tmp])\n\t" - "adoxq %%r11, %%r11\n\t" - "adcxq %%rcx, %%r11\n\t" - "movq %%r11, 8(%[tmp])\n\t" - "movq 16(%[tmp]), %%r10\n\t" - "movq 24(%[tmp]), %%r11\n\t" - "# A[1] x A[1]\n\t" - "movq 8(%[a]), %%rdx\n\t" - "mulxq %%rdx, %%rax, %%rcx\n\t" - "adoxq %%r10, %%r10\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r11, %%r11\n\t" - "adcxq %%rcx, %%r11\n\t" - "movq %%r10, 16(%[tmp])\n\t" - "movq %%r11, 24(%[tmp])\n\t" - "movq 32(%[tmp]), %%r10\n\t" - "movq 40(%[tmp]), %%r11\n\t" - "# A[2] x A[2]\n\t" - "movq 16(%[a]), %%rdx\n\t" - "mulxq %%rdx, %%rax, %%rcx\n\t" - "adoxq %%r10, %%r10\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r11, %%r11\n\t" - "adcxq %%rcx, %%r11\n\t" - "movq %%r10, 32(%[tmp])\n\t" - "movq %%r11, 40(%[tmp])\n\t" - "movq 48(%[tmp]), %%r10\n\t" - "movq 56(%[tmp]), %%r11\n\t" - "# A[3] x A[3]\n\t" - "movq 24(%[a]), %%rdx\n\t" - "mulxq %%rdx, %%rax, %%rcx\n\t" - "adoxq %%r10, %%r10\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r11, %%r11\n\t" - "adcxq %%rcx, %%r11\n\t" - "movq %%r10, 48(%[tmp])\n\t" - "movq %%r11, 56(%[tmp])\n\t" - "movq 64(%[tmp]), %%r10\n\t" - "movq 72(%[tmp]), %%r11\n\t" - "# A[4] x A[4]\n\t" - "movq 32(%[a]), %%rdx\n\t" - "mulxq %%rdx, %%rax, %%rcx\n\t" - "adoxq %%r10, %%r10\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r11, %%r11\n\t" - "adcxq %%rcx, %%r11\n\t" - "movq %%r10, 64(%[tmp])\n\t" - "movq %%r11, 72(%[tmp])\n\t" - "movq 80(%[tmp]), %%r10\n\t" - "movq 88(%[tmp]), %%r11\n\t" - "# A[5] x A[5]\n\t" - "movq 40(%[a]), %%rdx\n\t" - "mulxq %%rdx, %%rax, %%rcx\n\t" - "adoxq %%r10, %%r10\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r11, %%r11\n\t" - "adcxq %%rcx, %%r11\n\t" - "movq %%r10, 80(%[tmp])\n\t" - "movq %%r11, 88(%[tmp])\n\t" - "movq 96(%[tmp]), %%r10\n\t" - "movq 104(%[tmp]), %%r11\n\t" - "# A[6] x A[6]\n\t" - "movq 48(%[a]), %%rdx\n\t" - "mulxq %%rdx, %%rax, %%rcx\n\t" - "adoxq %%r10, %%r10\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r11, %%r11\n\t" - "adcxq %%rcx, %%r11\n\t" - "movq %%r10, 96(%[tmp])\n\t" - "movq %%r11, 104(%[tmp])\n\t" - "movq 112(%[tmp]), %%r10\n\t" - "movq 120(%[tmp]), %%r11\n\t" - "# A[7] x A[7]\n\t" - "movq 56(%[a]), %%rdx\n\t" - "mulxq %%rdx, %%rax, %%rcx\n\t" - "adoxq %%r10, %%r10\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r11, %%r11\n\t" - "adcxq %%rcx, %%r11\n\t" - "movq %%r10, 112(%[tmp])\n\t" - "movq %%r11, 120(%[tmp])\n\t" - "movq 128(%[tmp]), %%r10\n\t" - "movq 136(%[tmp]), %%r11\n\t" - "# A[8] x A[8]\n\t" - "movq 64(%[a]), %%rdx\n\t" - "mulxq %%rdx, %%rax, %%rcx\n\t" - "adoxq %%r10, %%r10\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r11, %%r11\n\t" - "adcxq %%rcx, %%r11\n\t" - "movq %%r10, 128(%[tmp])\n\t" - "movq %%r11, 136(%[tmp])\n\t" - "movq 144(%[tmp]), %%r10\n\t" - "movq 152(%[tmp]), %%r11\n\t" - "# A[9] x A[9]\n\t" - "movq 72(%[a]), %%rdx\n\t" - "mulxq %%rdx, %%rax, %%rcx\n\t" - "adoxq %%r10, %%r10\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r11, %%r11\n\t" - "adcxq %%rcx, %%r11\n\t" - "movq %%r10, 144(%[tmp])\n\t" - "movq %%r11, 152(%[tmp])\n\t" - "movq 160(%[tmp]), %%r10\n\t" - "movq 168(%[tmp]), %%r11\n\t" - "# A[10] x A[10]\n\t" - "movq 80(%[a]), %%rdx\n\t" - "mulxq %%rdx, %%rax, %%rcx\n\t" - "adoxq %%r10, %%r10\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r11, %%r11\n\t" - "adcxq %%rcx, %%r11\n\t" - "movq %%r10, 160(%[tmp])\n\t" - "movq %%r11, 168(%[tmp])\n\t" - "movq 176(%[tmp]), %%r10\n\t" - "movq 184(%[tmp]), %%r11\n\t" - "# A[11] x A[11]\n\t" - "movq 88(%[a]), %%rdx\n\t" - "mulxq %%rdx, %%rax, %%rcx\n\t" - "adoxq %%r10, %%r10\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r11, %%r11\n\t" - "adcxq %%rcx, %%r11\n\t" - "movq %%r10, 176(%[tmp])\n\t" - "movq %%r11, 184(%[tmp])\n\t" - "movq 192(%[tmp]), %%r10\n\t" - "movq 200(%[tmp]), %%r11\n\t" - "# A[12] x A[12]\n\t" - "movq 96(%[a]), %%rdx\n\t" - "mulxq %%rdx, %%rax, %%rcx\n\t" - "adoxq %%r10, %%r10\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r11, %%r11\n\t" - "adcxq %%rcx, %%r11\n\t" - "movq %%r10, 192(%[r])\n\t" - "movq %%r11, 200(%[r])\n\t" - "movq 208(%[tmp]), %%r10\n\t" - "movq 216(%[tmp]), %%r11\n\t" - "# A[13] x A[13]\n\t" - "movq 104(%[a]), %%rdx\n\t" - "mulxq %%rdx, %%rax, %%rcx\n\t" - "adoxq %%r10, %%r10\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r11, %%r11\n\t" - "adcxq %%rcx, %%r11\n\t" - "movq %%r10, 208(%[r])\n\t" - "movq %%r11, 216(%[r])\n\t" - "movq 224(%[tmp]), %%r10\n\t" - "movq 232(%[tmp]), %%r11\n\t" - "# A[14] x A[14]\n\t" - "movq 112(%[a]), %%rdx\n\t" - "mulxq %%rdx, %%rax, %%rcx\n\t" - "adoxq %%r10, %%r10\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r11, %%r11\n\t" - "adcxq %%rcx, %%r11\n\t" - "movq %%r10, 224(%[r])\n\t" - "movq %%r11, 232(%[r])\n\t" - "movq 240(%[tmp]), %%r10\n\t" - "movq 248(%[tmp]), %%r11\n\t" - "# A[15] x A[15]\n\t" - "movq 120(%[a]), %%rdx\n\t" - "mulxq %%rdx, %%rax, %%rcx\n\t" - "adoxq %%r10, %%r10\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r11, %%r11\n\t" - "adcxq %%rcx, %%r11\n\t" - "movq %%r10, 240(%[r])\n\t" - "movq %%r11, 248(%[r])\n\t" - "movq 256(%[tmp]), %%r10\n\t" - "movq 264(%[tmp]), %%r11\n\t" - "# A[16] x A[16]\n\t" - "movq 128(%[a]), %%rdx\n\t" - "mulxq %%rdx, %%rax, %%rcx\n\t" - "adoxq %%r10, %%r10\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r11, %%r11\n\t" - "adcxq %%rcx, %%r11\n\t" - "movq %%r10, 256(%[r])\n\t" - "movq %%r11, 264(%[r])\n\t" - "movq 272(%[tmp]), %%r10\n\t" - "movq 280(%[tmp]), %%r11\n\t" - "# A[17] x A[17]\n\t" - "movq 136(%[a]), %%rdx\n\t" - "mulxq %%rdx, %%rax, %%rcx\n\t" - "adoxq %%r10, %%r10\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r11, %%r11\n\t" - "adcxq %%rcx, %%r11\n\t" - "movq %%r10, 272(%[r])\n\t" - "movq %%r11, 280(%[r])\n\t" - "movq 288(%[tmp]), %%r10\n\t" - "movq 296(%[tmp]), %%r11\n\t" - "# A[18] x A[18]\n\t" - "movq 144(%[a]), %%rdx\n\t" - "mulxq %%rdx, %%rax, %%rcx\n\t" - "adoxq %%r10, %%r10\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r11, %%r11\n\t" - "adcxq %%rcx, %%r11\n\t" - "movq %%r10, 288(%[r])\n\t" - "movq %%r11, 296(%[r])\n\t" - "movq 304(%[tmp]), %%r10\n\t" - "movq 312(%[tmp]), %%r11\n\t" - "# A[19] x A[19]\n\t" - "movq 152(%[a]), %%rdx\n\t" - "mulxq %%rdx, %%rax, %%rcx\n\t" - "adoxq %%r10, %%r10\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r11, %%r11\n\t" - "adcxq %%rcx, %%r11\n\t" - "movq %%r10, 304(%[r])\n\t" - "movq %%r11, 312(%[r])\n\t" - "movq 320(%[tmp]), %%r10\n\t" - "movq 328(%[tmp]), %%r11\n\t" - "# A[20] x A[20]\n\t" - "movq 160(%[a]), %%rdx\n\t" - "mulxq %%rdx, %%rax, %%rcx\n\t" - "adoxq %%r10, %%r10\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r11, %%r11\n\t" - "adcxq %%rcx, %%r11\n\t" - "movq %%r10, 320(%[r])\n\t" - "movq %%r11, 328(%[r])\n\t" - "movq 336(%[tmp]), %%r10\n\t" - "movq 344(%[tmp]), %%r11\n\t" - "# A[21] x A[21]\n\t" - "movq 168(%[a]), %%rdx\n\t" - "mulxq %%rdx, %%rax, %%rcx\n\t" - "adoxq %%r10, %%r10\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r11, %%r11\n\t" - "adcxq %%rcx, %%r11\n\t" - "movq %%r10, 336(%[r])\n\t" - "movq %%r11, 344(%[r])\n\t" - "movq 352(%[tmp]), %%r10\n\t" - "movq 360(%[tmp]), %%r11\n\t" - "# A[22] x A[22]\n\t" - "movq 176(%[a]), %%rdx\n\t" - "mulxq %%rdx, %%rax, %%rcx\n\t" - "adoxq %%r10, %%r10\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r11, %%r11\n\t" - "adcxq %%rcx, %%r11\n\t" - "movq %%r10, 352(%[r])\n\t" - "movq %%r11, 360(%[r])\n\t" - "movq 368(%[tmp]), %%r10\n\t" - "movq 376(%[tmp]), %%r11\n\t" - "# A[23] x A[23]\n\t" - "movq 184(%[a]), %%rdx\n\t" - "mulxq %%rdx, %%rax, %%rcx\n\t" - "adoxq %%r10, %%r10\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r11, %%r11\n\t" - "adcxq %%rcx, %%r11\n\t" - "movq %%r10, 368(%[r])\n\t" - "movq %%r11, 376(%[r])\n\t" - : - : [r] "r" (r), [a] "r" (a), [tmp] "r" (tmp) - : "memory", "rax", "rdx", "rcx", "r8", "r9", "r10", "r11", - "r12", "r13", "r14", "r15" - ); - - XMEMCPY(r, tmp, sizeof(tmp)/2); -} -#endif /* HAVE_INTEL_AVX2 */ - -/* Add b to a into r. (r = a + b) - * - * r A single precision integer. - * a A single precision integer. - * b A single precision integer. - */ -SP_NOINLINE static sp_digit sp_3072_add_24(sp_digit* r, const sp_digit* a, - const sp_digit* b) -{ - sp_digit c = 0; - - __asm__ __volatile__ ( - "movq (%[a]), %%rax\n\t" - "addq (%[b]), %%rax\n\t" - "movq %%rax, (%[r])\n\t" - "movq 8(%[a]), %%rax\n\t" - "adcq 8(%[b]), %%rax\n\t" - "movq %%rax, 8(%[r])\n\t" - "movq 16(%[a]), %%rax\n\t" - "adcq 16(%[b]), %%rax\n\t" - "movq %%rax, 16(%[r])\n\t" - "movq 24(%[a]), %%rax\n\t" - "adcq 24(%[b]), %%rax\n\t" - "movq %%rax, 24(%[r])\n\t" - "movq 32(%[a]), %%rax\n\t" - "adcq 32(%[b]), %%rax\n\t" - "movq %%rax, 32(%[r])\n\t" - "movq 40(%[a]), %%rax\n\t" - "adcq 40(%[b]), %%rax\n\t" - "movq %%rax, 40(%[r])\n\t" - "movq 48(%[a]), %%rax\n\t" - "adcq 48(%[b]), %%rax\n\t" - "movq %%rax, 48(%[r])\n\t" - "movq 56(%[a]), %%rax\n\t" - "adcq 56(%[b]), %%rax\n\t" - "movq %%rax, 56(%[r])\n\t" - "movq 64(%[a]), %%rax\n\t" - "adcq 64(%[b]), %%rax\n\t" - "movq %%rax, 64(%[r])\n\t" - "movq 72(%[a]), %%rax\n\t" - "adcq 72(%[b]), %%rax\n\t" - "movq %%rax, 72(%[r])\n\t" - "movq 80(%[a]), %%rax\n\t" - "adcq 80(%[b]), %%rax\n\t" - "movq %%rax, 80(%[r])\n\t" - "movq 88(%[a]), %%rax\n\t" - "adcq 88(%[b]), %%rax\n\t" - "movq %%rax, 88(%[r])\n\t" - "movq 96(%[a]), %%rax\n\t" - "adcq 96(%[b]), %%rax\n\t" - "movq %%rax, 96(%[r])\n\t" - "movq 104(%[a]), %%rax\n\t" - "adcq 104(%[b]), %%rax\n\t" - "movq %%rax, 104(%[r])\n\t" - "movq 112(%[a]), %%rax\n\t" - "adcq 112(%[b]), %%rax\n\t" - "movq %%rax, 112(%[r])\n\t" - "movq 120(%[a]), %%rax\n\t" - "adcq 120(%[b]), %%rax\n\t" - "movq %%rax, 120(%[r])\n\t" - "movq 128(%[a]), %%rax\n\t" - "adcq 128(%[b]), %%rax\n\t" - "movq %%rax, 128(%[r])\n\t" - "movq 136(%[a]), %%rax\n\t" - "adcq 136(%[b]), %%rax\n\t" - "movq %%rax, 136(%[r])\n\t" - "movq 144(%[a]), %%rax\n\t" - "adcq 144(%[b]), %%rax\n\t" - "movq %%rax, 144(%[r])\n\t" - "movq 152(%[a]), %%rax\n\t" - "adcq 152(%[b]), %%rax\n\t" - "movq %%rax, 152(%[r])\n\t" - "movq 160(%[a]), %%rax\n\t" - "adcq 160(%[b]), %%rax\n\t" - "movq %%rax, 160(%[r])\n\t" - "movq 168(%[a]), %%rax\n\t" - "adcq 168(%[b]), %%rax\n\t" - "movq %%rax, 168(%[r])\n\t" - "movq 176(%[a]), %%rax\n\t" - "adcq 176(%[b]), %%rax\n\t" - "movq %%rax, 176(%[r])\n\t" - "movq 184(%[a]), %%rax\n\t" - "adcq 184(%[b]), %%rax\n\t" - "movq %%rax, 184(%[r])\n\t" - "adcq $0, %[c]\n\t" - : [c] "+r" (c) - : [r] "r" (r), [a] "r" (a), [b] "r" (b) - : "memory", "rax" - ); - - return c; -} - -/* Sub b from a into a. (a -= b) - * - * a A single precision integer and result. - * b A single precision integer. - */ -SP_NOINLINE static sp_digit sp_3072_sub_in_place_48(sp_digit* a, - const sp_digit* b) -{ - sp_digit c = 0; - - __asm__ __volatile__ ( - "movq 0(%[a]), %%r8\n\t" - "movq 8(%[a]), %%r9\n\t" - "movq 0(%[b]), %%rdx\n\t" - "movq 8(%[b]), %%rcx\n\t" - "subq %%rdx, %%r8\n\t" - "movq 16(%[b]), %%rdx\n\t" - "movq %%r8, 0(%[a])\n\t" - "movq 16(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 24(%[b]), %%rcx\n\t" - "movq %%r9, 8(%[a])\n\t" - "movq 24(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq 32(%[b]), %%rdx\n\t" - "movq %%r8, 16(%[a])\n\t" - "movq 32(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 40(%[b]), %%rcx\n\t" - "movq %%r9, 24(%[a])\n\t" - "movq 40(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq 48(%[b]), %%rdx\n\t" - "movq %%r8, 32(%[a])\n\t" - "movq 48(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 56(%[b]), %%rcx\n\t" - "movq %%r9, 40(%[a])\n\t" - "movq 56(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq 64(%[b]), %%rdx\n\t" - "movq %%r8, 48(%[a])\n\t" - "movq 64(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 72(%[b]), %%rcx\n\t" - "movq %%r9, 56(%[a])\n\t" - "movq 72(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq 80(%[b]), %%rdx\n\t" - "movq %%r8, 64(%[a])\n\t" - "movq 80(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 88(%[b]), %%rcx\n\t" - "movq %%r9, 72(%[a])\n\t" - "movq 88(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq 96(%[b]), %%rdx\n\t" - "movq %%r8, 80(%[a])\n\t" - "movq 96(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 104(%[b]), %%rcx\n\t" - "movq %%r9, 88(%[a])\n\t" - "movq 104(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq 112(%[b]), %%rdx\n\t" - "movq %%r8, 96(%[a])\n\t" - "movq 112(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 120(%[b]), %%rcx\n\t" - "movq %%r9, 104(%[a])\n\t" - "movq 120(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq 128(%[b]), %%rdx\n\t" - "movq %%r8, 112(%[a])\n\t" - "movq 128(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 136(%[b]), %%rcx\n\t" - "movq %%r9, 120(%[a])\n\t" - "movq 136(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq 144(%[b]), %%rdx\n\t" - "movq %%r8, 128(%[a])\n\t" - "movq 144(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 152(%[b]), %%rcx\n\t" - "movq %%r9, 136(%[a])\n\t" - "movq 152(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq 160(%[b]), %%rdx\n\t" - "movq %%r8, 144(%[a])\n\t" - "movq 160(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 168(%[b]), %%rcx\n\t" - "movq %%r9, 152(%[a])\n\t" - "movq 168(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq 176(%[b]), %%rdx\n\t" - "movq %%r8, 160(%[a])\n\t" - "movq 176(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 184(%[b]), %%rcx\n\t" - "movq %%r9, 168(%[a])\n\t" - "movq 184(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq 192(%[b]), %%rdx\n\t" - "movq %%r8, 176(%[a])\n\t" - "movq 192(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 200(%[b]), %%rcx\n\t" - "movq %%r9, 184(%[a])\n\t" - "movq 200(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq 208(%[b]), %%rdx\n\t" - "movq %%r8, 192(%[a])\n\t" - "movq 208(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 216(%[b]), %%rcx\n\t" - "movq %%r9, 200(%[a])\n\t" - "movq 216(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq 224(%[b]), %%rdx\n\t" - "movq %%r8, 208(%[a])\n\t" - "movq 224(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 232(%[b]), %%rcx\n\t" - "movq %%r9, 216(%[a])\n\t" - "movq 232(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq 240(%[b]), %%rdx\n\t" - "movq %%r8, 224(%[a])\n\t" - "movq 240(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 248(%[b]), %%rcx\n\t" - "movq %%r9, 232(%[a])\n\t" - "movq 248(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq 256(%[b]), %%rdx\n\t" - "movq %%r8, 240(%[a])\n\t" - "movq 256(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 264(%[b]), %%rcx\n\t" - "movq %%r9, 248(%[a])\n\t" - "movq 264(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq 272(%[b]), %%rdx\n\t" - "movq %%r8, 256(%[a])\n\t" - "movq 272(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 280(%[b]), %%rcx\n\t" - "movq %%r9, 264(%[a])\n\t" - "movq 280(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq 288(%[b]), %%rdx\n\t" - "movq %%r8, 272(%[a])\n\t" - "movq 288(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 296(%[b]), %%rcx\n\t" - "movq %%r9, 280(%[a])\n\t" - "movq 296(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq 304(%[b]), %%rdx\n\t" - "movq %%r8, 288(%[a])\n\t" - "movq 304(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 312(%[b]), %%rcx\n\t" - "movq %%r9, 296(%[a])\n\t" - "movq 312(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq 320(%[b]), %%rdx\n\t" - "movq %%r8, 304(%[a])\n\t" - "movq 320(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 328(%[b]), %%rcx\n\t" - "movq %%r9, 312(%[a])\n\t" - "movq 328(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq 336(%[b]), %%rdx\n\t" - "movq %%r8, 320(%[a])\n\t" - "movq 336(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 344(%[b]), %%rcx\n\t" - "movq %%r9, 328(%[a])\n\t" - "movq 344(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq 352(%[b]), %%rdx\n\t" - "movq %%r8, 336(%[a])\n\t" - "movq 352(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 360(%[b]), %%rcx\n\t" - "movq %%r9, 344(%[a])\n\t" - "movq 360(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq 368(%[b]), %%rdx\n\t" - "movq %%r8, 352(%[a])\n\t" - "movq 368(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 376(%[b]), %%rcx\n\t" - "movq %%r9, 360(%[a])\n\t" - "movq 376(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq %%r8, 368(%[a])\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq %%r9, 376(%[a])\n\t" - "sbbq $0, %[c]\n\t" - : [c] "+r" (c) - : [a] "r" (a), [b] "r" (b) - : "memory", "rdx", "rcx", "r8", "r9" - ); - - return c; -} - -/* Add b to a into r. (r = a + b) - * - * r A single precision integer. - * a A single precision integer. - * b A single precision integer. - */ -SP_NOINLINE static sp_digit sp_3072_add_48(sp_digit* r, const sp_digit* a, - const sp_digit* b) -{ - sp_digit c = 0; - - __asm__ __volatile__ ( - "movq (%[a]), %%rax\n\t" - "addq (%[b]), %%rax\n\t" - "movq %%rax, (%[r])\n\t" - "movq 8(%[a]), %%rax\n\t" - "adcq 8(%[b]), %%rax\n\t" - "movq %%rax, 8(%[r])\n\t" - "movq 16(%[a]), %%rax\n\t" - "adcq 16(%[b]), %%rax\n\t" - "movq %%rax, 16(%[r])\n\t" - "movq 24(%[a]), %%rax\n\t" - "adcq 24(%[b]), %%rax\n\t" - "movq %%rax, 24(%[r])\n\t" - "movq 32(%[a]), %%rax\n\t" - "adcq 32(%[b]), %%rax\n\t" - "movq %%rax, 32(%[r])\n\t" - "movq 40(%[a]), %%rax\n\t" - "adcq 40(%[b]), %%rax\n\t" - "movq %%rax, 40(%[r])\n\t" - "movq 48(%[a]), %%rax\n\t" - "adcq 48(%[b]), %%rax\n\t" - "movq %%rax, 48(%[r])\n\t" - "movq 56(%[a]), %%rax\n\t" - "adcq 56(%[b]), %%rax\n\t" - "movq %%rax, 56(%[r])\n\t" - "movq 64(%[a]), %%rax\n\t" - "adcq 64(%[b]), %%rax\n\t" - "movq %%rax, 64(%[r])\n\t" - "movq 72(%[a]), %%rax\n\t" - "adcq 72(%[b]), %%rax\n\t" - "movq %%rax, 72(%[r])\n\t" - "movq 80(%[a]), %%rax\n\t" - "adcq 80(%[b]), %%rax\n\t" - "movq %%rax, 80(%[r])\n\t" - "movq 88(%[a]), %%rax\n\t" - "adcq 88(%[b]), %%rax\n\t" - "movq %%rax, 88(%[r])\n\t" - "movq 96(%[a]), %%rax\n\t" - "adcq 96(%[b]), %%rax\n\t" - "movq %%rax, 96(%[r])\n\t" - "movq 104(%[a]), %%rax\n\t" - "adcq 104(%[b]), %%rax\n\t" - "movq %%rax, 104(%[r])\n\t" - "movq 112(%[a]), %%rax\n\t" - "adcq 112(%[b]), %%rax\n\t" - "movq %%rax, 112(%[r])\n\t" - "movq 120(%[a]), %%rax\n\t" - "adcq 120(%[b]), %%rax\n\t" - "movq %%rax, 120(%[r])\n\t" - "movq 128(%[a]), %%rax\n\t" - "adcq 128(%[b]), %%rax\n\t" - "movq %%rax, 128(%[r])\n\t" - "movq 136(%[a]), %%rax\n\t" - "adcq 136(%[b]), %%rax\n\t" - "movq %%rax, 136(%[r])\n\t" - "movq 144(%[a]), %%rax\n\t" - "adcq 144(%[b]), %%rax\n\t" - "movq %%rax, 144(%[r])\n\t" - "movq 152(%[a]), %%rax\n\t" - "adcq 152(%[b]), %%rax\n\t" - "movq %%rax, 152(%[r])\n\t" - "movq 160(%[a]), %%rax\n\t" - "adcq 160(%[b]), %%rax\n\t" - "movq %%rax, 160(%[r])\n\t" - "movq 168(%[a]), %%rax\n\t" - "adcq 168(%[b]), %%rax\n\t" - "movq %%rax, 168(%[r])\n\t" - "movq 176(%[a]), %%rax\n\t" - "adcq 176(%[b]), %%rax\n\t" - "movq %%rax, 176(%[r])\n\t" - "movq 184(%[a]), %%rax\n\t" - "adcq 184(%[b]), %%rax\n\t" - "movq %%rax, 184(%[r])\n\t" - "movq 192(%[a]), %%rax\n\t" - "adcq 192(%[b]), %%rax\n\t" - "movq %%rax, 192(%[r])\n\t" - "movq 200(%[a]), %%rax\n\t" - "adcq 200(%[b]), %%rax\n\t" - "movq %%rax, 200(%[r])\n\t" - "movq 208(%[a]), %%rax\n\t" - "adcq 208(%[b]), %%rax\n\t" - "movq %%rax, 208(%[r])\n\t" - "movq 216(%[a]), %%rax\n\t" - "adcq 216(%[b]), %%rax\n\t" - "movq %%rax, 216(%[r])\n\t" - "movq 224(%[a]), %%rax\n\t" - "adcq 224(%[b]), %%rax\n\t" - "movq %%rax, 224(%[r])\n\t" - "movq 232(%[a]), %%rax\n\t" - "adcq 232(%[b]), %%rax\n\t" - "movq %%rax, 232(%[r])\n\t" - "movq 240(%[a]), %%rax\n\t" - "adcq 240(%[b]), %%rax\n\t" - "movq %%rax, 240(%[r])\n\t" - "movq 248(%[a]), %%rax\n\t" - "adcq 248(%[b]), %%rax\n\t" - "movq %%rax, 248(%[r])\n\t" - "movq 256(%[a]), %%rax\n\t" - "adcq 256(%[b]), %%rax\n\t" - "movq %%rax, 256(%[r])\n\t" - "movq 264(%[a]), %%rax\n\t" - "adcq 264(%[b]), %%rax\n\t" - "movq %%rax, 264(%[r])\n\t" - "movq 272(%[a]), %%rax\n\t" - "adcq 272(%[b]), %%rax\n\t" - "movq %%rax, 272(%[r])\n\t" - "movq 280(%[a]), %%rax\n\t" - "adcq 280(%[b]), %%rax\n\t" - "movq %%rax, 280(%[r])\n\t" - "movq 288(%[a]), %%rax\n\t" - "adcq 288(%[b]), %%rax\n\t" - "movq %%rax, 288(%[r])\n\t" - "movq 296(%[a]), %%rax\n\t" - "adcq 296(%[b]), %%rax\n\t" - "movq %%rax, 296(%[r])\n\t" - "movq 304(%[a]), %%rax\n\t" - "adcq 304(%[b]), %%rax\n\t" - "movq %%rax, 304(%[r])\n\t" - "movq 312(%[a]), %%rax\n\t" - "adcq 312(%[b]), %%rax\n\t" - "movq %%rax, 312(%[r])\n\t" - "movq 320(%[a]), %%rax\n\t" - "adcq 320(%[b]), %%rax\n\t" - "movq %%rax, 320(%[r])\n\t" - "movq 328(%[a]), %%rax\n\t" - "adcq 328(%[b]), %%rax\n\t" - "movq %%rax, 328(%[r])\n\t" - "movq 336(%[a]), %%rax\n\t" - "adcq 336(%[b]), %%rax\n\t" - "movq %%rax, 336(%[r])\n\t" - "movq 344(%[a]), %%rax\n\t" - "adcq 344(%[b]), %%rax\n\t" - "movq %%rax, 344(%[r])\n\t" - "movq 352(%[a]), %%rax\n\t" - "adcq 352(%[b]), %%rax\n\t" - "movq %%rax, 352(%[r])\n\t" - "movq 360(%[a]), %%rax\n\t" - "adcq 360(%[b]), %%rax\n\t" - "movq %%rax, 360(%[r])\n\t" - "movq 368(%[a]), %%rax\n\t" - "adcq 368(%[b]), %%rax\n\t" - "movq %%rax, 368(%[r])\n\t" - "movq 376(%[a]), %%rax\n\t" - "adcq 376(%[b]), %%rax\n\t" - "movq %%rax, 376(%[r])\n\t" - "adcq $0, %[c]\n\t" - : [c] "+r" (c) - : [r] "r" (r), [a] "r" (a), [b] "r" (b) - : "memory", "rax" - ); - - return c; -} - +extern void sp_3072_mul_24(sp_digit* r, const sp_digit* a, const sp_digit* b); +extern void sp_3072_sqr_24(sp_digit* r, const sp_digit* a); +extern void sp_3072_mul_avx2_24(sp_digit* r, const sp_digit* a, const sp_digit* b); +extern void sp_3072_sqr_avx2_24(sp_digit* r, const sp_digit* a); +extern sp_digit sp_3072_add_24(sp_digit* r, const sp_digit* a, const sp_digit* b); +extern sp_digit sp_3072_sub_in_place_48(sp_digit* a, const sp_digit* b); +extern sp_digit sp_3072_add_48(sp_digit* r, const sp_digit* a, const sp_digit* b); /* AND m into each word of a and store in r. * * r A single precision integer. @@ -22230,122 +2147,7 @@ static void sp_3072_mont_setup(sp_digit* a, sp_digit* rho) } #if !defined(SP_RSA_PRIVATE_EXP_D) && defined(WOLFSSL_HAVE_SP_RSA) -/* Sub b from a into a. (a -= b) - * - * a A single precision integer and result. - * b A single precision integer. - */ -SP_NOINLINE static sp_digit sp_3072_sub_in_place_24(sp_digit* a, - const sp_digit* b) -{ - sp_digit c = 0; - - __asm__ __volatile__ ( - "movq 0(%[a]), %%r8\n\t" - "movq 8(%[a]), %%r9\n\t" - "movq 0(%[b]), %%rdx\n\t" - "movq 8(%[b]), %%rcx\n\t" - "subq %%rdx, %%r8\n\t" - "movq 16(%[b]), %%rdx\n\t" - "movq %%r8, 0(%[a])\n\t" - "movq 16(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 24(%[b]), %%rcx\n\t" - "movq %%r9, 8(%[a])\n\t" - "movq 24(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq 32(%[b]), %%rdx\n\t" - "movq %%r8, 16(%[a])\n\t" - "movq 32(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 40(%[b]), %%rcx\n\t" - "movq %%r9, 24(%[a])\n\t" - "movq 40(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq 48(%[b]), %%rdx\n\t" - "movq %%r8, 32(%[a])\n\t" - "movq 48(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 56(%[b]), %%rcx\n\t" - "movq %%r9, 40(%[a])\n\t" - "movq 56(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq 64(%[b]), %%rdx\n\t" - "movq %%r8, 48(%[a])\n\t" - "movq 64(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 72(%[b]), %%rcx\n\t" - "movq %%r9, 56(%[a])\n\t" - "movq 72(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq 80(%[b]), %%rdx\n\t" - "movq %%r8, 64(%[a])\n\t" - "movq 80(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 88(%[b]), %%rcx\n\t" - "movq %%r9, 72(%[a])\n\t" - "movq 88(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq 96(%[b]), %%rdx\n\t" - "movq %%r8, 80(%[a])\n\t" - "movq 96(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 104(%[b]), %%rcx\n\t" - "movq %%r9, 88(%[a])\n\t" - "movq 104(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq 112(%[b]), %%rdx\n\t" - "movq %%r8, 96(%[a])\n\t" - "movq 112(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 120(%[b]), %%rcx\n\t" - "movq %%r9, 104(%[a])\n\t" - "movq 120(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq 128(%[b]), %%rdx\n\t" - "movq %%r8, 112(%[a])\n\t" - "movq 128(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 136(%[b]), %%rcx\n\t" - "movq %%r9, 120(%[a])\n\t" - "movq 136(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq 144(%[b]), %%rdx\n\t" - "movq %%r8, 128(%[a])\n\t" - "movq 144(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 152(%[b]), %%rcx\n\t" - "movq %%r9, 136(%[a])\n\t" - "movq 152(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq 160(%[b]), %%rdx\n\t" - "movq %%r8, 144(%[a])\n\t" - "movq 160(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 168(%[b]), %%rcx\n\t" - "movq %%r9, 152(%[a])\n\t" - "movq 168(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq 176(%[b]), %%rdx\n\t" - "movq %%r8, 160(%[a])\n\t" - "movq 176(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 184(%[b]), %%rcx\n\t" - "movq %%r9, 168(%[a])\n\t" - "movq 184(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq %%r8, 176(%[a])\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq %%r9, 184(%[a])\n\t" - "sbbq $0, %[c]\n\t" - : [c] "+r" (c) - : [a] "r" (a), [b] "r" (b) - : "memory", "rdx", "rcx", "r8", "r9" - ); - - return c; -} - +extern sp_digit sp_3072_sub_in_place_24(sp_digit* a, const sp_digit* b); /* r = 2^n mod m where n is the number of bits to reduce by. * Given m must be 3072 bits, just need to subtract. * @@ -22360,470 +2162,8 @@ static void sp_3072_mont_norm_24(sp_digit* r, sp_digit* m) sp_3072_sub_in_place_24(r, m); } -/* Conditionally subtract b from a using the mask m. - * m is -1 to subtract and 0 when not copying. - * - * r A single precision number representing condition subtract result. - * a A single precision number to subtract from. - * b A single precision number to subtract. - * m Mask value to apply. - */ -static sp_digit sp_3072_cond_sub_24(sp_digit* r, sp_digit* a, sp_digit* b, - sp_digit m) -{ - sp_digit t[24]; - sp_digit c = 0; - - __asm__ __volatile__ ( - "movq 0(%[b]), %%rax\n\t" - "movq 8(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 0(%[t])\n\t" - "movq %%rcx, 8(%[t])\n\t" - "movq 16(%[b]), %%rax\n\t" - "movq 24(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 16(%[t])\n\t" - "movq %%rcx, 24(%[t])\n\t" - "movq 32(%[b]), %%rax\n\t" - "movq 40(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 32(%[t])\n\t" - "movq %%rcx, 40(%[t])\n\t" - "movq 48(%[b]), %%rax\n\t" - "movq 56(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 48(%[t])\n\t" - "movq %%rcx, 56(%[t])\n\t" - "movq 64(%[b]), %%rax\n\t" - "movq 72(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 64(%[t])\n\t" - "movq %%rcx, 72(%[t])\n\t" - "movq 80(%[b]), %%rax\n\t" - "movq 88(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 80(%[t])\n\t" - "movq %%rcx, 88(%[t])\n\t" - "movq 96(%[b]), %%rax\n\t" - "movq 104(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 96(%[t])\n\t" - "movq %%rcx, 104(%[t])\n\t" - "movq 112(%[b]), %%rax\n\t" - "movq 120(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 112(%[t])\n\t" - "movq %%rcx, 120(%[t])\n\t" - "movq 128(%[b]), %%rax\n\t" - "movq 136(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 128(%[t])\n\t" - "movq %%rcx, 136(%[t])\n\t" - "movq 144(%[b]), %%rax\n\t" - "movq 152(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 144(%[t])\n\t" - "movq %%rcx, 152(%[t])\n\t" - "movq 160(%[b]), %%rax\n\t" - "movq 168(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 160(%[t])\n\t" - "movq %%rcx, 168(%[t])\n\t" - "movq 176(%[b]), %%rax\n\t" - "movq 184(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 176(%[t])\n\t" - "movq %%rcx, 184(%[t])\n\t" - "movq (%[a]), %%rax\n\t" - "movq (%[t]), %%rdx\n\t" - "subq %%rdx,%%rax\n\t" - "movq 8(%[a]), %%rcx\n\t" - "movq 8(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 0(%[r])\n\t" - "movq 16(%[a]), %%rax\n\t" - "movq 16(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 8(%[r])\n\t" - "movq 24(%[a]), %%rcx\n\t" - "movq 24(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 16(%[r])\n\t" - "movq 32(%[a]), %%rax\n\t" - "movq 32(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 24(%[r])\n\t" - "movq 40(%[a]), %%rcx\n\t" - "movq 40(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 32(%[r])\n\t" - "movq 48(%[a]), %%rax\n\t" - "movq 48(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 40(%[r])\n\t" - "movq 56(%[a]), %%rcx\n\t" - "movq 56(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 48(%[r])\n\t" - "movq 64(%[a]), %%rax\n\t" - "movq 64(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 56(%[r])\n\t" - "movq 72(%[a]), %%rcx\n\t" - "movq 72(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 64(%[r])\n\t" - "movq 80(%[a]), %%rax\n\t" - "movq 80(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 72(%[r])\n\t" - "movq 88(%[a]), %%rcx\n\t" - "movq 88(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 80(%[r])\n\t" - "movq 96(%[a]), %%rax\n\t" - "movq 96(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 88(%[r])\n\t" - "movq 104(%[a]), %%rcx\n\t" - "movq 104(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 96(%[r])\n\t" - "movq 112(%[a]), %%rax\n\t" - "movq 112(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 104(%[r])\n\t" - "movq 120(%[a]), %%rcx\n\t" - "movq 120(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 112(%[r])\n\t" - "movq 128(%[a]), %%rax\n\t" - "movq 128(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 120(%[r])\n\t" - "movq 136(%[a]), %%rcx\n\t" - "movq 136(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 128(%[r])\n\t" - "movq 144(%[a]), %%rax\n\t" - "movq 144(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 136(%[r])\n\t" - "movq 152(%[a]), %%rcx\n\t" - "movq 152(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 144(%[r])\n\t" - "movq 160(%[a]), %%rax\n\t" - "movq 160(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 152(%[r])\n\t" - "movq 168(%[a]), %%rcx\n\t" - "movq 168(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 160(%[r])\n\t" - "movq 176(%[a]), %%rax\n\t" - "movq 176(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 168(%[r])\n\t" - "movq 184(%[a]), %%rcx\n\t" - "movq 184(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 176(%[r])\n\t" - "movq %%rcx, 184(%[r])\n\t" - "sbbq $0, %[c]\n\t" - : [c] "+r" (c) - : [r] "r" (r), [a] "r" (a), [b] "r" (b), [m] "r" (m), [t] "r" (t) - : "memory", "rax", "rcx", "rdx" - ); - - return c; -} - -/* Reduce the number back to 3072 bits using Montgomery reduction. - * - * a A single precision number to reduce in place. - * m The single precision number representing the modulus. - * mp The digit representing the negative inverse of m mod 2^n. - */ -SP_NOINLINE static void sp_3072_mont_reduce_24(sp_digit* a, sp_digit* m, - sp_digit mp) -{ - sp_digit ca = 0; - - __asm__ __volatile__ ( - "# i = 0\n\t" - "xorq %%rcx, %%rcx\n\t" - "movq 0(%[a]), %%r12\n\t" - "movq 8(%[a]), %%r13\n\t" - "\nL_mont_loop_24:\n\t" - "# mu = a[i] * mp\n\t" - "movq %%r12, %%r10\n\t" - "imulq %[mp], %%r10\n\t" - "# a[i+0] += m[0] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 0(%[m])\n\t" - "addq %%rax, %%r12\n\t" - "adcq %%rdx, %%r9\n\t" - "# a[i+1] += m[1] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 8(%[m])\n\t" - "movq %%r13, %%r12\n\t" - "addq %%rax, %%r12\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r12\n\t" - "adcq $0, %%r8\n\t" - "# a[i+2] += m[2] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 16(%[m])\n\t" - "movq 16(%[a]), %%r13\n\t" - "addq %%rax, %%r13\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r13\n\t" - "adcq $0, %%r9\n\t" - "# a[i+3] += m[3] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 24(%[m])\n\t" - "movq 24(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 24(%[a])\n\t" - "adcq $0, %%r8\n\t" - "# a[i+4] += m[4] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 32(%[m])\n\t" - "movq 32(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r11\n\t" - "movq %%r11, 32(%[a])\n\t" - "adcq $0, %%r9\n\t" - "# a[i+5] += m[5] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 40(%[m])\n\t" - "movq 40(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 40(%[a])\n\t" - "adcq $0, %%r8\n\t" - "# a[i+6] += m[6] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 48(%[m])\n\t" - "movq 48(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r11\n\t" - "movq %%r11, 48(%[a])\n\t" - "adcq $0, %%r9\n\t" - "# a[i+7] += m[7] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 56(%[m])\n\t" - "movq 56(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 56(%[a])\n\t" - "adcq $0, %%r8\n\t" - "# a[i+8] += m[8] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 64(%[m])\n\t" - "movq 64(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r11\n\t" - "movq %%r11, 64(%[a])\n\t" - "adcq $0, %%r9\n\t" - "# a[i+9] += m[9] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 72(%[m])\n\t" - "movq 72(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 72(%[a])\n\t" - "adcq $0, %%r8\n\t" - "# a[i+10] += m[10] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 80(%[m])\n\t" - "movq 80(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r11\n\t" - "movq %%r11, 80(%[a])\n\t" - "adcq $0, %%r9\n\t" - "# a[i+11] += m[11] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 88(%[m])\n\t" - "movq 88(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 88(%[a])\n\t" - "adcq $0, %%r8\n\t" - "# a[i+12] += m[12] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 96(%[m])\n\t" - "movq 96(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r11\n\t" - "movq %%r11, 96(%[a])\n\t" - "adcq $0, %%r9\n\t" - "# a[i+13] += m[13] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 104(%[m])\n\t" - "movq 104(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 104(%[a])\n\t" - "adcq $0, %%r8\n\t" - "# a[i+14] += m[14] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 112(%[m])\n\t" - "movq 112(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r11\n\t" - "movq %%r11, 112(%[a])\n\t" - "adcq $0, %%r9\n\t" - "# a[i+15] += m[15] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 120(%[m])\n\t" - "movq 120(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 120(%[a])\n\t" - "adcq $0, %%r8\n\t" - "# a[i+16] += m[16] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 128(%[m])\n\t" - "movq 128(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r11\n\t" - "movq %%r11, 128(%[a])\n\t" - "adcq $0, %%r9\n\t" - "# a[i+17] += m[17] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 136(%[m])\n\t" - "movq 136(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 136(%[a])\n\t" - "adcq $0, %%r8\n\t" - "# a[i+18] += m[18] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 144(%[m])\n\t" - "movq 144(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r11\n\t" - "movq %%r11, 144(%[a])\n\t" - "adcq $0, %%r9\n\t" - "# a[i+19] += m[19] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 152(%[m])\n\t" - "movq 152(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 152(%[a])\n\t" - "adcq $0, %%r8\n\t" - "# a[i+20] += m[20] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 160(%[m])\n\t" - "movq 160(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r11\n\t" - "movq %%r11, 160(%[a])\n\t" - "adcq $0, %%r9\n\t" - "# a[i+21] += m[21] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 168(%[m])\n\t" - "movq 168(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 168(%[a])\n\t" - "adcq $0, %%r8\n\t" - "# a[i+22] += m[22] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 176(%[m])\n\t" - "movq 176(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r11\n\t" - "movq %%r11, 176(%[a])\n\t" - "adcq $0, %%r9\n\t" - "# a[i+23] += m[23] * mu\n\t" - "movq %%r10, %%rax\n\t" - "mulq 184(%[m])\n\t" - "movq 184(%[a]), %%r11\n\t" - "addq %%rax, %%r9\n\t" - "adcq %[ca], %%rdx\n\t" - "movq $0, %[ca]\n\t" - "adcq $0, %[ca]\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 184(%[a])\n\t" - "adcq %%rdx, 192(%[a])\n\t" - "adcq $0, %[ca]\n\t" - "# i += 1\n\t" - "addq $8, %[a]\n\t" - "addq $8, %%rcx\n\t" - "cmpq $192, %%rcx\n\t" - "jl L_mont_loop_24\n\t" - "movq %%r12, 0(%[a])\n\t" - "movq %%r13, 8(%[a])\n\t" - : [ca] "+r" (ca), [a] "+r" (a) - : [m] "r" (m), [mp] "r" (mp) - : "memory", "rax", "rdx", "rcx", "r8", "r9", "r10", "r11", - "r12", "r13" - ); - - sp_3072_cond_sub_24(a - 24, a, m, (sp_digit)0 - ca); -} - +extern sp_digit sp_3072_cond_sub_24(sp_digit* r, sp_digit* a, sp_digit* b, sp_digit m); +extern void sp_3072_mont_reduce_24(sp_digit* a, sp_digit* m, sp_digit mp); /* Multiply two Montogmery form numbers mod the modulus (prime). * (r = a * b mod m) * @@ -22854,375 +2194,8 @@ static void sp_3072_mont_sqr_24(sp_digit* r, sp_digit* a, sp_digit* m, sp_3072_mont_reduce_24(r, m, mp); } -/* Mul a by digit b into r. (r = a * b) - * - * r A single precision integer. - * a A single precision integer. - * b A single precision digit. - */ -SP_NOINLINE static void sp_3072_mul_d_24(sp_digit* r, const sp_digit* a, - const sp_digit b) -{ - __asm__ __volatile__ ( - "# A[0] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq (%[a])\n\t" - "movq %%rax, %%rbx\n\t" - "movq %%rdx, %%rcx\n\t" - "movq %%rbx, 0(%[r])\n\t" - "# A[1] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rbx, %%rbx\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "movq %%rcx, 8(%[r])\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[2] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rcx, %%rcx\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "movq %%r8, 16(%[r])\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[3] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "movq %%rbx, 24(%[r])\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[4] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rbx, %%rbx\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "movq %%rcx, 32(%[r])\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[5] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rcx, %%rcx\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "movq %%r8, 40(%[r])\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[6] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "movq %%rbx, 48(%[r])\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[7] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rbx, %%rbx\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "movq %%rcx, 56(%[r])\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[8] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rcx, %%rcx\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "movq %%r8, 64(%[r])\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[9] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "movq %%rbx, 72(%[r])\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[10] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rbx, %%rbx\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "movq %%rcx, 80(%[r])\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[11] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rcx, %%rcx\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "movq %%r8, 88(%[r])\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[12] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 96(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "movq %%rbx, 96(%[r])\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[13] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rbx, %%rbx\n\t" - "mulq 104(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "movq %%rcx, 104(%[r])\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[14] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rcx, %%rcx\n\t" - "mulq 112(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "movq %%r8, 112(%[r])\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[15] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 120(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "movq %%rbx, 120(%[r])\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[16] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rbx, %%rbx\n\t" - "mulq 128(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "movq %%rcx, 128(%[r])\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[17] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rcx, %%rcx\n\t" - "mulq 136(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "movq %%r8, 136(%[r])\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[18] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 144(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "movq %%rbx, 144(%[r])\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[19] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rbx, %%rbx\n\t" - "mulq 152(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "movq %%rcx, 152(%[r])\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[20] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rcx, %%rcx\n\t" - "mulq 160(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "movq %%r8, 160(%[r])\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[21] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 168(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "movq %%rbx, 168(%[r])\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[22] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rbx, %%rbx\n\t" - "mulq 176(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "movq %%rcx, 176(%[r])\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[23] * B\n\t" - "movq %[b], %%rax\n\t" - "mulq 184(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "movq %%r8, 184(%[r])\n\t" - "movq %%rbx, 192(%[r])\n\t" - : - : [r] "r" (r), [a] "r" (a), [b] "r" (b) - : "memory", "rax", "rdx", "rbx", "rcx", "r8" - ); -} - -#ifdef HAVE_INTEL_AVX2 -/* Mul a by digit b into r. (r = a * b) - * - * r A single precision integer. - * a A single precision integer. - * b A single precision digit. - */ -SP_NOINLINE static void sp_3072_mul_d_avx2_24(sp_digit* r, const sp_digit* a, - const sp_digit b) -{ - __asm__ __volatile__ ( - "# A[0] * B\n\t" - "movq %[b], %%rdx\n\t" - "xorq %%r10, %%r10\n\t" - "mulxq (%[a]), %%r8, %%r9\n\t" - "movq %%r8, 0(%[r])\n\t" - "# A[1] * B\n\t" - "mulxq 8(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 8(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[2] * B\n\t" - "mulxq 16(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 16(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[3] * B\n\t" - "mulxq 24(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 24(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[4] * B\n\t" - "mulxq 32(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 32(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[5] * B\n\t" - "mulxq 40(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 40(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[6] * B\n\t" - "mulxq 48(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 48(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[7] * B\n\t" - "mulxq 56(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 56(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[8] * B\n\t" - "mulxq 64(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 64(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[9] * B\n\t" - "mulxq 72(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 72(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[10] * B\n\t" - "mulxq 80(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 80(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[11] * B\n\t" - "mulxq 88(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 88(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[12] * B\n\t" - "mulxq 96(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 96(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[13] * B\n\t" - "mulxq 104(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 104(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[14] * B\n\t" - "mulxq 112(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 112(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[15] * B\n\t" - "mulxq 120(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 120(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[16] * B\n\t" - "mulxq 128(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 128(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[17] * B\n\t" - "mulxq 136(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 136(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[18] * B\n\t" - "mulxq 144(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 144(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[19] * B\n\t" - "mulxq 152(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 152(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[20] * B\n\t" - "mulxq 160(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 160(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[21] * B\n\t" - "mulxq 168(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 168(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[22] * B\n\t" - "mulxq 176(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 176(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[23] * B\n\t" - "mulxq 184(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "adoxq %%rcx, %%r8\n\t" - "adcxq %%r10, %%r8\n\t" - "movq %%r9, 184(%[r])\n\t" - "movq %%r8, 192(%[r])\n\t" - : - : [r] "r" (r), [a] "r" (a), [b] "r" (b) - : "memory", "rax", "rdx", "rcx", "r8", "r9", "r10" - ); -} -#endif /* HAVE_INTEL_AVX2 */ - +extern void sp_3072_mul_d_24(sp_digit* r, const sp_digit* a, const sp_digit b); +extern void sp_3072_mul_d_avx2_24(sp_digit* r, const sp_digit* a, const sp_digit b); /* Divide the double width number (d1|d0) by the dividend. (d1|d0 / div) * * d1 The high order half of the number to divide. @@ -23230,239 +2203,19 @@ SP_NOINLINE static void sp_3072_mul_d_avx2_24(sp_digit* r, const sp_digit* a, * div The dividend. * returns the result of the division. */ -static sp_digit div_3072_word_24(sp_digit d1, sp_digit d0, sp_digit div) +static WC_INLINE sp_digit div_3072_word_24(sp_digit d1, sp_digit d0, + sp_digit div) { - sp_digit r; - + register sp_digit r asm("rax"); __asm__ __volatile__ ( - "movq %[d0], %%rax\n\t" - "movq %[d1], %%rdx\n\t" - "divq %[div]\n\t" - "movq %%rax, %[r]\n\t" - : [r] "=r" (r) - : [d1] "r" (d1), [d0] "r" (d0), [div] "r" (div) - : "rax", "rdx" + "divq %3" + : "=a" (r) + : "d" (d1), "a" (d0), "r" (div) + : ); - return r; } - -/* Compare a with b in constant time. - * - * a A single precision integer. - * b A single precision integer. - * return -ve, 0 or +ve if a is less than, equal to or greater than b - * respectively. - */ -static int64_t sp_3072_cmp_24(sp_digit* a, sp_digit* b) -{ - sp_digit r = -1; - sp_digit one = 1; - - __asm__ __volatile__ ( - "xorq %%rcx, %%rcx\n\t" - "movq $-1, %%rdx\n\t" - "movq 184(%[a]), %%rbx\n\t" - "movq 184(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 176(%[a]), %%rbx\n\t" - "movq 176(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 168(%[a]), %%rbx\n\t" - "movq 168(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 160(%[a]), %%rbx\n\t" - "movq 160(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 152(%[a]), %%rbx\n\t" - "movq 152(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 144(%[a]), %%rbx\n\t" - "movq 144(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 136(%[a]), %%rbx\n\t" - "movq 136(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 128(%[a]), %%rbx\n\t" - "movq 128(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 120(%[a]), %%rbx\n\t" - "movq 120(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 112(%[a]), %%rbx\n\t" - "movq 112(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 104(%[a]), %%rbx\n\t" - "movq 104(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 96(%[a]), %%rbx\n\t" - "movq 96(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 88(%[a]), %%rbx\n\t" - "movq 88(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 80(%[a]), %%rbx\n\t" - "movq 80(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 72(%[a]), %%rbx\n\t" - "movq 72(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 64(%[a]), %%rbx\n\t" - "movq 64(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 56(%[a]), %%rbx\n\t" - "movq 56(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 48(%[a]), %%rbx\n\t" - "movq 48(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 40(%[a]), %%rbx\n\t" - "movq 40(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 32(%[a]), %%rbx\n\t" - "movq 32(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 24(%[a]), %%rbx\n\t" - "movq 24(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 16(%[a]), %%rbx\n\t" - "movq 16(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 8(%[a]), %%rbx\n\t" - "movq 8(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 0(%[a]), %%rbx\n\t" - "movq 0(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "xorq %%rdx, %[r]\n\t" - : [r] "+r" (r) - : [a] "r" (a), [b] "r" (b), [one] "r" (one) - : "rax", "rdx", "rcx", "rbx", "r8" - ); - - return r; -} - +extern int64_t sp_3072_cmp_24(sp_digit* a, sp_digit* b); /* Divide d in a and put remainder into r (m*d + r = a) * m is not calculated as it is not needed at this time. * @@ -23665,190 +2418,7 @@ static int sp_3072_mod_exp_24(sp_digit* r, sp_digit* a, sp_digit* e, return err; } -#ifdef HAVE_INTEL_AVX2 -/* Reduce the number back to 3072 bits using Montgomery reduction. - * - * a A single precision number to reduce in place. - * m The single precision number representing the modulus. - * mp The digit representing the negative inverse of m mod 2^n. - */ -SP_NOINLINE static void sp_3072_mont_reduce_avx2_24(sp_digit* a, sp_digit* m, - sp_digit mp) -{ - sp_digit ca = 0; - - __asm__ __volatile__ ( - "# i = 0\n\t" - "movq 0(%[a]), %%r12\n\t" - "xorq %%rcx, %%rcx\n\t" - "\nL_mont_loop_avx2_24:\n\t" - "xorq %%r9, %%r9\n\t" - "movq %%r12, %%r10\n\t" - "# mu = a[i] * mp\n\t" - "movq %%r12, %%rdx\n\t" - "mulxq %[mp], %%rdx, %%r8\n\t" - "# a[i+0] += m[0] * mu\n\t" - "mulxq 0(%[m]), %%rax, %%r8\n\t" - "movq 8(%[a]), %%r12\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r12\n\t" - "# a[i+1] += m[1] * mu\n\t" - "mulxq 8(%[m]), %%rax, %%r8\n\t" - "movq 16(%[a]), %%r10\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%r8, %%r10\n\t" - "# a[i+2] += m[2] * mu\n\t" - "mulxq 16(%[m]), %%rax, %%r8\n\t" - "movq 24(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 16(%[a])\n\t" - "# a[i+3] += m[3] * mu\n\t" - "mulxq 24(%[m]), %%rax, %%r8\n\t" - "movq 32(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 24(%[a])\n\t" - "# a[i+4] += m[4] * mu\n\t" - "mulxq 32(%[m]), %%rax, %%r8\n\t" - "movq 40(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 32(%[a])\n\t" - "# a[i+5] += m[5] * mu\n\t" - "mulxq 40(%[m]), %%rax, %%r8\n\t" - "movq 48(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 40(%[a])\n\t" - "# a[i+6] += m[6] * mu\n\t" - "mulxq 48(%[m]), %%rax, %%r8\n\t" - "movq 56(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 48(%[a])\n\t" - "# a[i+7] += m[7] * mu\n\t" - "mulxq 56(%[m]), %%rax, %%r8\n\t" - "movq 64(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 56(%[a])\n\t" - "# a[i+8] += m[8] * mu\n\t" - "mulxq 64(%[m]), %%rax, %%r8\n\t" - "movq 72(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 64(%[a])\n\t" - "# a[i+9] += m[9] * mu\n\t" - "mulxq 72(%[m]), %%rax, %%r8\n\t" - "movq 80(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 72(%[a])\n\t" - "# a[i+10] += m[10] * mu\n\t" - "mulxq 80(%[m]), %%rax, %%r8\n\t" - "movq 88(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 80(%[a])\n\t" - "# a[i+11] += m[11] * mu\n\t" - "mulxq 88(%[m]), %%rax, %%r8\n\t" - "movq 96(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 88(%[a])\n\t" - "# a[i+12] += m[12] * mu\n\t" - "mulxq 96(%[m]), %%rax, %%r8\n\t" - "movq 104(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 96(%[a])\n\t" - "# a[i+13] += m[13] * mu\n\t" - "mulxq 104(%[m]), %%rax, %%r8\n\t" - "movq 112(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 104(%[a])\n\t" - "# a[i+14] += m[14] * mu\n\t" - "mulxq 112(%[m]), %%rax, %%r8\n\t" - "movq 120(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 112(%[a])\n\t" - "# a[i+15] += m[15] * mu\n\t" - "mulxq 120(%[m]), %%rax, %%r8\n\t" - "movq 128(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 120(%[a])\n\t" - "# a[i+16] += m[16] * mu\n\t" - "mulxq 128(%[m]), %%rax, %%r8\n\t" - "movq 136(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 128(%[a])\n\t" - "# a[i+17] += m[17] * mu\n\t" - "mulxq 136(%[m]), %%rax, %%r8\n\t" - "movq 144(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 136(%[a])\n\t" - "# a[i+18] += m[18] * mu\n\t" - "mulxq 144(%[m]), %%rax, %%r8\n\t" - "movq 152(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 144(%[a])\n\t" - "# a[i+19] += m[19] * mu\n\t" - "mulxq 152(%[m]), %%rax, %%r8\n\t" - "movq 160(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 152(%[a])\n\t" - "# a[i+20] += m[20] * mu\n\t" - "mulxq 160(%[m]), %%rax, %%r8\n\t" - "movq 168(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 160(%[a])\n\t" - "# a[i+21] += m[21] * mu\n\t" - "mulxq 168(%[m]), %%rax, %%r8\n\t" - "movq 176(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 168(%[a])\n\t" - "# a[i+22] += m[22] * mu\n\t" - "mulxq 176(%[m]), %%rax, %%r8\n\t" - "movq 184(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 176(%[a])\n\t" - "# a[i+23] += m[23] * mu\n\t" - "mulxq 184(%[m]), %%rax, %%r8\n\t" - "movq 192(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 184(%[a])\n\t" - "adcxq %[ca], %%r10\n\t" - "movq %%r9, %[ca]\n\t" - "adoxq %%r9, %[ca]\n\t" - "adcxq %%r9, %[ca]\n\t" - "movq %%r10, 192(%[a])\n\t" - "# i += 1\n\t" - "addq $8, %[a]\n\t" - "addq $1, %%rcx\n\t" - "cmpq $24, %%rcx\n\t" - "jl L_mont_loop_avx2_24\n\t" - "movq %%r12, 0(%[a])\n\t" - : [ca] "+r" (ca), [a] "+r" (a) - : [m] "r" (m), [mp] "r" (mp) - : "memory", "rax", "rdx", "rcx", "r8", "r9", "r10", "r11", "r12" - ); - - sp_3072_cond_sub_24(a - 24, a, m, (sp_digit)0 - ca); -} -#endif /* HAVE_INTEL_AVX2 */ - +extern void sp_3072_mont_reduce_avx2_24(sp_digit* a, sp_digit* m, sp_digit mp); #ifdef HAVE_INTEL_AVX2 /* Multiply two Montogmery form numbers mod the modulus (prime). * (r = a * b mod m) @@ -24047,878 +2617,8 @@ static void sp_3072_mont_norm_48(sp_digit* r, sp_digit* m) } #endif /* WOLFSSL_HAVE_SP_DH */ -/* Conditionally subtract b from a using the mask m. - * m is -1 to subtract and 0 when not copying. - * - * r A single precision number representing condition subtract result. - * a A single precision number to subtract from. - * b A single precision number to subtract. - * m Mask value to apply. - */ -static sp_digit sp_3072_cond_sub_48(sp_digit* r, sp_digit* a, sp_digit* b, - sp_digit m) -{ - sp_digit t[48]; - sp_digit c = 0; - - __asm__ __volatile__ ( - "movq 0(%[b]), %%rax\n\t" - "movq 8(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 0(%[t])\n\t" - "movq %%rcx, 8(%[t])\n\t" - "movq 16(%[b]), %%rax\n\t" - "movq 24(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 16(%[t])\n\t" - "movq %%rcx, 24(%[t])\n\t" - "movq 32(%[b]), %%rax\n\t" - "movq 40(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 32(%[t])\n\t" - "movq %%rcx, 40(%[t])\n\t" - "movq 48(%[b]), %%rax\n\t" - "movq 56(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 48(%[t])\n\t" - "movq %%rcx, 56(%[t])\n\t" - "movq 64(%[b]), %%rax\n\t" - "movq 72(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 64(%[t])\n\t" - "movq %%rcx, 72(%[t])\n\t" - "movq 80(%[b]), %%rax\n\t" - "movq 88(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 80(%[t])\n\t" - "movq %%rcx, 88(%[t])\n\t" - "movq 96(%[b]), %%rax\n\t" - "movq 104(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 96(%[t])\n\t" - "movq %%rcx, 104(%[t])\n\t" - "movq 112(%[b]), %%rax\n\t" - "movq 120(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 112(%[t])\n\t" - "movq %%rcx, 120(%[t])\n\t" - "movq 128(%[b]), %%rax\n\t" - "movq 136(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 128(%[t])\n\t" - "movq %%rcx, 136(%[t])\n\t" - "movq 144(%[b]), %%rax\n\t" - "movq 152(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 144(%[t])\n\t" - "movq %%rcx, 152(%[t])\n\t" - "movq 160(%[b]), %%rax\n\t" - "movq 168(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 160(%[t])\n\t" - "movq %%rcx, 168(%[t])\n\t" - "movq 176(%[b]), %%rax\n\t" - "movq 184(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 176(%[t])\n\t" - "movq %%rcx, 184(%[t])\n\t" - "movq 192(%[b]), %%rax\n\t" - "movq 200(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 192(%[t])\n\t" - "movq %%rcx, 200(%[t])\n\t" - "movq 208(%[b]), %%rax\n\t" - "movq 216(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 208(%[t])\n\t" - "movq %%rcx, 216(%[t])\n\t" - "movq 224(%[b]), %%rax\n\t" - "movq 232(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 224(%[t])\n\t" - "movq %%rcx, 232(%[t])\n\t" - "movq 240(%[b]), %%rax\n\t" - "movq 248(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 240(%[t])\n\t" - "movq %%rcx, 248(%[t])\n\t" - "movq 256(%[b]), %%rax\n\t" - "movq 264(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 256(%[t])\n\t" - "movq %%rcx, 264(%[t])\n\t" - "movq 272(%[b]), %%rax\n\t" - "movq 280(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 272(%[t])\n\t" - "movq %%rcx, 280(%[t])\n\t" - "movq 288(%[b]), %%rax\n\t" - "movq 296(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 288(%[t])\n\t" - "movq %%rcx, 296(%[t])\n\t" - "movq 304(%[b]), %%rax\n\t" - "movq 312(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 304(%[t])\n\t" - "movq %%rcx, 312(%[t])\n\t" - "movq 320(%[b]), %%rax\n\t" - "movq 328(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 320(%[t])\n\t" - "movq %%rcx, 328(%[t])\n\t" - "movq 336(%[b]), %%rax\n\t" - "movq 344(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 336(%[t])\n\t" - "movq %%rcx, 344(%[t])\n\t" - "movq 352(%[b]), %%rax\n\t" - "movq 360(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 352(%[t])\n\t" - "movq %%rcx, 360(%[t])\n\t" - "movq 368(%[b]), %%rax\n\t" - "movq 376(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 368(%[t])\n\t" - "movq %%rcx, 376(%[t])\n\t" - "movq (%[a]), %%rax\n\t" - "movq (%[t]), %%rdx\n\t" - "subq %%rdx,%%rax\n\t" - "movq 8(%[a]), %%rcx\n\t" - "movq 8(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 0(%[r])\n\t" - "movq 16(%[a]), %%rax\n\t" - "movq 16(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 8(%[r])\n\t" - "movq 24(%[a]), %%rcx\n\t" - "movq 24(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 16(%[r])\n\t" - "movq 32(%[a]), %%rax\n\t" - "movq 32(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 24(%[r])\n\t" - "movq 40(%[a]), %%rcx\n\t" - "movq 40(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 32(%[r])\n\t" - "movq 48(%[a]), %%rax\n\t" - "movq 48(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 40(%[r])\n\t" - "movq 56(%[a]), %%rcx\n\t" - "movq 56(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 48(%[r])\n\t" - "movq 64(%[a]), %%rax\n\t" - "movq 64(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 56(%[r])\n\t" - "movq 72(%[a]), %%rcx\n\t" - "movq 72(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 64(%[r])\n\t" - "movq 80(%[a]), %%rax\n\t" - "movq 80(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 72(%[r])\n\t" - "movq 88(%[a]), %%rcx\n\t" - "movq 88(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 80(%[r])\n\t" - "movq 96(%[a]), %%rax\n\t" - "movq 96(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 88(%[r])\n\t" - "movq 104(%[a]), %%rcx\n\t" - "movq 104(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 96(%[r])\n\t" - "movq 112(%[a]), %%rax\n\t" - "movq 112(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 104(%[r])\n\t" - "movq 120(%[a]), %%rcx\n\t" - "movq 120(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 112(%[r])\n\t" - "movq 128(%[a]), %%rax\n\t" - "movq 128(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 120(%[r])\n\t" - "movq 136(%[a]), %%rcx\n\t" - "movq 136(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 128(%[r])\n\t" - "movq 144(%[a]), %%rax\n\t" - "movq 144(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 136(%[r])\n\t" - "movq 152(%[a]), %%rcx\n\t" - "movq 152(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 144(%[r])\n\t" - "movq 160(%[a]), %%rax\n\t" - "movq 160(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 152(%[r])\n\t" - "movq 168(%[a]), %%rcx\n\t" - "movq 168(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 160(%[r])\n\t" - "movq 176(%[a]), %%rax\n\t" - "movq 176(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 168(%[r])\n\t" - "movq 184(%[a]), %%rcx\n\t" - "movq 184(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 176(%[r])\n\t" - "movq 192(%[a]), %%rax\n\t" - "movq 192(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 184(%[r])\n\t" - "movq 200(%[a]), %%rcx\n\t" - "movq 200(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 192(%[r])\n\t" - "movq 208(%[a]), %%rax\n\t" - "movq 208(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 200(%[r])\n\t" - "movq 216(%[a]), %%rcx\n\t" - "movq 216(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 208(%[r])\n\t" - "movq 224(%[a]), %%rax\n\t" - "movq 224(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 216(%[r])\n\t" - "movq 232(%[a]), %%rcx\n\t" - "movq 232(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 224(%[r])\n\t" - "movq 240(%[a]), %%rax\n\t" - "movq 240(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 232(%[r])\n\t" - "movq 248(%[a]), %%rcx\n\t" - "movq 248(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 240(%[r])\n\t" - "movq 256(%[a]), %%rax\n\t" - "movq 256(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 248(%[r])\n\t" - "movq 264(%[a]), %%rcx\n\t" - "movq 264(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 256(%[r])\n\t" - "movq 272(%[a]), %%rax\n\t" - "movq 272(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 264(%[r])\n\t" - "movq 280(%[a]), %%rcx\n\t" - "movq 280(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 272(%[r])\n\t" - "movq 288(%[a]), %%rax\n\t" - "movq 288(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 280(%[r])\n\t" - "movq 296(%[a]), %%rcx\n\t" - "movq 296(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 288(%[r])\n\t" - "movq 304(%[a]), %%rax\n\t" - "movq 304(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 296(%[r])\n\t" - "movq 312(%[a]), %%rcx\n\t" - "movq 312(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 304(%[r])\n\t" - "movq 320(%[a]), %%rax\n\t" - "movq 320(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 312(%[r])\n\t" - "movq 328(%[a]), %%rcx\n\t" - "movq 328(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 320(%[r])\n\t" - "movq 336(%[a]), %%rax\n\t" - "movq 336(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 328(%[r])\n\t" - "movq 344(%[a]), %%rcx\n\t" - "movq 344(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 336(%[r])\n\t" - "movq 352(%[a]), %%rax\n\t" - "movq 352(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 344(%[r])\n\t" - "movq 360(%[a]), %%rcx\n\t" - "movq 360(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 352(%[r])\n\t" - "movq 368(%[a]), %%rax\n\t" - "movq 368(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 360(%[r])\n\t" - "movq 376(%[a]), %%rcx\n\t" - "movq 376(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 368(%[r])\n\t" - "movq %%rcx, 376(%[r])\n\t" - "sbbq $0, %[c]\n\t" - : [c] "+r" (c) - : [r] "r" (r), [a] "r" (a), [b] "r" (b), [m] "r" (m), [t] "r" (t) - : "memory", "rax", "rcx", "rdx" - ); - - return c; -} - -/* Reduce the number back to 3072 bits using Montgomery reduction. - * - * a A single precision number to reduce in place. - * m The single precision number representing the modulus. - * mp The digit representing the negative inverse of m mod 2^n. - */ -SP_NOINLINE static void sp_3072_mont_reduce_48(sp_digit* a, sp_digit* m, - sp_digit mp) -{ - sp_digit ca = 0; - - __asm__ __volatile__ ( - "# i = 0\n\t" - "xorq %%rcx, %%rcx\n\t" - "movq 0(%[a]), %%r12\n\t" - "movq 8(%[a]), %%r13\n\t" - "\nL_mont_loop_48:\n\t" - "# mu = a[i] * mp\n\t" - "movq %%r12, %%r10\n\t" - "imulq %[mp], %%r10\n\t" - "# a[i+0] += m[0] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 0(%[m])\n\t" - "addq %%rax, %%r12\n\t" - "adcq %%rdx, %%r9\n\t" - "# a[i+1] += m[1] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 8(%[m])\n\t" - "movq %%r13, %%r12\n\t" - "addq %%rax, %%r12\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r12\n\t" - "adcq $0, %%r8\n\t" - "# a[i+2] += m[2] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 16(%[m])\n\t" - "movq 16(%[a]), %%r13\n\t" - "addq %%rax, %%r13\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r13\n\t" - "adcq $0, %%r9\n\t" - "# a[i+3] += m[3] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 24(%[m])\n\t" - "movq 24(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 24(%[a])\n\t" - "adcq $0, %%r8\n\t" - "# a[i+4] += m[4] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 32(%[m])\n\t" - "movq 32(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r11\n\t" - "movq %%r11, 32(%[a])\n\t" - "adcq $0, %%r9\n\t" - "# a[i+5] += m[5] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 40(%[m])\n\t" - "movq 40(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 40(%[a])\n\t" - "adcq $0, %%r8\n\t" - "# a[i+6] += m[6] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 48(%[m])\n\t" - "movq 48(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r11\n\t" - "movq %%r11, 48(%[a])\n\t" - "adcq $0, %%r9\n\t" - "# a[i+7] += m[7] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 56(%[m])\n\t" - "movq 56(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 56(%[a])\n\t" - "adcq $0, %%r8\n\t" - "# a[i+8] += m[8] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 64(%[m])\n\t" - "movq 64(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r11\n\t" - "movq %%r11, 64(%[a])\n\t" - "adcq $0, %%r9\n\t" - "# a[i+9] += m[9] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 72(%[m])\n\t" - "movq 72(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 72(%[a])\n\t" - "adcq $0, %%r8\n\t" - "# a[i+10] += m[10] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 80(%[m])\n\t" - "movq 80(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r11\n\t" - "movq %%r11, 80(%[a])\n\t" - "adcq $0, %%r9\n\t" - "# a[i+11] += m[11] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 88(%[m])\n\t" - "movq 88(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 88(%[a])\n\t" - "adcq $0, %%r8\n\t" - "# a[i+12] += m[12] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 96(%[m])\n\t" - "movq 96(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r11\n\t" - "movq %%r11, 96(%[a])\n\t" - "adcq $0, %%r9\n\t" - "# a[i+13] += m[13] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 104(%[m])\n\t" - "movq 104(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 104(%[a])\n\t" - "adcq $0, %%r8\n\t" - "# a[i+14] += m[14] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 112(%[m])\n\t" - "movq 112(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r11\n\t" - "movq %%r11, 112(%[a])\n\t" - "adcq $0, %%r9\n\t" - "# a[i+15] += m[15] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 120(%[m])\n\t" - "movq 120(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 120(%[a])\n\t" - "adcq $0, %%r8\n\t" - "# a[i+16] += m[16] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 128(%[m])\n\t" - "movq 128(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r11\n\t" - "movq %%r11, 128(%[a])\n\t" - "adcq $0, %%r9\n\t" - "# a[i+17] += m[17] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 136(%[m])\n\t" - "movq 136(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 136(%[a])\n\t" - "adcq $0, %%r8\n\t" - "# a[i+18] += m[18] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 144(%[m])\n\t" - "movq 144(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r11\n\t" - "movq %%r11, 144(%[a])\n\t" - "adcq $0, %%r9\n\t" - "# a[i+19] += m[19] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 152(%[m])\n\t" - "movq 152(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 152(%[a])\n\t" - "adcq $0, %%r8\n\t" - "# a[i+20] += m[20] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 160(%[m])\n\t" - "movq 160(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r11\n\t" - "movq %%r11, 160(%[a])\n\t" - "adcq $0, %%r9\n\t" - "# a[i+21] += m[21] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 168(%[m])\n\t" - "movq 168(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 168(%[a])\n\t" - "adcq $0, %%r8\n\t" - "# a[i+22] += m[22] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 176(%[m])\n\t" - "movq 176(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r11\n\t" - "movq %%r11, 176(%[a])\n\t" - "adcq $0, %%r9\n\t" - "# a[i+23] += m[23] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 184(%[m])\n\t" - "movq 184(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 184(%[a])\n\t" - "adcq $0, %%r8\n\t" - "# a[i+24] += m[24] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 192(%[m])\n\t" - "movq 192(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r11\n\t" - "movq %%r11, 192(%[a])\n\t" - "adcq $0, %%r9\n\t" - "# a[i+25] += m[25] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 200(%[m])\n\t" - "movq 200(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 200(%[a])\n\t" - "adcq $0, %%r8\n\t" - "# a[i+26] += m[26] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 208(%[m])\n\t" - "movq 208(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r11\n\t" - "movq %%r11, 208(%[a])\n\t" - "adcq $0, %%r9\n\t" - "# a[i+27] += m[27] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 216(%[m])\n\t" - "movq 216(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 216(%[a])\n\t" - "adcq $0, %%r8\n\t" - "# a[i+28] += m[28] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 224(%[m])\n\t" - "movq 224(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r11\n\t" - "movq %%r11, 224(%[a])\n\t" - "adcq $0, %%r9\n\t" - "# a[i+29] += m[29] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 232(%[m])\n\t" - "movq 232(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 232(%[a])\n\t" - "adcq $0, %%r8\n\t" - "# a[i+30] += m[30] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 240(%[m])\n\t" - "movq 240(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r11\n\t" - "movq %%r11, 240(%[a])\n\t" - "adcq $0, %%r9\n\t" - "# a[i+31] += m[31] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 248(%[m])\n\t" - "movq 248(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 248(%[a])\n\t" - "adcq $0, %%r8\n\t" - "# a[i+32] += m[32] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 256(%[m])\n\t" - "movq 256(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r11\n\t" - "movq %%r11, 256(%[a])\n\t" - "adcq $0, %%r9\n\t" - "# a[i+33] += m[33] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 264(%[m])\n\t" - "movq 264(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 264(%[a])\n\t" - "adcq $0, %%r8\n\t" - "# a[i+34] += m[34] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 272(%[m])\n\t" - "movq 272(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r11\n\t" - "movq %%r11, 272(%[a])\n\t" - "adcq $0, %%r9\n\t" - "# a[i+35] += m[35] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 280(%[m])\n\t" - "movq 280(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 280(%[a])\n\t" - "adcq $0, %%r8\n\t" - "# a[i+36] += m[36] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 288(%[m])\n\t" - "movq 288(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r11\n\t" - "movq %%r11, 288(%[a])\n\t" - "adcq $0, %%r9\n\t" - "# a[i+37] += m[37] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 296(%[m])\n\t" - "movq 296(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 296(%[a])\n\t" - "adcq $0, %%r8\n\t" - "# a[i+38] += m[38] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 304(%[m])\n\t" - "movq 304(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r11\n\t" - "movq %%r11, 304(%[a])\n\t" - "adcq $0, %%r9\n\t" - "# a[i+39] += m[39] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 312(%[m])\n\t" - "movq 312(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 312(%[a])\n\t" - "adcq $0, %%r8\n\t" - "# a[i+40] += m[40] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 320(%[m])\n\t" - "movq 320(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r11\n\t" - "movq %%r11, 320(%[a])\n\t" - "adcq $0, %%r9\n\t" - "# a[i+41] += m[41] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 328(%[m])\n\t" - "movq 328(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 328(%[a])\n\t" - "adcq $0, %%r8\n\t" - "# a[i+42] += m[42] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 336(%[m])\n\t" - "movq 336(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r11\n\t" - "movq %%r11, 336(%[a])\n\t" - "adcq $0, %%r9\n\t" - "# a[i+43] += m[43] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 344(%[m])\n\t" - "movq 344(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 344(%[a])\n\t" - "adcq $0, %%r8\n\t" - "# a[i+44] += m[44] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 352(%[m])\n\t" - "movq 352(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r11\n\t" - "movq %%r11, 352(%[a])\n\t" - "adcq $0, %%r9\n\t" - "# a[i+45] += m[45] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 360(%[m])\n\t" - "movq 360(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r8\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 360(%[a])\n\t" - "adcq $0, %%r8\n\t" - "# a[i+46] += m[46] * mu\n\t" - "movq %%r10, %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "mulq 368(%[m])\n\t" - "movq 368(%[a]), %%r11\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r9\n\t" - "addq %%r8, %%r11\n\t" - "movq %%r11, 368(%[a])\n\t" - "adcq $0, %%r9\n\t" - "# a[i+47] += m[47] * mu\n\t" - "movq %%r10, %%rax\n\t" - "mulq 376(%[m])\n\t" - "movq 376(%[a]), %%r11\n\t" - "addq %%rax, %%r9\n\t" - "adcq %[ca], %%rdx\n\t" - "movq $0, %[ca]\n\t" - "adcq $0, %[ca]\n\t" - "addq %%r9, %%r11\n\t" - "movq %%r11, 376(%[a])\n\t" - "adcq %%rdx, 384(%[a])\n\t" - "adcq $0, %[ca]\n\t" - "# i += 1\n\t" - "addq $8, %[a]\n\t" - "addq $8, %%rcx\n\t" - "cmpq $384, %%rcx\n\t" - "jl L_mont_loop_48\n\t" - "movq %%r12, 0(%[a])\n\t" - "movq %%r13, 8(%[a])\n\t" - : [ca] "+r" (ca), [a] "+r" (a) - : [m] "r" (m), [mp] "r" (mp) - : "memory", "rax", "rdx", "rcx", "r8", "r9", "r10", "r11", - "r12", "r13" - ); - - sp_3072_cond_sub_48(a - 48, a, m, (sp_digit)0 - ca); -} - +extern sp_digit sp_3072_cond_sub_48(sp_digit* r, sp_digit* a, sp_digit* b, sp_digit m); +extern void sp_3072_mont_reduce_48(sp_digit* a, sp_digit* m, sp_digit mp); /* Multiply two Montogmery form numbers mod the modulus (prime). * (r = a * b mod m) * @@ -24949,711 +2649,8 @@ static void sp_3072_mont_sqr_48(sp_digit* r, sp_digit* a, sp_digit* m, sp_3072_mont_reduce_48(r, m, mp); } -/* Mul a by digit b into r. (r = a * b) - * - * r A single precision integer. - * a A single precision integer. - * b A single precision digit. - */ -SP_NOINLINE static void sp_3072_mul_d_48(sp_digit* r, const sp_digit* a, - const sp_digit b) -{ - __asm__ __volatile__ ( - "# A[0] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq (%[a])\n\t" - "movq %%rax, %%rbx\n\t" - "movq %%rdx, %%rcx\n\t" - "movq %%rbx, 0(%[r])\n\t" - "# A[1] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rbx, %%rbx\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "movq %%rcx, 8(%[r])\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[2] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rcx, %%rcx\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "movq %%r8, 16(%[r])\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[3] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "movq %%rbx, 24(%[r])\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[4] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rbx, %%rbx\n\t" - "mulq 32(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "movq %%rcx, 32(%[r])\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[5] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rcx, %%rcx\n\t" - "mulq 40(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "movq %%r8, 40(%[r])\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[6] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 48(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "movq %%rbx, 48(%[r])\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[7] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rbx, %%rbx\n\t" - "mulq 56(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "movq %%rcx, 56(%[r])\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[8] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rcx, %%rcx\n\t" - "mulq 64(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "movq %%r8, 64(%[r])\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[9] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 72(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "movq %%rbx, 72(%[r])\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[10] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rbx, %%rbx\n\t" - "mulq 80(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "movq %%rcx, 80(%[r])\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[11] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rcx, %%rcx\n\t" - "mulq 88(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "movq %%r8, 88(%[r])\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[12] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 96(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "movq %%rbx, 96(%[r])\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[13] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rbx, %%rbx\n\t" - "mulq 104(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "movq %%rcx, 104(%[r])\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[14] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rcx, %%rcx\n\t" - "mulq 112(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "movq %%r8, 112(%[r])\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[15] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 120(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "movq %%rbx, 120(%[r])\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[16] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rbx, %%rbx\n\t" - "mulq 128(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "movq %%rcx, 128(%[r])\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[17] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rcx, %%rcx\n\t" - "mulq 136(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "movq %%r8, 136(%[r])\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[18] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 144(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "movq %%rbx, 144(%[r])\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[19] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rbx, %%rbx\n\t" - "mulq 152(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "movq %%rcx, 152(%[r])\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[20] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rcx, %%rcx\n\t" - "mulq 160(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "movq %%r8, 160(%[r])\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[21] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 168(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "movq %%rbx, 168(%[r])\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[22] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rbx, %%rbx\n\t" - "mulq 176(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "movq %%rcx, 176(%[r])\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[23] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rcx, %%rcx\n\t" - "mulq 184(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "movq %%r8, 184(%[r])\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[24] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 192(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "movq %%rbx, 192(%[r])\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[25] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rbx, %%rbx\n\t" - "mulq 200(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "movq %%rcx, 200(%[r])\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[26] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rcx, %%rcx\n\t" - "mulq 208(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "movq %%r8, 208(%[r])\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[27] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 216(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "movq %%rbx, 216(%[r])\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[28] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rbx, %%rbx\n\t" - "mulq 224(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "movq %%rcx, 224(%[r])\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[29] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rcx, %%rcx\n\t" - "mulq 232(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "movq %%r8, 232(%[r])\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[30] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 240(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "movq %%rbx, 240(%[r])\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[31] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rbx, %%rbx\n\t" - "mulq 248(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "movq %%rcx, 248(%[r])\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[32] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rcx, %%rcx\n\t" - "mulq 256(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "movq %%r8, 256(%[r])\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[33] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 264(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "movq %%rbx, 264(%[r])\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[34] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rbx, %%rbx\n\t" - "mulq 272(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "movq %%rcx, 272(%[r])\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[35] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rcx, %%rcx\n\t" - "mulq 280(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "movq %%r8, 280(%[r])\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[36] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 288(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "movq %%rbx, 288(%[r])\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[37] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rbx, %%rbx\n\t" - "mulq 296(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "movq %%rcx, 296(%[r])\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[38] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rcx, %%rcx\n\t" - "mulq 304(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "movq %%r8, 304(%[r])\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[39] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 312(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "movq %%rbx, 312(%[r])\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[40] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rbx, %%rbx\n\t" - "mulq 320(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "movq %%rcx, 320(%[r])\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[41] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rcx, %%rcx\n\t" - "mulq 328(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "movq %%r8, 328(%[r])\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[42] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 336(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "movq %%rbx, 336(%[r])\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[43] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rbx, %%rbx\n\t" - "mulq 344(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "movq %%rcx, 344(%[r])\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[44] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rcx, %%rcx\n\t" - "mulq 352(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "movq %%r8, 352(%[r])\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[45] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq 360(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "movq %%rbx, 360(%[r])\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[46] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rbx, %%rbx\n\t" - "mulq 368(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "movq %%rcx, 368(%[r])\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[47] * B\n\t" - "movq %[b], %%rax\n\t" - "mulq 376(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "movq %%r8, 376(%[r])\n\t" - "movq %%rbx, 384(%[r])\n\t" - : - : [r] "r" (r), [a] "r" (a), [b] "r" (b) - : "memory", "rax", "rdx", "rbx", "rcx", "r8" - ); -} - -#ifdef HAVE_INTEL_AVX2 -/* Mul a by digit b into r. (r = a * b) - * - * r A single precision integer. - * a A single precision integer. - * b A single precision digit. - */ -SP_NOINLINE static void sp_3072_mul_d_avx2_48(sp_digit* r, const sp_digit* a, - const sp_digit b) -{ - __asm__ __volatile__ ( - "# A[0] * B\n\t" - "movq %[b], %%rdx\n\t" - "xorq %%r10, %%r10\n\t" - "mulxq (%[a]), %%r8, %%r9\n\t" - "movq %%r8, 0(%[r])\n\t" - "# A[1] * B\n\t" - "mulxq 8(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 8(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[2] * B\n\t" - "mulxq 16(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 16(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[3] * B\n\t" - "mulxq 24(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 24(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[4] * B\n\t" - "mulxq 32(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 32(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[5] * B\n\t" - "mulxq 40(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 40(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[6] * B\n\t" - "mulxq 48(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 48(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[7] * B\n\t" - "mulxq 56(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 56(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[8] * B\n\t" - "mulxq 64(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 64(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[9] * B\n\t" - "mulxq 72(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 72(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[10] * B\n\t" - "mulxq 80(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 80(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[11] * B\n\t" - "mulxq 88(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 88(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[12] * B\n\t" - "mulxq 96(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 96(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[13] * B\n\t" - "mulxq 104(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 104(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[14] * B\n\t" - "mulxq 112(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 112(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[15] * B\n\t" - "mulxq 120(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 120(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[16] * B\n\t" - "mulxq 128(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 128(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[17] * B\n\t" - "mulxq 136(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 136(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[18] * B\n\t" - "mulxq 144(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 144(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[19] * B\n\t" - "mulxq 152(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 152(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[20] * B\n\t" - "mulxq 160(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 160(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[21] * B\n\t" - "mulxq 168(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 168(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[22] * B\n\t" - "mulxq 176(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 176(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[23] * B\n\t" - "mulxq 184(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 184(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[24] * B\n\t" - "mulxq 192(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 192(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[25] * B\n\t" - "mulxq 200(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 200(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[26] * B\n\t" - "mulxq 208(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 208(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[27] * B\n\t" - "mulxq 216(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 216(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[28] * B\n\t" - "mulxq 224(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 224(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[29] * B\n\t" - "mulxq 232(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 232(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[30] * B\n\t" - "mulxq 240(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 240(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[31] * B\n\t" - "mulxq 248(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 248(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[32] * B\n\t" - "mulxq 256(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 256(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[33] * B\n\t" - "mulxq 264(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 264(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[34] * B\n\t" - "mulxq 272(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 272(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[35] * B\n\t" - "mulxq 280(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 280(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[36] * B\n\t" - "mulxq 288(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 288(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[37] * B\n\t" - "mulxq 296(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 296(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[38] * B\n\t" - "mulxq 304(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 304(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[39] * B\n\t" - "mulxq 312(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 312(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[40] * B\n\t" - "mulxq 320(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 320(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[41] * B\n\t" - "mulxq 328(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 328(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[42] * B\n\t" - "mulxq 336(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 336(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[43] * B\n\t" - "mulxq 344(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 344(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[44] * B\n\t" - "mulxq 352(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 352(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[45] * B\n\t" - "mulxq 360(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 360(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[46] * B\n\t" - "mulxq 368(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 368(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[47] * B\n\t" - "mulxq 376(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "adoxq %%rcx, %%r8\n\t" - "adcxq %%r10, %%r8\n\t" - "movq %%r9, 376(%[r])\n\t" - "movq %%r8, 384(%[r])\n\t" - : - : [r] "r" (r), [a] "r" (a), [b] "r" (b) - : "memory", "rax", "rdx", "rcx", "r8", "r9", "r10" - ); -} -#endif /* HAVE_INTEL_AVX2 */ - +extern void sp_3072_mul_d_48(sp_digit* r, const sp_digit* a, const sp_digit b); +extern void sp_3072_mul_d_avx2_48(sp_digit* r, const sp_digit* a, const sp_digit b); /* Divide the double width number (d1|d0) by the dividend. (d1|d0 / div) * * d1 The high order half of the number to divide. @@ -25661,23 +2658,18 @@ SP_NOINLINE static void sp_3072_mul_d_avx2_48(sp_digit* r, const sp_digit* a, * div The dividend. * returns the result of the division. */ -static sp_digit div_3072_word_48(sp_digit d1, sp_digit d0, sp_digit div) +static WC_INLINE sp_digit div_3072_word_48(sp_digit d1, sp_digit d0, + sp_digit div) { - sp_digit r; - + register sp_digit r asm("rax"); __asm__ __volatile__ ( - "movq %[d0], %%rax\n\t" - "movq %[d1], %%rdx\n\t" - "divq %[div]\n\t" - "movq %%rax, %[r]\n\t" - : [r] "=r" (r) - : [d1] "r" (d1), [d0] "r" (d0), [div] "r" (div) - : "rax", "rdx" + "divq %3" + : "=a" (r) + : "d" (d1), "a" (d0), "r" (div) + : ); - return r; } - /* AND m into each word of a and store in r. * * r A single precision integer. @@ -25707,414 +2699,7 @@ static void sp_3072_mask_48(sp_digit* r, sp_digit* a, sp_digit m) #endif } -/* Compare a with b in constant time. - * - * a A single precision integer. - * b A single precision integer. - * return -ve, 0 or +ve if a is less than, equal to or greater than b - * respectively. - */ -static int64_t sp_3072_cmp_48(sp_digit* a, sp_digit* b) -{ - sp_digit r = -1; - sp_digit one = 1; - - __asm__ __volatile__ ( - "xorq %%rcx, %%rcx\n\t" - "movq $-1, %%rdx\n\t" - "movq 376(%[a]), %%rbx\n\t" - "movq 376(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 368(%[a]), %%rbx\n\t" - "movq 368(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 360(%[a]), %%rbx\n\t" - "movq 360(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 352(%[a]), %%rbx\n\t" - "movq 352(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 344(%[a]), %%rbx\n\t" - "movq 344(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 336(%[a]), %%rbx\n\t" - "movq 336(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 328(%[a]), %%rbx\n\t" - "movq 328(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 320(%[a]), %%rbx\n\t" - "movq 320(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 312(%[a]), %%rbx\n\t" - "movq 312(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 304(%[a]), %%rbx\n\t" - "movq 304(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 296(%[a]), %%rbx\n\t" - "movq 296(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 288(%[a]), %%rbx\n\t" - "movq 288(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 280(%[a]), %%rbx\n\t" - "movq 280(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 272(%[a]), %%rbx\n\t" - "movq 272(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 264(%[a]), %%rbx\n\t" - "movq 264(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 256(%[a]), %%rbx\n\t" - "movq 256(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 248(%[a]), %%rbx\n\t" - "movq 248(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 240(%[a]), %%rbx\n\t" - "movq 240(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 232(%[a]), %%rbx\n\t" - "movq 232(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 224(%[a]), %%rbx\n\t" - "movq 224(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 216(%[a]), %%rbx\n\t" - "movq 216(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 208(%[a]), %%rbx\n\t" - "movq 208(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 200(%[a]), %%rbx\n\t" - "movq 200(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 192(%[a]), %%rbx\n\t" - "movq 192(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 184(%[a]), %%rbx\n\t" - "movq 184(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 176(%[a]), %%rbx\n\t" - "movq 176(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 168(%[a]), %%rbx\n\t" - "movq 168(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 160(%[a]), %%rbx\n\t" - "movq 160(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 152(%[a]), %%rbx\n\t" - "movq 152(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 144(%[a]), %%rbx\n\t" - "movq 144(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 136(%[a]), %%rbx\n\t" - "movq 136(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 128(%[a]), %%rbx\n\t" - "movq 128(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 120(%[a]), %%rbx\n\t" - "movq 120(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 112(%[a]), %%rbx\n\t" - "movq 112(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 104(%[a]), %%rbx\n\t" - "movq 104(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 96(%[a]), %%rbx\n\t" - "movq 96(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 88(%[a]), %%rbx\n\t" - "movq 88(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 80(%[a]), %%rbx\n\t" - "movq 80(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 72(%[a]), %%rbx\n\t" - "movq 72(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 64(%[a]), %%rbx\n\t" - "movq 64(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 56(%[a]), %%rbx\n\t" - "movq 56(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 48(%[a]), %%rbx\n\t" - "movq 48(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 40(%[a]), %%rbx\n\t" - "movq 40(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 32(%[a]), %%rbx\n\t" - "movq 32(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 24(%[a]), %%rbx\n\t" - "movq 24(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 16(%[a]), %%rbx\n\t" - "movq 16(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 8(%[a]), %%rbx\n\t" - "movq 8(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 0(%[a]), %%rbx\n\t" - "movq 0(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "xorq %%rdx, %[r]\n\t" - : [r] "+r" (r) - : [a] "r" (a), [b] "r" (b), [one] "r" (one) - : "rax", "rdx", "rcx", "rbx", "r8" - ); - - return r; -} - +extern int64_t sp_3072_cmp_48(sp_digit* a, sp_digit* b); /* Divide d in a and put remainder into r (m*d + r = a) * m is not calculated as it is not needed at this time. * @@ -26378,334 +2963,7 @@ static int sp_3072_mod_exp_48(sp_digit* r, sp_digit* a, sp_digit* e, } #endif /* SP_RSA_PRIVATE_EXP_D || WOLFSSL_HAVE_SP_DH */ -#ifdef HAVE_INTEL_AVX2 -/* Reduce the number back to 3072 bits using Montgomery reduction. - * - * a A single precision number to reduce in place. - * m The single precision number representing the modulus. - * mp The digit representing the negative inverse of m mod 2^n. - */ -SP_NOINLINE static void sp_3072_mont_reduce_avx2_48(sp_digit* a, sp_digit* m, - sp_digit mp) -{ - sp_digit ca = 0; - - __asm__ __volatile__ ( - "# i = 0\n\t" - "movq 0(%[a]), %%r12\n\t" - "xorq %%rcx, %%rcx\n\t" - "\nL_mont_loop_avx2_48:\n\t" - "xorq %%r9, %%r9\n\t" - "movq %%r12, %%r10\n\t" - "# mu = a[i] * mp\n\t" - "movq %%r12, %%rdx\n\t" - "mulxq %[mp], %%rdx, %%r8\n\t" - "# a[i+0] += m[0] * mu\n\t" - "mulxq 0(%[m]), %%rax, %%r8\n\t" - "movq 8(%[a]), %%r12\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r12\n\t" - "# a[i+1] += m[1] * mu\n\t" - "mulxq 8(%[m]), %%rax, %%r8\n\t" - "movq 16(%[a]), %%r10\n\t" - "adcxq %%rax, %%r12\n\t" - "adoxq %%r8, %%r10\n\t" - "# a[i+2] += m[2] * mu\n\t" - "mulxq 16(%[m]), %%rax, %%r8\n\t" - "movq 24(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 16(%[a])\n\t" - "# a[i+3] += m[3] * mu\n\t" - "mulxq 24(%[m]), %%rax, %%r8\n\t" - "movq 32(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 24(%[a])\n\t" - "# a[i+4] += m[4] * mu\n\t" - "mulxq 32(%[m]), %%rax, %%r8\n\t" - "movq 40(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 32(%[a])\n\t" - "# a[i+5] += m[5] * mu\n\t" - "mulxq 40(%[m]), %%rax, %%r8\n\t" - "movq 48(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 40(%[a])\n\t" - "# a[i+6] += m[6] * mu\n\t" - "mulxq 48(%[m]), %%rax, %%r8\n\t" - "movq 56(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 48(%[a])\n\t" - "# a[i+7] += m[7] * mu\n\t" - "mulxq 56(%[m]), %%rax, %%r8\n\t" - "movq 64(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 56(%[a])\n\t" - "# a[i+8] += m[8] * mu\n\t" - "mulxq 64(%[m]), %%rax, %%r8\n\t" - "movq 72(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 64(%[a])\n\t" - "# a[i+9] += m[9] * mu\n\t" - "mulxq 72(%[m]), %%rax, %%r8\n\t" - "movq 80(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 72(%[a])\n\t" - "# a[i+10] += m[10] * mu\n\t" - "mulxq 80(%[m]), %%rax, %%r8\n\t" - "movq 88(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 80(%[a])\n\t" - "# a[i+11] += m[11] * mu\n\t" - "mulxq 88(%[m]), %%rax, %%r8\n\t" - "movq 96(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 88(%[a])\n\t" - "# a[i+12] += m[12] * mu\n\t" - "mulxq 96(%[m]), %%rax, %%r8\n\t" - "movq 104(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 96(%[a])\n\t" - "# a[i+13] += m[13] * mu\n\t" - "mulxq 104(%[m]), %%rax, %%r8\n\t" - "movq 112(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 104(%[a])\n\t" - "# a[i+14] += m[14] * mu\n\t" - "mulxq 112(%[m]), %%rax, %%r8\n\t" - "movq 120(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 112(%[a])\n\t" - "# a[i+15] += m[15] * mu\n\t" - "mulxq 120(%[m]), %%rax, %%r8\n\t" - "movq 128(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 120(%[a])\n\t" - "# a[i+16] += m[16] * mu\n\t" - "mulxq 128(%[m]), %%rax, %%r8\n\t" - "movq 136(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 128(%[a])\n\t" - "# a[i+17] += m[17] * mu\n\t" - "mulxq 136(%[m]), %%rax, %%r8\n\t" - "movq 144(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 136(%[a])\n\t" - "# a[i+18] += m[18] * mu\n\t" - "mulxq 144(%[m]), %%rax, %%r8\n\t" - "movq 152(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 144(%[a])\n\t" - "# a[i+19] += m[19] * mu\n\t" - "mulxq 152(%[m]), %%rax, %%r8\n\t" - "movq 160(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 152(%[a])\n\t" - "# a[i+20] += m[20] * mu\n\t" - "mulxq 160(%[m]), %%rax, %%r8\n\t" - "movq 168(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 160(%[a])\n\t" - "# a[i+21] += m[21] * mu\n\t" - "mulxq 168(%[m]), %%rax, %%r8\n\t" - "movq 176(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 168(%[a])\n\t" - "# a[i+22] += m[22] * mu\n\t" - "mulxq 176(%[m]), %%rax, %%r8\n\t" - "movq 184(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 176(%[a])\n\t" - "# a[i+23] += m[23] * mu\n\t" - "mulxq 184(%[m]), %%rax, %%r8\n\t" - "movq 192(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 184(%[a])\n\t" - "# a[i+24] += m[24] * mu\n\t" - "mulxq 192(%[m]), %%rax, %%r8\n\t" - "movq 200(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 192(%[a])\n\t" - "# a[i+25] += m[25] * mu\n\t" - "mulxq 200(%[m]), %%rax, %%r8\n\t" - "movq 208(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 200(%[a])\n\t" - "# a[i+26] += m[26] * mu\n\t" - "mulxq 208(%[m]), %%rax, %%r8\n\t" - "movq 216(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 208(%[a])\n\t" - "# a[i+27] += m[27] * mu\n\t" - "mulxq 216(%[m]), %%rax, %%r8\n\t" - "movq 224(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 216(%[a])\n\t" - "# a[i+28] += m[28] * mu\n\t" - "mulxq 224(%[m]), %%rax, %%r8\n\t" - "movq 232(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 224(%[a])\n\t" - "# a[i+29] += m[29] * mu\n\t" - "mulxq 232(%[m]), %%rax, %%r8\n\t" - "movq 240(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 232(%[a])\n\t" - "# a[i+30] += m[30] * mu\n\t" - "mulxq 240(%[m]), %%rax, %%r8\n\t" - "movq 248(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 240(%[a])\n\t" - "# a[i+31] += m[31] * mu\n\t" - "mulxq 248(%[m]), %%rax, %%r8\n\t" - "movq 256(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 248(%[a])\n\t" - "# a[i+32] += m[32] * mu\n\t" - "mulxq 256(%[m]), %%rax, %%r8\n\t" - "movq 264(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 256(%[a])\n\t" - "# a[i+33] += m[33] * mu\n\t" - "mulxq 264(%[m]), %%rax, %%r8\n\t" - "movq 272(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 264(%[a])\n\t" - "# a[i+34] += m[34] * mu\n\t" - "mulxq 272(%[m]), %%rax, %%r8\n\t" - "movq 280(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 272(%[a])\n\t" - "# a[i+35] += m[35] * mu\n\t" - "mulxq 280(%[m]), %%rax, %%r8\n\t" - "movq 288(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 280(%[a])\n\t" - "# a[i+36] += m[36] * mu\n\t" - "mulxq 288(%[m]), %%rax, %%r8\n\t" - "movq 296(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 288(%[a])\n\t" - "# a[i+37] += m[37] * mu\n\t" - "mulxq 296(%[m]), %%rax, %%r8\n\t" - "movq 304(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 296(%[a])\n\t" - "# a[i+38] += m[38] * mu\n\t" - "mulxq 304(%[m]), %%rax, %%r8\n\t" - "movq 312(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 304(%[a])\n\t" - "# a[i+39] += m[39] * mu\n\t" - "mulxq 312(%[m]), %%rax, %%r8\n\t" - "movq 320(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 312(%[a])\n\t" - "# a[i+40] += m[40] * mu\n\t" - "mulxq 320(%[m]), %%rax, %%r8\n\t" - "movq 328(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 320(%[a])\n\t" - "# a[i+41] += m[41] * mu\n\t" - "mulxq 328(%[m]), %%rax, %%r8\n\t" - "movq 336(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 328(%[a])\n\t" - "# a[i+42] += m[42] * mu\n\t" - "mulxq 336(%[m]), %%rax, %%r8\n\t" - "movq 344(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 336(%[a])\n\t" - "# a[i+43] += m[43] * mu\n\t" - "mulxq 344(%[m]), %%rax, %%r8\n\t" - "movq 352(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 344(%[a])\n\t" - "# a[i+44] += m[44] * mu\n\t" - "mulxq 352(%[m]), %%rax, %%r8\n\t" - "movq 360(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 352(%[a])\n\t" - "# a[i+45] += m[45] * mu\n\t" - "mulxq 360(%[m]), %%rax, %%r8\n\t" - "movq 368(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 360(%[a])\n\t" - "# a[i+46] += m[46] * mu\n\t" - "mulxq 368(%[m]), %%rax, %%r8\n\t" - "movq 376(%[a]), %%r11\n\t" - "adcxq %%rax, %%r10\n\t" - "adoxq %%r8, %%r11\n\t" - "movq %%r10, 368(%[a])\n\t" - "# a[i+47] += m[47] * mu\n\t" - "mulxq 376(%[m]), %%rax, %%r8\n\t" - "movq 384(%[a]), %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "adoxq %%r8, %%r10\n\t" - "movq %%r11, 376(%[a])\n\t" - "adcxq %[ca], %%r10\n\t" - "movq %%r9, %[ca]\n\t" - "adoxq %%r9, %[ca]\n\t" - "adcxq %%r9, %[ca]\n\t" - "movq %%r10, 384(%[a])\n\t" - "# i += 1\n\t" - "addq $8, %[a]\n\t" - "addq $1, %%rcx\n\t" - "cmpq $48, %%rcx\n\t" - "jl L_mont_loop_avx2_48\n\t" - "movq %%r12, 0(%[a])\n\t" - : [ca] "+r" (ca), [a] "+r" (a) - : [m] "r" (m), [mp] "r" (mp) - : "memory", "rax", "rdx", "rcx", "r8", "r9", "r10", "r11", "r12" - ); - - sp_3072_cond_sub_48(a - 48, a, m, (sp_digit)0 - ca); -} -#endif /* HAVE_INTEL_AVX2 */ - +extern void sp_3072_mont_reduce_avx2_48(sp_digit* a, sp_digit* m, sp_digit mp); #ifdef HAVE_INTEL_AVX2 /* Multiply two Montogmery form numbers mod the modulus (prime). * (r = a * b mod m) @@ -27356,7 +3614,7 @@ int sp_DhExp_3072(mp_int* base, const byte* exp, word32 expLen, } #endif /* WOLFSSL_HAVE_SP_DH */ -#endif /* WOLFSSL_SP_NO_3072 */ +#endif /* !WOLFSSL_SP_NO_3072 */ #endif /* WOLFSSL_HAVE_SP_RSA || WOLFSSL_HAVE_SP_DH */ #ifdef WOLFSSL_HAVE_SP_ECC @@ -27696,638 +3954,21 @@ static int sp_256_point_to_ecc_point_4(sp_point* p, ecc_point* pm) return err; } -/* Conditionally copy a into r using the mask m. - * m is -1 to copy and 0 when not. - * - * r A single precision number to copy over. - * a A single precision number to copy. - * m Mask value to apply. - */ -static void sp_256_cond_copy_4(sp_digit* r, const sp_digit* a, const sp_digit m) -{ - sp_digit t[4]; - __asm__ __volatile__ ( - "movq (%[r]), %%rax\n\t" - "movq 8(%[r]), %%rcx\n\t" - "movq 16(%[r]), %%rdx\n\t" - "movq 24(%[r]), %%r8\n\t" - "xorq (%[a]), %%rax\n\t" - "xorq 8(%[a]), %%rcx\n\t" - "xorq 16(%[a]), %%rdx\n\t" - "xorq 24(%[a]), %%r8\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "andq %[m], %%rdx\n\t" - "andq %[m], %%r8\n\t" - "xorq %%rax, (%[r])\n\t" - "xorq %%rcx, 8(%[r])\n\t" - "xorq %%rdx, 16(%[r])\n\t" - "xorq %%r8, 24(%[r])\n\t" - : - : [r] "r" (r), [a] "r" (a), [m] "r" (m), [t] "r" (t) - : "memory", "rax", "rcx", "rdx", "r8" - ); -} - -/* Compare a with b in constant time. - * - * a A single precision integer. - * b A single precision integer. - * return -ve, 0 or +ve if a is less than, equal to or greater than b - * respectively. - */ -static int64_t sp_256_cmp_4(sp_digit* a, sp_digit* b) -{ - sp_digit r = -1; - sp_digit one = 1; - - __asm__ __volatile__ ( - "xorq %%rcx, %%rcx\n\t" - "movq $-1, %%rdx\n\t" - "movq 24(%[a]), %%rbx\n\t" - "movq 24(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 16(%[a]), %%rbx\n\t" - "movq 16(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 8(%[a]), %%rbx\n\t" - "movq 8(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "movq 0(%[a]), %%rbx\n\t" - "movq 0(%[b]), %%r8\n\t" - "andq %%rdx, %%rbx\n\t" - "andq %%rdx, %%r8\n\t" - "subq %%r8, %%rbx\n\t" - "cmova %[one], %[r]\n\t" - "cmovc %%rdx, %[r]\n\t" - "cmovnz %%rcx, %%rdx\n\t" - "xorq %%rdx, %[r]\n\t" - : [r] "+r" (r) - : [a] "r" (a), [b] "r" (b), [one] "r" (one) - : "rax", "rdx", "rcx", "rbx", "r8" - ); - - return r; -} - +extern void sp_256_cond_copy_4(sp_digit* r, sp_digit* a, sp_digit m); +extern int64_t sp_256_cmp_4(sp_digit* a, sp_digit* b); /* Normalize the values in each word to 64. * * a Array of sp_digit to normalize. */ #define sp_256_norm_4(a) -/* Conditionally subtract b from a using the mask m. - * m is -1 to subtract and 0 when not copying. - * - * r A single precision number representing condition subtract result. - * a A single precision number to subtract from. - * b A single precision number to subtract. - * m Mask value to apply. - */ -static sp_digit sp_256_cond_sub_4(sp_digit* r, sp_digit* a, sp_digit* b, - sp_digit m) -{ - sp_digit t[4]; - sp_digit c = 0; - - __asm__ __volatile__ ( - "movq 0(%[b]), %%rax\n\t" - "movq 8(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 0(%[t])\n\t" - "movq %%rcx, 8(%[t])\n\t" - "movq 16(%[b]), %%rax\n\t" - "movq 24(%[b]), %%rcx\n\t" - "andq %[m], %%rax\n\t" - "andq %[m], %%rcx\n\t" - "movq %%rax, 16(%[t])\n\t" - "movq %%rcx, 24(%[t])\n\t" - "movq (%[a]), %%rax\n\t" - "movq (%[t]), %%rdx\n\t" - "subq %%rdx,%%rax\n\t" - "movq 8(%[a]), %%rcx\n\t" - "movq 8(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 0(%[r])\n\t" - "movq 16(%[a]), %%rax\n\t" - "movq 16(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 8(%[r])\n\t" - "movq 24(%[a]), %%rcx\n\t" - "movq 24(%[t]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 16(%[r])\n\t" - "movq %%rcx, 24(%[r])\n\t" - "sbbq $0, %[c]\n\t" - : [c] "+r" (c) - : [r] "r" (r), [a] "r" (a), [b] "r" (b), [m] "r" (m), [t] "r" (t) - : "memory", "rax", "rcx", "rdx" - ); - - return c; -} - -/* Sub b from a into r. (r = a - b) - * - * r A single precision integer. - * a A single precision integer. - * b A single precision integer. - */ -SP_NOINLINE static sp_digit sp_256_sub_4(sp_digit* r, const sp_digit* a, - const sp_digit* b) -{ - sp_digit c = 0; - - __asm__ __volatile__ ( - "movq (%[a]), %%rax\n\t" - "movq (%[b]), %%rdx\n\t" - "subq %%rdx, %%rax\n\t" - "movq 8(%[a]), %%rcx\n\t" - "movq 8(%[b]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 0(%[r])\n\t" - "movq 16(%[a]), %%rax\n\t" - "movq 16(%[b]), %%rdx\n\t" - "sbbq %%rdx, %%rax\n\t" - "movq %%rcx, 8(%[r])\n\t" - "movq 24(%[a]), %%rcx\n\t" - "movq 24(%[b]), %%rdx\n\t" - "sbbq %%rdx, %%rcx\n\t" - "movq %%rax, 16(%[r])\n\t" - "movq %%rcx, 24(%[r])\n\t" - "sbbq $0, %[c]\n\t" - : [c] "+r" (c) - : [r] "r" (r), [a] "r" (a), [b] "r" (b) - : "memory", "rax", "rcx", "rdx" - ); - - return c; -} - +extern sp_digit sp_256_cond_sub_4(sp_digit* r, sp_digit* a, sp_digit* b, sp_digit m); +extern sp_digit sp_256_sub_4(sp_digit* r, const sp_digit* a, const sp_digit* b); #define sp_256_mont_reduce_order_4 sp_256_mont_reduce_4 -/* Reduce the number back to 256 bits using Montgomery reduction. - * - * a A single precision number to reduce in place. - * m The single precision number representing the modulus. - * mp The digit representing the negative inverse of m mod 2^n. - */ -SP_NOINLINE static void sp_256_mont_reduce_4(sp_digit* a, sp_digit* m, - sp_digit mp) -{ - __asm__ __volatile__ ( - "# i = 0\n\t" - "xorq %%r13, %%r13\n\t" - "xorq %%rcx, %%rcx\n\t" - "movq %[a], %%r12\n\t" - "\nL_mont_loop_4:\n\t" - "# mu = a[i] * mp\n\t" - "movq 0(%%r12), %%r11\n\t" - "imulq %[mp], %%r11\n\t" - "# a[i+0] += m[0] * mu\n\t" - "movq 0(%[m]), %%rax\n\t" - "movq 8(%[m]), %%r9\n\t" - "mulq %%r11\n\t" - "movq 0(%%r12), %%rbx\n\t" - "addq %%rax, %%rbx\n\t" - "movq %%rdx, %%r8\n\t" - "movq %%rbx, 0(%%r12)\n\t" - "adcq $0, %%r8\n\t" - "# a[i+1] += m[1] * mu\n\t" - "movq %%r9, %%rax\n\t" - "mulq %%r11\n\t" - "movq 16(%[m]), %%r9\n\t" - "movq 8(%%r12), %%rbx\n\t" - "addq %%r8, %%rax\n\t" - "movq %%rdx, %%r10\n\t" - "adcq $0, %%r10\n\t" - "addq %%rax, %%rbx\n\t" - "movq %%rbx, 8(%%r12)\n\t" - "adcq $0, %%r10\n\t" - "# a[i+2] += m[2] * mu\n\t" - "movq %%r9, %%rax\n\t" - "mulq %%r11\n\t" - "movq 24(%[m]), %%r9\n\t" - "movq 16(%%r12), %%rbx\n\t" - "addq %%r10, %%rax\n\t" - "movq %%rdx, %%r8\n\t" - "adcq $0, %%r8\n\t" - "addq %%rax, %%rbx\n\t" - "movq %%rbx, 16(%%r12)\n\t" - "adcq $0, %%r8\n\t" - "# a[i+3] += m[3] * mu\n\t" - "movq %%r9, %%rax\n\t" - "mulq %%r11\n\t" - "movq 24(%%r12), %%rbx\n\t" - "addq %%r8, %%rax\n\t" - "adcq %%r13, %%rdx\n\t" - "movq $0, %%r13\n\t" - "adcq $0, %%r13\n\t" - "addq %%rax, %%rbx\n\t" - "movq %%rbx, 24(%%r12)\n\t" - "adcq %%rdx, 32(%%r12)\n\t" - "adcq $0, %%r13\n\t" - "# i += 1\n\t" - "addq $8, %%r12\n\t" - "addq $8, %%rcx\n\t" - "cmpq $32, %%rcx\n\t" - "jl L_mont_loop_4\n\t" - "xorq %%rax, %%rax\n\t" - "movq 32(%[a]), %%rdx\n\t" - "movq 40(%[a]), %%rcx\n\t" - "movq 48(%[a]), %%rbx\n\t" - "movq 56(%[a]), %%r8\n\t" - "subq %%r13, %%rax\n\t" - "movq 0(%[m]), %%r9\n\t" - "movq 8(%[m]), %%r10\n\t" - "movq 16(%[m]), %%r11\n\t" - "movq 24(%[m]), %%r12\n\t" - "andq %%rax, %%r9\n\t" - "andq %%rax, %%r10\n\t" - "andq %%rax, %%r11\n\t" - "andq %%rax, %%r12\n\t" - "subq %%r9, %%rdx\n\t" - "sbbq %%r10, %%rcx\n\t" - "sbbq %%r11, %%rbx\n\t" - "sbbq %%r12, %%r8\n\t" - "movq %%rdx, (%[a])\n\t" - "movq %%rcx, 8(%[a])\n\t" - "movq %%rbx, 16(%[a])\n\t" - "movq %%r8, 24(%[a])\n\t" - : - : [a] "r" (a), [m] "r" (m), [mp] "r" (mp) - : "memory", "rax", "rbx", "rdx", "rcx", "r8", "r9", "r10", "r11", - "r12", "r13" - ); -} - -/* Multiply two Montogmery form numbers mod the modulus (prime). - * (r = a * b mod m) - * - * r Result of multiplication. - * a First number to multiply in Montogmery form. - * b Second number to multiply in Montogmery form. - * m Modulus (prime). - * mp Montogmery mulitplier. - */ -SP_NOINLINE static void sp_256_mont_mul_4(sp_digit* r, sp_digit* a, sp_digit* b, - sp_digit* m, sp_digit mp) -{ - (void)mp; - - __asm__ __volatile__ ( - "# A[0] * B[0]\n\t" - "movq 0(%[b]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "movq %%rax, %%r8\n\t" - "movq %%rdx, %%r9\n\t" - "# A[0] * B[1]\n\t" - "movq 8(%[b]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%r10, %%r10\n\t" - "addq %%rax, %%r9\n\t" - "adcq %%rdx, %%r10\n\t" - "# A[1] * B[0]\n\t" - "movq 0(%[b]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "xorq %%r11, %%r11\n\t" - "addq %%rax, %%r9\n\t" - "adcq %%rdx, %%r10\n\t" - "adcq $0, %%r11\n\t" - "# A[0] * B[2]\n\t" - "movq 16(%[b]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "# A[1] * B[1]\n\t" - "movq 8(%[b]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "xorq %%r12, %%r12\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[2] * B[0]\n\t" - "movq 0(%[b]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "adcq $0, %%r12\n\t" - "# A[0] * B[3]\n\t" - "movq 24(%[b]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%r13, %%r13\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r12\n\t" - "adcq $0, %%r13\n\t" - "# A[1] * B[2]\n\t" - "movq 16(%[b]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r12\n\t" - "adcq $0, %%r13\n\t" - "# A[2] * B[1]\n\t" - "movq 8(%[b]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r12\n\t" - "adcq $0, %%r13\n\t" - "# A[3] * B[0]\n\t" - "movq 0(%[b]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r12\n\t" - "adcq $0, %%r13\n\t" - "# A[1] * B[3]\n\t" - "movq 24(%[b]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "xorq %%r14, %%r14\n\t" - "addq %%rax, %%r12\n\t" - "adcq %%rdx, %%r13\n\t" - "adcq $0, %%r14\n\t" - "# A[2] * B[2]\n\t" - "movq 16(%[b]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%r12\n\t" - "adcq %%rdx, %%r13\n\t" - "adcq $0, %%r14\n\t" - "# A[3] * B[1]\n\t" - "movq 8(%[b]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%r12\n\t" - "adcq %%rdx, %%r13\n\t" - "adcq $0, %%r14\n\t" - "# A[2] * B[3]\n\t" - "movq 24(%[b]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "xorq %%r15, %%r15\n\t" - "addq %%rax, %%r13\n\t" - "adcq %%rdx, %%r14\n\t" - "adcq $0, %%r15\n\t" - "# A[3] * B[2]\n\t" - "movq 16(%[b]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%r13\n\t" - "adcq %%rdx, %%r14\n\t" - "adcq $0, %%r15\n\t" - "# A[3] * B[3]\n\t" - "movq 24(%[b]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%r14\n\t" - "adcq %%rdx, %%r15\n\t" - "# Start Reduction\n\t" - "movq %%r8, %%rax\n\t" - "movq %%r9, %[a]\n\t" - "movq %%r10, %[b]\n\t" - "movq %%r11, %%rdx\n\t" - "# mu = a[0]-a[3] + a[0]-a[2] << 32 << 64 + (a[0] * 2) << 192\n\t" - "# - a[0] << 32 << 192\n\t" - "# + (a[0] * 2) << 192\n\t" - "addq %%r8, %%rdx\n\t" - "addq %%r8, %%rdx\n\t" - "# a[0]-a[2] << 32\n\t" - "shlq $32, %%r8\n\t" - "shldq $32, %[a], %%r10\n\t" - "shldq $32, %%rax, %%r9\n\t" - "# - a[0] << 32 << 192\n\t" - "subq %%r8, %%rdx\n\t" - "# + a[0]-a[2] << 32 << 64\n\t" - "addq %%r8, %[a]\n\t" - "adcq %%r9, %[b]\n\t" - "adcq %%r10, %%rdx\n\t" - "# a += (mu << 256) - (mu << 224) + (mu << 192) + (mu << 96) - mu\n\t" - "# a += mu << 256\n\t" - "xorq %%r8, %%r8\n\t" - "addq %%rax, %%r12\n\t" - "adcq %[a], %%r13\n\t" - "adcq %[b], %%r14\n\t" - "adcq %%rdx, %%r15\n\t" - "sbbq $0, %%r8\n\t" - "# a += mu << 192\n\t" - "addq %%rax, %%r11\n\t" - "adcq %[a], %%r12\n\t" - "adcq %[b], %%r13\n\t" - "adcq %%rdx, %%r14\n\t" - "adcq $0, %%r15\n\t" - "sbbq $0, %%r8\n\t" - "# mu <<= 32\n\t" - "movq %%rdx, %[m]\n\t" - "shldq $32, %[b], %%rdx\n\t" - "shldq $32, %[a], %[b]\n\t" - "shldq $32, %%rax, %[a]\n\t" - "shlq $32, %%rax\n\t" - "shrq $32, %[m]\n\t" - "# a += (mu << 32) << 64\n\t" - "addq %[b], %%r11\n\t" - "adcq %%rdx, %%r12\n\t" - "adcq %[m], %%r13\n\t" - "adcq $0, %%r14\n\t" - "adcq $0, %%r15\n\t" - "sbbq $0, %%r8\n\t" - "# a -= (mu << 32) << 192\n\t" - "subq %%rax, %%r11\n\t" - "movq $0xffffffff, %%rax\n\t" - "sbbq %[a], %%r12\n\t" - "movq $0xffffffff00000001, %[a]\n\t" - "sbbq %[b], %%r13\n\t" - "sbbq %%rdx, %%r14\n\t" - "sbbq %[m], %%r15\n\t" - "adcq $0, %%r8\n\t" - "# mask m and sub from result if overflow\n\t" - "# m[0] = -1 & mask = mask\n\t" - "andq %%r8, %%rax\n\t" - "# m[2] = 0 & mask = 0\n\t" - "andq %%r8, %[a]\n\t" - "subq %%r8, %%r12\n\t" - "sbbq %%rax, %%r13\n\t" - "sbbq $0, %%r14\n\t" - "sbbq %[a], %%r15\n\t" - "movq %%r12, 0(%[r])\n\t" - "movq %%r13, 8(%[r])\n\t" - "movq %%r14, 16(%[r])\n\t" - "movq %%r15, 24(%[r])\n\t" - : [m] "+r" (m), [a] "+r" (a), [b] "+r" (b) - : [r] "r" (r) - : "memory", "rax", "rdx", "r8", "r9", "r10", "r11", - "r12", "r13", "r14", "r15" - ); -} - -/* Square the Montgomery form number mod the modulus (prime). (r = a * a mod m) - * - * r Result of squaring. - * a Number to square in Montogmery form. - * m Modulus (prime). - * mp Montogmery mulitplier. - */ -SP_NOINLINE static void sp_256_mont_sqr_4(sp_digit* r, sp_digit* a, sp_digit* m, - sp_digit mp) -{ - __asm__ __volatile__ ( - "# A[0] * A[1]\n\t" - "movq 0(%[a]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "movq %%rax, %%r9\n\t" - "movq %%rdx, %%r10\n\t" - "# A[0] * A[2]\n\t" - "movq 0(%[a]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "xorq %%r11, %%r11\n\t" - "addq %%rax, %%r10\n\t" - "adcq %%rdx, %%r11\n\t" - "# A[0] * A[3]\n\t" - "movq 0(%[a]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "xorq %%r12, %%r12\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r12\n\t" - "# A[1] * A[2]\n\t" - "movq 8(%[a]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "xorq %%r13, %%r13\n\t" - "addq %%rax, %%r11\n\t" - "adcq %%rdx, %%r12\n\t" - "adcq $0, %%r13\n\t" - "# A[1] * A[3]\n\t" - "movq 8(%[a]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%r12\n\t" - "adcq %%rdx, %%r13\n\t" - "# A[2] * A[3]\n\t" - "movq 16(%[a]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "xorq %%r14, %%r14\n\t" - "addq %%rax, %%r13\n\t" - "adcq %%rdx, %%r14\n\t" - "# Double\n\t" - "xorq %%r15, %%r15\n\t" - "addq %%r9, %%r9\n\t" - "adcq %%r10, %%r10\n\t" - "adcq %%r11, %%r11\n\t" - "adcq %%r12, %%r12\n\t" - "adcq %%r13, %%r13\n\t" - "adcq %%r14, %%r14\n\t" - "adcq $0, %%r15\n\t" - "# A[0] * A[0]\n\t" - "movq 0(%[a]), %%rax\n\t" - "mulq %%rax\n\t" - "movq %%rax, %%r8\n\t" - "movq %%rdx, %[mp]\n\t" - "# A[1] * A[1]\n\t" - "movq 8(%[a]), %%rax\n\t" - "mulq %%rax\n\t" - "addq %[mp], %%r9\n\t" - "adcq %%rax, %%r10\n\t" - "adcq $0, %%rdx\n\t" - "movq %%rdx, %[mp]\n\t" - "# A[2] * A[2]\n\t" - "movq 16(%[a]), %%rax\n\t" - "mulq %%rax\n\t" - "addq %[mp], %%r11\n\t" - "adcq %%rax, %%r12\n\t" - "adcq $0, %%rdx\n\t" - "movq %%rdx, %[mp]\n\t" - "# A[3] * A[3]\n\t" - "movq 24(%[a]), %%rax\n\t" - "mulq %%rax\n\t" - "addq %%rax, %%r14\n\t" - "adcq %%rdx, %%r15\n\t" - "addq %[mp], %%r13\n\t" - "adcq $0, %%r14\n\t" - "adcq $0, %%r15\n\t" - "# Start Reduction\n\t" - "movq %%r8, %%rax\n\t" - "movq %%r9, %[a]\n\t" - "movq %%r10, %[mp]\n\t" - "movq %%r11, %%rdx\n\t" - "# mu = a[0]-a[3] + a[0]-a[2] << 32 << 64 + (a[0] * 2) << 192\n\t" - "# - a[0] << 32 << 192\n\t" - "# + (a[0] * 2) << 192\n\t" - "addq %%r8, %%rdx\n\t" - "addq %%r8, %%rdx\n\t" - "# a[0]-a[2] << 32\n\t" - "shlq $32, %%r8\n\t" - "shldq $32, %[a], %%r10\n\t" - "shldq $32, %%rax, %%r9\n\t" - "# - a[0] << 32 << 192\n\t" - "subq %%r8, %%rdx\n\t" - "# + a[0]-a[2] << 32 << 64\n\t" - "addq %%r8, %[a]\n\t" - "adcq %%r9, %[mp]\n\t" - "adcq %%r10, %%rdx\n\t" - "# a += (mu << 256) - (mu << 224) + (mu << 192) + (mu << 96) - mu\n\t" - "# a += mu << 256\n\t" - "xorq %%r8, %%r8\n\t" - "addq %%rax, %%r12\n\t" - "adcq %[a], %%r13\n\t" - "adcq %[mp], %%r14\n\t" - "adcq %%rdx, %%r15\n\t" - "sbbq $0, %%r8\n\t" - "# a += mu << 192\n\t" - "addq %%rax, %%r11\n\t" - "adcq %[a], %%r12\n\t" - "adcq %[mp], %%r13\n\t" - "adcq %%rdx, %%r14\n\t" - "adcq $0, %%r15\n\t" - "sbbq $0, %%r8\n\t" - "# mu <<= 32\n\t" - "movq %%rdx, %[m]\n\t" - "shldq $32, %[mp], %%rdx\n\t" - "shldq $32, %[a], %[mp]\n\t" - "shldq $32, %%rax, %[a]\n\t" - "shlq $32, %%rax\n\t" - "shrq $32, %[m]\n\t" - "# a += (mu << 32) << 64\n\t" - "addq %[mp], %%r11\n\t" - "adcq %%rdx, %%r12\n\t" - "adcq %[m], %%r13\n\t" - "adcq $0, %%r14\n\t" - "adcq $0, %%r15\n\t" - "sbbq $0, %%r8\n\t" - "# a -= (mu << 32) << 192\n\t" - "subq %%rax, %%r11\n\t" - "movq $0xffffffff, %%rax\n\t" - "sbbq %[a], %%r12\n\t" - "movq $0xffffffff00000001, %[a]\n\t" - "sbbq %[mp], %%r13\n\t" - "sbbq %%rdx, %%r14\n\t" - "sbbq %[m], %%r15\n\t" - "adcq $0, %%r8\n\t" - "# mask m and sub from result if overflow\n\t" - "# m[0] = -1 & mask = mask\n\t" - "andq %%r8, %%rax\n\t" - "# m[2] = 0 & mask = 0\n\t" - "andq %%r8, %[a]\n\t" - "subq %%r8, %%r12\n\t" - "sbbq %%rax, %%r13\n\t" - "sbbq $0, %%r14\n\t" - "sbbq %[a], %%r15\n\t" - "movq %%r12, 0(%[r])\n\t" - "movq %%r13, 8(%[r])\n\t" - "movq %%r14, 16(%[r])\n\t" - "movq %%r15, 24(%[r])\n\t" - : [m] "+r" (m), [a] "+r" (a), [mp] "+r" (mp) - : [r] "r" (r) - : "memory", "rax", "rdx", "r8", "r9", "r10", "r11", - "r12", "r13", "r14", "r15" - ); -} - +extern void sp_256_mont_reduce_4(sp_digit* a, sp_digit* m, sp_digit mp); +extern void sp_256_mont_mul_4(sp_digit* r, sp_digit* a, sp_digit* b, sp_digit* m, sp_digit mp); +extern void sp_256_mont_sqr_4(sp_digit* r, sp_digit* a, sp_digit* m, sp_digit mp); #ifndef WOLFSSL_SP_SMALL /* Square the Montgomery form number a number of times. (r = a ^ n mod m) * @@ -28467,218 +4108,11 @@ static void sp_256_map_4(sp_point* r, sp_point* p, sp_digit* t) } -/* Add two Montgomery form numbers (r = a + b % m). - * - * r Result of addition. - * a First number to add in Montogmery form. - * b Second number to add in Montogmery form. - * m Modulus (prime). - */ -static void sp_256_mont_add_4(sp_digit* r, sp_digit* a, sp_digit* b, - sp_digit* m) -{ - __asm__ __volatile__ ( - "movq 0(%[a]), %%rax\n\t" - "movq 8(%[a]), %%rcx\n\t" - "movq 16(%[a]), %%rdx\n\t" - "movq 24(%[a]), %%r10\n\t" - "xorq %%r11, %%r11\n\t" - "addq 0(%[b]), %%rax\n\t" - "adcq 8(%[b]), %%rcx\n\t" - "movq $0xffffffff, %%r8\n\t" - "adcq 16(%[b]), %%rdx\n\t" - "adcq 24(%[b]), %%r10\n\t" - "movq $0xffffffff00000001, %%r9\n\t" - "sbbq $0, %%r11\n\t" - "andq %%r11, %%r8\n\t" - "andq %%r11, %%r9\n\t" - "subq %%r11, %%rax\n\t" - "sbbq %%r8, %%rcx\n\t" - "movq %%rax, 0(%[r])\n\t" - "sbbq $0, %%rdx\n\t" - "movq %%rcx, 8(%[r])\n\t" - "sbbq %%r9, %%r10\n\t" - "movq %%rdx, 16(%[r])\n\t" - "movq %%r10, 24(%[r])\n\t" - : - : [r] "r" (r), [a] "r" (a), [b] "r" (b), [m] "r" (m) - : "memory", "rax", "rcx", "rdx", "r8", "r9", "r10", "r11" - ); -} - -/* Double a Montgomery form number (r = a + a % m). - * - * r Result of doubling. - * a Number to double in Montogmery form. - * m Modulus (prime). - */ -static void sp_256_mont_dbl_4(sp_digit* r, sp_digit* a, sp_digit* m) -{ - __asm__ __volatile__ ( - "movq (%[a]), %%rax\n\t" - "movq 8(%[a]), %%rcx\n\t" - "movq 16(%[a]), %%rdx\n\t" - "movq 24(%[a]), %%r10\n\t" - "xorq %%r11, %%r11\n\t" - "addq %%rax, %%rax\n\t" - "adcq %%rcx, %%rcx\n\t" - "movq $0xffffffff, %%r8\n\t" - "adcq %%rdx, %%rdx\n\t" - "movq $0xffffffff00000001, %%r9\n\t" - "adcq %%r10, %%r10\n\t" - "sbbq $0, %%r11\n\t" - "andq %%r11, %%r8\n\t" - "andq %%r11, %%r9\n\t" - "subq %%r11, %%rax\n\t" - "sbbq %%r8, %%rcx\n\t" - "movq %%rax, 0(%[r])\n\t" - "sbbq $0, %%rdx\n\t" - "movq %%rcx, 8(%[r])\n\t" - "sbbq %%r9, %%r10\n\t" - "movq %%rdx, 16(%[r])\n\t" - "movq %%r10, 24(%[r])\n\t" - : - : [r] "r" (r), [a] "r" (a) - : "memory", "rax", "rcx", "rdx", "r8", "r9", "r10", "r11" - ); - - (void)m; -} - -/* Triple a Montgomery form number (r = a + a + a % m). - * - * r Result of Tripling. - * a Number to triple in Montogmery form. - * m Modulus (prime). - */ -static void sp_256_mont_tpl_4(sp_digit* r, sp_digit* a, sp_digit* m) -{ - __asm__ __volatile__ ( - "movq (%[a]), %%rax\n\t" - "movq 8(%[a]), %%rcx\n\t" - "movq 16(%[a]), %%rdx\n\t" - "movq 24(%[a]), %%r10\n\t" - "xorq %%r11, %%r11\n\t" - "addq %%rax, %%rax\n\t" - "adcq %%rcx, %%rcx\n\t" - "movq $0xffffffff, %%r8\n\t" - "adcq %%rdx, %%rdx\n\t" - "adcq %%r10, %%r10\n\t" - "movq $0xffffffff00000001, %%r9\n\t" - "sbbq $0, %%r11\n\t" - "andq %%r11, %%r8\n\t" - "andq %%r11, %%r9\n\t" - "subq %%r11, %%rax\n\t" - "sbbq %%r8, %%rcx\n\t" - "sbbq $0, %%rdx\n\t" - "sbbq %%r9, %%r10\n\t" - "xorq %%r11, %%r11\n\t" - "addq (%[a]), %%rax\n\t" - "adcq 8(%[a]), %%rcx\n\t" - "movq $0xffffffff, %%r8\n\t" - "adcq 16(%[a]), %%rdx\n\t" - "adcq 24(%[a]), %%r10\n\t" - "movq $0xffffffff00000001, %%r9\n\t" - "sbbq $0, %%r11\n\t" - "andq %%r11, %%r8\n\t" - "andq %%r11, %%r9\n\t" - "subq %%r11, %%rax\n\t" - "sbbq %%r8, %%rcx\n\t" - "sbbq $0, %%rdx\n\t" - "sbbq %%r9, %%r10\n\t" - "movq %%rax, 0(%[r])\n\t" - "movq %%rcx, 8(%[r])\n\t" - "movq %%rdx, 16(%[r])\n\t" - "movq %%r10, 24(%[r])\n\t" - : - : [r] "r" (r), [a] "r" (a) - : "memory", "rax", "rcx", "rdx", "r8", "r9", "r10", "r11" - ); - - (void)m; -} - -/* Subtract two Montgomery form numbers (r = a - b % m). - * - * r Result of subtration. - * a Number to subtract from in Montogmery form. - * b Number to subtract with in Montogmery form. - * m Modulus (prime). - */ -static void sp_256_mont_sub_4(sp_digit* r, sp_digit* a, sp_digit* b, - sp_digit* m) -{ - __asm__ __volatile__ ( - "movq 0(%[a]), %%rax\n\t" - "movq 8(%[a]), %%rcx\n\t" - "movq 16(%[a]), %%rdx\n\t" - "movq 24(%[a]), %%r10\n\t" - "xorq %%r11, %%r11\n\t" - "subq 0(%[b]), %%rax\n\t" - "sbbq 8(%[b]), %%rcx\n\t" - "movq $0xffffffff, %%r8\n\t" - "sbbq 16(%[b]), %%rdx\n\t" - "sbbq 24(%[b]), %%r10\n\t" - "movq $0xffffffff00000001, %%r9\n\t" - "sbbq $0, %%r11\n\t" - "andq %%r11, %%r8\n\t" - "andq %%r11, %%r9\n\t" - "addq %%r11, %%rax\n\t" - "adcq %%r8, %%rcx\n\t" - "movq %%rax, 0(%[r])\n\t" - "adcq $0, %%rdx\n\t" - "movq %%rcx, 8(%[r])\n\t" - "adcq %%r9, %%r10\n\t" - "movq %%rdx, 16(%[r])\n\t" - "movq %%r10, 24(%[r])\n\t" - : - : [r] "r" (r), [a] "r" (a), [b] "r" (b), [m] "r" (m) - : "memory", "rax", "rcx", "rdx", "r8", "r9", "r10", "r11" - ); -} - -/* Divide the number by 2 mod the modulus (prime). (r = a / 2 % m) - * - * r Result of division by 2. - * a Number to divide. - * m Modulus (prime). - */ -SP_NOINLINE static void sp_256_div2_4(sp_digit* r, sp_digit* a, sp_digit* m) -{ - __asm__ __volatile__ ( - "movq 0(%[a]), %%rax\n\t" - "movq 8(%[a]), %%rdx\n\t" - "movq 16(%[a]), %%rcx\n\t" - "movq 24(%[a]), %%r10\n\t" - "movq $0xffffffff, %%r8\n\t" - "movq $0xffffffff00000001, %%r9\n\t" - "xorq %%r12, %%r12\n\t" - "movq %%rax, %%r11\n\t" - "andq $1, %%r11\n\t" - "subq %%r11, %%r12\n\t" - "andq %%r12, %%r8\n\t" - "andq %%r12, %%r9\n\t" - "xorq %%r11, %%r11\n\t" - "addq %%r12, %%rax\n\t" - "adcq %%r8, %%rdx\n\t" - "adcq $0, %%rcx\n\t" - "adcq %%r9, %%r10\n\t" - "adcq $0, %%r11\n\t" - "shrdq $1, %%rdx, %%rax\n\t" - "shrdq $1, %%rcx, %%rdx\n\t" - "shrdq $1, %%r10, %%rcx\n\t" - "shrdq $1, %%r11, %%r10\n\t" - "movq %%rax, 0(%[r])\n\t" - "movq %%rdx, 8(%[r])\n\t" - "movq %%rcx, 16(%[r])\n\t" - "movq %%r10, 24(%[r])\n\t" - : - : [r] "r" (r), [a] "r" (a), [m] "r" (m) - : "memory", "rax", "rdx", "rcx", "r8", "r9", "r10", "r11", "r12" - ); - -} - +extern void sp_256_mont_add_4(sp_digit* r, sp_digit* a, sp_digit* b, sp_digit* m); +extern void sp_256_mont_dbl_4(sp_digit* r, sp_digit* a, sp_digit* m); +extern void sp_256_mont_tpl_4(sp_digit* r, sp_digit* a, sp_digit* m); +extern void sp_256_mont_sub_4(sp_digit* r, sp_digit* a, sp_digit* b, sp_digit* m); +extern void sp_256_div2_4(sp_digit* r, const sp_digit* a, const sp_digit* m); /* Double the Montgomery form projective point p. * * r Result of doubling point. @@ -29255,318 +4689,8 @@ static int sp_256_ecc_mulmod_win_add_sub_4(sp_point* r, sp_point* g, } #ifdef HAVE_INTEL_AVX2 -/* Multiply two Montogmery form numbers mod the modulus (prime). - * (r = a * b mod m) - * - * r Result of multiplication. - * a First number to multiply in Montogmery form. - * b Second number to multiply in Montogmery form. - * m Modulus (prime). - * mp Montogmery mulitplier. - */ -SP_NOINLINE static void sp_256_mont_mul_avx2_4(sp_digit* r, sp_digit* a, - sp_digit* b, sp_digit* m, sp_digit mp) -{ - (void)mp; - - __asm__ __volatile__ ( - "# A[0] * B[0]\n\t" - "movq 0(%[b]), %%rdx\n\t" - "mulxq 0(%[a]), %%r8, %%r9\n\t" - "# A[2] * B[0]\n\t" - "mulxq 16(%[a]), %%r10, %%r11\n\t" - "# A[1] * B[0]\n\t" - "mulxq 8(%[a]), %%rax, %[m]\n\t" - "xorq %%r15, %%r15\n\t" - "adcxq %%rax, %%r9\n\t" - "# A[1] * B[3]\n\t" - "movq 24(%[b]), %%rdx\n\t" - "mulxq 8(%[a]), %%r12, %%r13\n\t" - "adcxq %[m], %%r10\n\t" - "# A[0] * B[1]\n\t" - "movq 8(%[b]), %%rdx\n\t" - "mulxq 0(%[a]), %%rax, %[m]\n\t" - "adoxq %%rax, %%r9\n\t" - "# A[2] * B[1]\n\t" - "mulxq 16(%[a]), %%rax, %%r14\n\t" - "adoxq %[m], %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "# A[1] * B[2]\n\t" - "movq 16(%[b]), %%rdx\n\t" - "mulxq 8(%[a]), %%rax, %[m]\n\t" - "adcxq %%r14, %%r12\n\t" - "adoxq %%rax, %%r11\n\t" - "adcxq %%r15, %%r13\n\t" - "adoxq %[m], %%r12\n\t" - "# A[0] * B[2]\n\t" - "mulxq 0(%[a]), %%rax, %[m]\n\t" - "adoxq %%r15, %%r13\n\t" - "xorq %%r14, %%r14\n\t" - "adcxq %%rax, %%r10\n\t" - "# A[1] * B[1]\n\t" - "movq 8(%[b]), %%rdx\n\t" - "mulxq 8(%[a]), %%rdx, %%rax\n\t" - "adcxq %[m], %%r11\n\t" - "adoxq %%rdx, %%r10\n\t" - "# A[3] * B[1]\n\t" - "movq 8(%[b]), %%rdx\n\t" - "adoxq %%rax, %%r11\n\t" - "mulxq 24(%[a]), %%rax, %[m]\n\t" - "adcxq %%rax, %%r12\n\t" - "# A[2] * B[2]\n\t" - "movq 16(%[b]), %%rdx\n\t" - "mulxq 16(%[a]), %%rdx, %%rax\n\t" - "adcxq %[m], %%r13\n\t" - "adoxq %%rdx, %%r12\n\t" - "# A[3] * B[3]\n\t" - "movq 24(%[b]), %%rdx\n\t" - "adoxq %%rax, %%r13\n\t" - "mulxq 24(%[a]), %%rax, %[m]\n\t" - "adoxq %%r15, %%r14\n\t" - "adcxq %%rax, %%r14\n\t" - "# A[0] * B[3]\n\t" - "mulxq 0(%[a]), %%rdx, %%rax\n\t" - "adcxq %[m], %%r15\n\t" - "xorq %[m], %[m]\n\t" - "adcxq %%rdx, %%r11\n\t" - "# A[3] * B[0]\n\t" - "movq 0(%[b]), %%rdx\n\t" - "adcxq %%rax, %%r12\n\t" - "mulxq 24(%[a]), %%rdx, %%rax\n\t" - "adoxq %%rdx, %%r11\n\t" - "adoxq %%rax, %%r12\n\t" - "# A[2] * B[3]\n\t" - "movq 24(%[b]), %%rdx\n\t" - "mulxq 16(%[a]), %%rdx, %%rax\n\t" - "adcxq %%rdx, %%r13\n\t" - "# A[3] * B[2]\n\t" - "movq 16(%[b]), %%rdx\n\t" - "adcxq %%rax, %%r14\n\t" - "mulxq 24(%[a]), %%rax, %%rdx\n\t" - "adcxq %[m], %%r15\n\t" - "adoxq %%rax, %%r13\n\t" - "adoxq %%rdx, %%r14\n\t" - "adoxq %[m], %%r15\n\t" - "# Start Reduction\n\t" - "movq %%r8, %%rax\n\t" - "movq %%r9, %[a]\n\t" - "movq %%r10, %[b]\n\t" - "movq %%r11, %%rdx\n\t" - "# mu = a[0]-a[3] + a[0]-a[2] << 32 << 64 + (a[0] * 2) << 192\n\t" - "# - a[0] << 32 << 192\n\t" - "# + (a[0] * 2) << 192\n\t" - "addq %%r8, %%rdx\n\t" - "addq %%r8, %%rdx\n\t" - "# a[0]-a[2] << 32\n\t" - "shlq $32, %%r8\n\t" - "shldq $32, %[a], %%r10\n\t" - "shldq $32, %%rax, %%r9\n\t" - "# - a[0] << 32 << 192\n\t" - "subq %%r8, %%rdx\n\t" - "# + a[0]-a[2] << 32 << 64\n\t" - "addq %%r8, %[a]\n\t" - "adcq %%r9, %[b]\n\t" - "adcq %%r10, %%rdx\n\t" - "# a += (mu << 256) - (mu << 224) + (mu << 192) + (mu << 96) - mu\n\t" - "# a += mu << 256\n\t" - "xorq %%r8, %%r8\n\t" - "addq %%rax, %%r12\n\t" - "adcq %[a], %%r13\n\t" - "adcq %[b], %%r14\n\t" - "adcq %%rdx, %%r15\n\t" - "sbbq $0, %%r8\n\t" - "# a += mu << 192\n\t" - "addq %%rax, %%r11\n\t" - "adcq %[a], %%r12\n\t" - "adcq %[b], %%r13\n\t" - "adcq %%rdx, %%r14\n\t" - "adcq $0, %%r15\n\t" - "sbbq $0, %%r8\n\t" - "# mu <<= 32\n\t" - "movq %%rdx, %[m]\n\t" - "shldq $32, %[b], %%rdx\n\t" - "shldq $32, %[a], %[b]\n\t" - "shldq $32, %%rax, %[a]\n\t" - "shlq $32, %%rax\n\t" - "shrq $32, %[m]\n\t" - "# a += (mu << 32) << 64\n\t" - "addq %[b], %%r11\n\t" - "adcq %%rdx, %%r12\n\t" - "adcq %[m], %%r13\n\t" - "adcq $0, %%r14\n\t" - "adcq $0, %%r15\n\t" - "sbbq $0, %%r8\n\t" - "# a -= (mu << 32) << 192\n\t" - "subq %%rax, %%r11\n\t" - "movq $0xffffffff, %%rax\n\t" - "sbbq %[a], %%r12\n\t" - "movq $0xffffffff00000001, %[a]\n\t" - "sbbq %[b], %%r13\n\t" - "sbbq %%rdx, %%r14\n\t" - "sbbq %[m], %%r15\n\t" - "adcq $0, %%r8\n\t" - "# mask m and sub from result if overflow\n\t" - "# m[0] = -1 & mask = mask\n\t" - "andq %%r8, %%rax\n\t" - "# m[2] = 0 & mask = 0\n\t" - "andq %%r8, %[a]\n\t" - "subq %%r8, %%r12\n\t" - "sbbq %%rax, %%r13\n\t" - "sbbq $0, %%r14\n\t" - "sbbq %[a], %%r15\n\t" - "movq %%r12, 0(%[r])\n\t" - "movq %%r13, 8(%[r])\n\t" - "movq %%r14, 16(%[r])\n\t" - "movq %%r15, 24(%[r])\n\t" - : [m] "+r" (m), [a] "+r" (a), [b] "+r" (b) - : [r] "r" (r) - : "memory", "rax", "rdx", "r8", "r9", "r10", "r11", - "r12", "r13", "r14", "r15" - ); -} - -/* Square the Montgomery form number mod the modulus (prime). (r = a * a mod m) - * - * r Result of squaring. - * a Number to square in Montogmery form. - * m Modulus (prime). - * mp Montogmery mulitplier. - */ -SP_NOINLINE static void sp_256_mont_sqr_avx2_4(sp_digit* r, sp_digit* a, - sp_digit* m, sp_digit mp) -{ - __asm__ __volatile__ ( - "# A[0] * A[1]\n\t" - "movq 0(%[a]), %%rdx\n\t" - "mulxq 8(%[a]), %%r9, %%r10\n\t" - "# A[0] * A[3]\n\t" - "mulxq 24(%[a]), %%r11, %%r12\n\t" - "# A[2] * A[1]\n\t" - "movq 16(%[a]), %%rdx\n\t" - "mulxq 8(%[a]), %[mp], %[m]\n\t" - "xorq %%r15, %%r15\n\t" - "adoxq %[mp], %%r11\n\t" - "# A[2] * A[3]\n\t" - "mulxq 24(%[a]), %%r13, %%r14\n\t" - "adoxq %[m], %%r12\n\t" - "# A[2] * A[0]\n\t" - "mulxq 0(%[a]), %[mp], %[m]\n\t" - "adoxq %%r15, %%r13\n\t" - "adcxq %[mp], %%r10\n\t" - "adoxq %%r15, %%r14\n\t" - "# A[1] * A[3]\n\t" - "movq 8(%[a]), %%rdx\n\t" - "mulxq 24(%[a]), %%rax, %%r8\n\t" - "adcxq %[m], %%r11\n\t" - "adcxq %%rax, %%r12\n\t" - "adcxq %%r8, %%r13\n\t" - "adcxq %%r15, %%r14\n\t" - "# Double with Carry Flag\n\t" - "xorq %%r15, %%r15\n\t" - "# A[0] * A[0]\n\t" - "movq 0(%[a]), %%rdx\n\t" - "mulxq %%rdx, %%r8, %%rax\n\t" - "adcxq %%r9, %%r9\n\t" - "# A[1] * A[1]\n\t" - "movq 8(%[a]), %%rdx\n\t" - "mulxq %%rdx, %[mp], %[m]\n\t" - "adcxq %%r10, %%r10\n\t" - "adoxq %%rax, %%r9\n\t" - "adcxq %%r11, %%r11\n\t" - "adoxq %[mp], %%r10\n\t" - "# A[2] * A[2]\n\t" - "movq 16(%[a]), %%rdx\n\t" - "mulxq %%rdx, %%rax, %[mp]\n\t" - "adcxq %%r12, %%r12\n\t" - "adoxq %[m], %%r11\n\t" - "adcxq %%r13, %%r13\n\t" - "adoxq %%rax, %%r12\n\t" - "# A[3] * A[3]\n\t" - "movq 24(%[a]), %%rdx\n\t" - "mulxq %%rdx, %%rax, %[m]\n\t" - "adcxq %%r14, %%r14\n\t" - "adoxq %[mp], %%r13\n\t" - "adcxq %%r15, %%r15\n\t" - "adoxq %%rax, %%r14\n\t" - "adoxq %[m], %%r15\n\t" - "# Start Reduction\n\t" - "movq %%r8, %%rax\n\t" - "movq %%r9, %[a]\n\t" - "movq %%r10, %[mp]\n\t" - "movq %%r11, %%rdx\n\t" - "# mu = a[0]-a[3] + a[0]-a[2] << 32 << 64 + (a[0] * 2) << 192\n\t" - "# - a[0] << 32 << 192\n\t" - "# + (a[0] * 2) << 192\n\t" - "addq %%r8, %%rdx\n\t" - "addq %%r8, %%rdx\n\t" - "# a[0]-a[2] << 32\n\t" - "shlq $32, %%r8\n\t" - "shldq $32, %[a], %%r10\n\t" - "shldq $32, %%rax, %%r9\n\t" - "# - a[0] << 32 << 192\n\t" - "subq %%r8, %%rdx\n\t" - "# + a[0]-a[2] << 32 << 64\n\t" - "addq %%r8, %[a]\n\t" - "adcq %%r9, %[mp]\n\t" - "adcq %%r10, %%rdx\n\t" - "# a += (mu << 256) - (mu << 224) + (mu << 192) + (mu << 96) - mu\n\t" - "# a += mu << 256\n\t" - "xorq %%r8, %%r8\n\t" - "addq %%rax, %%r12\n\t" - "adcq %[a], %%r13\n\t" - "adcq %[mp], %%r14\n\t" - "adcq %%rdx, %%r15\n\t" - "sbbq $0, %%r8\n\t" - "# a += mu << 192\n\t" - "addq %%rax, %%r11\n\t" - "adcq %[a], %%r12\n\t" - "adcq %[mp], %%r13\n\t" - "adcq %%rdx, %%r14\n\t" - "adcq $0, %%r15\n\t" - "sbbq $0, %%r8\n\t" - "# mu <<= 32\n\t" - "movq %%rdx, %[m]\n\t" - "shldq $32, %[mp], %%rdx\n\t" - "shldq $32, %[a], %[mp]\n\t" - "shldq $32, %%rax, %[a]\n\t" - "shlq $32, %%rax\n\t" - "shrq $32, %[m]\n\t" - "# a += (mu << 32) << 64\n\t" - "addq %[mp], %%r11\n\t" - "adcq %%rdx, %%r12\n\t" - "adcq %[m], %%r13\n\t" - "adcq $0, %%r14\n\t" - "adcq $0, %%r15\n\t" - "sbbq $0, %%r8\n\t" - "# a -= (mu << 32) << 192\n\t" - "subq %%rax, %%r11\n\t" - "movq $0xffffffff, %%rax\n\t" - "sbbq %[a], %%r12\n\t" - "movq $0xffffffff00000001, %[a]\n\t" - "sbbq %[mp], %%r13\n\t" - "sbbq %%rdx, %%r14\n\t" - "sbbq %[m], %%r15\n\t" - "adcq $0, %%r8\n\t" - "# mask m and sub from result if overflow\n\t" - "# m[0] = -1 & mask = mask\n\t" - "andq %%r8, %%rax\n\t" - "# m[2] = 0 & mask = 0\n\t" - "andq %%r8, %[a]\n\t" - "subq %%r8, %%r12\n\t" - "sbbq %%rax, %%r13\n\t" - "sbbq $0, %%r14\n\t" - "sbbq %[a], %%r15\n\t" - "movq %%r12, 0(%[r])\n\t" - "movq %%r13, 8(%[r])\n\t" - "movq %%r14, 16(%[r])\n\t" - "movq %%r15, 24(%[r])\n\t" - : [m] "+r" (m), [a] "+r" (a), [mp] "+r" (mp) - : [r] "r" (r) - : "memory", "rax", "rdx", "r8", "r9", "r10", "r11", - "r12", "r13", "r14", "r15" - ); -} - +extern void sp_256_mont_mul_avx2_4(sp_digit* r, sp_digit* a, sp_digit* b, sp_digit* m, sp_digit mp); +extern void sp_256_mont_sqr_avx2_4(sp_digit* r, sp_digit* a, sp_digit* m, sp_digit mp); #ifndef WOLFSSL_SP_SMALL /* Square the Montgomery form number a number of times. (r = a ^ n mod m) * @@ -44400,7 +19524,7 @@ int sp_ecc_mulmod_base_256(mp_int* km, ecc_point* r, int map, void* heap) return err; } -#if defined(WOLFSSL_VALIDATE_ECC_KEYGEN) || defined(HAVE_ECC_SIGN) || defined(HAVE_ECC_VERIFY) +#if defined(WOLFSSL_VALIDATE_ECC_KEYGEN) || defined(HAVE_ECC_SIGN) /* Returns 1 if the number of zero. * Implementation is constant time. * @@ -44412,24 +19536,8 @@ static int sp_256_iszero_4(const sp_digit* a) return (a[0] | a[1] | a[2] | a[3]) == 0; } -#endif /* WOLFSSL_VALIDATE_ECC_KEYGEN || HAVE_ECC_SIGN || HAVE_ECC_VERIFY */ -/* Add 1 to a. (a = a + 1) - * - * a A single precision integer. - */ -static void sp_256_add_one_4(sp_digit* a) -{ - __asm__ __volatile__ ( - "addq $1, (%[a])\n\t" - "adcq $0, 8(%[a])\n\t" - "adcq $0, 16(%[a])\n\t" - "adcq $0, 24(%[a])\n\t" - : - : [a] "r" (a) - : "memory" - ); -} - +#endif /* WOLFSSL_VALIDATE_ECC_KEYGEN || HAVE_ECC_SIGN */ +extern void sp_256_add_one_4(sp_digit* a); /* Read big endian unsigned byte aray into r. * * r A single precision integer. @@ -44674,399 +19782,18 @@ int sp_ecc_secret_gen_256(mp_int* priv, ecc_point* pub, byte* out, #endif /* HAVE_ECC_DHE */ #if defined(HAVE_ECC_SIGN) || defined(HAVE_ECC_VERIFY) -/* Add b to a into r. (r = a + b) - * - * r A single precision integer. - * a A single precision integer. - * b A single precision integer. - */ -SP_NOINLINE static sp_digit sp_256_add_4(sp_digit* r, const sp_digit* a, - const sp_digit* b) -{ - sp_digit c = 0; - - __asm__ __volatile__ ( - "movq (%[a]), %%rax\n\t" - "addq (%[b]), %%rax\n\t" - "movq %%rax, (%[r])\n\t" - "movq 8(%[a]), %%rax\n\t" - "adcq 8(%[b]), %%rax\n\t" - "movq %%rax, 8(%[r])\n\t" - "movq 16(%[a]), %%rax\n\t" - "adcq 16(%[b]), %%rax\n\t" - "movq %%rax, 16(%[r])\n\t" - "movq 24(%[a]), %%rax\n\t" - "adcq 24(%[b]), %%rax\n\t" - "movq %%rax, 24(%[r])\n\t" - "adcq $0, %[c]\n\t" - : [c] "+r" (c) - : [r] "r" (r), [a] "r" (a), [b] "r" (b) - : "memory", "rax" - ); - - return c; -} - +extern sp_digit sp_256_add_4(sp_digit* r, const sp_digit* a, const sp_digit* b); #endif #if defined(HAVE_ECC_SIGN) || defined(HAVE_ECC_VERIFY) -/* Multiply a and b into r. (r = a * b) - * - * r A single precision integer. - * a A single precision integer. - * b A single precision integer. - */ -SP_NOINLINE static void sp_256_mul_4(sp_digit* r, const sp_digit* a, - const sp_digit* b) -{ - sp_digit tmp[4]; - - __asm__ __volatile__ ( - "# A[0] * B[0]\n\t" - "movq 0(%[b]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "movq %%rax, (%[tmp])\n\t" - "movq %%rdx, %%rcx\n\t" - "# A[0] * B[1]\n\t" - "movq 8(%[b]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%rbx, %%rbx\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[1] * B[0]\n\t" - "movq 0(%[b]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "movq %%rcx, 8(%[tmp])\n\t" - "# A[0] * B[2]\n\t" - "movq 16(%[b]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%rcx, %%rcx\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[1] * B[1]\n\t" - "movq 8(%[b]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[2] * B[0]\n\t" - "movq 0(%[b]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "movq %%r8, 16(%[tmp])\n\t" - "# A[0] * B[3]\n\t" - "movq 24(%[b]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[1] * B[2]\n\t" - "movq 16(%[b]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[2] * B[1]\n\t" - "movq 8(%[b]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[3] * B[0]\n\t" - "movq 0(%[b]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "movq %%rbx, 24(%[tmp])\n\t" - "# A[1] * B[3]\n\t" - "movq 24(%[b]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "xorq %%rbx, %%rbx\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[2] * B[2]\n\t" - "movq 16(%[b]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[3] * B[1]\n\t" - "movq 8(%[b]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "movq %%rcx, 32(%[r])\n\t" - "# A[2] * B[3]\n\t" - "movq 24(%[b]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "xorq %%rcx, %%rcx\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[3] * B[2]\n\t" - "movq 16(%[b]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "movq %%r8, 40(%[r])\n\t" - "# A[3] * B[3]\n\t" - "movq 24(%[b]), %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "movq %%rbx, 48(%[r])\n\t" - "movq %%rcx, 56(%[r])\n\t" - : - : [r] "r" (r), [a] "r" (a), [b] "r" (b), [tmp] "r" (tmp) - : "memory", "rax", "rdx", "rbx", "rcx", "r8" - ); - - XMEMCPY(r, tmp, sizeof(tmp)); -} - +extern void sp_256_mul_4(sp_digit* r, const sp_digit* a, const sp_digit* b); #ifdef HAVE_INTEL_AVX2 -/* Multiply a and b into r. (r = a * b) - * - * r Result of multiplication. - * a First number to multiply. - * b Second number to multiply. - */ -SP_NOINLINE static void sp_256_mul_avx2_4(sp_digit* r, const sp_digit* a, - const sp_digit* b) -{ - __asm__ __volatile__ ( - "# A[0] * B[0]\n\t" - "movq 0(%[b]), %%rdx\n\t" - "mulxq 0(%[a]), %%r8, %%r9\n\t" - "# A[2] * B[0]\n\t" - "mulxq 16(%[a]), %%r10, %%r11\n\t" - "# A[1] * B[0]\n\t" - "mulxq 8(%[a]), %%rax, %%rcx\n\t" - "xorq %%r15, %%r15\n\t" - "adcxq %%rax, %%r9\n\t" - "# A[1] * B[3]\n\t" - "movq 24(%[b]), %%rdx\n\t" - "mulxq 8(%[a]), %%r12, %%r13\n\t" - "adcxq %%rcx, %%r10\n\t" - "# A[0] * B[1]\n\t" - "movq 8(%[b]), %%rdx\n\t" - "mulxq 0(%[a]), %%rax, %%rcx\n\t" - "adoxq %%rax, %%r9\n\t" - "# A[2] * B[1]\n\t" - "mulxq 16(%[a]), %%rax, %%r14\n\t" - "adoxq %%rcx, %%r10\n\t" - "adcxq %%rax, %%r11\n\t" - "# A[1] * B[2]\n\t" - "movq 16(%[b]), %%rdx\n\t" - "mulxq 8(%[a]), %%rax, %%rcx\n\t" - "adcxq %%r14, %%r12\n\t" - "adoxq %%rax, %%r11\n\t" - "adcxq %%r15, %%r13\n\t" - "adoxq %%rcx, %%r12\n\t" - "# A[0] * B[2]\n\t" - "mulxq 0(%[a]), %%rax, %%rcx\n\t" - "adoxq %%r15, %%r13\n\t" - "xorq %%r14, %%r14\n\t" - "adcxq %%rax, %%r10\n\t" - "# A[1] * B[1]\n\t" - "movq 8(%[b]), %%rdx\n\t" - "mulxq 8(%[a]), %%rdx, %%rax\n\t" - "adcxq %%rcx, %%r11\n\t" - "adoxq %%rdx, %%r10\n\t" - "# A[3] * B[1]\n\t" - "movq 8(%[b]), %%rdx\n\t" - "adoxq %%rax, %%r11\n\t" - "mulxq 24(%[a]), %%rax, %%rcx\n\t" - "adcxq %%rax, %%r12\n\t" - "# A[2] * B[2]\n\t" - "movq 16(%[b]), %%rdx\n\t" - "mulxq 16(%[a]), %%rdx, %%rax\n\t" - "adcxq %%rcx, %%r13\n\t" - "adoxq %%rdx, %%r12\n\t" - "# A[3] * B[3]\n\t" - "movq 24(%[b]), %%rdx\n\t" - "adoxq %%rax, %%r13\n\t" - "mulxq 24(%[a]), %%rax, %%rcx\n\t" - "adoxq %%r15, %%r14\n\t" - "adcxq %%rax, %%r14\n\t" - "# A[0] * B[3]\n\t" - "mulxq 0(%[a]), %%rdx, %%rax\n\t" - "adcxq %%rcx, %%r15\n\t" - "xorq %%rcx, %%rcx\n\t" - "adcxq %%rdx, %%r11\n\t" - "# A[3] * B[0]\n\t" - "movq 0(%[b]), %%rdx\n\t" - "adcxq %%rax, %%r12\n\t" - "mulxq 24(%[a]), %%rdx, %%rax\n\t" - "adoxq %%rdx, %%r11\n\t" - "adoxq %%rax, %%r12\n\t" - "# A[2] * B[3]\n\t" - "movq 24(%[b]), %%rdx\n\t" - "mulxq 16(%[a]), %%rdx, %%rax\n\t" - "adcxq %%rdx, %%r13\n\t" - "# A[3] * B[2]\n\t" - "movq 16(%[b]), %%rdx\n\t" - "adcxq %%rax, %%r14\n\t" - "mulxq 24(%[a]), %%rax, %%rdx\n\t" - "adcxq %%rcx, %%r15\n\t" - "adoxq %%rax, %%r13\n\t" - "adoxq %%rdx, %%r14\n\t" - "adoxq %%rcx, %%r15\n\t" - "movq %%r8, 0(%[r])\n\t" - "movq %%r9, 8(%[r])\n\t" - "movq %%r10, 16(%[r])\n\t" - "movq %%r11, 24(%[r])\n\t" - "movq %%r12, 32(%[r])\n\t" - "movq %%r13, 40(%[r])\n\t" - "movq %%r14, 48(%[r])\n\t" - "movq %%r15, 56(%[r])\n\t" - : - : [r] "r" (r), [a] "r" (a), [b] "r" (b) - : "memory", "rax", "rdx", "rcx", "r8", "r9", "r10", "r11", - "r12", "r13", "r14", "r15" - ); -} - +extern void sp_256_mul_avx2_4(sp_digit* r, const sp_digit* a, const sp_digit* b); #endif /* HAVE_INTEL_AVX2 */ #endif #if defined(HAVE_ECC_SIGN) || defined(HAVE_ECC_VERIFY) -/* Sub b from a into a. (a -= b) - * - * a A single precision integer and result. - * b A single precision integer. - */ -SP_NOINLINE static sp_digit sp_256_sub_in_place_4(sp_digit* a, - const sp_digit* b) -{ - sp_digit c = 0; - - __asm__ __volatile__ ( - "movq 0(%[a]), %%r8\n\t" - "movq 8(%[a]), %%r9\n\t" - "movq 0(%[b]), %%rdx\n\t" - "movq 8(%[b]), %%rcx\n\t" - "subq %%rdx, %%r8\n\t" - "movq 16(%[b]), %%rdx\n\t" - "movq %%r8, 0(%[a])\n\t" - "movq 16(%[a]), %%r8\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq 24(%[b]), %%rcx\n\t" - "movq %%r9, 8(%[a])\n\t" - "movq 24(%[a]), %%r9\n\t" - "sbbq %%rdx, %%r8\n\t" - "movq %%r8, 16(%[a])\n\t" - "sbbq %%rcx, %%r9\n\t" - "movq %%r9, 24(%[a])\n\t" - "sbbq $0, %[c]\n\t" - : [c] "+r" (c) - : [a] "r" (a), [b] "r" (b) - : "memory", "rdx", "rcx", "r8", "r9" - ); - - return c; -} - -/* Mul a by digit b into r. (r = a * b) - * - * r A single precision integer. - * a A single precision integer. - * b A single precision digit. - */ -SP_NOINLINE static void sp_256_mul_d_4(sp_digit* r, const sp_digit* a, - const sp_digit b) -{ - __asm__ __volatile__ ( - "# A[0] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%r8, %%r8\n\t" - "mulq (%[a])\n\t" - "movq %%rax, %%rbx\n\t" - "movq %%rdx, %%rcx\n\t" - "movq %%rbx, 0(%[r])\n\t" - "# A[1] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rbx, %%rbx\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "movq %%rcx, 8(%[r])\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%rbx\n\t" - "# A[2] * B\n\t" - "movq %[b], %%rax\n\t" - "xorq %%rcx, %%rcx\n\t" - "mulq 16(%[a])\n\t" - "addq %%rax, %%r8\n\t" - "movq %%r8, 16(%[r])\n\t" - "adcq %%rdx, %%rbx\n\t" - "adcq $0, %%rcx\n\t" - "# A[3] * B\n\t" - "movq %[b], %%rax\n\t" - "mulq 24(%[a])\n\t" - "addq %%rax, %%rbx\n\t" - "adcq %%rdx, %%rcx\n\t" - "movq %%rbx, 24(%[r])\n\t" - "movq %%rcx, 32(%[r])\n\t" - : - : [r] "r" (r), [a] "r" (a), [b] "r" (b) - : "memory", "rax", "rdx", "rbx", "rcx", "r8" - ); -} - -#ifdef HAVE_INTEL_AVX2 -/* Mul a by digit b into r. (r = a * b) - * - * r A single precision integer. - * a A single precision integer. - * b A single precision digit. - */ -SP_NOINLINE static void sp_256_mul_d_avx2_4(sp_digit* r, const sp_digit* a, - const sp_digit b) -{ - __asm__ __volatile__ ( - "# A[0] * B\n\t" - "movq %[b], %%rdx\n\t" - "xorq %%r10, %%r10\n\t" - "mulxq (%[a]), %%r8, %%r9\n\t" - "movq %%r8, 0(%[r])\n\t" - "# A[1] * B\n\t" - "mulxq 8(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "movq %%r9, 8(%[r])\n\t" - "adoxq %%rcx, %%r8\n\t" - "# A[2] * B\n\t" - "mulxq 16(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r9\n\t" - "adcxq %%rax, %%r8\n\t" - "movq %%r8, 16(%[r])\n\t" - "adoxq %%rcx, %%r9\n\t" - "# A[3] * B\n\t" - "mulxq 24(%[a]), %%rax, %%rcx\n\t" - "movq %%r10, %%r8\n\t" - "adcxq %%rax, %%r9\n\t" - "adoxq %%rcx, %%r8\n\t" - "adcxq %%r10, %%r8\n\t" - "movq %%r9, 24(%[r])\n\t" - "movq %%r8, 32(%[r])\n\t" - : - : [r] "r" (r), [a] "r" (a), [b] "r" (b) - : "memory", "rax", "rdx", "rcx", "r8", "r9", "r10" - ); -} -#endif /* HAVE_INTEL_AVX2 */ - +extern sp_digit sp_256_sub_in_place_4(sp_digit* a, const sp_digit* b); +extern void sp_256_mul_d_4(sp_digit* r, const sp_digit* a, const sp_digit b); +extern void sp_256_mul_d_avx2_4(sp_digit* r, const sp_digit* a, const sp_digit b); /* Divide the double width number (d1|d0) by the dividend. (d1|d0 / div) * * d1 The high order half of the number to divide. @@ -45074,23 +19801,18 @@ SP_NOINLINE static void sp_256_mul_d_avx2_4(sp_digit* r, const sp_digit* a, * div The dividend. * returns the result of the division. */ -static sp_digit div_256_word_4(sp_digit d1, sp_digit d0, sp_digit div) +static WC_INLINE sp_digit div_256_word_4(sp_digit d1, sp_digit d0, + sp_digit div) { - sp_digit r; - + register sp_digit r asm("rax"); __asm__ __volatile__ ( - "movq %[d0], %%rax\n\t" - "movq %[d1], %%rdx\n\t" - "divq %[div]\n\t" - "movq %%rax, %[r]\n\t" - : [r] "=r" (r) - : [d1] "r" (d1), [d0] "r" (d0), [div] "r" (div) - : "rax", "rdx" + "divq %3" + : "=a" (r) + : "d" (d1), "a" (d0), "r" (div) + : ); - return r; } - /* AND m into each word of a and store in r. * * r A single precision integer. @@ -45172,113 +19894,7 @@ static WC_INLINE int sp_256_mod_4(sp_digit* r, sp_digit* a, sp_digit* m) #endif #if defined(HAVE_ECC_SIGN) || defined(HAVE_ECC_VERIFY) -/* Square a and put result in r. (r = a * a) - * - * r A single precision integer. - * a A single precision integer. - */ -SP_NOINLINE static void sp_256_sqr_4(sp_digit* r, const sp_digit* a) -{ - sp_digit tmp[4]; - - __asm__ __volatile__ ( - "# A[0] * A[0]\n\t" - "movq 0(%[a]), %%rax\n\t" - "mulq %%rax\n\t" - "xorq %%r9, %%r9\n\t" - "movq %%rax, (%[tmp])\n\t" - "movq %%rdx, %%r8\n\t" - "# A[0] * A[1]\n\t" - "movq 8(%[a]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%rcx, %%rcx\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%r9\n\t" - "adcq $0, %%rcx\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%r9\n\t" - "adcq $0, %%rcx\n\t" - "movq %%r8, 8(%[tmp])\n\t" - "# A[0] * A[2]\n\t" - "movq 16(%[a]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "addq %%rax, %%r9\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "addq %%rax, %%r9\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "# A[1] * A[1]\n\t" - "movq 8(%[a]), %%rax\n\t" - "mulq %%rax\n\t" - "addq %%rax, %%r9\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "movq %%r9, 16(%[tmp])\n\t" - "# A[0] * A[3]\n\t" - "movq 24(%[a]), %%rax\n\t" - "mulq 0(%[a])\n\t" - "xorq %%r9, %%r9\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%r9\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%r9\n\t" - "# A[1] * A[2]\n\t" - "movq 16(%[a]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%r9\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "adcq $0, %%r9\n\t" - "movq %%rcx, 24(%[tmp])\n\t" - "# A[1] * A[3]\n\t" - "movq 24(%[a]), %%rax\n\t" - "mulq 8(%[a])\n\t" - "xorq %%rcx, %%rcx\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%r9\n\t" - "adcq $0, %%rcx\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%r9\n\t" - "adcq $0, %%rcx\n\t" - "# A[2] * A[2]\n\t" - "movq 16(%[a]), %%rax\n\t" - "mulq %%rax\n\t" - "addq %%rax, %%r8\n\t" - "adcq %%rdx, %%r9\n\t" - "adcq $0, %%rcx\n\t" - "movq %%r8, 32(%[r])\n\t" - "# A[2] * A[3]\n\t" - "movq 24(%[a]), %%rax\n\t" - "mulq 16(%[a])\n\t" - "xorq %%r8, %%r8\n\t" - "addq %%rax, %%r9\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "addq %%rax, %%r9\n\t" - "adcq %%rdx, %%rcx\n\t" - "adcq $0, %%r8\n\t" - "movq %%r9, 40(%[r])\n\t" - "# A[3] * A[3]\n\t" - "movq 24(%[a]), %%rax\n\t" - "mulq %%rax\n\t" - "addq %%rax, %%rcx\n\t" - "adcq %%rdx, %%r8\n\t" - "movq %%rcx, 48(%[r])\n\t" - "movq %%r8, 56(%[r])\n\t" - : - : [r] "r" (r), [a] "r" (a), [tmp] "r" (tmp) - : "memory", "rax", "rdx", "rcx", "r8", "r9", "r10", "r11", "r12" - ); - - XMEMCPY(r, tmp, sizeof(tmp)); -} - +extern void sp_256_sqr_4(sp_digit* r, const sp_digit* a); #ifdef WOLFSSL_SP_SMALL /* Order-2 for the P256 curve. */ static const uint64_t p256_order_2[4] = { @@ -45428,224 +20044,10 @@ static void sp_256_mont_inv_order_4(sp_digit* r, sp_digit* a, } #ifdef HAVE_INTEL_AVX2 -/* Square a and put result in r. (r = a * a) - * - * r Result of squaring. - * a Number to square in Montogmery form. - */ -SP_NOINLINE static void sp_256_sqr_avx2_4(sp_digit* r, const sp_digit* a) -{ - __asm__ __volatile__ ( - "# A[0] * A[1]\n\t" - "movq 0(%[a]), %%rdx\n\t" - "mulxq 8(%[a]), %%r9, %%r10\n\t" - "# A[0] * A[3]\n\t" - "mulxq 24(%[a]), %%r11, %%r12\n\t" - "# A[2] * A[1]\n\t" - "movq 16(%[a]), %%rdx\n\t" - "mulxq 8(%[a]), %%rcx, %%rbx\n\t" - "xorq %%r15, %%r15\n\t" - "adoxq %%rcx, %%r11\n\t" - "# A[2] * A[3]\n\t" - "mulxq 24(%[a]), %%r13, %%r14\n\t" - "adoxq %%rbx, %%r12\n\t" - "# A[2] * A[0]\n\t" - "mulxq 0(%[a]), %%rcx, %%rbx\n\t" - "adoxq %%r15, %%r13\n\t" - "adcxq %%rcx, %%r10\n\t" - "adoxq %%r15, %%r14\n\t" - "# A[1] * A[3]\n\t" - "movq 8(%[a]), %%rdx\n\t" - "mulxq 24(%[a]), %%rax, %%r8\n\t" - "adcxq %%rbx, %%r11\n\t" - "adcxq %%rax, %%r12\n\t" - "adcxq %%r8, %%r13\n\t" - "adcxq %%r15, %%r14\n\t" - "# Double with Carry Flag\n\t" - "xorq %%r15, %%r15\n\t" - "# A[0] * A[0]\n\t" - "movq 0(%[a]), %%rdx\n\t" - "mulxq %%rdx, %%r8, %%rax\n\t" - "adcxq %%r9, %%r9\n\t" - "# A[1] * A[1]\n\t" - "movq 8(%[a]), %%rdx\n\t" - "mulxq %%rdx, %%rcx, %%rbx\n\t" - "adcxq %%r10, %%r10\n\t" - "adoxq %%rax, %%r9\n\t" - "adcxq %%r11, %%r11\n\t" - "adoxq %%rcx, %%r10\n\t" - "# A[2] * A[2]\n\t" - "movq 16(%[a]), %%rdx\n\t" - "mulxq %%rdx, %%rax, %%rcx\n\t" - "adcxq %%r12, %%r12\n\t" - "adoxq %%rbx, %%r11\n\t" - "adcxq %%r13, %%r13\n\t" - "adoxq %%rax, %%r12\n\t" - "# A[3] * A[3]\n\t" - "movq 24(%[a]), %%rdx\n\t" - "mulxq %%rdx, %%rax, %%rbx\n\t" - "adcxq %%r14, %%r14\n\t" - "adoxq %%rcx, %%r13\n\t" - "adcxq %%r15, %%r15\n\t" - "adoxq %%rax, %%r14\n\t" - "adoxq %%rbx, %%r15\n\t" - "movq %%r8, 0(%[r])\n\t" - "movq %%r9, 8(%[r])\n\t" - "movq %%r10, 16(%[r])\n\t" - "movq %%r11, 24(%[r])\n\t" - "movq %%r12, 32(%[r])\n\t" - "movq %%r13, 40(%[r])\n\t" - "movq %%r14, 48(%[r])\n\t" - "movq %%r15, 56(%[r])\n\t" - : - : [r] "r" (r), [a] "r" (a) - : "memory", "rax", "rdx", "rcx", "rbx", "r8", "r9", "r10", "r11", - "r12", "r13", "r14", "r15" - ); -} - +extern void sp_256_sqr_avx2_4(sp_digit* r, const sp_digit* a); #define sp_256_mont_reduce_order_avx2_4 sp_256_mont_reduce_avx2_4 -/* Reduce the number back to 256 bits using Montgomery reduction. - * - * a A single precision number to reduce in place. - * m The single precision number representing the modulus. - * mp The digit representing the negative inverse of m mod 2^n. - */ -SP_NOINLINE static void sp_256_mont_reduce_avx2_4(sp_digit* a, sp_digit* m, - sp_digit mp) -{ - __asm__ __volatile__ ( - "movq 0(%[a]), %%r12\n\t" - "movq 8(%[a]), %%r13\n\t" - "movq 16(%[a]), %%r14\n\t" - "movq 24(%[a]), %%r15\n\t" - "xorq %%r10, %%r10\n\t" - "xorq %%r11, %%r11\n\t" - "# a[0-4] += m[0-3] * mu = m[0-3] * (a[0] * mp)\n\t" - "movq 32(%[a]), %%rax\n\t" - "# mu = a[0] * mp\n\t" - "movq %%r12, %%rdx\n\t" - "mulxq %[mp], %%rdx, %%rcx\n\t" - "# a[0] += m[0] * mu\n\t" - "mulx 0(%[m]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r12\n\t" - "# a[1] += m[1] * mu\n\t" - "mulx 8(%[m]), %%r8, %%rcx\n\t" - "adoxq %%r9, %%r13\n\t" - "adcxq %%r8, %%r13\n\t" - "# a[2] += m[2] * mu\n\t" - "mulx 16(%[m]), %%r8, %%r9\n\t" - "adoxq %%rcx, %%r14\n\t" - "adcxq %%r8, %%r14\n\t" - "# a[3] += m[3] * mu\n\t" - "mulx 24(%[m]), %%r8, %%rcx\n\t" - "adoxq %%r9, %%r15\n\t" - "adcxq %%r8, %%r15\n\t" - "adoxq %%rcx, %%rax\n\t" - "adcxq %%r11, %%rax\n\t" - "adoxq %%r11, %%r10\n\t" - "adcxq %%r11, %%r10\n\t" - "# a[1-5] += m[0-3] * mu = m[0-3] * (a[1] * mp)\n\t" - "movq 40(%[a]), %%r12\n\t" - "# mu = a[1] * mp\n\t" - "movq %%r13, %%rdx\n\t" - "mulxq %[mp], %%rdx, %%rcx\n\t" - "# a[1] += m[0] * mu\n\t" - "mulx 0(%[m]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r13\n\t" - "# a[2] += m[1] * mu\n\t" - "mulx 8(%[m]), %%r8, %%rcx\n\t" - "adoxq %%r9, %%r14\n\t" - "adcxq %%r8, %%r14\n\t" - "# a[3] += m[2] * mu\n\t" - "mulx 16(%[m]), %%r8, %%r9\n\t" - "adoxq %%rcx, %%r15\n\t" - "adcxq %%r8, %%r15\n\t" - "# a[4] += m[3] * mu\n\t" - "mulx 24(%[m]), %%r8, %%rcx\n\t" - "adoxq %%r9, %%rax\n\t" - "adcxq %%r8, %%rax\n\t" - "adoxq %%rcx, %%r12\n\t" - "adcxq %%r10, %%r12\n\t" - "movq %%r11, %%r10\n\t" - "adoxq %%r11, %%r10\n\t" - "adcxq %%r11, %%r10\n\t" - "# a[2-6] += m[0-3] * mu = m[0-3] * (a[2] * mp)\n\t" - "movq 48(%[a]), %%r13\n\t" - "# mu = a[2] * mp\n\t" - "movq %%r14, %%rdx\n\t" - "mulxq %[mp], %%rdx, %%rcx\n\t" - "# a[2] += m[0] * mu\n\t" - "mulx 0(%[m]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r14\n\t" - "# a[3] += m[1] * mu\n\t" - "mulx 8(%[m]), %%r8, %%rcx\n\t" - "adoxq %%r9, %%r15\n\t" - "adcxq %%r8, %%r15\n\t" - "# a[4] += m[2] * mu\n\t" - "mulx 16(%[m]), %%r8, %%r9\n\t" - "adoxq %%rcx, %%rax\n\t" - "adcxq %%r8, %%rax\n\t" - "# a[5] += m[3] * mu\n\t" - "mulx 24(%[m]), %%r8, %%rcx\n\t" - "adoxq %%r9, %%r12\n\t" - "adcxq %%r8, %%r12\n\t" - "adoxq %%rcx, %%r13\n\t" - "adcxq %%r10, %%r13\n\t" - "movq %%r11, %%r10\n\t" - "adoxq %%r11, %%r10\n\t" - "adcxq %%r11, %%r10\n\t" - "# a[3-7] += m[0-3] * mu = m[0-3] * (a[3] * mp)\n\t" - "movq 56(%[a]), %%r14\n\t" - "# mu = a[3] * mp\n\t" - "movq %%r15, %%rdx\n\t" - "mulxq %[mp], %%rdx, %%rcx\n\t" - "# a[3] += m[0] * mu\n\t" - "mulx 0(%[m]), %%r8, %%r9\n\t" - "adcxq %%r8, %%r15\n\t" - "# a[4] += m[1] * mu\n\t" - "mulx 8(%[m]), %%r8, %%rcx\n\t" - "adoxq %%r9, %%rax\n\t" - "adcxq %%r8, %%rax\n\t" - "# a[5] += m[2] * mu\n\t" - "mulx 16(%[m]), %%r8, %%r9\n\t" - "adoxq %%rcx, %%r12\n\t" - "adcxq %%r8, %%r12\n\t" - "# a[6] += m[3] * mu\n\t" - "mulx 24(%[m]), %%r8, %%rcx\n\t" - "adoxq %%r9, %%r13\n\t" - "adcxq %%r8, %%r13\n\t" - "adoxq %%rcx, %%r14\n\t" - "adcxq %%r10, %%r14\n\t" - "movq %%r11, %%r10\n\t" - "adoxq %%r11, %%r10\n\t" - "adcxq %%r11, %%r10\n\t" - "# Subtract mod if carry\n\t" - "subq %%r10, %%r11\n\t" - "movq 0(%[m]), %%r8\n\t" - "movq 8(%[m]), %%r9\n\t" - "movq 16(%[m]), %%r10\n\t" - "movq 24(%[m]), %%rdx\n\t" - "andq %%r11, %%r8\n\t" - "andq %%r11, %%r9\n\t" - "andq %%r11, %%r10\n\t" - "andq %%r11, %%rdx\n\t" - "subq %%r8, %%rax\n\t" - "sbbq %%r9, %%r12\n\t" - "sbbq %%r10, %%r13\n\t" - "sbbq %%rdx, %%r14\n\t" - "movq %%rax, (%[a])\n\t" - "movq %%r12, 8(%[a])\n\t" - "movq %%r13, 16(%[a])\n\t" - "movq %%r14, 24(%[a])\n\t" - : - : [a] "r" (a), [m] "r" (m), [mp] "r" (mp) - : "memory", "rax", "rcx", "rdx", "r8", "r9", "r10", "r11", - "r12", "r13", "r14", "r15" - ); -} - +extern void sp_256_mont_reduce_avx2_4(sp_digit* a, sp_digit* m, sp_digit mp); /* Multiply two number mod the order of P256 curve. (r = a * b mod order) * * r Result of the multiplication. @@ -46728,7 +21130,7 @@ int sp_ecc_uncompress_256(mp_int* xm, int odd, mp_int* ym) return err; } #endif -#endif /* WOLFSSL_SP_NO_256 */ +#endif /* !WOLFSSL_SP_NO_256 */ #endif /* WOLFSSL_HAVE_SP_ECC */ #endif /* WOLFSSL_SP_X86_64_ASM */ #endif /* WOLFSSL_HAVE_SP_RSA || WOLFSSL_HAVE_SP_DH || WOLFSSL_HAVE_SP_ECC */ diff --git a/wolfcrypt/src/sp_x86_64_asm.S b/wolfcrypt/src/sp_x86_64_asm.S new file mode 100644 index 0000000000..0d73aa7cbe --- /dev/null +++ b/wolfcrypt/src/sp_x86_64_asm.S @@ -0,0 +1,25934 @@ +/* sp_x86_64_asm + * + * Copyright (C) 2006-2018 wolfSSL Inc. + * + * This file is part of wolfSSL. + * + * wolfSSL is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * wolfSSL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA + */ + +#define HAVE_INTEL_AVX2 +#ifdef RSA_LOW_MEM +#ifndef SP_RSA_PRIVATE_EXP_D +#define SP_RSA_PRIVATE_EXP_D +#endif /* SP_RSA_PRIVATE_EXP_D */ +#ifndef WOLFSSL_SP_SMALL +#define WOLFSSL_SP_SMALL +#endif /* WOLFSSL_SP_SMALL */ +#endif /* RSA_LOW_MEM */ +#ifdef WOLFSSL_SP_X86_64_ASM +#if defined(WOLFSSL_HAVE_SP_RSA) || defined(WOLFSSL_HAVE_SP_DH) +#ifndef WOLFSSL_SP_NO_2048 +/* Multiply a and b into r. (r = a * b) + * + * r A single precision integer. + * a A single precision integer. + * b A single precision integer. + */ +.globl sp_2048_mul_16 +.type sp_2048_mul_16,@function +.align 16 +sp_2048_mul_16: + movq %rdx, %rcx + subq $128, %rsp + # A[0] * B[0] + movq (%rcx), %rax + mulq (%rsi) + xorq %r10, %r10 + movq %rax, (%rsp) + movq %rdx, %r9 + # A[0] * B[1] + movq 8(%rcx), %rax + mulq (%rsi) + xorq %r8, %r8 + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[1] * B[0] + movq (%rcx), %rax + mulq 8(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + movq %r9, 8(%rsp) + # A[0] * B[2] + movq 16(%rcx), %rax + mulq (%rsi) + xorq %r9, %r9 + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[1] * B[1] + movq 8(%rcx), %rax + mulq 8(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[2] * B[0] + movq (%rcx), %rax + mulq 16(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + movq %r10, 16(%rsp) + # A[0] * B[3] + movq 24(%rcx), %rax + mulq (%rsi) + xorq %r10, %r10 + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[1] * B[2] + movq 16(%rcx), %rax + mulq 8(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[2] * B[1] + movq 8(%rcx), %rax + mulq 16(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[3] * B[0] + movq (%rcx), %rax + mulq 24(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + movq %r8, 24(%rsp) + # A[0] * B[4] + movq 32(%rcx), %rax + mulq (%rsi) + xorq %r8, %r8 + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[1] * B[3] + movq 24(%rcx), %rax + mulq 8(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[2] * B[2] + movq 16(%rcx), %rax + mulq 16(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[3] * B[1] + movq 8(%rcx), %rax + mulq 24(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[4] * B[0] + movq (%rcx), %rax + mulq 32(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + movq %r9, 32(%rsp) + # A[0] * B[5] + movq 40(%rcx), %rax + mulq (%rsi) + xorq %r9, %r9 + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[1] * B[4] + movq 32(%rcx), %rax + mulq 8(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[2] * B[3] + movq 24(%rcx), %rax + mulq 16(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[3] * B[2] + movq 16(%rcx), %rax + mulq 24(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[4] * B[1] + movq 8(%rcx), %rax + mulq 32(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[5] * B[0] + movq (%rcx), %rax + mulq 40(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + movq %r10, 40(%rsp) + # A[0] * B[6] + movq 48(%rcx), %rax + mulq (%rsi) + xorq %r10, %r10 + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[1] * B[5] + movq 40(%rcx), %rax + mulq 8(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[2] * B[4] + movq 32(%rcx), %rax + mulq 16(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[3] * B[3] + movq 24(%rcx), %rax + mulq 24(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[4] * B[2] + movq 16(%rcx), %rax + mulq 32(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[5] * B[1] + movq 8(%rcx), %rax + mulq 40(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[6] * B[0] + movq (%rcx), %rax + mulq 48(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + movq %r8, 48(%rsp) + # A[0] * B[7] + movq 56(%rcx), %rax + mulq (%rsi) + xorq %r8, %r8 + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[1] * B[6] + movq 48(%rcx), %rax + mulq 8(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[2] * B[5] + movq 40(%rcx), %rax + mulq 16(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[3] * B[4] + movq 32(%rcx), %rax + mulq 24(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[4] * B[3] + movq 24(%rcx), %rax + mulq 32(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[5] * B[2] + movq 16(%rcx), %rax + mulq 40(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[6] * B[1] + movq 8(%rcx), %rax + mulq 48(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[7] * B[0] + movq (%rcx), %rax + mulq 56(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + movq %r9, 56(%rsp) + # A[0] * B[8] + movq 64(%rcx), %rax + mulq (%rsi) + xorq %r9, %r9 + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[1] * B[7] + movq 56(%rcx), %rax + mulq 8(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[2] * B[6] + movq 48(%rcx), %rax + mulq 16(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[3] * B[5] + movq 40(%rcx), %rax + mulq 24(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[4] * B[4] + movq 32(%rcx), %rax + mulq 32(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[5] * B[3] + movq 24(%rcx), %rax + mulq 40(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[6] * B[2] + movq 16(%rcx), %rax + mulq 48(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[7] * B[1] + movq 8(%rcx), %rax + mulq 56(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[8] * B[0] + movq (%rcx), %rax + mulq 64(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + movq %r10, 64(%rsp) + # A[0] * B[9] + movq 72(%rcx), %rax + mulq (%rsi) + xorq %r10, %r10 + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[1] * B[8] + movq 64(%rcx), %rax + mulq 8(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[2] * B[7] + movq 56(%rcx), %rax + mulq 16(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[3] * B[6] + movq 48(%rcx), %rax + mulq 24(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[4] * B[5] + movq 40(%rcx), %rax + mulq 32(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[5] * B[4] + movq 32(%rcx), %rax + mulq 40(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[6] * B[3] + movq 24(%rcx), %rax + mulq 48(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[7] * B[2] + movq 16(%rcx), %rax + mulq 56(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[8] * B[1] + movq 8(%rcx), %rax + mulq 64(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[9] * B[0] + movq (%rcx), %rax + mulq 72(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + movq %r8, 72(%rsp) + # A[0] * B[10] + movq 80(%rcx), %rax + mulq (%rsi) + xorq %r8, %r8 + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[1] * B[9] + movq 72(%rcx), %rax + mulq 8(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[2] * B[8] + movq 64(%rcx), %rax + mulq 16(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[3] * B[7] + movq 56(%rcx), %rax + mulq 24(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[4] * B[6] + movq 48(%rcx), %rax + mulq 32(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[5] * B[5] + movq 40(%rcx), %rax + mulq 40(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[6] * B[4] + movq 32(%rcx), %rax + mulq 48(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[7] * B[3] + movq 24(%rcx), %rax + mulq 56(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[8] * B[2] + movq 16(%rcx), %rax + mulq 64(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[9] * B[1] + movq 8(%rcx), %rax + mulq 72(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[10] * B[0] + movq (%rcx), %rax + mulq 80(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + movq %r9, 80(%rsp) + # A[0] * B[11] + movq 88(%rcx), %rax + mulq (%rsi) + xorq %r9, %r9 + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[1] * B[10] + movq 80(%rcx), %rax + mulq 8(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[2] * B[9] + movq 72(%rcx), %rax + mulq 16(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[3] * B[8] + movq 64(%rcx), %rax + mulq 24(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[4] * B[7] + movq 56(%rcx), %rax + mulq 32(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[5] * B[6] + movq 48(%rcx), %rax + mulq 40(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[6] * B[5] + movq 40(%rcx), %rax + mulq 48(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[7] * B[4] + movq 32(%rcx), %rax + mulq 56(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[8] * B[3] + movq 24(%rcx), %rax + mulq 64(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[9] * B[2] + movq 16(%rcx), %rax + mulq 72(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[10] * B[1] + movq 8(%rcx), %rax + mulq 80(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[11] * B[0] + movq (%rcx), %rax + mulq 88(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + movq %r10, 88(%rsp) + # A[0] * B[12] + movq 96(%rcx), %rax + mulq (%rsi) + xorq %r10, %r10 + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[1] * B[11] + movq 88(%rcx), %rax + mulq 8(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[2] * B[10] + movq 80(%rcx), %rax + mulq 16(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[3] * B[9] + movq 72(%rcx), %rax + mulq 24(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[4] * B[8] + movq 64(%rcx), %rax + mulq 32(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[5] * B[7] + movq 56(%rcx), %rax + mulq 40(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[6] * B[6] + movq 48(%rcx), %rax + mulq 48(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[7] * B[5] + movq 40(%rcx), %rax + mulq 56(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[8] * B[4] + movq 32(%rcx), %rax + mulq 64(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[9] * B[3] + movq 24(%rcx), %rax + mulq 72(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[10] * B[2] + movq 16(%rcx), %rax + mulq 80(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[11] * B[1] + movq 8(%rcx), %rax + mulq 88(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[12] * B[0] + movq (%rcx), %rax + mulq 96(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + movq %r8, 96(%rsp) + # A[0] * B[13] + movq 104(%rcx), %rax + mulq (%rsi) + xorq %r8, %r8 + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[1] * B[12] + movq 96(%rcx), %rax + mulq 8(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[2] * B[11] + movq 88(%rcx), %rax + mulq 16(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[3] * B[10] + movq 80(%rcx), %rax + mulq 24(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[4] * B[9] + movq 72(%rcx), %rax + mulq 32(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[5] * B[8] + movq 64(%rcx), %rax + mulq 40(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[6] * B[7] + movq 56(%rcx), %rax + mulq 48(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[7] * B[6] + movq 48(%rcx), %rax + mulq 56(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[8] * B[5] + movq 40(%rcx), %rax + mulq 64(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[9] * B[4] + movq 32(%rcx), %rax + mulq 72(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[10] * B[3] + movq 24(%rcx), %rax + mulq 80(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[11] * B[2] + movq 16(%rcx), %rax + mulq 88(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[12] * B[1] + movq 8(%rcx), %rax + mulq 96(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[13] * B[0] + movq (%rcx), %rax + mulq 104(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + movq %r9, 104(%rsp) + # A[0] * B[14] + movq 112(%rcx), %rax + mulq (%rsi) + xorq %r9, %r9 + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[1] * B[13] + movq 104(%rcx), %rax + mulq 8(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[2] * B[12] + movq 96(%rcx), %rax + mulq 16(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[3] * B[11] + movq 88(%rcx), %rax + mulq 24(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[4] * B[10] + movq 80(%rcx), %rax + mulq 32(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[5] * B[9] + movq 72(%rcx), %rax + mulq 40(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[6] * B[8] + movq 64(%rcx), %rax + mulq 48(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[7] * B[7] + movq 56(%rcx), %rax + mulq 56(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[8] * B[6] + movq 48(%rcx), %rax + mulq 64(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[9] * B[5] + movq 40(%rcx), %rax + mulq 72(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[10] * B[4] + movq 32(%rcx), %rax + mulq 80(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[11] * B[3] + movq 24(%rcx), %rax + mulq 88(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[12] * B[2] + movq 16(%rcx), %rax + mulq 96(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[13] * B[1] + movq 8(%rcx), %rax + mulq 104(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[14] * B[0] + movq (%rcx), %rax + mulq 112(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + movq %r10, 112(%rsp) + # A[0] * B[15] + movq 120(%rcx), %rax + mulq (%rsi) + xorq %r10, %r10 + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[1] * B[14] + movq 112(%rcx), %rax + mulq 8(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[2] * B[13] + movq 104(%rcx), %rax + mulq 16(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[3] * B[12] + movq 96(%rcx), %rax + mulq 24(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[4] * B[11] + movq 88(%rcx), %rax + mulq 32(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[5] * B[10] + movq 80(%rcx), %rax + mulq 40(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[6] * B[9] + movq 72(%rcx), %rax + mulq 48(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[7] * B[8] + movq 64(%rcx), %rax + mulq 56(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[8] * B[7] + movq 56(%rcx), %rax + mulq 64(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[9] * B[6] + movq 48(%rcx), %rax + mulq 72(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[10] * B[5] + movq 40(%rcx), %rax + mulq 80(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[11] * B[4] + movq 32(%rcx), %rax + mulq 88(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[12] * B[3] + movq 24(%rcx), %rax + mulq 96(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[13] * B[2] + movq 16(%rcx), %rax + mulq 104(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[14] * B[1] + movq 8(%rcx), %rax + mulq 112(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[15] * B[0] + movq (%rcx), %rax + mulq 120(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + movq %r8, 120(%rsp) + # A[1] * B[15] + movq 120(%rcx), %rax + mulq 8(%rsi) + xorq %r8, %r8 + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[2] * B[14] + movq 112(%rcx), %rax + mulq 16(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[3] * B[13] + movq 104(%rcx), %rax + mulq 24(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[4] * B[12] + movq 96(%rcx), %rax + mulq 32(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[5] * B[11] + movq 88(%rcx), %rax + mulq 40(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[6] * B[10] + movq 80(%rcx), %rax + mulq 48(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[7] * B[9] + movq 72(%rcx), %rax + mulq 56(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[8] * B[8] + movq 64(%rcx), %rax + mulq 64(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[9] * B[7] + movq 56(%rcx), %rax + mulq 72(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[10] * B[6] + movq 48(%rcx), %rax + mulq 80(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[11] * B[5] + movq 40(%rcx), %rax + mulq 88(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[12] * B[4] + movq 32(%rcx), %rax + mulq 96(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[13] * B[3] + movq 24(%rcx), %rax + mulq 104(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[14] * B[2] + movq 16(%rcx), %rax + mulq 112(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[15] * B[1] + movq 8(%rcx), %rax + mulq 120(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + movq %r9, 128(%rdi) + # A[2] * B[15] + movq 120(%rcx), %rax + mulq 16(%rsi) + xorq %r9, %r9 + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[3] * B[14] + movq 112(%rcx), %rax + mulq 24(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[4] * B[13] + movq 104(%rcx), %rax + mulq 32(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[5] * B[12] + movq 96(%rcx), %rax + mulq 40(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[6] * B[11] + movq 88(%rcx), %rax + mulq 48(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[7] * B[10] + movq 80(%rcx), %rax + mulq 56(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[8] * B[9] + movq 72(%rcx), %rax + mulq 64(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[9] * B[8] + movq 64(%rcx), %rax + mulq 72(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[10] * B[7] + movq 56(%rcx), %rax + mulq 80(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[11] * B[6] + movq 48(%rcx), %rax + mulq 88(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[12] * B[5] + movq 40(%rcx), %rax + mulq 96(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[13] * B[4] + movq 32(%rcx), %rax + mulq 104(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[14] * B[3] + movq 24(%rcx), %rax + mulq 112(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[15] * B[2] + movq 16(%rcx), %rax + mulq 120(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + movq %r10, 136(%rdi) + # A[3] * B[15] + movq 120(%rcx), %rax + mulq 24(%rsi) + xorq %r10, %r10 + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[4] * B[14] + movq 112(%rcx), %rax + mulq 32(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[5] * B[13] + movq 104(%rcx), %rax + mulq 40(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[6] * B[12] + movq 96(%rcx), %rax + mulq 48(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[7] * B[11] + movq 88(%rcx), %rax + mulq 56(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[8] * B[10] + movq 80(%rcx), %rax + mulq 64(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[9] * B[9] + movq 72(%rcx), %rax + mulq 72(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[10] * B[8] + movq 64(%rcx), %rax + mulq 80(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[11] * B[7] + movq 56(%rcx), %rax + mulq 88(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[12] * B[6] + movq 48(%rcx), %rax + mulq 96(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[13] * B[5] + movq 40(%rcx), %rax + mulq 104(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[14] * B[4] + movq 32(%rcx), %rax + mulq 112(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[15] * B[3] + movq 24(%rcx), %rax + mulq 120(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + movq %r8, 144(%rdi) + # A[4] * B[15] + movq 120(%rcx), %rax + mulq 32(%rsi) + xorq %r8, %r8 + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[5] * B[14] + movq 112(%rcx), %rax + mulq 40(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[6] * B[13] + movq 104(%rcx), %rax + mulq 48(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[7] * B[12] + movq 96(%rcx), %rax + mulq 56(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[8] * B[11] + movq 88(%rcx), %rax + mulq 64(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[9] * B[10] + movq 80(%rcx), %rax + mulq 72(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[10] * B[9] + movq 72(%rcx), %rax + mulq 80(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[11] * B[8] + movq 64(%rcx), %rax + mulq 88(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[12] * B[7] + movq 56(%rcx), %rax + mulq 96(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[13] * B[6] + movq 48(%rcx), %rax + mulq 104(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[14] * B[5] + movq 40(%rcx), %rax + mulq 112(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[15] * B[4] + movq 32(%rcx), %rax + mulq 120(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + movq %r9, 152(%rdi) + # A[5] * B[15] + movq 120(%rcx), %rax + mulq 40(%rsi) + xorq %r9, %r9 + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[6] * B[14] + movq 112(%rcx), %rax + mulq 48(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[7] * B[13] + movq 104(%rcx), %rax + mulq 56(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[8] * B[12] + movq 96(%rcx), %rax + mulq 64(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[9] * B[11] + movq 88(%rcx), %rax + mulq 72(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[10] * B[10] + movq 80(%rcx), %rax + mulq 80(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[11] * B[9] + movq 72(%rcx), %rax + mulq 88(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[12] * B[8] + movq 64(%rcx), %rax + mulq 96(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[13] * B[7] + movq 56(%rcx), %rax + mulq 104(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[14] * B[6] + movq 48(%rcx), %rax + mulq 112(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[15] * B[5] + movq 40(%rcx), %rax + mulq 120(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + movq %r10, 160(%rdi) + # A[6] * B[15] + movq 120(%rcx), %rax + mulq 48(%rsi) + xorq %r10, %r10 + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[7] * B[14] + movq 112(%rcx), %rax + mulq 56(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[8] * B[13] + movq 104(%rcx), %rax + mulq 64(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[9] * B[12] + movq 96(%rcx), %rax + mulq 72(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[10] * B[11] + movq 88(%rcx), %rax + mulq 80(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[11] * B[10] + movq 80(%rcx), %rax + mulq 88(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[12] * B[9] + movq 72(%rcx), %rax + mulq 96(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[13] * B[8] + movq 64(%rcx), %rax + mulq 104(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[14] * B[7] + movq 56(%rcx), %rax + mulq 112(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[15] * B[6] + movq 48(%rcx), %rax + mulq 120(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + movq %r8, 168(%rdi) + # A[7] * B[15] + movq 120(%rcx), %rax + mulq 56(%rsi) + xorq %r8, %r8 + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[8] * B[14] + movq 112(%rcx), %rax + mulq 64(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[9] * B[13] + movq 104(%rcx), %rax + mulq 72(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[10] * B[12] + movq 96(%rcx), %rax + mulq 80(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[11] * B[11] + movq 88(%rcx), %rax + mulq 88(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[12] * B[10] + movq 80(%rcx), %rax + mulq 96(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[13] * B[9] + movq 72(%rcx), %rax + mulq 104(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[14] * B[8] + movq 64(%rcx), %rax + mulq 112(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[15] * B[7] + movq 56(%rcx), %rax + mulq 120(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + movq %r9, 176(%rdi) + # A[8] * B[15] + movq 120(%rcx), %rax + mulq 64(%rsi) + xorq %r9, %r9 + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[9] * B[14] + movq 112(%rcx), %rax + mulq 72(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[10] * B[13] + movq 104(%rcx), %rax + mulq 80(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[11] * B[12] + movq 96(%rcx), %rax + mulq 88(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[12] * B[11] + movq 88(%rcx), %rax + mulq 96(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[13] * B[10] + movq 80(%rcx), %rax + mulq 104(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[14] * B[9] + movq 72(%rcx), %rax + mulq 112(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[15] * B[8] + movq 64(%rcx), %rax + mulq 120(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + movq %r10, 184(%rdi) + # A[9] * B[15] + movq 120(%rcx), %rax + mulq 72(%rsi) + xorq %r10, %r10 + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[10] * B[14] + movq 112(%rcx), %rax + mulq 80(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[11] * B[13] + movq 104(%rcx), %rax + mulq 88(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[12] * B[12] + movq 96(%rcx), %rax + mulq 96(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[13] * B[11] + movq 88(%rcx), %rax + mulq 104(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[14] * B[10] + movq 80(%rcx), %rax + mulq 112(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[15] * B[9] + movq 72(%rcx), %rax + mulq 120(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + movq %r8, 192(%rdi) + # A[10] * B[15] + movq 120(%rcx), %rax + mulq 80(%rsi) + xorq %r8, %r8 + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[11] * B[14] + movq 112(%rcx), %rax + mulq 88(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[12] * B[13] + movq 104(%rcx), %rax + mulq 96(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[13] * B[12] + movq 96(%rcx), %rax + mulq 104(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[14] * B[11] + movq 88(%rcx), %rax + mulq 112(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[15] * B[10] + movq 80(%rcx), %rax + mulq 120(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + movq %r9, 200(%rdi) + # A[11] * B[15] + movq 120(%rcx), %rax + mulq 88(%rsi) + xorq %r9, %r9 + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[12] * B[14] + movq 112(%rcx), %rax + mulq 96(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[13] * B[13] + movq 104(%rcx), %rax + mulq 104(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[14] * B[12] + movq 96(%rcx), %rax + mulq 112(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[15] * B[11] + movq 88(%rcx), %rax + mulq 120(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + movq %r10, 208(%rdi) + # A[12] * B[15] + movq 120(%rcx), %rax + mulq 96(%rsi) + xorq %r10, %r10 + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[13] * B[14] + movq 112(%rcx), %rax + mulq 104(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[14] * B[13] + movq 104(%rcx), %rax + mulq 112(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[15] * B[12] + movq 96(%rcx), %rax + mulq 120(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + movq %r8, 216(%rdi) + # A[13] * B[15] + movq 120(%rcx), %rax + mulq 104(%rsi) + xorq %r8, %r8 + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[14] * B[14] + movq 112(%rcx), %rax + mulq 112(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[15] * B[13] + movq 104(%rcx), %rax + mulq 120(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + movq %r9, 224(%rdi) + # A[14] * B[15] + movq 120(%rcx), %rax + mulq 112(%rsi) + xorq %r9, %r9 + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[15] * B[14] + movq 112(%rcx), %rax + mulq 120(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + movq %r10, 232(%rdi) + # A[15] * B[15] + movq 120(%rcx), %rax + mulq 120(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + movq %r8, 240(%rdi) + movq %r9, 248(%rdi) + movq (%rsp), %rax + movq 8(%rsp), %rdx + movq 16(%rsp), %r8 + movq 24(%rsp), %r9 + movq %rax, (%rdi) + movq %rdx, 8(%rdi) + movq %r8, 16(%rdi) + movq %r9, 24(%rdi) + movq 32(%rsp), %rax + movq 40(%rsp), %rdx + movq 48(%rsp), %r8 + movq 56(%rsp), %r9 + movq %rax, 32(%rdi) + movq %rdx, 40(%rdi) + movq %r8, 48(%rdi) + movq %r9, 56(%rdi) + movq 64(%rsp), %rax + movq 72(%rsp), %rdx + movq 80(%rsp), %r8 + movq 88(%rsp), %r9 + movq %rax, 64(%rdi) + movq %rdx, 72(%rdi) + movq %r8, 80(%rdi) + movq %r9, 88(%rdi) + movq 96(%rsp), %rax + movq 104(%rsp), %rdx + movq 112(%rsp), %r8 + movq 120(%rsp), %r9 + movq %rax, 96(%rdi) + movq %rdx, 104(%rdi) + movq %r8, 112(%rdi) + movq %r9, 120(%rdi) + addq $128, %rsp + repz retq +.size sp_2048_mul_16,.-sp_2048_mul_16 +/* Square a and put result in r. (r = a * a) + * + * r A single precision integer. + * a A single precision integer. + */ +.globl sp_2048_sqr_16 +.type sp_2048_sqr_16,@function +.align 16 +sp_2048_sqr_16: + push %r12 + subq $128, %rsp + # A[0] * A[0] + movq (%rsi), %rax + mulq %rax + xorq %r9, %r9 + movq %rax, (%rsp) + movq %rdx, %r8 + # A[0] * A[1] + movq 8(%rsi), %rax + mulq (%rsi) + xorq %rcx, %rcx + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %rcx + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %rcx + movq %r8, 8(%rsp) + # A[0] * A[2] + movq 16(%rsi), %rax + mulq (%rsi) + xorq %r8, %r8 + addq %rax, %r9 + adcq %rdx, %rcx + adcq $0, %r8 + addq %rax, %r9 + adcq %rdx, %rcx + adcq $0, %r8 + # A[1] * A[1] + movq 8(%rsi), %rax + mulq %rax + addq %rax, %r9 + adcq %rdx, %rcx + adcq $0, %r8 + movq %r9, 16(%rsp) + # A[0] * A[3] + movq 24(%rsi), %rax + mulq (%rsi) + xorq %r9, %r9 + addq %rax, %rcx + adcq %rdx, %r8 + adcq $0, %r9 + addq %rax, %rcx + adcq %rdx, %r8 + adcq $0, %r9 + # A[1] * A[2] + movq 16(%rsi), %rax + mulq 8(%rsi) + addq %rax, %rcx + adcq %rdx, %r8 + adcq $0, %r9 + addq %rax, %rcx + adcq %rdx, %r8 + adcq $0, %r9 + movq %rcx, 24(%rsp) + # A[0] * A[4] + movq 32(%rsi), %rax + mulq (%rsi) + xorq %rcx, %rcx + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %rcx + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %rcx + # A[1] * A[3] + movq 24(%rsi), %rax + mulq 8(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %rcx + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %rcx + # A[2] * A[2] + movq 16(%rsi), %rax + mulq %rax + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %rcx + movq %r8, 32(%rsp) + # A[0] * A[5] + movq 40(%rsi), %rax + mulq (%rsi) + xorq %r8, %r8 + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[1] * A[4] + movq 32(%rsi), %rax + mulq 8(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[2] * A[3] + movq 24(%rsi), %rax + mulq 16(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %r10, %r9 + adcq %r11, %rcx + adcq %r12, %r8 + movq %r9, 40(%rsp) + # A[0] * A[6] + movq 48(%rsi), %rax + mulq (%rsi) + xorq %r9, %r9 + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[1] * A[5] + movq 40(%rsi), %rax + mulq 8(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[2] * A[4] + movq 32(%rsi), %rax + mulq 16(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[3] * A[3] + movq 24(%rsi), %rax + mulq %rax + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %rcx + adcq %r11, %r8 + adcq %r12, %r9 + movq %rcx, 48(%rsp) + # A[0] * A[7] + movq 56(%rsi), %rax + mulq (%rsi) + xorq %rcx, %rcx + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[1] * A[6] + movq 48(%rsi), %rax + mulq 8(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[2] * A[5] + movq 40(%rsi), %rax + mulq 16(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[3] * A[4] + movq 32(%rsi), %rax + mulq 24(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %r10, %r8 + adcq %r11, %r9 + adcq %r12, %rcx + movq %r8, 56(%rsp) + # A[0] * A[8] + movq 64(%rsi), %rax + mulq (%rsi) + xorq %r8, %r8 + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[1] * A[7] + movq 56(%rsi), %rax + mulq 8(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[2] * A[6] + movq 48(%rsi), %rax + mulq 16(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[3] * A[5] + movq 40(%rsi), %rax + mulq 24(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[4] * A[4] + movq 32(%rsi), %rax + mulq %rax + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %r9 + adcq %r11, %rcx + adcq %r12, %r8 + movq %r9, 64(%rsp) + # A[0] * A[9] + movq 72(%rsi), %rax + mulq (%rsi) + xorq %r9, %r9 + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[1] * A[8] + movq 64(%rsi), %rax + mulq 8(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[2] * A[7] + movq 56(%rsi), %rax + mulq 16(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[3] * A[6] + movq 48(%rsi), %rax + mulq 24(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[4] * A[5] + movq 40(%rsi), %rax + mulq 32(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %r10, %rcx + adcq %r11, %r8 + adcq %r12, %r9 + movq %rcx, 72(%rsp) + # A[0] * A[10] + movq 80(%rsi), %rax + mulq (%rsi) + xorq %rcx, %rcx + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[1] * A[9] + movq 72(%rsi), %rax + mulq 8(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[2] * A[8] + movq 64(%rsi), %rax + mulq 16(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[3] * A[7] + movq 56(%rsi), %rax + mulq 24(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[4] * A[6] + movq 48(%rsi), %rax + mulq 32(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[5] * A[5] + movq 40(%rsi), %rax + mulq %rax + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %r8 + adcq %r11, %r9 + adcq %r12, %rcx + movq %r8, 80(%rsp) + # A[0] * A[11] + movq 88(%rsi), %rax + mulq (%rsi) + xorq %r8, %r8 + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[1] * A[10] + movq 80(%rsi), %rax + mulq 8(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[2] * A[9] + movq 72(%rsi), %rax + mulq 16(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[3] * A[8] + movq 64(%rsi), %rax + mulq 24(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[4] * A[7] + movq 56(%rsi), %rax + mulq 32(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[5] * A[6] + movq 48(%rsi), %rax + mulq 40(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %r10, %r9 + adcq %r11, %rcx + adcq %r12, %r8 + movq %r9, 88(%rsp) + # A[0] * A[12] + movq 96(%rsi), %rax + mulq (%rsi) + xorq %r9, %r9 + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[1] * A[11] + movq 88(%rsi), %rax + mulq 8(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[2] * A[10] + movq 80(%rsi), %rax + mulq 16(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[3] * A[9] + movq 72(%rsi), %rax + mulq 24(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[4] * A[8] + movq 64(%rsi), %rax + mulq 32(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[5] * A[7] + movq 56(%rsi), %rax + mulq 40(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[6] * A[6] + movq 48(%rsi), %rax + mulq %rax + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %rcx + adcq %r11, %r8 + adcq %r12, %r9 + movq %rcx, 96(%rsp) + # A[0] * A[13] + movq 104(%rsi), %rax + mulq (%rsi) + xorq %rcx, %rcx + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[1] * A[12] + movq 96(%rsi), %rax + mulq 8(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[2] * A[11] + movq 88(%rsi), %rax + mulq 16(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[3] * A[10] + movq 80(%rsi), %rax + mulq 24(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[4] * A[9] + movq 72(%rsi), %rax + mulq 32(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[5] * A[8] + movq 64(%rsi), %rax + mulq 40(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[6] * A[7] + movq 56(%rsi), %rax + mulq 48(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %r10, %r8 + adcq %r11, %r9 + adcq %r12, %rcx + movq %r8, 104(%rsp) + # A[0] * A[14] + movq 112(%rsi), %rax + mulq (%rsi) + xorq %r8, %r8 + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[1] * A[13] + movq 104(%rsi), %rax + mulq 8(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[2] * A[12] + movq 96(%rsi), %rax + mulq 16(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[3] * A[11] + movq 88(%rsi), %rax + mulq 24(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[4] * A[10] + movq 80(%rsi), %rax + mulq 32(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[5] * A[9] + movq 72(%rsi), %rax + mulq 40(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[6] * A[8] + movq 64(%rsi), %rax + mulq 48(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[7] * A[7] + movq 56(%rsi), %rax + mulq %rax + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %r9 + adcq %r11, %rcx + adcq %r12, %r8 + movq %r9, 112(%rsp) + # A[0] * A[15] + movq 120(%rsi), %rax + mulq (%rsi) + xorq %r9, %r9 + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[1] * A[14] + movq 112(%rsi), %rax + mulq 8(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[2] * A[13] + movq 104(%rsi), %rax + mulq 16(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[3] * A[12] + movq 96(%rsi), %rax + mulq 24(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[4] * A[11] + movq 88(%rsi), %rax + mulq 32(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[5] * A[10] + movq 80(%rsi), %rax + mulq 40(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[6] * A[9] + movq 72(%rsi), %rax + mulq 48(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[7] * A[8] + movq 64(%rsi), %rax + mulq 56(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %r10, %rcx + adcq %r11, %r8 + adcq %r12, %r9 + movq %rcx, 120(%rsp) + # A[1] * A[15] + movq 120(%rsi), %rax + mulq 8(%rsi) + xorq %rcx, %rcx + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[2] * A[14] + movq 112(%rsi), %rax + mulq 16(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[3] * A[13] + movq 104(%rsi), %rax + mulq 24(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[4] * A[12] + movq 96(%rsi), %rax + mulq 32(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[5] * A[11] + movq 88(%rsi), %rax + mulq 40(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[6] * A[10] + movq 80(%rsi), %rax + mulq 48(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[7] * A[9] + movq 72(%rsi), %rax + mulq 56(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[8] * A[8] + movq 64(%rsi), %rax + mulq %rax + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %r8 + adcq %r11, %r9 + adcq %r12, %rcx + movq %r8, 128(%rdi) + # A[2] * A[15] + movq 120(%rsi), %rax + mulq 16(%rsi) + xorq %r8, %r8 + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[3] * A[14] + movq 112(%rsi), %rax + mulq 24(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[4] * A[13] + movq 104(%rsi), %rax + mulq 32(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[5] * A[12] + movq 96(%rsi), %rax + mulq 40(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[6] * A[11] + movq 88(%rsi), %rax + mulq 48(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[7] * A[10] + movq 80(%rsi), %rax + mulq 56(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[8] * A[9] + movq 72(%rsi), %rax + mulq 64(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %r10, %r9 + adcq %r11, %rcx + adcq %r12, %r8 + movq %r9, 136(%rdi) + # A[3] * A[15] + movq 120(%rsi), %rax + mulq 24(%rsi) + xorq %r9, %r9 + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[4] * A[14] + movq 112(%rsi), %rax + mulq 32(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[5] * A[13] + movq 104(%rsi), %rax + mulq 40(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[6] * A[12] + movq 96(%rsi), %rax + mulq 48(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[7] * A[11] + movq 88(%rsi), %rax + mulq 56(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[8] * A[10] + movq 80(%rsi), %rax + mulq 64(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[9] * A[9] + movq 72(%rsi), %rax + mulq %rax + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %rcx + adcq %r11, %r8 + adcq %r12, %r9 + movq %rcx, 144(%rdi) + # A[4] * A[15] + movq 120(%rsi), %rax + mulq 32(%rsi) + xorq %rcx, %rcx + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[5] * A[14] + movq 112(%rsi), %rax + mulq 40(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[6] * A[13] + movq 104(%rsi), %rax + mulq 48(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[7] * A[12] + movq 96(%rsi), %rax + mulq 56(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[8] * A[11] + movq 88(%rsi), %rax + mulq 64(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[9] * A[10] + movq 80(%rsi), %rax + mulq 72(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %r10, %r8 + adcq %r11, %r9 + adcq %r12, %rcx + movq %r8, 152(%rdi) + # A[5] * A[15] + movq 120(%rsi), %rax + mulq 40(%rsi) + xorq %r8, %r8 + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[6] * A[14] + movq 112(%rsi), %rax + mulq 48(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[7] * A[13] + movq 104(%rsi), %rax + mulq 56(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[8] * A[12] + movq 96(%rsi), %rax + mulq 64(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[9] * A[11] + movq 88(%rsi), %rax + mulq 72(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[10] * A[10] + movq 80(%rsi), %rax + mulq %rax + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %r9 + adcq %r11, %rcx + adcq %r12, %r8 + movq %r9, 160(%rdi) + # A[6] * A[15] + movq 120(%rsi), %rax + mulq 48(%rsi) + xorq %r9, %r9 + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[7] * A[14] + movq 112(%rsi), %rax + mulq 56(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[8] * A[13] + movq 104(%rsi), %rax + mulq 64(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[9] * A[12] + movq 96(%rsi), %rax + mulq 72(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[10] * A[11] + movq 88(%rsi), %rax + mulq 80(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %r10, %rcx + adcq %r11, %r8 + adcq %r12, %r9 + movq %rcx, 168(%rdi) + # A[7] * A[15] + movq 120(%rsi), %rax + mulq 56(%rsi) + xorq %rcx, %rcx + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[8] * A[14] + movq 112(%rsi), %rax + mulq 64(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[9] * A[13] + movq 104(%rsi), %rax + mulq 72(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[10] * A[12] + movq 96(%rsi), %rax + mulq 80(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[11] * A[11] + movq 88(%rsi), %rax + mulq %rax + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %r8 + adcq %r11, %r9 + adcq %r12, %rcx + movq %r8, 176(%rdi) + # A[8] * A[15] + movq 120(%rsi), %rax + mulq 64(%rsi) + xorq %r8, %r8 + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[9] * A[14] + movq 112(%rsi), %rax + mulq 72(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[10] * A[13] + movq 104(%rsi), %rax + mulq 80(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[11] * A[12] + movq 96(%rsi), %rax + mulq 88(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %r10, %r9 + adcq %r11, %rcx + adcq %r12, %r8 + movq %r9, 184(%rdi) + # A[9] * A[15] + movq 120(%rsi), %rax + mulq 72(%rsi) + xorq %r9, %r9 + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[10] * A[14] + movq 112(%rsi), %rax + mulq 80(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[11] * A[13] + movq 104(%rsi), %rax + mulq 88(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[12] * A[12] + movq 96(%rsi), %rax + mulq %rax + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %rcx + adcq %r11, %r8 + adcq %r12, %r9 + movq %rcx, 192(%rdi) + # A[10] * A[15] + movq 120(%rsi), %rax + mulq 80(%rsi) + xorq %rcx, %rcx + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[11] * A[14] + movq 112(%rsi), %rax + mulq 88(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[12] * A[13] + movq 104(%rsi), %rax + mulq 96(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %r10, %r8 + adcq %r11, %r9 + adcq %r12, %rcx + movq %r8, 200(%rdi) + # A[11] * A[15] + movq 120(%rsi), %rax + mulq 88(%rsi) + xorq %r8, %r8 + addq %rax, %r9 + adcq %rdx, %rcx + adcq $0, %r8 + addq %rax, %r9 + adcq %rdx, %rcx + adcq $0, %r8 + # A[12] * A[14] + movq 112(%rsi), %rax + mulq 96(%rsi) + addq %rax, %r9 + adcq %rdx, %rcx + adcq $0, %r8 + addq %rax, %r9 + adcq %rdx, %rcx + adcq $0, %r8 + # A[13] * A[13] + movq 104(%rsi), %rax + mulq %rax + addq %rax, %r9 + adcq %rdx, %rcx + adcq $0, %r8 + movq %r9, 208(%rdi) + # A[12] * A[15] + movq 120(%rsi), %rax + mulq 96(%rsi) + xorq %r9, %r9 + addq %rax, %rcx + adcq %rdx, %r8 + adcq $0, %r9 + addq %rax, %rcx + adcq %rdx, %r8 + adcq $0, %r9 + # A[13] * A[14] + movq 112(%rsi), %rax + mulq 104(%rsi) + addq %rax, %rcx + adcq %rdx, %r8 + adcq $0, %r9 + addq %rax, %rcx + adcq %rdx, %r8 + adcq $0, %r9 + movq %rcx, 216(%rdi) + # A[13] * A[15] + movq 120(%rsi), %rax + mulq 104(%rsi) + xorq %rcx, %rcx + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %rcx + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %rcx + # A[14] * A[14] + movq 112(%rsi), %rax + mulq %rax + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %rcx + movq %r8, 224(%rdi) + # A[14] * A[15] + movq 120(%rsi), %rax + mulq 112(%rsi) + xorq %r8, %r8 + addq %rax, %r9 + adcq %rdx, %rcx + adcq $0, %r8 + addq %rax, %r9 + adcq %rdx, %rcx + adcq $0, %r8 + movq %r9, 232(%rdi) + # A[15] * A[15] + movq 120(%rsi), %rax + mulq %rax + addq %rax, %rcx + adcq %rdx, %r8 + movq %rcx, 240(%rdi) + movq %r8, 248(%rdi) + movq (%rsp), %rax + movq 8(%rsp), %rdx + movq 16(%rsp), %r10 + movq 24(%rsp), %r11 + movq %rax, (%rdi) + movq %rdx, 8(%rdi) + movq %r10, 16(%rdi) + movq %r11, 24(%rdi) + movq 32(%rsp), %rax + movq 40(%rsp), %rdx + movq 48(%rsp), %r10 + movq 56(%rsp), %r11 + movq %rax, 32(%rdi) + movq %rdx, 40(%rdi) + movq %r10, 48(%rdi) + movq %r11, 56(%rdi) + movq 64(%rsp), %rax + movq 72(%rsp), %rdx + movq 80(%rsp), %r10 + movq 88(%rsp), %r11 + movq %rax, 64(%rdi) + movq %rdx, 72(%rdi) + movq %r10, 80(%rdi) + movq %r11, 88(%rdi) + movq 96(%rsp), %rax + movq 104(%rsp), %rdx + movq 112(%rsp), %r10 + movq 120(%rsp), %r11 + movq %rax, 96(%rdi) + movq %rdx, 104(%rdi) + movq %r10, 112(%rdi) + movq %r11, 120(%rdi) + addq $128, %rsp + pop %r12 + repz retq +.size sp_2048_sqr_16,.-sp_2048_sqr_16 +#ifdef HAVE_INTEL_AVX2 +/* Multiply a and b into r. (r = a * b) + * + * r Result of multiplication. + * a First number to multiply. + * b Second number to multiply. + */ +.globl sp_2048_mul_avx2_16 +.type sp_2048_mul_avx2_16,@function +.align 16 +sp_2048_mul_avx2_16: + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 + push %rbx + movq %rdx, %rbp + subq $128, %rsp + movq $0, %r14 + movq (%rsi), %rdx + # A[0] * B[0] + mulx (%rbp), %r8, %r9 + # A[0] * B[1] + mulx 8(%rbp), %rax, %r10 + movq %r8, (%rsp) + adcxq %rax, %r9 + # A[0] * B[2] + mulx 16(%rbp), %rax, %r11 + movq %r9, 8(%rsp) + adcxq %rax, %r10 + # A[0] * B[3] + mulx 24(%rbp), %rax, %r12 + movq %r10, 16(%rsp) + adcxq %rax, %r11 + movq %r11, 24(%rsp) + # A[0] * B[4] + mulx 32(%rbp), %rax, %r8 + adcxq %rax, %r12 + # A[0] * B[5] + mulx 40(%rbp), %rax, %r9 + movq %r12, 32(%rsp) + adcxq %rax, %r8 + # A[0] * B[6] + mulx 48(%rbp), %rax, %r10 + movq %r8, 40(%rsp) + adcxq %rax, %r9 + # A[0] * B[7] + mulx 56(%rbp), %rax, %r11 + movq %r9, 48(%rsp) + adcxq %rax, %r10 + movq %r10, 56(%rsp) + # A[0] * B[8] + mulx 64(%rbp), %rax, %r12 + adcxq %rax, %r11 + # A[0] * B[9] + mulx 72(%rbp), %rax, %r8 + movq %r11, 64(%rsp) + adcxq %rax, %r12 + # A[0] * B[10] + mulx 80(%rbp), %rax, %r9 + movq %r12, 72(%rsp) + adcxq %rax, %r8 + # A[0] * B[11] + mulx 88(%rbp), %rax, %r10 + movq %r8, 80(%rsp) + adcxq %rax, %r9 + movq %r9, 88(%rsp) + # A[0] * B[12] + mulx 96(%rbp), %rax, %r11 + adcxq %rax, %r10 + # A[0] * B[13] + mulx 104(%rbp), %rax, %r12 + movq %r10, 96(%rsp) + adcxq %rax, %r11 + # A[0] * B[14] + mulx 112(%rbp), %rax, %r8 + movq %r11, 104(%rsp) + adcxq %rax, %r12 + # A[0] * B[15] + mulx 120(%rbp), %rax, %r9 + movq %r12, %r15 + adcxq %rax, %r8 + adcxq %r14, %r9 + movq %r14, %r13 + adcxq %r14, %r13 + movq %r8, %rbx + movq %r9, 128(%rdi) + movq 8(%rsi), %rdx + movq 8(%rsp), %r9 + movq 16(%rsp), %r10 + movq 24(%rsp), %r11 + movq 32(%rsp), %r12 + movq 40(%rsp), %r8 + # A[1] * B[0] + mulx (%rbp), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[1] * B[1] + mulx 8(%rbp), %rax, %rcx + movq %r9, 8(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[1] * B[2] + mulx 16(%rbp), %rax, %rcx + movq %r10, 16(%rsp) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[1] * B[3] + mulx 24(%rbp), %rax, %rcx + movq %r11, 24(%rsp) + adcxq %rax, %r12 + adoxq %rcx, %r8 + movq %r12, 32(%rsp) + movq 48(%rsp), %r9 + movq 56(%rsp), %r10 + movq 64(%rsp), %r11 + movq 72(%rsp), %r12 + # A[1] * B[4] + mulx 32(%rbp), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[1] * B[5] + mulx 40(%rbp), %rax, %rcx + movq %r8, 40(%rsp) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[1] * B[6] + mulx 48(%rbp), %rax, %rcx + movq %r9, 48(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[1] * B[7] + mulx 56(%rbp), %rax, %rcx + movq %r10, 56(%rsp) + adcxq %rax, %r11 + adoxq %rcx, %r12 + movq %r11, 64(%rsp) + movq 80(%rsp), %r8 + movq 88(%rsp), %r9 + movq 96(%rsp), %r10 + movq 104(%rsp), %r11 + # A[1] * B[8] + mulx 64(%rbp), %rax, %rcx + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[1] * B[9] + mulx 72(%rbp), %rax, %rcx + movq %r12, 72(%rsp) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[1] * B[10] + mulx 80(%rbp), %rax, %rcx + movq %r8, 80(%rsp) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[1] * B[11] + mulx 88(%rbp), %rax, %rcx + movq %r9, 88(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r10, 96(%rsp) + movq %r15, %r12 + movq %rbx, %r8 + movq 128(%rdi), %r9 + # A[1] * B[12] + mulx 96(%rbp), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[1] * B[13] + mulx 104(%rbp), %rax, %rcx + movq %r11, 104(%rsp) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[1] * B[14] + mulx 112(%rbp), %rax, %rcx + movq %r12, %r15 + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[1] * B[15] + mulx 120(%rbp), %rax, %rcx + movq %r8, %rbx + movq %r14, %r10 + adcxq %rax, %r9 + adoxq %rcx, %r10 + adcxq %r13, %r10 + movq %r14, %r13 + adoxq %r14, %r13 + adcxq %r14, %r13 + movq %r9, 128(%rdi) + movq %r10, 136(%rdi) + movq 16(%rsi), %rdx + movq 16(%rsp), %r10 + movq 24(%rsp), %r11 + movq 32(%rsp), %r12 + movq 40(%rsp), %r8 + movq 48(%rsp), %r9 + # A[2] * B[0] + mulx (%rbp), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[2] * B[1] + mulx 8(%rbp), %rax, %rcx + movq %r10, 16(%rsp) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[2] * B[2] + mulx 16(%rbp), %rax, %rcx + movq %r11, 24(%rsp) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[2] * B[3] + mulx 24(%rbp), %rax, %rcx + movq %r12, 32(%rsp) + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r8, 40(%rsp) + movq 56(%rsp), %r10 + movq 64(%rsp), %r11 + movq 72(%rsp), %r12 + movq 80(%rsp), %r8 + # A[2] * B[4] + mulx 32(%rbp), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[2] * B[5] + mulx 40(%rbp), %rax, %rcx + movq %r9, 48(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[2] * B[6] + mulx 48(%rbp), %rax, %rcx + movq %r10, 56(%rsp) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[2] * B[7] + mulx 56(%rbp), %rax, %rcx + movq %r11, 64(%rsp) + adcxq %rax, %r12 + adoxq %rcx, %r8 + movq %r12, 72(%rsp) + movq 88(%rsp), %r9 + movq 96(%rsp), %r10 + movq 104(%rsp), %r11 + movq %r15, %r12 + # A[2] * B[8] + mulx 64(%rbp), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[2] * B[9] + mulx 72(%rbp), %rax, %rcx + movq %r8, 80(%rsp) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[2] * B[10] + mulx 80(%rbp), %rax, %rcx + movq %r9, 88(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[2] * B[11] + mulx 88(%rbp), %rax, %rcx + movq %r10, 96(%rsp) + adcxq %rax, %r11 + adoxq %rcx, %r12 + movq %r11, 104(%rsp) + movq %rbx, %r8 + movq 128(%rdi), %r9 + movq 136(%rdi), %r10 + # A[2] * B[12] + mulx 96(%rbp), %rax, %rcx + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[2] * B[13] + mulx 104(%rbp), %rax, %rcx + movq %r12, %r15 + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[2] * B[14] + mulx 112(%rbp), %rax, %rcx + movq %r8, %rbx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[2] * B[15] + mulx 120(%rbp), %rax, %rcx + movq %r9, 128(%rdi) + movq %r14, %r11 + adcxq %rax, %r10 + adoxq %rcx, %r11 + adcxq %r13, %r11 + movq %r14, %r13 + adoxq %r14, %r13 + adcxq %r14, %r13 + movq %r10, 136(%rdi) + movq %r11, 144(%rdi) + movq 24(%rsi), %rdx + movq 24(%rsp), %r11 + movq 32(%rsp), %r12 + movq 40(%rsp), %r8 + movq 48(%rsp), %r9 + movq 56(%rsp), %r10 + # A[3] * B[0] + mulx (%rbp), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[3] * B[1] + mulx 8(%rbp), %rax, %rcx + movq %r11, 24(%rsp) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[3] * B[2] + mulx 16(%rbp), %rax, %rcx + movq %r12, 32(%rsp) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[3] * B[3] + mulx 24(%rbp), %rax, %rcx + movq %r8, 40(%rsp) + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r9, 48(%rsp) + movq 64(%rsp), %r11 + movq 72(%rsp), %r12 + movq 80(%rsp), %r8 + movq 88(%rsp), %r9 + # A[3] * B[4] + mulx 32(%rbp), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[3] * B[5] + mulx 40(%rbp), %rax, %rcx + movq %r10, 56(%rsp) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[3] * B[6] + mulx 48(%rbp), %rax, %rcx + movq %r11, 64(%rsp) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[3] * B[7] + mulx 56(%rbp), %rax, %rcx + movq %r12, 72(%rsp) + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r8, 80(%rsp) + movq 96(%rsp), %r10 + movq 104(%rsp), %r11 + movq %r15, %r12 + movq %rbx, %r8 + # A[3] * B[8] + mulx 64(%rbp), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[3] * B[9] + mulx 72(%rbp), %rax, %rcx + movq %r9, 88(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[3] * B[10] + mulx 80(%rbp), %rax, %rcx + movq %r10, 96(%rsp) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[3] * B[11] + mulx 88(%rbp), %rax, %rcx + movq %r11, 104(%rsp) + adcxq %rax, %r12 + adoxq %rcx, %r8 + movq %r12, %r15 + movq 128(%rdi), %r9 + movq 136(%rdi), %r10 + movq 144(%rdi), %r11 + # A[3] * B[12] + mulx 96(%rbp), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[3] * B[13] + mulx 104(%rbp), %rax, %rcx + movq %r8, %rbx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[3] * B[14] + mulx 112(%rbp), %rax, %rcx + movq %r9, 128(%rdi) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[3] * B[15] + mulx 120(%rbp), %rax, %rcx + movq %r10, 136(%rdi) + movq %r14, %r12 + adcxq %rax, %r11 + adoxq %rcx, %r12 + adcxq %r13, %r12 + movq %r14, %r13 + adoxq %r14, %r13 + adcxq %r14, %r13 + movq %r11, 144(%rdi) + movq %r12, 152(%rdi) + movq 32(%rsi), %rdx + movq 32(%rsp), %r12 + movq 40(%rsp), %r8 + movq 48(%rsp), %r9 + movq 56(%rsp), %r10 + movq 64(%rsp), %r11 + # A[4] * B[0] + mulx (%rbp), %rax, %rcx + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[4] * B[1] + mulx 8(%rbp), %rax, %rcx + movq %r12, 32(%rsp) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[4] * B[2] + mulx 16(%rbp), %rax, %rcx + movq %r8, 40(%rsp) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[4] * B[3] + mulx 24(%rbp), %rax, %rcx + movq %r9, 48(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r10, 56(%rsp) + movq 72(%rsp), %r12 + movq 80(%rsp), %r8 + movq 88(%rsp), %r9 + movq 96(%rsp), %r10 + # A[4] * B[4] + mulx 32(%rbp), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[4] * B[5] + mulx 40(%rbp), %rax, %rcx + movq %r11, 64(%rsp) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[4] * B[6] + mulx 48(%rbp), %rax, %rcx + movq %r12, 72(%rsp) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[4] * B[7] + mulx 56(%rbp), %rax, %rcx + movq %r8, 80(%rsp) + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r9, 88(%rsp) + movq 104(%rsp), %r11 + movq %r15, %r12 + movq %rbx, %r8 + movq 128(%rdi), %r9 + # A[4] * B[8] + mulx 64(%rbp), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[4] * B[9] + mulx 72(%rbp), %rax, %rcx + movq %r10, 96(%rsp) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[4] * B[10] + mulx 80(%rbp), %rax, %rcx + movq %r11, 104(%rsp) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[4] * B[11] + mulx 88(%rbp), %rax, %rcx + movq %r12, %r15 + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r8, %rbx + movq 136(%rdi), %r10 + movq 144(%rdi), %r11 + movq 152(%rdi), %r12 + # A[4] * B[12] + mulx 96(%rbp), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[4] * B[13] + mulx 104(%rbp), %rax, %rcx + movq %r9, 128(%rdi) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[4] * B[14] + mulx 112(%rbp), %rax, %rcx + movq %r10, 136(%rdi) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[4] * B[15] + mulx 120(%rbp), %rax, %rcx + movq %r11, 144(%rdi) + movq %r14, %r8 + adcxq %rax, %r12 + adoxq %rcx, %r8 + adcxq %r13, %r8 + movq %r14, %r13 + adoxq %r14, %r13 + adcxq %r14, %r13 + movq %r12, 152(%rdi) + movq %r8, 160(%rdi) + movq 40(%rsi), %rdx + movq 40(%rsp), %r8 + movq 48(%rsp), %r9 + movq 56(%rsp), %r10 + movq 64(%rsp), %r11 + movq 72(%rsp), %r12 + # A[5] * B[0] + mulx (%rbp), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[5] * B[1] + mulx 8(%rbp), %rax, %rcx + movq %r8, 40(%rsp) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[5] * B[2] + mulx 16(%rbp), %rax, %rcx + movq %r9, 48(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[5] * B[3] + mulx 24(%rbp), %rax, %rcx + movq %r10, 56(%rsp) + adcxq %rax, %r11 + adoxq %rcx, %r12 + movq %r11, 64(%rsp) + movq 80(%rsp), %r8 + movq 88(%rsp), %r9 + movq 96(%rsp), %r10 + movq 104(%rsp), %r11 + # A[5] * B[4] + mulx 32(%rbp), %rax, %rcx + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[5] * B[5] + mulx 40(%rbp), %rax, %rcx + movq %r12, 72(%rsp) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[5] * B[6] + mulx 48(%rbp), %rax, %rcx + movq %r8, 80(%rsp) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[5] * B[7] + mulx 56(%rbp), %rax, %rcx + movq %r9, 88(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r10, 96(%rsp) + movq %r15, %r12 + movq %rbx, %r8 + movq 128(%rdi), %r9 + movq 136(%rdi), %r10 + # A[5] * B[8] + mulx 64(%rbp), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[5] * B[9] + mulx 72(%rbp), %rax, %rcx + movq %r11, 104(%rsp) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[5] * B[10] + mulx 80(%rbp), %rax, %rcx + movq %r12, %r15 + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[5] * B[11] + mulx 88(%rbp), %rax, %rcx + movq %r8, %rbx + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r9, 128(%rdi) + movq 144(%rdi), %r11 + movq 152(%rdi), %r12 + movq 160(%rdi), %r8 + # A[5] * B[12] + mulx 96(%rbp), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[5] * B[13] + mulx 104(%rbp), %rax, %rcx + movq %r10, 136(%rdi) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[5] * B[14] + mulx 112(%rbp), %rax, %rcx + movq %r11, 144(%rdi) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[5] * B[15] + mulx 120(%rbp), %rax, %rcx + movq %r12, 152(%rdi) + movq %r14, %r9 + adcxq %rax, %r8 + adoxq %rcx, %r9 + adcxq %r13, %r9 + movq %r14, %r13 + adoxq %r14, %r13 + adcxq %r14, %r13 + movq %r8, 160(%rdi) + movq %r9, 168(%rdi) + movq 48(%rsi), %rdx + movq 48(%rsp), %r9 + movq 56(%rsp), %r10 + movq 64(%rsp), %r11 + movq 72(%rsp), %r12 + movq 80(%rsp), %r8 + # A[6] * B[0] + mulx (%rbp), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[6] * B[1] + mulx 8(%rbp), %rax, %rcx + movq %r9, 48(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[6] * B[2] + mulx 16(%rbp), %rax, %rcx + movq %r10, 56(%rsp) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[6] * B[3] + mulx 24(%rbp), %rax, %rcx + movq %r11, 64(%rsp) + adcxq %rax, %r12 + adoxq %rcx, %r8 + movq %r12, 72(%rsp) + movq 88(%rsp), %r9 + movq 96(%rsp), %r10 + movq 104(%rsp), %r11 + movq %r15, %r12 + # A[6] * B[4] + mulx 32(%rbp), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[6] * B[5] + mulx 40(%rbp), %rax, %rcx + movq %r8, 80(%rsp) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[6] * B[6] + mulx 48(%rbp), %rax, %rcx + movq %r9, 88(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[6] * B[7] + mulx 56(%rbp), %rax, %rcx + movq %r10, 96(%rsp) + adcxq %rax, %r11 + adoxq %rcx, %r12 + movq %r11, 104(%rsp) + movq %rbx, %r8 + movq 128(%rdi), %r9 + movq 136(%rdi), %r10 + movq 144(%rdi), %r11 + # A[6] * B[8] + mulx 64(%rbp), %rax, %rcx + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[6] * B[9] + mulx 72(%rbp), %rax, %rcx + movq %r12, %r15 + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[6] * B[10] + mulx 80(%rbp), %rax, %rcx + movq %r8, %rbx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[6] * B[11] + mulx 88(%rbp), %rax, %rcx + movq %r9, 128(%rdi) + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r10, 136(%rdi) + movq 152(%rdi), %r12 + movq 160(%rdi), %r8 + movq 168(%rdi), %r9 + # A[6] * B[12] + mulx 96(%rbp), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[6] * B[13] + mulx 104(%rbp), %rax, %rcx + movq %r11, 144(%rdi) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[6] * B[14] + mulx 112(%rbp), %rax, %rcx + movq %r12, 152(%rdi) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[6] * B[15] + mulx 120(%rbp), %rax, %rcx + movq %r8, 160(%rdi) + movq %r14, %r10 + adcxq %rax, %r9 + adoxq %rcx, %r10 + adcxq %r13, %r10 + movq %r14, %r13 + adoxq %r14, %r13 + adcxq %r14, %r13 + movq %r9, 168(%rdi) + movq %r10, 176(%rdi) + movq 56(%rsi), %rdx + movq 56(%rsp), %r10 + movq 64(%rsp), %r11 + movq 72(%rsp), %r12 + movq 80(%rsp), %r8 + movq 88(%rsp), %r9 + # A[7] * B[0] + mulx (%rbp), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[7] * B[1] + mulx 8(%rbp), %rax, %rcx + movq %r10, 56(%rsp) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[7] * B[2] + mulx 16(%rbp), %rax, %rcx + movq %r11, 64(%rsp) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[7] * B[3] + mulx 24(%rbp), %rax, %rcx + movq %r12, 72(%rsp) + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r8, 80(%rsp) + movq 96(%rsp), %r10 + movq 104(%rsp), %r11 + movq %r15, %r12 + movq %rbx, %r8 + # A[7] * B[4] + mulx 32(%rbp), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[7] * B[5] + mulx 40(%rbp), %rax, %rcx + movq %r9, 88(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[7] * B[6] + mulx 48(%rbp), %rax, %rcx + movq %r10, 96(%rsp) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[7] * B[7] + mulx 56(%rbp), %rax, %rcx + movq %r11, 104(%rsp) + adcxq %rax, %r12 + adoxq %rcx, %r8 + movq %r12, %r15 + movq 128(%rdi), %r9 + movq 136(%rdi), %r10 + movq 144(%rdi), %r11 + movq 152(%rdi), %r12 + # A[7] * B[8] + mulx 64(%rbp), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[7] * B[9] + mulx 72(%rbp), %rax, %rcx + movq %r8, %rbx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[7] * B[10] + mulx 80(%rbp), %rax, %rcx + movq %r9, 128(%rdi) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[7] * B[11] + mulx 88(%rbp), %rax, %rcx + movq %r10, 136(%rdi) + adcxq %rax, %r11 + adoxq %rcx, %r12 + movq %r11, 144(%rdi) + movq 160(%rdi), %r8 + movq 168(%rdi), %r9 + movq 176(%rdi), %r10 + # A[7] * B[12] + mulx 96(%rbp), %rax, %rcx + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[7] * B[13] + mulx 104(%rbp), %rax, %rcx + movq %r12, 152(%rdi) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[7] * B[14] + mulx 112(%rbp), %rax, %rcx + movq %r8, 160(%rdi) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[7] * B[15] + mulx 120(%rbp), %rax, %rcx + movq %r9, 168(%rdi) + movq %r14, %r11 + adcxq %rax, %r10 + adoxq %rcx, %r11 + adcxq %r13, %r11 + movq %r14, %r13 + adoxq %r14, %r13 + adcxq %r14, %r13 + movq %r10, 176(%rdi) + movq %r11, 184(%rdi) + movq 64(%rsi), %rdx + movq 64(%rsp), %r11 + movq 72(%rsp), %r12 + movq 80(%rsp), %r8 + movq 88(%rsp), %r9 + movq 96(%rsp), %r10 + # A[8] * B[0] + mulx (%rbp), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[8] * B[1] + mulx 8(%rbp), %rax, %rcx + movq %r11, 64(%rsp) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[8] * B[2] + mulx 16(%rbp), %rax, %rcx + movq %r12, 72(%rsp) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[8] * B[3] + mulx 24(%rbp), %rax, %rcx + movq %r8, 80(%rsp) + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r9, 88(%rsp) + movq 104(%rsp), %r11 + movq %r15, %r12 + movq %rbx, %r8 + movq 128(%rdi), %r9 + # A[8] * B[4] + mulx 32(%rbp), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[8] * B[5] + mulx 40(%rbp), %rax, %rcx + movq %r10, 96(%rsp) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[8] * B[6] + mulx 48(%rbp), %rax, %rcx + movq %r11, 104(%rsp) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[8] * B[7] + mulx 56(%rbp), %rax, %rcx + movq %r12, %r15 + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r8, %rbx + movq 136(%rdi), %r10 + movq 144(%rdi), %r11 + movq 152(%rdi), %r12 + movq 160(%rdi), %r8 + # A[8] * B[8] + mulx 64(%rbp), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[8] * B[9] + mulx 72(%rbp), %rax, %rcx + movq %r9, 128(%rdi) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[8] * B[10] + mulx 80(%rbp), %rax, %rcx + movq %r10, 136(%rdi) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[8] * B[11] + mulx 88(%rbp), %rax, %rcx + movq %r11, 144(%rdi) + adcxq %rax, %r12 + adoxq %rcx, %r8 + movq %r12, 152(%rdi) + movq 168(%rdi), %r9 + movq 176(%rdi), %r10 + movq 184(%rdi), %r11 + # A[8] * B[12] + mulx 96(%rbp), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[8] * B[13] + mulx 104(%rbp), %rax, %rcx + movq %r8, 160(%rdi) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[8] * B[14] + mulx 112(%rbp), %rax, %rcx + movq %r9, 168(%rdi) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[8] * B[15] + mulx 120(%rbp), %rax, %rcx + movq %r10, 176(%rdi) + movq %r14, %r12 + adcxq %rax, %r11 + adoxq %rcx, %r12 + adcxq %r13, %r12 + movq %r14, %r13 + adoxq %r14, %r13 + adcxq %r14, %r13 + movq %r11, 184(%rdi) + movq %r12, 192(%rdi) + movq 72(%rsi), %rdx + movq 72(%rsp), %r12 + movq 80(%rsp), %r8 + movq 88(%rsp), %r9 + movq 96(%rsp), %r10 + movq 104(%rsp), %r11 + # A[9] * B[0] + mulx (%rbp), %rax, %rcx + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[9] * B[1] + mulx 8(%rbp), %rax, %rcx + movq %r12, 72(%rsp) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[9] * B[2] + mulx 16(%rbp), %rax, %rcx + movq %r8, 80(%rsp) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[9] * B[3] + mulx 24(%rbp), %rax, %rcx + movq %r9, 88(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r10, 96(%rsp) + movq %r15, %r12 + movq %rbx, %r8 + movq 128(%rdi), %r9 + movq 136(%rdi), %r10 + # A[9] * B[4] + mulx 32(%rbp), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[9] * B[5] + mulx 40(%rbp), %rax, %rcx + movq %r11, 104(%rsp) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[9] * B[6] + mulx 48(%rbp), %rax, %rcx + movq %r12, %r15 + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[9] * B[7] + mulx 56(%rbp), %rax, %rcx + movq %r8, %rbx + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r9, 128(%rdi) + movq 144(%rdi), %r11 + movq 152(%rdi), %r12 + movq 160(%rdi), %r8 + movq 168(%rdi), %r9 + # A[9] * B[8] + mulx 64(%rbp), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[9] * B[9] + mulx 72(%rbp), %rax, %rcx + movq %r10, 136(%rdi) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[9] * B[10] + mulx 80(%rbp), %rax, %rcx + movq %r11, 144(%rdi) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[9] * B[11] + mulx 88(%rbp), %rax, %rcx + movq %r12, 152(%rdi) + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r8, 160(%rdi) + movq 176(%rdi), %r10 + movq 184(%rdi), %r11 + movq 192(%rdi), %r12 + # A[9] * B[12] + mulx 96(%rbp), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[9] * B[13] + mulx 104(%rbp), %rax, %rcx + movq %r9, 168(%rdi) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[9] * B[14] + mulx 112(%rbp), %rax, %rcx + movq %r10, 176(%rdi) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[9] * B[15] + mulx 120(%rbp), %rax, %rcx + movq %r11, 184(%rdi) + movq %r14, %r8 + adcxq %rax, %r12 + adoxq %rcx, %r8 + adcxq %r13, %r8 + movq %r14, %r13 + adoxq %r14, %r13 + adcxq %r14, %r13 + movq %r12, 192(%rdi) + movq %r8, 200(%rdi) + movq 80(%rsi), %rdx + movq 80(%rsp), %r8 + movq 88(%rsp), %r9 + movq 96(%rsp), %r10 + movq 104(%rsp), %r11 + movq %r15, %r12 + # A[10] * B[0] + mulx (%rbp), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[10] * B[1] + mulx 8(%rbp), %rax, %rcx + movq %r8, 80(%rsp) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[10] * B[2] + mulx 16(%rbp), %rax, %rcx + movq %r9, 88(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[10] * B[3] + mulx 24(%rbp), %rax, %rcx + movq %r10, 96(%rsp) + adcxq %rax, %r11 + adoxq %rcx, %r12 + movq %r11, 104(%rsp) + movq %rbx, %r8 + movq 128(%rdi), %r9 + movq 136(%rdi), %r10 + movq 144(%rdi), %r11 + # A[10] * B[4] + mulx 32(%rbp), %rax, %rcx + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[10] * B[5] + mulx 40(%rbp), %rax, %rcx + movq %r12, %r15 + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[10] * B[6] + mulx 48(%rbp), %rax, %rcx + movq %r8, %rbx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[10] * B[7] + mulx 56(%rbp), %rax, %rcx + movq %r9, 128(%rdi) + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r10, 136(%rdi) + movq 152(%rdi), %r12 + movq 160(%rdi), %r8 + movq 168(%rdi), %r9 + movq 176(%rdi), %r10 + # A[10] * B[8] + mulx 64(%rbp), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[10] * B[9] + mulx 72(%rbp), %rax, %rcx + movq %r11, 144(%rdi) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[10] * B[10] + mulx 80(%rbp), %rax, %rcx + movq %r12, 152(%rdi) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[10] * B[11] + mulx 88(%rbp), %rax, %rcx + movq %r8, 160(%rdi) + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r9, 168(%rdi) + movq 184(%rdi), %r11 + movq 192(%rdi), %r12 + movq 200(%rdi), %r8 + # A[10] * B[12] + mulx 96(%rbp), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[10] * B[13] + mulx 104(%rbp), %rax, %rcx + movq %r10, 176(%rdi) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[10] * B[14] + mulx 112(%rbp), %rax, %rcx + movq %r11, 184(%rdi) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[10] * B[15] + mulx 120(%rbp), %rax, %rcx + movq %r12, 192(%rdi) + movq %r14, %r9 + adcxq %rax, %r8 + adoxq %rcx, %r9 + adcxq %r13, %r9 + movq %r14, %r13 + adoxq %r14, %r13 + adcxq %r14, %r13 + movq %r8, 200(%rdi) + movq %r9, 208(%rdi) + movq 88(%rsi), %rdx + movq 88(%rsp), %r9 + movq 96(%rsp), %r10 + movq 104(%rsp), %r11 + movq %r15, %r12 + movq %rbx, %r8 + # A[11] * B[0] + mulx (%rbp), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[11] * B[1] + mulx 8(%rbp), %rax, %rcx + movq %r9, 88(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[11] * B[2] + mulx 16(%rbp), %rax, %rcx + movq %r10, 96(%rsp) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[11] * B[3] + mulx 24(%rbp), %rax, %rcx + movq %r11, 104(%rsp) + adcxq %rax, %r12 + adoxq %rcx, %r8 + movq %r12, %r15 + movq 128(%rdi), %r9 + movq 136(%rdi), %r10 + movq 144(%rdi), %r11 + movq 152(%rdi), %r12 + # A[11] * B[4] + mulx 32(%rbp), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[11] * B[5] + mulx 40(%rbp), %rax, %rcx + movq %r8, %rbx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[11] * B[6] + mulx 48(%rbp), %rax, %rcx + movq %r9, 128(%rdi) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[11] * B[7] + mulx 56(%rbp), %rax, %rcx + movq %r10, 136(%rdi) + adcxq %rax, %r11 + adoxq %rcx, %r12 + movq %r11, 144(%rdi) + movq 160(%rdi), %r8 + movq 168(%rdi), %r9 + movq 176(%rdi), %r10 + movq 184(%rdi), %r11 + # A[11] * B[8] + mulx 64(%rbp), %rax, %rcx + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[11] * B[9] + mulx 72(%rbp), %rax, %rcx + movq %r12, 152(%rdi) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[11] * B[10] + mulx 80(%rbp), %rax, %rcx + movq %r8, 160(%rdi) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[11] * B[11] + mulx 88(%rbp), %rax, %rcx + movq %r9, 168(%rdi) + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r10, 176(%rdi) + movq 192(%rdi), %r12 + movq 200(%rdi), %r8 + movq 208(%rdi), %r9 + # A[11] * B[12] + mulx 96(%rbp), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[11] * B[13] + mulx 104(%rbp), %rax, %rcx + movq %r11, 184(%rdi) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[11] * B[14] + mulx 112(%rbp), %rax, %rcx + movq %r12, 192(%rdi) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[11] * B[15] + mulx 120(%rbp), %rax, %rcx + movq %r8, 200(%rdi) + movq %r14, %r10 + adcxq %rax, %r9 + adoxq %rcx, %r10 + adcxq %r13, %r10 + movq %r14, %r13 + adoxq %r14, %r13 + adcxq %r14, %r13 + movq %r9, 208(%rdi) + movq %r10, 216(%rdi) + movq 96(%rsi), %rdx + movq 96(%rsp), %r10 + movq 104(%rsp), %r11 + movq %r15, %r12 + movq %rbx, %r8 + movq 128(%rdi), %r9 + # A[12] * B[0] + mulx (%rbp), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[12] * B[1] + mulx 8(%rbp), %rax, %rcx + movq %r10, 96(%rsp) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[12] * B[2] + mulx 16(%rbp), %rax, %rcx + movq %r11, 104(%rsp) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[12] * B[3] + mulx 24(%rbp), %rax, %rcx + movq %r12, %r15 + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r8, %rbx + movq 136(%rdi), %r10 + movq 144(%rdi), %r11 + movq 152(%rdi), %r12 + movq 160(%rdi), %r8 + # A[12] * B[4] + mulx 32(%rbp), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[12] * B[5] + mulx 40(%rbp), %rax, %rcx + movq %r9, 128(%rdi) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[12] * B[6] + mulx 48(%rbp), %rax, %rcx + movq %r10, 136(%rdi) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[12] * B[7] + mulx 56(%rbp), %rax, %rcx + movq %r11, 144(%rdi) + adcxq %rax, %r12 + adoxq %rcx, %r8 + movq %r12, 152(%rdi) + movq 168(%rdi), %r9 + movq 176(%rdi), %r10 + movq 184(%rdi), %r11 + movq 192(%rdi), %r12 + # A[12] * B[8] + mulx 64(%rbp), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[12] * B[9] + mulx 72(%rbp), %rax, %rcx + movq %r8, 160(%rdi) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[12] * B[10] + mulx 80(%rbp), %rax, %rcx + movq %r9, 168(%rdi) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[12] * B[11] + mulx 88(%rbp), %rax, %rcx + movq %r10, 176(%rdi) + adcxq %rax, %r11 + adoxq %rcx, %r12 + movq %r11, 184(%rdi) + movq 200(%rdi), %r8 + movq 208(%rdi), %r9 + movq 216(%rdi), %r10 + # A[12] * B[12] + mulx 96(%rbp), %rax, %rcx + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[12] * B[13] + mulx 104(%rbp), %rax, %rcx + movq %r12, 192(%rdi) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[12] * B[14] + mulx 112(%rbp), %rax, %rcx + movq %r8, 200(%rdi) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[12] * B[15] + mulx 120(%rbp), %rax, %rcx + movq %r9, 208(%rdi) + movq %r14, %r11 + adcxq %rax, %r10 + adoxq %rcx, %r11 + adcxq %r13, %r11 + movq %r14, %r13 + adoxq %r14, %r13 + adcxq %r14, %r13 + movq %r10, 216(%rdi) + movq %r11, 224(%rdi) + movq 104(%rsi), %rdx + movq 104(%rsp), %r11 + movq %r15, %r12 + movq %rbx, %r8 + movq 128(%rdi), %r9 + movq 136(%rdi), %r10 + # A[13] * B[0] + mulx (%rbp), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[13] * B[1] + mulx 8(%rbp), %rax, %rcx + movq %r11, 104(%rsp) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[13] * B[2] + mulx 16(%rbp), %rax, %rcx + movq %r12, %r15 + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[13] * B[3] + mulx 24(%rbp), %rax, %rcx + movq %r8, %rbx + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r9, 128(%rdi) + movq 144(%rdi), %r11 + movq 152(%rdi), %r12 + movq 160(%rdi), %r8 + movq 168(%rdi), %r9 + # A[13] * B[4] + mulx 32(%rbp), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[13] * B[5] + mulx 40(%rbp), %rax, %rcx + movq %r10, 136(%rdi) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[13] * B[6] + mulx 48(%rbp), %rax, %rcx + movq %r11, 144(%rdi) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[13] * B[7] + mulx 56(%rbp), %rax, %rcx + movq %r12, 152(%rdi) + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r8, 160(%rdi) + movq 176(%rdi), %r10 + movq 184(%rdi), %r11 + movq 192(%rdi), %r12 + movq 200(%rdi), %r8 + # A[13] * B[8] + mulx 64(%rbp), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[13] * B[9] + mulx 72(%rbp), %rax, %rcx + movq %r9, 168(%rdi) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[13] * B[10] + mulx 80(%rbp), %rax, %rcx + movq %r10, 176(%rdi) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[13] * B[11] + mulx 88(%rbp), %rax, %rcx + movq %r11, 184(%rdi) + adcxq %rax, %r12 + adoxq %rcx, %r8 + movq %r12, 192(%rdi) + movq 208(%rdi), %r9 + movq 216(%rdi), %r10 + movq 224(%rdi), %r11 + # A[13] * B[12] + mulx 96(%rbp), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[13] * B[13] + mulx 104(%rbp), %rax, %rcx + movq %r8, 200(%rdi) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[13] * B[14] + mulx 112(%rbp), %rax, %rcx + movq %r9, 208(%rdi) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[13] * B[15] + mulx 120(%rbp), %rax, %rcx + movq %r10, 216(%rdi) + movq %r14, %r12 + adcxq %rax, %r11 + adoxq %rcx, %r12 + adcxq %r13, %r12 + movq %r14, %r13 + adoxq %r14, %r13 + adcxq %r14, %r13 + movq %r11, 224(%rdi) + movq %r12, 232(%rdi) + movq 112(%rsi), %rdx + movq %r15, %r12 + movq %rbx, %r8 + movq 128(%rdi), %r9 + movq 136(%rdi), %r10 + movq 144(%rdi), %r11 + # A[14] * B[0] + mulx (%rbp), %rax, %rcx + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[14] * B[1] + mulx 8(%rbp), %rax, %rcx + movq %r12, %r15 + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[14] * B[2] + mulx 16(%rbp), %rax, %rcx + movq %r8, %rbx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[14] * B[3] + mulx 24(%rbp), %rax, %rcx + movq %r9, 128(%rdi) + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r10, 136(%rdi) + movq 152(%rdi), %r12 + movq 160(%rdi), %r8 + movq 168(%rdi), %r9 + movq 176(%rdi), %r10 + # A[14] * B[4] + mulx 32(%rbp), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[14] * B[5] + mulx 40(%rbp), %rax, %rcx + movq %r11, 144(%rdi) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[14] * B[6] + mulx 48(%rbp), %rax, %rcx + movq %r12, 152(%rdi) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[14] * B[7] + mulx 56(%rbp), %rax, %rcx + movq %r8, 160(%rdi) + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r9, 168(%rdi) + movq 184(%rdi), %r11 + movq 192(%rdi), %r12 + movq 200(%rdi), %r8 + movq 208(%rdi), %r9 + # A[14] * B[8] + mulx 64(%rbp), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[14] * B[9] + mulx 72(%rbp), %rax, %rcx + movq %r10, 176(%rdi) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[14] * B[10] + mulx 80(%rbp), %rax, %rcx + movq %r11, 184(%rdi) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[14] * B[11] + mulx 88(%rbp), %rax, %rcx + movq %r12, 192(%rdi) + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r8, 200(%rdi) + movq 216(%rdi), %r10 + movq 224(%rdi), %r11 + movq 232(%rdi), %r12 + # A[14] * B[12] + mulx 96(%rbp), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[14] * B[13] + mulx 104(%rbp), %rax, %rcx + movq %r9, 208(%rdi) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[14] * B[14] + mulx 112(%rbp), %rax, %rcx + movq %r10, 216(%rdi) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[14] * B[15] + mulx 120(%rbp), %rax, %rcx + movq %r11, 224(%rdi) + movq %r14, %r8 + adcxq %rax, %r12 + adoxq %rcx, %r8 + adcxq %r13, %r8 + movq %r14, %r13 + adoxq %r14, %r13 + adcxq %r14, %r13 + movq %r12, 232(%rdi) + movq %r8, 240(%rdi) + movq 120(%rsi), %rdx + movq %rbx, %r8 + movq 128(%rdi), %r9 + movq 136(%rdi), %r10 + movq 144(%rdi), %r11 + movq 152(%rdi), %r12 + # A[15] * B[0] + mulx (%rbp), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[15] * B[1] + mulx 8(%rbp), %rax, %rcx + movq %r8, %rbx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[15] * B[2] + mulx 16(%rbp), %rax, %rcx + movq %r9, 128(%rdi) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[15] * B[3] + mulx 24(%rbp), %rax, %rcx + movq %r10, 136(%rdi) + adcxq %rax, %r11 + adoxq %rcx, %r12 + movq %r11, 144(%rdi) + movq 160(%rdi), %r8 + movq 168(%rdi), %r9 + movq 176(%rdi), %r10 + movq 184(%rdi), %r11 + # A[15] * B[4] + mulx 32(%rbp), %rax, %rcx + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[15] * B[5] + mulx 40(%rbp), %rax, %rcx + movq %r12, 152(%rdi) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[15] * B[6] + mulx 48(%rbp), %rax, %rcx + movq %r8, 160(%rdi) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[15] * B[7] + mulx 56(%rbp), %rax, %rcx + movq %r9, 168(%rdi) + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r10, 176(%rdi) + movq 192(%rdi), %r12 + movq 200(%rdi), %r8 + movq 208(%rdi), %r9 + movq 216(%rdi), %r10 + # A[15] * B[8] + mulx 64(%rbp), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[15] * B[9] + mulx 72(%rbp), %rax, %rcx + movq %r11, 184(%rdi) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[15] * B[10] + mulx 80(%rbp), %rax, %rcx + movq %r12, 192(%rdi) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[15] * B[11] + mulx 88(%rbp), %rax, %rcx + movq %r8, 200(%rdi) + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r9, 208(%rdi) + movq 224(%rdi), %r11 + movq 232(%rdi), %r12 + movq 240(%rdi), %r8 + # A[15] * B[12] + mulx 96(%rbp), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[15] * B[13] + mulx 104(%rbp), %rax, %rcx + movq %r10, 216(%rdi) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[15] * B[14] + mulx 112(%rbp), %rax, %rcx + movq %r11, 224(%rdi) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[15] * B[15] + mulx 120(%rbp), %rax, %rcx + movq %r12, 232(%rdi) + movq %r14, %r9 + adcxq %rax, %r8 + adoxq %rcx, %r9 + adcxq %r13, %r9 + movq %r8, 240(%rdi) + movq %r9, 248(%rdi) + movq (%rsp), %r8 + movq 8(%rsp), %r9 + movq 16(%rsp), %r10 + movq 24(%rsp), %r11 + movq %r8, (%rdi) + movq %r9, 8(%rdi) + movq %r10, 16(%rdi) + movq %r11, 24(%rdi) + movq 32(%rsp), %r8 + movq 40(%rsp), %r9 + movq 48(%rsp), %r10 + movq 56(%rsp), %r11 + movq %r8, 32(%rdi) + movq %r9, 40(%rdi) + movq %r10, 48(%rdi) + movq %r11, 56(%rdi) + movq 64(%rsp), %r8 + movq 72(%rsp), %r9 + movq 80(%rsp), %r10 + movq 88(%rsp), %r11 + movq %r8, 64(%rdi) + movq %r9, 72(%rdi) + movq %r10, 80(%rdi) + movq %r11, 88(%rdi) + movq 96(%rsp), %r8 + movq 104(%rsp), %r9 + movq %r15, %r10 + movq %rbx, %r11 + movq %r8, 96(%rdi) + movq %r9, 104(%rdi) + movq %r10, 112(%rdi) + movq %r11, 120(%rdi) + addq $128, %rsp + pop %rbx + pop %r15 + pop %r14 + pop %r13 + pop %r12 + pop %rbp + repz retq +.size sp_2048_mul_avx2_16,.-sp_2048_mul_avx2_16 +#endif /* HAVE_INTEL_AVX2 */ +#ifdef HAVE_INTEL_AVX2 +/* Square a and put result in r. (r = a * a) + * + * r A single precision integer. + * a A single precision integer. + */ +.globl sp_2048_sqr_avx2_16 +.type sp_2048_sqr_avx2_16,@function +.align 16 +sp_2048_sqr_avx2_16: + push %rbp + push %r12 + push %r13 + subq $128, %rsp + cmpq %rdi, %rsi + movq %rsp, %rbp + cmovne %rdi, %rbp + xorq %r12, %r12 + # Diagonal 1 + xorq %r8, %r8 + xorq %r9, %r9 + xorq %r10, %r10 + xorq %r11, %r11 + # A[1] x A[0] + movq (%rsi), %rdx + mulxq 8(%rsi), %r8, %r9 + # A[2] x A[0] + mulxq 16(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[3] x A[0] + mulxq 24(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r8, 8(%rbp) + movq %r9, 16(%rbp) + movq %r10, 24(%rbp) + movq %r12, %r8 + movq %r12, %r9 + movq %r12, %r10 + # A[4] x A[0] + mulxq 32(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[5] x A[0] + mulxq 40(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[6] x A[0] + mulxq 48(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r11, 32(%rbp) + movq %r8, 40(%rbp) + movq %r9, 48(%rbp) + movq %r12, %r11 + movq %r12, %r8 + movq %r12, %r9 + # A[7] x A[0] + mulxq 56(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[8] x A[0] + mulxq 64(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[9] x A[0] + mulxq 72(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r10, 56(%rbp) + movq %r11, 64(%rbp) + movq %r8, 72(%rbp) + movq %r12, %r10 + movq %r12, %r11 + movq %r12, %r8 + # A[10] x A[0] + mulxq 80(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[11] x A[0] + mulxq 88(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[12] x A[0] + mulxq 96(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + movq %r9, 80(%rbp) + movq %r10, 88(%rbp) + movq %r11, 96(%rbp) + movq %r12, %r9 + movq %r12, %r10 + movq %r12, %r11 + # A[13] x A[0] + mulxq 104(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[14] x A[0] + mulxq 112(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[15] x A[0] + mulxq 120(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r8, 104(%rbp) + movq %r9, 112(%rbp) + movq %r10, 120(%rbp) + # Carry + adcxq %r12, %r11 + movq %r12, %r13 + adcxq %r12, %r13 + adoxq %r12, %r13 + movq %r11, 128(%rdi) + # Diagonal 2 + movq 24(%rbp), %r11 + movq 32(%rbp), %r8 + movq 40(%rbp), %r9 + movq 48(%rbp), %r10 + # A[2] x A[1] + movq 8(%rsi), %rdx + mulxq 16(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[3] x A[1] + mulxq 24(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[4] x A[1] + mulxq 32(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r11, 24(%rbp) + movq %r8, 32(%rbp) + movq %r9, 40(%rbp) + movq 56(%rbp), %r11 + movq 64(%rbp), %r8 + movq 72(%rbp), %r9 + # A[5] x A[1] + mulxq 40(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[6] x A[1] + mulxq 48(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[7] x A[1] + mulxq 56(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r10, 48(%rbp) + movq %r11, 56(%rbp) + movq %r8, 64(%rbp) + movq 80(%rbp), %r10 + movq 88(%rbp), %r11 + movq 96(%rbp), %r8 + # A[8] x A[1] + mulxq 64(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[9] x A[1] + mulxq 72(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[10] x A[1] + mulxq 80(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + movq %r9, 72(%rbp) + movq %r10, 80(%rbp) + movq %r11, 88(%rbp) + movq 104(%rbp), %r9 + movq 112(%rbp), %r10 + movq 120(%rbp), %r11 + # A[11] x A[1] + mulxq 88(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[12] x A[1] + mulxq 96(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[13] x A[1] + mulxq 104(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r8, 96(%rbp) + movq %r9, 104(%rbp) + movq %r10, 112(%rbp) + movq 128(%rdi), %r8 + movq %r12, %r9 + movq %r12, %r10 + # A[14] x A[1] + mulxq 112(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[15] x A[1] + mulxq 120(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[15] x A[2] + movq 16(%rsi), %rdx + mulxq 120(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r11, 120(%rbp) + movq %r8, 128(%rdi) + movq %r9, 136(%rdi) + # Carry + adcxq %r13, %r10 + movq %r12, %r13 + adcxq %r12, %r13 + adoxq %r12, %r13 + movq %r10, 144(%rdi) + # Diagonal 3 + movq 40(%rbp), %r10 + movq 48(%rbp), %r11 + movq 56(%rbp), %r8 + movq 64(%rbp), %r9 + # A[3] x A[2] + mulxq 24(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[4] x A[2] + mulxq 32(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[5] x A[2] + mulxq 40(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r10, 40(%rbp) + movq %r11, 48(%rbp) + movq %r8, 56(%rbp) + movq 72(%rbp), %r10 + movq 80(%rbp), %r11 + movq 88(%rbp), %r8 + # A[6] x A[2] + mulxq 48(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[7] x A[2] + mulxq 56(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[8] x A[2] + mulxq 64(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + movq %r9, 64(%rbp) + movq %r10, 72(%rbp) + movq %r11, 80(%rbp) + movq 96(%rbp), %r9 + movq 104(%rbp), %r10 + movq 112(%rbp), %r11 + # A[9] x A[2] + mulxq 72(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[10] x A[2] + mulxq 80(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[11] x A[2] + mulxq 88(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r8, 88(%rbp) + movq %r9, 96(%rbp) + movq %r10, 104(%rbp) + movq 120(%rbp), %r8 + movq 128(%rdi), %r9 + movq 136(%rdi), %r10 + # A[12] x A[2] + mulxq 96(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[13] x A[2] + mulxq 104(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[14] x A[2] + mulxq 112(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r11, 112(%rbp) + movq %r8, 120(%rbp) + movq %r9, 128(%rdi) + movq 144(%rdi), %r11 + movq %r12, %r8 + movq %r12, %r9 + # A[14] x A[3] + movq 112(%rsi), %rdx + mulxq 24(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[14] x A[4] + mulxq 32(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[14] x A[5] + mulxq 40(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r10, 136(%rdi) + movq %r11, 144(%rdi) + movq %r8, 152(%rdi) + # Carry + adcxq %r13, %r9 + movq %r12, %r13 + adcxq %r12, %r13 + adoxq %r12, %r13 + movq %r9, 160(%rdi) + # Diagonal 4 + movq 56(%rbp), %r9 + movq 64(%rbp), %r10 + movq 72(%rbp), %r11 + movq 80(%rbp), %r8 + # A[4] x A[3] + movq 24(%rsi), %rdx + mulxq 32(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[5] x A[3] + mulxq 40(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[6] x A[3] + mulxq 48(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + movq %r9, 56(%rbp) + movq %r10, 64(%rbp) + movq %r11, 72(%rbp) + movq 88(%rbp), %r9 + movq 96(%rbp), %r10 + movq 104(%rbp), %r11 + # A[7] x A[3] + mulxq 56(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[8] x A[3] + mulxq 64(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[9] x A[3] + mulxq 72(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r8, 80(%rbp) + movq %r9, 88(%rbp) + movq %r10, 96(%rbp) + movq 112(%rbp), %r8 + movq 120(%rbp), %r9 + movq 128(%rdi), %r10 + # A[10] x A[3] + mulxq 80(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[11] x A[3] + mulxq 88(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[12] x A[3] + mulxq 96(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r11, 104(%rbp) + movq %r8, 112(%rbp) + movq %r9, 120(%rbp) + movq 136(%rdi), %r11 + movq 144(%rdi), %r8 + movq 152(%rdi), %r9 + # A[13] x A[3] + mulxq 104(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[13] x A[4] + movq 104(%rsi), %rdx + mulxq 32(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[13] x A[5] + mulxq 40(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r10, 128(%rdi) + movq %r11, 136(%rdi) + movq %r8, 144(%rdi) + movq 160(%rdi), %r10 + movq %r12, %r11 + movq %r12, %r8 + # A[13] x A[6] + mulxq 48(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[13] x A[7] + mulxq 56(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[13] x A[8] + mulxq 64(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + movq %r9, 152(%rdi) + movq %r10, 160(%rdi) + movq %r11, 168(%rdi) + # Carry + adcxq %r13, %r8 + movq %r12, %r13 + adcxq %r12, %r13 + adoxq %r12, %r13 + movq %r8, 176(%rdi) + # Diagonal 5 + movq 72(%rbp), %r8 + movq 80(%rbp), %r9 + movq 88(%rbp), %r10 + movq 96(%rbp), %r11 + # A[5] x A[4] + movq 32(%rsi), %rdx + mulxq 40(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[6] x A[4] + mulxq 48(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[7] x A[4] + mulxq 56(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r8, 72(%rbp) + movq %r9, 80(%rbp) + movq %r10, 88(%rbp) + movq 104(%rbp), %r8 + movq 112(%rbp), %r9 + movq 120(%rbp), %r10 + # A[8] x A[4] + mulxq 64(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[9] x A[4] + mulxq 72(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[10] x A[4] + mulxq 80(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r11, 96(%rbp) + movq %r8, 104(%rbp) + movq %r9, 112(%rbp) + movq 128(%rdi), %r11 + movq 136(%rdi), %r8 + movq 144(%rdi), %r9 + # A[11] x A[4] + mulxq 88(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[12] x A[4] + mulxq 96(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[12] x A[5] + movq 96(%rsi), %rdx + mulxq 40(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r10, 120(%rbp) + movq %r11, 128(%rdi) + movq %r8, 136(%rdi) + movq 152(%rdi), %r10 + movq 160(%rdi), %r11 + movq 168(%rdi), %r8 + # A[12] x A[6] + mulxq 48(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[12] x A[7] + mulxq 56(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[12] x A[8] + mulxq 64(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + movq %r9, 144(%rdi) + movq %r10, 152(%rdi) + movq %r11, 160(%rdi) + movq 176(%rdi), %r9 + movq %r12, %r10 + movq %r12, %r11 + # A[12] x A[9] + mulxq 72(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[12] x A[10] + mulxq 80(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[12] x A[11] + mulxq 88(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r8, 168(%rdi) + movq %r9, 176(%rdi) + movq %r10, 184(%rdi) + # Carry + adcxq %r13, %r11 + movq %r12, %r13 + adcxq %r12, %r13 + adoxq %r12, %r13 + movq %r11, 192(%rdi) + # Diagonal 6 + movq 88(%rbp), %r11 + movq 96(%rbp), %r8 + movq 104(%rbp), %r9 + movq 112(%rbp), %r10 + # A[6] x A[5] + movq 40(%rsi), %rdx + mulxq 48(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[7] x A[5] + mulxq 56(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[8] x A[5] + mulxq 64(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r11, 88(%rbp) + movq %r8, 96(%rbp) + movq %r9, 104(%rbp) + movq 120(%rbp), %r11 + movq 128(%rdi), %r8 + movq 136(%rdi), %r9 + # A[9] x A[5] + mulxq 72(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[10] x A[5] + mulxq 80(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[11] x A[5] + mulxq 88(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r10, 112(%rbp) + movq %r11, 120(%rbp) + movq %r8, 128(%rdi) + movq 144(%rdi), %r10 + movq 152(%rdi), %r11 + movq 160(%rdi), %r8 + # A[11] x A[6] + movq 88(%rsi), %rdx + mulxq 48(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[11] x A[7] + mulxq 56(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[11] x A[8] + mulxq 64(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + movq %r9, 136(%rdi) + movq %r10, 144(%rdi) + movq %r11, 152(%rdi) + movq 168(%rdi), %r9 + movq 176(%rdi), %r10 + movq 184(%rdi), %r11 + # A[11] x A[9] + mulxq 72(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[11] x A[10] + mulxq 80(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[13] x A[9] + movq 104(%rsi), %rdx + mulxq 72(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r8, 160(%rdi) + movq %r9, 168(%rdi) + movq %r10, 176(%rdi) + movq 192(%rdi), %r8 + movq %r12, %r9 + movq %r12, %r10 + # A[13] x A[10] + mulxq 80(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[13] x A[11] + mulxq 88(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[13] x A[12] + mulxq 96(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r11, 184(%rdi) + movq %r8, 192(%rdi) + movq %r9, 200(%rdi) + # Carry + adcxq %r13, %r10 + movq %r12, %r13 + adcxq %r12, %r13 + adoxq %r12, %r13 + movq %r10, 208(%rdi) + # Diagonal 7 + movq 104(%rbp), %r10 + movq 112(%rbp), %r11 + movq 120(%rbp), %r8 + movq 128(%rdi), %r9 + # A[7] x A[6] + movq 48(%rsi), %rdx + mulxq 56(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[8] x A[6] + mulxq 64(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[9] x A[6] + mulxq 72(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r10, 104(%rbp) + movq %r11, 112(%rbp) + movq %r8, 120(%rbp) + movq 136(%rdi), %r10 + movq 144(%rdi), %r11 + movq 152(%rdi), %r8 + # A[10] x A[6] + mulxq 80(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[10] x A[7] + movq 80(%rsi), %rdx + mulxq 56(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[10] x A[8] + mulxq 64(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + movq %r9, 128(%rdi) + movq %r10, 136(%rdi) + movq %r11, 144(%rdi) + movq 160(%rdi), %r9 + movq 168(%rdi), %r10 + movq 176(%rdi), %r11 + # A[10] x A[9] + mulxq 72(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[14] x A[6] + movq 112(%rsi), %rdx + mulxq 48(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[14] x A[7] + mulxq 56(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r8, 152(%rdi) + movq %r9, 160(%rdi) + movq %r10, 168(%rdi) + movq 184(%rdi), %r8 + movq 192(%rdi), %r9 + movq 200(%rdi), %r10 + # A[14] x A[8] + mulxq 64(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[14] x A[9] + mulxq 72(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[14] x A[10] + mulxq 80(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r11, 176(%rdi) + movq %r8, 184(%rdi) + movq %r9, 192(%rdi) + movq 208(%rdi), %r11 + movq %r12, %r8 + movq %r12, %r9 + # A[14] x A[11] + mulxq 88(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[14] x A[12] + mulxq 96(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[14] x A[13] + mulxq 104(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r10, 200(%rdi) + movq %r11, 208(%rdi) + movq %r8, 216(%rdi) + # Carry + adcxq %r13, %r9 + movq %r12, %r13 + adcxq %r12, %r13 + adoxq %r12, %r13 + movq %r9, 224(%rdi) + # Diagonal 8 + movq 120(%rbp), %r9 + movq 128(%rdi), %r10 + movq 136(%rdi), %r11 + movq 144(%rdi), %r8 + # A[8] x A[7] + movq 56(%rsi), %rdx + mulxq 64(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[9] x A[7] + mulxq 72(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[9] x A[8] + movq 64(%rsi), %rdx + mulxq 72(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + movq %r9, 120(%rbp) + movq %r10, 128(%rdi) + movq %r11, 136(%rdi) + movq 152(%rdi), %r9 + movq 160(%rdi), %r10 + movq 168(%rdi), %r11 + # A[15] x A[3] + movq 120(%rsi), %rdx + mulxq 24(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[15] x A[4] + mulxq 32(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[15] x A[5] + mulxq 40(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r8, 144(%rdi) + movq %r9, 152(%rdi) + movq %r10, 160(%rdi) + movq 176(%rdi), %r8 + movq 184(%rdi), %r9 + movq 192(%rdi), %r10 + # A[15] x A[6] + mulxq 48(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[15] x A[7] + mulxq 56(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[15] x A[8] + mulxq 64(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r11, 168(%rdi) + movq %r8, 176(%rdi) + movq %r9, 184(%rdi) + movq 200(%rdi), %r11 + movq 208(%rdi), %r8 + movq 216(%rdi), %r9 + # A[15] x A[9] + mulxq 72(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[15] x A[10] + mulxq 80(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[15] x A[11] + mulxq 88(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r10, 192(%rdi) + movq %r11, 200(%rdi) + movq %r8, 208(%rdi) + movq 224(%rdi), %r10 + movq %r12, %r11 + movq %r12, %r8 + # A[15] x A[12] + mulxq 96(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[15] x A[13] + mulxq 104(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[15] x A[14] + mulxq 112(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + movq %r9, 216(%rdi) + movq %r10, 224(%rdi) + movq %r11, 232(%rdi) + # Carry + adcxq %r13, %r8 + movq %r12, %r13 + adcxq %r12, %r13 + adoxq %r12, %r13 + movq %r8, 240(%rdi) + movq %r13, 248(%rdi) + # Double and Add in A[i] x A[i] + movq 8(%rbp), %r9 + # A[0] x A[0] + movq (%rsi), %rdx + mulxq %rdx, %rax, %rcx + movq %rax, (%rbp) + adoxq %r9, %r9 + adcxq %rcx, %r9 + movq %r9, 8(%rbp) + movq 16(%rbp), %r8 + movq 24(%rbp), %r9 + # A[1] x A[1] + movq 8(%rsi), %rdx + mulxq %rdx, %rax, %rcx + adoxq %r8, %r8 + adcxq %rax, %r8 + adoxq %r9, %r9 + adcxq %rcx, %r9 + movq %r8, 16(%rbp) + movq %r9, 24(%rbp) + movq 32(%rbp), %r8 + movq 40(%rbp), %r9 + # A[2] x A[2] + movq 16(%rsi), %rdx + mulxq %rdx, %rax, %rcx + adoxq %r8, %r8 + adcxq %rax, %r8 + adoxq %r9, %r9 + adcxq %rcx, %r9 + movq %r8, 32(%rbp) + movq %r9, 40(%rbp) + movq 48(%rbp), %r8 + movq 56(%rbp), %r9 + # A[3] x A[3] + movq 24(%rsi), %rdx + mulxq %rdx, %rax, %rcx + adoxq %r8, %r8 + adcxq %rax, %r8 + adoxq %r9, %r9 + adcxq %rcx, %r9 + movq %r8, 48(%rbp) + movq %r9, 56(%rbp) + movq 64(%rbp), %r8 + movq 72(%rbp), %r9 + # A[4] x A[4] + movq 32(%rsi), %rdx + mulxq %rdx, %rax, %rcx + adoxq %r8, %r8 + adcxq %rax, %r8 + adoxq %r9, %r9 + adcxq %rcx, %r9 + movq %r8, 64(%rbp) + movq %r9, 72(%rbp) + movq 80(%rbp), %r8 + movq 88(%rbp), %r9 + # A[5] x A[5] + movq 40(%rsi), %rdx + mulxq %rdx, %rax, %rcx + adoxq %r8, %r8 + adcxq %rax, %r8 + adoxq %r9, %r9 + adcxq %rcx, %r9 + movq %r8, 80(%rbp) + movq %r9, 88(%rbp) + movq 96(%rbp), %r8 + movq 104(%rbp), %r9 + # A[6] x A[6] + movq 48(%rsi), %rdx + mulxq %rdx, %rax, %rcx + adoxq %r8, %r8 + adcxq %rax, %r8 + adoxq %r9, %r9 + adcxq %rcx, %r9 + movq %r8, 96(%rbp) + movq %r9, 104(%rbp) + movq 112(%rbp), %r8 + movq 120(%rbp), %r9 + # A[7] x A[7] + movq 56(%rsi), %rdx + mulxq %rdx, %rax, %rcx + adoxq %r8, %r8 + adcxq %rax, %r8 + adoxq %r9, %r9 + adcxq %rcx, %r9 + movq %r8, 112(%rbp) + movq %r9, 120(%rbp) + movq 128(%rdi), %r8 + movq 136(%rdi), %r9 + # A[8] x A[8] + movq 64(%rsi), %rdx + mulxq %rdx, %rax, %rcx + adoxq %r8, %r8 + adcxq %rax, %r8 + adoxq %r9, %r9 + adcxq %rcx, %r9 + movq %r8, 128(%rdi) + movq %r9, 136(%rdi) + movq 144(%rdi), %r8 + movq 152(%rdi), %r9 + # A[9] x A[9] + movq 72(%rsi), %rdx + mulxq %rdx, %rax, %rcx + adoxq %r8, %r8 + adcxq %rax, %r8 + adoxq %r9, %r9 + adcxq %rcx, %r9 + movq %r8, 144(%rdi) + movq %r9, 152(%rdi) + movq 160(%rdi), %r8 + movq 168(%rdi), %r9 + # A[10] x A[10] + movq 80(%rsi), %rdx + mulxq %rdx, %rax, %rcx + adoxq %r8, %r8 + adcxq %rax, %r8 + adoxq %r9, %r9 + adcxq %rcx, %r9 + movq %r8, 160(%rdi) + movq %r9, 168(%rdi) + movq 176(%rdi), %r8 + movq 184(%rdi), %r9 + # A[11] x A[11] + movq 88(%rsi), %rdx + mulxq %rdx, %rax, %rcx + adoxq %r8, %r8 + adcxq %rax, %r8 + adoxq %r9, %r9 + adcxq %rcx, %r9 + movq %r8, 176(%rdi) + movq %r9, 184(%rdi) + movq 192(%rdi), %r8 + movq 200(%rdi), %r9 + # A[12] x A[12] + movq 96(%rsi), %rdx + mulxq %rdx, %rax, %rcx + adoxq %r8, %r8 + adcxq %rax, %r8 + adoxq %r9, %r9 + adcxq %rcx, %r9 + movq %r8, 192(%rdi) + movq %r9, 200(%rdi) + movq 208(%rdi), %r8 + movq 216(%rdi), %r9 + # A[13] x A[13] + movq 104(%rsi), %rdx + mulxq %rdx, %rax, %rcx + adoxq %r8, %r8 + adcxq %rax, %r8 + adoxq %r9, %r9 + adcxq %rcx, %r9 + movq %r8, 208(%rdi) + movq %r9, 216(%rdi) + movq 224(%rdi), %r8 + movq 232(%rdi), %r9 + # A[14] x A[14] + movq 112(%rsi), %rdx + mulxq %rdx, %rax, %rcx + adoxq %r8, %r8 + adcxq %rax, %r8 + adoxq %r9, %r9 + adcxq %rcx, %r9 + movq %r8, 224(%rdi) + movq %r9, 232(%rdi) + movq 240(%rdi), %r8 + movq 248(%rdi), %r9 + # A[15] x A[15] + movq 120(%rsi), %rdx + mulxq %rdx, %rax, %rcx + adoxq %r8, %r8 + adcxq %rax, %r8 + adoxq %r9, %r9 + adcxq %rcx, %r9 + movq %r8, 240(%rdi) + movq %r9, 248(%rdi) + cmpq %rdi, %rsi + jne L_end_2048_sqr_avx2_16 + movq (%rbp), %r8 + movq 8(%rbp), %r9 + movq 16(%rbp), %r10 + movq 24(%rbp), %r11 + movq %r8, (%rdi) + movq %r9, 8(%rdi) + movq %r10, 16(%rdi) + movq %r11, 24(%rdi) + movq 32(%rbp), %r8 + movq 40(%rbp), %r9 + movq 48(%rbp), %r10 + movq 56(%rbp), %r11 + movq %r8, 32(%rdi) + movq %r9, 40(%rdi) + movq %r10, 48(%rdi) + movq %r11, 56(%rdi) + movq 64(%rbp), %r8 + movq 72(%rbp), %r9 + movq 80(%rbp), %r10 + movq 88(%rbp), %r11 + movq %r8, 64(%rdi) + movq %r9, 72(%rdi) + movq %r10, 80(%rdi) + movq %r11, 88(%rdi) + movq 96(%rbp), %r8 + movq 104(%rbp), %r9 + movq 112(%rbp), %r10 + movq 120(%rbp), %r11 + movq %r8, 96(%rdi) + movq %r9, 104(%rdi) + movq %r10, 112(%rdi) + movq %r11, 120(%rdi) +L_end_2048_sqr_avx2_16: + addq $128, %rsp + pop %r13 + pop %r12 + pop %rbp + repz retq +.size sp_2048_sqr_avx2_16,.-sp_2048_sqr_avx2_16 +#endif /* HAVE_INTEL_AVX2 */ +/* Add b to a into r. (r = a + b) + * + * r A single precision integer. + * a A single precision integer. + * b A single precision integer. + */ +.globl sp_2048_add_16 +.type sp_2048_add_16,@function +.align 16 +sp_2048_add_16: + xorq %rax, %rax + movq (%rsi), %rcx + addq (%rdx), %rcx + movq %rcx, (%rdi) + movq 8(%rsi), %rcx + adcq 8(%rdx), %rcx + movq %rcx, 8(%rdi) + movq 16(%rsi), %rcx + adcq 16(%rdx), %rcx + movq %rcx, 16(%rdi) + movq 24(%rsi), %rcx + adcq 24(%rdx), %rcx + movq %rcx, 24(%rdi) + movq 32(%rsi), %rcx + adcq 32(%rdx), %rcx + movq %rcx, 32(%rdi) + movq 40(%rsi), %rcx + adcq 40(%rdx), %rcx + movq %rcx, 40(%rdi) + movq 48(%rsi), %rcx + adcq 48(%rdx), %rcx + movq %rcx, 48(%rdi) + movq 56(%rsi), %rcx + adcq 56(%rdx), %rcx + movq %rcx, 56(%rdi) + movq 64(%rsi), %rcx + adcq 64(%rdx), %rcx + movq %rcx, 64(%rdi) + movq 72(%rsi), %rcx + adcq 72(%rdx), %rcx + movq %rcx, 72(%rdi) + movq 80(%rsi), %rcx + adcq 80(%rdx), %rcx + movq %rcx, 80(%rdi) + movq 88(%rsi), %rcx + adcq 88(%rdx), %rcx + movq %rcx, 88(%rdi) + movq 96(%rsi), %rcx + adcq 96(%rdx), %rcx + movq %rcx, 96(%rdi) + movq 104(%rsi), %rcx + adcq 104(%rdx), %rcx + movq %rcx, 104(%rdi) + movq 112(%rsi), %rcx + adcq 112(%rdx), %rcx + movq %rcx, 112(%rdi) + movq 120(%rsi), %rcx + adcq 120(%rdx), %rcx + movq %rcx, 120(%rdi) + adcq $0, %rax + repz retq +.size sp_2048_add_16,.-sp_2048_add_16 +/* Sub b from a into a. (a -= b) + * + * a A single precision integer and result. + * b A single precision integer. + */ +.globl sp_2048_sub_in_place_32 +.type sp_2048_sub_in_place_32,@function +.align 16 +sp_2048_sub_in_place_32: + xorq %rax, %rax + movq (%rdi), %rdx + movq 8(%rdi), %rcx + movq (%rsi), %r8 + movq 8(%rsi), %r9 + subq %r8, %rdx + movq 16(%rsi), %r8 + movq %rdx, (%rdi) + movq 16(%rdi), %rdx + sbbq %r9, %rcx + movq 24(%rsi), %r9 + movq %rcx, 8(%rdi) + movq 24(%rdi), %rcx + sbbq %r8, %rdx + movq 32(%rsi), %r8 + movq %rdx, 16(%rdi) + movq 32(%rdi), %rdx + sbbq %r9, %rcx + movq 40(%rsi), %r9 + movq %rcx, 24(%rdi) + movq 40(%rdi), %rcx + sbbq %r8, %rdx + movq 48(%rsi), %r8 + movq %rdx, 32(%rdi) + movq 48(%rdi), %rdx + sbbq %r9, %rcx + movq 56(%rsi), %r9 + movq %rcx, 40(%rdi) + movq 56(%rdi), %rcx + sbbq %r8, %rdx + movq 64(%rsi), %r8 + movq %rdx, 48(%rdi) + movq 64(%rdi), %rdx + sbbq %r9, %rcx + movq 72(%rsi), %r9 + movq %rcx, 56(%rdi) + movq 72(%rdi), %rcx + sbbq %r8, %rdx + movq 80(%rsi), %r8 + movq %rdx, 64(%rdi) + movq 80(%rdi), %rdx + sbbq %r9, %rcx + movq 88(%rsi), %r9 + movq %rcx, 72(%rdi) + movq 88(%rdi), %rcx + sbbq %r8, %rdx + movq 96(%rsi), %r8 + movq %rdx, 80(%rdi) + movq 96(%rdi), %rdx + sbbq %r9, %rcx + movq 104(%rsi), %r9 + movq %rcx, 88(%rdi) + movq 104(%rdi), %rcx + sbbq %r8, %rdx + movq 112(%rsi), %r8 + movq %rdx, 96(%rdi) + movq 112(%rdi), %rdx + sbbq %r9, %rcx + movq 120(%rsi), %r9 + movq %rcx, 104(%rdi) + movq 120(%rdi), %rcx + sbbq %r8, %rdx + movq 128(%rsi), %r8 + movq %rdx, 112(%rdi) + movq 128(%rdi), %rdx + sbbq %r9, %rcx + movq 136(%rsi), %r9 + movq %rcx, 120(%rdi) + movq 136(%rdi), %rcx + sbbq %r8, %rdx + movq 144(%rsi), %r8 + movq %rdx, 128(%rdi) + movq 144(%rdi), %rdx + sbbq %r9, %rcx + movq 152(%rsi), %r9 + movq %rcx, 136(%rdi) + movq 152(%rdi), %rcx + sbbq %r8, %rdx + movq 160(%rsi), %r8 + movq %rdx, 144(%rdi) + movq 160(%rdi), %rdx + sbbq %r9, %rcx + movq 168(%rsi), %r9 + movq %rcx, 152(%rdi) + movq 168(%rdi), %rcx + sbbq %r8, %rdx + movq 176(%rsi), %r8 + movq %rdx, 160(%rdi) + movq 176(%rdi), %rdx + sbbq %r9, %rcx + movq 184(%rsi), %r9 + movq %rcx, 168(%rdi) + movq 184(%rdi), %rcx + sbbq %r8, %rdx + movq 192(%rsi), %r8 + movq %rdx, 176(%rdi) + movq 192(%rdi), %rdx + sbbq %r9, %rcx + movq 200(%rsi), %r9 + movq %rcx, 184(%rdi) + movq 200(%rdi), %rcx + sbbq %r8, %rdx + movq 208(%rsi), %r8 + movq %rdx, 192(%rdi) + movq 208(%rdi), %rdx + sbbq %r9, %rcx + movq 216(%rsi), %r9 + movq %rcx, 200(%rdi) + movq 216(%rdi), %rcx + sbbq %r8, %rdx + movq 224(%rsi), %r8 + movq %rdx, 208(%rdi) + movq 224(%rdi), %rdx + sbbq %r9, %rcx + movq 232(%rsi), %r9 + movq %rcx, 216(%rdi) + movq 232(%rdi), %rcx + sbbq %r8, %rdx + movq 240(%rsi), %r8 + movq %rdx, 224(%rdi) + movq 240(%rdi), %rdx + sbbq %r9, %rcx + movq 248(%rsi), %r9 + movq %rcx, 232(%rdi) + movq 248(%rdi), %rcx + sbbq %r8, %rdx + movq %rdx, 240(%rdi) + sbbq %r9, %rcx + movq %rcx, 248(%rdi) + sbbq $0, %rax + repz retq +.size sp_2048_sub_in_place_32,.-sp_2048_sub_in_place_32 +/* Add b to a into r. (r = a + b) + * + * r A single precision integer. + * a A single precision integer. + * b A single precision integer. + */ +.globl sp_2048_add_32 +.type sp_2048_add_32,@function +.align 16 +sp_2048_add_32: + xorq %rax, %rax + movq (%rsi), %rcx + addq (%rdx), %rcx + movq %rcx, (%rdi) + movq 8(%rsi), %rcx + adcq 8(%rdx), %rcx + movq %rcx, 8(%rdi) + movq 16(%rsi), %rcx + adcq 16(%rdx), %rcx + movq %rcx, 16(%rdi) + movq 24(%rsi), %rcx + adcq 24(%rdx), %rcx + movq %rcx, 24(%rdi) + movq 32(%rsi), %rcx + adcq 32(%rdx), %rcx + movq %rcx, 32(%rdi) + movq 40(%rsi), %rcx + adcq 40(%rdx), %rcx + movq %rcx, 40(%rdi) + movq 48(%rsi), %rcx + adcq 48(%rdx), %rcx + movq %rcx, 48(%rdi) + movq 56(%rsi), %rcx + adcq 56(%rdx), %rcx + movq %rcx, 56(%rdi) + movq 64(%rsi), %rcx + adcq 64(%rdx), %rcx + movq %rcx, 64(%rdi) + movq 72(%rsi), %rcx + adcq 72(%rdx), %rcx + movq %rcx, 72(%rdi) + movq 80(%rsi), %rcx + adcq 80(%rdx), %rcx + movq %rcx, 80(%rdi) + movq 88(%rsi), %rcx + adcq 88(%rdx), %rcx + movq %rcx, 88(%rdi) + movq 96(%rsi), %rcx + adcq 96(%rdx), %rcx + movq %rcx, 96(%rdi) + movq 104(%rsi), %rcx + adcq 104(%rdx), %rcx + movq %rcx, 104(%rdi) + movq 112(%rsi), %rcx + adcq 112(%rdx), %rcx + movq %rcx, 112(%rdi) + movq 120(%rsi), %rcx + adcq 120(%rdx), %rcx + movq %rcx, 120(%rdi) + movq 128(%rsi), %rcx + adcq 128(%rdx), %rcx + movq %rcx, 128(%rdi) + movq 136(%rsi), %rcx + adcq 136(%rdx), %rcx + movq %rcx, 136(%rdi) + movq 144(%rsi), %rcx + adcq 144(%rdx), %rcx + movq %rcx, 144(%rdi) + movq 152(%rsi), %rcx + adcq 152(%rdx), %rcx + movq %rcx, 152(%rdi) + movq 160(%rsi), %rcx + adcq 160(%rdx), %rcx + movq %rcx, 160(%rdi) + movq 168(%rsi), %rcx + adcq 168(%rdx), %rcx + movq %rcx, 168(%rdi) + movq 176(%rsi), %rcx + adcq 176(%rdx), %rcx + movq %rcx, 176(%rdi) + movq 184(%rsi), %rcx + adcq 184(%rdx), %rcx + movq %rcx, 184(%rdi) + movq 192(%rsi), %rcx + adcq 192(%rdx), %rcx + movq %rcx, 192(%rdi) + movq 200(%rsi), %rcx + adcq 200(%rdx), %rcx + movq %rcx, 200(%rdi) + movq 208(%rsi), %rcx + adcq 208(%rdx), %rcx + movq %rcx, 208(%rdi) + movq 216(%rsi), %rcx + adcq 216(%rdx), %rcx + movq %rcx, 216(%rdi) + movq 224(%rsi), %rcx + adcq 224(%rdx), %rcx + movq %rcx, 224(%rdi) + movq 232(%rsi), %rcx + adcq 232(%rdx), %rcx + movq %rcx, 232(%rdi) + movq 240(%rsi), %rcx + adcq 240(%rdx), %rcx + movq %rcx, 240(%rdi) + movq 248(%rsi), %rcx + adcq 248(%rdx), %rcx + movq %rcx, 248(%rdi) + adcq $0, %rax + repz retq +.size sp_2048_add_32,.-sp_2048_add_32 +#ifdef HAVE_INTEL_AVX2 +#endif /* HAVE_INTEL_AVX2 */ +#ifdef HAVE_INTEL_AVX2 +#endif /* HAVE_INTEL_AVX2 */ +#if !defined(SP_RSA_PRIVATE_EXP_D) && defined(WOLFSSL_HAVE_SP_RSA) +#endif /* !SP_RSA_PRIVATE_EXP_D && WOLFSSL_HAVE_SP_RSA */ +#if !defined(SP_RSA_PRIVATE_EXP_D) && defined(WOLFSSL_HAVE_SP_RSA) +/* Sub b from a into a. (a -= b) + * + * a A single precision integer and result. + * b A single precision integer. + */ +.globl sp_2048_sub_in_place_16 +.type sp_2048_sub_in_place_16,@function +.align 16 +sp_2048_sub_in_place_16: + xorq %rax, %rax + movq (%rdi), %rdx + movq 8(%rdi), %rcx + movq (%rsi), %r8 + movq 8(%rsi), %r9 + subq %r8, %rdx + movq 16(%rsi), %r8 + movq %rdx, (%rdi) + movq 16(%rdi), %rdx + sbbq %r9, %rcx + movq 24(%rsi), %r9 + movq %rcx, 8(%rdi) + movq 24(%rdi), %rcx + sbbq %r8, %rdx + movq 32(%rsi), %r8 + movq %rdx, 16(%rdi) + movq 32(%rdi), %rdx + sbbq %r9, %rcx + movq 40(%rsi), %r9 + movq %rcx, 24(%rdi) + movq 40(%rdi), %rcx + sbbq %r8, %rdx + movq 48(%rsi), %r8 + movq %rdx, 32(%rdi) + movq 48(%rdi), %rdx + sbbq %r9, %rcx + movq 56(%rsi), %r9 + movq %rcx, 40(%rdi) + movq 56(%rdi), %rcx + sbbq %r8, %rdx + movq 64(%rsi), %r8 + movq %rdx, 48(%rdi) + movq 64(%rdi), %rdx + sbbq %r9, %rcx + movq 72(%rsi), %r9 + movq %rcx, 56(%rdi) + movq 72(%rdi), %rcx + sbbq %r8, %rdx + movq 80(%rsi), %r8 + movq %rdx, 64(%rdi) + movq 80(%rdi), %rdx + sbbq %r9, %rcx + movq 88(%rsi), %r9 + movq %rcx, 72(%rdi) + movq 88(%rdi), %rcx + sbbq %r8, %rdx + movq 96(%rsi), %r8 + movq %rdx, 80(%rdi) + movq 96(%rdi), %rdx + sbbq %r9, %rcx + movq 104(%rsi), %r9 + movq %rcx, 88(%rdi) + movq 104(%rdi), %rcx + sbbq %r8, %rdx + movq 112(%rsi), %r8 + movq %rdx, 96(%rdi) + movq 112(%rdi), %rdx + sbbq %r9, %rcx + movq 120(%rsi), %r9 + movq %rcx, 104(%rdi) + movq 120(%rdi), %rcx + sbbq %r8, %rdx + movq %rdx, 112(%rdi) + sbbq %r9, %rcx + movq %rcx, 120(%rdi) + sbbq $0, %rax + repz retq +.size sp_2048_sub_in_place_16,.-sp_2048_sub_in_place_16 +/* Conditionally subtract b from a using the mask m. + * m is -1 to subtract and 0 when not copying. + * + * r A single precision number representing condition subtract result. + * a A single precision number to subtract from. + * b A single precision number to subtract. + * m Mask value to apply. + */ +.globl sp_2048_cond_sub_16 +.type sp_2048_cond_sub_16,@function +.align 16 +sp_2048_cond_sub_16: + subq $128, %rsp + movq $0, %rax + movq (%rdx), %r8 + movq 8(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, (%rsp) + movq %r9, 8(%rsp) + movq 16(%rdx), %r8 + movq 24(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 16(%rsp) + movq %r9, 24(%rsp) + movq 32(%rdx), %r8 + movq 40(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 32(%rsp) + movq %r9, 40(%rsp) + movq 48(%rdx), %r8 + movq 56(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 48(%rsp) + movq %r9, 56(%rsp) + movq 64(%rdx), %r8 + movq 72(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 64(%rsp) + movq %r9, 72(%rsp) + movq 80(%rdx), %r8 + movq 88(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 80(%rsp) + movq %r9, 88(%rsp) + movq 96(%rdx), %r8 + movq 104(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 96(%rsp) + movq %r9, 104(%rsp) + movq 112(%rdx), %r8 + movq 120(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 112(%rsp) + movq %r9, 120(%rsp) + movq (%rsi), %r8 + movq (%rsp), %rdx + subq %rdx, %r8 + movq 8(%rsi), %r9 + movq 8(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, (%rdi) + movq 16(%rsi), %r8 + movq 16(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 8(%rdi) + movq 24(%rsi), %r9 + movq 24(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 16(%rdi) + movq 32(%rsi), %r8 + movq 32(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 24(%rdi) + movq 40(%rsi), %r9 + movq 40(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 32(%rdi) + movq 48(%rsi), %r8 + movq 48(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 40(%rdi) + movq 56(%rsi), %r9 + movq 56(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 48(%rdi) + movq 64(%rsi), %r8 + movq 64(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 56(%rdi) + movq 72(%rsi), %r9 + movq 72(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 64(%rdi) + movq 80(%rsi), %r8 + movq 80(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 72(%rdi) + movq 88(%rsi), %r9 + movq 88(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 80(%rdi) + movq 96(%rsi), %r8 + movq 96(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 88(%rdi) + movq 104(%rsi), %r9 + movq 104(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 96(%rdi) + movq 112(%rsi), %r8 + movq 112(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 104(%rdi) + movq 120(%rsi), %r9 + movq 120(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 112(%rdi) + movq %r9, 120(%rdi) + sbbq $0, %rax + addq $128, %rsp + repz retq +.size sp_2048_cond_sub_16,.-sp_2048_cond_sub_16 +/* Reduce the number back to 2048 bits using Montgomery reduction. + * + * a A single precision number to reduce in place. + * m The single precision number representing the modulus. + * mp The digit representing the negative inverse of m mod 2^n. + */ +.globl sp_2048_mont_reduce_16 +.type sp_2048_mont_reduce_16,@function +.align 16 +sp_2048_mont_reduce_16: + push %r12 + push %r13 + push %r14 + push %r15 + movq %rdx, %rcx + xorq %r15, %r15 + # i = 0 + movq $16, %r8 + movq (%rdi), %r13 + movq 8(%rdi), %r14 +L_mont_loop_16: + # mu = a[i] * mp + movq %r13, %r11 + imulq %rcx, %r11 + # a[i+0] += m[0] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq (%rsi) + addq %rax, %r13 + adcq %rdx, %r10 + # a[i+1] += m[1] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 8(%rsi) + movq %r14, %r13 + addq %rax, %r13 + adcq %rdx, %r9 + addq %r10, %r13 + adcq $0, %r9 + # a[i+2] += m[2] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 16(%rsi) + movq 16(%rdi), %r14 + addq %rax, %r14 + adcq %rdx, %r10 + addq %r9, %r14 + adcq $0, %r10 + # a[i+3] += m[3] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 24(%rsi) + movq 24(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r9 + addq %r10, %r12 + movq %r12, 24(%rdi) + adcq $0, %r9 + # a[i+4] += m[4] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 32(%rsi) + movq 32(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r10 + addq %r9, %r12 + movq %r12, 32(%rdi) + adcq $0, %r10 + # a[i+5] += m[5] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 40(%rsi) + movq 40(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r9 + addq %r10, %r12 + movq %r12, 40(%rdi) + adcq $0, %r9 + # a[i+6] += m[6] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 48(%rsi) + movq 48(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r10 + addq %r9, %r12 + movq %r12, 48(%rdi) + adcq $0, %r10 + # a[i+7] += m[7] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 56(%rsi) + movq 56(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r9 + addq %r10, %r12 + movq %r12, 56(%rdi) + adcq $0, %r9 + # a[i+8] += m[8] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 64(%rsi) + movq 64(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r10 + addq %r9, %r12 + movq %r12, 64(%rdi) + adcq $0, %r10 + # a[i+9] += m[9] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 72(%rsi) + movq 72(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r9 + addq %r10, %r12 + movq %r12, 72(%rdi) + adcq $0, %r9 + # a[i+10] += m[10] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 80(%rsi) + movq 80(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r10 + addq %r9, %r12 + movq %r12, 80(%rdi) + adcq $0, %r10 + # a[i+11] += m[11] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 88(%rsi) + movq 88(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r9 + addq %r10, %r12 + movq %r12, 88(%rdi) + adcq $0, %r9 + # a[i+12] += m[12] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 96(%rsi) + movq 96(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r10 + addq %r9, %r12 + movq %r12, 96(%rdi) + adcq $0, %r10 + # a[i+13] += m[13] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 104(%rsi) + movq 104(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r9 + addq %r10, %r12 + movq %r12, 104(%rdi) + adcq $0, %r9 + # a[i+14] += m[14] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 112(%rsi) + movq 112(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r10 + addq %r9, %r12 + movq %r12, 112(%rdi) + adcq $0, %r10 + # a[i+15] += m[15] * mu + movq %r11, %rax + mulq 120(%rsi) + movq 120(%rdi), %r12 + addq %rax, %r10 + adcq %r15, %rdx + movq $0, %r15 + adcq $0, %r15 + addq %r10, %r12 + movq %r12, 120(%rdi) + adcq %rdx, 128(%rdi) + adcq $0, %r15 + # i += 1 + addq $8, %rdi + decq %r8 + jnz L_mont_loop_16 + movq %r13, (%rdi) + movq %r14, 8(%rdi) + negq %r15 + movq %r15, %rcx + movq %rsi, %rdx + movq %rdi, %rsi + movq %rdi, %rdi + subq $128, %rdi + callq sp_2048_cond_sub_16@plt + pop %r15 + pop %r14 + pop %r13 + pop %r12 + repz retq +.size sp_2048_mont_reduce_16,.-sp_2048_mont_reduce_16 +/* Mul a by digit b into r. (r = a * b) + * + * r A single precision integer. + * a A single precision integer. + * b A single precision digit. + */ +.globl sp_2048_mul_d_16 +.type sp_2048_mul_d_16,@function +.align 16 +sp_2048_mul_d_16: + movq %rdx, %rcx + # A[0] * B + movq %rcx, %rax + xorq %r10, %r10 + mulq (%rsi) + movq %rax, %r8 + movq %rdx, %r9 + movq %r8, (%rdi) + # A[1] * B + movq %rcx, %rax + xorq %r8, %r8 + mulq 8(%rsi) + addq %rax, %r9 + movq %r9, 8(%rdi) + adcq %rdx, %r10 + adcq $0, %r8 + # A[2] * B + movq %rcx, %rax + xorq %r9, %r9 + mulq 16(%rsi) + addq %rax, %r10 + movq %r10, 16(%rdi) + adcq %rdx, %r8 + adcq $0, %r9 + # A[3] * B + movq %rcx, %rax + xorq %r10, %r10 + mulq 24(%rsi) + addq %rax, %r8 + movq %r8, 24(%rdi) + adcq %rdx, %r9 + adcq $0, %r10 + # A[4] * B + movq %rcx, %rax + xorq %r8, %r8 + mulq 32(%rsi) + addq %rax, %r9 + movq %r9, 32(%rdi) + adcq %rdx, %r10 + adcq $0, %r8 + # A[5] * B + movq %rcx, %rax + xorq %r9, %r9 + mulq 40(%rsi) + addq %rax, %r10 + movq %r10, 40(%rdi) + adcq %rdx, %r8 + adcq $0, %r9 + # A[6] * B + movq %rcx, %rax + xorq %r10, %r10 + mulq 48(%rsi) + addq %rax, %r8 + movq %r8, 48(%rdi) + adcq %rdx, %r9 + adcq $0, %r10 + # A[7] * B + movq %rcx, %rax + xorq %r8, %r8 + mulq 56(%rsi) + addq %rax, %r9 + movq %r9, 56(%rdi) + adcq %rdx, %r10 + adcq $0, %r8 + # A[8] * B + movq %rcx, %rax + xorq %r9, %r9 + mulq 64(%rsi) + addq %rax, %r10 + movq %r10, 64(%rdi) + adcq %rdx, %r8 + adcq $0, %r9 + # A[9] * B + movq %rcx, %rax + xorq %r10, %r10 + mulq 72(%rsi) + addq %rax, %r8 + movq %r8, 72(%rdi) + adcq %rdx, %r9 + adcq $0, %r10 + # A[10] * B + movq %rcx, %rax + xorq %r8, %r8 + mulq 80(%rsi) + addq %rax, %r9 + movq %r9, 80(%rdi) + adcq %rdx, %r10 + adcq $0, %r8 + # A[11] * B + movq %rcx, %rax + xorq %r9, %r9 + mulq 88(%rsi) + addq %rax, %r10 + movq %r10, 88(%rdi) + adcq %rdx, %r8 + adcq $0, %r9 + # A[12] * B + movq %rcx, %rax + xorq %r10, %r10 + mulq 96(%rsi) + addq %rax, %r8 + movq %r8, 96(%rdi) + adcq %rdx, %r9 + adcq $0, %r10 + # A[13] * B + movq %rcx, %rax + xorq %r8, %r8 + mulq 104(%rsi) + addq %rax, %r9 + movq %r9, 104(%rdi) + adcq %rdx, %r10 + adcq $0, %r8 + # A[14] * B + movq %rcx, %rax + xorq %r9, %r9 + mulq 112(%rsi) + addq %rax, %r10 + movq %r10, 112(%rdi) + adcq %rdx, %r8 + adcq $0, %r9 + # # A[15] * B + movq %rcx, %rax + mulq 120(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + movq %r8, 120(%rdi) + movq %r9, 128(%rdi) + repz retq +.size sp_2048_mul_d_16,.-sp_2048_mul_d_16 +#ifdef HAVE_INTEL_AVX2 +/* Mul a by digit b into r. (r = a * b) + * + * r A single precision integer. + * a A single precision integer. + * b A single precision digit. + */ +.globl sp_2048_mul_d_avx2_16 +.type sp_2048_mul_d_avx2_16,@function +.align 16 +sp_2048_mul_d_avx2_16: + movq %rdx, %rax + # A[0] * B + movq %rax, %rdx + xorq %r11, %r11 + mulxq (%rsi), %r9, %r10 + movq %r9, (%rdi) + # A[1] * B + mulxq 8(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 8(%rdi) + adoxq %r8, %r9 + # A[2] * B + mulxq 16(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 16(%rdi) + adoxq %r8, %r10 + # A[3] * B + mulxq 24(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 24(%rdi) + adoxq %r8, %r9 + # A[4] * B + mulxq 32(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 32(%rdi) + adoxq %r8, %r10 + # A[5] * B + mulxq 40(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 40(%rdi) + adoxq %r8, %r9 + # A[6] * B + mulxq 48(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 48(%rdi) + adoxq %r8, %r10 + # A[7] * B + mulxq 56(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 56(%rdi) + adoxq %r8, %r9 + # A[8] * B + mulxq 64(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 64(%rdi) + adoxq %r8, %r10 + # A[9] * B + mulxq 72(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 72(%rdi) + adoxq %r8, %r9 + # A[10] * B + mulxq 80(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 80(%rdi) + adoxq %r8, %r10 + # A[11] * B + mulxq 88(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 88(%rdi) + adoxq %r8, %r9 + # A[12] * B + mulxq 96(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 96(%rdi) + adoxq %r8, %r10 + # A[13] * B + mulxq 104(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 104(%rdi) + adoxq %r8, %r9 + # A[14] * B + mulxq 112(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 112(%rdi) + adoxq %r8, %r10 + # A[15] * B + mulxq 120(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + adoxq %r8, %r9 + adcxq %r11, %r9 + movq %r10, 120(%rdi) + movq %r9, 128(%rdi) + repz retq +.size sp_2048_mul_d_avx2_16,.-sp_2048_mul_d_avx2_16 +#endif /* HAVE_INTEL_AVX2 */ +/* Compare a with b in constant time. + * + * a A single precision integer. + * b A single precision integer. + * return -ve, 0 or +ve if a is less than, equal to or greater than b + * respectively. + */ +.globl sp_2048_cmp_16 +.type sp_2048_cmp_16,@function +.align 16 +sp_2048_cmp_16: + xorq %rcx, %rcx + movq $-1, %rdx + movq $-1, %rax + movq $1, %r8 + movq 120(%rdi), %r9 + movq 120(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 112(%rdi), %r9 + movq 112(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 104(%rdi), %r9 + movq 104(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 96(%rdi), %r9 + movq 96(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 88(%rdi), %r9 + movq 88(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 80(%rdi), %r9 + movq 80(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 72(%rdi), %r9 + movq 72(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 64(%rdi), %r9 + movq 64(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 56(%rdi), %r9 + movq 56(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 48(%rdi), %r9 + movq 48(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 40(%rdi), %r9 + movq 40(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 32(%rdi), %r9 + movq 32(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 24(%rdi), %r9 + movq 24(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 16(%rdi), %r9 + movq 16(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 8(%rdi), %r9 + movq 8(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq (%rdi), %r9 + movq (%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + xorq %rdx, %rax + repz retq +.size sp_2048_cmp_16,.-sp_2048_cmp_16 +#ifdef HAVE_INTEL_AVX2 +/* Reduce the number back to 2048 bits using Montgomery reduction. + * + * a A single precision number to reduce in place. + * m The single precision number representing the modulus. + * mp The digit representing the negative inverse of m mod 2^n. + */ +.globl sp_2048_mont_reduce_avx2_16 +.type sp_2048_mont_reduce_avx2_16,@function +.align 16 +sp_2048_mont_reduce_avx2_16: + push %r12 + push %r13 + push %r14 + movq %rdx, %rax + xorq %r14, %r14 + # i = 0 + movq $16, %r9 + movq (%rdi), %r13 + xorq %r12, %r12 +L_mont_loop_avx2_16: + # mu = a[i] * mp + movq %r13, %rdx + mulxq %rax, %rdx, %r8 + movq %r13, %r10 + # a[i+0] += m[0] * mu + mulxq (%rsi), %rcx, %r8 + movq 8(%rdi), %r13 + adcxq %rcx, %r10 + adoxq %r8, %r13 + # a[i+1] += m[1] * mu + mulxq 8(%rsi), %rcx, %r8 + movq 16(%rdi), %r10 + adcxq %rcx, %r13 + adoxq %r8, %r10 + # a[i+2] += m[2] * mu + mulxq 16(%rsi), %rcx, %r8 + movq 24(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 16(%rdi) + # a[i+3] += m[3] * mu + mulxq 24(%rsi), %rcx, %r8 + movq 32(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 24(%rdi) + # a[i+4] += m[4] * mu + mulxq 32(%rsi), %rcx, %r8 + movq 40(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 32(%rdi) + # a[i+5] += m[5] * mu + mulxq 40(%rsi), %rcx, %r8 + movq 48(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 40(%rdi) + # a[i+6] += m[6] * mu + mulxq 48(%rsi), %rcx, %r8 + movq 56(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 48(%rdi) + # a[i+7] += m[7] * mu + mulxq 56(%rsi), %rcx, %r8 + movq 64(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 56(%rdi) + # a[i+8] += m[8] * mu + mulxq 64(%rsi), %rcx, %r8 + movq 72(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 64(%rdi) + # a[i+9] += m[9] * mu + mulxq 72(%rsi), %rcx, %r8 + movq 80(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 72(%rdi) + # a[i+10] += m[10] * mu + mulxq 80(%rsi), %rcx, %r8 + movq 88(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 80(%rdi) + # a[i+11] += m[11] * mu + mulxq 88(%rsi), %rcx, %r8 + movq 96(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 88(%rdi) + # a[i+12] += m[12] * mu + mulxq 96(%rsi), %rcx, %r8 + movq 104(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 96(%rdi) + # a[i+13] += m[13] * mu + mulxq 104(%rsi), %rcx, %r8 + movq 112(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 104(%rdi) + # a[i+14] += m[14] * mu + mulxq 112(%rsi), %rcx, %r8 + movq 120(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 112(%rdi) + # a[i+15] += m[15] * mu + mulxq 120(%rsi), %rcx, %r8 + movq 128(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 120(%rdi) + adcxq %r14, %r10 + movq %r12, %r14 + adoxq %r12, %r14 + adcxq %r12, %r14 + movq %r10, 128(%rdi) + # i += 1 + addq $8, %rdi + decq %r9 + jnz L_mont_loop_avx2_16 + movq %r13, (%rdi) + negq %r14 + movq %r14, %rcx + movq %rsi, %rdx + movq %rdi, %rsi + movq %rdi, %rdi + subq $128, %rdi + callq sp_2048_cond_sub_16@plt + pop %r14 + pop %r13 + pop %r12 + repz retq +.size sp_2048_mont_reduce_avx2_16,.-sp_2048_mont_reduce_avx2_16 +#endif /* HAVE_INTEL_AVX2 */ +#ifdef HAVE_INTEL_AVX2 +#endif /* HAVE_INTEL_AVX2 */ +#ifdef HAVE_INTEL_AVX2 +#endif /* HAVE_INTEL_AVX2 */ +#ifdef HAVE_INTEL_AVX2 +#endif /* HAVE_INTEL_AVX2 */ +#endif /* !SP_RSA_PRIVATE_EXP_D && WOLFSSL_HAVE_SP_RSA */ +#ifdef WOLFSSL_HAVE_SP_DH +#endif /* WOLFSSL_HAVE_SP_DH */ +/* Conditionally subtract b from a using the mask m. + * m is -1 to subtract and 0 when not copying. + * + * r A single precision number representing condition subtract result. + * a A single precision number to subtract from. + * b A single precision number to subtract. + * m Mask value to apply. + */ +.globl sp_2048_cond_sub_32 +.type sp_2048_cond_sub_32,@function +.align 16 +sp_2048_cond_sub_32: + subq $256, %rsp + movq $0, %rax + movq (%rdx), %r8 + movq 8(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, (%rsp) + movq %r9, 8(%rsp) + movq 16(%rdx), %r8 + movq 24(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 16(%rsp) + movq %r9, 24(%rsp) + movq 32(%rdx), %r8 + movq 40(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 32(%rsp) + movq %r9, 40(%rsp) + movq 48(%rdx), %r8 + movq 56(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 48(%rsp) + movq %r9, 56(%rsp) + movq 64(%rdx), %r8 + movq 72(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 64(%rsp) + movq %r9, 72(%rsp) + movq 80(%rdx), %r8 + movq 88(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 80(%rsp) + movq %r9, 88(%rsp) + movq 96(%rdx), %r8 + movq 104(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 96(%rsp) + movq %r9, 104(%rsp) + movq 112(%rdx), %r8 + movq 120(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 112(%rsp) + movq %r9, 120(%rsp) + movq 128(%rdx), %r8 + movq 136(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 128(%rsp) + movq %r9, 136(%rsp) + movq 144(%rdx), %r8 + movq 152(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 144(%rsp) + movq %r9, 152(%rsp) + movq 160(%rdx), %r8 + movq 168(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 160(%rsp) + movq %r9, 168(%rsp) + movq 176(%rdx), %r8 + movq 184(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 176(%rsp) + movq %r9, 184(%rsp) + movq 192(%rdx), %r8 + movq 200(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 192(%rsp) + movq %r9, 200(%rsp) + movq 208(%rdx), %r8 + movq 216(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 208(%rsp) + movq %r9, 216(%rsp) + movq 224(%rdx), %r8 + movq 232(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 224(%rsp) + movq %r9, 232(%rsp) + movq 240(%rdx), %r8 + movq 248(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 240(%rsp) + movq %r9, 248(%rsp) + movq (%rsi), %r8 + movq (%rsp), %rdx + subq %rdx, %r8 + movq 8(%rsi), %r9 + movq 8(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, (%rdi) + movq 16(%rsi), %r8 + movq 16(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 8(%rdi) + movq 24(%rsi), %r9 + movq 24(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 16(%rdi) + movq 32(%rsi), %r8 + movq 32(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 24(%rdi) + movq 40(%rsi), %r9 + movq 40(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 32(%rdi) + movq 48(%rsi), %r8 + movq 48(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 40(%rdi) + movq 56(%rsi), %r9 + movq 56(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 48(%rdi) + movq 64(%rsi), %r8 + movq 64(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 56(%rdi) + movq 72(%rsi), %r9 + movq 72(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 64(%rdi) + movq 80(%rsi), %r8 + movq 80(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 72(%rdi) + movq 88(%rsi), %r9 + movq 88(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 80(%rdi) + movq 96(%rsi), %r8 + movq 96(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 88(%rdi) + movq 104(%rsi), %r9 + movq 104(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 96(%rdi) + movq 112(%rsi), %r8 + movq 112(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 104(%rdi) + movq 120(%rsi), %r9 + movq 120(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 112(%rdi) + movq 128(%rsi), %r8 + movq 128(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 120(%rdi) + movq 136(%rsi), %r9 + movq 136(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 128(%rdi) + movq 144(%rsi), %r8 + movq 144(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 136(%rdi) + movq 152(%rsi), %r9 + movq 152(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 144(%rdi) + movq 160(%rsi), %r8 + movq 160(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 152(%rdi) + movq 168(%rsi), %r9 + movq 168(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 160(%rdi) + movq 176(%rsi), %r8 + movq 176(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 168(%rdi) + movq 184(%rsi), %r9 + movq 184(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 176(%rdi) + movq 192(%rsi), %r8 + movq 192(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 184(%rdi) + movq 200(%rsi), %r9 + movq 200(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 192(%rdi) + movq 208(%rsi), %r8 + movq 208(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 200(%rdi) + movq 216(%rsi), %r9 + movq 216(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 208(%rdi) + movq 224(%rsi), %r8 + movq 224(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 216(%rdi) + movq 232(%rsi), %r9 + movq 232(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 224(%rdi) + movq 240(%rsi), %r8 + movq 240(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 232(%rdi) + movq 248(%rsi), %r9 + movq 248(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 240(%rdi) + movq %r9, 248(%rdi) + sbbq $0, %rax + addq $256, %rsp + repz retq +.size sp_2048_cond_sub_32,.-sp_2048_cond_sub_32 +/* Reduce the number back to 2048 bits using Montgomery reduction. + * + * a A single precision number to reduce in place. + * m The single precision number representing the modulus. + * mp The digit representing the negative inverse of m mod 2^n. + */ +.globl sp_2048_mont_reduce_32 +.type sp_2048_mont_reduce_32,@function +.align 16 +sp_2048_mont_reduce_32: + push %r12 + push %r13 + push %r14 + push %r15 + movq %rdx, %rcx + xorq %r15, %r15 + # i = 0 + movq $32, %r8 + movq (%rdi), %r13 + movq 8(%rdi), %r14 +L_mont_loop_32: + # mu = a[i] * mp + movq %r13, %r11 + imulq %rcx, %r11 + # a[i+0] += m[0] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq (%rsi) + addq %rax, %r13 + adcq %rdx, %r10 + # a[i+1] += m[1] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 8(%rsi) + movq %r14, %r13 + addq %rax, %r13 + adcq %rdx, %r9 + addq %r10, %r13 + adcq $0, %r9 + # a[i+2] += m[2] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 16(%rsi) + movq 16(%rdi), %r14 + addq %rax, %r14 + adcq %rdx, %r10 + addq %r9, %r14 + adcq $0, %r10 + # a[i+3] += m[3] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 24(%rsi) + movq 24(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r9 + addq %r10, %r12 + movq %r12, 24(%rdi) + adcq $0, %r9 + # a[i+4] += m[4] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 32(%rsi) + movq 32(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r10 + addq %r9, %r12 + movq %r12, 32(%rdi) + adcq $0, %r10 + # a[i+5] += m[5] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 40(%rsi) + movq 40(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r9 + addq %r10, %r12 + movq %r12, 40(%rdi) + adcq $0, %r9 + # a[i+6] += m[6] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 48(%rsi) + movq 48(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r10 + addq %r9, %r12 + movq %r12, 48(%rdi) + adcq $0, %r10 + # a[i+7] += m[7] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 56(%rsi) + movq 56(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r9 + addq %r10, %r12 + movq %r12, 56(%rdi) + adcq $0, %r9 + # a[i+8] += m[8] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 64(%rsi) + movq 64(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r10 + addq %r9, %r12 + movq %r12, 64(%rdi) + adcq $0, %r10 + # a[i+9] += m[9] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 72(%rsi) + movq 72(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r9 + addq %r10, %r12 + movq %r12, 72(%rdi) + adcq $0, %r9 + # a[i+10] += m[10] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 80(%rsi) + movq 80(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r10 + addq %r9, %r12 + movq %r12, 80(%rdi) + adcq $0, %r10 + # a[i+11] += m[11] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 88(%rsi) + movq 88(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r9 + addq %r10, %r12 + movq %r12, 88(%rdi) + adcq $0, %r9 + # a[i+12] += m[12] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 96(%rsi) + movq 96(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r10 + addq %r9, %r12 + movq %r12, 96(%rdi) + adcq $0, %r10 + # a[i+13] += m[13] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 104(%rsi) + movq 104(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r9 + addq %r10, %r12 + movq %r12, 104(%rdi) + adcq $0, %r9 + # a[i+14] += m[14] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 112(%rsi) + movq 112(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r10 + addq %r9, %r12 + movq %r12, 112(%rdi) + adcq $0, %r10 + # a[i+15] += m[15] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 120(%rsi) + movq 120(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r9 + addq %r10, %r12 + movq %r12, 120(%rdi) + adcq $0, %r9 + # a[i+16] += m[16] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 128(%rsi) + movq 128(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r10 + addq %r9, %r12 + movq %r12, 128(%rdi) + adcq $0, %r10 + # a[i+17] += m[17] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 136(%rsi) + movq 136(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r9 + addq %r10, %r12 + movq %r12, 136(%rdi) + adcq $0, %r9 + # a[i+18] += m[18] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 144(%rsi) + movq 144(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r10 + addq %r9, %r12 + movq %r12, 144(%rdi) + adcq $0, %r10 + # a[i+19] += m[19] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 152(%rsi) + movq 152(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r9 + addq %r10, %r12 + movq %r12, 152(%rdi) + adcq $0, %r9 + # a[i+20] += m[20] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 160(%rsi) + movq 160(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r10 + addq %r9, %r12 + movq %r12, 160(%rdi) + adcq $0, %r10 + # a[i+21] += m[21] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 168(%rsi) + movq 168(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r9 + addq %r10, %r12 + movq %r12, 168(%rdi) + adcq $0, %r9 + # a[i+22] += m[22] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 176(%rsi) + movq 176(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r10 + addq %r9, %r12 + movq %r12, 176(%rdi) + adcq $0, %r10 + # a[i+23] += m[23] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 184(%rsi) + movq 184(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r9 + addq %r10, %r12 + movq %r12, 184(%rdi) + adcq $0, %r9 + # a[i+24] += m[24] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 192(%rsi) + movq 192(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r10 + addq %r9, %r12 + movq %r12, 192(%rdi) + adcq $0, %r10 + # a[i+25] += m[25] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 200(%rsi) + movq 200(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r9 + addq %r10, %r12 + movq %r12, 200(%rdi) + adcq $0, %r9 + # a[i+26] += m[26] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 208(%rsi) + movq 208(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r10 + addq %r9, %r12 + movq %r12, 208(%rdi) + adcq $0, %r10 + # a[i+27] += m[27] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 216(%rsi) + movq 216(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r9 + addq %r10, %r12 + movq %r12, 216(%rdi) + adcq $0, %r9 + # a[i+28] += m[28] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 224(%rsi) + movq 224(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r10 + addq %r9, %r12 + movq %r12, 224(%rdi) + adcq $0, %r10 + # a[i+29] += m[29] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 232(%rsi) + movq 232(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r9 + addq %r10, %r12 + movq %r12, 232(%rdi) + adcq $0, %r9 + # a[i+30] += m[30] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 240(%rsi) + movq 240(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r10 + addq %r9, %r12 + movq %r12, 240(%rdi) + adcq $0, %r10 + # a[i+31] += m[31] * mu + movq %r11, %rax + mulq 248(%rsi) + movq 248(%rdi), %r12 + addq %rax, %r10 + adcq %r15, %rdx + movq $0, %r15 + adcq $0, %r15 + addq %r10, %r12 + movq %r12, 248(%rdi) + adcq %rdx, 256(%rdi) + adcq $0, %r15 + # i += 1 + addq $8, %rdi + decq %r8 + jnz L_mont_loop_32 + movq %r13, (%rdi) + movq %r14, 8(%rdi) + negq %r15 + movq %r15, %rcx + movq %rsi, %rdx + movq %rdi, %rsi + movq %rdi, %rdi + subq $256, %rdi + callq sp_2048_cond_sub_32@plt + pop %r15 + pop %r14 + pop %r13 + pop %r12 + repz retq +.size sp_2048_mont_reduce_32,.-sp_2048_mont_reduce_32 +/* Mul a by digit b into r. (r = a * b) + * + * r A single precision integer. + * a A single precision integer. + * b A single precision digit. + */ +.globl sp_2048_mul_d_32 +.type sp_2048_mul_d_32,@function +.align 16 +sp_2048_mul_d_32: + movq %rdx, %rcx + # A[0] * B + movq %rcx, %rax + xorq %r10, %r10 + mulq (%rsi) + movq %rax, %r8 + movq %rdx, %r9 + movq %r8, (%rdi) + # A[1] * B + movq %rcx, %rax + xorq %r8, %r8 + mulq 8(%rsi) + addq %rax, %r9 + movq %r9, 8(%rdi) + adcq %rdx, %r10 + adcq $0, %r8 + # A[2] * B + movq %rcx, %rax + xorq %r9, %r9 + mulq 16(%rsi) + addq %rax, %r10 + movq %r10, 16(%rdi) + adcq %rdx, %r8 + adcq $0, %r9 + # A[3] * B + movq %rcx, %rax + xorq %r10, %r10 + mulq 24(%rsi) + addq %rax, %r8 + movq %r8, 24(%rdi) + adcq %rdx, %r9 + adcq $0, %r10 + # A[4] * B + movq %rcx, %rax + xorq %r8, %r8 + mulq 32(%rsi) + addq %rax, %r9 + movq %r9, 32(%rdi) + adcq %rdx, %r10 + adcq $0, %r8 + # A[5] * B + movq %rcx, %rax + xorq %r9, %r9 + mulq 40(%rsi) + addq %rax, %r10 + movq %r10, 40(%rdi) + adcq %rdx, %r8 + adcq $0, %r9 + # A[6] * B + movq %rcx, %rax + xorq %r10, %r10 + mulq 48(%rsi) + addq %rax, %r8 + movq %r8, 48(%rdi) + adcq %rdx, %r9 + adcq $0, %r10 + # A[7] * B + movq %rcx, %rax + xorq %r8, %r8 + mulq 56(%rsi) + addq %rax, %r9 + movq %r9, 56(%rdi) + adcq %rdx, %r10 + adcq $0, %r8 + # A[8] * B + movq %rcx, %rax + xorq %r9, %r9 + mulq 64(%rsi) + addq %rax, %r10 + movq %r10, 64(%rdi) + adcq %rdx, %r8 + adcq $0, %r9 + # A[9] * B + movq %rcx, %rax + xorq %r10, %r10 + mulq 72(%rsi) + addq %rax, %r8 + movq %r8, 72(%rdi) + adcq %rdx, %r9 + adcq $0, %r10 + # A[10] * B + movq %rcx, %rax + xorq %r8, %r8 + mulq 80(%rsi) + addq %rax, %r9 + movq %r9, 80(%rdi) + adcq %rdx, %r10 + adcq $0, %r8 + # A[11] * B + movq %rcx, %rax + xorq %r9, %r9 + mulq 88(%rsi) + addq %rax, %r10 + movq %r10, 88(%rdi) + adcq %rdx, %r8 + adcq $0, %r9 + # A[12] * B + movq %rcx, %rax + xorq %r10, %r10 + mulq 96(%rsi) + addq %rax, %r8 + movq %r8, 96(%rdi) + adcq %rdx, %r9 + adcq $0, %r10 + # A[13] * B + movq %rcx, %rax + xorq %r8, %r8 + mulq 104(%rsi) + addq %rax, %r9 + movq %r9, 104(%rdi) + adcq %rdx, %r10 + adcq $0, %r8 + # A[14] * B + movq %rcx, %rax + xorq %r9, %r9 + mulq 112(%rsi) + addq %rax, %r10 + movq %r10, 112(%rdi) + adcq %rdx, %r8 + adcq $0, %r9 + # A[15] * B + movq %rcx, %rax + xorq %r10, %r10 + mulq 120(%rsi) + addq %rax, %r8 + movq %r8, 120(%rdi) + adcq %rdx, %r9 + adcq $0, %r10 + # A[16] * B + movq %rcx, %rax + xorq %r8, %r8 + mulq 128(%rsi) + addq %rax, %r9 + movq %r9, 128(%rdi) + adcq %rdx, %r10 + adcq $0, %r8 + # A[17] * B + movq %rcx, %rax + xorq %r9, %r9 + mulq 136(%rsi) + addq %rax, %r10 + movq %r10, 136(%rdi) + adcq %rdx, %r8 + adcq $0, %r9 + # A[18] * B + movq %rcx, %rax + xorq %r10, %r10 + mulq 144(%rsi) + addq %rax, %r8 + movq %r8, 144(%rdi) + adcq %rdx, %r9 + adcq $0, %r10 + # A[19] * B + movq %rcx, %rax + xorq %r8, %r8 + mulq 152(%rsi) + addq %rax, %r9 + movq %r9, 152(%rdi) + adcq %rdx, %r10 + adcq $0, %r8 + # A[20] * B + movq %rcx, %rax + xorq %r9, %r9 + mulq 160(%rsi) + addq %rax, %r10 + movq %r10, 160(%rdi) + adcq %rdx, %r8 + adcq $0, %r9 + # A[21] * B + movq %rcx, %rax + xorq %r10, %r10 + mulq 168(%rsi) + addq %rax, %r8 + movq %r8, 168(%rdi) + adcq %rdx, %r9 + adcq $0, %r10 + # A[22] * B + movq %rcx, %rax + xorq %r8, %r8 + mulq 176(%rsi) + addq %rax, %r9 + movq %r9, 176(%rdi) + adcq %rdx, %r10 + adcq $0, %r8 + # A[23] * B + movq %rcx, %rax + xorq %r9, %r9 + mulq 184(%rsi) + addq %rax, %r10 + movq %r10, 184(%rdi) + adcq %rdx, %r8 + adcq $0, %r9 + # A[24] * B + movq %rcx, %rax + xorq %r10, %r10 + mulq 192(%rsi) + addq %rax, %r8 + movq %r8, 192(%rdi) + adcq %rdx, %r9 + adcq $0, %r10 + # A[25] * B + movq %rcx, %rax + xorq %r8, %r8 + mulq 200(%rsi) + addq %rax, %r9 + movq %r9, 200(%rdi) + adcq %rdx, %r10 + adcq $0, %r8 + # A[26] * B + movq %rcx, %rax + xorq %r9, %r9 + mulq 208(%rsi) + addq %rax, %r10 + movq %r10, 208(%rdi) + adcq %rdx, %r8 + adcq $0, %r9 + # A[27] * B + movq %rcx, %rax + xorq %r10, %r10 + mulq 216(%rsi) + addq %rax, %r8 + movq %r8, 216(%rdi) + adcq %rdx, %r9 + adcq $0, %r10 + # A[28] * B + movq %rcx, %rax + xorq %r8, %r8 + mulq 224(%rsi) + addq %rax, %r9 + movq %r9, 224(%rdi) + adcq %rdx, %r10 + adcq $0, %r8 + # A[29] * B + movq %rcx, %rax + xorq %r9, %r9 + mulq 232(%rsi) + addq %rax, %r10 + movq %r10, 232(%rdi) + adcq %rdx, %r8 + adcq $0, %r9 + # A[30] * B + movq %rcx, %rax + xorq %r10, %r10 + mulq 240(%rsi) + addq %rax, %r8 + movq %r8, 240(%rdi) + adcq %rdx, %r9 + adcq $0, %r10 + # # A[31] * B + movq %rcx, %rax + mulq 248(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + movq %r9, 248(%rdi) + movq %r10, 256(%rdi) + repz retq +.size sp_2048_mul_d_32,.-sp_2048_mul_d_32 +#ifdef HAVE_INTEL_AVX2 +/* Mul a by digit b into r. (r = a * b) + * + * r A single precision integer. + * a A single precision integer. + * b A single precision digit. + */ +.globl sp_2048_mul_d_avx2_32 +.type sp_2048_mul_d_avx2_32,@function +.align 16 +sp_2048_mul_d_avx2_32: + movq %rdx, %rax + # A[0] * B + movq %rax, %rdx + xorq %r11, %r11 + mulxq (%rsi), %r9, %r10 + movq %r9, (%rdi) + # A[1] * B + mulxq 8(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 8(%rdi) + adoxq %r8, %r9 + # A[2] * B + mulxq 16(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 16(%rdi) + adoxq %r8, %r10 + # A[3] * B + mulxq 24(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 24(%rdi) + adoxq %r8, %r9 + # A[4] * B + mulxq 32(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 32(%rdi) + adoxq %r8, %r10 + # A[5] * B + mulxq 40(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 40(%rdi) + adoxq %r8, %r9 + # A[6] * B + mulxq 48(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 48(%rdi) + adoxq %r8, %r10 + # A[7] * B + mulxq 56(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 56(%rdi) + adoxq %r8, %r9 + # A[8] * B + mulxq 64(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 64(%rdi) + adoxq %r8, %r10 + # A[9] * B + mulxq 72(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 72(%rdi) + adoxq %r8, %r9 + # A[10] * B + mulxq 80(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 80(%rdi) + adoxq %r8, %r10 + # A[11] * B + mulxq 88(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 88(%rdi) + adoxq %r8, %r9 + # A[12] * B + mulxq 96(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 96(%rdi) + adoxq %r8, %r10 + # A[13] * B + mulxq 104(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 104(%rdi) + adoxq %r8, %r9 + # A[14] * B + mulxq 112(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 112(%rdi) + adoxq %r8, %r10 + # A[15] * B + mulxq 120(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 120(%rdi) + adoxq %r8, %r9 + # A[16] * B + mulxq 128(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 128(%rdi) + adoxq %r8, %r10 + # A[17] * B + mulxq 136(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 136(%rdi) + adoxq %r8, %r9 + # A[18] * B + mulxq 144(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 144(%rdi) + adoxq %r8, %r10 + # A[19] * B + mulxq 152(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 152(%rdi) + adoxq %r8, %r9 + # A[20] * B + mulxq 160(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 160(%rdi) + adoxq %r8, %r10 + # A[21] * B + mulxq 168(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 168(%rdi) + adoxq %r8, %r9 + # A[22] * B + mulxq 176(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 176(%rdi) + adoxq %r8, %r10 + # A[23] * B + mulxq 184(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 184(%rdi) + adoxq %r8, %r9 + # A[24] * B + mulxq 192(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 192(%rdi) + adoxq %r8, %r10 + # A[25] * B + mulxq 200(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 200(%rdi) + adoxq %r8, %r9 + # A[26] * B + mulxq 208(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 208(%rdi) + adoxq %r8, %r10 + # A[27] * B + mulxq 216(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 216(%rdi) + adoxq %r8, %r9 + # A[28] * B + mulxq 224(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 224(%rdi) + adoxq %r8, %r10 + # A[29] * B + mulxq 232(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 232(%rdi) + adoxq %r8, %r9 + # A[30] * B + mulxq 240(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 240(%rdi) + adoxq %r8, %r10 + # A[31] * B + mulxq 248(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + adoxq %r8, %r9 + adcxq %r11, %r9 + movq %r10, 248(%rdi) + movq %r9, 256(%rdi) + repz retq +.size sp_2048_mul_d_avx2_32,.-sp_2048_mul_d_avx2_32 +#endif /* HAVE_INTEL_AVX2 */ +/* Compare a with b in constant time. + * + * a A single precision integer. + * b A single precision integer. + * return -ve, 0 or +ve if a is less than, equal to or greater than b + * respectively. + */ +.globl sp_2048_cmp_32 +.type sp_2048_cmp_32,@function +.align 16 +sp_2048_cmp_32: + xorq %rcx, %rcx + movq $-1, %rdx + movq $-1, %rax + movq $1, %r8 + movq 248(%rdi), %r9 + movq 248(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 240(%rdi), %r9 + movq 240(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 232(%rdi), %r9 + movq 232(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 224(%rdi), %r9 + movq 224(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 216(%rdi), %r9 + movq 216(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 208(%rdi), %r9 + movq 208(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 200(%rdi), %r9 + movq 200(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 192(%rdi), %r9 + movq 192(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 184(%rdi), %r9 + movq 184(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 176(%rdi), %r9 + movq 176(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 168(%rdi), %r9 + movq 168(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 160(%rdi), %r9 + movq 160(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 152(%rdi), %r9 + movq 152(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 144(%rdi), %r9 + movq 144(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 136(%rdi), %r9 + movq 136(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 128(%rdi), %r9 + movq 128(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 120(%rdi), %r9 + movq 120(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 112(%rdi), %r9 + movq 112(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 104(%rdi), %r9 + movq 104(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 96(%rdi), %r9 + movq 96(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 88(%rdi), %r9 + movq 88(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 80(%rdi), %r9 + movq 80(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 72(%rdi), %r9 + movq 72(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 64(%rdi), %r9 + movq 64(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 56(%rdi), %r9 + movq 56(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 48(%rdi), %r9 + movq 48(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 40(%rdi), %r9 + movq 40(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 32(%rdi), %r9 + movq 32(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 24(%rdi), %r9 + movq 24(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 16(%rdi), %r9 + movq 16(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 8(%rdi), %r9 + movq 8(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq (%rdi), %r9 + movq (%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + xorq %rdx, %rax + repz retq +.size sp_2048_cmp_32,.-sp_2048_cmp_32 +#if defined(SP_RSA_PRIVATE_EXP_D) || defined(WOLFSSL_HAVE_SP_DH) +#endif /* SP_RSA_PRIVATE_EXP_D || WOLFSSL_HAVE_SP_DH */ +#ifdef HAVE_INTEL_AVX2 +/* Reduce the number back to 2048 bits using Montgomery reduction. + * + * a A single precision number to reduce in place. + * m The single precision number representing the modulus. + * mp The digit representing the negative inverse of m mod 2^n. + */ +.globl sp_2048_mont_reduce_avx2_32 +.type sp_2048_mont_reduce_avx2_32,@function +.align 16 +sp_2048_mont_reduce_avx2_32: + push %r12 + push %r13 + push %r14 + movq %rdx, %rax + xorq %r14, %r14 + # i = 0 + movq $32, %r9 + movq (%rdi), %r13 + xorq %r12, %r12 +L_mont_loop_avx2_32: + # mu = a[i] * mp + movq %r13, %rdx + mulxq %rax, %rdx, %r8 + movq %r13, %r10 + # a[i+0] += m[0] * mu + mulxq (%rsi), %rcx, %r8 + movq 8(%rdi), %r13 + adcxq %rcx, %r10 + adoxq %r8, %r13 + # a[i+1] += m[1] * mu + mulxq 8(%rsi), %rcx, %r8 + movq 16(%rdi), %r10 + adcxq %rcx, %r13 + adoxq %r8, %r10 + # a[i+2] += m[2] * mu + mulxq 16(%rsi), %rcx, %r8 + movq 24(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 16(%rdi) + # a[i+3] += m[3] * mu + mulxq 24(%rsi), %rcx, %r8 + movq 32(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 24(%rdi) + # a[i+4] += m[4] * mu + mulxq 32(%rsi), %rcx, %r8 + movq 40(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 32(%rdi) + # a[i+5] += m[5] * mu + mulxq 40(%rsi), %rcx, %r8 + movq 48(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 40(%rdi) + # a[i+6] += m[6] * mu + mulxq 48(%rsi), %rcx, %r8 + movq 56(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 48(%rdi) + # a[i+7] += m[7] * mu + mulxq 56(%rsi), %rcx, %r8 + movq 64(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 56(%rdi) + # a[i+8] += m[8] * mu + mulxq 64(%rsi), %rcx, %r8 + movq 72(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 64(%rdi) + # a[i+9] += m[9] * mu + mulxq 72(%rsi), %rcx, %r8 + movq 80(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 72(%rdi) + # a[i+10] += m[10] * mu + mulxq 80(%rsi), %rcx, %r8 + movq 88(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 80(%rdi) + # a[i+11] += m[11] * mu + mulxq 88(%rsi), %rcx, %r8 + movq 96(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 88(%rdi) + # a[i+12] += m[12] * mu + mulxq 96(%rsi), %rcx, %r8 + movq 104(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 96(%rdi) + # a[i+13] += m[13] * mu + mulxq 104(%rsi), %rcx, %r8 + movq 112(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 104(%rdi) + # a[i+14] += m[14] * mu + mulxq 112(%rsi), %rcx, %r8 + movq 120(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 112(%rdi) + # a[i+15] += m[15] * mu + mulxq 120(%rsi), %rcx, %r8 + movq 128(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 120(%rdi) + # a[i+16] += m[16] * mu + mulxq 128(%rsi), %rcx, %r8 + movq 136(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 128(%rdi) + # a[i+17] += m[17] * mu + mulxq 136(%rsi), %rcx, %r8 + movq 144(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 136(%rdi) + # a[i+18] += m[18] * mu + mulxq 144(%rsi), %rcx, %r8 + movq 152(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 144(%rdi) + # a[i+19] += m[19] * mu + mulxq 152(%rsi), %rcx, %r8 + movq 160(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 152(%rdi) + # a[i+20] += m[20] * mu + mulxq 160(%rsi), %rcx, %r8 + movq 168(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 160(%rdi) + # a[i+21] += m[21] * mu + mulxq 168(%rsi), %rcx, %r8 + movq 176(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 168(%rdi) + # a[i+22] += m[22] * mu + mulxq 176(%rsi), %rcx, %r8 + movq 184(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 176(%rdi) + # a[i+23] += m[23] * mu + mulxq 184(%rsi), %rcx, %r8 + movq 192(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 184(%rdi) + # a[i+24] += m[24] * mu + mulxq 192(%rsi), %rcx, %r8 + movq 200(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 192(%rdi) + # a[i+25] += m[25] * mu + mulxq 200(%rsi), %rcx, %r8 + movq 208(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 200(%rdi) + # a[i+26] += m[26] * mu + mulxq 208(%rsi), %rcx, %r8 + movq 216(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 208(%rdi) + # a[i+27] += m[27] * mu + mulxq 216(%rsi), %rcx, %r8 + movq 224(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 216(%rdi) + # a[i+28] += m[28] * mu + mulxq 224(%rsi), %rcx, %r8 + movq 232(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 224(%rdi) + # a[i+29] += m[29] * mu + mulxq 232(%rsi), %rcx, %r8 + movq 240(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 232(%rdi) + # a[i+30] += m[30] * mu + mulxq 240(%rsi), %rcx, %r8 + movq 248(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 240(%rdi) + # a[i+31] += m[31] * mu + mulxq 248(%rsi), %rcx, %r8 + movq 256(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 248(%rdi) + adcxq %r14, %r10 + movq %r12, %r14 + adoxq %r12, %r14 + adcxq %r12, %r14 + movq %r10, 256(%rdi) + # i += 1 + addq $8, %rdi + decq %r9 + jnz L_mont_loop_avx2_32 + movq %r13, (%rdi) + negq %r14 + movq %r14, %rcx + movq %rsi, %rdx + movq %rdi, %rsi + movq %rdi, %rdi + subq $256, %rdi + callq sp_2048_cond_sub_32@plt + pop %r14 + pop %r13 + pop %r12 + repz retq +.size sp_2048_mont_reduce_avx2_32,.-sp_2048_mont_reduce_avx2_32 +#endif /* HAVE_INTEL_AVX2 */ +#ifdef HAVE_INTEL_AVX2 +#endif /* HAVE_INTEL_AVX2 */ +#ifdef HAVE_INTEL_AVX2 +#endif /* HAVE_INTEL_AVX2 */ +#if defined(SP_RSA_PRIVATE_EXP_D) || defined(WOLFSSL_HAVE_SP_DH) +#ifdef HAVE_INTEL_AVX2 +#endif /* HAVE_INTEL_AVX2 */ +#endif /* SP_RSA_PRIVATE_EXP_D || WOLFSSL_HAVE_SP_DH */ +#ifdef WOLFSSL_HAVE_SP_RSA +#endif /* WOLFSSL_HAVE_SP_RSA */ +#endif /* !WOLFSSL_SP_NO_2048 */ +#ifndef WOLFSSL_SP_NO_3072 +/* Multiply a and b into r. (r = a * b) + * + * r A single precision integer. + * a A single precision integer. + * b A single precision integer. + */ +.globl sp_3072_mul_24 +.type sp_3072_mul_24,@function +.align 16 +sp_3072_mul_24: + movq %rdx, %rcx + subq $192, %rsp + # A[0] * B[0] + movq (%rcx), %rax + mulq (%rsi) + xorq %r10, %r10 + movq %rax, (%rsp) + movq %rdx, %r9 + # A[0] * B[1] + movq 8(%rcx), %rax + mulq (%rsi) + xorq %r8, %r8 + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[1] * B[0] + movq (%rcx), %rax + mulq 8(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + movq %r9, 8(%rsp) + # A[0] * B[2] + movq 16(%rcx), %rax + mulq (%rsi) + xorq %r9, %r9 + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[1] * B[1] + movq 8(%rcx), %rax + mulq 8(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[2] * B[0] + movq (%rcx), %rax + mulq 16(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + movq %r10, 16(%rsp) + # A[0] * B[3] + movq 24(%rcx), %rax + mulq (%rsi) + xorq %r10, %r10 + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[1] * B[2] + movq 16(%rcx), %rax + mulq 8(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[2] * B[1] + movq 8(%rcx), %rax + mulq 16(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[3] * B[0] + movq (%rcx), %rax + mulq 24(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + movq %r8, 24(%rsp) + # A[0] * B[4] + movq 32(%rcx), %rax + mulq (%rsi) + xorq %r8, %r8 + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[1] * B[3] + movq 24(%rcx), %rax + mulq 8(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[2] * B[2] + movq 16(%rcx), %rax + mulq 16(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[3] * B[1] + movq 8(%rcx), %rax + mulq 24(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[4] * B[0] + movq (%rcx), %rax + mulq 32(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + movq %r9, 32(%rsp) + # A[0] * B[5] + movq 40(%rcx), %rax + mulq (%rsi) + xorq %r9, %r9 + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[1] * B[4] + movq 32(%rcx), %rax + mulq 8(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[2] * B[3] + movq 24(%rcx), %rax + mulq 16(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[3] * B[2] + movq 16(%rcx), %rax + mulq 24(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[4] * B[1] + movq 8(%rcx), %rax + mulq 32(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[5] * B[0] + movq (%rcx), %rax + mulq 40(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + movq %r10, 40(%rsp) + # A[0] * B[6] + movq 48(%rcx), %rax + mulq (%rsi) + xorq %r10, %r10 + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[1] * B[5] + movq 40(%rcx), %rax + mulq 8(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[2] * B[4] + movq 32(%rcx), %rax + mulq 16(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[3] * B[3] + movq 24(%rcx), %rax + mulq 24(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[4] * B[2] + movq 16(%rcx), %rax + mulq 32(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[5] * B[1] + movq 8(%rcx), %rax + mulq 40(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[6] * B[0] + movq (%rcx), %rax + mulq 48(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + movq %r8, 48(%rsp) + # A[0] * B[7] + movq 56(%rcx), %rax + mulq (%rsi) + xorq %r8, %r8 + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[1] * B[6] + movq 48(%rcx), %rax + mulq 8(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[2] * B[5] + movq 40(%rcx), %rax + mulq 16(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[3] * B[4] + movq 32(%rcx), %rax + mulq 24(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[4] * B[3] + movq 24(%rcx), %rax + mulq 32(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[5] * B[2] + movq 16(%rcx), %rax + mulq 40(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[6] * B[1] + movq 8(%rcx), %rax + mulq 48(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[7] * B[0] + movq (%rcx), %rax + mulq 56(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + movq %r9, 56(%rsp) + # A[0] * B[8] + movq 64(%rcx), %rax + mulq (%rsi) + xorq %r9, %r9 + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[1] * B[7] + movq 56(%rcx), %rax + mulq 8(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[2] * B[6] + movq 48(%rcx), %rax + mulq 16(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[3] * B[5] + movq 40(%rcx), %rax + mulq 24(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[4] * B[4] + movq 32(%rcx), %rax + mulq 32(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[5] * B[3] + movq 24(%rcx), %rax + mulq 40(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[6] * B[2] + movq 16(%rcx), %rax + mulq 48(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[7] * B[1] + movq 8(%rcx), %rax + mulq 56(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[8] * B[0] + movq (%rcx), %rax + mulq 64(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + movq %r10, 64(%rsp) + # A[0] * B[9] + movq 72(%rcx), %rax + mulq (%rsi) + xorq %r10, %r10 + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[1] * B[8] + movq 64(%rcx), %rax + mulq 8(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[2] * B[7] + movq 56(%rcx), %rax + mulq 16(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[3] * B[6] + movq 48(%rcx), %rax + mulq 24(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[4] * B[5] + movq 40(%rcx), %rax + mulq 32(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[5] * B[4] + movq 32(%rcx), %rax + mulq 40(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[6] * B[3] + movq 24(%rcx), %rax + mulq 48(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[7] * B[2] + movq 16(%rcx), %rax + mulq 56(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[8] * B[1] + movq 8(%rcx), %rax + mulq 64(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[9] * B[0] + movq (%rcx), %rax + mulq 72(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + movq %r8, 72(%rsp) + # A[0] * B[10] + movq 80(%rcx), %rax + mulq (%rsi) + xorq %r8, %r8 + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[1] * B[9] + movq 72(%rcx), %rax + mulq 8(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[2] * B[8] + movq 64(%rcx), %rax + mulq 16(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[3] * B[7] + movq 56(%rcx), %rax + mulq 24(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[4] * B[6] + movq 48(%rcx), %rax + mulq 32(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[5] * B[5] + movq 40(%rcx), %rax + mulq 40(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[6] * B[4] + movq 32(%rcx), %rax + mulq 48(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[7] * B[3] + movq 24(%rcx), %rax + mulq 56(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[8] * B[2] + movq 16(%rcx), %rax + mulq 64(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[9] * B[1] + movq 8(%rcx), %rax + mulq 72(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[10] * B[0] + movq (%rcx), %rax + mulq 80(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + movq %r9, 80(%rsp) + # A[0] * B[11] + movq 88(%rcx), %rax + mulq (%rsi) + xorq %r9, %r9 + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[1] * B[10] + movq 80(%rcx), %rax + mulq 8(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[2] * B[9] + movq 72(%rcx), %rax + mulq 16(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[3] * B[8] + movq 64(%rcx), %rax + mulq 24(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[4] * B[7] + movq 56(%rcx), %rax + mulq 32(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[5] * B[6] + movq 48(%rcx), %rax + mulq 40(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[6] * B[5] + movq 40(%rcx), %rax + mulq 48(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[7] * B[4] + movq 32(%rcx), %rax + mulq 56(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[8] * B[3] + movq 24(%rcx), %rax + mulq 64(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[9] * B[2] + movq 16(%rcx), %rax + mulq 72(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[10] * B[1] + movq 8(%rcx), %rax + mulq 80(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[11] * B[0] + movq (%rcx), %rax + mulq 88(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + movq %r10, 88(%rsp) + # A[0] * B[12] + movq 96(%rcx), %rax + mulq (%rsi) + xorq %r10, %r10 + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[1] * B[11] + movq 88(%rcx), %rax + mulq 8(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[2] * B[10] + movq 80(%rcx), %rax + mulq 16(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[3] * B[9] + movq 72(%rcx), %rax + mulq 24(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[4] * B[8] + movq 64(%rcx), %rax + mulq 32(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[5] * B[7] + movq 56(%rcx), %rax + mulq 40(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[6] * B[6] + movq 48(%rcx), %rax + mulq 48(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[7] * B[5] + movq 40(%rcx), %rax + mulq 56(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[8] * B[4] + movq 32(%rcx), %rax + mulq 64(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[9] * B[3] + movq 24(%rcx), %rax + mulq 72(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[10] * B[2] + movq 16(%rcx), %rax + mulq 80(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[11] * B[1] + movq 8(%rcx), %rax + mulq 88(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[12] * B[0] + movq (%rcx), %rax + mulq 96(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + movq %r8, 96(%rsp) + # A[0] * B[13] + movq 104(%rcx), %rax + mulq (%rsi) + xorq %r8, %r8 + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[1] * B[12] + movq 96(%rcx), %rax + mulq 8(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[2] * B[11] + movq 88(%rcx), %rax + mulq 16(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[3] * B[10] + movq 80(%rcx), %rax + mulq 24(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[4] * B[9] + movq 72(%rcx), %rax + mulq 32(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[5] * B[8] + movq 64(%rcx), %rax + mulq 40(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[6] * B[7] + movq 56(%rcx), %rax + mulq 48(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[7] * B[6] + movq 48(%rcx), %rax + mulq 56(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[8] * B[5] + movq 40(%rcx), %rax + mulq 64(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[9] * B[4] + movq 32(%rcx), %rax + mulq 72(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[10] * B[3] + movq 24(%rcx), %rax + mulq 80(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[11] * B[2] + movq 16(%rcx), %rax + mulq 88(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[12] * B[1] + movq 8(%rcx), %rax + mulq 96(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[13] * B[0] + movq (%rcx), %rax + mulq 104(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + movq %r9, 104(%rsp) + # A[0] * B[14] + movq 112(%rcx), %rax + mulq (%rsi) + xorq %r9, %r9 + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[1] * B[13] + movq 104(%rcx), %rax + mulq 8(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[2] * B[12] + movq 96(%rcx), %rax + mulq 16(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[3] * B[11] + movq 88(%rcx), %rax + mulq 24(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[4] * B[10] + movq 80(%rcx), %rax + mulq 32(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[5] * B[9] + movq 72(%rcx), %rax + mulq 40(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[6] * B[8] + movq 64(%rcx), %rax + mulq 48(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[7] * B[7] + movq 56(%rcx), %rax + mulq 56(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[8] * B[6] + movq 48(%rcx), %rax + mulq 64(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[9] * B[5] + movq 40(%rcx), %rax + mulq 72(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[10] * B[4] + movq 32(%rcx), %rax + mulq 80(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[11] * B[3] + movq 24(%rcx), %rax + mulq 88(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[12] * B[2] + movq 16(%rcx), %rax + mulq 96(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[13] * B[1] + movq 8(%rcx), %rax + mulq 104(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[14] * B[0] + movq (%rcx), %rax + mulq 112(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + movq %r10, 112(%rsp) + # A[0] * B[15] + movq 120(%rcx), %rax + mulq (%rsi) + xorq %r10, %r10 + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[1] * B[14] + movq 112(%rcx), %rax + mulq 8(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[2] * B[13] + movq 104(%rcx), %rax + mulq 16(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[3] * B[12] + movq 96(%rcx), %rax + mulq 24(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[4] * B[11] + movq 88(%rcx), %rax + mulq 32(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[5] * B[10] + movq 80(%rcx), %rax + mulq 40(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[6] * B[9] + movq 72(%rcx), %rax + mulq 48(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[7] * B[8] + movq 64(%rcx), %rax + mulq 56(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[8] * B[7] + movq 56(%rcx), %rax + mulq 64(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[9] * B[6] + movq 48(%rcx), %rax + mulq 72(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[10] * B[5] + movq 40(%rcx), %rax + mulq 80(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[11] * B[4] + movq 32(%rcx), %rax + mulq 88(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[12] * B[3] + movq 24(%rcx), %rax + mulq 96(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[13] * B[2] + movq 16(%rcx), %rax + mulq 104(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[14] * B[1] + movq 8(%rcx), %rax + mulq 112(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[15] * B[0] + movq (%rcx), %rax + mulq 120(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + movq %r8, 120(%rsp) + # A[0] * B[16] + movq 128(%rcx), %rax + mulq (%rsi) + xorq %r8, %r8 + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[1] * B[15] + movq 120(%rcx), %rax + mulq 8(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[2] * B[14] + movq 112(%rcx), %rax + mulq 16(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[3] * B[13] + movq 104(%rcx), %rax + mulq 24(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[4] * B[12] + movq 96(%rcx), %rax + mulq 32(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[5] * B[11] + movq 88(%rcx), %rax + mulq 40(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[6] * B[10] + movq 80(%rcx), %rax + mulq 48(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[7] * B[9] + movq 72(%rcx), %rax + mulq 56(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[8] * B[8] + movq 64(%rcx), %rax + mulq 64(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[9] * B[7] + movq 56(%rcx), %rax + mulq 72(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[10] * B[6] + movq 48(%rcx), %rax + mulq 80(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[11] * B[5] + movq 40(%rcx), %rax + mulq 88(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[12] * B[4] + movq 32(%rcx), %rax + mulq 96(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[13] * B[3] + movq 24(%rcx), %rax + mulq 104(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[14] * B[2] + movq 16(%rcx), %rax + mulq 112(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[15] * B[1] + movq 8(%rcx), %rax + mulq 120(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[16] * B[0] + movq (%rcx), %rax + mulq 128(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + movq %r9, 128(%rsp) + # A[0] * B[17] + movq 136(%rcx), %rax + mulq (%rsi) + xorq %r9, %r9 + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[1] * B[16] + movq 128(%rcx), %rax + mulq 8(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[2] * B[15] + movq 120(%rcx), %rax + mulq 16(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[3] * B[14] + movq 112(%rcx), %rax + mulq 24(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[4] * B[13] + movq 104(%rcx), %rax + mulq 32(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[5] * B[12] + movq 96(%rcx), %rax + mulq 40(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[6] * B[11] + movq 88(%rcx), %rax + mulq 48(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[7] * B[10] + movq 80(%rcx), %rax + mulq 56(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[8] * B[9] + movq 72(%rcx), %rax + mulq 64(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[9] * B[8] + movq 64(%rcx), %rax + mulq 72(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[10] * B[7] + movq 56(%rcx), %rax + mulq 80(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[11] * B[6] + movq 48(%rcx), %rax + mulq 88(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[12] * B[5] + movq 40(%rcx), %rax + mulq 96(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[13] * B[4] + movq 32(%rcx), %rax + mulq 104(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[14] * B[3] + movq 24(%rcx), %rax + mulq 112(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[15] * B[2] + movq 16(%rcx), %rax + mulq 120(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[16] * B[1] + movq 8(%rcx), %rax + mulq 128(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[17] * B[0] + movq (%rcx), %rax + mulq 136(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + movq %r10, 136(%rsp) + # A[0] * B[18] + movq 144(%rcx), %rax + mulq (%rsi) + xorq %r10, %r10 + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[1] * B[17] + movq 136(%rcx), %rax + mulq 8(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[2] * B[16] + movq 128(%rcx), %rax + mulq 16(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[3] * B[15] + movq 120(%rcx), %rax + mulq 24(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[4] * B[14] + movq 112(%rcx), %rax + mulq 32(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[5] * B[13] + movq 104(%rcx), %rax + mulq 40(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[6] * B[12] + movq 96(%rcx), %rax + mulq 48(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[7] * B[11] + movq 88(%rcx), %rax + mulq 56(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[8] * B[10] + movq 80(%rcx), %rax + mulq 64(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[9] * B[9] + movq 72(%rcx), %rax + mulq 72(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[10] * B[8] + movq 64(%rcx), %rax + mulq 80(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[11] * B[7] + movq 56(%rcx), %rax + mulq 88(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[12] * B[6] + movq 48(%rcx), %rax + mulq 96(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[13] * B[5] + movq 40(%rcx), %rax + mulq 104(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[14] * B[4] + movq 32(%rcx), %rax + mulq 112(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[15] * B[3] + movq 24(%rcx), %rax + mulq 120(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[16] * B[2] + movq 16(%rcx), %rax + mulq 128(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[17] * B[1] + movq 8(%rcx), %rax + mulq 136(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[18] * B[0] + movq (%rcx), %rax + mulq 144(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + movq %r8, 144(%rsp) + # A[0] * B[19] + movq 152(%rcx), %rax + mulq (%rsi) + xorq %r8, %r8 + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[1] * B[18] + movq 144(%rcx), %rax + mulq 8(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[2] * B[17] + movq 136(%rcx), %rax + mulq 16(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[3] * B[16] + movq 128(%rcx), %rax + mulq 24(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[4] * B[15] + movq 120(%rcx), %rax + mulq 32(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[5] * B[14] + movq 112(%rcx), %rax + mulq 40(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[6] * B[13] + movq 104(%rcx), %rax + mulq 48(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[7] * B[12] + movq 96(%rcx), %rax + mulq 56(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[8] * B[11] + movq 88(%rcx), %rax + mulq 64(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[9] * B[10] + movq 80(%rcx), %rax + mulq 72(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[10] * B[9] + movq 72(%rcx), %rax + mulq 80(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[11] * B[8] + movq 64(%rcx), %rax + mulq 88(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[12] * B[7] + movq 56(%rcx), %rax + mulq 96(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[13] * B[6] + movq 48(%rcx), %rax + mulq 104(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[14] * B[5] + movq 40(%rcx), %rax + mulq 112(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[15] * B[4] + movq 32(%rcx), %rax + mulq 120(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[16] * B[3] + movq 24(%rcx), %rax + mulq 128(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[17] * B[2] + movq 16(%rcx), %rax + mulq 136(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[18] * B[1] + movq 8(%rcx), %rax + mulq 144(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[19] * B[0] + movq (%rcx), %rax + mulq 152(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + movq %r9, 152(%rsp) + # A[0] * B[20] + movq 160(%rcx), %rax + mulq (%rsi) + xorq %r9, %r9 + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[1] * B[19] + movq 152(%rcx), %rax + mulq 8(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[2] * B[18] + movq 144(%rcx), %rax + mulq 16(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[3] * B[17] + movq 136(%rcx), %rax + mulq 24(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[4] * B[16] + movq 128(%rcx), %rax + mulq 32(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[5] * B[15] + movq 120(%rcx), %rax + mulq 40(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[6] * B[14] + movq 112(%rcx), %rax + mulq 48(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[7] * B[13] + movq 104(%rcx), %rax + mulq 56(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[8] * B[12] + movq 96(%rcx), %rax + mulq 64(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[9] * B[11] + movq 88(%rcx), %rax + mulq 72(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[10] * B[10] + movq 80(%rcx), %rax + mulq 80(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[11] * B[9] + movq 72(%rcx), %rax + mulq 88(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[12] * B[8] + movq 64(%rcx), %rax + mulq 96(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[13] * B[7] + movq 56(%rcx), %rax + mulq 104(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[14] * B[6] + movq 48(%rcx), %rax + mulq 112(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[15] * B[5] + movq 40(%rcx), %rax + mulq 120(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[16] * B[4] + movq 32(%rcx), %rax + mulq 128(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[17] * B[3] + movq 24(%rcx), %rax + mulq 136(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[18] * B[2] + movq 16(%rcx), %rax + mulq 144(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[19] * B[1] + movq 8(%rcx), %rax + mulq 152(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[20] * B[0] + movq (%rcx), %rax + mulq 160(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + movq %r10, 160(%rsp) + # A[0] * B[21] + movq 168(%rcx), %rax + mulq (%rsi) + xorq %r10, %r10 + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[1] * B[20] + movq 160(%rcx), %rax + mulq 8(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[2] * B[19] + movq 152(%rcx), %rax + mulq 16(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[3] * B[18] + movq 144(%rcx), %rax + mulq 24(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[4] * B[17] + movq 136(%rcx), %rax + mulq 32(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[5] * B[16] + movq 128(%rcx), %rax + mulq 40(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[6] * B[15] + movq 120(%rcx), %rax + mulq 48(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[7] * B[14] + movq 112(%rcx), %rax + mulq 56(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[8] * B[13] + movq 104(%rcx), %rax + mulq 64(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[9] * B[12] + movq 96(%rcx), %rax + mulq 72(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[10] * B[11] + movq 88(%rcx), %rax + mulq 80(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[11] * B[10] + movq 80(%rcx), %rax + mulq 88(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[12] * B[9] + movq 72(%rcx), %rax + mulq 96(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[13] * B[8] + movq 64(%rcx), %rax + mulq 104(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[14] * B[7] + movq 56(%rcx), %rax + mulq 112(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[15] * B[6] + movq 48(%rcx), %rax + mulq 120(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[16] * B[5] + movq 40(%rcx), %rax + mulq 128(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[17] * B[4] + movq 32(%rcx), %rax + mulq 136(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[18] * B[3] + movq 24(%rcx), %rax + mulq 144(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[19] * B[2] + movq 16(%rcx), %rax + mulq 152(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[20] * B[1] + movq 8(%rcx), %rax + mulq 160(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[21] * B[0] + movq (%rcx), %rax + mulq 168(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + movq %r8, 168(%rsp) + # A[0] * B[22] + movq 176(%rcx), %rax + mulq (%rsi) + xorq %r8, %r8 + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[1] * B[21] + movq 168(%rcx), %rax + mulq 8(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[2] * B[20] + movq 160(%rcx), %rax + mulq 16(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[3] * B[19] + movq 152(%rcx), %rax + mulq 24(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[4] * B[18] + movq 144(%rcx), %rax + mulq 32(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[5] * B[17] + movq 136(%rcx), %rax + mulq 40(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[6] * B[16] + movq 128(%rcx), %rax + mulq 48(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[7] * B[15] + movq 120(%rcx), %rax + mulq 56(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[8] * B[14] + movq 112(%rcx), %rax + mulq 64(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[9] * B[13] + movq 104(%rcx), %rax + mulq 72(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[10] * B[12] + movq 96(%rcx), %rax + mulq 80(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[11] * B[11] + movq 88(%rcx), %rax + mulq 88(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[12] * B[10] + movq 80(%rcx), %rax + mulq 96(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[13] * B[9] + movq 72(%rcx), %rax + mulq 104(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[14] * B[8] + movq 64(%rcx), %rax + mulq 112(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[15] * B[7] + movq 56(%rcx), %rax + mulq 120(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[16] * B[6] + movq 48(%rcx), %rax + mulq 128(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[17] * B[5] + movq 40(%rcx), %rax + mulq 136(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[18] * B[4] + movq 32(%rcx), %rax + mulq 144(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[19] * B[3] + movq 24(%rcx), %rax + mulq 152(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[20] * B[2] + movq 16(%rcx), %rax + mulq 160(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[21] * B[1] + movq 8(%rcx), %rax + mulq 168(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[22] * B[0] + movq (%rcx), %rax + mulq 176(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + movq %r9, 176(%rsp) + # A[0] * B[23] + movq 184(%rcx), %rax + mulq (%rsi) + xorq %r9, %r9 + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[1] * B[22] + movq 176(%rcx), %rax + mulq 8(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[2] * B[21] + movq 168(%rcx), %rax + mulq 16(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[3] * B[20] + movq 160(%rcx), %rax + mulq 24(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[4] * B[19] + movq 152(%rcx), %rax + mulq 32(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[5] * B[18] + movq 144(%rcx), %rax + mulq 40(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[6] * B[17] + movq 136(%rcx), %rax + mulq 48(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[7] * B[16] + movq 128(%rcx), %rax + mulq 56(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[8] * B[15] + movq 120(%rcx), %rax + mulq 64(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[9] * B[14] + movq 112(%rcx), %rax + mulq 72(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[10] * B[13] + movq 104(%rcx), %rax + mulq 80(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[11] * B[12] + movq 96(%rcx), %rax + mulq 88(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[12] * B[11] + movq 88(%rcx), %rax + mulq 96(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[13] * B[10] + movq 80(%rcx), %rax + mulq 104(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[14] * B[9] + movq 72(%rcx), %rax + mulq 112(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[15] * B[8] + movq 64(%rcx), %rax + mulq 120(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[16] * B[7] + movq 56(%rcx), %rax + mulq 128(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[17] * B[6] + movq 48(%rcx), %rax + mulq 136(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[18] * B[5] + movq 40(%rcx), %rax + mulq 144(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[19] * B[4] + movq 32(%rcx), %rax + mulq 152(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[20] * B[3] + movq 24(%rcx), %rax + mulq 160(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[21] * B[2] + movq 16(%rcx), %rax + mulq 168(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[22] * B[1] + movq 8(%rcx), %rax + mulq 176(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[23] * B[0] + movq (%rcx), %rax + mulq 184(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + movq %r10, 184(%rsp) + # A[1] * B[23] + movq 184(%rcx), %rax + mulq 8(%rsi) + xorq %r10, %r10 + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[2] * B[22] + movq 176(%rcx), %rax + mulq 16(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[3] * B[21] + movq 168(%rcx), %rax + mulq 24(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[4] * B[20] + movq 160(%rcx), %rax + mulq 32(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[5] * B[19] + movq 152(%rcx), %rax + mulq 40(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[6] * B[18] + movq 144(%rcx), %rax + mulq 48(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[7] * B[17] + movq 136(%rcx), %rax + mulq 56(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[8] * B[16] + movq 128(%rcx), %rax + mulq 64(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[9] * B[15] + movq 120(%rcx), %rax + mulq 72(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[10] * B[14] + movq 112(%rcx), %rax + mulq 80(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[11] * B[13] + movq 104(%rcx), %rax + mulq 88(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[12] * B[12] + movq 96(%rcx), %rax + mulq 96(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[13] * B[11] + movq 88(%rcx), %rax + mulq 104(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[14] * B[10] + movq 80(%rcx), %rax + mulq 112(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[15] * B[9] + movq 72(%rcx), %rax + mulq 120(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[16] * B[8] + movq 64(%rcx), %rax + mulq 128(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[17] * B[7] + movq 56(%rcx), %rax + mulq 136(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[18] * B[6] + movq 48(%rcx), %rax + mulq 144(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[19] * B[5] + movq 40(%rcx), %rax + mulq 152(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[20] * B[4] + movq 32(%rcx), %rax + mulq 160(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[21] * B[3] + movq 24(%rcx), %rax + mulq 168(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[22] * B[2] + movq 16(%rcx), %rax + mulq 176(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[23] * B[1] + movq 8(%rcx), %rax + mulq 184(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + movq %r8, 192(%rdi) + # A[2] * B[23] + movq 184(%rcx), %rax + mulq 16(%rsi) + xorq %r8, %r8 + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[3] * B[22] + movq 176(%rcx), %rax + mulq 24(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[4] * B[21] + movq 168(%rcx), %rax + mulq 32(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[5] * B[20] + movq 160(%rcx), %rax + mulq 40(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[6] * B[19] + movq 152(%rcx), %rax + mulq 48(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[7] * B[18] + movq 144(%rcx), %rax + mulq 56(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[8] * B[17] + movq 136(%rcx), %rax + mulq 64(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[9] * B[16] + movq 128(%rcx), %rax + mulq 72(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[10] * B[15] + movq 120(%rcx), %rax + mulq 80(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[11] * B[14] + movq 112(%rcx), %rax + mulq 88(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[12] * B[13] + movq 104(%rcx), %rax + mulq 96(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[13] * B[12] + movq 96(%rcx), %rax + mulq 104(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[14] * B[11] + movq 88(%rcx), %rax + mulq 112(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[15] * B[10] + movq 80(%rcx), %rax + mulq 120(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[16] * B[9] + movq 72(%rcx), %rax + mulq 128(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[17] * B[8] + movq 64(%rcx), %rax + mulq 136(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[18] * B[7] + movq 56(%rcx), %rax + mulq 144(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[19] * B[6] + movq 48(%rcx), %rax + mulq 152(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[20] * B[5] + movq 40(%rcx), %rax + mulq 160(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[21] * B[4] + movq 32(%rcx), %rax + mulq 168(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[22] * B[3] + movq 24(%rcx), %rax + mulq 176(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[23] * B[2] + movq 16(%rcx), %rax + mulq 184(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + movq %r9, 200(%rdi) + # A[3] * B[23] + movq 184(%rcx), %rax + mulq 24(%rsi) + xorq %r9, %r9 + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[4] * B[22] + movq 176(%rcx), %rax + mulq 32(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[5] * B[21] + movq 168(%rcx), %rax + mulq 40(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[6] * B[20] + movq 160(%rcx), %rax + mulq 48(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[7] * B[19] + movq 152(%rcx), %rax + mulq 56(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[8] * B[18] + movq 144(%rcx), %rax + mulq 64(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[9] * B[17] + movq 136(%rcx), %rax + mulq 72(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[10] * B[16] + movq 128(%rcx), %rax + mulq 80(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[11] * B[15] + movq 120(%rcx), %rax + mulq 88(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[12] * B[14] + movq 112(%rcx), %rax + mulq 96(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[13] * B[13] + movq 104(%rcx), %rax + mulq 104(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[14] * B[12] + movq 96(%rcx), %rax + mulq 112(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[15] * B[11] + movq 88(%rcx), %rax + mulq 120(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[16] * B[10] + movq 80(%rcx), %rax + mulq 128(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[17] * B[9] + movq 72(%rcx), %rax + mulq 136(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[18] * B[8] + movq 64(%rcx), %rax + mulq 144(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[19] * B[7] + movq 56(%rcx), %rax + mulq 152(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[20] * B[6] + movq 48(%rcx), %rax + mulq 160(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[21] * B[5] + movq 40(%rcx), %rax + mulq 168(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[22] * B[4] + movq 32(%rcx), %rax + mulq 176(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[23] * B[3] + movq 24(%rcx), %rax + mulq 184(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + movq %r10, 208(%rdi) + # A[4] * B[23] + movq 184(%rcx), %rax + mulq 32(%rsi) + xorq %r10, %r10 + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[5] * B[22] + movq 176(%rcx), %rax + mulq 40(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[6] * B[21] + movq 168(%rcx), %rax + mulq 48(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[7] * B[20] + movq 160(%rcx), %rax + mulq 56(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[8] * B[19] + movq 152(%rcx), %rax + mulq 64(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[9] * B[18] + movq 144(%rcx), %rax + mulq 72(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[10] * B[17] + movq 136(%rcx), %rax + mulq 80(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[11] * B[16] + movq 128(%rcx), %rax + mulq 88(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[12] * B[15] + movq 120(%rcx), %rax + mulq 96(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[13] * B[14] + movq 112(%rcx), %rax + mulq 104(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[14] * B[13] + movq 104(%rcx), %rax + mulq 112(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[15] * B[12] + movq 96(%rcx), %rax + mulq 120(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[16] * B[11] + movq 88(%rcx), %rax + mulq 128(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[17] * B[10] + movq 80(%rcx), %rax + mulq 136(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[18] * B[9] + movq 72(%rcx), %rax + mulq 144(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[19] * B[8] + movq 64(%rcx), %rax + mulq 152(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[20] * B[7] + movq 56(%rcx), %rax + mulq 160(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[21] * B[6] + movq 48(%rcx), %rax + mulq 168(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[22] * B[5] + movq 40(%rcx), %rax + mulq 176(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[23] * B[4] + movq 32(%rcx), %rax + mulq 184(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + movq %r8, 216(%rdi) + # A[5] * B[23] + movq 184(%rcx), %rax + mulq 40(%rsi) + xorq %r8, %r8 + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[6] * B[22] + movq 176(%rcx), %rax + mulq 48(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[7] * B[21] + movq 168(%rcx), %rax + mulq 56(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[8] * B[20] + movq 160(%rcx), %rax + mulq 64(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[9] * B[19] + movq 152(%rcx), %rax + mulq 72(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[10] * B[18] + movq 144(%rcx), %rax + mulq 80(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[11] * B[17] + movq 136(%rcx), %rax + mulq 88(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[12] * B[16] + movq 128(%rcx), %rax + mulq 96(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[13] * B[15] + movq 120(%rcx), %rax + mulq 104(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[14] * B[14] + movq 112(%rcx), %rax + mulq 112(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[15] * B[13] + movq 104(%rcx), %rax + mulq 120(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[16] * B[12] + movq 96(%rcx), %rax + mulq 128(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[17] * B[11] + movq 88(%rcx), %rax + mulq 136(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[18] * B[10] + movq 80(%rcx), %rax + mulq 144(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[19] * B[9] + movq 72(%rcx), %rax + mulq 152(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[20] * B[8] + movq 64(%rcx), %rax + mulq 160(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[21] * B[7] + movq 56(%rcx), %rax + mulq 168(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[22] * B[6] + movq 48(%rcx), %rax + mulq 176(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[23] * B[5] + movq 40(%rcx), %rax + mulq 184(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + movq %r9, 224(%rdi) + # A[6] * B[23] + movq 184(%rcx), %rax + mulq 48(%rsi) + xorq %r9, %r9 + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[7] * B[22] + movq 176(%rcx), %rax + mulq 56(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[8] * B[21] + movq 168(%rcx), %rax + mulq 64(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[9] * B[20] + movq 160(%rcx), %rax + mulq 72(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[10] * B[19] + movq 152(%rcx), %rax + mulq 80(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[11] * B[18] + movq 144(%rcx), %rax + mulq 88(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[12] * B[17] + movq 136(%rcx), %rax + mulq 96(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[13] * B[16] + movq 128(%rcx), %rax + mulq 104(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[14] * B[15] + movq 120(%rcx), %rax + mulq 112(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[15] * B[14] + movq 112(%rcx), %rax + mulq 120(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[16] * B[13] + movq 104(%rcx), %rax + mulq 128(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[17] * B[12] + movq 96(%rcx), %rax + mulq 136(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[18] * B[11] + movq 88(%rcx), %rax + mulq 144(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[19] * B[10] + movq 80(%rcx), %rax + mulq 152(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[20] * B[9] + movq 72(%rcx), %rax + mulq 160(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[21] * B[8] + movq 64(%rcx), %rax + mulq 168(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[22] * B[7] + movq 56(%rcx), %rax + mulq 176(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[23] * B[6] + movq 48(%rcx), %rax + mulq 184(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + movq %r10, 232(%rdi) + # A[7] * B[23] + movq 184(%rcx), %rax + mulq 56(%rsi) + xorq %r10, %r10 + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[8] * B[22] + movq 176(%rcx), %rax + mulq 64(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[9] * B[21] + movq 168(%rcx), %rax + mulq 72(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[10] * B[20] + movq 160(%rcx), %rax + mulq 80(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[11] * B[19] + movq 152(%rcx), %rax + mulq 88(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[12] * B[18] + movq 144(%rcx), %rax + mulq 96(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[13] * B[17] + movq 136(%rcx), %rax + mulq 104(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[14] * B[16] + movq 128(%rcx), %rax + mulq 112(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[15] * B[15] + movq 120(%rcx), %rax + mulq 120(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[16] * B[14] + movq 112(%rcx), %rax + mulq 128(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[17] * B[13] + movq 104(%rcx), %rax + mulq 136(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[18] * B[12] + movq 96(%rcx), %rax + mulq 144(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[19] * B[11] + movq 88(%rcx), %rax + mulq 152(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[20] * B[10] + movq 80(%rcx), %rax + mulq 160(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[21] * B[9] + movq 72(%rcx), %rax + mulq 168(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[22] * B[8] + movq 64(%rcx), %rax + mulq 176(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[23] * B[7] + movq 56(%rcx), %rax + mulq 184(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + movq %r8, 240(%rdi) + # A[8] * B[23] + movq 184(%rcx), %rax + mulq 64(%rsi) + xorq %r8, %r8 + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[9] * B[22] + movq 176(%rcx), %rax + mulq 72(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[10] * B[21] + movq 168(%rcx), %rax + mulq 80(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[11] * B[20] + movq 160(%rcx), %rax + mulq 88(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[12] * B[19] + movq 152(%rcx), %rax + mulq 96(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[13] * B[18] + movq 144(%rcx), %rax + mulq 104(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[14] * B[17] + movq 136(%rcx), %rax + mulq 112(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[15] * B[16] + movq 128(%rcx), %rax + mulq 120(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[16] * B[15] + movq 120(%rcx), %rax + mulq 128(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[17] * B[14] + movq 112(%rcx), %rax + mulq 136(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[18] * B[13] + movq 104(%rcx), %rax + mulq 144(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[19] * B[12] + movq 96(%rcx), %rax + mulq 152(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[20] * B[11] + movq 88(%rcx), %rax + mulq 160(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[21] * B[10] + movq 80(%rcx), %rax + mulq 168(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[22] * B[9] + movq 72(%rcx), %rax + mulq 176(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[23] * B[8] + movq 64(%rcx), %rax + mulq 184(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + movq %r9, 248(%rdi) + # A[9] * B[23] + movq 184(%rcx), %rax + mulq 72(%rsi) + xorq %r9, %r9 + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[10] * B[22] + movq 176(%rcx), %rax + mulq 80(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[11] * B[21] + movq 168(%rcx), %rax + mulq 88(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[12] * B[20] + movq 160(%rcx), %rax + mulq 96(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[13] * B[19] + movq 152(%rcx), %rax + mulq 104(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[14] * B[18] + movq 144(%rcx), %rax + mulq 112(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[15] * B[17] + movq 136(%rcx), %rax + mulq 120(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[16] * B[16] + movq 128(%rcx), %rax + mulq 128(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[17] * B[15] + movq 120(%rcx), %rax + mulq 136(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[18] * B[14] + movq 112(%rcx), %rax + mulq 144(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[19] * B[13] + movq 104(%rcx), %rax + mulq 152(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[20] * B[12] + movq 96(%rcx), %rax + mulq 160(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[21] * B[11] + movq 88(%rcx), %rax + mulq 168(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[22] * B[10] + movq 80(%rcx), %rax + mulq 176(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[23] * B[9] + movq 72(%rcx), %rax + mulq 184(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + movq %r10, 256(%rdi) + # A[10] * B[23] + movq 184(%rcx), %rax + mulq 80(%rsi) + xorq %r10, %r10 + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[11] * B[22] + movq 176(%rcx), %rax + mulq 88(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[12] * B[21] + movq 168(%rcx), %rax + mulq 96(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[13] * B[20] + movq 160(%rcx), %rax + mulq 104(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[14] * B[19] + movq 152(%rcx), %rax + mulq 112(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[15] * B[18] + movq 144(%rcx), %rax + mulq 120(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[16] * B[17] + movq 136(%rcx), %rax + mulq 128(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[17] * B[16] + movq 128(%rcx), %rax + mulq 136(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[18] * B[15] + movq 120(%rcx), %rax + mulq 144(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[19] * B[14] + movq 112(%rcx), %rax + mulq 152(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[20] * B[13] + movq 104(%rcx), %rax + mulq 160(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[21] * B[12] + movq 96(%rcx), %rax + mulq 168(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[22] * B[11] + movq 88(%rcx), %rax + mulq 176(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[23] * B[10] + movq 80(%rcx), %rax + mulq 184(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + movq %r8, 264(%rdi) + # A[11] * B[23] + movq 184(%rcx), %rax + mulq 88(%rsi) + xorq %r8, %r8 + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[12] * B[22] + movq 176(%rcx), %rax + mulq 96(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[13] * B[21] + movq 168(%rcx), %rax + mulq 104(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[14] * B[20] + movq 160(%rcx), %rax + mulq 112(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[15] * B[19] + movq 152(%rcx), %rax + mulq 120(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[16] * B[18] + movq 144(%rcx), %rax + mulq 128(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[17] * B[17] + movq 136(%rcx), %rax + mulq 136(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[18] * B[16] + movq 128(%rcx), %rax + mulq 144(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[19] * B[15] + movq 120(%rcx), %rax + mulq 152(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[20] * B[14] + movq 112(%rcx), %rax + mulq 160(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[21] * B[13] + movq 104(%rcx), %rax + mulq 168(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[22] * B[12] + movq 96(%rcx), %rax + mulq 176(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[23] * B[11] + movq 88(%rcx), %rax + mulq 184(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + movq %r9, 272(%rdi) + # A[12] * B[23] + movq 184(%rcx), %rax + mulq 96(%rsi) + xorq %r9, %r9 + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[13] * B[22] + movq 176(%rcx), %rax + mulq 104(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[14] * B[21] + movq 168(%rcx), %rax + mulq 112(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[15] * B[20] + movq 160(%rcx), %rax + mulq 120(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[16] * B[19] + movq 152(%rcx), %rax + mulq 128(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[17] * B[18] + movq 144(%rcx), %rax + mulq 136(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[18] * B[17] + movq 136(%rcx), %rax + mulq 144(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[19] * B[16] + movq 128(%rcx), %rax + mulq 152(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[20] * B[15] + movq 120(%rcx), %rax + mulq 160(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[21] * B[14] + movq 112(%rcx), %rax + mulq 168(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[22] * B[13] + movq 104(%rcx), %rax + mulq 176(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[23] * B[12] + movq 96(%rcx), %rax + mulq 184(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + movq %r10, 280(%rdi) + # A[13] * B[23] + movq 184(%rcx), %rax + mulq 104(%rsi) + xorq %r10, %r10 + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[14] * B[22] + movq 176(%rcx), %rax + mulq 112(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[15] * B[21] + movq 168(%rcx), %rax + mulq 120(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[16] * B[20] + movq 160(%rcx), %rax + mulq 128(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[17] * B[19] + movq 152(%rcx), %rax + mulq 136(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[18] * B[18] + movq 144(%rcx), %rax + mulq 144(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[19] * B[17] + movq 136(%rcx), %rax + mulq 152(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[20] * B[16] + movq 128(%rcx), %rax + mulq 160(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[21] * B[15] + movq 120(%rcx), %rax + mulq 168(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[22] * B[14] + movq 112(%rcx), %rax + mulq 176(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[23] * B[13] + movq 104(%rcx), %rax + mulq 184(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + movq %r8, 288(%rdi) + # A[14] * B[23] + movq 184(%rcx), %rax + mulq 112(%rsi) + xorq %r8, %r8 + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[15] * B[22] + movq 176(%rcx), %rax + mulq 120(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[16] * B[21] + movq 168(%rcx), %rax + mulq 128(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[17] * B[20] + movq 160(%rcx), %rax + mulq 136(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[18] * B[19] + movq 152(%rcx), %rax + mulq 144(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[19] * B[18] + movq 144(%rcx), %rax + mulq 152(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[20] * B[17] + movq 136(%rcx), %rax + mulq 160(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[21] * B[16] + movq 128(%rcx), %rax + mulq 168(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[22] * B[15] + movq 120(%rcx), %rax + mulq 176(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[23] * B[14] + movq 112(%rcx), %rax + mulq 184(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + movq %r9, 296(%rdi) + # A[15] * B[23] + movq 184(%rcx), %rax + mulq 120(%rsi) + xorq %r9, %r9 + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[16] * B[22] + movq 176(%rcx), %rax + mulq 128(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[17] * B[21] + movq 168(%rcx), %rax + mulq 136(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[18] * B[20] + movq 160(%rcx), %rax + mulq 144(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[19] * B[19] + movq 152(%rcx), %rax + mulq 152(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[20] * B[18] + movq 144(%rcx), %rax + mulq 160(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[21] * B[17] + movq 136(%rcx), %rax + mulq 168(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[22] * B[16] + movq 128(%rcx), %rax + mulq 176(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[23] * B[15] + movq 120(%rcx), %rax + mulq 184(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + movq %r10, 304(%rdi) + # A[16] * B[23] + movq 184(%rcx), %rax + mulq 128(%rsi) + xorq %r10, %r10 + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[17] * B[22] + movq 176(%rcx), %rax + mulq 136(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[18] * B[21] + movq 168(%rcx), %rax + mulq 144(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[19] * B[20] + movq 160(%rcx), %rax + mulq 152(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[20] * B[19] + movq 152(%rcx), %rax + mulq 160(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[21] * B[18] + movq 144(%rcx), %rax + mulq 168(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[22] * B[17] + movq 136(%rcx), %rax + mulq 176(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[23] * B[16] + movq 128(%rcx), %rax + mulq 184(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + movq %r8, 312(%rdi) + # A[17] * B[23] + movq 184(%rcx), %rax + mulq 136(%rsi) + xorq %r8, %r8 + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[18] * B[22] + movq 176(%rcx), %rax + mulq 144(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[19] * B[21] + movq 168(%rcx), %rax + mulq 152(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[20] * B[20] + movq 160(%rcx), %rax + mulq 160(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[21] * B[19] + movq 152(%rcx), %rax + mulq 168(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[22] * B[18] + movq 144(%rcx), %rax + mulq 176(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[23] * B[17] + movq 136(%rcx), %rax + mulq 184(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + movq %r9, 320(%rdi) + # A[18] * B[23] + movq 184(%rcx), %rax + mulq 144(%rsi) + xorq %r9, %r9 + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[19] * B[22] + movq 176(%rcx), %rax + mulq 152(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[20] * B[21] + movq 168(%rcx), %rax + mulq 160(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[21] * B[20] + movq 160(%rcx), %rax + mulq 168(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[22] * B[19] + movq 152(%rcx), %rax + mulq 176(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[23] * B[18] + movq 144(%rcx), %rax + mulq 184(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + movq %r10, 328(%rdi) + # A[19] * B[23] + movq 184(%rcx), %rax + mulq 152(%rsi) + xorq %r10, %r10 + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[20] * B[22] + movq 176(%rcx), %rax + mulq 160(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[21] * B[21] + movq 168(%rcx), %rax + mulq 168(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[22] * B[20] + movq 160(%rcx), %rax + mulq 176(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[23] * B[19] + movq 152(%rcx), %rax + mulq 184(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + movq %r8, 336(%rdi) + # A[20] * B[23] + movq 184(%rcx), %rax + mulq 160(%rsi) + xorq %r8, %r8 + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[21] * B[22] + movq 176(%rcx), %rax + mulq 168(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[22] * B[21] + movq 168(%rcx), %rax + mulq 176(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[23] * B[20] + movq 160(%rcx), %rax + mulq 184(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + movq %r9, 344(%rdi) + # A[21] * B[23] + movq 184(%rcx), %rax + mulq 168(%rsi) + xorq %r9, %r9 + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[22] * B[22] + movq 176(%rcx), %rax + mulq 176(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[23] * B[21] + movq 168(%rcx), %rax + mulq 184(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + movq %r10, 352(%rdi) + # A[22] * B[23] + movq 184(%rcx), %rax + mulq 176(%rsi) + xorq %r10, %r10 + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[23] * B[22] + movq 176(%rcx), %rax + mulq 184(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + movq %r8, 360(%rdi) + # A[23] * B[23] + movq 184(%rcx), %rax + mulq 184(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + movq %r9, 368(%rdi) + movq %r10, 376(%rdi) + movq (%rsp), %rax + movq 8(%rsp), %rdx + movq 16(%rsp), %r8 + movq 24(%rsp), %r9 + movq %rax, (%rdi) + movq %rdx, 8(%rdi) + movq %r8, 16(%rdi) + movq %r9, 24(%rdi) + movq 32(%rsp), %rax + movq 40(%rsp), %rdx + movq 48(%rsp), %r8 + movq 56(%rsp), %r9 + movq %rax, 32(%rdi) + movq %rdx, 40(%rdi) + movq %r8, 48(%rdi) + movq %r9, 56(%rdi) + movq 64(%rsp), %rax + movq 72(%rsp), %rdx + movq 80(%rsp), %r8 + movq 88(%rsp), %r9 + movq %rax, 64(%rdi) + movq %rdx, 72(%rdi) + movq %r8, 80(%rdi) + movq %r9, 88(%rdi) + movq 96(%rsp), %rax + movq 104(%rsp), %rdx + movq 112(%rsp), %r8 + movq 120(%rsp), %r9 + movq %rax, 96(%rdi) + movq %rdx, 104(%rdi) + movq %r8, 112(%rdi) + movq %r9, 120(%rdi) + movq 128(%rsp), %rax + movq 136(%rsp), %rdx + movq 144(%rsp), %r8 + movq 152(%rsp), %r9 + movq %rax, 128(%rdi) + movq %rdx, 136(%rdi) + movq %r8, 144(%rdi) + movq %r9, 152(%rdi) + movq 160(%rsp), %rax + movq 168(%rsp), %rdx + movq 176(%rsp), %r8 + movq 184(%rsp), %r9 + movq %rax, 160(%rdi) + movq %rdx, 168(%rdi) + movq %r8, 176(%rdi) + movq %r9, 184(%rdi) + addq $192, %rsp + repz retq +.size sp_3072_mul_24,.-sp_3072_mul_24 +/* Square a and put result in r. (r = a * a) + * + * r A single precision integer. + * a A single precision integer. + */ +.globl sp_3072_sqr_24 +.type sp_3072_sqr_24,@function +.align 16 +sp_3072_sqr_24: + push %r12 + subq $192, %rsp + # A[0] * A[0] + movq (%rsi), %rax + mulq %rax + xorq %r9, %r9 + movq %rax, (%rsp) + movq %rdx, %r8 + # A[0] * A[1] + movq 8(%rsi), %rax + mulq (%rsi) + xorq %rcx, %rcx + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %rcx + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %rcx + movq %r8, 8(%rsp) + # A[0] * A[2] + movq 16(%rsi), %rax + mulq (%rsi) + xorq %r8, %r8 + addq %rax, %r9 + adcq %rdx, %rcx + adcq $0, %r8 + addq %rax, %r9 + adcq %rdx, %rcx + adcq $0, %r8 + # A[1] * A[1] + movq 8(%rsi), %rax + mulq %rax + addq %rax, %r9 + adcq %rdx, %rcx + adcq $0, %r8 + movq %r9, 16(%rsp) + # A[0] * A[3] + movq 24(%rsi), %rax + mulq (%rsi) + xorq %r9, %r9 + addq %rax, %rcx + adcq %rdx, %r8 + adcq $0, %r9 + addq %rax, %rcx + adcq %rdx, %r8 + adcq $0, %r9 + # A[1] * A[2] + movq 16(%rsi), %rax + mulq 8(%rsi) + addq %rax, %rcx + adcq %rdx, %r8 + adcq $0, %r9 + addq %rax, %rcx + adcq %rdx, %r8 + adcq $0, %r9 + movq %rcx, 24(%rsp) + # A[0] * A[4] + movq 32(%rsi), %rax + mulq (%rsi) + xorq %rcx, %rcx + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %rcx + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %rcx + # A[1] * A[3] + movq 24(%rsi), %rax + mulq 8(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %rcx + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %rcx + # A[2] * A[2] + movq 16(%rsi), %rax + mulq %rax + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %rcx + movq %r8, 32(%rsp) + # A[0] * A[5] + movq 40(%rsi), %rax + mulq (%rsi) + xorq %r8, %r8 + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[1] * A[4] + movq 32(%rsi), %rax + mulq 8(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[2] * A[3] + movq 24(%rsi), %rax + mulq 16(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %r10, %r9 + adcq %r11, %rcx + adcq %r12, %r8 + movq %r9, 40(%rsp) + # A[0] * A[6] + movq 48(%rsi), %rax + mulq (%rsi) + xorq %r9, %r9 + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[1] * A[5] + movq 40(%rsi), %rax + mulq 8(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[2] * A[4] + movq 32(%rsi), %rax + mulq 16(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[3] * A[3] + movq 24(%rsi), %rax + mulq %rax + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %rcx + adcq %r11, %r8 + adcq %r12, %r9 + movq %rcx, 48(%rsp) + # A[0] * A[7] + movq 56(%rsi), %rax + mulq (%rsi) + xorq %rcx, %rcx + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[1] * A[6] + movq 48(%rsi), %rax + mulq 8(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[2] * A[5] + movq 40(%rsi), %rax + mulq 16(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[3] * A[4] + movq 32(%rsi), %rax + mulq 24(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %r10, %r8 + adcq %r11, %r9 + adcq %r12, %rcx + movq %r8, 56(%rsp) + # A[0] * A[8] + movq 64(%rsi), %rax + mulq (%rsi) + xorq %r8, %r8 + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[1] * A[7] + movq 56(%rsi), %rax + mulq 8(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[2] * A[6] + movq 48(%rsi), %rax + mulq 16(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[3] * A[5] + movq 40(%rsi), %rax + mulq 24(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[4] * A[4] + movq 32(%rsi), %rax + mulq %rax + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %r9 + adcq %r11, %rcx + adcq %r12, %r8 + movq %r9, 64(%rsp) + # A[0] * A[9] + movq 72(%rsi), %rax + mulq (%rsi) + xorq %r9, %r9 + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[1] * A[8] + movq 64(%rsi), %rax + mulq 8(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[2] * A[7] + movq 56(%rsi), %rax + mulq 16(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[3] * A[6] + movq 48(%rsi), %rax + mulq 24(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[4] * A[5] + movq 40(%rsi), %rax + mulq 32(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %r10, %rcx + adcq %r11, %r8 + adcq %r12, %r9 + movq %rcx, 72(%rsp) + # A[0] * A[10] + movq 80(%rsi), %rax + mulq (%rsi) + xorq %rcx, %rcx + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[1] * A[9] + movq 72(%rsi), %rax + mulq 8(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[2] * A[8] + movq 64(%rsi), %rax + mulq 16(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[3] * A[7] + movq 56(%rsi), %rax + mulq 24(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[4] * A[6] + movq 48(%rsi), %rax + mulq 32(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[5] * A[5] + movq 40(%rsi), %rax + mulq %rax + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %r8 + adcq %r11, %r9 + adcq %r12, %rcx + movq %r8, 80(%rsp) + # A[0] * A[11] + movq 88(%rsi), %rax + mulq (%rsi) + xorq %r8, %r8 + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[1] * A[10] + movq 80(%rsi), %rax + mulq 8(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[2] * A[9] + movq 72(%rsi), %rax + mulq 16(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[3] * A[8] + movq 64(%rsi), %rax + mulq 24(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[4] * A[7] + movq 56(%rsi), %rax + mulq 32(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[5] * A[6] + movq 48(%rsi), %rax + mulq 40(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %r10, %r9 + adcq %r11, %rcx + adcq %r12, %r8 + movq %r9, 88(%rsp) + # A[0] * A[12] + movq 96(%rsi), %rax + mulq (%rsi) + xorq %r9, %r9 + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[1] * A[11] + movq 88(%rsi), %rax + mulq 8(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[2] * A[10] + movq 80(%rsi), %rax + mulq 16(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[3] * A[9] + movq 72(%rsi), %rax + mulq 24(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[4] * A[8] + movq 64(%rsi), %rax + mulq 32(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[5] * A[7] + movq 56(%rsi), %rax + mulq 40(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[6] * A[6] + movq 48(%rsi), %rax + mulq %rax + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %rcx + adcq %r11, %r8 + adcq %r12, %r9 + movq %rcx, 96(%rsp) + # A[0] * A[13] + movq 104(%rsi), %rax + mulq (%rsi) + xorq %rcx, %rcx + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[1] * A[12] + movq 96(%rsi), %rax + mulq 8(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[2] * A[11] + movq 88(%rsi), %rax + mulq 16(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[3] * A[10] + movq 80(%rsi), %rax + mulq 24(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[4] * A[9] + movq 72(%rsi), %rax + mulq 32(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[5] * A[8] + movq 64(%rsi), %rax + mulq 40(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[6] * A[7] + movq 56(%rsi), %rax + mulq 48(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %r10, %r8 + adcq %r11, %r9 + adcq %r12, %rcx + movq %r8, 104(%rsp) + # A[0] * A[14] + movq 112(%rsi), %rax + mulq (%rsi) + xorq %r8, %r8 + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[1] * A[13] + movq 104(%rsi), %rax + mulq 8(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[2] * A[12] + movq 96(%rsi), %rax + mulq 16(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[3] * A[11] + movq 88(%rsi), %rax + mulq 24(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[4] * A[10] + movq 80(%rsi), %rax + mulq 32(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[5] * A[9] + movq 72(%rsi), %rax + mulq 40(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[6] * A[8] + movq 64(%rsi), %rax + mulq 48(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[7] * A[7] + movq 56(%rsi), %rax + mulq %rax + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %r9 + adcq %r11, %rcx + adcq %r12, %r8 + movq %r9, 112(%rsp) + # A[0] * A[15] + movq 120(%rsi), %rax + mulq (%rsi) + xorq %r9, %r9 + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[1] * A[14] + movq 112(%rsi), %rax + mulq 8(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[2] * A[13] + movq 104(%rsi), %rax + mulq 16(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[3] * A[12] + movq 96(%rsi), %rax + mulq 24(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[4] * A[11] + movq 88(%rsi), %rax + mulq 32(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[5] * A[10] + movq 80(%rsi), %rax + mulq 40(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[6] * A[9] + movq 72(%rsi), %rax + mulq 48(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[7] * A[8] + movq 64(%rsi), %rax + mulq 56(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %r10, %rcx + adcq %r11, %r8 + adcq %r12, %r9 + movq %rcx, 120(%rsp) + # A[0] * A[16] + movq 128(%rsi), %rax + mulq (%rsi) + xorq %rcx, %rcx + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[1] * A[15] + movq 120(%rsi), %rax + mulq 8(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[2] * A[14] + movq 112(%rsi), %rax + mulq 16(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[3] * A[13] + movq 104(%rsi), %rax + mulq 24(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[4] * A[12] + movq 96(%rsi), %rax + mulq 32(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[5] * A[11] + movq 88(%rsi), %rax + mulq 40(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[6] * A[10] + movq 80(%rsi), %rax + mulq 48(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[7] * A[9] + movq 72(%rsi), %rax + mulq 56(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[8] * A[8] + movq 64(%rsi), %rax + mulq %rax + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %r8 + adcq %r11, %r9 + adcq %r12, %rcx + movq %r8, 128(%rsp) + # A[0] * A[17] + movq 136(%rsi), %rax + mulq (%rsi) + xorq %r8, %r8 + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[1] * A[16] + movq 128(%rsi), %rax + mulq 8(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[2] * A[15] + movq 120(%rsi), %rax + mulq 16(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[3] * A[14] + movq 112(%rsi), %rax + mulq 24(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[4] * A[13] + movq 104(%rsi), %rax + mulq 32(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[5] * A[12] + movq 96(%rsi), %rax + mulq 40(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[6] * A[11] + movq 88(%rsi), %rax + mulq 48(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[7] * A[10] + movq 80(%rsi), %rax + mulq 56(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[8] * A[9] + movq 72(%rsi), %rax + mulq 64(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %r10, %r9 + adcq %r11, %rcx + adcq %r12, %r8 + movq %r9, 136(%rsp) + # A[0] * A[18] + movq 144(%rsi), %rax + mulq (%rsi) + xorq %r9, %r9 + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[1] * A[17] + movq 136(%rsi), %rax + mulq 8(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[2] * A[16] + movq 128(%rsi), %rax + mulq 16(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[3] * A[15] + movq 120(%rsi), %rax + mulq 24(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[4] * A[14] + movq 112(%rsi), %rax + mulq 32(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[5] * A[13] + movq 104(%rsi), %rax + mulq 40(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[6] * A[12] + movq 96(%rsi), %rax + mulq 48(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[7] * A[11] + movq 88(%rsi), %rax + mulq 56(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[8] * A[10] + movq 80(%rsi), %rax + mulq 64(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[9] * A[9] + movq 72(%rsi), %rax + mulq %rax + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %rcx + adcq %r11, %r8 + adcq %r12, %r9 + movq %rcx, 144(%rsp) + # A[0] * A[19] + movq 152(%rsi), %rax + mulq (%rsi) + xorq %rcx, %rcx + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[1] * A[18] + movq 144(%rsi), %rax + mulq 8(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[2] * A[17] + movq 136(%rsi), %rax + mulq 16(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[3] * A[16] + movq 128(%rsi), %rax + mulq 24(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[4] * A[15] + movq 120(%rsi), %rax + mulq 32(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[5] * A[14] + movq 112(%rsi), %rax + mulq 40(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[6] * A[13] + movq 104(%rsi), %rax + mulq 48(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[7] * A[12] + movq 96(%rsi), %rax + mulq 56(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[8] * A[11] + movq 88(%rsi), %rax + mulq 64(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[9] * A[10] + movq 80(%rsi), %rax + mulq 72(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %r10, %r8 + adcq %r11, %r9 + adcq %r12, %rcx + movq %r8, 152(%rsp) + # A[0] * A[20] + movq 160(%rsi), %rax + mulq (%rsi) + xorq %r8, %r8 + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[1] * A[19] + movq 152(%rsi), %rax + mulq 8(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[2] * A[18] + movq 144(%rsi), %rax + mulq 16(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[3] * A[17] + movq 136(%rsi), %rax + mulq 24(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[4] * A[16] + movq 128(%rsi), %rax + mulq 32(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[5] * A[15] + movq 120(%rsi), %rax + mulq 40(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[6] * A[14] + movq 112(%rsi), %rax + mulq 48(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[7] * A[13] + movq 104(%rsi), %rax + mulq 56(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[8] * A[12] + movq 96(%rsi), %rax + mulq 64(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[9] * A[11] + movq 88(%rsi), %rax + mulq 72(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[10] * A[10] + movq 80(%rsi), %rax + mulq %rax + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %r9 + adcq %r11, %rcx + adcq %r12, %r8 + movq %r9, 160(%rsp) + # A[0] * A[21] + movq 168(%rsi), %rax + mulq (%rsi) + xorq %r9, %r9 + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[1] * A[20] + movq 160(%rsi), %rax + mulq 8(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[2] * A[19] + movq 152(%rsi), %rax + mulq 16(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[3] * A[18] + movq 144(%rsi), %rax + mulq 24(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[4] * A[17] + movq 136(%rsi), %rax + mulq 32(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[5] * A[16] + movq 128(%rsi), %rax + mulq 40(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[6] * A[15] + movq 120(%rsi), %rax + mulq 48(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[7] * A[14] + movq 112(%rsi), %rax + mulq 56(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[8] * A[13] + movq 104(%rsi), %rax + mulq 64(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[9] * A[12] + movq 96(%rsi), %rax + mulq 72(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[10] * A[11] + movq 88(%rsi), %rax + mulq 80(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %r10, %rcx + adcq %r11, %r8 + adcq %r12, %r9 + movq %rcx, 168(%rsp) + # A[0] * A[22] + movq 176(%rsi), %rax + mulq (%rsi) + xorq %rcx, %rcx + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[1] * A[21] + movq 168(%rsi), %rax + mulq 8(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[2] * A[20] + movq 160(%rsi), %rax + mulq 16(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[3] * A[19] + movq 152(%rsi), %rax + mulq 24(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[4] * A[18] + movq 144(%rsi), %rax + mulq 32(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[5] * A[17] + movq 136(%rsi), %rax + mulq 40(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[6] * A[16] + movq 128(%rsi), %rax + mulq 48(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[7] * A[15] + movq 120(%rsi), %rax + mulq 56(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[8] * A[14] + movq 112(%rsi), %rax + mulq 64(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[9] * A[13] + movq 104(%rsi), %rax + mulq 72(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[10] * A[12] + movq 96(%rsi), %rax + mulq 80(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[11] * A[11] + movq 88(%rsi), %rax + mulq %rax + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %r8 + adcq %r11, %r9 + adcq %r12, %rcx + movq %r8, 176(%rsp) + # A[0] * A[23] + movq 184(%rsi), %rax + mulq (%rsi) + xorq %r8, %r8 + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[1] * A[22] + movq 176(%rsi), %rax + mulq 8(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[2] * A[21] + movq 168(%rsi), %rax + mulq 16(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[3] * A[20] + movq 160(%rsi), %rax + mulq 24(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[4] * A[19] + movq 152(%rsi), %rax + mulq 32(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[5] * A[18] + movq 144(%rsi), %rax + mulq 40(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[6] * A[17] + movq 136(%rsi), %rax + mulq 48(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[7] * A[16] + movq 128(%rsi), %rax + mulq 56(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[8] * A[15] + movq 120(%rsi), %rax + mulq 64(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[9] * A[14] + movq 112(%rsi), %rax + mulq 72(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[10] * A[13] + movq 104(%rsi), %rax + mulq 80(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[11] * A[12] + movq 96(%rsi), %rax + mulq 88(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %r10, %r9 + adcq %r11, %rcx + adcq %r12, %r8 + movq %r9, 184(%rsp) + # A[1] * A[23] + movq 184(%rsi), %rax + mulq 8(%rsi) + xorq %r9, %r9 + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[2] * A[22] + movq 176(%rsi), %rax + mulq 16(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[3] * A[21] + movq 168(%rsi), %rax + mulq 24(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[4] * A[20] + movq 160(%rsi), %rax + mulq 32(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[5] * A[19] + movq 152(%rsi), %rax + mulq 40(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[6] * A[18] + movq 144(%rsi), %rax + mulq 48(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[7] * A[17] + movq 136(%rsi), %rax + mulq 56(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[8] * A[16] + movq 128(%rsi), %rax + mulq 64(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[9] * A[15] + movq 120(%rsi), %rax + mulq 72(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[10] * A[14] + movq 112(%rsi), %rax + mulq 80(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[11] * A[13] + movq 104(%rsi), %rax + mulq 88(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[12] * A[12] + movq 96(%rsi), %rax + mulq %rax + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %rcx + adcq %r11, %r8 + adcq %r12, %r9 + movq %rcx, 192(%rdi) + # A[2] * A[23] + movq 184(%rsi), %rax + mulq 16(%rsi) + xorq %rcx, %rcx + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[3] * A[22] + movq 176(%rsi), %rax + mulq 24(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[4] * A[21] + movq 168(%rsi), %rax + mulq 32(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[5] * A[20] + movq 160(%rsi), %rax + mulq 40(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[6] * A[19] + movq 152(%rsi), %rax + mulq 48(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[7] * A[18] + movq 144(%rsi), %rax + mulq 56(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[8] * A[17] + movq 136(%rsi), %rax + mulq 64(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[9] * A[16] + movq 128(%rsi), %rax + mulq 72(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[10] * A[15] + movq 120(%rsi), %rax + mulq 80(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[11] * A[14] + movq 112(%rsi), %rax + mulq 88(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[12] * A[13] + movq 104(%rsi), %rax + mulq 96(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %r10, %r8 + adcq %r11, %r9 + adcq %r12, %rcx + movq %r8, 200(%rdi) + # A[3] * A[23] + movq 184(%rsi), %rax + mulq 24(%rsi) + xorq %r8, %r8 + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[4] * A[22] + movq 176(%rsi), %rax + mulq 32(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[5] * A[21] + movq 168(%rsi), %rax + mulq 40(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[6] * A[20] + movq 160(%rsi), %rax + mulq 48(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[7] * A[19] + movq 152(%rsi), %rax + mulq 56(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[8] * A[18] + movq 144(%rsi), %rax + mulq 64(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[9] * A[17] + movq 136(%rsi), %rax + mulq 72(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[10] * A[16] + movq 128(%rsi), %rax + mulq 80(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[11] * A[15] + movq 120(%rsi), %rax + mulq 88(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[12] * A[14] + movq 112(%rsi), %rax + mulq 96(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[13] * A[13] + movq 104(%rsi), %rax + mulq %rax + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %r9 + adcq %r11, %rcx + adcq %r12, %r8 + movq %r9, 208(%rdi) + # A[4] * A[23] + movq 184(%rsi), %rax + mulq 32(%rsi) + xorq %r9, %r9 + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[5] * A[22] + movq 176(%rsi), %rax + mulq 40(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[6] * A[21] + movq 168(%rsi), %rax + mulq 48(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[7] * A[20] + movq 160(%rsi), %rax + mulq 56(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[8] * A[19] + movq 152(%rsi), %rax + mulq 64(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[9] * A[18] + movq 144(%rsi), %rax + mulq 72(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[10] * A[17] + movq 136(%rsi), %rax + mulq 80(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[11] * A[16] + movq 128(%rsi), %rax + mulq 88(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[12] * A[15] + movq 120(%rsi), %rax + mulq 96(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[13] * A[14] + movq 112(%rsi), %rax + mulq 104(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %r10, %rcx + adcq %r11, %r8 + adcq %r12, %r9 + movq %rcx, 216(%rdi) + # A[5] * A[23] + movq 184(%rsi), %rax + mulq 40(%rsi) + xorq %rcx, %rcx + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[6] * A[22] + movq 176(%rsi), %rax + mulq 48(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[7] * A[21] + movq 168(%rsi), %rax + mulq 56(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[8] * A[20] + movq 160(%rsi), %rax + mulq 64(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[9] * A[19] + movq 152(%rsi), %rax + mulq 72(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[10] * A[18] + movq 144(%rsi), %rax + mulq 80(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[11] * A[17] + movq 136(%rsi), %rax + mulq 88(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[12] * A[16] + movq 128(%rsi), %rax + mulq 96(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[13] * A[15] + movq 120(%rsi), %rax + mulq 104(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[14] * A[14] + movq 112(%rsi), %rax + mulq %rax + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %r8 + adcq %r11, %r9 + adcq %r12, %rcx + movq %r8, 224(%rdi) + # A[6] * A[23] + movq 184(%rsi), %rax + mulq 48(%rsi) + xorq %r8, %r8 + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[7] * A[22] + movq 176(%rsi), %rax + mulq 56(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[8] * A[21] + movq 168(%rsi), %rax + mulq 64(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[9] * A[20] + movq 160(%rsi), %rax + mulq 72(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[10] * A[19] + movq 152(%rsi), %rax + mulq 80(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[11] * A[18] + movq 144(%rsi), %rax + mulq 88(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[12] * A[17] + movq 136(%rsi), %rax + mulq 96(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[13] * A[16] + movq 128(%rsi), %rax + mulq 104(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[14] * A[15] + movq 120(%rsi), %rax + mulq 112(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %r10, %r9 + adcq %r11, %rcx + adcq %r12, %r8 + movq %r9, 232(%rdi) + # A[7] * A[23] + movq 184(%rsi), %rax + mulq 56(%rsi) + xorq %r9, %r9 + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[8] * A[22] + movq 176(%rsi), %rax + mulq 64(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[9] * A[21] + movq 168(%rsi), %rax + mulq 72(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[10] * A[20] + movq 160(%rsi), %rax + mulq 80(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[11] * A[19] + movq 152(%rsi), %rax + mulq 88(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[12] * A[18] + movq 144(%rsi), %rax + mulq 96(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[13] * A[17] + movq 136(%rsi), %rax + mulq 104(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[14] * A[16] + movq 128(%rsi), %rax + mulq 112(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[15] * A[15] + movq 120(%rsi), %rax + mulq %rax + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %rcx + adcq %r11, %r8 + adcq %r12, %r9 + movq %rcx, 240(%rdi) + # A[8] * A[23] + movq 184(%rsi), %rax + mulq 64(%rsi) + xorq %rcx, %rcx + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[9] * A[22] + movq 176(%rsi), %rax + mulq 72(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[10] * A[21] + movq 168(%rsi), %rax + mulq 80(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[11] * A[20] + movq 160(%rsi), %rax + mulq 88(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[12] * A[19] + movq 152(%rsi), %rax + mulq 96(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[13] * A[18] + movq 144(%rsi), %rax + mulq 104(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[14] * A[17] + movq 136(%rsi), %rax + mulq 112(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[15] * A[16] + movq 128(%rsi), %rax + mulq 120(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %r10, %r8 + adcq %r11, %r9 + adcq %r12, %rcx + movq %r8, 248(%rdi) + # A[9] * A[23] + movq 184(%rsi), %rax + mulq 72(%rsi) + xorq %r8, %r8 + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[10] * A[22] + movq 176(%rsi), %rax + mulq 80(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[11] * A[21] + movq 168(%rsi), %rax + mulq 88(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[12] * A[20] + movq 160(%rsi), %rax + mulq 96(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[13] * A[19] + movq 152(%rsi), %rax + mulq 104(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[14] * A[18] + movq 144(%rsi), %rax + mulq 112(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[15] * A[17] + movq 136(%rsi), %rax + mulq 120(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[16] * A[16] + movq 128(%rsi), %rax + mulq %rax + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %r9 + adcq %r11, %rcx + adcq %r12, %r8 + movq %r9, 256(%rdi) + # A[10] * A[23] + movq 184(%rsi), %rax + mulq 80(%rsi) + xorq %r9, %r9 + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[11] * A[22] + movq 176(%rsi), %rax + mulq 88(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[12] * A[21] + movq 168(%rsi), %rax + mulq 96(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[13] * A[20] + movq 160(%rsi), %rax + mulq 104(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[14] * A[19] + movq 152(%rsi), %rax + mulq 112(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[15] * A[18] + movq 144(%rsi), %rax + mulq 120(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[16] * A[17] + movq 136(%rsi), %rax + mulq 128(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %r10, %rcx + adcq %r11, %r8 + adcq %r12, %r9 + movq %rcx, 264(%rdi) + # A[11] * A[23] + movq 184(%rsi), %rax + mulq 88(%rsi) + xorq %rcx, %rcx + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[12] * A[22] + movq 176(%rsi), %rax + mulq 96(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[13] * A[21] + movq 168(%rsi), %rax + mulq 104(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[14] * A[20] + movq 160(%rsi), %rax + mulq 112(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[15] * A[19] + movq 152(%rsi), %rax + mulq 120(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[16] * A[18] + movq 144(%rsi), %rax + mulq 128(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[17] * A[17] + movq 136(%rsi), %rax + mulq %rax + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %r8 + adcq %r11, %r9 + adcq %r12, %rcx + movq %r8, 272(%rdi) + # A[12] * A[23] + movq 184(%rsi), %rax + mulq 96(%rsi) + xorq %r8, %r8 + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[13] * A[22] + movq 176(%rsi), %rax + mulq 104(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[14] * A[21] + movq 168(%rsi), %rax + mulq 112(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[15] * A[20] + movq 160(%rsi), %rax + mulq 120(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[16] * A[19] + movq 152(%rsi), %rax + mulq 128(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[17] * A[18] + movq 144(%rsi), %rax + mulq 136(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %r10, %r9 + adcq %r11, %rcx + adcq %r12, %r8 + movq %r9, 280(%rdi) + # A[13] * A[23] + movq 184(%rsi), %rax + mulq 104(%rsi) + xorq %r9, %r9 + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[14] * A[22] + movq 176(%rsi), %rax + mulq 112(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[15] * A[21] + movq 168(%rsi), %rax + mulq 120(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[16] * A[20] + movq 160(%rsi), %rax + mulq 128(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[17] * A[19] + movq 152(%rsi), %rax + mulq 136(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[18] * A[18] + movq 144(%rsi), %rax + mulq %rax + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %rcx + adcq %r11, %r8 + adcq %r12, %r9 + movq %rcx, 288(%rdi) + # A[14] * A[23] + movq 184(%rsi), %rax + mulq 112(%rsi) + xorq %rcx, %rcx + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[15] * A[22] + movq 176(%rsi), %rax + mulq 120(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[16] * A[21] + movq 168(%rsi), %rax + mulq 128(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[17] * A[20] + movq 160(%rsi), %rax + mulq 136(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[18] * A[19] + movq 152(%rsi), %rax + mulq 144(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %r10, %r8 + adcq %r11, %r9 + adcq %r12, %rcx + movq %r8, 296(%rdi) + # A[15] * A[23] + movq 184(%rsi), %rax + mulq 120(%rsi) + xorq %r8, %r8 + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[16] * A[22] + movq 176(%rsi), %rax + mulq 128(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[17] * A[21] + movq 168(%rsi), %rax + mulq 136(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[18] * A[20] + movq 160(%rsi), %rax + mulq 144(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[19] * A[19] + movq 152(%rsi), %rax + mulq %rax + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %r9 + adcq %r11, %rcx + adcq %r12, %r8 + movq %r9, 304(%rdi) + # A[16] * A[23] + movq 184(%rsi), %rax + mulq 128(%rsi) + xorq %r9, %r9 + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[17] * A[22] + movq 176(%rsi), %rax + mulq 136(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[18] * A[21] + movq 168(%rsi), %rax + mulq 144(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[19] * A[20] + movq 160(%rsi), %rax + mulq 152(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %r10, %rcx + adcq %r11, %r8 + adcq %r12, %r9 + movq %rcx, 312(%rdi) + # A[17] * A[23] + movq 184(%rsi), %rax + mulq 136(%rsi) + xorq %rcx, %rcx + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[18] * A[22] + movq 176(%rsi), %rax + mulq 144(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[19] * A[21] + movq 168(%rsi), %rax + mulq 152(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[20] * A[20] + movq 160(%rsi), %rax + mulq %rax + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %r8 + adcq %r11, %r9 + adcq %r12, %rcx + movq %r8, 320(%rdi) + # A[18] * A[23] + movq 184(%rsi), %rax + mulq 144(%rsi) + xorq %r8, %r8 + xorq %r12, %r12 + movq %rax, %r10 + movq %rdx, %r11 + # A[19] * A[22] + movq 176(%rsi), %rax + mulq 152(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[20] * A[21] + movq 168(%rsi), %rax + mulq 160(%rsi) + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + addq %r10, %r9 + adcq %r11, %rcx + adcq %r12, %r8 + movq %r9, 328(%rdi) + # A[19] * A[23] + movq 184(%rsi), %rax + mulq 152(%rsi) + xorq %r9, %r9 + addq %rax, %rcx + adcq %rdx, %r8 + adcq $0, %r9 + addq %rax, %rcx + adcq %rdx, %r8 + adcq $0, %r9 + # A[20] * A[22] + movq 176(%rsi), %rax + mulq 160(%rsi) + addq %rax, %rcx + adcq %rdx, %r8 + adcq $0, %r9 + addq %rax, %rcx + adcq %rdx, %r8 + adcq $0, %r9 + # A[21] * A[21] + movq 168(%rsi), %rax + mulq %rax + addq %rax, %rcx + adcq %rdx, %r8 + adcq $0, %r9 + movq %rcx, 336(%rdi) + # A[20] * A[23] + movq 184(%rsi), %rax + mulq 160(%rsi) + xorq %rcx, %rcx + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %rcx + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %rcx + # A[21] * A[22] + movq 176(%rsi), %rax + mulq 168(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %rcx + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %rcx + movq %r8, 344(%rdi) + # A[21] * A[23] + movq 184(%rsi), %rax + mulq 168(%rsi) + xorq %r8, %r8 + addq %rax, %r9 + adcq %rdx, %rcx + adcq $0, %r8 + addq %rax, %r9 + adcq %rdx, %rcx + adcq $0, %r8 + # A[22] * A[22] + movq 176(%rsi), %rax + mulq %rax + addq %rax, %r9 + adcq %rdx, %rcx + adcq $0, %r8 + movq %r9, 352(%rdi) + # A[22] * A[23] + movq 184(%rsi), %rax + mulq 176(%rsi) + xorq %r9, %r9 + addq %rax, %rcx + adcq %rdx, %r8 + adcq $0, %r9 + addq %rax, %rcx + adcq %rdx, %r8 + adcq $0, %r9 + movq %rcx, 360(%rdi) + # A[23] * A[23] + movq 184(%rsi), %rax + mulq %rax + addq %rax, %r8 + adcq %rdx, %r9 + movq %r8, 368(%rdi) + movq %r9, 376(%rdi) + movq (%rsp), %rax + movq 8(%rsp), %rdx + movq 16(%rsp), %r10 + movq 24(%rsp), %r11 + movq %rax, (%rdi) + movq %rdx, 8(%rdi) + movq %r10, 16(%rdi) + movq %r11, 24(%rdi) + movq 32(%rsp), %rax + movq 40(%rsp), %rdx + movq 48(%rsp), %r10 + movq 56(%rsp), %r11 + movq %rax, 32(%rdi) + movq %rdx, 40(%rdi) + movq %r10, 48(%rdi) + movq %r11, 56(%rdi) + movq 64(%rsp), %rax + movq 72(%rsp), %rdx + movq 80(%rsp), %r10 + movq 88(%rsp), %r11 + movq %rax, 64(%rdi) + movq %rdx, 72(%rdi) + movq %r10, 80(%rdi) + movq %r11, 88(%rdi) + movq 96(%rsp), %rax + movq 104(%rsp), %rdx + movq 112(%rsp), %r10 + movq 120(%rsp), %r11 + movq %rax, 96(%rdi) + movq %rdx, 104(%rdi) + movq %r10, 112(%rdi) + movq %r11, 120(%rdi) + movq 128(%rsp), %rax + movq 136(%rsp), %rdx + movq 144(%rsp), %r10 + movq 152(%rsp), %r11 + movq %rax, 128(%rdi) + movq %rdx, 136(%rdi) + movq %r10, 144(%rdi) + movq %r11, 152(%rdi) + movq 160(%rsp), %rax + movq 168(%rsp), %rdx + movq 176(%rsp), %r10 + movq 184(%rsp), %r11 + movq %rax, 160(%rdi) + movq %rdx, 168(%rdi) + movq %r10, 176(%rdi) + movq %r11, 184(%rdi) + addq $192, %rsp + pop %r12 + repz retq +.size sp_3072_sqr_24,.-sp_3072_sqr_24 +#ifdef HAVE_INTEL_AVX2 +/* Multiply a and b into r. (r = a * b) + * + * r Result of multiplication. + * a First number to multiply. + * b Second number to multiply. + */ +.globl sp_3072_mul_avx2_24 +.type sp_3072_mul_avx2_24,@function +.align 16 +sp_3072_mul_avx2_24: + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 + push %rbx + movq %rdx, %rbp + subq $192, %rsp + movq $0, %r14 + movq (%rsi), %rdx + # A[0] * B[0] + mulx (%rbp), %r8, %r9 + # A[0] * B[1] + mulx 8(%rbp), %rax, %r10 + movq %r8, (%rsp) + adcxq %rax, %r9 + # A[0] * B[2] + mulx 16(%rbp), %rax, %r11 + movq %r9, 8(%rsp) + adcxq %rax, %r10 + # A[0] * B[3] + mulx 24(%rbp), %rax, %r12 + movq %r10, 16(%rsp) + adcxq %rax, %r11 + movq %r11, 24(%rsp) + # A[0] * B[4] + mulx 32(%rbp), %rax, %r8 + adcxq %rax, %r12 + # A[0] * B[5] + mulx 40(%rbp), %rax, %r9 + movq %r12, 32(%rsp) + adcxq %rax, %r8 + # A[0] * B[6] + mulx 48(%rbp), %rax, %r10 + movq %r8, 40(%rsp) + adcxq %rax, %r9 + # A[0] * B[7] + mulx 56(%rbp), %rax, %r11 + movq %r9, 48(%rsp) + adcxq %rax, %r10 + movq %r10, 56(%rsp) + # A[0] * B[8] + mulx 64(%rbp), %rax, %r12 + adcxq %rax, %r11 + # A[0] * B[9] + mulx 72(%rbp), %rax, %r8 + movq %r11, 64(%rsp) + adcxq %rax, %r12 + # A[0] * B[10] + mulx 80(%rbp), %rax, %r9 + movq %r12, 72(%rsp) + adcxq %rax, %r8 + # A[0] * B[11] + mulx 88(%rbp), %rax, %r10 + movq %r8, 80(%rsp) + adcxq %rax, %r9 + movq %r9, 88(%rsp) + # A[0] * B[12] + mulx 96(%rbp), %rax, %r11 + adcxq %rax, %r10 + # A[0] * B[13] + mulx 104(%rbp), %rax, %r12 + movq %r10, 96(%rsp) + adcxq %rax, %r11 + # A[0] * B[14] + mulx 112(%rbp), %rax, %r8 + movq %r11, 104(%rsp) + adcxq %rax, %r12 + # A[0] * B[15] + mulx 120(%rbp), %rax, %r9 + movq %r12, 112(%rsp) + adcxq %rax, %r8 + movq %r8, 120(%rsp) + # A[0] * B[16] + mulx 128(%rbp), %rax, %r10 + adcxq %rax, %r9 + # A[0] * B[17] + mulx 136(%rbp), %rax, %r11 + movq %r9, 128(%rsp) + adcxq %rax, %r10 + # A[0] * B[18] + mulx 144(%rbp), %rax, %r12 + movq %r10, 136(%rsp) + adcxq %rax, %r11 + # A[0] * B[19] + mulx 152(%rbp), %rax, %r8 + movq %r11, 144(%rsp) + adcxq %rax, %r12 + movq %r12, 152(%rsp) + # A[0] * B[20] + mulx 160(%rbp), %rax, %r9 + adcxq %rax, %r8 + # A[0] * B[21] + mulx 168(%rbp), %rax, %r10 + movq %r8, 160(%rsp) + adcxq %rax, %r9 + # A[0] * B[22] + mulx 176(%rbp), %rax, %r11 + movq %r9, 168(%rsp) + adcxq %rax, %r10 + # A[0] * B[23] + mulx 184(%rbp), %rax, %r12 + movq %r10, %r15 + adcxq %rax, %r11 + adcxq %r14, %r12 + movq %r14, %r13 + adcxq %r14, %r13 + movq %r11, %rbx + movq %r12, 192(%rdi) + movq 8(%rsi), %rdx + movq 8(%rsp), %r9 + movq 16(%rsp), %r10 + movq 24(%rsp), %r11 + movq 32(%rsp), %r12 + movq 40(%rsp), %r8 + # A[1] * B[0] + mulx (%rbp), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[1] * B[1] + mulx 8(%rbp), %rax, %rcx + movq %r9, 8(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[1] * B[2] + mulx 16(%rbp), %rax, %rcx + movq %r10, 16(%rsp) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[1] * B[3] + mulx 24(%rbp), %rax, %rcx + movq %r11, 24(%rsp) + adcxq %rax, %r12 + adoxq %rcx, %r8 + movq %r12, 32(%rsp) + movq 48(%rsp), %r9 + movq 56(%rsp), %r10 + movq 64(%rsp), %r11 + movq 72(%rsp), %r12 + # A[1] * B[4] + mulx 32(%rbp), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[1] * B[5] + mulx 40(%rbp), %rax, %rcx + movq %r8, 40(%rsp) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[1] * B[6] + mulx 48(%rbp), %rax, %rcx + movq %r9, 48(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[1] * B[7] + mulx 56(%rbp), %rax, %rcx + movq %r10, 56(%rsp) + adcxq %rax, %r11 + adoxq %rcx, %r12 + movq %r11, 64(%rsp) + movq 80(%rsp), %r8 + movq 88(%rsp), %r9 + movq 96(%rsp), %r10 + movq 104(%rsp), %r11 + # A[1] * B[8] + mulx 64(%rbp), %rax, %rcx + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[1] * B[9] + mulx 72(%rbp), %rax, %rcx + movq %r12, 72(%rsp) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[1] * B[10] + mulx 80(%rbp), %rax, %rcx + movq %r8, 80(%rsp) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[1] * B[11] + mulx 88(%rbp), %rax, %rcx + movq %r9, 88(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r10, 96(%rsp) + movq 112(%rsp), %r12 + movq 120(%rsp), %r8 + movq 128(%rsp), %r9 + movq 136(%rsp), %r10 + # A[1] * B[12] + mulx 96(%rbp), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[1] * B[13] + mulx 104(%rbp), %rax, %rcx + movq %r11, 104(%rsp) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[1] * B[14] + mulx 112(%rbp), %rax, %rcx + movq %r12, 112(%rsp) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[1] * B[15] + mulx 120(%rbp), %rax, %rcx + movq %r8, 120(%rsp) + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r9, 128(%rsp) + movq 144(%rsp), %r11 + movq 152(%rsp), %r12 + movq 160(%rsp), %r8 + movq 168(%rsp), %r9 + # A[1] * B[16] + mulx 128(%rbp), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[1] * B[17] + mulx 136(%rbp), %rax, %rcx + movq %r10, 136(%rsp) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[1] * B[18] + mulx 144(%rbp), %rax, %rcx + movq %r11, 144(%rsp) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[1] * B[19] + mulx 152(%rbp), %rax, %rcx + movq %r12, 152(%rsp) + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r8, 160(%rsp) + movq %r15, %r10 + movq %rbx, %r11 + movq 192(%rdi), %r12 + # A[1] * B[20] + mulx 160(%rbp), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[1] * B[21] + mulx 168(%rbp), %rax, %rcx + movq %r9, 168(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[1] * B[22] + mulx 176(%rbp), %rax, %rcx + movq %r10, %r15 + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[1] * B[23] + mulx 184(%rbp), %rax, %rcx + movq %r11, %rbx + movq %r14, %r8 + adcxq %rax, %r12 + adoxq %rcx, %r8 + adcxq %r13, %r8 + movq %r14, %r13 + adoxq %r14, %r13 + adcxq %r14, %r13 + movq %r12, 192(%rdi) + movq %r8, 200(%rdi) + movq 16(%rsi), %rdx + movq 16(%rsp), %r10 + movq 24(%rsp), %r11 + movq 32(%rsp), %r12 + movq 40(%rsp), %r8 + movq 48(%rsp), %r9 + # A[2] * B[0] + mulx (%rbp), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[2] * B[1] + mulx 8(%rbp), %rax, %rcx + movq %r10, 16(%rsp) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[2] * B[2] + mulx 16(%rbp), %rax, %rcx + movq %r11, 24(%rsp) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[2] * B[3] + mulx 24(%rbp), %rax, %rcx + movq %r12, 32(%rsp) + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r8, 40(%rsp) + movq 56(%rsp), %r10 + movq 64(%rsp), %r11 + movq 72(%rsp), %r12 + movq 80(%rsp), %r8 + # A[2] * B[4] + mulx 32(%rbp), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[2] * B[5] + mulx 40(%rbp), %rax, %rcx + movq %r9, 48(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[2] * B[6] + mulx 48(%rbp), %rax, %rcx + movq %r10, 56(%rsp) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[2] * B[7] + mulx 56(%rbp), %rax, %rcx + movq %r11, 64(%rsp) + adcxq %rax, %r12 + adoxq %rcx, %r8 + movq %r12, 72(%rsp) + movq 88(%rsp), %r9 + movq 96(%rsp), %r10 + movq 104(%rsp), %r11 + movq 112(%rsp), %r12 + # A[2] * B[8] + mulx 64(%rbp), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[2] * B[9] + mulx 72(%rbp), %rax, %rcx + movq %r8, 80(%rsp) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[2] * B[10] + mulx 80(%rbp), %rax, %rcx + movq %r9, 88(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[2] * B[11] + mulx 88(%rbp), %rax, %rcx + movq %r10, 96(%rsp) + adcxq %rax, %r11 + adoxq %rcx, %r12 + movq %r11, 104(%rsp) + movq 120(%rsp), %r8 + movq 128(%rsp), %r9 + movq 136(%rsp), %r10 + movq 144(%rsp), %r11 + # A[2] * B[12] + mulx 96(%rbp), %rax, %rcx + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[2] * B[13] + mulx 104(%rbp), %rax, %rcx + movq %r12, 112(%rsp) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[2] * B[14] + mulx 112(%rbp), %rax, %rcx + movq %r8, 120(%rsp) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[2] * B[15] + mulx 120(%rbp), %rax, %rcx + movq %r9, 128(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r10, 136(%rsp) + movq 152(%rsp), %r12 + movq 160(%rsp), %r8 + movq 168(%rsp), %r9 + movq %r15, %r10 + # A[2] * B[16] + mulx 128(%rbp), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[2] * B[17] + mulx 136(%rbp), %rax, %rcx + movq %r11, 144(%rsp) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[2] * B[18] + mulx 144(%rbp), %rax, %rcx + movq %r12, 152(%rsp) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[2] * B[19] + mulx 152(%rbp), %rax, %rcx + movq %r8, 160(%rsp) + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r9, 168(%rsp) + movq %rbx, %r11 + movq 192(%rdi), %r12 + movq 200(%rdi), %r8 + # A[2] * B[20] + mulx 160(%rbp), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[2] * B[21] + mulx 168(%rbp), %rax, %rcx + movq %r10, %r15 + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[2] * B[22] + mulx 176(%rbp), %rax, %rcx + movq %r11, %rbx + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[2] * B[23] + mulx 184(%rbp), %rax, %rcx + movq %r12, 192(%rdi) + movq %r14, %r9 + adcxq %rax, %r8 + adoxq %rcx, %r9 + adcxq %r13, %r9 + movq %r14, %r13 + adoxq %r14, %r13 + adcxq %r14, %r13 + movq %r8, 200(%rdi) + movq %r9, 208(%rdi) + movq 24(%rsi), %rdx + movq 24(%rsp), %r11 + movq 32(%rsp), %r12 + movq 40(%rsp), %r8 + movq 48(%rsp), %r9 + movq 56(%rsp), %r10 + # A[3] * B[0] + mulx (%rbp), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[3] * B[1] + mulx 8(%rbp), %rax, %rcx + movq %r11, 24(%rsp) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[3] * B[2] + mulx 16(%rbp), %rax, %rcx + movq %r12, 32(%rsp) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[3] * B[3] + mulx 24(%rbp), %rax, %rcx + movq %r8, 40(%rsp) + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r9, 48(%rsp) + movq 64(%rsp), %r11 + movq 72(%rsp), %r12 + movq 80(%rsp), %r8 + movq 88(%rsp), %r9 + # A[3] * B[4] + mulx 32(%rbp), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[3] * B[5] + mulx 40(%rbp), %rax, %rcx + movq %r10, 56(%rsp) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[3] * B[6] + mulx 48(%rbp), %rax, %rcx + movq %r11, 64(%rsp) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[3] * B[7] + mulx 56(%rbp), %rax, %rcx + movq %r12, 72(%rsp) + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r8, 80(%rsp) + movq 96(%rsp), %r10 + movq 104(%rsp), %r11 + movq 112(%rsp), %r12 + movq 120(%rsp), %r8 + # A[3] * B[8] + mulx 64(%rbp), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[3] * B[9] + mulx 72(%rbp), %rax, %rcx + movq %r9, 88(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[3] * B[10] + mulx 80(%rbp), %rax, %rcx + movq %r10, 96(%rsp) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[3] * B[11] + mulx 88(%rbp), %rax, %rcx + movq %r11, 104(%rsp) + adcxq %rax, %r12 + adoxq %rcx, %r8 + movq %r12, 112(%rsp) + movq 128(%rsp), %r9 + movq 136(%rsp), %r10 + movq 144(%rsp), %r11 + movq 152(%rsp), %r12 + # A[3] * B[12] + mulx 96(%rbp), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[3] * B[13] + mulx 104(%rbp), %rax, %rcx + movq %r8, 120(%rsp) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[3] * B[14] + mulx 112(%rbp), %rax, %rcx + movq %r9, 128(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[3] * B[15] + mulx 120(%rbp), %rax, %rcx + movq %r10, 136(%rsp) + adcxq %rax, %r11 + adoxq %rcx, %r12 + movq %r11, 144(%rsp) + movq 160(%rsp), %r8 + movq 168(%rsp), %r9 + movq %r15, %r10 + movq %rbx, %r11 + # A[3] * B[16] + mulx 128(%rbp), %rax, %rcx + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[3] * B[17] + mulx 136(%rbp), %rax, %rcx + movq %r12, 152(%rsp) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[3] * B[18] + mulx 144(%rbp), %rax, %rcx + movq %r8, 160(%rsp) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[3] * B[19] + mulx 152(%rbp), %rax, %rcx + movq %r9, 168(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r10, %r15 + movq 192(%rdi), %r12 + movq 200(%rdi), %r8 + movq 208(%rdi), %r9 + # A[3] * B[20] + mulx 160(%rbp), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[3] * B[21] + mulx 168(%rbp), %rax, %rcx + movq %r11, %rbx + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[3] * B[22] + mulx 176(%rbp), %rax, %rcx + movq %r12, 192(%rdi) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[3] * B[23] + mulx 184(%rbp), %rax, %rcx + movq %r8, 200(%rdi) + movq %r14, %r10 + adcxq %rax, %r9 + adoxq %rcx, %r10 + adcxq %r13, %r10 + movq %r14, %r13 + adoxq %r14, %r13 + adcxq %r14, %r13 + movq %r9, 208(%rdi) + movq %r10, 216(%rdi) + movq 32(%rsi), %rdx + movq 32(%rsp), %r12 + movq 40(%rsp), %r8 + movq 48(%rsp), %r9 + movq 56(%rsp), %r10 + movq 64(%rsp), %r11 + # A[4] * B[0] + mulx (%rbp), %rax, %rcx + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[4] * B[1] + mulx 8(%rbp), %rax, %rcx + movq %r12, 32(%rsp) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[4] * B[2] + mulx 16(%rbp), %rax, %rcx + movq %r8, 40(%rsp) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[4] * B[3] + mulx 24(%rbp), %rax, %rcx + movq %r9, 48(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r10, 56(%rsp) + movq 72(%rsp), %r12 + movq 80(%rsp), %r8 + movq 88(%rsp), %r9 + movq 96(%rsp), %r10 + # A[4] * B[4] + mulx 32(%rbp), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[4] * B[5] + mulx 40(%rbp), %rax, %rcx + movq %r11, 64(%rsp) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[4] * B[6] + mulx 48(%rbp), %rax, %rcx + movq %r12, 72(%rsp) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[4] * B[7] + mulx 56(%rbp), %rax, %rcx + movq %r8, 80(%rsp) + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r9, 88(%rsp) + movq 104(%rsp), %r11 + movq 112(%rsp), %r12 + movq 120(%rsp), %r8 + movq 128(%rsp), %r9 + # A[4] * B[8] + mulx 64(%rbp), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[4] * B[9] + mulx 72(%rbp), %rax, %rcx + movq %r10, 96(%rsp) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[4] * B[10] + mulx 80(%rbp), %rax, %rcx + movq %r11, 104(%rsp) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[4] * B[11] + mulx 88(%rbp), %rax, %rcx + movq %r12, 112(%rsp) + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r8, 120(%rsp) + movq 136(%rsp), %r10 + movq 144(%rsp), %r11 + movq 152(%rsp), %r12 + movq 160(%rsp), %r8 + # A[4] * B[12] + mulx 96(%rbp), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[4] * B[13] + mulx 104(%rbp), %rax, %rcx + movq %r9, 128(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[4] * B[14] + mulx 112(%rbp), %rax, %rcx + movq %r10, 136(%rsp) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[4] * B[15] + mulx 120(%rbp), %rax, %rcx + movq %r11, 144(%rsp) + adcxq %rax, %r12 + adoxq %rcx, %r8 + movq %r12, 152(%rsp) + movq 168(%rsp), %r9 + movq %r15, %r10 + movq %rbx, %r11 + movq 192(%rdi), %r12 + # A[4] * B[16] + mulx 128(%rbp), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[4] * B[17] + mulx 136(%rbp), %rax, %rcx + movq %r8, 160(%rsp) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[4] * B[18] + mulx 144(%rbp), %rax, %rcx + movq %r9, 168(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[4] * B[19] + mulx 152(%rbp), %rax, %rcx + movq %r10, %r15 + adcxq %rax, %r11 + adoxq %rcx, %r12 + movq %r11, %rbx + movq 200(%rdi), %r8 + movq 208(%rdi), %r9 + movq 216(%rdi), %r10 + # A[4] * B[20] + mulx 160(%rbp), %rax, %rcx + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[4] * B[21] + mulx 168(%rbp), %rax, %rcx + movq %r12, 192(%rdi) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[4] * B[22] + mulx 176(%rbp), %rax, %rcx + movq %r8, 200(%rdi) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[4] * B[23] + mulx 184(%rbp), %rax, %rcx + movq %r9, 208(%rdi) + movq %r14, %r11 + adcxq %rax, %r10 + adoxq %rcx, %r11 + adcxq %r13, %r11 + movq %r14, %r13 + adoxq %r14, %r13 + adcxq %r14, %r13 + movq %r10, 216(%rdi) + movq %r11, 224(%rdi) + movq 40(%rsi), %rdx + movq 40(%rsp), %r8 + movq 48(%rsp), %r9 + movq 56(%rsp), %r10 + movq 64(%rsp), %r11 + movq 72(%rsp), %r12 + # A[5] * B[0] + mulx (%rbp), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[5] * B[1] + mulx 8(%rbp), %rax, %rcx + movq %r8, 40(%rsp) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[5] * B[2] + mulx 16(%rbp), %rax, %rcx + movq %r9, 48(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[5] * B[3] + mulx 24(%rbp), %rax, %rcx + movq %r10, 56(%rsp) + adcxq %rax, %r11 + adoxq %rcx, %r12 + movq %r11, 64(%rsp) + movq 80(%rsp), %r8 + movq 88(%rsp), %r9 + movq 96(%rsp), %r10 + movq 104(%rsp), %r11 + # A[5] * B[4] + mulx 32(%rbp), %rax, %rcx + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[5] * B[5] + mulx 40(%rbp), %rax, %rcx + movq %r12, 72(%rsp) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[5] * B[6] + mulx 48(%rbp), %rax, %rcx + movq %r8, 80(%rsp) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[5] * B[7] + mulx 56(%rbp), %rax, %rcx + movq %r9, 88(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r10, 96(%rsp) + movq 112(%rsp), %r12 + movq 120(%rsp), %r8 + movq 128(%rsp), %r9 + movq 136(%rsp), %r10 + # A[5] * B[8] + mulx 64(%rbp), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[5] * B[9] + mulx 72(%rbp), %rax, %rcx + movq %r11, 104(%rsp) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[5] * B[10] + mulx 80(%rbp), %rax, %rcx + movq %r12, 112(%rsp) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[5] * B[11] + mulx 88(%rbp), %rax, %rcx + movq %r8, 120(%rsp) + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r9, 128(%rsp) + movq 144(%rsp), %r11 + movq 152(%rsp), %r12 + movq 160(%rsp), %r8 + movq 168(%rsp), %r9 + # A[5] * B[12] + mulx 96(%rbp), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[5] * B[13] + mulx 104(%rbp), %rax, %rcx + movq %r10, 136(%rsp) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[5] * B[14] + mulx 112(%rbp), %rax, %rcx + movq %r11, 144(%rsp) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[5] * B[15] + mulx 120(%rbp), %rax, %rcx + movq %r12, 152(%rsp) + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r8, 160(%rsp) + movq %r15, %r10 + movq %rbx, %r11 + movq 192(%rdi), %r12 + movq 200(%rdi), %r8 + # A[5] * B[16] + mulx 128(%rbp), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[5] * B[17] + mulx 136(%rbp), %rax, %rcx + movq %r9, 168(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[5] * B[18] + mulx 144(%rbp), %rax, %rcx + movq %r10, %r15 + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[5] * B[19] + mulx 152(%rbp), %rax, %rcx + movq %r11, %rbx + adcxq %rax, %r12 + adoxq %rcx, %r8 + movq %r12, 192(%rdi) + movq 208(%rdi), %r9 + movq 216(%rdi), %r10 + movq 224(%rdi), %r11 + # A[5] * B[20] + mulx 160(%rbp), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[5] * B[21] + mulx 168(%rbp), %rax, %rcx + movq %r8, 200(%rdi) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[5] * B[22] + mulx 176(%rbp), %rax, %rcx + movq %r9, 208(%rdi) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[5] * B[23] + mulx 184(%rbp), %rax, %rcx + movq %r10, 216(%rdi) + movq %r14, %r12 + adcxq %rax, %r11 + adoxq %rcx, %r12 + adcxq %r13, %r12 + movq %r14, %r13 + adoxq %r14, %r13 + adcxq %r14, %r13 + movq %r11, 224(%rdi) + movq %r12, 232(%rdi) + movq 48(%rsi), %rdx + movq 48(%rsp), %r9 + movq 56(%rsp), %r10 + movq 64(%rsp), %r11 + movq 72(%rsp), %r12 + movq 80(%rsp), %r8 + # A[6] * B[0] + mulx (%rbp), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[6] * B[1] + mulx 8(%rbp), %rax, %rcx + movq %r9, 48(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[6] * B[2] + mulx 16(%rbp), %rax, %rcx + movq %r10, 56(%rsp) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[6] * B[3] + mulx 24(%rbp), %rax, %rcx + movq %r11, 64(%rsp) + adcxq %rax, %r12 + adoxq %rcx, %r8 + movq %r12, 72(%rsp) + movq 88(%rsp), %r9 + movq 96(%rsp), %r10 + movq 104(%rsp), %r11 + movq 112(%rsp), %r12 + # A[6] * B[4] + mulx 32(%rbp), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[6] * B[5] + mulx 40(%rbp), %rax, %rcx + movq %r8, 80(%rsp) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[6] * B[6] + mulx 48(%rbp), %rax, %rcx + movq %r9, 88(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[6] * B[7] + mulx 56(%rbp), %rax, %rcx + movq %r10, 96(%rsp) + adcxq %rax, %r11 + adoxq %rcx, %r12 + movq %r11, 104(%rsp) + movq 120(%rsp), %r8 + movq 128(%rsp), %r9 + movq 136(%rsp), %r10 + movq 144(%rsp), %r11 + # A[6] * B[8] + mulx 64(%rbp), %rax, %rcx + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[6] * B[9] + mulx 72(%rbp), %rax, %rcx + movq %r12, 112(%rsp) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[6] * B[10] + mulx 80(%rbp), %rax, %rcx + movq %r8, 120(%rsp) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[6] * B[11] + mulx 88(%rbp), %rax, %rcx + movq %r9, 128(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r10, 136(%rsp) + movq 152(%rsp), %r12 + movq 160(%rsp), %r8 + movq 168(%rsp), %r9 + movq %r15, %r10 + # A[6] * B[12] + mulx 96(%rbp), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[6] * B[13] + mulx 104(%rbp), %rax, %rcx + movq %r11, 144(%rsp) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[6] * B[14] + mulx 112(%rbp), %rax, %rcx + movq %r12, 152(%rsp) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[6] * B[15] + mulx 120(%rbp), %rax, %rcx + movq %r8, 160(%rsp) + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r9, 168(%rsp) + movq %rbx, %r11 + movq 192(%rdi), %r12 + movq 200(%rdi), %r8 + movq 208(%rdi), %r9 + # A[6] * B[16] + mulx 128(%rbp), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[6] * B[17] + mulx 136(%rbp), %rax, %rcx + movq %r10, %r15 + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[6] * B[18] + mulx 144(%rbp), %rax, %rcx + movq %r11, %rbx + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[6] * B[19] + mulx 152(%rbp), %rax, %rcx + movq %r12, 192(%rdi) + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r8, 200(%rdi) + movq 216(%rdi), %r10 + movq 224(%rdi), %r11 + movq 232(%rdi), %r12 + # A[6] * B[20] + mulx 160(%rbp), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[6] * B[21] + mulx 168(%rbp), %rax, %rcx + movq %r9, 208(%rdi) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[6] * B[22] + mulx 176(%rbp), %rax, %rcx + movq %r10, 216(%rdi) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[6] * B[23] + mulx 184(%rbp), %rax, %rcx + movq %r11, 224(%rdi) + movq %r14, %r8 + adcxq %rax, %r12 + adoxq %rcx, %r8 + adcxq %r13, %r8 + movq %r14, %r13 + adoxq %r14, %r13 + adcxq %r14, %r13 + movq %r12, 232(%rdi) + movq %r8, 240(%rdi) + movq 56(%rsi), %rdx + movq 56(%rsp), %r10 + movq 64(%rsp), %r11 + movq 72(%rsp), %r12 + movq 80(%rsp), %r8 + movq 88(%rsp), %r9 + # A[7] * B[0] + mulx (%rbp), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[7] * B[1] + mulx 8(%rbp), %rax, %rcx + movq %r10, 56(%rsp) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[7] * B[2] + mulx 16(%rbp), %rax, %rcx + movq %r11, 64(%rsp) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[7] * B[3] + mulx 24(%rbp), %rax, %rcx + movq %r12, 72(%rsp) + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r8, 80(%rsp) + movq 96(%rsp), %r10 + movq 104(%rsp), %r11 + movq 112(%rsp), %r12 + movq 120(%rsp), %r8 + # A[7] * B[4] + mulx 32(%rbp), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[7] * B[5] + mulx 40(%rbp), %rax, %rcx + movq %r9, 88(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[7] * B[6] + mulx 48(%rbp), %rax, %rcx + movq %r10, 96(%rsp) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[7] * B[7] + mulx 56(%rbp), %rax, %rcx + movq %r11, 104(%rsp) + adcxq %rax, %r12 + adoxq %rcx, %r8 + movq %r12, 112(%rsp) + movq 128(%rsp), %r9 + movq 136(%rsp), %r10 + movq 144(%rsp), %r11 + movq 152(%rsp), %r12 + # A[7] * B[8] + mulx 64(%rbp), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[7] * B[9] + mulx 72(%rbp), %rax, %rcx + movq %r8, 120(%rsp) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[7] * B[10] + mulx 80(%rbp), %rax, %rcx + movq %r9, 128(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[7] * B[11] + mulx 88(%rbp), %rax, %rcx + movq %r10, 136(%rsp) + adcxq %rax, %r11 + adoxq %rcx, %r12 + movq %r11, 144(%rsp) + movq 160(%rsp), %r8 + movq 168(%rsp), %r9 + movq %r15, %r10 + movq %rbx, %r11 + # A[7] * B[12] + mulx 96(%rbp), %rax, %rcx + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[7] * B[13] + mulx 104(%rbp), %rax, %rcx + movq %r12, 152(%rsp) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[7] * B[14] + mulx 112(%rbp), %rax, %rcx + movq %r8, 160(%rsp) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[7] * B[15] + mulx 120(%rbp), %rax, %rcx + movq %r9, 168(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r10, %r15 + movq 192(%rdi), %r12 + movq 200(%rdi), %r8 + movq 208(%rdi), %r9 + movq 216(%rdi), %r10 + # A[7] * B[16] + mulx 128(%rbp), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[7] * B[17] + mulx 136(%rbp), %rax, %rcx + movq %r11, %rbx + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[7] * B[18] + mulx 144(%rbp), %rax, %rcx + movq %r12, 192(%rdi) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[7] * B[19] + mulx 152(%rbp), %rax, %rcx + movq %r8, 200(%rdi) + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r9, 208(%rdi) + movq 224(%rdi), %r11 + movq 232(%rdi), %r12 + movq 240(%rdi), %r8 + # A[7] * B[20] + mulx 160(%rbp), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[7] * B[21] + mulx 168(%rbp), %rax, %rcx + movq %r10, 216(%rdi) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[7] * B[22] + mulx 176(%rbp), %rax, %rcx + movq %r11, 224(%rdi) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[7] * B[23] + mulx 184(%rbp), %rax, %rcx + movq %r12, 232(%rdi) + movq %r14, %r9 + adcxq %rax, %r8 + adoxq %rcx, %r9 + adcxq %r13, %r9 + movq %r14, %r13 + adoxq %r14, %r13 + adcxq %r14, %r13 + movq %r8, 240(%rdi) + movq %r9, 248(%rdi) + movq 64(%rsi), %rdx + movq 64(%rsp), %r11 + movq 72(%rsp), %r12 + movq 80(%rsp), %r8 + movq 88(%rsp), %r9 + movq 96(%rsp), %r10 + # A[8] * B[0] + mulx (%rbp), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[8] * B[1] + mulx 8(%rbp), %rax, %rcx + movq %r11, 64(%rsp) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[8] * B[2] + mulx 16(%rbp), %rax, %rcx + movq %r12, 72(%rsp) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[8] * B[3] + mulx 24(%rbp), %rax, %rcx + movq %r8, 80(%rsp) + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r9, 88(%rsp) + movq 104(%rsp), %r11 + movq 112(%rsp), %r12 + movq 120(%rsp), %r8 + movq 128(%rsp), %r9 + # A[8] * B[4] + mulx 32(%rbp), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[8] * B[5] + mulx 40(%rbp), %rax, %rcx + movq %r10, 96(%rsp) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[8] * B[6] + mulx 48(%rbp), %rax, %rcx + movq %r11, 104(%rsp) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[8] * B[7] + mulx 56(%rbp), %rax, %rcx + movq %r12, 112(%rsp) + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r8, 120(%rsp) + movq 136(%rsp), %r10 + movq 144(%rsp), %r11 + movq 152(%rsp), %r12 + movq 160(%rsp), %r8 + # A[8] * B[8] + mulx 64(%rbp), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[8] * B[9] + mulx 72(%rbp), %rax, %rcx + movq %r9, 128(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[8] * B[10] + mulx 80(%rbp), %rax, %rcx + movq %r10, 136(%rsp) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[8] * B[11] + mulx 88(%rbp), %rax, %rcx + movq %r11, 144(%rsp) + adcxq %rax, %r12 + adoxq %rcx, %r8 + movq %r12, 152(%rsp) + movq 168(%rsp), %r9 + movq %r15, %r10 + movq %rbx, %r11 + movq 192(%rdi), %r12 + # A[8] * B[12] + mulx 96(%rbp), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[8] * B[13] + mulx 104(%rbp), %rax, %rcx + movq %r8, 160(%rsp) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[8] * B[14] + mulx 112(%rbp), %rax, %rcx + movq %r9, 168(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[8] * B[15] + mulx 120(%rbp), %rax, %rcx + movq %r10, %r15 + adcxq %rax, %r11 + adoxq %rcx, %r12 + movq %r11, %rbx + movq 200(%rdi), %r8 + movq 208(%rdi), %r9 + movq 216(%rdi), %r10 + movq 224(%rdi), %r11 + # A[8] * B[16] + mulx 128(%rbp), %rax, %rcx + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[8] * B[17] + mulx 136(%rbp), %rax, %rcx + movq %r12, 192(%rdi) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[8] * B[18] + mulx 144(%rbp), %rax, %rcx + movq %r8, 200(%rdi) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[8] * B[19] + mulx 152(%rbp), %rax, %rcx + movq %r9, 208(%rdi) + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r10, 216(%rdi) + movq 232(%rdi), %r12 + movq 240(%rdi), %r8 + movq 248(%rdi), %r9 + # A[8] * B[20] + mulx 160(%rbp), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[8] * B[21] + mulx 168(%rbp), %rax, %rcx + movq %r11, 224(%rdi) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[8] * B[22] + mulx 176(%rbp), %rax, %rcx + movq %r12, 232(%rdi) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[8] * B[23] + mulx 184(%rbp), %rax, %rcx + movq %r8, 240(%rdi) + movq %r14, %r10 + adcxq %rax, %r9 + adoxq %rcx, %r10 + adcxq %r13, %r10 + movq %r14, %r13 + adoxq %r14, %r13 + adcxq %r14, %r13 + movq %r9, 248(%rdi) + movq %r10, 256(%rdi) + movq 72(%rsi), %rdx + movq 72(%rsp), %r12 + movq 80(%rsp), %r8 + movq 88(%rsp), %r9 + movq 96(%rsp), %r10 + movq 104(%rsp), %r11 + # A[9] * B[0] + mulx (%rbp), %rax, %rcx + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[9] * B[1] + mulx 8(%rbp), %rax, %rcx + movq %r12, 72(%rsp) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[9] * B[2] + mulx 16(%rbp), %rax, %rcx + movq %r8, 80(%rsp) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[9] * B[3] + mulx 24(%rbp), %rax, %rcx + movq %r9, 88(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r10, 96(%rsp) + movq 112(%rsp), %r12 + movq 120(%rsp), %r8 + movq 128(%rsp), %r9 + movq 136(%rsp), %r10 + # A[9] * B[4] + mulx 32(%rbp), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[9] * B[5] + mulx 40(%rbp), %rax, %rcx + movq %r11, 104(%rsp) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[9] * B[6] + mulx 48(%rbp), %rax, %rcx + movq %r12, 112(%rsp) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[9] * B[7] + mulx 56(%rbp), %rax, %rcx + movq %r8, 120(%rsp) + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r9, 128(%rsp) + movq 144(%rsp), %r11 + movq 152(%rsp), %r12 + movq 160(%rsp), %r8 + movq 168(%rsp), %r9 + # A[9] * B[8] + mulx 64(%rbp), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[9] * B[9] + mulx 72(%rbp), %rax, %rcx + movq %r10, 136(%rsp) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[9] * B[10] + mulx 80(%rbp), %rax, %rcx + movq %r11, 144(%rsp) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[9] * B[11] + mulx 88(%rbp), %rax, %rcx + movq %r12, 152(%rsp) + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r8, 160(%rsp) + movq %r15, %r10 + movq %rbx, %r11 + movq 192(%rdi), %r12 + movq 200(%rdi), %r8 + # A[9] * B[12] + mulx 96(%rbp), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[9] * B[13] + mulx 104(%rbp), %rax, %rcx + movq %r9, 168(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[9] * B[14] + mulx 112(%rbp), %rax, %rcx + movq %r10, %r15 + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[9] * B[15] + mulx 120(%rbp), %rax, %rcx + movq %r11, %rbx + adcxq %rax, %r12 + adoxq %rcx, %r8 + movq %r12, 192(%rdi) + movq 208(%rdi), %r9 + movq 216(%rdi), %r10 + movq 224(%rdi), %r11 + movq 232(%rdi), %r12 + # A[9] * B[16] + mulx 128(%rbp), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[9] * B[17] + mulx 136(%rbp), %rax, %rcx + movq %r8, 200(%rdi) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[9] * B[18] + mulx 144(%rbp), %rax, %rcx + movq %r9, 208(%rdi) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[9] * B[19] + mulx 152(%rbp), %rax, %rcx + movq %r10, 216(%rdi) + adcxq %rax, %r11 + adoxq %rcx, %r12 + movq %r11, 224(%rdi) + movq 240(%rdi), %r8 + movq 248(%rdi), %r9 + movq 256(%rdi), %r10 + # A[9] * B[20] + mulx 160(%rbp), %rax, %rcx + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[9] * B[21] + mulx 168(%rbp), %rax, %rcx + movq %r12, 232(%rdi) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[9] * B[22] + mulx 176(%rbp), %rax, %rcx + movq %r8, 240(%rdi) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[9] * B[23] + mulx 184(%rbp), %rax, %rcx + movq %r9, 248(%rdi) + movq %r14, %r11 + adcxq %rax, %r10 + adoxq %rcx, %r11 + adcxq %r13, %r11 + movq %r14, %r13 + adoxq %r14, %r13 + adcxq %r14, %r13 + movq %r10, 256(%rdi) + movq %r11, 264(%rdi) + movq 80(%rsi), %rdx + movq 80(%rsp), %r8 + movq 88(%rsp), %r9 + movq 96(%rsp), %r10 + movq 104(%rsp), %r11 + movq 112(%rsp), %r12 + # A[10] * B[0] + mulx (%rbp), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[10] * B[1] + mulx 8(%rbp), %rax, %rcx + movq %r8, 80(%rsp) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[10] * B[2] + mulx 16(%rbp), %rax, %rcx + movq %r9, 88(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[10] * B[3] + mulx 24(%rbp), %rax, %rcx + movq %r10, 96(%rsp) + adcxq %rax, %r11 + adoxq %rcx, %r12 + movq %r11, 104(%rsp) + movq 120(%rsp), %r8 + movq 128(%rsp), %r9 + movq 136(%rsp), %r10 + movq 144(%rsp), %r11 + # A[10] * B[4] + mulx 32(%rbp), %rax, %rcx + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[10] * B[5] + mulx 40(%rbp), %rax, %rcx + movq %r12, 112(%rsp) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[10] * B[6] + mulx 48(%rbp), %rax, %rcx + movq %r8, 120(%rsp) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[10] * B[7] + mulx 56(%rbp), %rax, %rcx + movq %r9, 128(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r10, 136(%rsp) + movq 152(%rsp), %r12 + movq 160(%rsp), %r8 + movq 168(%rsp), %r9 + movq %r15, %r10 + # A[10] * B[8] + mulx 64(%rbp), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[10] * B[9] + mulx 72(%rbp), %rax, %rcx + movq %r11, 144(%rsp) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[10] * B[10] + mulx 80(%rbp), %rax, %rcx + movq %r12, 152(%rsp) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[10] * B[11] + mulx 88(%rbp), %rax, %rcx + movq %r8, 160(%rsp) + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r9, 168(%rsp) + movq %rbx, %r11 + movq 192(%rdi), %r12 + movq 200(%rdi), %r8 + movq 208(%rdi), %r9 + # A[10] * B[12] + mulx 96(%rbp), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[10] * B[13] + mulx 104(%rbp), %rax, %rcx + movq %r10, %r15 + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[10] * B[14] + mulx 112(%rbp), %rax, %rcx + movq %r11, %rbx + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[10] * B[15] + mulx 120(%rbp), %rax, %rcx + movq %r12, 192(%rdi) + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r8, 200(%rdi) + movq 216(%rdi), %r10 + movq 224(%rdi), %r11 + movq 232(%rdi), %r12 + movq 240(%rdi), %r8 + # A[10] * B[16] + mulx 128(%rbp), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[10] * B[17] + mulx 136(%rbp), %rax, %rcx + movq %r9, 208(%rdi) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[10] * B[18] + mulx 144(%rbp), %rax, %rcx + movq %r10, 216(%rdi) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[10] * B[19] + mulx 152(%rbp), %rax, %rcx + movq %r11, 224(%rdi) + adcxq %rax, %r12 + adoxq %rcx, %r8 + movq %r12, 232(%rdi) + movq 248(%rdi), %r9 + movq 256(%rdi), %r10 + movq 264(%rdi), %r11 + # A[10] * B[20] + mulx 160(%rbp), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[10] * B[21] + mulx 168(%rbp), %rax, %rcx + movq %r8, 240(%rdi) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[10] * B[22] + mulx 176(%rbp), %rax, %rcx + movq %r9, 248(%rdi) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[10] * B[23] + mulx 184(%rbp), %rax, %rcx + movq %r10, 256(%rdi) + movq %r14, %r12 + adcxq %rax, %r11 + adoxq %rcx, %r12 + adcxq %r13, %r12 + movq %r14, %r13 + adoxq %r14, %r13 + adcxq %r14, %r13 + movq %r11, 264(%rdi) + movq %r12, 272(%rdi) + movq 88(%rsi), %rdx + movq 88(%rsp), %r9 + movq 96(%rsp), %r10 + movq 104(%rsp), %r11 + movq 112(%rsp), %r12 + movq 120(%rsp), %r8 + # A[11] * B[0] + mulx (%rbp), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[11] * B[1] + mulx 8(%rbp), %rax, %rcx + movq %r9, 88(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[11] * B[2] + mulx 16(%rbp), %rax, %rcx + movq %r10, 96(%rsp) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[11] * B[3] + mulx 24(%rbp), %rax, %rcx + movq %r11, 104(%rsp) + adcxq %rax, %r12 + adoxq %rcx, %r8 + movq %r12, 112(%rsp) + movq 128(%rsp), %r9 + movq 136(%rsp), %r10 + movq 144(%rsp), %r11 + movq 152(%rsp), %r12 + # A[11] * B[4] + mulx 32(%rbp), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[11] * B[5] + mulx 40(%rbp), %rax, %rcx + movq %r8, 120(%rsp) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[11] * B[6] + mulx 48(%rbp), %rax, %rcx + movq %r9, 128(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[11] * B[7] + mulx 56(%rbp), %rax, %rcx + movq %r10, 136(%rsp) + adcxq %rax, %r11 + adoxq %rcx, %r12 + movq %r11, 144(%rsp) + movq 160(%rsp), %r8 + movq 168(%rsp), %r9 + movq %r15, %r10 + movq %rbx, %r11 + # A[11] * B[8] + mulx 64(%rbp), %rax, %rcx + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[11] * B[9] + mulx 72(%rbp), %rax, %rcx + movq %r12, 152(%rsp) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[11] * B[10] + mulx 80(%rbp), %rax, %rcx + movq %r8, 160(%rsp) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[11] * B[11] + mulx 88(%rbp), %rax, %rcx + movq %r9, 168(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r10, %r15 + movq 192(%rdi), %r12 + movq 200(%rdi), %r8 + movq 208(%rdi), %r9 + movq 216(%rdi), %r10 + # A[11] * B[12] + mulx 96(%rbp), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[11] * B[13] + mulx 104(%rbp), %rax, %rcx + movq %r11, %rbx + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[11] * B[14] + mulx 112(%rbp), %rax, %rcx + movq %r12, 192(%rdi) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[11] * B[15] + mulx 120(%rbp), %rax, %rcx + movq %r8, 200(%rdi) + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r9, 208(%rdi) + movq 224(%rdi), %r11 + movq 232(%rdi), %r12 + movq 240(%rdi), %r8 + movq 248(%rdi), %r9 + # A[11] * B[16] + mulx 128(%rbp), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[11] * B[17] + mulx 136(%rbp), %rax, %rcx + movq %r10, 216(%rdi) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[11] * B[18] + mulx 144(%rbp), %rax, %rcx + movq %r11, 224(%rdi) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[11] * B[19] + mulx 152(%rbp), %rax, %rcx + movq %r12, 232(%rdi) + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r8, 240(%rdi) + movq 256(%rdi), %r10 + movq 264(%rdi), %r11 + movq 272(%rdi), %r12 + # A[11] * B[20] + mulx 160(%rbp), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[11] * B[21] + mulx 168(%rbp), %rax, %rcx + movq %r9, 248(%rdi) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[11] * B[22] + mulx 176(%rbp), %rax, %rcx + movq %r10, 256(%rdi) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[11] * B[23] + mulx 184(%rbp), %rax, %rcx + movq %r11, 264(%rdi) + movq %r14, %r8 + adcxq %rax, %r12 + adoxq %rcx, %r8 + adcxq %r13, %r8 + movq %r14, %r13 + adoxq %r14, %r13 + adcxq %r14, %r13 + movq %r12, 272(%rdi) + movq %r8, 280(%rdi) + movq 96(%rsi), %rdx + movq 96(%rsp), %r10 + movq 104(%rsp), %r11 + movq 112(%rsp), %r12 + movq 120(%rsp), %r8 + movq 128(%rsp), %r9 + # A[12] * B[0] + mulx (%rbp), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[12] * B[1] + mulx 8(%rbp), %rax, %rcx + movq %r10, 96(%rsp) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[12] * B[2] + mulx 16(%rbp), %rax, %rcx + movq %r11, 104(%rsp) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[12] * B[3] + mulx 24(%rbp), %rax, %rcx + movq %r12, 112(%rsp) + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r8, 120(%rsp) + movq 136(%rsp), %r10 + movq 144(%rsp), %r11 + movq 152(%rsp), %r12 + movq 160(%rsp), %r8 + # A[12] * B[4] + mulx 32(%rbp), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[12] * B[5] + mulx 40(%rbp), %rax, %rcx + movq %r9, 128(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[12] * B[6] + mulx 48(%rbp), %rax, %rcx + movq %r10, 136(%rsp) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[12] * B[7] + mulx 56(%rbp), %rax, %rcx + movq %r11, 144(%rsp) + adcxq %rax, %r12 + adoxq %rcx, %r8 + movq %r12, 152(%rsp) + movq 168(%rsp), %r9 + movq %r15, %r10 + movq %rbx, %r11 + movq 192(%rdi), %r12 + # A[12] * B[8] + mulx 64(%rbp), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[12] * B[9] + mulx 72(%rbp), %rax, %rcx + movq %r8, 160(%rsp) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[12] * B[10] + mulx 80(%rbp), %rax, %rcx + movq %r9, 168(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[12] * B[11] + mulx 88(%rbp), %rax, %rcx + movq %r10, %r15 + adcxq %rax, %r11 + adoxq %rcx, %r12 + movq %r11, %rbx + movq 200(%rdi), %r8 + movq 208(%rdi), %r9 + movq 216(%rdi), %r10 + movq 224(%rdi), %r11 + # A[12] * B[12] + mulx 96(%rbp), %rax, %rcx + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[12] * B[13] + mulx 104(%rbp), %rax, %rcx + movq %r12, 192(%rdi) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[12] * B[14] + mulx 112(%rbp), %rax, %rcx + movq %r8, 200(%rdi) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[12] * B[15] + mulx 120(%rbp), %rax, %rcx + movq %r9, 208(%rdi) + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r10, 216(%rdi) + movq 232(%rdi), %r12 + movq 240(%rdi), %r8 + movq 248(%rdi), %r9 + movq 256(%rdi), %r10 + # A[12] * B[16] + mulx 128(%rbp), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[12] * B[17] + mulx 136(%rbp), %rax, %rcx + movq %r11, 224(%rdi) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[12] * B[18] + mulx 144(%rbp), %rax, %rcx + movq %r12, 232(%rdi) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[12] * B[19] + mulx 152(%rbp), %rax, %rcx + movq %r8, 240(%rdi) + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r9, 248(%rdi) + movq 264(%rdi), %r11 + movq 272(%rdi), %r12 + movq 280(%rdi), %r8 + # A[12] * B[20] + mulx 160(%rbp), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[12] * B[21] + mulx 168(%rbp), %rax, %rcx + movq %r10, 256(%rdi) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[12] * B[22] + mulx 176(%rbp), %rax, %rcx + movq %r11, 264(%rdi) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[12] * B[23] + mulx 184(%rbp), %rax, %rcx + movq %r12, 272(%rdi) + movq %r14, %r9 + adcxq %rax, %r8 + adoxq %rcx, %r9 + adcxq %r13, %r9 + movq %r14, %r13 + adoxq %r14, %r13 + adcxq %r14, %r13 + movq %r8, 280(%rdi) + movq %r9, 288(%rdi) + movq 104(%rsi), %rdx + movq 104(%rsp), %r11 + movq 112(%rsp), %r12 + movq 120(%rsp), %r8 + movq 128(%rsp), %r9 + movq 136(%rsp), %r10 + # A[13] * B[0] + mulx (%rbp), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[13] * B[1] + mulx 8(%rbp), %rax, %rcx + movq %r11, 104(%rsp) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[13] * B[2] + mulx 16(%rbp), %rax, %rcx + movq %r12, 112(%rsp) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[13] * B[3] + mulx 24(%rbp), %rax, %rcx + movq %r8, 120(%rsp) + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r9, 128(%rsp) + movq 144(%rsp), %r11 + movq 152(%rsp), %r12 + movq 160(%rsp), %r8 + movq 168(%rsp), %r9 + # A[13] * B[4] + mulx 32(%rbp), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[13] * B[5] + mulx 40(%rbp), %rax, %rcx + movq %r10, 136(%rsp) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[13] * B[6] + mulx 48(%rbp), %rax, %rcx + movq %r11, 144(%rsp) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[13] * B[7] + mulx 56(%rbp), %rax, %rcx + movq %r12, 152(%rsp) + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r8, 160(%rsp) + movq %r15, %r10 + movq %rbx, %r11 + movq 192(%rdi), %r12 + movq 200(%rdi), %r8 + # A[13] * B[8] + mulx 64(%rbp), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[13] * B[9] + mulx 72(%rbp), %rax, %rcx + movq %r9, 168(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[13] * B[10] + mulx 80(%rbp), %rax, %rcx + movq %r10, %r15 + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[13] * B[11] + mulx 88(%rbp), %rax, %rcx + movq %r11, %rbx + adcxq %rax, %r12 + adoxq %rcx, %r8 + movq %r12, 192(%rdi) + movq 208(%rdi), %r9 + movq 216(%rdi), %r10 + movq 224(%rdi), %r11 + movq 232(%rdi), %r12 + # A[13] * B[12] + mulx 96(%rbp), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[13] * B[13] + mulx 104(%rbp), %rax, %rcx + movq %r8, 200(%rdi) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[13] * B[14] + mulx 112(%rbp), %rax, %rcx + movq %r9, 208(%rdi) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[13] * B[15] + mulx 120(%rbp), %rax, %rcx + movq %r10, 216(%rdi) + adcxq %rax, %r11 + adoxq %rcx, %r12 + movq %r11, 224(%rdi) + movq 240(%rdi), %r8 + movq 248(%rdi), %r9 + movq 256(%rdi), %r10 + movq 264(%rdi), %r11 + # A[13] * B[16] + mulx 128(%rbp), %rax, %rcx + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[13] * B[17] + mulx 136(%rbp), %rax, %rcx + movq %r12, 232(%rdi) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[13] * B[18] + mulx 144(%rbp), %rax, %rcx + movq %r8, 240(%rdi) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[13] * B[19] + mulx 152(%rbp), %rax, %rcx + movq %r9, 248(%rdi) + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r10, 256(%rdi) + movq 272(%rdi), %r12 + movq 280(%rdi), %r8 + movq 288(%rdi), %r9 + # A[13] * B[20] + mulx 160(%rbp), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[13] * B[21] + mulx 168(%rbp), %rax, %rcx + movq %r11, 264(%rdi) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[13] * B[22] + mulx 176(%rbp), %rax, %rcx + movq %r12, 272(%rdi) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[13] * B[23] + mulx 184(%rbp), %rax, %rcx + movq %r8, 280(%rdi) + movq %r14, %r10 + adcxq %rax, %r9 + adoxq %rcx, %r10 + adcxq %r13, %r10 + movq %r14, %r13 + adoxq %r14, %r13 + adcxq %r14, %r13 + movq %r9, 288(%rdi) + movq %r10, 296(%rdi) + movq 112(%rsi), %rdx + movq 112(%rsp), %r12 + movq 120(%rsp), %r8 + movq 128(%rsp), %r9 + movq 136(%rsp), %r10 + movq 144(%rsp), %r11 + # A[14] * B[0] + mulx (%rbp), %rax, %rcx + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[14] * B[1] + mulx 8(%rbp), %rax, %rcx + movq %r12, 112(%rsp) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[14] * B[2] + mulx 16(%rbp), %rax, %rcx + movq %r8, 120(%rsp) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[14] * B[3] + mulx 24(%rbp), %rax, %rcx + movq %r9, 128(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r10, 136(%rsp) + movq 152(%rsp), %r12 + movq 160(%rsp), %r8 + movq 168(%rsp), %r9 + movq %r15, %r10 + # A[14] * B[4] + mulx 32(%rbp), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[14] * B[5] + mulx 40(%rbp), %rax, %rcx + movq %r11, 144(%rsp) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[14] * B[6] + mulx 48(%rbp), %rax, %rcx + movq %r12, 152(%rsp) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[14] * B[7] + mulx 56(%rbp), %rax, %rcx + movq %r8, 160(%rsp) + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r9, 168(%rsp) + movq %rbx, %r11 + movq 192(%rdi), %r12 + movq 200(%rdi), %r8 + movq 208(%rdi), %r9 + # A[14] * B[8] + mulx 64(%rbp), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[14] * B[9] + mulx 72(%rbp), %rax, %rcx + movq %r10, %r15 + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[14] * B[10] + mulx 80(%rbp), %rax, %rcx + movq %r11, %rbx + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[14] * B[11] + mulx 88(%rbp), %rax, %rcx + movq %r12, 192(%rdi) + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r8, 200(%rdi) + movq 216(%rdi), %r10 + movq 224(%rdi), %r11 + movq 232(%rdi), %r12 + movq 240(%rdi), %r8 + # A[14] * B[12] + mulx 96(%rbp), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[14] * B[13] + mulx 104(%rbp), %rax, %rcx + movq %r9, 208(%rdi) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[14] * B[14] + mulx 112(%rbp), %rax, %rcx + movq %r10, 216(%rdi) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[14] * B[15] + mulx 120(%rbp), %rax, %rcx + movq %r11, 224(%rdi) + adcxq %rax, %r12 + adoxq %rcx, %r8 + movq %r12, 232(%rdi) + movq 248(%rdi), %r9 + movq 256(%rdi), %r10 + movq 264(%rdi), %r11 + movq 272(%rdi), %r12 + # A[14] * B[16] + mulx 128(%rbp), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[14] * B[17] + mulx 136(%rbp), %rax, %rcx + movq %r8, 240(%rdi) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[14] * B[18] + mulx 144(%rbp), %rax, %rcx + movq %r9, 248(%rdi) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[14] * B[19] + mulx 152(%rbp), %rax, %rcx + movq %r10, 256(%rdi) + adcxq %rax, %r11 + adoxq %rcx, %r12 + movq %r11, 264(%rdi) + movq 280(%rdi), %r8 + movq 288(%rdi), %r9 + movq 296(%rdi), %r10 + # A[14] * B[20] + mulx 160(%rbp), %rax, %rcx + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[14] * B[21] + mulx 168(%rbp), %rax, %rcx + movq %r12, 272(%rdi) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[14] * B[22] + mulx 176(%rbp), %rax, %rcx + movq %r8, 280(%rdi) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[14] * B[23] + mulx 184(%rbp), %rax, %rcx + movq %r9, 288(%rdi) + movq %r14, %r11 + adcxq %rax, %r10 + adoxq %rcx, %r11 + adcxq %r13, %r11 + movq %r14, %r13 + adoxq %r14, %r13 + adcxq %r14, %r13 + movq %r10, 296(%rdi) + movq %r11, 304(%rdi) + movq 120(%rsi), %rdx + movq 120(%rsp), %r8 + movq 128(%rsp), %r9 + movq 136(%rsp), %r10 + movq 144(%rsp), %r11 + movq 152(%rsp), %r12 + # A[15] * B[0] + mulx (%rbp), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[15] * B[1] + mulx 8(%rbp), %rax, %rcx + movq %r8, 120(%rsp) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[15] * B[2] + mulx 16(%rbp), %rax, %rcx + movq %r9, 128(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[15] * B[3] + mulx 24(%rbp), %rax, %rcx + movq %r10, 136(%rsp) + adcxq %rax, %r11 + adoxq %rcx, %r12 + movq %r11, 144(%rsp) + movq 160(%rsp), %r8 + movq 168(%rsp), %r9 + movq %r15, %r10 + movq %rbx, %r11 + # A[15] * B[4] + mulx 32(%rbp), %rax, %rcx + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[15] * B[5] + mulx 40(%rbp), %rax, %rcx + movq %r12, 152(%rsp) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[15] * B[6] + mulx 48(%rbp), %rax, %rcx + movq %r8, 160(%rsp) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[15] * B[7] + mulx 56(%rbp), %rax, %rcx + movq %r9, 168(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r10, %r15 + movq 192(%rdi), %r12 + movq 200(%rdi), %r8 + movq 208(%rdi), %r9 + movq 216(%rdi), %r10 + # A[15] * B[8] + mulx 64(%rbp), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[15] * B[9] + mulx 72(%rbp), %rax, %rcx + movq %r11, %rbx + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[15] * B[10] + mulx 80(%rbp), %rax, %rcx + movq %r12, 192(%rdi) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[15] * B[11] + mulx 88(%rbp), %rax, %rcx + movq %r8, 200(%rdi) + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r9, 208(%rdi) + movq 224(%rdi), %r11 + movq 232(%rdi), %r12 + movq 240(%rdi), %r8 + movq 248(%rdi), %r9 + # A[15] * B[12] + mulx 96(%rbp), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[15] * B[13] + mulx 104(%rbp), %rax, %rcx + movq %r10, 216(%rdi) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[15] * B[14] + mulx 112(%rbp), %rax, %rcx + movq %r11, 224(%rdi) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[15] * B[15] + mulx 120(%rbp), %rax, %rcx + movq %r12, 232(%rdi) + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r8, 240(%rdi) + movq 256(%rdi), %r10 + movq 264(%rdi), %r11 + movq 272(%rdi), %r12 + movq 280(%rdi), %r8 + # A[15] * B[16] + mulx 128(%rbp), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[15] * B[17] + mulx 136(%rbp), %rax, %rcx + movq %r9, 248(%rdi) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[15] * B[18] + mulx 144(%rbp), %rax, %rcx + movq %r10, 256(%rdi) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[15] * B[19] + mulx 152(%rbp), %rax, %rcx + movq %r11, 264(%rdi) + adcxq %rax, %r12 + adoxq %rcx, %r8 + movq %r12, 272(%rdi) + movq 288(%rdi), %r9 + movq 296(%rdi), %r10 + movq 304(%rdi), %r11 + # A[15] * B[20] + mulx 160(%rbp), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[15] * B[21] + mulx 168(%rbp), %rax, %rcx + movq %r8, 280(%rdi) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[15] * B[22] + mulx 176(%rbp), %rax, %rcx + movq %r9, 288(%rdi) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[15] * B[23] + mulx 184(%rbp), %rax, %rcx + movq %r10, 296(%rdi) + movq %r14, %r12 + adcxq %rax, %r11 + adoxq %rcx, %r12 + adcxq %r13, %r12 + movq %r14, %r13 + adoxq %r14, %r13 + adcxq %r14, %r13 + movq %r11, 304(%rdi) + movq %r12, 312(%rdi) + movq 128(%rsi), %rdx + movq 128(%rsp), %r9 + movq 136(%rsp), %r10 + movq 144(%rsp), %r11 + movq 152(%rsp), %r12 + movq 160(%rsp), %r8 + # A[16] * B[0] + mulx (%rbp), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[16] * B[1] + mulx 8(%rbp), %rax, %rcx + movq %r9, 128(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[16] * B[2] + mulx 16(%rbp), %rax, %rcx + movq %r10, 136(%rsp) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[16] * B[3] + mulx 24(%rbp), %rax, %rcx + movq %r11, 144(%rsp) + adcxq %rax, %r12 + adoxq %rcx, %r8 + movq %r12, 152(%rsp) + movq 168(%rsp), %r9 + movq %r15, %r10 + movq %rbx, %r11 + movq 192(%rdi), %r12 + # A[16] * B[4] + mulx 32(%rbp), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[16] * B[5] + mulx 40(%rbp), %rax, %rcx + movq %r8, 160(%rsp) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[16] * B[6] + mulx 48(%rbp), %rax, %rcx + movq %r9, 168(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[16] * B[7] + mulx 56(%rbp), %rax, %rcx + movq %r10, %r15 + adcxq %rax, %r11 + adoxq %rcx, %r12 + movq %r11, %rbx + movq 200(%rdi), %r8 + movq 208(%rdi), %r9 + movq 216(%rdi), %r10 + movq 224(%rdi), %r11 + # A[16] * B[8] + mulx 64(%rbp), %rax, %rcx + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[16] * B[9] + mulx 72(%rbp), %rax, %rcx + movq %r12, 192(%rdi) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[16] * B[10] + mulx 80(%rbp), %rax, %rcx + movq %r8, 200(%rdi) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[16] * B[11] + mulx 88(%rbp), %rax, %rcx + movq %r9, 208(%rdi) + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r10, 216(%rdi) + movq 232(%rdi), %r12 + movq 240(%rdi), %r8 + movq 248(%rdi), %r9 + movq 256(%rdi), %r10 + # A[16] * B[12] + mulx 96(%rbp), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[16] * B[13] + mulx 104(%rbp), %rax, %rcx + movq %r11, 224(%rdi) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[16] * B[14] + mulx 112(%rbp), %rax, %rcx + movq %r12, 232(%rdi) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[16] * B[15] + mulx 120(%rbp), %rax, %rcx + movq %r8, 240(%rdi) + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r9, 248(%rdi) + movq 264(%rdi), %r11 + movq 272(%rdi), %r12 + movq 280(%rdi), %r8 + movq 288(%rdi), %r9 + # A[16] * B[16] + mulx 128(%rbp), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[16] * B[17] + mulx 136(%rbp), %rax, %rcx + movq %r10, 256(%rdi) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[16] * B[18] + mulx 144(%rbp), %rax, %rcx + movq %r11, 264(%rdi) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[16] * B[19] + mulx 152(%rbp), %rax, %rcx + movq %r12, 272(%rdi) + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r8, 280(%rdi) + movq 296(%rdi), %r10 + movq 304(%rdi), %r11 + movq 312(%rdi), %r12 + # A[16] * B[20] + mulx 160(%rbp), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[16] * B[21] + mulx 168(%rbp), %rax, %rcx + movq %r9, 288(%rdi) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[16] * B[22] + mulx 176(%rbp), %rax, %rcx + movq %r10, 296(%rdi) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[16] * B[23] + mulx 184(%rbp), %rax, %rcx + movq %r11, 304(%rdi) + movq %r14, %r8 + adcxq %rax, %r12 + adoxq %rcx, %r8 + adcxq %r13, %r8 + movq %r14, %r13 + adoxq %r14, %r13 + adcxq %r14, %r13 + movq %r12, 312(%rdi) + movq %r8, 320(%rdi) + movq 136(%rsi), %rdx + movq 136(%rsp), %r10 + movq 144(%rsp), %r11 + movq 152(%rsp), %r12 + movq 160(%rsp), %r8 + movq 168(%rsp), %r9 + # A[17] * B[0] + mulx (%rbp), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[17] * B[1] + mulx 8(%rbp), %rax, %rcx + movq %r10, 136(%rsp) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[17] * B[2] + mulx 16(%rbp), %rax, %rcx + movq %r11, 144(%rsp) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[17] * B[3] + mulx 24(%rbp), %rax, %rcx + movq %r12, 152(%rsp) + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r8, 160(%rsp) + movq %r15, %r10 + movq %rbx, %r11 + movq 192(%rdi), %r12 + movq 200(%rdi), %r8 + # A[17] * B[4] + mulx 32(%rbp), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[17] * B[5] + mulx 40(%rbp), %rax, %rcx + movq %r9, 168(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[17] * B[6] + mulx 48(%rbp), %rax, %rcx + movq %r10, %r15 + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[17] * B[7] + mulx 56(%rbp), %rax, %rcx + movq %r11, %rbx + adcxq %rax, %r12 + adoxq %rcx, %r8 + movq %r12, 192(%rdi) + movq 208(%rdi), %r9 + movq 216(%rdi), %r10 + movq 224(%rdi), %r11 + movq 232(%rdi), %r12 + # A[17] * B[8] + mulx 64(%rbp), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[17] * B[9] + mulx 72(%rbp), %rax, %rcx + movq %r8, 200(%rdi) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[17] * B[10] + mulx 80(%rbp), %rax, %rcx + movq %r9, 208(%rdi) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[17] * B[11] + mulx 88(%rbp), %rax, %rcx + movq %r10, 216(%rdi) + adcxq %rax, %r11 + adoxq %rcx, %r12 + movq %r11, 224(%rdi) + movq 240(%rdi), %r8 + movq 248(%rdi), %r9 + movq 256(%rdi), %r10 + movq 264(%rdi), %r11 + # A[17] * B[12] + mulx 96(%rbp), %rax, %rcx + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[17] * B[13] + mulx 104(%rbp), %rax, %rcx + movq %r12, 232(%rdi) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[17] * B[14] + mulx 112(%rbp), %rax, %rcx + movq %r8, 240(%rdi) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[17] * B[15] + mulx 120(%rbp), %rax, %rcx + movq %r9, 248(%rdi) + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r10, 256(%rdi) + movq 272(%rdi), %r12 + movq 280(%rdi), %r8 + movq 288(%rdi), %r9 + movq 296(%rdi), %r10 + # A[17] * B[16] + mulx 128(%rbp), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[17] * B[17] + mulx 136(%rbp), %rax, %rcx + movq %r11, 264(%rdi) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[17] * B[18] + mulx 144(%rbp), %rax, %rcx + movq %r12, 272(%rdi) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[17] * B[19] + mulx 152(%rbp), %rax, %rcx + movq %r8, 280(%rdi) + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r9, 288(%rdi) + movq 304(%rdi), %r11 + movq 312(%rdi), %r12 + movq 320(%rdi), %r8 + # A[17] * B[20] + mulx 160(%rbp), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[17] * B[21] + mulx 168(%rbp), %rax, %rcx + movq %r10, 296(%rdi) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[17] * B[22] + mulx 176(%rbp), %rax, %rcx + movq %r11, 304(%rdi) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[17] * B[23] + mulx 184(%rbp), %rax, %rcx + movq %r12, 312(%rdi) + movq %r14, %r9 + adcxq %rax, %r8 + adoxq %rcx, %r9 + adcxq %r13, %r9 + movq %r14, %r13 + adoxq %r14, %r13 + adcxq %r14, %r13 + movq %r8, 320(%rdi) + movq %r9, 328(%rdi) + movq 144(%rsi), %rdx + movq 144(%rsp), %r11 + movq 152(%rsp), %r12 + movq 160(%rsp), %r8 + movq 168(%rsp), %r9 + movq %r15, %r10 + # A[18] * B[0] + mulx (%rbp), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[18] * B[1] + mulx 8(%rbp), %rax, %rcx + movq %r11, 144(%rsp) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[18] * B[2] + mulx 16(%rbp), %rax, %rcx + movq %r12, 152(%rsp) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[18] * B[3] + mulx 24(%rbp), %rax, %rcx + movq %r8, 160(%rsp) + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r9, 168(%rsp) + movq %rbx, %r11 + movq 192(%rdi), %r12 + movq 200(%rdi), %r8 + movq 208(%rdi), %r9 + # A[18] * B[4] + mulx 32(%rbp), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[18] * B[5] + mulx 40(%rbp), %rax, %rcx + movq %r10, %r15 + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[18] * B[6] + mulx 48(%rbp), %rax, %rcx + movq %r11, %rbx + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[18] * B[7] + mulx 56(%rbp), %rax, %rcx + movq %r12, 192(%rdi) + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r8, 200(%rdi) + movq 216(%rdi), %r10 + movq 224(%rdi), %r11 + movq 232(%rdi), %r12 + movq 240(%rdi), %r8 + # A[18] * B[8] + mulx 64(%rbp), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[18] * B[9] + mulx 72(%rbp), %rax, %rcx + movq %r9, 208(%rdi) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[18] * B[10] + mulx 80(%rbp), %rax, %rcx + movq %r10, 216(%rdi) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[18] * B[11] + mulx 88(%rbp), %rax, %rcx + movq %r11, 224(%rdi) + adcxq %rax, %r12 + adoxq %rcx, %r8 + movq %r12, 232(%rdi) + movq 248(%rdi), %r9 + movq 256(%rdi), %r10 + movq 264(%rdi), %r11 + movq 272(%rdi), %r12 + # A[18] * B[12] + mulx 96(%rbp), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[18] * B[13] + mulx 104(%rbp), %rax, %rcx + movq %r8, 240(%rdi) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[18] * B[14] + mulx 112(%rbp), %rax, %rcx + movq %r9, 248(%rdi) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[18] * B[15] + mulx 120(%rbp), %rax, %rcx + movq %r10, 256(%rdi) + adcxq %rax, %r11 + adoxq %rcx, %r12 + movq %r11, 264(%rdi) + movq 280(%rdi), %r8 + movq 288(%rdi), %r9 + movq 296(%rdi), %r10 + movq 304(%rdi), %r11 + # A[18] * B[16] + mulx 128(%rbp), %rax, %rcx + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[18] * B[17] + mulx 136(%rbp), %rax, %rcx + movq %r12, 272(%rdi) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[18] * B[18] + mulx 144(%rbp), %rax, %rcx + movq %r8, 280(%rdi) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[18] * B[19] + mulx 152(%rbp), %rax, %rcx + movq %r9, 288(%rdi) + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r10, 296(%rdi) + movq 312(%rdi), %r12 + movq 320(%rdi), %r8 + movq 328(%rdi), %r9 + # A[18] * B[20] + mulx 160(%rbp), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[18] * B[21] + mulx 168(%rbp), %rax, %rcx + movq %r11, 304(%rdi) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[18] * B[22] + mulx 176(%rbp), %rax, %rcx + movq %r12, 312(%rdi) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[18] * B[23] + mulx 184(%rbp), %rax, %rcx + movq %r8, 320(%rdi) + movq %r14, %r10 + adcxq %rax, %r9 + adoxq %rcx, %r10 + adcxq %r13, %r10 + movq %r14, %r13 + adoxq %r14, %r13 + adcxq %r14, %r13 + movq %r9, 328(%rdi) + movq %r10, 336(%rdi) + movq 152(%rsi), %rdx + movq 152(%rsp), %r12 + movq 160(%rsp), %r8 + movq 168(%rsp), %r9 + movq %r15, %r10 + movq %rbx, %r11 + # A[19] * B[0] + mulx (%rbp), %rax, %rcx + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[19] * B[1] + mulx 8(%rbp), %rax, %rcx + movq %r12, 152(%rsp) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[19] * B[2] + mulx 16(%rbp), %rax, %rcx + movq %r8, 160(%rsp) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[19] * B[3] + mulx 24(%rbp), %rax, %rcx + movq %r9, 168(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r10, %r15 + movq 192(%rdi), %r12 + movq 200(%rdi), %r8 + movq 208(%rdi), %r9 + movq 216(%rdi), %r10 + # A[19] * B[4] + mulx 32(%rbp), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[19] * B[5] + mulx 40(%rbp), %rax, %rcx + movq %r11, %rbx + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[19] * B[6] + mulx 48(%rbp), %rax, %rcx + movq %r12, 192(%rdi) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[19] * B[7] + mulx 56(%rbp), %rax, %rcx + movq %r8, 200(%rdi) + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r9, 208(%rdi) + movq 224(%rdi), %r11 + movq 232(%rdi), %r12 + movq 240(%rdi), %r8 + movq 248(%rdi), %r9 + # A[19] * B[8] + mulx 64(%rbp), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[19] * B[9] + mulx 72(%rbp), %rax, %rcx + movq %r10, 216(%rdi) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[19] * B[10] + mulx 80(%rbp), %rax, %rcx + movq %r11, 224(%rdi) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[19] * B[11] + mulx 88(%rbp), %rax, %rcx + movq %r12, 232(%rdi) + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r8, 240(%rdi) + movq 256(%rdi), %r10 + movq 264(%rdi), %r11 + movq 272(%rdi), %r12 + movq 280(%rdi), %r8 + # A[19] * B[12] + mulx 96(%rbp), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[19] * B[13] + mulx 104(%rbp), %rax, %rcx + movq %r9, 248(%rdi) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[19] * B[14] + mulx 112(%rbp), %rax, %rcx + movq %r10, 256(%rdi) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[19] * B[15] + mulx 120(%rbp), %rax, %rcx + movq %r11, 264(%rdi) + adcxq %rax, %r12 + adoxq %rcx, %r8 + movq %r12, 272(%rdi) + movq 288(%rdi), %r9 + movq 296(%rdi), %r10 + movq 304(%rdi), %r11 + movq 312(%rdi), %r12 + # A[19] * B[16] + mulx 128(%rbp), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[19] * B[17] + mulx 136(%rbp), %rax, %rcx + movq %r8, 280(%rdi) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[19] * B[18] + mulx 144(%rbp), %rax, %rcx + movq %r9, 288(%rdi) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[19] * B[19] + mulx 152(%rbp), %rax, %rcx + movq %r10, 296(%rdi) + adcxq %rax, %r11 + adoxq %rcx, %r12 + movq %r11, 304(%rdi) + movq 320(%rdi), %r8 + movq 328(%rdi), %r9 + movq 336(%rdi), %r10 + # A[19] * B[20] + mulx 160(%rbp), %rax, %rcx + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[19] * B[21] + mulx 168(%rbp), %rax, %rcx + movq %r12, 312(%rdi) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[19] * B[22] + mulx 176(%rbp), %rax, %rcx + movq %r8, 320(%rdi) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[19] * B[23] + mulx 184(%rbp), %rax, %rcx + movq %r9, 328(%rdi) + movq %r14, %r11 + adcxq %rax, %r10 + adoxq %rcx, %r11 + adcxq %r13, %r11 + movq %r14, %r13 + adoxq %r14, %r13 + adcxq %r14, %r13 + movq %r10, 336(%rdi) + movq %r11, 344(%rdi) + movq 160(%rsi), %rdx + movq 160(%rsp), %r8 + movq 168(%rsp), %r9 + movq %r15, %r10 + movq %rbx, %r11 + movq 192(%rdi), %r12 + # A[20] * B[0] + mulx (%rbp), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[20] * B[1] + mulx 8(%rbp), %rax, %rcx + movq %r8, 160(%rsp) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[20] * B[2] + mulx 16(%rbp), %rax, %rcx + movq %r9, 168(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[20] * B[3] + mulx 24(%rbp), %rax, %rcx + movq %r10, %r15 + adcxq %rax, %r11 + adoxq %rcx, %r12 + movq %r11, %rbx + movq 200(%rdi), %r8 + movq 208(%rdi), %r9 + movq 216(%rdi), %r10 + movq 224(%rdi), %r11 + # A[20] * B[4] + mulx 32(%rbp), %rax, %rcx + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[20] * B[5] + mulx 40(%rbp), %rax, %rcx + movq %r12, 192(%rdi) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[20] * B[6] + mulx 48(%rbp), %rax, %rcx + movq %r8, 200(%rdi) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[20] * B[7] + mulx 56(%rbp), %rax, %rcx + movq %r9, 208(%rdi) + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r10, 216(%rdi) + movq 232(%rdi), %r12 + movq 240(%rdi), %r8 + movq 248(%rdi), %r9 + movq 256(%rdi), %r10 + # A[20] * B[8] + mulx 64(%rbp), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[20] * B[9] + mulx 72(%rbp), %rax, %rcx + movq %r11, 224(%rdi) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[20] * B[10] + mulx 80(%rbp), %rax, %rcx + movq %r12, 232(%rdi) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[20] * B[11] + mulx 88(%rbp), %rax, %rcx + movq %r8, 240(%rdi) + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r9, 248(%rdi) + movq 264(%rdi), %r11 + movq 272(%rdi), %r12 + movq 280(%rdi), %r8 + movq 288(%rdi), %r9 + # A[20] * B[12] + mulx 96(%rbp), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[20] * B[13] + mulx 104(%rbp), %rax, %rcx + movq %r10, 256(%rdi) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[20] * B[14] + mulx 112(%rbp), %rax, %rcx + movq %r11, 264(%rdi) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[20] * B[15] + mulx 120(%rbp), %rax, %rcx + movq %r12, 272(%rdi) + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r8, 280(%rdi) + movq 296(%rdi), %r10 + movq 304(%rdi), %r11 + movq 312(%rdi), %r12 + movq 320(%rdi), %r8 + # A[20] * B[16] + mulx 128(%rbp), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[20] * B[17] + mulx 136(%rbp), %rax, %rcx + movq %r9, 288(%rdi) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[20] * B[18] + mulx 144(%rbp), %rax, %rcx + movq %r10, 296(%rdi) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[20] * B[19] + mulx 152(%rbp), %rax, %rcx + movq %r11, 304(%rdi) + adcxq %rax, %r12 + adoxq %rcx, %r8 + movq %r12, 312(%rdi) + movq 328(%rdi), %r9 + movq 336(%rdi), %r10 + movq 344(%rdi), %r11 + # A[20] * B[20] + mulx 160(%rbp), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[20] * B[21] + mulx 168(%rbp), %rax, %rcx + movq %r8, 320(%rdi) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[20] * B[22] + mulx 176(%rbp), %rax, %rcx + movq %r9, 328(%rdi) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[20] * B[23] + mulx 184(%rbp), %rax, %rcx + movq %r10, 336(%rdi) + movq %r14, %r12 + adcxq %rax, %r11 + adoxq %rcx, %r12 + adcxq %r13, %r12 + movq %r14, %r13 + adoxq %r14, %r13 + adcxq %r14, %r13 + movq %r11, 344(%rdi) + movq %r12, 352(%rdi) + movq 168(%rsi), %rdx + movq 168(%rsp), %r9 + movq %r15, %r10 + movq %rbx, %r11 + movq 192(%rdi), %r12 + movq 200(%rdi), %r8 + # A[21] * B[0] + mulx (%rbp), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[21] * B[1] + mulx 8(%rbp), %rax, %rcx + movq %r9, 168(%rsp) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[21] * B[2] + mulx 16(%rbp), %rax, %rcx + movq %r10, %r15 + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[21] * B[3] + mulx 24(%rbp), %rax, %rcx + movq %r11, %rbx + adcxq %rax, %r12 + adoxq %rcx, %r8 + movq %r12, 192(%rdi) + movq 208(%rdi), %r9 + movq 216(%rdi), %r10 + movq 224(%rdi), %r11 + movq 232(%rdi), %r12 + # A[21] * B[4] + mulx 32(%rbp), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[21] * B[5] + mulx 40(%rbp), %rax, %rcx + movq %r8, 200(%rdi) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[21] * B[6] + mulx 48(%rbp), %rax, %rcx + movq %r9, 208(%rdi) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[21] * B[7] + mulx 56(%rbp), %rax, %rcx + movq %r10, 216(%rdi) + adcxq %rax, %r11 + adoxq %rcx, %r12 + movq %r11, 224(%rdi) + movq 240(%rdi), %r8 + movq 248(%rdi), %r9 + movq 256(%rdi), %r10 + movq 264(%rdi), %r11 + # A[21] * B[8] + mulx 64(%rbp), %rax, %rcx + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[21] * B[9] + mulx 72(%rbp), %rax, %rcx + movq %r12, 232(%rdi) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[21] * B[10] + mulx 80(%rbp), %rax, %rcx + movq %r8, 240(%rdi) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[21] * B[11] + mulx 88(%rbp), %rax, %rcx + movq %r9, 248(%rdi) + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r10, 256(%rdi) + movq 272(%rdi), %r12 + movq 280(%rdi), %r8 + movq 288(%rdi), %r9 + movq 296(%rdi), %r10 + # A[21] * B[12] + mulx 96(%rbp), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[21] * B[13] + mulx 104(%rbp), %rax, %rcx + movq %r11, 264(%rdi) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[21] * B[14] + mulx 112(%rbp), %rax, %rcx + movq %r12, 272(%rdi) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[21] * B[15] + mulx 120(%rbp), %rax, %rcx + movq %r8, 280(%rdi) + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r9, 288(%rdi) + movq 304(%rdi), %r11 + movq 312(%rdi), %r12 + movq 320(%rdi), %r8 + movq 328(%rdi), %r9 + # A[21] * B[16] + mulx 128(%rbp), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[21] * B[17] + mulx 136(%rbp), %rax, %rcx + movq %r10, 296(%rdi) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[21] * B[18] + mulx 144(%rbp), %rax, %rcx + movq %r11, 304(%rdi) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[21] * B[19] + mulx 152(%rbp), %rax, %rcx + movq %r12, 312(%rdi) + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r8, 320(%rdi) + movq 336(%rdi), %r10 + movq 344(%rdi), %r11 + movq 352(%rdi), %r12 + # A[21] * B[20] + mulx 160(%rbp), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[21] * B[21] + mulx 168(%rbp), %rax, %rcx + movq %r9, 328(%rdi) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[21] * B[22] + mulx 176(%rbp), %rax, %rcx + movq %r10, 336(%rdi) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[21] * B[23] + mulx 184(%rbp), %rax, %rcx + movq %r11, 344(%rdi) + movq %r14, %r8 + adcxq %rax, %r12 + adoxq %rcx, %r8 + adcxq %r13, %r8 + movq %r14, %r13 + adoxq %r14, %r13 + adcxq %r14, %r13 + movq %r12, 352(%rdi) + movq %r8, 360(%rdi) + movq 176(%rsi), %rdx + movq %r15, %r10 + movq %rbx, %r11 + movq 192(%rdi), %r12 + movq 200(%rdi), %r8 + movq 208(%rdi), %r9 + # A[22] * B[0] + mulx (%rbp), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[22] * B[1] + mulx 8(%rbp), %rax, %rcx + movq %r10, %r15 + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[22] * B[2] + mulx 16(%rbp), %rax, %rcx + movq %r11, %rbx + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[22] * B[3] + mulx 24(%rbp), %rax, %rcx + movq %r12, 192(%rdi) + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r8, 200(%rdi) + movq 216(%rdi), %r10 + movq 224(%rdi), %r11 + movq 232(%rdi), %r12 + movq 240(%rdi), %r8 + # A[22] * B[4] + mulx 32(%rbp), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[22] * B[5] + mulx 40(%rbp), %rax, %rcx + movq %r9, 208(%rdi) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[22] * B[6] + mulx 48(%rbp), %rax, %rcx + movq %r10, 216(%rdi) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[22] * B[7] + mulx 56(%rbp), %rax, %rcx + movq %r11, 224(%rdi) + adcxq %rax, %r12 + adoxq %rcx, %r8 + movq %r12, 232(%rdi) + movq 248(%rdi), %r9 + movq 256(%rdi), %r10 + movq 264(%rdi), %r11 + movq 272(%rdi), %r12 + # A[22] * B[8] + mulx 64(%rbp), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[22] * B[9] + mulx 72(%rbp), %rax, %rcx + movq %r8, 240(%rdi) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[22] * B[10] + mulx 80(%rbp), %rax, %rcx + movq %r9, 248(%rdi) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[22] * B[11] + mulx 88(%rbp), %rax, %rcx + movq %r10, 256(%rdi) + adcxq %rax, %r11 + adoxq %rcx, %r12 + movq %r11, 264(%rdi) + movq 280(%rdi), %r8 + movq 288(%rdi), %r9 + movq 296(%rdi), %r10 + movq 304(%rdi), %r11 + # A[22] * B[12] + mulx 96(%rbp), %rax, %rcx + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[22] * B[13] + mulx 104(%rbp), %rax, %rcx + movq %r12, 272(%rdi) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[22] * B[14] + mulx 112(%rbp), %rax, %rcx + movq %r8, 280(%rdi) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[22] * B[15] + mulx 120(%rbp), %rax, %rcx + movq %r9, 288(%rdi) + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r10, 296(%rdi) + movq 312(%rdi), %r12 + movq 320(%rdi), %r8 + movq 328(%rdi), %r9 + movq 336(%rdi), %r10 + # A[22] * B[16] + mulx 128(%rbp), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[22] * B[17] + mulx 136(%rbp), %rax, %rcx + movq %r11, 304(%rdi) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[22] * B[18] + mulx 144(%rbp), %rax, %rcx + movq %r12, 312(%rdi) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[22] * B[19] + mulx 152(%rbp), %rax, %rcx + movq %r8, 320(%rdi) + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r9, 328(%rdi) + movq 344(%rdi), %r11 + movq 352(%rdi), %r12 + movq 360(%rdi), %r8 + # A[22] * B[20] + mulx 160(%rbp), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[22] * B[21] + mulx 168(%rbp), %rax, %rcx + movq %r10, 336(%rdi) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[22] * B[22] + mulx 176(%rbp), %rax, %rcx + movq %r11, 344(%rdi) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[22] * B[23] + mulx 184(%rbp), %rax, %rcx + movq %r12, 352(%rdi) + movq %r14, %r9 + adcxq %rax, %r8 + adoxq %rcx, %r9 + adcxq %r13, %r9 + movq %r14, %r13 + adoxq %r14, %r13 + adcxq %r14, %r13 + movq %r8, 360(%rdi) + movq %r9, 368(%rdi) + movq 184(%rsi), %rdx + movq %rbx, %r11 + movq 192(%rdi), %r12 + movq 200(%rdi), %r8 + movq 208(%rdi), %r9 + movq 216(%rdi), %r10 + # A[23] * B[0] + mulx (%rbp), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[23] * B[1] + mulx 8(%rbp), %rax, %rcx + movq %r11, %rbx + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[23] * B[2] + mulx 16(%rbp), %rax, %rcx + movq %r12, 192(%rdi) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[23] * B[3] + mulx 24(%rbp), %rax, %rcx + movq %r8, 200(%rdi) + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r9, 208(%rdi) + movq 224(%rdi), %r11 + movq 232(%rdi), %r12 + movq 240(%rdi), %r8 + movq 248(%rdi), %r9 + # A[23] * B[4] + mulx 32(%rbp), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[23] * B[5] + mulx 40(%rbp), %rax, %rcx + movq %r10, 216(%rdi) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[23] * B[6] + mulx 48(%rbp), %rax, %rcx + movq %r11, 224(%rdi) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[23] * B[7] + mulx 56(%rbp), %rax, %rcx + movq %r12, 232(%rdi) + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r8, 240(%rdi) + movq 256(%rdi), %r10 + movq 264(%rdi), %r11 + movq 272(%rdi), %r12 + movq 280(%rdi), %r8 + # A[23] * B[8] + mulx 64(%rbp), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[23] * B[9] + mulx 72(%rbp), %rax, %rcx + movq %r9, 248(%rdi) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[23] * B[10] + mulx 80(%rbp), %rax, %rcx + movq %r10, 256(%rdi) + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[23] * B[11] + mulx 88(%rbp), %rax, %rcx + movq %r11, 264(%rdi) + adcxq %rax, %r12 + adoxq %rcx, %r8 + movq %r12, 272(%rdi) + movq 288(%rdi), %r9 + movq 296(%rdi), %r10 + movq 304(%rdi), %r11 + movq 312(%rdi), %r12 + # A[23] * B[12] + mulx 96(%rbp), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[23] * B[13] + mulx 104(%rbp), %rax, %rcx + movq %r8, 280(%rdi) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[23] * B[14] + mulx 112(%rbp), %rax, %rcx + movq %r9, 288(%rdi) + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[23] * B[15] + mulx 120(%rbp), %rax, %rcx + movq %r10, 296(%rdi) + adcxq %rax, %r11 + adoxq %rcx, %r12 + movq %r11, 304(%rdi) + movq 320(%rdi), %r8 + movq 328(%rdi), %r9 + movq 336(%rdi), %r10 + movq 344(%rdi), %r11 + # A[23] * B[16] + mulx 128(%rbp), %rax, %rcx + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[23] * B[17] + mulx 136(%rbp), %rax, %rcx + movq %r12, 312(%rdi) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[23] * B[18] + mulx 144(%rbp), %rax, %rcx + movq %r8, 320(%rdi) + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[23] * B[19] + mulx 152(%rbp), %rax, %rcx + movq %r9, 328(%rdi) + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r10, 336(%rdi) + movq 352(%rdi), %r12 + movq 360(%rdi), %r8 + movq 368(%rdi), %r9 + # A[23] * B[20] + mulx 160(%rbp), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r12 + # A[23] * B[21] + mulx 168(%rbp), %rax, %rcx + movq %r11, 344(%rdi) + adcxq %rax, %r12 + adoxq %rcx, %r8 + # A[23] * B[22] + mulx 176(%rbp), %rax, %rcx + movq %r12, 352(%rdi) + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[23] * B[23] + mulx 184(%rbp), %rax, %rcx + movq %r8, 360(%rdi) + movq %r14, %r10 + adcxq %rax, %r9 + adoxq %rcx, %r10 + adcxq %r13, %r10 + movq %r9, 368(%rdi) + movq %r10, 376(%rdi) + movq (%rsp), %r8 + movq 8(%rsp), %r9 + movq 16(%rsp), %r10 + movq 24(%rsp), %r11 + movq %r8, (%rdi) + movq %r9, 8(%rdi) + movq %r10, 16(%rdi) + movq %r11, 24(%rdi) + movq 32(%rsp), %r8 + movq 40(%rsp), %r9 + movq 48(%rsp), %r10 + movq 56(%rsp), %r11 + movq %r8, 32(%rdi) + movq %r9, 40(%rdi) + movq %r10, 48(%rdi) + movq %r11, 56(%rdi) + movq 64(%rsp), %r8 + movq 72(%rsp), %r9 + movq 80(%rsp), %r10 + movq 88(%rsp), %r11 + movq %r8, 64(%rdi) + movq %r9, 72(%rdi) + movq %r10, 80(%rdi) + movq %r11, 88(%rdi) + movq 96(%rsp), %r8 + movq 104(%rsp), %r9 + movq 112(%rsp), %r10 + movq 120(%rsp), %r11 + movq %r8, 96(%rdi) + movq %r9, 104(%rdi) + movq %r10, 112(%rdi) + movq %r11, 120(%rdi) + movq 128(%rsp), %r8 + movq 136(%rsp), %r9 + movq 144(%rsp), %r10 + movq 152(%rsp), %r11 + movq %r8, 128(%rdi) + movq %r9, 136(%rdi) + movq %r10, 144(%rdi) + movq %r11, 152(%rdi) + movq 160(%rsp), %r8 + movq 168(%rsp), %r9 + movq %r15, %r10 + movq %rbx, %r11 + movq %r8, 160(%rdi) + movq %r9, 168(%rdi) + movq %r10, 176(%rdi) + movq %r11, 184(%rdi) + addq $192, %rsp + pop %rbx + pop %r15 + pop %r14 + pop %r13 + pop %r12 + pop %rbp + repz retq +.size sp_3072_mul_avx2_24,.-sp_3072_mul_avx2_24 +#endif /* HAVE_INTEL_AVX2 */ +#ifdef HAVE_INTEL_AVX2 +/* Square a and put result in r. (r = a * a) + * + * r A single precision integer. + * a A single precision integer. + */ +.globl sp_3072_sqr_avx2_24 +.type sp_3072_sqr_avx2_24,@function +.align 16 +sp_3072_sqr_avx2_24: + push %rbp + push %r12 + push %r13 + subq $192, %rsp + cmpq %rdi, %rsi + movq %rsp, %rbp + cmovne %rdi, %rbp + xorq %r12, %r12 + # Diagonal 1 + xorq %r8, %r8 + xorq %r9, %r9 + xorq %r10, %r10 + xorq %r11, %r11 + # A[1] x A[0] + movq (%rsi), %rdx + mulxq 8(%rsi), %r8, %r9 + # A[2] x A[0] + mulxq 16(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[3] x A[0] + mulxq 24(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r8, 8(%rbp) + movq %r9, 16(%rbp) + movq %r10, 24(%rbp) + movq %r12, %r8 + movq %r12, %r9 + movq %r12, %r10 + # A[4] x A[0] + mulxq 32(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[5] x A[0] + mulxq 40(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[6] x A[0] + mulxq 48(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r11, 32(%rbp) + movq %r8, 40(%rbp) + movq %r9, 48(%rbp) + movq %r12, %r11 + movq %r12, %r8 + movq %r12, %r9 + # A[7] x A[0] + mulxq 56(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[8] x A[0] + mulxq 64(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[9] x A[0] + mulxq 72(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r10, 56(%rbp) + movq %r11, 64(%rbp) + movq %r8, 72(%rbp) + movq %r12, %r10 + movq %r12, %r11 + movq %r12, %r8 + # A[10] x A[0] + mulxq 80(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[11] x A[0] + mulxq 88(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[12] x A[0] + mulxq 96(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + movq %r9, 80(%rbp) + movq %r10, 88(%rbp) + movq %r11, 96(%rbp) + movq %r12, %r9 + movq %r12, %r10 + movq %r12, %r11 + # A[13] x A[0] + mulxq 104(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[14] x A[0] + mulxq 112(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[15] x A[0] + mulxq 120(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r8, 104(%rbp) + movq %r9, 112(%rbp) + movq %r10, 120(%rbp) + movq %r12, %r8 + movq %r12, %r9 + movq %r12, %r10 + # A[16] x A[0] + mulxq 128(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[17] x A[0] + mulxq 136(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[18] x A[0] + mulxq 144(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r11, 128(%rbp) + movq %r8, 136(%rbp) + movq %r9, 144(%rbp) + movq %r12, %r11 + movq %r12, %r8 + movq %r12, %r9 + # A[19] x A[0] + mulxq 152(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[20] x A[0] + mulxq 160(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[21] x A[0] + mulxq 168(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r10, 152(%rbp) + movq %r11, 160(%rbp) + movq %r8, 168(%rbp) + movq %r12, %r10 + movq %r12, %r11 + # A[22] x A[0] + mulxq 176(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[23] x A[0] + mulxq 184(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r9, 176(%rbp) + movq %r10, 184(%rbp) + # Carry + adcxq %r12, %r11 + movq %r12, %r13 + adcxq %r12, %r13 + adoxq %r12, %r13 + movq %r11, 192(%rdi) + # Diagonal 2 + movq 24(%rbp), %r11 + movq 32(%rbp), %r8 + movq 40(%rbp), %r9 + movq 48(%rbp), %r10 + # A[2] x A[1] + movq 8(%rsi), %rdx + mulxq 16(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[3] x A[1] + mulxq 24(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[4] x A[1] + mulxq 32(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r11, 24(%rbp) + movq %r8, 32(%rbp) + movq %r9, 40(%rbp) + movq 56(%rbp), %r11 + movq 64(%rbp), %r8 + movq 72(%rbp), %r9 + # A[5] x A[1] + mulxq 40(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[6] x A[1] + mulxq 48(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[7] x A[1] + mulxq 56(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r10, 48(%rbp) + movq %r11, 56(%rbp) + movq %r8, 64(%rbp) + movq 80(%rbp), %r10 + movq 88(%rbp), %r11 + movq 96(%rbp), %r8 + # A[8] x A[1] + mulxq 64(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[9] x A[1] + mulxq 72(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[10] x A[1] + mulxq 80(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + movq %r9, 72(%rbp) + movq %r10, 80(%rbp) + movq %r11, 88(%rbp) + movq 104(%rbp), %r9 + movq 112(%rbp), %r10 + movq 120(%rbp), %r11 + # A[11] x A[1] + mulxq 88(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[12] x A[1] + mulxq 96(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[13] x A[1] + mulxq 104(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r8, 96(%rbp) + movq %r9, 104(%rbp) + movq %r10, 112(%rbp) + movq 128(%rbp), %r8 + movq 136(%rbp), %r9 + movq 144(%rbp), %r10 + # A[14] x A[1] + mulxq 112(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[15] x A[1] + mulxq 120(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[16] x A[1] + mulxq 128(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r11, 120(%rbp) + movq %r8, 128(%rbp) + movq %r9, 136(%rbp) + movq 152(%rbp), %r11 + movq 160(%rbp), %r8 + movq 168(%rbp), %r9 + # A[17] x A[1] + mulxq 136(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[18] x A[1] + mulxq 144(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[19] x A[1] + mulxq 152(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r10, 144(%rbp) + movq %r11, 152(%rbp) + movq %r8, 160(%rbp) + movq 176(%rbp), %r10 + movq 184(%rbp), %r11 + movq 192(%rdi), %r8 + # A[20] x A[1] + mulxq 160(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[21] x A[1] + mulxq 168(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[22] x A[1] + mulxq 176(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + movq %r9, 168(%rbp) + movq %r10, 176(%rbp) + movq %r11, 184(%rbp) + movq %r12, %r9 + movq %r12, %r10 + # A[23] x A[1] + mulxq 184(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[23] x A[2] + movq 16(%rsi), %rdx + mulxq 184(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r8, 192(%rdi) + movq %r9, 200(%rdi) + # Carry + adcxq %r13, %r10 + movq %r12, %r13 + adcxq %r12, %r13 + adoxq %r12, %r13 + movq %r10, 208(%rdi) + # Diagonal 3 + movq 40(%rbp), %r10 + movq 48(%rbp), %r11 + movq 56(%rbp), %r8 + movq 64(%rbp), %r9 + # A[3] x A[2] + mulxq 24(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[4] x A[2] + mulxq 32(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[5] x A[2] + mulxq 40(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r10, 40(%rbp) + movq %r11, 48(%rbp) + movq %r8, 56(%rbp) + movq 72(%rbp), %r10 + movq 80(%rbp), %r11 + movq 88(%rbp), %r8 + # A[6] x A[2] + mulxq 48(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[7] x A[2] + mulxq 56(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[8] x A[2] + mulxq 64(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + movq %r9, 64(%rbp) + movq %r10, 72(%rbp) + movq %r11, 80(%rbp) + movq 96(%rbp), %r9 + movq 104(%rbp), %r10 + movq 112(%rbp), %r11 + # A[9] x A[2] + mulxq 72(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[10] x A[2] + mulxq 80(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[11] x A[2] + mulxq 88(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r8, 88(%rbp) + movq %r9, 96(%rbp) + movq %r10, 104(%rbp) + movq 120(%rbp), %r8 + movq 128(%rbp), %r9 + movq 136(%rbp), %r10 + # A[12] x A[2] + mulxq 96(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[13] x A[2] + mulxq 104(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[14] x A[2] + mulxq 112(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r11, 112(%rbp) + movq %r8, 120(%rbp) + movq %r9, 128(%rbp) + movq 144(%rbp), %r11 + movq 152(%rbp), %r8 + movq 160(%rbp), %r9 + # A[15] x A[2] + mulxq 120(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[16] x A[2] + mulxq 128(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[17] x A[2] + mulxq 136(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r10, 136(%rbp) + movq %r11, 144(%rbp) + movq %r8, 152(%rbp) + movq 168(%rbp), %r10 + movq 176(%rbp), %r11 + movq 184(%rbp), %r8 + # A[18] x A[2] + mulxq 144(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[19] x A[2] + mulxq 152(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[20] x A[2] + mulxq 160(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + movq %r9, 160(%rbp) + movq %r10, 168(%rbp) + movq %r11, 176(%rbp) + movq 192(%rdi), %r9 + movq 200(%rdi), %r10 + movq 208(%rdi), %r11 + # A[21] x A[2] + mulxq 168(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[22] x A[2] + mulxq 176(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[22] x A[3] + movq 176(%rsi), %rdx + mulxq 24(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r8, 184(%rbp) + movq %r9, 192(%rdi) + movq %r10, 200(%rdi) + movq %r12, %r8 + movq %r12, %r9 + # A[22] x A[4] + mulxq 32(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[22] x A[5] + mulxq 40(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r11, 208(%rdi) + movq %r8, 216(%rdi) + # Carry + adcxq %r13, %r9 + movq %r12, %r13 + adcxq %r12, %r13 + adoxq %r12, %r13 + movq %r9, 224(%rdi) + # Diagonal 4 + movq 56(%rbp), %r9 + movq 64(%rbp), %r10 + movq 72(%rbp), %r11 + movq 80(%rbp), %r8 + # A[4] x A[3] + movq 24(%rsi), %rdx + mulxq 32(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[5] x A[3] + mulxq 40(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[6] x A[3] + mulxq 48(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + movq %r9, 56(%rbp) + movq %r10, 64(%rbp) + movq %r11, 72(%rbp) + movq 88(%rbp), %r9 + movq 96(%rbp), %r10 + movq 104(%rbp), %r11 + # A[7] x A[3] + mulxq 56(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[8] x A[3] + mulxq 64(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[9] x A[3] + mulxq 72(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r8, 80(%rbp) + movq %r9, 88(%rbp) + movq %r10, 96(%rbp) + movq 112(%rbp), %r8 + movq 120(%rbp), %r9 + movq 128(%rbp), %r10 + # A[10] x A[3] + mulxq 80(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[11] x A[3] + mulxq 88(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[12] x A[3] + mulxq 96(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r11, 104(%rbp) + movq %r8, 112(%rbp) + movq %r9, 120(%rbp) + movq 136(%rbp), %r11 + movq 144(%rbp), %r8 + movq 152(%rbp), %r9 + # A[13] x A[3] + mulxq 104(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[14] x A[3] + mulxq 112(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[15] x A[3] + mulxq 120(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r10, 128(%rbp) + movq %r11, 136(%rbp) + movq %r8, 144(%rbp) + movq 160(%rbp), %r10 + movq 168(%rbp), %r11 + movq 176(%rbp), %r8 + # A[16] x A[3] + mulxq 128(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[17] x A[3] + mulxq 136(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[18] x A[3] + mulxq 144(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + movq %r9, 152(%rbp) + movq %r10, 160(%rbp) + movq %r11, 168(%rbp) + movq 184(%rbp), %r9 + movq 192(%rdi), %r10 + movq 200(%rdi), %r11 + # A[19] x A[3] + mulxq 152(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[20] x A[3] + mulxq 160(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[21] x A[3] + mulxq 168(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r8, 176(%rbp) + movq %r9, 184(%rbp) + movq %r10, 192(%rdi) + movq 208(%rdi), %r8 + movq 216(%rdi), %r9 + movq 224(%rdi), %r10 + # A[21] x A[4] + movq 168(%rsi), %rdx + mulxq 32(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[21] x A[5] + mulxq 40(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[21] x A[6] + mulxq 48(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r11, 200(%rdi) + movq %r8, 208(%rdi) + movq %r9, 216(%rdi) + movq %r12, %r11 + movq %r12, %r8 + # A[21] x A[7] + mulxq 56(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[21] x A[8] + mulxq 64(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + movq %r10, 224(%rdi) + movq %r11, 232(%rdi) + # Carry + adcxq %r13, %r8 + movq %r12, %r13 + adcxq %r12, %r13 + adoxq %r12, %r13 + movq %r8, 240(%rdi) + # Diagonal 5 + movq 72(%rbp), %r8 + movq 80(%rbp), %r9 + movq 88(%rbp), %r10 + movq 96(%rbp), %r11 + # A[5] x A[4] + movq 32(%rsi), %rdx + mulxq 40(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[6] x A[4] + mulxq 48(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[7] x A[4] + mulxq 56(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r8, 72(%rbp) + movq %r9, 80(%rbp) + movq %r10, 88(%rbp) + movq 104(%rbp), %r8 + movq 112(%rbp), %r9 + movq 120(%rbp), %r10 + # A[8] x A[4] + mulxq 64(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[9] x A[4] + mulxq 72(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[10] x A[4] + mulxq 80(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r11, 96(%rbp) + movq %r8, 104(%rbp) + movq %r9, 112(%rbp) + movq 128(%rbp), %r11 + movq 136(%rbp), %r8 + movq 144(%rbp), %r9 + # A[11] x A[4] + mulxq 88(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[12] x A[4] + mulxq 96(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[13] x A[4] + mulxq 104(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r10, 120(%rbp) + movq %r11, 128(%rbp) + movq %r8, 136(%rbp) + movq 152(%rbp), %r10 + movq 160(%rbp), %r11 + movq 168(%rbp), %r8 + # A[14] x A[4] + mulxq 112(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[15] x A[4] + mulxq 120(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[16] x A[4] + mulxq 128(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + movq %r9, 144(%rbp) + movq %r10, 152(%rbp) + movq %r11, 160(%rbp) + movq 176(%rbp), %r9 + movq 184(%rbp), %r10 + movq 192(%rdi), %r11 + # A[17] x A[4] + mulxq 136(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[18] x A[4] + mulxq 144(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[19] x A[4] + mulxq 152(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r8, 168(%rbp) + movq %r9, 176(%rbp) + movq %r10, 184(%rbp) + movq 200(%rdi), %r8 + movq 208(%rdi), %r9 + movq 216(%rdi), %r10 + # A[20] x A[4] + mulxq 160(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[20] x A[5] + movq 160(%rsi), %rdx + mulxq 40(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[20] x A[6] + mulxq 48(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r11, 192(%rdi) + movq %r8, 200(%rdi) + movq %r9, 208(%rdi) + movq 224(%rdi), %r11 + movq 232(%rdi), %r8 + movq 240(%rdi), %r9 + # A[20] x A[7] + mulxq 56(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[20] x A[8] + mulxq 64(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[20] x A[9] + mulxq 72(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r10, 216(%rdi) + movq %r11, 224(%rdi) + movq %r8, 232(%rdi) + movq %r12, %r10 + movq %r12, %r11 + # A[20] x A[10] + mulxq 80(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[20] x A[11] + mulxq 88(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r9, 240(%rdi) + movq %r10, 248(%rdi) + # Carry + adcxq %r13, %r11 + movq %r12, %r13 + adcxq %r12, %r13 + adoxq %r12, %r13 + movq %r11, 256(%rdi) + # Diagonal 6 + movq 88(%rbp), %r11 + movq 96(%rbp), %r8 + movq 104(%rbp), %r9 + movq 112(%rbp), %r10 + # A[6] x A[5] + movq 40(%rsi), %rdx + mulxq 48(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[7] x A[5] + mulxq 56(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[8] x A[5] + mulxq 64(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r11, 88(%rbp) + movq %r8, 96(%rbp) + movq %r9, 104(%rbp) + movq 120(%rbp), %r11 + movq 128(%rbp), %r8 + movq 136(%rbp), %r9 + # A[9] x A[5] + mulxq 72(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[10] x A[5] + mulxq 80(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[11] x A[5] + mulxq 88(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r10, 112(%rbp) + movq %r11, 120(%rbp) + movq %r8, 128(%rbp) + movq 144(%rbp), %r10 + movq 152(%rbp), %r11 + movq 160(%rbp), %r8 + # A[12] x A[5] + mulxq 96(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[13] x A[5] + mulxq 104(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[14] x A[5] + mulxq 112(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + movq %r9, 136(%rbp) + movq %r10, 144(%rbp) + movq %r11, 152(%rbp) + movq 168(%rbp), %r9 + movq 176(%rbp), %r10 + movq 184(%rbp), %r11 + # A[15] x A[5] + mulxq 120(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[16] x A[5] + mulxq 128(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[17] x A[5] + mulxq 136(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r8, 160(%rbp) + movq %r9, 168(%rbp) + movq %r10, 176(%rbp) + movq 192(%rdi), %r8 + movq 200(%rdi), %r9 + movq 208(%rdi), %r10 + # A[18] x A[5] + mulxq 144(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[19] x A[5] + mulxq 152(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[19] x A[6] + movq 152(%rsi), %rdx + mulxq 48(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r11, 184(%rbp) + movq %r8, 192(%rdi) + movq %r9, 200(%rdi) + movq 216(%rdi), %r11 + movq 224(%rdi), %r8 + movq 232(%rdi), %r9 + # A[19] x A[7] + mulxq 56(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[19] x A[8] + mulxq 64(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[19] x A[9] + mulxq 72(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r10, 208(%rdi) + movq %r11, 216(%rdi) + movq %r8, 224(%rdi) + movq 240(%rdi), %r10 + movq 248(%rdi), %r11 + movq 256(%rdi), %r8 + # A[19] x A[10] + mulxq 80(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[19] x A[11] + mulxq 88(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[19] x A[12] + mulxq 96(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + movq %r9, 232(%rdi) + movq %r10, 240(%rdi) + movq %r11, 248(%rdi) + movq %r12, %r9 + movq %r12, %r10 + # A[19] x A[13] + mulxq 104(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[19] x A[14] + mulxq 112(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r8, 256(%rdi) + movq %r9, 264(%rdi) + # Carry + adcxq %r13, %r10 + movq %r12, %r13 + adcxq %r12, %r13 + adoxq %r12, %r13 + movq %r10, 272(%rdi) + # Diagonal 7 + movq 104(%rbp), %r10 + movq 112(%rbp), %r11 + movq 120(%rbp), %r8 + movq 128(%rbp), %r9 + # A[7] x A[6] + movq 48(%rsi), %rdx + mulxq 56(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[8] x A[6] + mulxq 64(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[9] x A[6] + mulxq 72(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r10, 104(%rbp) + movq %r11, 112(%rbp) + movq %r8, 120(%rbp) + movq 136(%rbp), %r10 + movq 144(%rbp), %r11 + movq 152(%rbp), %r8 + # A[10] x A[6] + mulxq 80(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[11] x A[6] + mulxq 88(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[12] x A[6] + mulxq 96(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + movq %r9, 128(%rbp) + movq %r10, 136(%rbp) + movq %r11, 144(%rbp) + movq 160(%rbp), %r9 + movq 168(%rbp), %r10 + movq 176(%rbp), %r11 + # A[13] x A[6] + mulxq 104(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[14] x A[6] + mulxq 112(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[15] x A[6] + mulxq 120(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r8, 152(%rbp) + movq %r9, 160(%rbp) + movq %r10, 168(%rbp) + movq 184(%rbp), %r8 + movq 192(%rdi), %r9 + movq 200(%rdi), %r10 + # A[16] x A[6] + mulxq 128(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[17] x A[6] + mulxq 136(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[18] x A[6] + mulxq 144(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r11, 176(%rbp) + movq %r8, 184(%rbp) + movq %r9, 192(%rdi) + movq 208(%rdi), %r11 + movq 216(%rdi), %r8 + movq 224(%rdi), %r9 + # A[18] x A[7] + movq 144(%rsi), %rdx + mulxq 56(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[18] x A[8] + mulxq 64(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[18] x A[9] + mulxq 72(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r10, 200(%rdi) + movq %r11, 208(%rdi) + movq %r8, 216(%rdi) + movq 232(%rdi), %r10 + movq 240(%rdi), %r11 + movq 248(%rdi), %r8 + # A[18] x A[10] + mulxq 80(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[18] x A[11] + mulxq 88(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[18] x A[12] + mulxq 96(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + movq %r9, 224(%rdi) + movq %r10, 232(%rdi) + movq %r11, 240(%rdi) + movq 256(%rdi), %r9 + movq 264(%rdi), %r10 + movq 272(%rdi), %r11 + # A[18] x A[13] + mulxq 104(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[18] x A[14] + mulxq 112(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[18] x A[15] + mulxq 120(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r8, 248(%rdi) + movq %r9, 256(%rdi) + movq %r10, 264(%rdi) + movq %r12, %r8 + movq %r12, %r9 + # A[18] x A[16] + mulxq 128(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[18] x A[17] + mulxq 136(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r11, 272(%rdi) + movq %r8, 280(%rdi) + # Carry + adcxq %r13, %r9 + movq %r12, %r13 + adcxq %r12, %r13 + adoxq %r12, %r13 + movq %r9, 288(%rdi) + # Diagonal 8 + movq 120(%rbp), %r9 + movq 128(%rbp), %r10 + movq 136(%rbp), %r11 + movq 144(%rbp), %r8 + # A[8] x A[7] + movq 56(%rsi), %rdx + mulxq 64(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[9] x A[7] + mulxq 72(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[10] x A[7] + mulxq 80(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + movq %r9, 120(%rbp) + movq %r10, 128(%rbp) + movq %r11, 136(%rbp) + movq 152(%rbp), %r9 + movq 160(%rbp), %r10 + movq 168(%rbp), %r11 + # A[11] x A[7] + mulxq 88(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[12] x A[7] + mulxq 96(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[13] x A[7] + mulxq 104(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r8, 144(%rbp) + movq %r9, 152(%rbp) + movq %r10, 160(%rbp) + movq 176(%rbp), %r8 + movq 184(%rbp), %r9 + movq 192(%rdi), %r10 + # A[14] x A[7] + mulxq 112(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[15] x A[7] + mulxq 120(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[16] x A[7] + mulxq 128(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r11, 168(%rbp) + movq %r8, 176(%rbp) + movq %r9, 184(%rbp) + movq 200(%rdi), %r11 + movq 208(%rdi), %r8 + movq 216(%rdi), %r9 + # A[17] x A[7] + mulxq 136(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[17] x A[8] + movq 136(%rsi), %rdx + mulxq 64(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[17] x A[9] + mulxq 72(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r10, 192(%rdi) + movq %r11, 200(%rdi) + movq %r8, 208(%rdi) + movq 224(%rdi), %r10 + movq 232(%rdi), %r11 + movq 240(%rdi), %r8 + # A[17] x A[10] + mulxq 80(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[17] x A[11] + mulxq 88(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[17] x A[12] + mulxq 96(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + movq %r9, 216(%rdi) + movq %r10, 224(%rdi) + movq %r11, 232(%rdi) + movq 248(%rdi), %r9 + movq 256(%rdi), %r10 + movq 264(%rdi), %r11 + # A[17] x A[13] + mulxq 104(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[17] x A[14] + mulxq 112(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[17] x A[15] + mulxq 120(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r8, 240(%rdi) + movq %r9, 248(%rdi) + movq %r10, 256(%rdi) + movq 272(%rdi), %r8 + movq 280(%rdi), %r9 + movq 288(%rdi), %r10 + # A[17] x A[16] + mulxq 128(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[19] x A[15] + movq 152(%rsi), %rdx + mulxq 120(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[19] x A[16] + mulxq 128(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r11, 264(%rdi) + movq %r8, 272(%rdi) + movq %r9, 280(%rdi) + movq %r12, %r11 + movq %r12, %r8 + # A[19] x A[17] + mulxq 136(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[19] x A[18] + mulxq 144(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + movq %r10, 288(%rdi) + movq %r11, 296(%rdi) + # Carry + adcxq %r13, %r8 + movq %r12, %r13 + adcxq %r12, %r13 + adoxq %r12, %r13 + movq %r8, 304(%rdi) + # Diagonal 9 + movq 136(%rbp), %r8 + movq 144(%rbp), %r9 + movq 152(%rbp), %r10 + movq 160(%rbp), %r11 + # A[9] x A[8] + movq 64(%rsi), %rdx + mulxq 72(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[10] x A[8] + mulxq 80(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[11] x A[8] + mulxq 88(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r8, 136(%rbp) + movq %r9, 144(%rbp) + movq %r10, 152(%rbp) + movq 168(%rbp), %r8 + movq 176(%rbp), %r9 + movq 184(%rbp), %r10 + # A[12] x A[8] + mulxq 96(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[13] x A[8] + mulxq 104(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[14] x A[8] + mulxq 112(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r11, 160(%rbp) + movq %r8, 168(%rbp) + movq %r9, 176(%rbp) + movq 192(%rdi), %r11 + movq 200(%rdi), %r8 + movq 208(%rdi), %r9 + # A[15] x A[8] + mulxq 120(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[16] x A[8] + mulxq 128(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[16] x A[9] + movq 128(%rsi), %rdx + mulxq 72(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r10, 184(%rbp) + movq %r11, 192(%rdi) + movq %r8, 200(%rdi) + movq 216(%rdi), %r10 + movq 224(%rdi), %r11 + movq 232(%rdi), %r8 + # A[16] x A[10] + mulxq 80(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[16] x A[11] + mulxq 88(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[16] x A[12] + mulxq 96(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + movq %r9, 208(%rdi) + movq %r10, 216(%rdi) + movq %r11, 224(%rdi) + movq 240(%rdi), %r9 + movq 248(%rdi), %r10 + movq 256(%rdi), %r11 + # A[16] x A[13] + mulxq 104(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[16] x A[14] + mulxq 112(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[16] x A[15] + mulxq 120(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r8, 232(%rdi) + movq %r9, 240(%rdi) + movq %r10, 248(%rdi) + movq 264(%rdi), %r8 + movq 272(%rdi), %r9 + movq 280(%rdi), %r10 + # A[20] x A[12] + movq 160(%rsi), %rdx + mulxq 96(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[20] x A[13] + mulxq 104(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[20] x A[14] + mulxq 112(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r11, 256(%rdi) + movq %r8, 264(%rdi) + movq %r9, 272(%rdi) + movq 288(%rdi), %r11 + movq 296(%rdi), %r8 + movq 304(%rdi), %r9 + # A[20] x A[15] + mulxq 120(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[20] x A[16] + mulxq 128(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[20] x A[17] + mulxq 136(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r10, 280(%rdi) + movq %r11, 288(%rdi) + movq %r8, 296(%rdi) + movq %r12, %r10 + movq %r12, %r11 + # A[20] x A[18] + mulxq 144(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[20] x A[19] + mulxq 152(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r9, 304(%rdi) + movq %r10, 312(%rdi) + # Carry + adcxq %r13, %r11 + movq %r12, %r13 + adcxq %r12, %r13 + adoxq %r12, %r13 + movq %r11, 320(%rdi) + # Diagonal 10 + movq 152(%rbp), %r11 + movq 160(%rbp), %r8 + movq 168(%rbp), %r9 + movq 176(%rbp), %r10 + # A[10] x A[9] + movq 72(%rsi), %rdx + mulxq 80(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[11] x A[9] + mulxq 88(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[12] x A[9] + mulxq 96(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r11, 152(%rbp) + movq %r8, 160(%rbp) + movq %r9, 168(%rbp) + movq 184(%rbp), %r11 + movq 192(%rdi), %r8 + movq 200(%rdi), %r9 + # A[13] x A[9] + mulxq 104(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[14] x A[9] + mulxq 112(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[15] x A[9] + mulxq 120(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r10, 176(%rbp) + movq %r11, 184(%rbp) + movq %r8, 192(%rdi) + movq 208(%rdi), %r10 + movq 216(%rdi), %r11 + movq 224(%rdi), %r8 + # A[15] x A[10] + movq 120(%rsi), %rdx + mulxq 80(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[15] x A[11] + mulxq 88(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[15] x A[12] + mulxq 96(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + movq %r9, 200(%rdi) + movq %r10, 208(%rdi) + movq %r11, 216(%rdi) + movq 232(%rdi), %r9 + movq 240(%rdi), %r10 + movq 248(%rdi), %r11 + # A[15] x A[13] + mulxq 104(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[15] x A[14] + mulxq 112(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[21] x A[9] + movq 168(%rsi), %rdx + mulxq 72(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r8, 224(%rdi) + movq %r9, 232(%rdi) + movq %r10, 240(%rdi) + movq 256(%rdi), %r8 + movq 264(%rdi), %r9 + movq 272(%rdi), %r10 + # A[21] x A[10] + mulxq 80(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[21] x A[11] + mulxq 88(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[21] x A[12] + mulxq 96(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r11, 248(%rdi) + movq %r8, 256(%rdi) + movq %r9, 264(%rdi) + movq 280(%rdi), %r11 + movq 288(%rdi), %r8 + movq 296(%rdi), %r9 + # A[21] x A[13] + mulxq 104(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[21] x A[14] + mulxq 112(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[21] x A[15] + mulxq 120(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r10, 272(%rdi) + movq %r11, 280(%rdi) + movq %r8, 288(%rdi) + movq 304(%rdi), %r10 + movq 312(%rdi), %r11 + movq 320(%rdi), %r8 + # A[21] x A[16] + mulxq 128(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[21] x A[17] + mulxq 136(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[21] x A[18] + mulxq 144(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + movq %r9, 296(%rdi) + movq %r10, 304(%rdi) + movq %r11, 312(%rdi) + movq %r12, %r9 + movq %r12, %r10 + # A[21] x A[19] + mulxq 152(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[21] x A[20] + mulxq 160(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r8, 320(%rdi) + movq %r9, 328(%rdi) + # Carry + adcxq %r13, %r10 + movq %r12, %r13 + adcxq %r12, %r13 + adoxq %r12, %r13 + movq %r10, 336(%rdi) + # Diagonal 11 + movq 168(%rbp), %r10 + movq 176(%rbp), %r11 + movq 184(%rbp), %r8 + movq 192(%rdi), %r9 + # A[11] x A[10] + movq 80(%rsi), %rdx + mulxq 88(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[12] x A[10] + mulxq 96(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[13] x A[10] + mulxq 104(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r10, 168(%rbp) + movq %r11, 176(%rbp) + movq %r8, 184(%rbp) + movq 200(%rdi), %r10 + movq 208(%rdi), %r11 + movq 216(%rdi), %r8 + # A[14] x A[10] + mulxq 112(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[14] x A[11] + movq 112(%rsi), %rdx + mulxq 88(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[14] x A[12] + mulxq 96(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + movq %r9, 192(%rdi) + movq %r10, 200(%rdi) + movq %r11, 208(%rdi) + movq 224(%rdi), %r9 + movq 232(%rdi), %r10 + movq 240(%rdi), %r11 + # A[14] x A[13] + mulxq 104(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[22] x A[6] + movq 176(%rsi), %rdx + mulxq 48(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[22] x A[7] + mulxq 56(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r8, 216(%rdi) + movq %r9, 224(%rdi) + movq %r10, 232(%rdi) + movq 248(%rdi), %r8 + movq 256(%rdi), %r9 + movq 264(%rdi), %r10 + # A[22] x A[8] + mulxq 64(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[22] x A[9] + mulxq 72(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[22] x A[10] + mulxq 80(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r11, 240(%rdi) + movq %r8, 248(%rdi) + movq %r9, 256(%rdi) + movq 272(%rdi), %r11 + movq 280(%rdi), %r8 + movq 288(%rdi), %r9 + # A[22] x A[11] + mulxq 88(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[22] x A[12] + mulxq 96(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[22] x A[13] + mulxq 104(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r10, 264(%rdi) + movq %r11, 272(%rdi) + movq %r8, 280(%rdi) + movq 296(%rdi), %r10 + movq 304(%rdi), %r11 + movq 312(%rdi), %r8 + # A[22] x A[14] + mulxq 112(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[22] x A[15] + mulxq 120(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[22] x A[16] + mulxq 128(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + movq %r9, 288(%rdi) + movq %r10, 296(%rdi) + movq %r11, 304(%rdi) + movq 320(%rdi), %r9 + movq 328(%rdi), %r10 + movq 336(%rdi), %r11 + # A[22] x A[17] + mulxq 136(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[22] x A[18] + mulxq 144(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[22] x A[19] + mulxq 152(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r8, 312(%rdi) + movq %r9, 320(%rdi) + movq %r10, 328(%rdi) + movq %r12, %r8 + movq %r12, %r9 + # A[22] x A[20] + mulxq 160(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[22] x A[21] + mulxq 168(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r11, 336(%rdi) + movq %r8, 344(%rdi) + # Carry + adcxq %r13, %r9 + movq %r12, %r13 + adcxq %r12, %r13 + adoxq %r12, %r13 + movq %r9, 352(%rdi) + # Diagonal 12 + movq 184(%rbp), %r9 + movq 192(%rdi), %r10 + movq 200(%rdi), %r11 + movq 208(%rdi), %r8 + # A[12] x A[11] + movq 88(%rsi), %rdx + mulxq 96(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[13] x A[11] + mulxq 104(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[13] x A[12] + movq 96(%rsi), %rdx + mulxq 104(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + movq %r9, 184(%rbp) + movq %r10, 192(%rdi) + movq %r11, 200(%rdi) + movq 216(%rdi), %r9 + movq 224(%rdi), %r10 + movq 232(%rdi), %r11 + # A[23] x A[3] + movq 184(%rsi), %rdx + mulxq 24(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[23] x A[4] + mulxq 32(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[23] x A[5] + mulxq 40(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r8, 208(%rdi) + movq %r9, 216(%rdi) + movq %r10, 224(%rdi) + movq 240(%rdi), %r8 + movq 248(%rdi), %r9 + movq 256(%rdi), %r10 + # A[23] x A[6] + mulxq 48(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[23] x A[7] + mulxq 56(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[23] x A[8] + mulxq 64(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r11, 232(%rdi) + movq %r8, 240(%rdi) + movq %r9, 248(%rdi) + movq 264(%rdi), %r11 + movq 272(%rdi), %r8 + movq 280(%rdi), %r9 + # A[23] x A[9] + mulxq 72(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[23] x A[10] + mulxq 80(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[23] x A[11] + mulxq 88(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + movq %r10, 256(%rdi) + movq %r11, 264(%rdi) + movq %r8, 272(%rdi) + movq 288(%rdi), %r10 + movq 296(%rdi), %r11 + movq 304(%rdi), %r8 + # A[23] x A[12] + mulxq 96(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[23] x A[13] + mulxq 104(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[23] x A[14] + mulxq 112(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + movq %r9, 280(%rdi) + movq %r10, 288(%rdi) + movq %r11, 296(%rdi) + movq 312(%rdi), %r9 + movq 320(%rdi), %r10 + movq 328(%rdi), %r11 + # A[23] x A[15] + mulxq 120(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[23] x A[16] + mulxq 128(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + # A[23] x A[17] + mulxq 136(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + movq %r8, 304(%rdi) + movq %r9, 312(%rdi) + movq %r10, 320(%rdi) + movq 336(%rdi), %r8 + movq 344(%rdi), %r9 + movq 352(%rdi), %r10 + # A[23] x A[18] + mulxq 144(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + # A[23] x A[19] + mulxq 152(%rsi), %rax, %rcx + adcxq %rax, %r8 + adoxq %rcx, %r9 + # A[23] x A[20] + mulxq 160(%rsi), %rax, %rcx + adcxq %rax, %r9 + adoxq %rcx, %r10 + movq %r11, 328(%rdi) + movq %r8, 336(%rdi) + movq %r9, 344(%rdi) + movq %r12, %r11 + movq %r12, %r8 + # A[23] x A[21] + mulxq 168(%rsi), %rax, %rcx + adcxq %rax, %r10 + adoxq %rcx, %r11 + # A[23] x A[22] + mulxq 176(%rsi), %rax, %rcx + adcxq %rax, %r11 + adoxq %rcx, %r8 + movq %r10, 352(%rdi) + movq %r11, 360(%rdi) + # Carry + adcxq %r13, %r8 + movq %r12, %r13 + adcxq %r12, %r13 + adoxq %r12, %r13 + movq %r8, 368(%rdi) + movq %r13, 376(%rdi) + # Double and Add in A[i] x A[i] + movq 8(%rbp), %r9 + # A[0] x A[0] + movq (%rsi), %rdx + mulxq %rdx, %rax, %rcx + movq %rax, (%rbp) + adoxq %r9, %r9 + adcxq %rcx, %r9 + movq %r9, 8(%rbp) + movq 16(%rbp), %r8 + movq 24(%rbp), %r9 + # A[1] x A[1] + movq 8(%rsi), %rdx + mulxq %rdx, %rax, %rcx + adoxq %r8, %r8 + adcxq %rax, %r8 + adoxq %r9, %r9 + adcxq %rcx, %r9 + movq %r8, 16(%rbp) + movq %r9, 24(%rbp) + movq 32(%rbp), %r8 + movq 40(%rbp), %r9 + # A[2] x A[2] + movq 16(%rsi), %rdx + mulxq %rdx, %rax, %rcx + adoxq %r8, %r8 + adcxq %rax, %r8 + adoxq %r9, %r9 + adcxq %rcx, %r9 + movq %r8, 32(%rbp) + movq %r9, 40(%rbp) + movq 48(%rbp), %r8 + movq 56(%rbp), %r9 + # A[3] x A[3] + movq 24(%rsi), %rdx + mulxq %rdx, %rax, %rcx + adoxq %r8, %r8 + adcxq %rax, %r8 + adoxq %r9, %r9 + adcxq %rcx, %r9 + movq %r8, 48(%rbp) + movq %r9, 56(%rbp) + movq 64(%rbp), %r8 + movq 72(%rbp), %r9 + # A[4] x A[4] + movq 32(%rsi), %rdx + mulxq %rdx, %rax, %rcx + adoxq %r8, %r8 + adcxq %rax, %r8 + adoxq %r9, %r9 + adcxq %rcx, %r9 + movq %r8, 64(%rbp) + movq %r9, 72(%rbp) + movq 80(%rbp), %r8 + movq 88(%rbp), %r9 + # A[5] x A[5] + movq 40(%rsi), %rdx + mulxq %rdx, %rax, %rcx + adoxq %r8, %r8 + adcxq %rax, %r8 + adoxq %r9, %r9 + adcxq %rcx, %r9 + movq %r8, 80(%rbp) + movq %r9, 88(%rbp) + movq 96(%rbp), %r8 + movq 104(%rbp), %r9 + # A[6] x A[6] + movq 48(%rsi), %rdx + mulxq %rdx, %rax, %rcx + adoxq %r8, %r8 + adcxq %rax, %r8 + adoxq %r9, %r9 + adcxq %rcx, %r9 + movq %r8, 96(%rbp) + movq %r9, 104(%rbp) + movq 112(%rbp), %r8 + movq 120(%rbp), %r9 + # A[7] x A[7] + movq 56(%rsi), %rdx + mulxq %rdx, %rax, %rcx + adoxq %r8, %r8 + adcxq %rax, %r8 + adoxq %r9, %r9 + adcxq %rcx, %r9 + movq %r8, 112(%rbp) + movq %r9, 120(%rbp) + movq 128(%rbp), %r8 + movq 136(%rbp), %r9 + # A[8] x A[8] + movq 64(%rsi), %rdx + mulxq %rdx, %rax, %rcx + adoxq %r8, %r8 + adcxq %rax, %r8 + adoxq %r9, %r9 + adcxq %rcx, %r9 + movq %r8, 128(%rbp) + movq %r9, 136(%rbp) + movq 144(%rbp), %r8 + movq 152(%rbp), %r9 + # A[9] x A[9] + movq 72(%rsi), %rdx + mulxq %rdx, %rax, %rcx + adoxq %r8, %r8 + adcxq %rax, %r8 + adoxq %r9, %r9 + adcxq %rcx, %r9 + movq %r8, 144(%rbp) + movq %r9, 152(%rbp) + movq 160(%rbp), %r8 + movq 168(%rbp), %r9 + # A[10] x A[10] + movq 80(%rsi), %rdx + mulxq %rdx, %rax, %rcx + adoxq %r8, %r8 + adcxq %rax, %r8 + adoxq %r9, %r9 + adcxq %rcx, %r9 + movq %r8, 160(%rbp) + movq %r9, 168(%rbp) + movq 176(%rbp), %r8 + movq 184(%rbp), %r9 + # A[11] x A[11] + movq 88(%rsi), %rdx + mulxq %rdx, %rax, %rcx + adoxq %r8, %r8 + adcxq %rax, %r8 + adoxq %r9, %r9 + adcxq %rcx, %r9 + movq %r8, 176(%rbp) + movq %r9, 184(%rbp) + movq 192(%rdi), %r8 + movq 200(%rdi), %r9 + # A[12] x A[12] + movq 96(%rsi), %rdx + mulxq %rdx, %rax, %rcx + adoxq %r8, %r8 + adcxq %rax, %r8 + adoxq %r9, %r9 + adcxq %rcx, %r9 + movq %r8, 192(%rdi) + movq %r9, 200(%rdi) + movq 208(%rdi), %r8 + movq 216(%rdi), %r9 + # A[13] x A[13] + movq 104(%rsi), %rdx + mulxq %rdx, %rax, %rcx + adoxq %r8, %r8 + adcxq %rax, %r8 + adoxq %r9, %r9 + adcxq %rcx, %r9 + movq %r8, 208(%rdi) + movq %r9, 216(%rdi) + movq 224(%rdi), %r8 + movq 232(%rdi), %r9 + # A[14] x A[14] + movq 112(%rsi), %rdx + mulxq %rdx, %rax, %rcx + adoxq %r8, %r8 + adcxq %rax, %r8 + adoxq %r9, %r9 + adcxq %rcx, %r9 + movq %r8, 224(%rdi) + movq %r9, 232(%rdi) + movq 240(%rdi), %r8 + movq 248(%rdi), %r9 + # A[15] x A[15] + movq 120(%rsi), %rdx + mulxq %rdx, %rax, %rcx + adoxq %r8, %r8 + adcxq %rax, %r8 + adoxq %r9, %r9 + adcxq %rcx, %r9 + movq %r8, 240(%rdi) + movq %r9, 248(%rdi) + movq 256(%rdi), %r8 + movq 264(%rdi), %r9 + # A[16] x A[16] + movq 128(%rsi), %rdx + mulxq %rdx, %rax, %rcx + adoxq %r8, %r8 + adcxq %rax, %r8 + adoxq %r9, %r9 + adcxq %rcx, %r9 + movq %r8, 256(%rdi) + movq %r9, 264(%rdi) + movq 272(%rdi), %r8 + movq 280(%rdi), %r9 + # A[17] x A[17] + movq 136(%rsi), %rdx + mulxq %rdx, %rax, %rcx + adoxq %r8, %r8 + adcxq %rax, %r8 + adoxq %r9, %r9 + adcxq %rcx, %r9 + movq %r8, 272(%rdi) + movq %r9, 280(%rdi) + movq 288(%rdi), %r8 + movq 296(%rdi), %r9 + # A[18] x A[18] + movq 144(%rsi), %rdx + mulxq %rdx, %rax, %rcx + adoxq %r8, %r8 + adcxq %rax, %r8 + adoxq %r9, %r9 + adcxq %rcx, %r9 + movq %r8, 288(%rdi) + movq %r9, 296(%rdi) + movq 304(%rdi), %r8 + movq 312(%rdi), %r9 + # A[19] x A[19] + movq 152(%rsi), %rdx + mulxq %rdx, %rax, %rcx + adoxq %r8, %r8 + adcxq %rax, %r8 + adoxq %r9, %r9 + adcxq %rcx, %r9 + movq %r8, 304(%rdi) + movq %r9, 312(%rdi) + movq 320(%rdi), %r8 + movq 328(%rdi), %r9 + # A[20] x A[20] + movq 160(%rsi), %rdx + mulxq %rdx, %rax, %rcx + adoxq %r8, %r8 + adcxq %rax, %r8 + adoxq %r9, %r9 + adcxq %rcx, %r9 + movq %r8, 320(%rdi) + movq %r9, 328(%rdi) + movq 336(%rdi), %r8 + movq 344(%rdi), %r9 + # A[21] x A[21] + movq 168(%rsi), %rdx + mulxq %rdx, %rax, %rcx + adoxq %r8, %r8 + adcxq %rax, %r8 + adoxq %r9, %r9 + adcxq %rcx, %r9 + movq %r8, 336(%rdi) + movq %r9, 344(%rdi) + movq 352(%rdi), %r8 + movq 360(%rdi), %r9 + # A[22] x A[22] + movq 176(%rsi), %rdx + mulxq %rdx, %rax, %rcx + adoxq %r8, %r8 + adcxq %rax, %r8 + adoxq %r9, %r9 + adcxq %rcx, %r9 + movq %r8, 352(%rdi) + movq %r9, 360(%rdi) + movq 368(%rdi), %r8 + movq 376(%rdi), %r9 + # A[23] x A[23] + movq 184(%rsi), %rdx + mulxq %rdx, %rax, %rcx + adoxq %r8, %r8 + adcxq %rax, %r8 + adoxq %r9, %r9 + adcxq %rcx, %r9 + movq %r8, 368(%rdi) + movq %r9, 376(%rdi) + cmpq %rdi, %rsi + jne L_end_3072_sqr_avx2_24 + movq (%rbp), %r8 + movq 8(%rbp), %r9 + movq 16(%rbp), %r10 + movq 24(%rbp), %r11 + movq %r8, (%rdi) + movq %r9, 8(%rdi) + movq %r10, 16(%rdi) + movq %r11, 24(%rdi) + movq 32(%rbp), %r8 + movq 40(%rbp), %r9 + movq 48(%rbp), %r10 + movq 56(%rbp), %r11 + movq %r8, 32(%rdi) + movq %r9, 40(%rdi) + movq %r10, 48(%rdi) + movq %r11, 56(%rdi) + movq 64(%rbp), %r8 + movq 72(%rbp), %r9 + movq 80(%rbp), %r10 + movq 88(%rbp), %r11 + movq %r8, 64(%rdi) + movq %r9, 72(%rdi) + movq %r10, 80(%rdi) + movq %r11, 88(%rdi) + movq 96(%rbp), %r8 + movq 104(%rbp), %r9 + movq 112(%rbp), %r10 + movq 120(%rbp), %r11 + movq %r8, 96(%rdi) + movq %r9, 104(%rdi) + movq %r10, 112(%rdi) + movq %r11, 120(%rdi) + movq 128(%rbp), %r8 + movq 136(%rbp), %r9 + movq 144(%rbp), %r10 + movq 152(%rbp), %r11 + movq %r8, 128(%rdi) + movq %r9, 136(%rdi) + movq %r10, 144(%rdi) + movq %r11, 152(%rdi) + movq 160(%rbp), %r8 + movq 168(%rbp), %r9 + movq 176(%rbp), %r10 + movq 184(%rbp), %r11 + movq %r8, 160(%rdi) + movq %r9, 168(%rdi) + movq %r10, 176(%rdi) + movq %r11, 184(%rdi) +L_end_3072_sqr_avx2_24: + addq $192, %rsp + pop %r13 + pop %r12 + pop %rbp + repz retq +.size sp_3072_sqr_avx2_24,.-sp_3072_sqr_avx2_24 +#endif /* HAVE_INTEL_AVX2 */ +/* Add b to a into r. (r = a + b) + * + * r A single precision integer. + * a A single precision integer. + * b A single precision integer. + */ +.globl sp_3072_add_24 +.type sp_3072_add_24,@function +.align 16 +sp_3072_add_24: + xorq %rax, %rax + movq (%rsi), %rcx + addq (%rdx), %rcx + movq %rcx, (%rdi) + movq 8(%rsi), %rcx + adcq 8(%rdx), %rcx + movq %rcx, 8(%rdi) + movq 16(%rsi), %rcx + adcq 16(%rdx), %rcx + movq %rcx, 16(%rdi) + movq 24(%rsi), %rcx + adcq 24(%rdx), %rcx + movq %rcx, 24(%rdi) + movq 32(%rsi), %rcx + adcq 32(%rdx), %rcx + movq %rcx, 32(%rdi) + movq 40(%rsi), %rcx + adcq 40(%rdx), %rcx + movq %rcx, 40(%rdi) + movq 48(%rsi), %rcx + adcq 48(%rdx), %rcx + movq %rcx, 48(%rdi) + movq 56(%rsi), %rcx + adcq 56(%rdx), %rcx + movq %rcx, 56(%rdi) + movq 64(%rsi), %rcx + adcq 64(%rdx), %rcx + movq %rcx, 64(%rdi) + movq 72(%rsi), %rcx + adcq 72(%rdx), %rcx + movq %rcx, 72(%rdi) + movq 80(%rsi), %rcx + adcq 80(%rdx), %rcx + movq %rcx, 80(%rdi) + movq 88(%rsi), %rcx + adcq 88(%rdx), %rcx + movq %rcx, 88(%rdi) + movq 96(%rsi), %rcx + adcq 96(%rdx), %rcx + movq %rcx, 96(%rdi) + movq 104(%rsi), %rcx + adcq 104(%rdx), %rcx + movq %rcx, 104(%rdi) + movq 112(%rsi), %rcx + adcq 112(%rdx), %rcx + movq %rcx, 112(%rdi) + movq 120(%rsi), %rcx + adcq 120(%rdx), %rcx + movq %rcx, 120(%rdi) + movq 128(%rsi), %rcx + adcq 128(%rdx), %rcx + movq %rcx, 128(%rdi) + movq 136(%rsi), %rcx + adcq 136(%rdx), %rcx + movq %rcx, 136(%rdi) + movq 144(%rsi), %rcx + adcq 144(%rdx), %rcx + movq %rcx, 144(%rdi) + movq 152(%rsi), %rcx + adcq 152(%rdx), %rcx + movq %rcx, 152(%rdi) + movq 160(%rsi), %rcx + adcq 160(%rdx), %rcx + movq %rcx, 160(%rdi) + movq 168(%rsi), %rcx + adcq 168(%rdx), %rcx + movq %rcx, 168(%rdi) + movq 176(%rsi), %rcx + adcq 176(%rdx), %rcx + movq %rcx, 176(%rdi) + movq 184(%rsi), %rcx + adcq 184(%rdx), %rcx + movq %rcx, 184(%rdi) + adcq $0, %rax + repz retq +.size sp_3072_add_24,.-sp_3072_add_24 +/* Sub b from a into a. (a -= b) + * + * a A single precision integer and result. + * b A single precision integer. + */ +.globl sp_3072_sub_in_place_48 +.type sp_3072_sub_in_place_48,@function +.align 16 +sp_3072_sub_in_place_48: + xorq %rax, %rax + movq (%rdi), %rdx + movq 8(%rdi), %rcx + movq (%rsi), %r8 + movq 8(%rsi), %r9 + subq %r8, %rdx + movq 16(%rsi), %r8 + movq %rdx, (%rdi) + movq 16(%rdi), %rdx + sbbq %r9, %rcx + movq 24(%rsi), %r9 + movq %rcx, 8(%rdi) + movq 24(%rdi), %rcx + sbbq %r8, %rdx + movq 32(%rsi), %r8 + movq %rdx, 16(%rdi) + movq 32(%rdi), %rdx + sbbq %r9, %rcx + movq 40(%rsi), %r9 + movq %rcx, 24(%rdi) + movq 40(%rdi), %rcx + sbbq %r8, %rdx + movq 48(%rsi), %r8 + movq %rdx, 32(%rdi) + movq 48(%rdi), %rdx + sbbq %r9, %rcx + movq 56(%rsi), %r9 + movq %rcx, 40(%rdi) + movq 56(%rdi), %rcx + sbbq %r8, %rdx + movq 64(%rsi), %r8 + movq %rdx, 48(%rdi) + movq 64(%rdi), %rdx + sbbq %r9, %rcx + movq 72(%rsi), %r9 + movq %rcx, 56(%rdi) + movq 72(%rdi), %rcx + sbbq %r8, %rdx + movq 80(%rsi), %r8 + movq %rdx, 64(%rdi) + movq 80(%rdi), %rdx + sbbq %r9, %rcx + movq 88(%rsi), %r9 + movq %rcx, 72(%rdi) + movq 88(%rdi), %rcx + sbbq %r8, %rdx + movq 96(%rsi), %r8 + movq %rdx, 80(%rdi) + movq 96(%rdi), %rdx + sbbq %r9, %rcx + movq 104(%rsi), %r9 + movq %rcx, 88(%rdi) + movq 104(%rdi), %rcx + sbbq %r8, %rdx + movq 112(%rsi), %r8 + movq %rdx, 96(%rdi) + movq 112(%rdi), %rdx + sbbq %r9, %rcx + movq 120(%rsi), %r9 + movq %rcx, 104(%rdi) + movq 120(%rdi), %rcx + sbbq %r8, %rdx + movq 128(%rsi), %r8 + movq %rdx, 112(%rdi) + movq 128(%rdi), %rdx + sbbq %r9, %rcx + movq 136(%rsi), %r9 + movq %rcx, 120(%rdi) + movq 136(%rdi), %rcx + sbbq %r8, %rdx + movq 144(%rsi), %r8 + movq %rdx, 128(%rdi) + movq 144(%rdi), %rdx + sbbq %r9, %rcx + movq 152(%rsi), %r9 + movq %rcx, 136(%rdi) + movq 152(%rdi), %rcx + sbbq %r8, %rdx + movq 160(%rsi), %r8 + movq %rdx, 144(%rdi) + movq 160(%rdi), %rdx + sbbq %r9, %rcx + movq 168(%rsi), %r9 + movq %rcx, 152(%rdi) + movq 168(%rdi), %rcx + sbbq %r8, %rdx + movq 176(%rsi), %r8 + movq %rdx, 160(%rdi) + movq 176(%rdi), %rdx + sbbq %r9, %rcx + movq 184(%rsi), %r9 + movq %rcx, 168(%rdi) + movq 184(%rdi), %rcx + sbbq %r8, %rdx + movq 192(%rsi), %r8 + movq %rdx, 176(%rdi) + movq 192(%rdi), %rdx + sbbq %r9, %rcx + movq 200(%rsi), %r9 + movq %rcx, 184(%rdi) + movq 200(%rdi), %rcx + sbbq %r8, %rdx + movq 208(%rsi), %r8 + movq %rdx, 192(%rdi) + movq 208(%rdi), %rdx + sbbq %r9, %rcx + movq 216(%rsi), %r9 + movq %rcx, 200(%rdi) + movq 216(%rdi), %rcx + sbbq %r8, %rdx + movq 224(%rsi), %r8 + movq %rdx, 208(%rdi) + movq 224(%rdi), %rdx + sbbq %r9, %rcx + movq 232(%rsi), %r9 + movq %rcx, 216(%rdi) + movq 232(%rdi), %rcx + sbbq %r8, %rdx + movq 240(%rsi), %r8 + movq %rdx, 224(%rdi) + movq 240(%rdi), %rdx + sbbq %r9, %rcx + movq 248(%rsi), %r9 + movq %rcx, 232(%rdi) + movq 248(%rdi), %rcx + sbbq %r8, %rdx + movq 256(%rsi), %r8 + movq %rdx, 240(%rdi) + movq 256(%rdi), %rdx + sbbq %r9, %rcx + movq 264(%rsi), %r9 + movq %rcx, 248(%rdi) + movq 264(%rdi), %rcx + sbbq %r8, %rdx + movq 272(%rsi), %r8 + movq %rdx, 256(%rdi) + movq 272(%rdi), %rdx + sbbq %r9, %rcx + movq 280(%rsi), %r9 + movq %rcx, 264(%rdi) + movq 280(%rdi), %rcx + sbbq %r8, %rdx + movq 288(%rsi), %r8 + movq %rdx, 272(%rdi) + movq 288(%rdi), %rdx + sbbq %r9, %rcx + movq 296(%rsi), %r9 + movq %rcx, 280(%rdi) + movq 296(%rdi), %rcx + sbbq %r8, %rdx + movq 304(%rsi), %r8 + movq %rdx, 288(%rdi) + movq 304(%rdi), %rdx + sbbq %r9, %rcx + movq 312(%rsi), %r9 + movq %rcx, 296(%rdi) + movq 312(%rdi), %rcx + sbbq %r8, %rdx + movq 320(%rsi), %r8 + movq %rdx, 304(%rdi) + movq 320(%rdi), %rdx + sbbq %r9, %rcx + movq 328(%rsi), %r9 + movq %rcx, 312(%rdi) + movq 328(%rdi), %rcx + sbbq %r8, %rdx + movq 336(%rsi), %r8 + movq %rdx, 320(%rdi) + movq 336(%rdi), %rdx + sbbq %r9, %rcx + movq 344(%rsi), %r9 + movq %rcx, 328(%rdi) + movq 344(%rdi), %rcx + sbbq %r8, %rdx + movq 352(%rsi), %r8 + movq %rdx, 336(%rdi) + movq 352(%rdi), %rdx + sbbq %r9, %rcx + movq 360(%rsi), %r9 + movq %rcx, 344(%rdi) + movq 360(%rdi), %rcx + sbbq %r8, %rdx + movq 368(%rsi), %r8 + movq %rdx, 352(%rdi) + movq 368(%rdi), %rdx + sbbq %r9, %rcx + movq 376(%rsi), %r9 + movq %rcx, 360(%rdi) + movq 376(%rdi), %rcx + sbbq %r8, %rdx + movq %rdx, 368(%rdi) + sbbq %r9, %rcx + movq %rcx, 376(%rdi) + sbbq $0, %rax + repz retq +.size sp_3072_sub_in_place_48,.-sp_3072_sub_in_place_48 +/* Add b to a into r. (r = a + b) + * + * r A single precision integer. + * a A single precision integer. + * b A single precision integer. + */ +.globl sp_3072_add_48 +.type sp_3072_add_48,@function +.align 16 +sp_3072_add_48: + xorq %rax, %rax + movq (%rsi), %rcx + addq (%rdx), %rcx + movq %rcx, (%rdi) + movq 8(%rsi), %rcx + adcq 8(%rdx), %rcx + movq %rcx, 8(%rdi) + movq 16(%rsi), %rcx + adcq 16(%rdx), %rcx + movq %rcx, 16(%rdi) + movq 24(%rsi), %rcx + adcq 24(%rdx), %rcx + movq %rcx, 24(%rdi) + movq 32(%rsi), %rcx + adcq 32(%rdx), %rcx + movq %rcx, 32(%rdi) + movq 40(%rsi), %rcx + adcq 40(%rdx), %rcx + movq %rcx, 40(%rdi) + movq 48(%rsi), %rcx + adcq 48(%rdx), %rcx + movq %rcx, 48(%rdi) + movq 56(%rsi), %rcx + adcq 56(%rdx), %rcx + movq %rcx, 56(%rdi) + movq 64(%rsi), %rcx + adcq 64(%rdx), %rcx + movq %rcx, 64(%rdi) + movq 72(%rsi), %rcx + adcq 72(%rdx), %rcx + movq %rcx, 72(%rdi) + movq 80(%rsi), %rcx + adcq 80(%rdx), %rcx + movq %rcx, 80(%rdi) + movq 88(%rsi), %rcx + adcq 88(%rdx), %rcx + movq %rcx, 88(%rdi) + movq 96(%rsi), %rcx + adcq 96(%rdx), %rcx + movq %rcx, 96(%rdi) + movq 104(%rsi), %rcx + adcq 104(%rdx), %rcx + movq %rcx, 104(%rdi) + movq 112(%rsi), %rcx + adcq 112(%rdx), %rcx + movq %rcx, 112(%rdi) + movq 120(%rsi), %rcx + adcq 120(%rdx), %rcx + movq %rcx, 120(%rdi) + movq 128(%rsi), %rcx + adcq 128(%rdx), %rcx + movq %rcx, 128(%rdi) + movq 136(%rsi), %rcx + adcq 136(%rdx), %rcx + movq %rcx, 136(%rdi) + movq 144(%rsi), %rcx + adcq 144(%rdx), %rcx + movq %rcx, 144(%rdi) + movq 152(%rsi), %rcx + adcq 152(%rdx), %rcx + movq %rcx, 152(%rdi) + movq 160(%rsi), %rcx + adcq 160(%rdx), %rcx + movq %rcx, 160(%rdi) + movq 168(%rsi), %rcx + adcq 168(%rdx), %rcx + movq %rcx, 168(%rdi) + movq 176(%rsi), %rcx + adcq 176(%rdx), %rcx + movq %rcx, 176(%rdi) + movq 184(%rsi), %rcx + adcq 184(%rdx), %rcx + movq %rcx, 184(%rdi) + movq 192(%rsi), %rcx + adcq 192(%rdx), %rcx + movq %rcx, 192(%rdi) + movq 200(%rsi), %rcx + adcq 200(%rdx), %rcx + movq %rcx, 200(%rdi) + movq 208(%rsi), %rcx + adcq 208(%rdx), %rcx + movq %rcx, 208(%rdi) + movq 216(%rsi), %rcx + adcq 216(%rdx), %rcx + movq %rcx, 216(%rdi) + movq 224(%rsi), %rcx + adcq 224(%rdx), %rcx + movq %rcx, 224(%rdi) + movq 232(%rsi), %rcx + adcq 232(%rdx), %rcx + movq %rcx, 232(%rdi) + movq 240(%rsi), %rcx + adcq 240(%rdx), %rcx + movq %rcx, 240(%rdi) + movq 248(%rsi), %rcx + adcq 248(%rdx), %rcx + movq %rcx, 248(%rdi) + movq 256(%rsi), %rcx + adcq 256(%rdx), %rcx + movq %rcx, 256(%rdi) + movq 264(%rsi), %rcx + adcq 264(%rdx), %rcx + movq %rcx, 264(%rdi) + movq 272(%rsi), %rcx + adcq 272(%rdx), %rcx + movq %rcx, 272(%rdi) + movq 280(%rsi), %rcx + adcq 280(%rdx), %rcx + movq %rcx, 280(%rdi) + movq 288(%rsi), %rcx + adcq 288(%rdx), %rcx + movq %rcx, 288(%rdi) + movq 296(%rsi), %rcx + adcq 296(%rdx), %rcx + movq %rcx, 296(%rdi) + movq 304(%rsi), %rcx + adcq 304(%rdx), %rcx + movq %rcx, 304(%rdi) + movq 312(%rsi), %rcx + adcq 312(%rdx), %rcx + movq %rcx, 312(%rdi) + movq 320(%rsi), %rcx + adcq 320(%rdx), %rcx + movq %rcx, 320(%rdi) + movq 328(%rsi), %rcx + adcq 328(%rdx), %rcx + movq %rcx, 328(%rdi) + movq 336(%rsi), %rcx + adcq 336(%rdx), %rcx + movq %rcx, 336(%rdi) + movq 344(%rsi), %rcx + adcq 344(%rdx), %rcx + movq %rcx, 344(%rdi) + movq 352(%rsi), %rcx + adcq 352(%rdx), %rcx + movq %rcx, 352(%rdi) + movq 360(%rsi), %rcx + adcq 360(%rdx), %rcx + movq %rcx, 360(%rdi) + movq 368(%rsi), %rcx + adcq 368(%rdx), %rcx + movq %rcx, 368(%rdi) + movq 376(%rsi), %rcx + adcq 376(%rdx), %rcx + movq %rcx, 376(%rdi) + adcq $0, %rax + repz retq +.size sp_3072_add_48,.-sp_3072_add_48 +#ifdef HAVE_INTEL_AVX2 +#endif /* HAVE_INTEL_AVX2 */ +#ifdef HAVE_INTEL_AVX2 +#endif /* HAVE_INTEL_AVX2 */ +#if !defined(SP_RSA_PRIVATE_EXP_D) && defined(WOLFSSL_HAVE_SP_RSA) +#endif /* !SP_RSA_PRIVATE_EXP_D && WOLFSSL_HAVE_SP_RSA */ +#if !defined(SP_RSA_PRIVATE_EXP_D) && defined(WOLFSSL_HAVE_SP_RSA) +/* Sub b from a into a. (a -= b) + * + * a A single precision integer and result. + * b A single precision integer. + */ +.globl sp_3072_sub_in_place_24 +.type sp_3072_sub_in_place_24,@function +.align 16 +sp_3072_sub_in_place_24: + xorq %rax, %rax + movq (%rdi), %rdx + movq 8(%rdi), %rcx + movq (%rsi), %r8 + movq 8(%rsi), %r9 + subq %r8, %rdx + movq 16(%rsi), %r8 + movq %rdx, (%rdi) + movq 16(%rdi), %rdx + sbbq %r9, %rcx + movq 24(%rsi), %r9 + movq %rcx, 8(%rdi) + movq 24(%rdi), %rcx + sbbq %r8, %rdx + movq 32(%rsi), %r8 + movq %rdx, 16(%rdi) + movq 32(%rdi), %rdx + sbbq %r9, %rcx + movq 40(%rsi), %r9 + movq %rcx, 24(%rdi) + movq 40(%rdi), %rcx + sbbq %r8, %rdx + movq 48(%rsi), %r8 + movq %rdx, 32(%rdi) + movq 48(%rdi), %rdx + sbbq %r9, %rcx + movq 56(%rsi), %r9 + movq %rcx, 40(%rdi) + movq 56(%rdi), %rcx + sbbq %r8, %rdx + movq 64(%rsi), %r8 + movq %rdx, 48(%rdi) + movq 64(%rdi), %rdx + sbbq %r9, %rcx + movq 72(%rsi), %r9 + movq %rcx, 56(%rdi) + movq 72(%rdi), %rcx + sbbq %r8, %rdx + movq 80(%rsi), %r8 + movq %rdx, 64(%rdi) + movq 80(%rdi), %rdx + sbbq %r9, %rcx + movq 88(%rsi), %r9 + movq %rcx, 72(%rdi) + movq 88(%rdi), %rcx + sbbq %r8, %rdx + movq 96(%rsi), %r8 + movq %rdx, 80(%rdi) + movq 96(%rdi), %rdx + sbbq %r9, %rcx + movq 104(%rsi), %r9 + movq %rcx, 88(%rdi) + movq 104(%rdi), %rcx + sbbq %r8, %rdx + movq 112(%rsi), %r8 + movq %rdx, 96(%rdi) + movq 112(%rdi), %rdx + sbbq %r9, %rcx + movq 120(%rsi), %r9 + movq %rcx, 104(%rdi) + movq 120(%rdi), %rcx + sbbq %r8, %rdx + movq 128(%rsi), %r8 + movq %rdx, 112(%rdi) + movq 128(%rdi), %rdx + sbbq %r9, %rcx + movq 136(%rsi), %r9 + movq %rcx, 120(%rdi) + movq 136(%rdi), %rcx + sbbq %r8, %rdx + movq 144(%rsi), %r8 + movq %rdx, 128(%rdi) + movq 144(%rdi), %rdx + sbbq %r9, %rcx + movq 152(%rsi), %r9 + movq %rcx, 136(%rdi) + movq 152(%rdi), %rcx + sbbq %r8, %rdx + movq 160(%rsi), %r8 + movq %rdx, 144(%rdi) + movq 160(%rdi), %rdx + sbbq %r9, %rcx + movq 168(%rsi), %r9 + movq %rcx, 152(%rdi) + movq 168(%rdi), %rcx + sbbq %r8, %rdx + movq 176(%rsi), %r8 + movq %rdx, 160(%rdi) + movq 176(%rdi), %rdx + sbbq %r9, %rcx + movq 184(%rsi), %r9 + movq %rcx, 168(%rdi) + movq 184(%rdi), %rcx + sbbq %r8, %rdx + movq %rdx, 176(%rdi) + sbbq %r9, %rcx + movq %rcx, 184(%rdi) + sbbq $0, %rax + repz retq +.size sp_3072_sub_in_place_24,.-sp_3072_sub_in_place_24 +/* Conditionally subtract b from a using the mask m. + * m is -1 to subtract and 0 when not copying. + * + * r A single precision number representing condition subtract result. + * a A single precision number to subtract from. + * b A single precision number to subtract. + * m Mask value to apply. + */ +.globl sp_3072_cond_sub_24 +.type sp_3072_cond_sub_24,@function +.align 16 +sp_3072_cond_sub_24: + subq $192, %rsp + movq $0, %rax + movq (%rdx), %r8 + movq 8(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, (%rsp) + movq %r9, 8(%rsp) + movq 16(%rdx), %r8 + movq 24(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 16(%rsp) + movq %r9, 24(%rsp) + movq 32(%rdx), %r8 + movq 40(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 32(%rsp) + movq %r9, 40(%rsp) + movq 48(%rdx), %r8 + movq 56(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 48(%rsp) + movq %r9, 56(%rsp) + movq 64(%rdx), %r8 + movq 72(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 64(%rsp) + movq %r9, 72(%rsp) + movq 80(%rdx), %r8 + movq 88(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 80(%rsp) + movq %r9, 88(%rsp) + movq 96(%rdx), %r8 + movq 104(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 96(%rsp) + movq %r9, 104(%rsp) + movq 112(%rdx), %r8 + movq 120(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 112(%rsp) + movq %r9, 120(%rsp) + movq 128(%rdx), %r8 + movq 136(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 128(%rsp) + movq %r9, 136(%rsp) + movq 144(%rdx), %r8 + movq 152(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 144(%rsp) + movq %r9, 152(%rsp) + movq 160(%rdx), %r8 + movq 168(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 160(%rsp) + movq %r9, 168(%rsp) + movq 176(%rdx), %r8 + movq 184(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 176(%rsp) + movq %r9, 184(%rsp) + movq (%rsi), %r8 + movq (%rsp), %rdx + subq %rdx, %r8 + movq 8(%rsi), %r9 + movq 8(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, (%rdi) + movq 16(%rsi), %r8 + movq 16(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 8(%rdi) + movq 24(%rsi), %r9 + movq 24(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 16(%rdi) + movq 32(%rsi), %r8 + movq 32(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 24(%rdi) + movq 40(%rsi), %r9 + movq 40(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 32(%rdi) + movq 48(%rsi), %r8 + movq 48(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 40(%rdi) + movq 56(%rsi), %r9 + movq 56(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 48(%rdi) + movq 64(%rsi), %r8 + movq 64(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 56(%rdi) + movq 72(%rsi), %r9 + movq 72(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 64(%rdi) + movq 80(%rsi), %r8 + movq 80(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 72(%rdi) + movq 88(%rsi), %r9 + movq 88(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 80(%rdi) + movq 96(%rsi), %r8 + movq 96(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 88(%rdi) + movq 104(%rsi), %r9 + movq 104(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 96(%rdi) + movq 112(%rsi), %r8 + movq 112(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 104(%rdi) + movq 120(%rsi), %r9 + movq 120(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 112(%rdi) + movq 128(%rsi), %r8 + movq 128(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 120(%rdi) + movq 136(%rsi), %r9 + movq 136(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 128(%rdi) + movq 144(%rsi), %r8 + movq 144(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 136(%rdi) + movq 152(%rsi), %r9 + movq 152(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 144(%rdi) + movq 160(%rsi), %r8 + movq 160(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 152(%rdi) + movq 168(%rsi), %r9 + movq 168(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 160(%rdi) + movq 176(%rsi), %r8 + movq 176(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 168(%rdi) + movq 184(%rsi), %r9 + movq 184(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 176(%rdi) + movq %r9, 184(%rdi) + sbbq $0, %rax + addq $192, %rsp + repz retq +.size sp_3072_cond_sub_24,.-sp_3072_cond_sub_24 +/* Reduce the number back to 3072 bits using Montgomery reduction. + * + * a A single precision number to reduce in place. + * m The single precision number representing the modulus. + * mp The digit representing the negative inverse of m mod 2^n. + */ +.globl sp_3072_mont_reduce_24 +.type sp_3072_mont_reduce_24,@function +.align 16 +sp_3072_mont_reduce_24: + push %r12 + push %r13 + push %r14 + push %r15 + movq %rdx, %rcx + xorq %r15, %r15 + # i = 0 + movq $24, %r8 + movq (%rdi), %r13 + movq 8(%rdi), %r14 +L_mont_loop_24: + # mu = a[i] * mp + movq %r13, %r11 + imulq %rcx, %r11 + # a[i+0] += m[0] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq (%rsi) + addq %rax, %r13 + adcq %rdx, %r10 + # a[i+1] += m[1] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 8(%rsi) + movq %r14, %r13 + addq %rax, %r13 + adcq %rdx, %r9 + addq %r10, %r13 + adcq $0, %r9 + # a[i+2] += m[2] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 16(%rsi) + movq 16(%rdi), %r14 + addq %rax, %r14 + adcq %rdx, %r10 + addq %r9, %r14 + adcq $0, %r10 + # a[i+3] += m[3] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 24(%rsi) + movq 24(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r9 + addq %r10, %r12 + movq %r12, 24(%rdi) + adcq $0, %r9 + # a[i+4] += m[4] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 32(%rsi) + movq 32(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r10 + addq %r9, %r12 + movq %r12, 32(%rdi) + adcq $0, %r10 + # a[i+5] += m[5] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 40(%rsi) + movq 40(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r9 + addq %r10, %r12 + movq %r12, 40(%rdi) + adcq $0, %r9 + # a[i+6] += m[6] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 48(%rsi) + movq 48(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r10 + addq %r9, %r12 + movq %r12, 48(%rdi) + adcq $0, %r10 + # a[i+7] += m[7] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 56(%rsi) + movq 56(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r9 + addq %r10, %r12 + movq %r12, 56(%rdi) + adcq $0, %r9 + # a[i+8] += m[8] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 64(%rsi) + movq 64(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r10 + addq %r9, %r12 + movq %r12, 64(%rdi) + adcq $0, %r10 + # a[i+9] += m[9] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 72(%rsi) + movq 72(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r9 + addq %r10, %r12 + movq %r12, 72(%rdi) + adcq $0, %r9 + # a[i+10] += m[10] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 80(%rsi) + movq 80(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r10 + addq %r9, %r12 + movq %r12, 80(%rdi) + adcq $0, %r10 + # a[i+11] += m[11] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 88(%rsi) + movq 88(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r9 + addq %r10, %r12 + movq %r12, 88(%rdi) + adcq $0, %r9 + # a[i+12] += m[12] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 96(%rsi) + movq 96(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r10 + addq %r9, %r12 + movq %r12, 96(%rdi) + adcq $0, %r10 + # a[i+13] += m[13] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 104(%rsi) + movq 104(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r9 + addq %r10, %r12 + movq %r12, 104(%rdi) + adcq $0, %r9 + # a[i+14] += m[14] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 112(%rsi) + movq 112(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r10 + addq %r9, %r12 + movq %r12, 112(%rdi) + adcq $0, %r10 + # a[i+15] += m[15] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 120(%rsi) + movq 120(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r9 + addq %r10, %r12 + movq %r12, 120(%rdi) + adcq $0, %r9 + # a[i+16] += m[16] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 128(%rsi) + movq 128(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r10 + addq %r9, %r12 + movq %r12, 128(%rdi) + adcq $0, %r10 + # a[i+17] += m[17] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 136(%rsi) + movq 136(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r9 + addq %r10, %r12 + movq %r12, 136(%rdi) + adcq $0, %r9 + # a[i+18] += m[18] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 144(%rsi) + movq 144(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r10 + addq %r9, %r12 + movq %r12, 144(%rdi) + adcq $0, %r10 + # a[i+19] += m[19] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 152(%rsi) + movq 152(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r9 + addq %r10, %r12 + movq %r12, 152(%rdi) + adcq $0, %r9 + # a[i+20] += m[20] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 160(%rsi) + movq 160(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r10 + addq %r9, %r12 + movq %r12, 160(%rdi) + adcq $0, %r10 + # a[i+21] += m[21] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 168(%rsi) + movq 168(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r9 + addq %r10, %r12 + movq %r12, 168(%rdi) + adcq $0, %r9 + # a[i+22] += m[22] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 176(%rsi) + movq 176(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r10 + addq %r9, %r12 + movq %r12, 176(%rdi) + adcq $0, %r10 + # a[i+23] += m[23] * mu + movq %r11, %rax + mulq 184(%rsi) + movq 184(%rdi), %r12 + addq %rax, %r10 + adcq %r15, %rdx + movq $0, %r15 + adcq $0, %r15 + addq %r10, %r12 + movq %r12, 184(%rdi) + adcq %rdx, 192(%rdi) + adcq $0, %r15 + # i += 1 + addq $8, %rdi + decq %r8 + jnz L_mont_loop_24 + movq %r13, (%rdi) + movq %r14, 8(%rdi) + negq %r15 + movq %r15, %rcx + movq %rsi, %rdx + movq %rdi, %rsi + movq %rdi, %rdi + subq $192, %rdi + callq sp_3072_cond_sub_24@plt + pop %r15 + pop %r14 + pop %r13 + pop %r12 + repz retq +.size sp_3072_mont_reduce_24,.-sp_3072_mont_reduce_24 +/* Mul a by digit b into r. (r = a * b) + * + * r A single precision integer. + * a A single precision integer. + * b A single precision digit. + */ +.globl sp_3072_mul_d_24 +.type sp_3072_mul_d_24,@function +.align 16 +sp_3072_mul_d_24: + movq %rdx, %rcx + # A[0] * B + movq %rcx, %rax + xorq %r10, %r10 + mulq (%rsi) + movq %rax, %r8 + movq %rdx, %r9 + movq %r8, (%rdi) + # A[1] * B + movq %rcx, %rax + xorq %r8, %r8 + mulq 8(%rsi) + addq %rax, %r9 + movq %r9, 8(%rdi) + adcq %rdx, %r10 + adcq $0, %r8 + # A[2] * B + movq %rcx, %rax + xorq %r9, %r9 + mulq 16(%rsi) + addq %rax, %r10 + movq %r10, 16(%rdi) + adcq %rdx, %r8 + adcq $0, %r9 + # A[3] * B + movq %rcx, %rax + xorq %r10, %r10 + mulq 24(%rsi) + addq %rax, %r8 + movq %r8, 24(%rdi) + adcq %rdx, %r9 + adcq $0, %r10 + # A[4] * B + movq %rcx, %rax + xorq %r8, %r8 + mulq 32(%rsi) + addq %rax, %r9 + movq %r9, 32(%rdi) + adcq %rdx, %r10 + adcq $0, %r8 + # A[5] * B + movq %rcx, %rax + xorq %r9, %r9 + mulq 40(%rsi) + addq %rax, %r10 + movq %r10, 40(%rdi) + adcq %rdx, %r8 + adcq $0, %r9 + # A[6] * B + movq %rcx, %rax + xorq %r10, %r10 + mulq 48(%rsi) + addq %rax, %r8 + movq %r8, 48(%rdi) + adcq %rdx, %r9 + adcq $0, %r10 + # A[7] * B + movq %rcx, %rax + xorq %r8, %r8 + mulq 56(%rsi) + addq %rax, %r9 + movq %r9, 56(%rdi) + adcq %rdx, %r10 + adcq $0, %r8 + # A[8] * B + movq %rcx, %rax + xorq %r9, %r9 + mulq 64(%rsi) + addq %rax, %r10 + movq %r10, 64(%rdi) + adcq %rdx, %r8 + adcq $0, %r9 + # A[9] * B + movq %rcx, %rax + xorq %r10, %r10 + mulq 72(%rsi) + addq %rax, %r8 + movq %r8, 72(%rdi) + adcq %rdx, %r9 + adcq $0, %r10 + # A[10] * B + movq %rcx, %rax + xorq %r8, %r8 + mulq 80(%rsi) + addq %rax, %r9 + movq %r9, 80(%rdi) + adcq %rdx, %r10 + adcq $0, %r8 + # A[11] * B + movq %rcx, %rax + xorq %r9, %r9 + mulq 88(%rsi) + addq %rax, %r10 + movq %r10, 88(%rdi) + adcq %rdx, %r8 + adcq $0, %r9 + # A[12] * B + movq %rcx, %rax + xorq %r10, %r10 + mulq 96(%rsi) + addq %rax, %r8 + movq %r8, 96(%rdi) + adcq %rdx, %r9 + adcq $0, %r10 + # A[13] * B + movq %rcx, %rax + xorq %r8, %r8 + mulq 104(%rsi) + addq %rax, %r9 + movq %r9, 104(%rdi) + adcq %rdx, %r10 + adcq $0, %r8 + # A[14] * B + movq %rcx, %rax + xorq %r9, %r9 + mulq 112(%rsi) + addq %rax, %r10 + movq %r10, 112(%rdi) + adcq %rdx, %r8 + adcq $0, %r9 + # A[15] * B + movq %rcx, %rax + xorq %r10, %r10 + mulq 120(%rsi) + addq %rax, %r8 + movq %r8, 120(%rdi) + adcq %rdx, %r9 + adcq $0, %r10 + # A[16] * B + movq %rcx, %rax + xorq %r8, %r8 + mulq 128(%rsi) + addq %rax, %r9 + movq %r9, 128(%rdi) + adcq %rdx, %r10 + adcq $0, %r8 + # A[17] * B + movq %rcx, %rax + xorq %r9, %r9 + mulq 136(%rsi) + addq %rax, %r10 + movq %r10, 136(%rdi) + adcq %rdx, %r8 + adcq $0, %r9 + # A[18] * B + movq %rcx, %rax + xorq %r10, %r10 + mulq 144(%rsi) + addq %rax, %r8 + movq %r8, 144(%rdi) + adcq %rdx, %r9 + adcq $0, %r10 + # A[19] * B + movq %rcx, %rax + xorq %r8, %r8 + mulq 152(%rsi) + addq %rax, %r9 + movq %r9, 152(%rdi) + adcq %rdx, %r10 + adcq $0, %r8 + # A[20] * B + movq %rcx, %rax + xorq %r9, %r9 + mulq 160(%rsi) + addq %rax, %r10 + movq %r10, 160(%rdi) + adcq %rdx, %r8 + adcq $0, %r9 + # A[21] * B + movq %rcx, %rax + xorq %r10, %r10 + mulq 168(%rsi) + addq %rax, %r8 + movq %r8, 168(%rdi) + adcq %rdx, %r9 + adcq $0, %r10 + # A[22] * B + movq %rcx, %rax + xorq %r8, %r8 + mulq 176(%rsi) + addq %rax, %r9 + movq %r9, 176(%rdi) + adcq %rdx, %r10 + adcq $0, %r8 + # # A[23] * B + movq %rcx, %rax + mulq 184(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + movq %r10, 184(%rdi) + movq %r8, 192(%rdi) + repz retq +.size sp_3072_mul_d_24,.-sp_3072_mul_d_24 +#ifdef HAVE_INTEL_AVX2 +/* Mul a by digit b into r. (r = a * b) + * + * r A single precision integer. + * a A single precision integer. + * b A single precision digit. + */ +.globl sp_3072_mul_d_avx2_24 +.type sp_3072_mul_d_avx2_24,@function +.align 16 +sp_3072_mul_d_avx2_24: + movq %rdx, %rax + # A[0] * B + movq %rax, %rdx + xorq %r11, %r11 + mulxq (%rsi), %r9, %r10 + movq %r9, (%rdi) + # A[1] * B + mulxq 8(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 8(%rdi) + adoxq %r8, %r9 + # A[2] * B + mulxq 16(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 16(%rdi) + adoxq %r8, %r10 + # A[3] * B + mulxq 24(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 24(%rdi) + adoxq %r8, %r9 + # A[4] * B + mulxq 32(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 32(%rdi) + adoxq %r8, %r10 + # A[5] * B + mulxq 40(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 40(%rdi) + adoxq %r8, %r9 + # A[6] * B + mulxq 48(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 48(%rdi) + adoxq %r8, %r10 + # A[7] * B + mulxq 56(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 56(%rdi) + adoxq %r8, %r9 + # A[8] * B + mulxq 64(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 64(%rdi) + adoxq %r8, %r10 + # A[9] * B + mulxq 72(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 72(%rdi) + adoxq %r8, %r9 + # A[10] * B + mulxq 80(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 80(%rdi) + adoxq %r8, %r10 + # A[11] * B + mulxq 88(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 88(%rdi) + adoxq %r8, %r9 + # A[12] * B + mulxq 96(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 96(%rdi) + adoxq %r8, %r10 + # A[13] * B + mulxq 104(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 104(%rdi) + adoxq %r8, %r9 + # A[14] * B + mulxq 112(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 112(%rdi) + adoxq %r8, %r10 + # A[15] * B + mulxq 120(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 120(%rdi) + adoxq %r8, %r9 + # A[16] * B + mulxq 128(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 128(%rdi) + adoxq %r8, %r10 + # A[17] * B + mulxq 136(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 136(%rdi) + adoxq %r8, %r9 + # A[18] * B + mulxq 144(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 144(%rdi) + adoxq %r8, %r10 + # A[19] * B + mulxq 152(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 152(%rdi) + adoxq %r8, %r9 + # A[20] * B + mulxq 160(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 160(%rdi) + adoxq %r8, %r10 + # A[21] * B + mulxq 168(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 168(%rdi) + adoxq %r8, %r9 + # A[22] * B + mulxq 176(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 176(%rdi) + adoxq %r8, %r10 + # A[23] * B + mulxq 184(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + adoxq %r8, %r9 + adcxq %r11, %r9 + movq %r10, 184(%rdi) + movq %r9, 192(%rdi) + repz retq +.size sp_3072_mul_d_avx2_24,.-sp_3072_mul_d_avx2_24 +#endif /* HAVE_INTEL_AVX2 */ +/* Compare a with b in constant time. + * + * a A single precision integer. + * b A single precision integer. + * return -ve, 0 or +ve if a is less than, equal to or greater than b + * respectively. + */ +.globl sp_3072_cmp_24 +.type sp_3072_cmp_24,@function +.align 16 +sp_3072_cmp_24: + xorq %rcx, %rcx + movq $-1, %rdx + movq $-1, %rax + movq $1, %r8 + movq 184(%rdi), %r9 + movq 184(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 176(%rdi), %r9 + movq 176(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 168(%rdi), %r9 + movq 168(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 160(%rdi), %r9 + movq 160(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 152(%rdi), %r9 + movq 152(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 144(%rdi), %r9 + movq 144(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 136(%rdi), %r9 + movq 136(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 128(%rdi), %r9 + movq 128(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 120(%rdi), %r9 + movq 120(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 112(%rdi), %r9 + movq 112(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 104(%rdi), %r9 + movq 104(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 96(%rdi), %r9 + movq 96(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 88(%rdi), %r9 + movq 88(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 80(%rdi), %r9 + movq 80(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 72(%rdi), %r9 + movq 72(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 64(%rdi), %r9 + movq 64(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 56(%rdi), %r9 + movq 56(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 48(%rdi), %r9 + movq 48(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 40(%rdi), %r9 + movq 40(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 32(%rdi), %r9 + movq 32(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 24(%rdi), %r9 + movq 24(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 16(%rdi), %r9 + movq 16(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 8(%rdi), %r9 + movq 8(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq (%rdi), %r9 + movq (%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + xorq %rdx, %rax + repz retq +.size sp_3072_cmp_24,.-sp_3072_cmp_24 +#ifdef HAVE_INTEL_AVX2 +/* Reduce the number back to 3072 bits using Montgomery reduction. + * + * a A single precision number to reduce in place. + * m The single precision number representing the modulus. + * mp The digit representing the negative inverse of m mod 2^n. + */ +.globl sp_3072_mont_reduce_avx2_24 +.type sp_3072_mont_reduce_avx2_24,@function +.align 16 +sp_3072_mont_reduce_avx2_24: + push %r12 + push %r13 + push %r14 + movq %rdx, %rax + xorq %r14, %r14 + # i = 0 + movq $24, %r9 + movq (%rdi), %r13 + xorq %r12, %r12 +L_mont_loop_avx2_24: + # mu = a[i] * mp + movq %r13, %rdx + mulxq %rax, %rdx, %r8 + movq %r13, %r10 + # a[i+0] += m[0] * mu + mulxq (%rsi), %rcx, %r8 + movq 8(%rdi), %r13 + adcxq %rcx, %r10 + adoxq %r8, %r13 + # a[i+1] += m[1] * mu + mulxq 8(%rsi), %rcx, %r8 + movq 16(%rdi), %r10 + adcxq %rcx, %r13 + adoxq %r8, %r10 + # a[i+2] += m[2] * mu + mulxq 16(%rsi), %rcx, %r8 + movq 24(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 16(%rdi) + # a[i+3] += m[3] * mu + mulxq 24(%rsi), %rcx, %r8 + movq 32(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 24(%rdi) + # a[i+4] += m[4] * mu + mulxq 32(%rsi), %rcx, %r8 + movq 40(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 32(%rdi) + # a[i+5] += m[5] * mu + mulxq 40(%rsi), %rcx, %r8 + movq 48(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 40(%rdi) + # a[i+6] += m[6] * mu + mulxq 48(%rsi), %rcx, %r8 + movq 56(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 48(%rdi) + # a[i+7] += m[7] * mu + mulxq 56(%rsi), %rcx, %r8 + movq 64(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 56(%rdi) + # a[i+8] += m[8] * mu + mulxq 64(%rsi), %rcx, %r8 + movq 72(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 64(%rdi) + # a[i+9] += m[9] * mu + mulxq 72(%rsi), %rcx, %r8 + movq 80(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 72(%rdi) + # a[i+10] += m[10] * mu + mulxq 80(%rsi), %rcx, %r8 + movq 88(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 80(%rdi) + # a[i+11] += m[11] * mu + mulxq 88(%rsi), %rcx, %r8 + movq 96(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 88(%rdi) + # a[i+12] += m[12] * mu + mulxq 96(%rsi), %rcx, %r8 + movq 104(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 96(%rdi) + # a[i+13] += m[13] * mu + mulxq 104(%rsi), %rcx, %r8 + movq 112(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 104(%rdi) + # a[i+14] += m[14] * mu + mulxq 112(%rsi), %rcx, %r8 + movq 120(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 112(%rdi) + # a[i+15] += m[15] * mu + mulxq 120(%rsi), %rcx, %r8 + movq 128(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 120(%rdi) + # a[i+16] += m[16] * mu + mulxq 128(%rsi), %rcx, %r8 + movq 136(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 128(%rdi) + # a[i+17] += m[17] * mu + mulxq 136(%rsi), %rcx, %r8 + movq 144(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 136(%rdi) + # a[i+18] += m[18] * mu + mulxq 144(%rsi), %rcx, %r8 + movq 152(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 144(%rdi) + # a[i+19] += m[19] * mu + mulxq 152(%rsi), %rcx, %r8 + movq 160(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 152(%rdi) + # a[i+20] += m[20] * mu + mulxq 160(%rsi), %rcx, %r8 + movq 168(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 160(%rdi) + # a[i+21] += m[21] * mu + mulxq 168(%rsi), %rcx, %r8 + movq 176(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 168(%rdi) + # a[i+22] += m[22] * mu + mulxq 176(%rsi), %rcx, %r8 + movq 184(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 176(%rdi) + # a[i+23] += m[23] * mu + mulxq 184(%rsi), %rcx, %r8 + movq 192(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 184(%rdi) + adcxq %r14, %r10 + movq %r12, %r14 + adoxq %r12, %r14 + adcxq %r12, %r14 + movq %r10, 192(%rdi) + # i += 1 + addq $8, %rdi + decq %r9 + jnz L_mont_loop_avx2_24 + movq %r13, (%rdi) + negq %r14 + movq %r14, %rcx + movq %rsi, %rdx + movq %rdi, %rsi + movq %rdi, %rdi + subq $192, %rdi + callq sp_3072_cond_sub_24@plt + pop %r14 + pop %r13 + pop %r12 + repz retq +.size sp_3072_mont_reduce_avx2_24,.-sp_3072_mont_reduce_avx2_24 +#endif /* HAVE_INTEL_AVX2 */ +#ifdef HAVE_INTEL_AVX2 +#endif /* HAVE_INTEL_AVX2 */ +#ifdef HAVE_INTEL_AVX2 +#endif /* HAVE_INTEL_AVX2 */ +#ifdef HAVE_INTEL_AVX2 +#endif /* HAVE_INTEL_AVX2 */ +#endif /* !SP_RSA_PRIVATE_EXP_D && WOLFSSL_HAVE_SP_RSA */ +#ifdef WOLFSSL_HAVE_SP_DH +#endif /* WOLFSSL_HAVE_SP_DH */ +/* Conditionally subtract b from a using the mask m. + * m is -1 to subtract and 0 when not copying. + * + * r A single precision number representing condition subtract result. + * a A single precision number to subtract from. + * b A single precision number to subtract. + * m Mask value to apply. + */ +.globl sp_3072_cond_sub_48 +.type sp_3072_cond_sub_48,@function +.align 16 +sp_3072_cond_sub_48: + subq $384, %rsp + movq $0, %rax + movq (%rdx), %r8 + movq 8(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, (%rsp) + movq %r9, 8(%rsp) + movq 16(%rdx), %r8 + movq 24(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 16(%rsp) + movq %r9, 24(%rsp) + movq 32(%rdx), %r8 + movq 40(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 32(%rsp) + movq %r9, 40(%rsp) + movq 48(%rdx), %r8 + movq 56(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 48(%rsp) + movq %r9, 56(%rsp) + movq 64(%rdx), %r8 + movq 72(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 64(%rsp) + movq %r9, 72(%rsp) + movq 80(%rdx), %r8 + movq 88(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 80(%rsp) + movq %r9, 88(%rsp) + movq 96(%rdx), %r8 + movq 104(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 96(%rsp) + movq %r9, 104(%rsp) + movq 112(%rdx), %r8 + movq 120(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 112(%rsp) + movq %r9, 120(%rsp) + movq 128(%rdx), %r8 + movq 136(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 128(%rsp) + movq %r9, 136(%rsp) + movq 144(%rdx), %r8 + movq 152(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 144(%rsp) + movq %r9, 152(%rsp) + movq 160(%rdx), %r8 + movq 168(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 160(%rsp) + movq %r9, 168(%rsp) + movq 176(%rdx), %r8 + movq 184(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 176(%rsp) + movq %r9, 184(%rsp) + movq 192(%rdx), %r8 + movq 200(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 192(%rsp) + movq %r9, 200(%rsp) + movq 208(%rdx), %r8 + movq 216(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 208(%rsp) + movq %r9, 216(%rsp) + movq 224(%rdx), %r8 + movq 232(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 224(%rsp) + movq %r9, 232(%rsp) + movq 240(%rdx), %r8 + movq 248(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 240(%rsp) + movq %r9, 248(%rsp) + movq 256(%rdx), %r8 + movq 264(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 256(%rsp) + movq %r9, 264(%rsp) + movq 272(%rdx), %r8 + movq 280(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 272(%rsp) + movq %r9, 280(%rsp) + movq 288(%rdx), %r8 + movq 296(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 288(%rsp) + movq %r9, 296(%rsp) + movq 304(%rdx), %r8 + movq 312(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 304(%rsp) + movq %r9, 312(%rsp) + movq 320(%rdx), %r8 + movq 328(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 320(%rsp) + movq %r9, 328(%rsp) + movq 336(%rdx), %r8 + movq 344(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 336(%rsp) + movq %r9, 344(%rsp) + movq 352(%rdx), %r8 + movq 360(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 352(%rsp) + movq %r9, 360(%rsp) + movq 368(%rdx), %r8 + movq 376(%rdx), %r9 + andq %rcx, %r8 + andq %rcx, %r9 + movq %r8, 368(%rsp) + movq %r9, 376(%rsp) + movq (%rsi), %r8 + movq (%rsp), %rdx + subq %rdx, %r8 + movq 8(%rsi), %r9 + movq 8(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, (%rdi) + movq 16(%rsi), %r8 + movq 16(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 8(%rdi) + movq 24(%rsi), %r9 + movq 24(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 16(%rdi) + movq 32(%rsi), %r8 + movq 32(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 24(%rdi) + movq 40(%rsi), %r9 + movq 40(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 32(%rdi) + movq 48(%rsi), %r8 + movq 48(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 40(%rdi) + movq 56(%rsi), %r9 + movq 56(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 48(%rdi) + movq 64(%rsi), %r8 + movq 64(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 56(%rdi) + movq 72(%rsi), %r9 + movq 72(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 64(%rdi) + movq 80(%rsi), %r8 + movq 80(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 72(%rdi) + movq 88(%rsi), %r9 + movq 88(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 80(%rdi) + movq 96(%rsi), %r8 + movq 96(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 88(%rdi) + movq 104(%rsi), %r9 + movq 104(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 96(%rdi) + movq 112(%rsi), %r8 + movq 112(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 104(%rdi) + movq 120(%rsi), %r9 + movq 120(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 112(%rdi) + movq 128(%rsi), %r8 + movq 128(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 120(%rdi) + movq 136(%rsi), %r9 + movq 136(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 128(%rdi) + movq 144(%rsi), %r8 + movq 144(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 136(%rdi) + movq 152(%rsi), %r9 + movq 152(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 144(%rdi) + movq 160(%rsi), %r8 + movq 160(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 152(%rdi) + movq 168(%rsi), %r9 + movq 168(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 160(%rdi) + movq 176(%rsi), %r8 + movq 176(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 168(%rdi) + movq 184(%rsi), %r9 + movq 184(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 176(%rdi) + movq 192(%rsi), %r8 + movq 192(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 184(%rdi) + movq 200(%rsi), %r9 + movq 200(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 192(%rdi) + movq 208(%rsi), %r8 + movq 208(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 200(%rdi) + movq 216(%rsi), %r9 + movq 216(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 208(%rdi) + movq 224(%rsi), %r8 + movq 224(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 216(%rdi) + movq 232(%rsi), %r9 + movq 232(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 224(%rdi) + movq 240(%rsi), %r8 + movq 240(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 232(%rdi) + movq 248(%rsi), %r9 + movq 248(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 240(%rdi) + movq 256(%rsi), %r8 + movq 256(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 248(%rdi) + movq 264(%rsi), %r9 + movq 264(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 256(%rdi) + movq 272(%rsi), %r8 + movq 272(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 264(%rdi) + movq 280(%rsi), %r9 + movq 280(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 272(%rdi) + movq 288(%rsi), %r8 + movq 288(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 280(%rdi) + movq 296(%rsi), %r9 + movq 296(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 288(%rdi) + movq 304(%rsi), %r8 + movq 304(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 296(%rdi) + movq 312(%rsi), %r9 + movq 312(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 304(%rdi) + movq 320(%rsi), %r8 + movq 320(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 312(%rdi) + movq 328(%rsi), %r9 + movq 328(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 320(%rdi) + movq 336(%rsi), %r8 + movq 336(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 328(%rdi) + movq 344(%rsi), %r9 + movq 344(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 336(%rdi) + movq 352(%rsi), %r8 + movq 352(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 344(%rdi) + movq 360(%rsi), %r9 + movq 360(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 352(%rdi) + movq 368(%rsi), %r8 + movq 368(%rsp), %rdx + sbbq %rdx, %r8 + movq %r9, 360(%rdi) + movq 376(%rsi), %r9 + movq 376(%rsp), %rdx + sbbq %rdx, %r9 + movq %r8, 368(%rdi) + movq %r9, 376(%rdi) + sbbq $0, %rax + addq $384, %rsp + repz retq +.size sp_3072_cond_sub_48,.-sp_3072_cond_sub_48 +/* Reduce the number back to 3072 bits using Montgomery reduction. + * + * a A single precision number to reduce in place. + * m The single precision number representing the modulus. + * mp The digit representing the negative inverse of m mod 2^n. + */ +.globl sp_3072_mont_reduce_48 +.type sp_3072_mont_reduce_48,@function +.align 16 +sp_3072_mont_reduce_48: + push %r12 + push %r13 + push %r14 + push %r15 + movq %rdx, %rcx + xorq %r15, %r15 + # i = 0 + movq $48, %r8 + movq (%rdi), %r13 + movq 8(%rdi), %r14 +L_mont_loop_48: + # mu = a[i] * mp + movq %r13, %r11 + imulq %rcx, %r11 + # a[i+0] += m[0] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq (%rsi) + addq %rax, %r13 + adcq %rdx, %r10 + # a[i+1] += m[1] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 8(%rsi) + movq %r14, %r13 + addq %rax, %r13 + adcq %rdx, %r9 + addq %r10, %r13 + adcq $0, %r9 + # a[i+2] += m[2] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 16(%rsi) + movq 16(%rdi), %r14 + addq %rax, %r14 + adcq %rdx, %r10 + addq %r9, %r14 + adcq $0, %r10 + # a[i+3] += m[3] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 24(%rsi) + movq 24(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r9 + addq %r10, %r12 + movq %r12, 24(%rdi) + adcq $0, %r9 + # a[i+4] += m[4] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 32(%rsi) + movq 32(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r10 + addq %r9, %r12 + movq %r12, 32(%rdi) + adcq $0, %r10 + # a[i+5] += m[5] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 40(%rsi) + movq 40(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r9 + addq %r10, %r12 + movq %r12, 40(%rdi) + adcq $0, %r9 + # a[i+6] += m[6] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 48(%rsi) + movq 48(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r10 + addq %r9, %r12 + movq %r12, 48(%rdi) + adcq $0, %r10 + # a[i+7] += m[7] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 56(%rsi) + movq 56(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r9 + addq %r10, %r12 + movq %r12, 56(%rdi) + adcq $0, %r9 + # a[i+8] += m[8] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 64(%rsi) + movq 64(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r10 + addq %r9, %r12 + movq %r12, 64(%rdi) + adcq $0, %r10 + # a[i+9] += m[9] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 72(%rsi) + movq 72(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r9 + addq %r10, %r12 + movq %r12, 72(%rdi) + adcq $0, %r9 + # a[i+10] += m[10] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 80(%rsi) + movq 80(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r10 + addq %r9, %r12 + movq %r12, 80(%rdi) + adcq $0, %r10 + # a[i+11] += m[11] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 88(%rsi) + movq 88(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r9 + addq %r10, %r12 + movq %r12, 88(%rdi) + adcq $0, %r9 + # a[i+12] += m[12] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 96(%rsi) + movq 96(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r10 + addq %r9, %r12 + movq %r12, 96(%rdi) + adcq $0, %r10 + # a[i+13] += m[13] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 104(%rsi) + movq 104(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r9 + addq %r10, %r12 + movq %r12, 104(%rdi) + adcq $0, %r9 + # a[i+14] += m[14] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 112(%rsi) + movq 112(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r10 + addq %r9, %r12 + movq %r12, 112(%rdi) + adcq $0, %r10 + # a[i+15] += m[15] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 120(%rsi) + movq 120(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r9 + addq %r10, %r12 + movq %r12, 120(%rdi) + adcq $0, %r9 + # a[i+16] += m[16] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 128(%rsi) + movq 128(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r10 + addq %r9, %r12 + movq %r12, 128(%rdi) + adcq $0, %r10 + # a[i+17] += m[17] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 136(%rsi) + movq 136(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r9 + addq %r10, %r12 + movq %r12, 136(%rdi) + adcq $0, %r9 + # a[i+18] += m[18] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 144(%rsi) + movq 144(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r10 + addq %r9, %r12 + movq %r12, 144(%rdi) + adcq $0, %r10 + # a[i+19] += m[19] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 152(%rsi) + movq 152(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r9 + addq %r10, %r12 + movq %r12, 152(%rdi) + adcq $0, %r9 + # a[i+20] += m[20] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 160(%rsi) + movq 160(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r10 + addq %r9, %r12 + movq %r12, 160(%rdi) + adcq $0, %r10 + # a[i+21] += m[21] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 168(%rsi) + movq 168(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r9 + addq %r10, %r12 + movq %r12, 168(%rdi) + adcq $0, %r9 + # a[i+22] += m[22] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 176(%rsi) + movq 176(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r10 + addq %r9, %r12 + movq %r12, 176(%rdi) + adcq $0, %r10 + # a[i+23] += m[23] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 184(%rsi) + movq 184(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r9 + addq %r10, %r12 + movq %r12, 184(%rdi) + adcq $0, %r9 + # a[i+24] += m[24] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 192(%rsi) + movq 192(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r10 + addq %r9, %r12 + movq %r12, 192(%rdi) + adcq $0, %r10 + # a[i+25] += m[25] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 200(%rsi) + movq 200(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r9 + addq %r10, %r12 + movq %r12, 200(%rdi) + adcq $0, %r9 + # a[i+26] += m[26] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 208(%rsi) + movq 208(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r10 + addq %r9, %r12 + movq %r12, 208(%rdi) + adcq $0, %r10 + # a[i+27] += m[27] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 216(%rsi) + movq 216(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r9 + addq %r10, %r12 + movq %r12, 216(%rdi) + adcq $0, %r9 + # a[i+28] += m[28] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 224(%rsi) + movq 224(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r10 + addq %r9, %r12 + movq %r12, 224(%rdi) + adcq $0, %r10 + # a[i+29] += m[29] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 232(%rsi) + movq 232(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r9 + addq %r10, %r12 + movq %r12, 232(%rdi) + adcq $0, %r9 + # a[i+30] += m[30] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 240(%rsi) + movq 240(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r10 + addq %r9, %r12 + movq %r12, 240(%rdi) + adcq $0, %r10 + # a[i+31] += m[31] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 248(%rsi) + movq 248(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r9 + addq %r10, %r12 + movq %r12, 248(%rdi) + adcq $0, %r9 + # a[i+32] += m[32] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 256(%rsi) + movq 256(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r10 + addq %r9, %r12 + movq %r12, 256(%rdi) + adcq $0, %r10 + # a[i+33] += m[33] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 264(%rsi) + movq 264(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r9 + addq %r10, %r12 + movq %r12, 264(%rdi) + adcq $0, %r9 + # a[i+34] += m[34] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 272(%rsi) + movq 272(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r10 + addq %r9, %r12 + movq %r12, 272(%rdi) + adcq $0, %r10 + # a[i+35] += m[35] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 280(%rsi) + movq 280(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r9 + addq %r10, %r12 + movq %r12, 280(%rdi) + adcq $0, %r9 + # a[i+36] += m[36] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 288(%rsi) + movq 288(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r10 + addq %r9, %r12 + movq %r12, 288(%rdi) + adcq $0, %r10 + # a[i+37] += m[37] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 296(%rsi) + movq 296(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r9 + addq %r10, %r12 + movq %r12, 296(%rdi) + adcq $0, %r9 + # a[i+38] += m[38] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 304(%rsi) + movq 304(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r10 + addq %r9, %r12 + movq %r12, 304(%rdi) + adcq $0, %r10 + # a[i+39] += m[39] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 312(%rsi) + movq 312(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r9 + addq %r10, %r12 + movq %r12, 312(%rdi) + adcq $0, %r9 + # a[i+40] += m[40] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 320(%rsi) + movq 320(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r10 + addq %r9, %r12 + movq %r12, 320(%rdi) + adcq $0, %r10 + # a[i+41] += m[41] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 328(%rsi) + movq 328(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r9 + addq %r10, %r12 + movq %r12, 328(%rdi) + adcq $0, %r9 + # a[i+42] += m[42] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 336(%rsi) + movq 336(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r10 + addq %r9, %r12 + movq %r12, 336(%rdi) + adcq $0, %r10 + # a[i+43] += m[43] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 344(%rsi) + movq 344(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r9 + addq %r10, %r12 + movq %r12, 344(%rdi) + adcq $0, %r9 + # a[i+44] += m[44] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 352(%rsi) + movq 352(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r10 + addq %r9, %r12 + movq %r12, 352(%rdi) + adcq $0, %r10 + # a[i+45] += m[45] * mu + movq %r11, %rax + xorq %r9, %r9 + mulq 360(%rsi) + movq 360(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r9 + addq %r10, %r12 + movq %r12, 360(%rdi) + adcq $0, %r9 + # a[i+46] += m[46] * mu + movq %r11, %rax + xorq %r10, %r10 + mulq 368(%rsi) + movq 368(%rdi), %r12 + addq %rax, %r12 + adcq %rdx, %r10 + addq %r9, %r12 + movq %r12, 368(%rdi) + adcq $0, %r10 + # a[i+47] += m[47] * mu + movq %r11, %rax + mulq 376(%rsi) + movq 376(%rdi), %r12 + addq %rax, %r10 + adcq %r15, %rdx + movq $0, %r15 + adcq $0, %r15 + addq %r10, %r12 + movq %r12, 376(%rdi) + adcq %rdx, 384(%rdi) + adcq $0, %r15 + # i += 1 + addq $8, %rdi + decq %r8 + jnz L_mont_loop_48 + movq %r13, (%rdi) + movq %r14, 8(%rdi) + negq %r15 + movq %r15, %rcx + movq %rsi, %rdx + movq %rdi, %rsi + movq %rdi, %rdi + subq $384, %rdi + callq sp_3072_cond_sub_48@plt + pop %r15 + pop %r14 + pop %r13 + pop %r12 + repz retq +.size sp_3072_mont_reduce_48,.-sp_3072_mont_reduce_48 +/* Mul a by digit b into r. (r = a * b) + * + * r A single precision integer. + * a A single precision integer. + * b A single precision digit. + */ +.globl sp_3072_mul_d_48 +.type sp_3072_mul_d_48,@function +.align 16 +sp_3072_mul_d_48: + movq %rdx, %rcx + # A[0] * B + movq %rcx, %rax + xorq %r10, %r10 + mulq (%rsi) + movq %rax, %r8 + movq %rdx, %r9 + movq %r8, (%rdi) + # A[1] * B + movq %rcx, %rax + xorq %r8, %r8 + mulq 8(%rsi) + addq %rax, %r9 + movq %r9, 8(%rdi) + adcq %rdx, %r10 + adcq $0, %r8 + # A[2] * B + movq %rcx, %rax + xorq %r9, %r9 + mulq 16(%rsi) + addq %rax, %r10 + movq %r10, 16(%rdi) + adcq %rdx, %r8 + adcq $0, %r9 + # A[3] * B + movq %rcx, %rax + xorq %r10, %r10 + mulq 24(%rsi) + addq %rax, %r8 + movq %r8, 24(%rdi) + adcq %rdx, %r9 + adcq $0, %r10 + # A[4] * B + movq %rcx, %rax + xorq %r8, %r8 + mulq 32(%rsi) + addq %rax, %r9 + movq %r9, 32(%rdi) + adcq %rdx, %r10 + adcq $0, %r8 + # A[5] * B + movq %rcx, %rax + xorq %r9, %r9 + mulq 40(%rsi) + addq %rax, %r10 + movq %r10, 40(%rdi) + adcq %rdx, %r8 + adcq $0, %r9 + # A[6] * B + movq %rcx, %rax + xorq %r10, %r10 + mulq 48(%rsi) + addq %rax, %r8 + movq %r8, 48(%rdi) + adcq %rdx, %r9 + adcq $0, %r10 + # A[7] * B + movq %rcx, %rax + xorq %r8, %r8 + mulq 56(%rsi) + addq %rax, %r9 + movq %r9, 56(%rdi) + adcq %rdx, %r10 + adcq $0, %r8 + # A[8] * B + movq %rcx, %rax + xorq %r9, %r9 + mulq 64(%rsi) + addq %rax, %r10 + movq %r10, 64(%rdi) + adcq %rdx, %r8 + adcq $0, %r9 + # A[9] * B + movq %rcx, %rax + xorq %r10, %r10 + mulq 72(%rsi) + addq %rax, %r8 + movq %r8, 72(%rdi) + adcq %rdx, %r9 + adcq $0, %r10 + # A[10] * B + movq %rcx, %rax + xorq %r8, %r8 + mulq 80(%rsi) + addq %rax, %r9 + movq %r9, 80(%rdi) + adcq %rdx, %r10 + adcq $0, %r8 + # A[11] * B + movq %rcx, %rax + xorq %r9, %r9 + mulq 88(%rsi) + addq %rax, %r10 + movq %r10, 88(%rdi) + adcq %rdx, %r8 + adcq $0, %r9 + # A[12] * B + movq %rcx, %rax + xorq %r10, %r10 + mulq 96(%rsi) + addq %rax, %r8 + movq %r8, 96(%rdi) + adcq %rdx, %r9 + adcq $0, %r10 + # A[13] * B + movq %rcx, %rax + xorq %r8, %r8 + mulq 104(%rsi) + addq %rax, %r9 + movq %r9, 104(%rdi) + adcq %rdx, %r10 + adcq $0, %r8 + # A[14] * B + movq %rcx, %rax + xorq %r9, %r9 + mulq 112(%rsi) + addq %rax, %r10 + movq %r10, 112(%rdi) + adcq %rdx, %r8 + adcq $0, %r9 + # A[15] * B + movq %rcx, %rax + xorq %r10, %r10 + mulq 120(%rsi) + addq %rax, %r8 + movq %r8, 120(%rdi) + adcq %rdx, %r9 + adcq $0, %r10 + # A[16] * B + movq %rcx, %rax + xorq %r8, %r8 + mulq 128(%rsi) + addq %rax, %r9 + movq %r9, 128(%rdi) + adcq %rdx, %r10 + adcq $0, %r8 + # A[17] * B + movq %rcx, %rax + xorq %r9, %r9 + mulq 136(%rsi) + addq %rax, %r10 + movq %r10, 136(%rdi) + adcq %rdx, %r8 + adcq $0, %r9 + # A[18] * B + movq %rcx, %rax + xorq %r10, %r10 + mulq 144(%rsi) + addq %rax, %r8 + movq %r8, 144(%rdi) + adcq %rdx, %r9 + adcq $0, %r10 + # A[19] * B + movq %rcx, %rax + xorq %r8, %r8 + mulq 152(%rsi) + addq %rax, %r9 + movq %r9, 152(%rdi) + adcq %rdx, %r10 + adcq $0, %r8 + # A[20] * B + movq %rcx, %rax + xorq %r9, %r9 + mulq 160(%rsi) + addq %rax, %r10 + movq %r10, 160(%rdi) + adcq %rdx, %r8 + adcq $0, %r9 + # A[21] * B + movq %rcx, %rax + xorq %r10, %r10 + mulq 168(%rsi) + addq %rax, %r8 + movq %r8, 168(%rdi) + adcq %rdx, %r9 + adcq $0, %r10 + # A[22] * B + movq %rcx, %rax + xorq %r8, %r8 + mulq 176(%rsi) + addq %rax, %r9 + movq %r9, 176(%rdi) + adcq %rdx, %r10 + adcq $0, %r8 + # A[23] * B + movq %rcx, %rax + xorq %r9, %r9 + mulq 184(%rsi) + addq %rax, %r10 + movq %r10, 184(%rdi) + adcq %rdx, %r8 + adcq $0, %r9 + # A[24] * B + movq %rcx, %rax + xorq %r10, %r10 + mulq 192(%rsi) + addq %rax, %r8 + movq %r8, 192(%rdi) + adcq %rdx, %r9 + adcq $0, %r10 + # A[25] * B + movq %rcx, %rax + xorq %r8, %r8 + mulq 200(%rsi) + addq %rax, %r9 + movq %r9, 200(%rdi) + adcq %rdx, %r10 + adcq $0, %r8 + # A[26] * B + movq %rcx, %rax + xorq %r9, %r9 + mulq 208(%rsi) + addq %rax, %r10 + movq %r10, 208(%rdi) + adcq %rdx, %r8 + adcq $0, %r9 + # A[27] * B + movq %rcx, %rax + xorq %r10, %r10 + mulq 216(%rsi) + addq %rax, %r8 + movq %r8, 216(%rdi) + adcq %rdx, %r9 + adcq $0, %r10 + # A[28] * B + movq %rcx, %rax + xorq %r8, %r8 + mulq 224(%rsi) + addq %rax, %r9 + movq %r9, 224(%rdi) + adcq %rdx, %r10 + adcq $0, %r8 + # A[29] * B + movq %rcx, %rax + xorq %r9, %r9 + mulq 232(%rsi) + addq %rax, %r10 + movq %r10, 232(%rdi) + adcq %rdx, %r8 + adcq $0, %r9 + # A[30] * B + movq %rcx, %rax + xorq %r10, %r10 + mulq 240(%rsi) + addq %rax, %r8 + movq %r8, 240(%rdi) + adcq %rdx, %r9 + adcq $0, %r10 + # A[31] * B + movq %rcx, %rax + xorq %r8, %r8 + mulq 248(%rsi) + addq %rax, %r9 + movq %r9, 248(%rdi) + adcq %rdx, %r10 + adcq $0, %r8 + # A[32] * B + movq %rcx, %rax + xorq %r9, %r9 + mulq 256(%rsi) + addq %rax, %r10 + movq %r10, 256(%rdi) + adcq %rdx, %r8 + adcq $0, %r9 + # A[33] * B + movq %rcx, %rax + xorq %r10, %r10 + mulq 264(%rsi) + addq %rax, %r8 + movq %r8, 264(%rdi) + adcq %rdx, %r9 + adcq $0, %r10 + # A[34] * B + movq %rcx, %rax + xorq %r8, %r8 + mulq 272(%rsi) + addq %rax, %r9 + movq %r9, 272(%rdi) + adcq %rdx, %r10 + adcq $0, %r8 + # A[35] * B + movq %rcx, %rax + xorq %r9, %r9 + mulq 280(%rsi) + addq %rax, %r10 + movq %r10, 280(%rdi) + adcq %rdx, %r8 + adcq $0, %r9 + # A[36] * B + movq %rcx, %rax + xorq %r10, %r10 + mulq 288(%rsi) + addq %rax, %r8 + movq %r8, 288(%rdi) + adcq %rdx, %r9 + adcq $0, %r10 + # A[37] * B + movq %rcx, %rax + xorq %r8, %r8 + mulq 296(%rsi) + addq %rax, %r9 + movq %r9, 296(%rdi) + adcq %rdx, %r10 + adcq $0, %r8 + # A[38] * B + movq %rcx, %rax + xorq %r9, %r9 + mulq 304(%rsi) + addq %rax, %r10 + movq %r10, 304(%rdi) + adcq %rdx, %r8 + adcq $0, %r9 + # A[39] * B + movq %rcx, %rax + xorq %r10, %r10 + mulq 312(%rsi) + addq %rax, %r8 + movq %r8, 312(%rdi) + adcq %rdx, %r9 + adcq $0, %r10 + # A[40] * B + movq %rcx, %rax + xorq %r8, %r8 + mulq 320(%rsi) + addq %rax, %r9 + movq %r9, 320(%rdi) + adcq %rdx, %r10 + adcq $0, %r8 + # A[41] * B + movq %rcx, %rax + xorq %r9, %r9 + mulq 328(%rsi) + addq %rax, %r10 + movq %r10, 328(%rdi) + adcq %rdx, %r8 + adcq $0, %r9 + # A[42] * B + movq %rcx, %rax + xorq %r10, %r10 + mulq 336(%rsi) + addq %rax, %r8 + movq %r8, 336(%rdi) + adcq %rdx, %r9 + adcq $0, %r10 + # A[43] * B + movq %rcx, %rax + xorq %r8, %r8 + mulq 344(%rsi) + addq %rax, %r9 + movq %r9, 344(%rdi) + adcq %rdx, %r10 + adcq $0, %r8 + # A[44] * B + movq %rcx, %rax + xorq %r9, %r9 + mulq 352(%rsi) + addq %rax, %r10 + movq %r10, 352(%rdi) + adcq %rdx, %r8 + adcq $0, %r9 + # A[45] * B + movq %rcx, %rax + xorq %r10, %r10 + mulq 360(%rsi) + addq %rax, %r8 + movq %r8, 360(%rdi) + adcq %rdx, %r9 + adcq $0, %r10 + # A[46] * B + movq %rcx, %rax + xorq %r8, %r8 + mulq 368(%rsi) + addq %rax, %r9 + movq %r9, 368(%rdi) + adcq %rdx, %r10 + adcq $0, %r8 + # # A[47] * B + movq %rcx, %rax + mulq 376(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + movq %r10, 376(%rdi) + movq %r8, 384(%rdi) + repz retq +.size sp_3072_mul_d_48,.-sp_3072_mul_d_48 +#ifdef HAVE_INTEL_AVX2 +/* Mul a by digit b into r. (r = a * b) + * + * r A single precision integer. + * a A single precision integer. + * b A single precision digit. + */ +.globl sp_3072_mul_d_avx2_48 +.type sp_3072_mul_d_avx2_48,@function +.align 16 +sp_3072_mul_d_avx2_48: + movq %rdx, %rax + # A[0] * B + movq %rax, %rdx + xorq %r11, %r11 + mulxq (%rsi), %r9, %r10 + movq %r9, (%rdi) + # A[1] * B + mulxq 8(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 8(%rdi) + adoxq %r8, %r9 + # A[2] * B + mulxq 16(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 16(%rdi) + adoxq %r8, %r10 + # A[3] * B + mulxq 24(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 24(%rdi) + adoxq %r8, %r9 + # A[4] * B + mulxq 32(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 32(%rdi) + adoxq %r8, %r10 + # A[5] * B + mulxq 40(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 40(%rdi) + adoxq %r8, %r9 + # A[6] * B + mulxq 48(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 48(%rdi) + adoxq %r8, %r10 + # A[7] * B + mulxq 56(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 56(%rdi) + adoxq %r8, %r9 + # A[8] * B + mulxq 64(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 64(%rdi) + adoxq %r8, %r10 + # A[9] * B + mulxq 72(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 72(%rdi) + adoxq %r8, %r9 + # A[10] * B + mulxq 80(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 80(%rdi) + adoxq %r8, %r10 + # A[11] * B + mulxq 88(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 88(%rdi) + adoxq %r8, %r9 + # A[12] * B + mulxq 96(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 96(%rdi) + adoxq %r8, %r10 + # A[13] * B + mulxq 104(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 104(%rdi) + adoxq %r8, %r9 + # A[14] * B + mulxq 112(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 112(%rdi) + adoxq %r8, %r10 + # A[15] * B + mulxq 120(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 120(%rdi) + adoxq %r8, %r9 + # A[16] * B + mulxq 128(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 128(%rdi) + adoxq %r8, %r10 + # A[17] * B + mulxq 136(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 136(%rdi) + adoxq %r8, %r9 + # A[18] * B + mulxq 144(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 144(%rdi) + adoxq %r8, %r10 + # A[19] * B + mulxq 152(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 152(%rdi) + adoxq %r8, %r9 + # A[20] * B + mulxq 160(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 160(%rdi) + adoxq %r8, %r10 + # A[21] * B + mulxq 168(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 168(%rdi) + adoxq %r8, %r9 + # A[22] * B + mulxq 176(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 176(%rdi) + adoxq %r8, %r10 + # A[23] * B + mulxq 184(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 184(%rdi) + adoxq %r8, %r9 + # A[24] * B + mulxq 192(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 192(%rdi) + adoxq %r8, %r10 + # A[25] * B + mulxq 200(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 200(%rdi) + adoxq %r8, %r9 + # A[26] * B + mulxq 208(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 208(%rdi) + adoxq %r8, %r10 + # A[27] * B + mulxq 216(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 216(%rdi) + adoxq %r8, %r9 + # A[28] * B + mulxq 224(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 224(%rdi) + adoxq %r8, %r10 + # A[29] * B + mulxq 232(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 232(%rdi) + adoxq %r8, %r9 + # A[30] * B + mulxq 240(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 240(%rdi) + adoxq %r8, %r10 + # A[31] * B + mulxq 248(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 248(%rdi) + adoxq %r8, %r9 + # A[32] * B + mulxq 256(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 256(%rdi) + adoxq %r8, %r10 + # A[33] * B + mulxq 264(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 264(%rdi) + adoxq %r8, %r9 + # A[34] * B + mulxq 272(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 272(%rdi) + adoxq %r8, %r10 + # A[35] * B + mulxq 280(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 280(%rdi) + adoxq %r8, %r9 + # A[36] * B + mulxq 288(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 288(%rdi) + adoxq %r8, %r10 + # A[37] * B + mulxq 296(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 296(%rdi) + adoxq %r8, %r9 + # A[38] * B + mulxq 304(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 304(%rdi) + adoxq %r8, %r10 + # A[39] * B + mulxq 312(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 312(%rdi) + adoxq %r8, %r9 + # A[40] * B + mulxq 320(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 320(%rdi) + adoxq %r8, %r10 + # A[41] * B + mulxq 328(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 328(%rdi) + adoxq %r8, %r9 + # A[42] * B + mulxq 336(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 336(%rdi) + adoxq %r8, %r10 + # A[43] * B + mulxq 344(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 344(%rdi) + adoxq %r8, %r9 + # A[44] * B + mulxq 352(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 352(%rdi) + adoxq %r8, %r10 + # A[45] * B + mulxq 360(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 360(%rdi) + adoxq %r8, %r9 + # A[46] * B + mulxq 368(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 368(%rdi) + adoxq %r8, %r10 + # A[47] * B + mulxq 376(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + adoxq %r8, %r9 + adcxq %r11, %r9 + movq %r10, 376(%rdi) + movq %r9, 384(%rdi) + repz retq +.size sp_3072_mul_d_avx2_48,.-sp_3072_mul_d_avx2_48 +#endif /* HAVE_INTEL_AVX2 */ +/* Compare a with b in constant time. + * + * a A single precision integer. + * b A single precision integer. + * return -ve, 0 or +ve if a is less than, equal to or greater than b + * respectively. + */ +.globl sp_3072_cmp_48 +.type sp_3072_cmp_48,@function +.align 16 +sp_3072_cmp_48: + xorq %rcx, %rcx + movq $-1, %rdx + movq $-1, %rax + movq $1, %r8 + movq 376(%rdi), %r9 + movq 376(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 368(%rdi), %r9 + movq 368(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 360(%rdi), %r9 + movq 360(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 352(%rdi), %r9 + movq 352(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 344(%rdi), %r9 + movq 344(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 336(%rdi), %r9 + movq 336(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 328(%rdi), %r9 + movq 328(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 320(%rdi), %r9 + movq 320(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 312(%rdi), %r9 + movq 312(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 304(%rdi), %r9 + movq 304(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 296(%rdi), %r9 + movq 296(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 288(%rdi), %r9 + movq 288(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 280(%rdi), %r9 + movq 280(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 272(%rdi), %r9 + movq 272(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 264(%rdi), %r9 + movq 264(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 256(%rdi), %r9 + movq 256(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 248(%rdi), %r9 + movq 248(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 240(%rdi), %r9 + movq 240(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 232(%rdi), %r9 + movq 232(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 224(%rdi), %r9 + movq 224(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 216(%rdi), %r9 + movq 216(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 208(%rdi), %r9 + movq 208(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 200(%rdi), %r9 + movq 200(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 192(%rdi), %r9 + movq 192(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 184(%rdi), %r9 + movq 184(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 176(%rdi), %r9 + movq 176(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 168(%rdi), %r9 + movq 168(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 160(%rdi), %r9 + movq 160(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 152(%rdi), %r9 + movq 152(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 144(%rdi), %r9 + movq 144(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 136(%rdi), %r9 + movq 136(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 128(%rdi), %r9 + movq 128(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 120(%rdi), %r9 + movq 120(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 112(%rdi), %r9 + movq 112(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 104(%rdi), %r9 + movq 104(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 96(%rdi), %r9 + movq 96(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 88(%rdi), %r9 + movq 88(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 80(%rdi), %r9 + movq 80(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 72(%rdi), %r9 + movq 72(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 64(%rdi), %r9 + movq 64(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 56(%rdi), %r9 + movq 56(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 48(%rdi), %r9 + movq 48(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 40(%rdi), %r9 + movq 40(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 32(%rdi), %r9 + movq 32(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 24(%rdi), %r9 + movq 24(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 16(%rdi), %r9 + movq 16(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 8(%rdi), %r9 + movq 8(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq (%rdi), %r9 + movq (%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + xorq %rdx, %rax + repz retq +.size sp_3072_cmp_48,.-sp_3072_cmp_48 +#if defined(SP_RSA_PRIVATE_EXP_D) || defined(WOLFSSL_HAVE_SP_DH) +#endif /* SP_RSA_PRIVATE_EXP_D || WOLFSSL_HAVE_SP_DH */ +#ifdef HAVE_INTEL_AVX2 +/* Reduce the number back to 3072 bits using Montgomery reduction. + * + * a A single precision number to reduce in place. + * m The single precision number representing the modulus. + * mp The digit representing the negative inverse of m mod 2^n. + */ +.globl sp_3072_mont_reduce_avx2_48 +.type sp_3072_mont_reduce_avx2_48,@function +.align 16 +sp_3072_mont_reduce_avx2_48: + push %r12 + push %r13 + push %r14 + movq %rdx, %rax + xorq %r14, %r14 + # i = 0 + movq $48, %r9 + movq (%rdi), %r13 + xorq %r12, %r12 +L_mont_loop_avx2_48: + # mu = a[i] * mp + movq %r13, %rdx + mulxq %rax, %rdx, %r8 + movq %r13, %r10 + # a[i+0] += m[0] * mu + mulxq (%rsi), %rcx, %r8 + movq 8(%rdi), %r13 + adcxq %rcx, %r10 + adoxq %r8, %r13 + # a[i+1] += m[1] * mu + mulxq 8(%rsi), %rcx, %r8 + movq 16(%rdi), %r10 + adcxq %rcx, %r13 + adoxq %r8, %r10 + # a[i+2] += m[2] * mu + mulxq 16(%rsi), %rcx, %r8 + movq 24(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 16(%rdi) + # a[i+3] += m[3] * mu + mulxq 24(%rsi), %rcx, %r8 + movq 32(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 24(%rdi) + # a[i+4] += m[4] * mu + mulxq 32(%rsi), %rcx, %r8 + movq 40(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 32(%rdi) + # a[i+5] += m[5] * mu + mulxq 40(%rsi), %rcx, %r8 + movq 48(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 40(%rdi) + # a[i+6] += m[6] * mu + mulxq 48(%rsi), %rcx, %r8 + movq 56(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 48(%rdi) + # a[i+7] += m[7] * mu + mulxq 56(%rsi), %rcx, %r8 + movq 64(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 56(%rdi) + # a[i+8] += m[8] * mu + mulxq 64(%rsi), %rcx, %r8 + movq 72(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 64(%rdi) + # a[i+9] += m[9] * mu + mulxq 72(%rsi), %rcx, %r8 + movq 80(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 72(%rdi) + # a[i+10] += m[10] * mu + mulxq 80(%rsi), %rcx, %r8 + movq 88(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 80(%rdi) + # a[i+11] += m[11] * mu + mulxq 88(%rsi), %rcx, %r8 + movq 96(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 88(%rdi) + # a[i+12] += m[12] * mu + mulxq 96(%rsi), %rcx, %r8 + movq 104(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 96(%rdi) + # a[i+13] += m[13] * mu + mulxq 104(%rsi), %rcx, %r8 + movq 112(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 104(%rdi) + # a[i+14] += m[14] * mu + mulxq 112(%rsi), %rcx, %r8 + movq 120(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 112(%rdi) + # a[i+15] += m[15] * mu + mulxq 120(%rsi), %rcx, %r8 + movq 128(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 120(%rdi) + # a[i+16] += m[16] * mu + mulxq 128(%rsi), %rcx, %r8 + movq 136(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 128(%rdi) + # a[i+17] += m[17] * mu + mulxq 136(%rsi), %rcx, %r8 + movq 144(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 136(%rdi) + # a[i+18] += m[18] * mu + mulxq 144(%rsi), %rcx, %r8 + movq 152(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 144(%rdi) + # a[i+19] += m[19] * mu + mulxq 152(%rsi), %rcx, %r8 + movq 160(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 152(%rdi) + # a[i+20] += m[20] * mu + mulxq 160(%rsi), %rcx, %r8 + movq 168(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 160(%rdi) + # a[i+21] += m[21] * mu + mulxq 168(%rsi), %rcx, %r8 + movq 176(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 168(%rdi) + # a[i+22] += m[22] * mu + mulxq 176(%rsi), %rcx, %r8 + movq 184(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 176(%rdi) + # a[i+23] += m[23] * mu + mulxq 184(%rsi), %rcx, %r8 + movq 192(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 184(%rdi) + # a[i+24] += m[24] * mu + mulxq 192(%rsi), %rcx, %r8 + movq 200(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 192(%rdi) + # a[i+25] += m[25] * mu + mulxq 200(%rsi), %rcx, %r8 + movq 208(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 200(%rdi) + # a[i+26] += m[26] * mu + mulxq 208(%rsi), %rcx, %r8 + movq 216(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 208(%rdi) + # a[i+27] += m[27] * mu + mulxq 216(%rsi), %rcx, %r8 + movq 224(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 216(%rdi) + # a[i+28] += m[28] * mu + mulxq 224(%rsi), %rcx, %r8 + movq 232(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 224(%rdi) + # a[i+29] += m[29] * mu + mulxq 232(%rsi), %rcx, %r8 + movq 240(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 232(%rdi) + # a[i+30] += m[30] * mu + mulxq 240(%rsi), %rcx, %r8 + movq 248(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 240(%rdi) + # a[i+31] += m[31] * mu + mulxq 248(%rsi), %rcx, %r8 + movq 256(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 248(%rdi) + # a[i+32] += m[32] * mu + mulxq 256(%rsi), %rcx, %r8 + movq 264(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 256(%rdi) + # a[i+33] += m[33] * mu + mulxq 264(%rsi), %rcx, %r8 + movq 272(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 264(%rdi) + # a[i+34] += m[34] * mu + mulxq 272(%rsi), %rcx, %r8 + movq 280(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 272(%rdi) + # a[i+35] += m[35] * mu + mulxq 280(%rsi), %rcx, %r8 + movq 288(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 280(%rdi) + # a[i+36] += m[36] * mu + mulxq 288(%rsi), %rcx, %r8 + movq 296(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 288(%rdi) + # a[i+37] += m[37] * mu + mulxq 296(%rsi), %rcx, %r8 + movq 304(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 296(%rdi) + # a[i+38] += m[38] * mu + mulxq 304(%rsi), %rcx, %r8 + movq 312(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 304(%rdi) + # a[i+39] += m[39] * mu + mulxq 312(%rsi), %rcx, %r8 + movq 320(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 312(%rdi) + # a[i+40] += m[40] * mu + mulxq 320(%rsi), %rcx, %r8 + movq 328(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 320(%rdi) + # a[i+41] += m[41] * mu + mulxq 328(%rsi), %rcx, %r8 + movq 336(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 328(%rdi) + # a[i+42] += m[42] * mu + mulxq 336(%rsi), %rcx, %r8 + movq 344(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 336(%rdi) + # a[i+43] += m[43] * mu + mulxq 344(%rsi), %rcx, %r8 + movq 352(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 344(%rdi) + # a[i+44] += m[44] * mu + mulxq 352(%rsi), %rcx, %r8 + movq 360(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 352(%rdi) + # a[i+45] += m[45] * mu + mulxq 360(%rsi), %rcx, %r8 + movq 368(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 360(%rdi) + # a[i+46] += m[46] * mu + mulxq 368(%rsi), %rcx, %r8 + movq 376(%rdi), %r11 + adcxq %rcx, %r10 + adoxq %r8, %r11 + movq %r10, 368(%rdi) + # a[i+47] += m[47] * mu + mulxq 376(%rsi), %rcx, %r8 + movq 384(%rdi), %r10 + adcxq %rcx, %r11 + adoxq %r8, %r10 + movq %r11, 376(%rdi) + adcxq %r14, %r10 + movq %r12, %r14 + adoxq %r12, %r14 + adcxq %r12, %r14 + movq %r10, 384(%rdi) + # i += 1 + addq $8, %rdi + decq %r9 + jnz L_mont_loop_avx2_48 + movq %r13, (%rdi) + negq %r14 + movq %r14, %rcx + movq %rsi, %rdx + movq %rdi, %rsi + movq %rdi, %rdi + subq $384, %rdi + callq sp_3072_cond_sub_48@plt + pop %r14 + pop %r13 + pop %r12 + repz retq +.size sp_3072_mont_reduce_avx2_48,.-sp_3072_mont_reduce_avx2_48 +#endif /* HAVE_INTEL_AVX2 */ +#ifdef HAVE_INTEL_AVX2 +#endif /* HAVE_INTEL_AVX2 */ +#ifdef HAVE_INTEL_AVX2 +#endif /* HAVE_INTEL_AVX2 */ +#if defined(SP_RSA_PRIVATE_EXP_D) || defined(WOLFSSL_HAVE_SP_DH) +#ifdef HAVE_INTEL_AVX2 +#endif /* HAVE_INTEL_AVX2 */ +#endif /* SP_RSA_PRIVATE_EXP_D || WOLFSSL_HAVE_SP_DH */ +#ifdef WOLFSSL_HAVE_SP_RSA +#endif /* WOLFSSL_HAVE_SP_RSA */ +#endif /* !WOLFSSL_SP_NO_3072 */ +#endif /* WOLFSSL_HAVE_SP_RSA || WOLFSSL_HAVE_SP_DH */ +#ifdef WOLFSSL_HAVE_SP_ECC +#ifndef WOLFSSL_SP_NO_256 +/* Conditionally copy a into r using the mask m. + * m is -1 to copy and 0 when not. + * + * r A single precision number to copy over. + * a A single precision number to copy. + * m Mask value to apply. + */ +.globl sp_256_cond_copy_4 +.type sp_256_cond_copy_4,@function +.align 16 +sp_256_cond_copy_4: + movq (%rdi), %rax + movq 8(%rdi), %rcx + movq 16(%rdi), %r8 + movq 24(%rdi), %r9 + xorq (%rsi), %rax + xorq 8(%rsi), %rcx + xorq 16(%rsi), %r8 + xorq 24(%rsi), %r9 + andq %rdx, %rax + andq %rdx, %rcx + andq %rdx, %r8 + andq %rdx, %r9 + xorq %rax, (%rdi) + xorq %rcx, 8(%rdi) + xorq %r8, 16(%rdi) + xorq %r9, 24(%rdi) + repz retq +.size sp_256_cond_copy_4,.-sp_256_cond_copy_4 +/* Compare a with b in constant time. + * + * a A single precision integer. + * b A single precision integer. + * return -ve, 0 or +ve if a is less than, equal to or greater than b + * respectively. + */ +.globl sp_256_cmp_4 +.type sp_256_cmp_4,@function +.align 16 +sp_256_cmp_4: + xorq %rcx, %rcx + movq $-1, %rdx + movq $-1, %rax + movq $1, %r8 + movq 24(%rdi), %r9 + movq 24(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 16(%rdi), %r9 + movq 16(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq 8(%rdi), %r9 + movq 8(%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + movq (%rdi), %r9 + movq (%rsi), %r10 + andq %rdx, %r9 + andq %rdx, %r10 + subq %r10, %r9 + cmova %r8, %rax + cmovc %rdx, %rax + cmovnz %rcx, %rdx + xorq %rdx, %rax + repz retq +.size sp_256_cmp_4,.-sp_256_cmp_4 +/* Conditionally subtract b from a using the mask m. + * m is -1 to subtract and 0 when not copying. + * + * r A single precision number representing condition subtract result. + * a A single precision number to subtract from. + * b A single precision number to subtract. + * m Mask value to apply. + */ +.globl sp_256_cond_sub_4 +.type sp_256_cond_sub_4,@function +.align 16 +sp_256_cond_sub_4: + push %r12 + push %r13 + push %r14 + push %r15 + movq $0, %rax + movq (%rdx), %r12 + movq 8(%rdx), %r13 + movq 16(%rdx), %r14 + movq 24(%rdx), %r15 + andq %rcx, %r12 + andq %rcx, %r13 + andq %rcx, %r14 + andq %rcx, %r15 + movq (%rsi), %r8 + movq 8(%rsi), %r9 + movq 16(%rsi), %r10 + movq 24(%rsi), %r11 + subq %r12, %r8 + sbbq %r13, %r9 + sbbq %r14, %r10 + sbbq %r15, %r11 + movq %r8, (%rdi) + movq %r9, 8(%rdi) + movq %r10, 16(%rdi) + movq %r11, 24(%rdi) + sbbq $0, %rax + pop %r15 + pop %r14 + pop %r13 + pop %r12 + repz retq +.size sp_256_cond_sub_4,.-sp_256_cond_sub_4 +/* Sub b from a into r. (r = a - b) + * + * r A single precision integer. + * a A single precision integer. + * b A single precision integer. + */ +.globl sp_256_sub_4 +.type sp_256_sub_4,@function +.align 16 +sp_256_sub_4: + push %r12 + push %r13 + push %r14 + push %r15 + xorq %rax, %rax + movq (%rsi), %r8 + movq 8(%rsi), %r9 + movq 16(%rsi), %r10 + movq 24(%rsi), %r11 + movq (%rdx), %r12 + movq 8(%rdx), %r13 + movq 16(%rdx), %r14 + movq 24(%rdx), %r15 + subq %r12, %r8 + sbbq %r13, %r9 + sbbq %r14, %r10 + sbbq %r15, %r11 + movq %r8, (%rdi) + movq %r9, 8(%rdi) + movq %r10, 16(%rdi) + movq %r11, 24(%rdi) + sbbq $0, %rax + pop %r15 + pop %r14 + pop %r13 + pop %r12 + repz retq +.size sp_256_sub_4,.-sp_256_sub_4 +/* Reduce the number back to 256 bits using Montgomery reduction. + * + * a A single precision number to reduce in place. + * m The single precision number representing the modulus. + * mp The digit representing the negative inverse of m mod 2^n. + */ +.globl sp_256_mont_reduce_4 +.type sp_256_mont_reduce_4,@function +.align 16 +sp_256_mont_reduce_4: + push %r12 + push %r13 + push %r14 + push %r15 + movq %rdx, %rcx + # i = 0 + xorq %r14, %r14 + movq $4, %r8 + movq %rdi, %r13 +L_mont_loop_4: + # mu = a[i] * mp + movq (%r13), %r12 + imulq %rcx, %r12 + # a[i+0] += m[0] * mu + movq (%rsi), %rax + movq 8(%rsi), %r10 + mulq %r12 + movq (%r13), %r15 + addq %rax, %r15 + movq %rdx, %r9 + movq %r15, (%r13) + adcq $0, %r9 + # a[i+1] += m[1] * mu + movq %r10, %rax + mulq %r12 + movq 16(%rsi), %r10 + movq 8(%r13), %r15 + addq %r9, %rax + movq %rdx, %r11 + adcq $0, %r11 + addq %rax, %r15 + movq %r15, 8(%r13) + adcq $0, %r11 + # a[i+2] += m[2] * mu + movq %r10, %rax + mulq %r12 + movq 24(%rsi), %r10 + movq 16(%r13), %r15 + addq %r11, %rax + movq %rdx, %r9 + adcq $0, %r9 + addq %rax, %r15 + movq %r15, 16(%r13) + adcq $0, %r9 + # a[i+3] += m[3] * mu + movq %r10, %rax + mulq %r12 + movq 24(%r13), %r15 + addq %r9, %rax + adcq %r14, %rdx + movq $0, %r14 + adcq $0, %r14 + addq %rax, %r15 + movq %r15, 24(%r13) + adcq %rdx, 32(%r13) + adcq $0, %r14 + # i += 1 + addq $8, %r13 + decq %r8 + jnz L_mont_loop_4 + xorq %rax, %rax + movq 32(%rdi), %rdx + movq 40(%rdi), %r8 + movq 48(%rdi), %r15 + movq 56(%rdi), %r9 + subq %r14, %rax + movq (%rsi), %r10 + movq 8(%rsi), %r11 + movq 16(%rsi), %r12 + movq 24(%rsi), %r13 + andq %rax, %r10 + andq %rax, %r11 + andq %rax, %r12 + andq %rax, %r13 + subq %r10, %rdx + sbbq %r11, %r8 + sbbq %r12, %r15 + sbbq %r13, %r9 + movq %rdx, (%rdi) + movq %r8, 8(%rdi) + movq %r15, 16(%rdi) + movq %r9, 24(%rdi) + pop %r15 + pop %r14 + pop %r13 + pop %r12 + repz retq +.size sp_256_mont_reduce_4,.-sp_256_mont_reduce_4 +/* Multiply two Montogmery form numbers mod the modulus (prime). + * (r = a * b mod m) + * + * r Result of multiplication. + * a First number to multiply in Montogmery form. + * b Second number to multiply in Montogmery form. + * m Modulus (prime). + * mp Montogmery mulitplier. + */ +.globl sp_256_mont_mul_4 +.type sp_256_mont_mul_4,@function +.align 16 +sp_256_mont_mul_4: + push %r12 + push %r13 + push %r14 + push %r15 + push %rbx + movq %rdx, %r8 + # A[0] * B[0] + movq (%r8), %rax + mulq (%rsi) + movq %rax, %r9 + movq %rdx, %r10 + # A[0] * B[1] + movq 8(%r8), %rax + mulq (%rsi) + xorq %r11, %r11 + addq %rax, %r10 + adcq %rdx, %r11 + # A[1] * B[0] + movq (%r8), %rax + mulq 8(%rsi) + xorq %r12, %r12 + addq %rax, %r10 + adcq %rdx, %r11 + adcq $0, %r12 + # A[0] * B[2] + movq 16(%r8), %rax + mulq (%rsi) + addq %rax, %r11 + adcq %rdx, %r12 + # A[1] * B[1] + movq 8(%r8), %rax + mulq 8(%rsi) + xorq %r13, %r13 + addq %rax, %r11 + adcq %rdx, %r12 + adcq $0, %r13 + # A[2] * B[0] + movq (%r8), %rax + mulq 16(%rsi) + addq %rax, %r11 + adcq %rdx, %r12 + adcq $0, %r13 + # A[0] * B[3] + movq 24(%r8), %rax + mulq (%rsi) + xorq %r14, %r14 + addq %rax, %r12 + adcq %rdx, %r13 + adcq $0, %r14 + # A[1] * B[2] + movq 16(%r8), %rax + mulq 8(%rsi) + addq %rax, %r12 + adcq %rdx, %r13 + adcq $0, %r14 + # A[2] * B[1] + movq 8(%r8), %rax + mulq 16(%rsi) + addq %rax, %r12 + adcq %rdx, %r13 + adcq $0, %r14 + # A[3] * B[0] + movq (%r8), %rax + mulq 24(%rsi) + addq %rax, %r12 + adcq %rdx, %r13 + adcq $0, %r14 + # A[1] * B[3] + movq 24(%r8), %rax + mulq 8(%rsi) + xorq %r15, %r15 + addq %rax, %r13 + adcq %rdx, %r14 + adcq $0, %r15 + # A[2] * B[2] + movq 16(%r8), %rax + mulq 16(%rsi) + addq %rax, %r13 + adcq %rdx, %r14 + adcq $0, %r15 + # A[3] * B[1] + movq 8(%r8), %rax + mulq 24(%rsi) + addq %rax, %r13 + adcq %rdx, %r14 + adcq $0, %r15 + # A[2] * B[3] + movq 24(%r8), %rax + mulq 16(%rsi) + xorq %rbx, %rbx + addq %rax, %r14 + adcq %rdx, %r15 + adcq $0, %rbx + # A[3] * B[2] + movq 16(%r8), %rax + mulq 24(%rsi) + addq %rax, %r14 + adcq %rdx, %r15 + adcq $0, %rbx + # A[3] * B[3] + movq 24(%r8), %rax + mulq 24(%rsi) + addq %rax, %r15 + adcq %rdx, %rbx + # Start Reduction + # mu = a[0]-a[3] + a[0]-a[2] << 32 << 64 + (a[0] * 2) << 192 + # - a[0] << 32 << 192 + # + (a[0] * 2) << 192 + movq %r9, %rax + movq %r12, %rdx + addq %r9, %rdx + movq %r10, %rsi + addq %r9, %rdx + movq %r11, %r8 + # a[0]-a[2] << 32 + shlq $32, %r9 + shldq $32, %rsi, %r11 + shldq $32, %rax, %r10 + # - a[0] << 32 << 192 + subq %r9, %rdx + # + a[0]-a[2] << 32 << 64 + addq %r9, %rsi + adcq %r10, %r8 + adcq %r11, %rdx + # a += (mu << 256) - (mu << 224) + (mu << 192) + (mu << 96) - mu + # a += mu << 256 + xorq %r9, %r9 + addq %rax, %r13 + adcq %rsi, %r14 + adcq %r8, %r15 + adcq %rdx, %rbx + sbbq $0, %r9 + # a += mu << 192 + addq %rax, %r12 + adcq %rsi, %r13 + adcq %r8, %r14 + adcq %rdx, %r15 + adcq $0, %rbx + sbbq $0, %r9 + # mu <<= 32 + movq %rdx, %rcx + shldq $32, %r8, %rdx + shldq $32, %rsi, %r8 + shldq $32, %rax, %rsi + shrq $32, %rcx + shlq $32, %rax + # a += (mu << 32) << 64 + addq %r8, %r12 + adcq %rdx, %r13 + adcq %rcx, %r14 + adcq $0, %r15 + adcq $0, %rbx + sbbq $0, %r9 + # a -= (mu << 32) << 192 + subq %rax, %r12 + movq $4294967295, %rax + sbbq %rsi, %r13 + movq $18446744069414584321, %rsi + sbbq %r8, %r14 + sbbq %rdx, %r15 + sbbq %rcx, %rbx + adcq $0, %r9 + # mask m and sub from result if overflow + # m[0] = -1 & mask = mask + andq %r9, %rax + # m[2] = 0 & mask = 0 + andq %r9, %rsi + subq %r9, %r13 + sbbq %rax, %r14 + sbbq $0, %r15 + sbbq %rsi, %rbx + movq %r13, (%rdi) + movq %r14, 8(%rdi) + movq %r15, 16(%rdi) + movq %rbx, 24(%rdi) + pop %rbx + pop %r15 + pop %r14 + pop %r13 + pop %r12 + repz retq +.size sp_256_mont_mul_4,.-sp_256_mont_mul_4 +/* Square the Montgomery form number mod the modulus (prime). (r = a * a mod m) + * + * r Result of squaring. + * a Number to square in Montogmery form. + * m Modulus (prime). + * mp Montogmery mulitplier. + */ +.globl sp_256_mont_sqr_4 +.type sp_256_mont_sqr_4,@function +.align 16 +sp_256_mont_sqr_4: + push %r12 + push %r13 + push %r14 + push %r15 + push %rbx + # A[0] * A[1] + movq (%rsi), %rax + mulq 8(%rsi) + movq %rax, %r10 + movq %rdx, %r11 + # A[0] * A[2] + movq (%rsi), %rax + mulq 16(%rsi) + xorq %r12, %r12 + addq %rax, %r11 + adcq %rdx, %r12 + # A[0] * A[3] + movq (%rsi), %rax + mulq 24(%rsi) + xorq %r13, %r13 + addq %rax, %r12 + adcq %rdx, %r13 + # A[1] * A[2] + movq 8(%rsi), %rax + mulq 16(%rsi) + xorq %r14, %r14 + addq %rax, %r12 + adcq %rdx, %r13 + adcq $0, %r14 + # A[1] * A[3] + movq 8(%rsi), %rax + mulq 24(%rsi) + addq %rax, %r13 + adcq %rdx, %r14 + # A[2] * A[3] + movq 16(%rsi), %rax + mulq 24(%rsi) + xorq %r15, %r15 + addq %rax, %r14 + adcq %rdx, %r15 + # Double + xorq %rbx, %rbx + addq %r10, %r10 + adcq %r11, %r11 + adcq %r12, %r12 + adcq %r13, %r13 + adcq %r14, %r14 + adcq %r15, %r15 + adcq $0, %rbx + # A[0] * A[0] + movq (%rsi), %rax + mulq %rax + movq %rax, %rax + movq %rdx, %rdx + movq %rax, %r9 + movq %rdx, %r8 + # A[1] * A[1] + movq 8(%rsi), %rax + mulq %rax + movq %rax, %rax + movq %rdx, %rdx + addq %r8, %r10 + adcq %rax, %r11 + adcq $0, %rdx + movq %rdx, %r8 + # A[2] * A[2] + movq 16(%rsi), %rax + mulq %rax + movq %rax, %rax + movq %rdx, %rdx + addq %r8, %r12 + adcq %rax, %r13 + adcq $0, %rdx + movq %rdx, %r8 + # A[3] * A[3] + movq 24(%rsi), %rax + mulq %rax + addq %rax, %r15 + adcq %rdx, %rbx + addq %r8, %r14 + adcq $0, %r15 + adcq $0, %rbx + # Start Reduction + # mu = a[0]-a[3] + a[0]-a[2] << 32 << 64 + (a[0] * 2) << 192 + # - a[0] << 32 << 192 + # + (a[0] * 2) << 192 + movq %r9, %rax + movq %r12, %rdx + addq %r9, %rdx + movq %r10, %rsi + addq %r9, %rdx + movq %r11, %r8 + # a[0]-a[2] << 32 + shlq $32, %r9 + shldq $32, %rsi, %r11 + shldq $32, %rax, %r10 + # - a[0] << 32 << 192 + subq %r9, %rdx + # + a[0]-a[2] << 32 << 64 + addq %r9, %rsi + adcq %r10, %r8 + adcq %r11, %rdx + # a += (mu << 256) - (mu << 224) + (mu << 192) + (mu << 96) - mu + # a += mu << 256 + xorq %r9, %r9 + addq %rax, %r13 + adcq %rsi, %r14 + adcq %r8, %r15 + adcq %rdx, %rbx + sbbq $0, %r9 + # a += mu << 192 + addq %rax, %r12 + adcq %rsi, %r13 + adcq %r8, %r14 + adcq %rdx, %r15 + adcq $0, %rbx + sbbq $0, %r9 + # mu <<= 32 + movq %rdx, %rcx + shldq $32, %r8, %rdx + shldq $32, %rsi, %r8 + shldq $32, %rax, %rsi + shrq $32, %rcx + shlq $32, %rax + # a += (mu << 32) << 64 + addq %r8, %r12 + adcq %rdx, %r13 + adcq %rcx, %r14 + adcq $0, %r15 + adcq $0, %rbx + sbbq $0, %r9 + # a -= (mu << 32) << 192 + subq %rax, %r12 + movq $4294967295, %rax + sbbq %rsi, %r13 + movq $18446744069414584321, %rsi + sbbq %r8, %r14 + sbbq %rdx, %r15 + sbbq %rcx, %rbx + adcq $0, %r9 + # mask m and sub from result if overflow + # m[0] = -1 & mask = mask + andq %r9, %rax + # m[2] = 0 & mask = 0 + andq %r9, %rsi + subq %r9, %r13 + sbbq %rax, %r14 + sbbq $0, %r15 + sbbq %rsi, %rbx + movq %r13, (%rdi) + movq %r14, 8(%rdi) + movq %r15, 16(%rdi) + movq %rbx, 24(%rdi) + pop %rbx + pop %r15 + pop %r14 + pop %r13 + pop %r12 + repz retq +.size sp_256_mont_sqr_4,.-sp_256_mont_sqr_4 +/* Add two Montgomery form numbers (r = a + b % m). + * + * r Result of addition. + * a First number to add in Montogmery form. + * b Second number to add in Montogmery form. + * m Modulus (prime). + */ +.globl sp_256_mont_add_4 +.type sp_256_mont_add_4,@function +.align 16 +sp_256_mont_add_4: + movq (%rsi), %rax + movq 8(%rsi), %rcx + movq 16(%rsi), %r8 + movq 24(%rsi), %r9 + addq (%rdx), %rax + adcq 8(%rdx), %rcx + movq $4294967295, %r10 + adcq 16(%rdx), %r8 + movq $18446744069414584321, %r11 + adcq 24(%rdx), %r9 + movq $0, %rdx + sbbq $0, %rdx + andq %rdx, %r10 + andq %rdx, %r11 + subq %rdx, %rax + sbbq %r10, %rcx + movq %rax, (%rdi) + sbbq $0, %r8 + movq %rcx, 8(%rdi) + sbbq %r11, %r9 + movq %r8, 16(%rdi) + movq %r9, 24(%rdi) + repz retq +.size sp_256_mont_add_4,.-sp_256_mont_add_4 +/* Double a Montgomery form number (r = a + a % m). + * + * r Result of doubling. + * a Number to double in Montogmery form. + * m Modulus (prime). + */ +.globl sp_256_mont_dbl_4 +.type sp_256_mont_dbl_4,@function +.align 16 +sp_256_mont_dbl_4: + movq (%rsi), %rdx + movq 8(%rsi), %rax + movq 16(%rsi), %rcx + movq 24(%rsi), %r8 + xorq %r11, %r11 + addq %rdx, %rdx + adcq %rax, %rax + movq $4294967295, %r9 + adcq %rcx, %rcx + adcq %r8, %r8 + movq $18446744069414584321, %r10 + sbbq $0, %r11 + andq %r11, %r9 + andq %r11, %r10 + subq %r11, %rdx + sbbq %r9, %rax + movq %rdx, (%rdi) + sbbq $0, %rcx + movq %rax, 8(%rdi) + sbbq %r10, %r8 + movq %rcx, 16(%rdi) + movq %r8, 24(%rdi) + repz retq +.size sp_256_mont_dbl_4,.-sp_256_mont_dbl_4 +/* Triple a Montgomery form number (r = a + a + a % m). + * + * r Result of Tripling. + * a Number to triple in Montogmery form. + * m Modulus (prime). + */ +.globl sp_256_mont_tpl_4 +.type sp_256_mont_tpl_4,@function +.align 16 +sp_256_mont_tpl_4: + movq (%rsi), %rdx + movq 8(%rsi), %rax + movq 16(%rsi), %rcx + movq 24(%rsi), %r8 + xorq %r11, %r11 + addq %rdx, %rdx + adcq %rax, %rax + movq $4294967295, %r9 + adcq %rcx, %rcx + adcq %r8, %r8 + movq $18446744069414584321, %r10 + sbbq $0, %r11 + andq %r11, %r9 + andq %r11, %r10 + subq %r11, %rdx + sbbq %r9, %rax + sbbq $0, %rcx + sbbq %r10, %r8 + xorq %r11, %r11 + addq (%rsi), %rdx + adcq 8(%rsi), %rax + movq $4294967295, %r9 + adcq 16(%rsi), %rcx + adcq 24(%rsi), %r8 + movq $18446744069414584321, %r10 + sbbq $0, %r11 + andq %r11, %r9 + andq %r11, %r10 + subq %r11, %rdx + sbbq %r9, %rax + movq %rdx, (%rdi) + sbbq $0, %rcx + movq %rax, 8(%rdi) + sbbq %r10, %r8 + movq %rcx, 16(%rdi) + movq %r8, 24(%rdi) + repz retq +.size sp_256_mont_tpl_4,.-sp_256_mont_tpl_4 +/* Subtract two Montgomery form numbers (r = a - b % m). + * + * r Result of subtration. + * a Number to subtract from in Montogmery form. + * b Number to subtract with in Montogmery form. + * m Modulus (prime). + */ +.globl sp_256_mont_sub_4 +.type sp_256_mont_sub_4,@function +.align 16 +sp_256_mont_sub_4: + movq (%rsi), %rax + movq 8(%rsi), %rcx + movq 16(%rsi), %r8 + movq 24(%rsi), %r9 + subq (%rdx), %rax + sbbq 8(%rdx), %rcx + movq $4294967295, %r10 + sbbq 16(%rdx), %r8 + movq $18446744069414584321, %r11 + sbbq 24(%rdx), %r9 + movq $0, %rdx + sbbq $0, %rdx + andq %rdx, %r10 + andq %rdx, %r11 + addq %rdx, %rax + adcq %r10, %rcx + movq %rax, (%rdi) + adcq $0, %r8 + movq %rcx, 8(%rdi) + adcq %r11, %r9 + movq %r8, 16(%rdi) + movq %r9, 24(%rdi) + repz retq +.size sp_256_mont_sub_4,.-sp_256_mont_sub_4 +/* Divide the number by 2 mod the modulus (prime). (r = a / 2 % m) + * + * r Result of division by 2. + * a Number to divide. + * m Modulus (prime). + */ +.globl sp_256_div2_4 +.type sp_256_div2_4,@function +.align 16 +sp_256_div2_4: + movq (%rsi), %rdx + movq 8(%rsi), %rax + movq 16(%rsi), %rcx + movq 24(%rsi), %r8 + movq $4294967295, %r9 + movq $18446744069414584321, %r10 + movq %rdx, %r11 + andq $1, %r11 + negq %r11 + andq %r11, %r9 + andq %r11, %r10 + addq %r11, %rdx + adcq %r9, %rax + adcq $0, %rcx + adcq %r10, %r8 + movq $0, %r11 + adcq $0, %r11 + shrdq $1, %rax, %rdx + shrdq $1, %rcx, %rax + shrdq $1, %r8, %rcx + shrdq $1, %r11, %r8 + movq %rdx, (%rdi) + movq %rax, 8(%rdi) + movq %rcx, 16(%rdi) + movq %r8, 24(%rdi) + repz retq +.size sp_256_div2_4,.-sp_256_div2_4 +#ifdef HAVE_INTEL_AVX2 +/* Multiply two Montogmery form numbers mod the modulus (prime). + * (r = a * b mod m) + * + * r Result of multiplication. + * a First number to multiply in Montogmery form. + * b Second number to multiply in Montogmery form. + * m Modulus (prime). + * mp Montogmery mulitplier. + */ +.globl sp_256_mont_mul_avx2_4 +.type sp_256_mont_mul_avx2_4,@function +.align 16 +sp_256_mont_mul_avx2_4: + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 + movq %rdx, %rbp + # A[0] * B[0] + movq (%rbp), %rdx + mulxq (%rsi), %r8, %r9 + # A[2] * B[0] + mulxq 16(%rsi), %r10, %r11 + # A[1] * B[0] + mulxq 8(%rsi), %rax, %rcx + xorq %r15, %r15 + adcxq %rax, %r9 + # A[1] * B[3] + movq 24(%rbp), %rdx + mulxq 8(%rsi), %r12, %r13 + adcxq %rcx, %r10 + # A[0] * B[1] + movq 8(%rbp), %rdx + mulxq (%rsi), %rax, %rcx + adoxq %rax, %r9 + # A[2] * B[1] + mulxq 16(%rsi), %rax, %r14 + adoxq %rcx, %r10 + adcxq %rax, %r11 + # A[1] * B[2] + movq 16(%rbp), %rdx + mulxq 8(%rsi), %rax, %rcx + adcxq %r14, %r12 + adoxq %rax, %r11 + adcxq %r15, %r13 + adoxq %rcx, %r12 + # A[0] * B[2] + mulxq (%rsi), %rax, %rcx + adoxq %r15, %r13 + xorq %r14, %r14 + adcxq %rax, %r10 + # A[1] * B[1] + movq 8(%rbp), %rdx + mulxq 8(%rsi), %rdx, %rax + adcxq %rcx, %r11 + adoxq %rdx, %r10 + # A[3] * B[1] + movq 8(%rbp), %rdx + adoxq %rax, %r11 + mulxq 24(%rsi), %rax, %rcx + adcxq %rax, %r12 + # A[2] * B[2] + movq 16(%rbp), %rdx + mulxq 16(%rsi), %rdx, %rax + adcxq %rcx, %r13 + adoxq %rdx, %r12 + # A[3] * B[3] + movq 24(%rbp), %rdx + adoxq %rax, %r13 + mulxq 24(%rsi), %rax, %rcx + adoxq %r15, %r14 + adcxq %rax, %r14 + # A[0] * B[3] + mulxq (%rsi), %rdx, %rax + adcxq %rcx, %r15 + xorq %rcx, %rcx + adcxq %rdx, %r11 + # A[3] * B[0] + movq 24(%rsi), %rdx + adcxq %rax, %r12 + mulxq (%rbp), %rbx, %rax + adoxq %rbx, %r11 + adoxq %rax, %r12 + # A[3] * B[2] + mulxq 16(%rbp), %rdx, %rax + adcxq %rdx, %r13 + # A[2] * B[3] + movq 24(%rbp), %rdx + adcxq %rax, %r14 + mulxq 16(%rsi), %rax, %rdx + adcxq %rcx, %r15 + adoxq %rax, %r13 + adoxq %rdx, %r14 + adoxq %rcx, %r15 + # Start Reduction + # mu = a[0]-a[3] + a[0]-a[2] << 32 << 64 + (a[0] * 2) << 192 + # - a[0] << 32 << 192 + # + (a[0] * 2) << 192 + movq %r8, %rax + movq %r11, %rdx + addq %r8, %rdx + movq %r9, %rsi + addq %r8, %rdx + movq %r10, %rbp + # a[0]-a[2] << 32 + shlq $32, %r8 + shldq $32, %rsi, %r10 + shldq $32, %rax, %r9 + # - a[0] << 32 << 192 + subq %r8, %rdx + # + a[0]-a[2] << 32 << 64 + addq %r8, %rsi + adcq %r9, %rbp + adcq %r10, %rdx + # a += (mu << 256) - (mu << 224) + (mu << 192) + (mu << 96) - mu + # a += mu << 256 + xorq %r8, %r8 + addq %rax, %r12 + adcq %rsi, %r13 + adcq %rbp, %r14 + adcq %rdx, %r15 + sbbq $0, %r8 + # a += mu << 192 + addq %rax, %r11 + adcq %rsi, %r12 + adcq %rbp, %r13 + adcq %rdx, %r14 + adcq $0, %r15 + sbbq $0, %r8 + # mu <<= 32 + movq %rdx, %rcx + shldq $32, %rbp, %rdx + shldq $32, %rsi, %rbp + shldq $32, %rax, %rsi + shrq $32, %rcx + shlq $32, %rax + # a += (mu << 32) << 64 + addq %rbp, %r11 + adcq %rdx, %r12 + adcq %rcx, %r13 + adcq $0, %r14 + adcq $0, %r15 + sbbq $0, %r8 + # a -= (mu << 32) << 192 + subq %rax, %r11 + movq $4294967295, %rax + sbbq %rsi, %r12 + movq $18446744069414584321, %rsi + sbbq %rbp, %r13 + sbbq %rdx, %r14 + sbbq %rcx, %r15 + adcq $0, %r8 + # mask m and sub from result if overflow + # m[0] = -1 & mask = mask + andq %r8, %rax + # m[2] = 0 & mask = 0 + andq %r8, %rsi + subq %r8, %r12 + sbbq %rax, %r13 + sbbq $0, %r14 + sbbq %rsi, %r15 + movq %r12, (%rdi) + movq %r13, 8(%rdi) + movq %r14, 16(%rdi) + movq %r15, 24(%rdi) + pop %r15 + pop %r14 + pop %r13 + pop %r12 + pop %rbp + pop %rbx + repz retq +.size sp_256_mont_mul_avx2_4,.-sp_256_mont_mul_avx2_4 +/* Square the Montgomery form number mod the modulus (prime). (r = a * a mod m) + * + * r Result of squaring. + * a Number to square in Montogmery form. + * m Modulus (prime). + * mp Montogmery mulitplier. + */ +.globl sp_256_mont_sqr_avx2_4 +.type sp_256_mont_sqr_avx2_4,@function +.align 16 +sp_256_mont_sqr_avx2_4: + push %r12 + push %r13 + push %r14 + push %r15 + push %rbx + # A[0] * A[1] + movq (%rsi), %rdx + mulxq 8(%rsi), %r9, %r10 + # A[0] * A[3] + mulxq 24(%rsi), %r11, %r12 + # A[2] * A[1] + movq 16(%rsi), %rdx + mulxq 8(%rsi), %rcx, %rbx + xorq %r15, %r15 + adoxq %rcx, %r11 + # A[2] * A[3] + mulxq 24(%rsi), %r13, %r14 + adoxq %rbx, %r12 + # A[2] * A[0] + mulxq (%rsi), %rcx, %rbx + adoxq %r15, %r13 + adcxq %rcx, %r10 + adoxq %r15, %r14 + # A[1] * A[3] + movq 8(%rsi), %rdx + mulxq 24(%rsi), %rax, %r8 + adcxq %rbx, %r11 + adcxq %rax, %r12 + adcxq %r8, %r13 + adcxq %r15, %r14 + # Double with Carry Flag + # A[0] * A[0] + movq (%rsi), %rdx + mulxq %rdx, %r8, %rax + adcxq %r9, %r9 + adcxq %r10, %r10 + adoxq %rax, %r9 + # A[1] * A[1] + movq 8(%rsi), %rdx + mulxq %rdx, %rcx, %rbx + adcxq %r11, %r11 + adoxq %rcx, %r10 + # A[2] * A[2] + movq 16(%rsi), %rdx + mulxq %rdx, %rax, %rcx + adcxq %r12, %r12 + adoxq %rbx, %r11 + adcxq %r13, %r13 + adoxq %rax, %r12 + adcxq %r14, %r14 + # A[3] * A[3] + movq 24(%rsi), %rdx + mulxq %rdx, %rax, %rbx + adoxq %rcx, %r13 + adcxq %r15, %r15 + adoxq %rax, %r14 + adoxq %rbx, %r15 + # Start Reduction + # mu = a[0]-a[3] + a[0]-a[2] << 32 << 64 + (a[0] * 2) << 192 + # - a[0] << 32 << 192 + # + (a[0] * 2) << 192 + movq %r8, %rax + movq %r11, %rdx + addq %r8, %rdx + movq %r9, %rsi + addq %r8, %rdx + movq %r10, %rcx + # a[0]-a[2] << 32 + shlq $32, %r8 + shldq $32, %rsi, %r10 + shldq $32, %rax, %r9 + # - a[0] << 32 << 192 + subq %r8, %rdx + # + a[0]-a[2] << 32 << 64 + addq %r8, %rsi + adcq %r9, %rcx + adcq %r10, %rdx + # a += (mu << 256) - (mu << 224) + (mu << 192) + (mu << 96) - mu + # a += mu << 256 + movq $0, %r8 + addq %rax, %r12 + adcq %rsi, %r13 + adcq %rcx, %r14 + adcq %rdx, %r15 + sbbq $0, %r8 + # a += mu << 192 + addq %rax, %r11 + adcq %rsi, %r12 + adcq %rcx, %r13 + adcq %rdx, %r14 + adcq $0, %r15 + sbbq $0, %r8 + # mu <<= 32 + movq %rdx, %rbx + shldq $32, %rcx, %rdx + shldq $32, %rsi, %rcx + shldq $32, %rax, %rsi + shrq $32, %rbx + shlq $32, %rax + # a += (mu << 32) << 64 + addq %rcx, %r11 + adcq %rdx, %r12 + adcq %rbx, %r13 + adcq $0, %r14 + adcq $0, %r15 + sbbq $0, %r8 + # a -= (mu << 32) << 192 + subq %rax, %r11 + movq $4294967295, %rax + sbbq %rsi, %r12 + movq $18446744069414584321, %rsi + sbbq %rcx, %r13 + sbbq %rdx, %r14 + sbbq %rbx, %r15 + adcq $0, %r8 + # mask m and sub from result if overflow + # m[0] = -1 & mask = mask + andq %r8, %rax + # m[2] = 0 & mask = 0 + andq %r8, %rsi + subq %r8, %r12 + sbbq %rax, %r13 + sbbq $0, %r14 + sbbq %rsi, %r15 + movq %r12, (%rdi) + movq %r13, 8(%rdi) + movq %r14, 16(%rdi) + movq %r15, 24(%rdi) + pop %rbx + pop %r15 + pop %r14 + pop %r13 + pop %r12 + repz retq +.size sp_256_mont_sqr_avx2_4,.-sp_256_mont_sqr_avx2_4 +#endif /* HAVE_INTEL_AVX2 */ +#ifdef HAVE_INTEL_AVX2 +#endif /* HAVE_INTEL_AVX2 */ +#ifdef WOLFSSL_SP_SMALL +#ifdef HAVE_INTEL_AVX2 +#endif /* HAVE_INTEL_AVX2 */ +#else +#ifdef HAVE_INTEL_AVX2 +#endif /* HAVE_INTEL_AVX2 */ +#endif /* WOLFSSL_SP_SMALL */ +/* Add 1 to a. (a = a + 1) + * + * a A single precision integer. + */ +.globl sp_256_add_one_4 +.type sp_256_add_one_4,@function +.align 16 +sp_256_add_one_4: + addq $1, (%rdi) + adcq $0, 8(%rdi) + adcq $0, 16(%rdi) + adcq $0, 24(%rdi) + repz retq +.size sp_256_add_one_4,.-sp_256_add_one_4 +/* Add b to a into r. (r = a + b) + * + * r A single precision integer. + * a A single precision integer. + * b A single precision integer. + */ +.globl sp_256_add_4 +.type sp_256_add_4,@function +.align 16 +sp_256_add_4: + xorq %rax, %rax + movq (%rsi), %rcx + addq (%rdx), %rcx + movq %rcx, (%rdi) + movq 8(%rsi), %rcx + adcq 8(%rdx), %rcx + movq %rcx, 8(%rdi) + movq 16(%rsi), %rcx + adcq 16(%rdx), %rcx + movq %rcx, 16(%rdi) + movq 24(%rsi), %rcx + adcq 24(%rdx), %rcx + movq %rcx, 24(%rdi) + adcq $0, %rax + repz retq +.size sp_256_add_4,.-sp_256_add_4 +/* Multiply a and b into r. (r = a * b) + * + * r A single precision integer. + * a A single precision integer. + * b A single precision integer. + */ +.globl sp_256_mul_4 +.type sp_256_mul_4,@function +.align 16 +sp_256_mul_4: + movq %rdx, %rcx + subq $32, %rsp + # A[0] * B[0] + movq (%rcx), %rax + mulq (%rsi) + xorq %r10, %r10 + movq %rax, (%rsp) + movq %rdx, %r9 + # A[0] * B[1] + movq 8(%rcx), %rax + mulq (%rsi) + xorq %r8, %r8 + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[1] * B[0] + movq (%rcx), %rax + mulq 8(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + movq %r9, 8(%rsp) + # A[0] * B[2] + movq 16(%rcx), %rax + mulq (%rsi) + xorq %r9, %r9 + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[1] * B[1] + movq 8(%rcx), %rax + mulq 8(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[2] * B[0] + movq (%rcx), %rax + mulq 16(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + movq %r10, 16(%rsp) + # A[0] * B[3] + movq 24(%rcx), %rax + mulq (%rsi) + xorq %r10, %r10 + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[1] * B[2] + movq 16(%rcx), %rax + mulq 8(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[2] * B[1] + movq 8(%rcx), %rax + mulq 16(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + # A[3] * B[0] + movq (%rcx), %rax + mulq 24(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %r10 + movq %r8, 24(%rsp) + # A[1] * B[3] + movq 24(%rcx), %rax + mulq 8(%rsi) + xorq %r8, %r8 + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[2] * B[2] + movq 16(%rcx), %rax + mulq 16(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + # A[3] * B[1] + movq 8(%rcx), %rax + mulq 24(%rsi) + addq %rax, %r9 + adcq %rdx, %r10 + adcq $0, %r8 + movq %r9, 32(%rdi) + # A[2] * B[3] + movq 24(%rcx), %rax + mulq 16(%rsi) + xorq %r9, %r9 + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + # A[3] * B[2] + movq 16(%rcx), %rax + mulq 24(%rsi) + addq %rax, %r10 + adcq %rdx, %r8 + adcq $0, %r9 + movq %r10, 40(%rdi) + # A[3] * B[3] + movq 24(%rcx), %rax + mulq 24(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + movq %r8, 48(%rdi) + movq %r9, 56(%rdi) + movq (%rsp), %rax + movq 8(%rsp), %rdx + movq 16(%rsp), %r8 + movq 24(%rsp), %r9 + movq %rax, (%rdi) + movq %rdx, 8(%rdi) + movq %r8, 16(%rdi) + movq %r9, 24(%rdi) + addq $32, %rsp + repz retq +.size sp_256_mul_4,.-sp_256_mul_4 +/* Multiply a and b into r. (r = a * b) + * + * r Result of multiplication. + * a First number to multiply. + * b Second number to multiply. + */ +.globl sp_256_mul_avx2_4 +.type sp_256_mul_avx2_4,@function +.align 16 +sp_256_mul_avx2_4: + push %r12 + push %r13 + push %r14 + push %r15 + push %rbx + movq %rdx, %rax + # A[0] * B[0] + movq (%rax), %rdx + mulxq (%rsi), %r9, %r10 + # A[2] * B[0] + mulxq 16(%rsi), %r11, %r12 + # A[1] * B[0] + mulxq 8(%rsi), %rcx, %r8 + xorq %rbx, %rbx + adcxq %rcx, %r10 + # A[1] * B[3] + movq 24(%rax), %rdx + mulxq 8(%rsi), %r13, %r14 + adcxq %r8, %r11 + # A[0] * B[1] + movq 8(%rax), %rdx + mulxq (%rsi), %rcx, %r8 + adoxq %rcx, %r10 + # A[2] * B[1] + mulxq 16(%rsi), %rcx, %r15 + adoxq %r8, %r11 + adcxq %rcx, %r12 + # A[1] * B[2] + movq 16(%rax), %rdx + mulxq 8(%rsi), %rcx, %r8 + adcxq %r15, %r13 + adoxq %rcx, %r12 + adcxq %rbx, %r14 + adoxq %r8, %r13 + # A[0] * B[2] + mulxq (%rsi), %rcx, %r8 + adoxq %rbx, %r14 + xorq %r15, %r15 + adcxq %rcx, %r11 + # A[1] * B[1] + movq 8(%rax), %rdx + mulxq 8(%rsi), %rdx, %rcx + adcxq %r8, %r12 + adoxq %rdx, %r11 + # A[3] * B[1] + movq 8(%rax), %rdx + adoxq %rcx, %r12 + mulxq 24(%rsi), %rcx, %r8 + adcxq %rcx, %r13 + # A[2] * B[2] + movq 16(%rax), %rdx + mulxq 16(%rsi), %rdx, %rcx + adcxq %r8, %r14 + adoxq %rdx, %r13 + # A[3] * B[3] + movq 24(%rax), %rdx + adoxq %rcx, %r14 + mulxq 24(%rsi), %rcx, %r8 + adoxq %rbx, %r15 + adcxq %rcx, %r15 + # A[0] * B[3] + mulxq (%rsi), %rdx, %rcx + adcxq %r8, %rbx + xorq %r8, %r8 + adcxq %rdx, %r12 + # A[3] * B[0] + movq (%rax), %rdx + adcxq %rcx, %r13 + mulxq 24(%rsi), %rdx, %rcx + adoxq %rdx, %r12 + adoxq %rcx, %r13 + # A[2] * B[3] + movq 24(%rax), %rdx + mulxq 16(%rsi), %rdx, %rcx + adcxq %rdx, %r14 + # A[3] * B[2] + movq 16(%rax), %rdx + adcxq %rcx, %r15 + mulxq 24(%rsi), %rcx, %rdx + adcxq %r8, %rbx + adoxq %rcx, %r14 + adoxq %rdx, %r15 + adoxq %r8, %rbx + movq %r9, (%rdi) + movq %r10, 8(%rdi) + movq %r11, 16(%rdi) + movq %r12, 24(%rdi) + movq %r13, 32(%rdi) + movq %r14, 40(%rdi) + movq %r15, 48(%rdi) + movq %rbx, 56(%rdi) + pop %rbx + pop %r15 + pop %r14 + pop %r13 + pop %r12 + repz retq +.size sp_256_mul_avx2_4,.-sp_256_mul_avx2_4 +/* Sub b from a into a. (a -= b) + * + * a A single precision integer and result. + * b A single precision integer. + */ +.globl sp_256_sub_in_place_4 +.type sp_256_sub_in_place_4,@function +.align 16 +sp_256_sub_in_place_4: + xorq %rax, %rax + movq (%rsi), %rdx + movq 8(%rsi), %rcx + movq 16(%rsi), %r8 + movq 24(%rsi), %r9 + subq %rdx, (%rdi) + sbbq %rcx, 8(%rdi) + sbbq %r8, 16(%rdi) + sbbq %r9, 24(%rdi) + sbbq $0, %rax + repz retq +.size sp_256_sub_in_place_4,.-sp_256_sub_in_place_4 +/* Mul a by digit b into r. (r = a * b) + * + * r A single precision integer. + * a A single precision integer. + * b A single precision digit. + */ +.globl sp_256_mul_d_4 +.type sp_256_mul_d_4,@function +.align 16 +sp_256_mul_d_4: + movq %rdx, %rcx + # A[0] * B + movq %rcx, %rax + xorq %r10, %r10 + mulq (%rsi) + movq %rax, %r8 + movq %rdx, %r9 + movq %r8, (%rdi) + # A[1] * B + movq %rcx, %rax + xorq %r8, %r8 + mulq 8(%rsi) + addq %rax, %r9 + movq %r9, 8(%rdi) + adcq %rdx, %r10 + adcq $0, %r8 + # A[2] * B + movq %rcx, %rax + xorq %r9, %r9 + mulq 16(%rsi) + addq %rax, %r10 + movq %r10, 16(%rdi) + adcq %rdx, %r8 + adcq $0, %r9 + # # A[3] * B + movq %rcx, %rax + mulq 24(%rsi) + addq %rax, %r8 + adcq %rdx, %r9 + movq %r8, 24(%rdi) + movq %r9, 32(%rdi) + repz retq +.size sp_256_mul_d_4,.-sp_256_mul_d_4 +#ifdef HAVE_INTEL_AVX2 +/* Mul a by digit b into r. (r = a * b) + * + * r A single precision integer. + * a A single precision integer. + * b A single precision digit. + */ +.globl sp_256_mul_d_avx2_4 +.type sp_256_mul_d_avx2_4,@function +.align 16 +sp_256_mul_d_avx2_4: + movq %rdx, %rax + # A[0] * B + movq %rax, %rdx + xorq %r11, %r11 + mulxq (%rsi), %r9, %r10 + movq %r9, (%rdi) + # A[1] * B + mulxq 8(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + movq %r10, 8(%rdi) + adoxq %r8, %r9 + # A[2] * B + mulxq 16(%rsi), %rcx, %r8 + movq %r11, %r10 + adcxq %rcx, %r9 + movq %r9, 16(%rdi) + adoxq %r8, %r10 + # A[3] * B + mulxq 24(%rsi), %rcx, %r8 + movq %r11, %r9 + adcxq %rcx, %r10 + adoxq %r8, %r9 + adcxq %r11, %r9 + movq %r10, 24(%rdi) + movq %r9, 32(%rdi) + repz retq +.size sp_256_mul_d_avx2_4,.-sp_256_mul_d_avx2_4 +#endif /* HAVE_INTEL_AVX2 */ +/* Square a and put result in r. (r = a * a) + * + * r A single precision integer. + * a A single precision integer. + */ +.globl sp_256_sqr_4 +.type sp_256_sqr_4,@function +.align 16 +sp_256_sqr_4: + push %r12 + subq $32, %rsp + # A[0] * A[0] + movq (%rsi), %rax + mulq %rax + xorq %r9, %r9 + movq %rax, (%rsp) + movq %rdx, %r8 + # A[0] * A[1] + movq 8(%rsi), %rax + mulq (%rsi) + xorq %rcx, %rcx + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %rcx + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %rcx + movq %r8, 8(%rsp) + # A[0] * A[2] + movq 16(%rsi), %rax + mulq (%rsi) + xorq %r8, %r8 + addq %rax, %r9 + adcq %rdx, %rcx + adcq $0, %r8 + addq %rax, %r9 + adcq %rdx, %rcx + adcq $0, %r8 + # A[1] * A[1] + movq 8(%rsi), %rax + mulq %rax + addq %rax, %r9 + adcq %rdx, %rcx + adcq $0, %r8 + movq %r9, 16(%rsp) + # A[0] * A[3] + movq 24(%rsi), %rax + mulq (%rsi) + xorq %r9, %r9 + addq %rax, %rcx + adcq %rdx, %r8 + adcq $0, %r9 + addq %rax, %rcx + adcq %rdx, %r8 + adcq $0, %r9 + # A[1] * A[2] + movq 16(%rsi), %rax + mulq 8(%rsi) + addq %rax, %rcx + adcq %rdx, %r8 + adcq $0, %r9 + addq %rax, %rcx + adcq %rdx, %r8 + adcq $0, %r9 + movq %rcx, 24(%rsp) + # A[1] * A[3] + movq 24(%rsi), %rax + mulq 8(%rsi) + xorq %rcx, %rcx + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %rcx + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %rcx + # A[2] * A[2] + movq 16(%rsi), %rax + mulq %rax + addq %rax, %r8 + adcq %rdx, %r9 + adcq $0, %rcx + movq %r8, 32(%rdi) + # A[2] * A[3] + movq 24(%rsi), %rax + mulq 16(%rsi) + xorq %r8, %r8 + addq %rax, %r9 + adcq %rdx, %rcx + adcq $0, %r8 + addq %rax, %r9 + adcq %rdx, %rcx + adcq $0, %r8 + movq %r9, 40(%rdi) + # A[3] * A[3] + movq 24(%rsi), %rax + mulq %rax + addq %rax, %rcx + adcq %rdx, %r8 + movq %rcx, 48(%rdi) + movq %r8, 56(%rdi) + movq (%rsp), %rax + movq 8(%rsp), %rdx + movq 16(%rsp), %r10 + movq 24(%rsp), %r11 + movq %rax, (%rdi) + movq %rdx, 8(%rdi) + movq %r10, 16(%rdi) + movq %r11, 24(%rdi) + addq $32, %rsp + pop %r12 + repz retq +.size sp_256_sqr_4,.-sp_256_sqr_4 + +/* Square a and put result in r. (r = a * a) + * + * r Result of squaring. + * a Number to square in Montogmery form. + */ +.globl sp_256_sqr_avx2_4 +.type sp_256_sqr_avx2_4,@function +.align 16 +sp_256_sqr_avx2_4: + push %rbx + push %r12 + push %r13 + push %r14 + push %r15 + # A[0] * A[1] + movq (%rsi), %rdx + mulxq 8(%rsi), %r9, %r10 + # A[0] * A[3] + mulxq 24(%rsi), %r11, %r12 + # A[2] * A[1] + movq 16(%rsi), %rdx + mulxq 8(%rsi), %rcx, %rbx + xorq %r15, %r15 + adoxq %rcx, %r11 + # A[2] * A[3] + mulxq 24(%rsi), %r13, %r14 + adoxq %rbx, %r12 + # A[2] * A[0] + mulxq (%rsi), %rcx, %rbx + adoxq %r15, %r13 + adcxq %rcx, %r10 + adoxq %r15, %r14 + # A[1] * A[3] + movq 8(%rsi), %rdx + mulxq 24(%rsi), %rax, %r8 + adcxq %rbx, %r11 + adcxq %rax, %r12 + adcxq %r8, %r13 + adcxq %r15, %r14 + # Double with Carry Flag + xorq %r15, %r15 + # A[0] * A[0] + movq (%rsi), %rdx + mulxq %rdx, %r8, %rax + adcxq %r9, %r9 + # A[1] * A[1] + movq 8(%rsi), %rdx + mulxq %rdx, %rcx, %rbx + adcxq %r10, %r10 + adoxq %rax, %r9 + adcxq %r11, %r11 + adoxq %rcx, %r10 + # A[2] * A[2] + movq 16(%rsi), %rdx + mulxq %rdx, %rax, %rcx + adcxq %r12, %r12 + adoxq %rbx, %r11 + adcxq %r13, %r13 + adoxq %rax, %r12 + # A[3] * A[3] + movq 24(%rsi), %rdx + mulxq %rdx, %rax, %rbx + adcxq %r14, %r14 + adoxq %rcx, %r13 + adcxq %r15, %r15 + adoxq %rax, %r14 + adoxq %rbx, %r15 + movq %r8, (%rdi) + movq %r9, 8(%rdi) + movq %r10, 16(%rdi) + movq %r11, 24(%rdi) + movq %r12, 32(%rdi) + movq %r13, 40(%rdi) + movq %r14, 48(%rdi) + movq %r15, 56(%rdi) + pop %r15 + pop %r14 + pop %r13 + pop %r12 + pop %rbx + repz retq +.size sp_256_sqr_avx2_4,.-sp_256_sqr_avx2_4 +#ifdef HAVE_INTEL_AVX2 +/* Reduce the number back to 256 bits using Montgomery reduction. + * + * a A single precision number to reduce in place. + * m The single precision number representing the modulus. + * mp The digit representing the negative inverse of m mod 2^n. + */ +.globl sp_256_mont_reduce_avx2_4 +.type sp_256_mont_reduce_avx2_4,@function +.align 16 +sp_256_mont_reduce_avx2_4: + push %r12 + push %r13 + push %r14 + push %r15 + push %rbx + movq %rdx, %rax + movq (%rdi), %r12 + movq 8(%rdi), %r13 + movq 16(%rdi), %r14 + movq 24(%rdi), %r15 + xorq %r11, %r11 + xorq %r10, %r10 + # a[0-4] += m[0-3] * mu = m[0-3] * (a[0] * mp) + movq 32(%rdi), %rbx + # mu = a[0] * mp + movq %r12, %rdx + mulxq %rax, %rdx, %rcx + # a[0] += m[0] * mu + mulx (%rsi), %r8, %r9 + adcxq %r8, %r12 + # a[1] += m[1] * mu + mulx 8(%rsi), %r8, %rcx + adoxq %r9, %r13 + adcxq %r8, %r13 + # a[2] += m[2] * mu + mulx 16(%rsi), %r8, %r9 + adoxq %rcx, %r14 + adcxq %r8, %r14 + # a[3] += m[3] * mu + mulx 24(%rsi), %r8, %rcx + adoxq %r9, %r15 + adcxq %r8, %r15 + # a[4] += carry + adoxq %rcx, %rbx + adcxq %r10, %rbx + # carry + adoxq %r10, %r11 + adcxq %r10, %r11 + # a[1-5] += m[0-3] * mu = m[0-3] * (a[1] * mp) + movq 40(%rdi), %r12 + # mu = a[1] * mp + movq %r13, %rdx + mulxq %rax, %rdx, %rcx + # a[1] += m[0] * mu + mulx (%rsi), %r8, %r9 + adcxq %r8, %r13 + # a[2] += m[1] * mu + mulx 8(%rsi), %r8, %rcx + adoxq %r9, %r14 + adcxq %r8, %r14 + # a[3] += m[2] * mu + mulx 16(%rsi), %r8, %r9 + adoxq %rcx, %r15 + adcxq %r8, %r15 + # a[4] += m[3] * mu + mulx 24(%rsi), %r8, %rcx + adoxq %r9, %rbx + adcxq %r8, %rbx + # a[5] += carry + adoxq %rcx, %r12 + adcxq %r11, %r12 + movq %r10, %r11 + # carry + adoxq %r10, %r11 + adcxq %r10, %r11 + # a[2-6] += m[0-3] * mu = m[0-3] * (a[2] * mp) + movq 48(%rdi), %r13 + # mu = a[2] * mp + movq %r14, %rdx + mulxq %rax, %rdx, %rcx + # a[2] += m[0] * mu + mulx (%rsi), %r8, %r9 + adcxq %r8, %r14 + # a[3] += m[1] * mu + mulx 8(%rsi), %r8, %rcx + adoxq %r9, %r15 + adcxq %r8, %r15 + # a[4] += m[2] * mu + mulx 16(%rsi), %r8, %r9 + adoxq %rcx, %rbx + adcxq %r8, %rbx + # a[5] += m[3] * mu + mulx 24(%rsi), %r8, %rcx + adoxq %r9, %r12 + adcxq %r8, %r12 + # a[6] += carry + adoxq %rcx, %r13 + adcxq %r11, %r13 + movq %r10, %r11 + # carry + adoxq %r10, %r11 + adcxq %r10, %r11 + # a[3-7] += m[0-3] * mu = m[0-3] * (a[3] * mp) + movq 56(%rdi), %r14 + # mu = a[3] * mp + movq %r15, %rdx + mulxq %rax, %rdx, %rcx + # a[3] += m[0] * mu + mulx (%rsi), %r8, %r9 + adcxq %r8, %r15 + # a[4] += m[1] * mu + mulx 8(%rsi), %r8, %rcx + adoxq %r9, %rbx + adcxq %r8, %rbx + # a[5] += m[2] * mu + mulx 16(%rsi), %r8, %r9 + adoxq %rcx, %r12 + adcxq %r8, %r12 + # a[6] += m[3] * mu + mulx 24(%rsi), %r8, %rcx + adoxq %r9, %r13 + adcxq %r8, %r13 + # a[7] += carry + adoxq %rcx, %r14 + adcxq %r11, %r14 + movq %r10, %r11 + # carry + adoxq %r10, %r11 + adcxq %r10, %r11 + # Subtract mod if carry + negq %r11 + movq $17562291160714782033, %r8 + movq $13611842547513532036, %r9 + movq $18446744069414584320, %rdx + andq %r11, %r8 + andq %r11, %r9 + movq %r11, %r11 + andq %r11, %rdx + subq %r8, %rbx + sbbq %r9, %r12 + sbbq %r11, %r13 + sbbq %rdx, %r14 + movq %rbx, (%rdi) + movq %r12, 8(%rdi) + movq %r13, 16(%rdi) + movq %r14, 24(%rdi) + pop %rbx + pop %r15 + pop %r14 + pop %r13 + pop %r12 + repz retq +.size sp_256_mont_reduce_avx2_4,.-sp_256_mont_reduce_avx2_4 +#endif /* HAVE_INTEL_AVX2 */ +#endif /* !WOLFSSL_SP_NO_256 */ +#endif /* WOLFSSL_HAVE_SP_ECC */ +#endif /* WOLFSSL_SP_X86_64_ASM */ diff --git a/wolfcrypt/test/test.c b/wolfcrypt/test/test.c index 08ada73e58..7e2c349853 100644 --- a/wolfcrypt/test/test.c +++ b/wolfcrypt/test/test.c @@ -56,8 +56,8 @@ #include #include -#if defined(WC_NO_RNG) && defined(USE_FAST_MATH) - #include +#if defined(WC_NO_RNG) + #include #else #include #endif diff --git a/wolfssl/wolfcrypt/sha256.h b/wolfssl/wolfcrypt/sha256.h index 27c2231f74..0a39d32931 100644 --- a/wolfssl/wolfcrypt/sha256.h +++ b/wolfssl/wolfcrypt/sha256.h @@ -135,9 +135,6 @@ typedef struct wc_Sha256 { word32 loLen; /* length in bytes */ word32 hiLen; /* length in bytes */ void* heap; -#ifdef USE_INTEL_SPEEDUP - const byte* data; -#endif #ifdef WOLFSSL_PIC32MZ_HASH hashUpdCache cache; /* cache for updates */ #endif