diff --git a/IDE/GCC-ARM/README.md b/IDE/GCC-ARM/README.md index f14c11fdca..f865bf99bc 100644 --- a/IDE/GCC-ARM/README.md +++ b/IDE/GCC-ARM/README.md @@ -60,6 +60,7 @@ These settings are located in `Header/user_settings.h`. AES GCM: `GCM_SMALL`, `GCM_WORD32` or `GCM_TABLE`: Tunes performance and flash/memory usage. * `CURVED25519_SMALL`: Enables small versions of Ed/Curve (FE/GE math). * `USE_SLOW_SHA`: Enables smaller/slower version of SHA. -* `USE_SLOW_SHA2`: Over twice as small, but 50% slower +* `USE_SLOW_SHA256`: About 2k smaller and about 25% slower +* `USE_SLOW_SHA512`: Over twice as small, but 50% slower * `USE_CERT_BUFFERS_1024` or `USE_CERT_BUFFERS_2048`: Size of RSA certs / keys to test with. * `BENCH_EMBEDDED`: Define this if using the wolfCrypt test/benchmark and using a low memory target. diff --git a/configure.ac b/configure.ac index cca0bb0e66..6a6edefe49 100644 --- a/configure.ac +++ b/configure.ac @@ -222,6 +222,33 @@ fi AM_CONDITIONAL([BUILD_ALL], [test "x$ENABLED_ALL" = "xyes"]) +# Support for forcing 32-bit mode +AC_ARG_ENABLE([32bit], + [AS_HELP_STRING([--enable-32bit],[Enables 32-bit support (default: disabled)])], + [ ENABLED_32BIT=$enableval ], + [ ENABLED_32BIT=no ] + ) + +if test "$ENABLED_32BIT" = "yes" +then + AM_CFLAGS="$AM_CFLAGS -DNO_64BIT -DNO_CURVED25519_128BIT -m32" + AM_LDFLAGS="$AM_LDFLAGS -m32" +fi + + +# Support for disabling all ASM +AC_ARG_ENABLE([asm], + [AS_HELP_STRING([--enable-asm],[Enables option for assembly (default: enabled)])], + [ ENABLED_ASM=$enableval ], + [ ENABLED_ASM=yes ] + ) + +if test "$ENABLED_ASM" = "no" +then + AM_CFLAGS="$AM_CFLAGS -DTFM_NO_ASM -DWOLFSSL_NO_ASM" +fi + + # SINGLE THREADED AC_ARG_ENABLE([singlethreaded], [AS_HELP_STRING([--enable-singlethreaded],[Enable wolfSSL single threaded (default: disabled)])], @@ -485,7 +512,7 @@ then AM_CFLAGS="$AM_CFLAGS -DHAVE_EXT_CACHE" AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_ALWAYS_VERIFY_CB -DOPENSSL_EXTRA" AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_AES_DIRECT -DWOLFSSL_DER_LOAD" - AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_SHA512 -DWOLFSSL_SHA384 -DWOLFSSL_KEY_GEN" + AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_KEY_GEN" AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_WPAS" fi @@ -504,7 +531,7 @@ fi if test "$ENABLED_FORTRESS" = "yes" then - AM_CFLAGS="$AM_CFLAGS -DFORTRESS -DWOLFSSL_ALWAYS_VERIFY_CB -DOPENSSL_EXTRA -DWOLFSSL_AES_COUNTER -DWOLFSSL_AES_DIRECT -DWOLFSSL_DER_LOAD -DWOLFSSL_SHA512 -DWOLFSSL_SHA384 -DWOLFSSL_KEY_GEN" + AM_CFLAGS="$AM_CFLAGS -DFORTRESS -DWOLFSSL_ALWAYS_VERIFY_CB -DOPENSSL_EXTRA -DWOLFSSL_AES_COUNTER -DWOLFSSL_AES_DIRECT -DWOLFSSL_DER_LOAD -DWOLFSSL_KEY_GEN" fi @@ -531,9 +558,10 @@ AC_ARG_ENABLE([leanpsk], if test "$ENABLED_LEANPSK" = "yes" then - AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_LEANPSK -DWOLFSSL_STATIC_PSK -DHAVE_NULL_CIPHER -DSINGLE_THREADED -DNO_AES -DNO_FILESYSTEM -DNO_RABBIT -DNO_RSA -DNO_DSA -DNO_DH -DNO_CERTS -DNO_PWDBASED -DNO_MD4 -DNO_MD5 -DNO_ERROR_STRINGS -DNO_OLD_TLS -DNO_RC4 -DNO_WRITEV -DNO_SESSION_CACHE -DNO_DEV_RANDOM -DWOLFSSL_USER_IO -DNO_SHA -DUSE_SLOW_SHA" + AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_LEANPSK -DWOLFSSL_STATIC_PSK -DHAVE_NULL_CIPHER -DSINGLE_THREADED -DNO_AES -DNO_FILESYSTEM -DNO_RABBIT -DNO_RSA -DNO_DSA -DNO_DH -DNO_CERTS -DNO_PWDBASED -DNO_MD4 -DNO_MD5 -DNO_ERROR_STRINGS -DNO_OLD_TLS -DNO_RC4 -DNO_WRITEV -DNO_DEV_RANDOM -DWOLFSSL_USER_IO -DNO_SHA" ENABLED_SLOWMATH="no" ENABLED_SINGLETHREADED="yes" + enable_lowresource=yes fi AM_CONDITIONAL([BUILD_LEANPSK], [test "x$ENABLED_LEANPSK" = "xyes"]) @@ -548,12 +576,32 @@ AC_ARG_ENABLE([leantls], if test "$ENABLED_LEANTLS" = "yes" then - AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_LEANTLS -DNO_WRITEV -DHAVE_ECC -DTFM_ECC256 -DECC_USER_CURVES -DNO_WOLFSSL_SERVER -DNO_RABBIT -DNO_RSA -DNO_DSA -DNO_DH -DNO_PWDBASED -DNO_MD5 -DNO_ERROR_STRINGS -DNO_OLD_TLS -DNO_RC4 -DNO_SESSION_CACHE -DNO_SHA -DUSE_SLOW_SHA -DUSE_SLOW_SHA2 -DNO_PSK -DNO_WOLFSSL_MEMORY" + AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_LEANTLS -DNO_WRITEV -DHAVE_ECC -DTFM_ECC256 -DECC_USER_CURVES -DNO_WOLFSSL_SERVER -DNO_RABBIT -DNO_RSA -DNO_DSA -DNO_DH -DNO_PWDBASED -DNO_MD5 -DNO_ERROR_STRINGS -DNO_OLD_TLS -DNO_RC4 -DNO_SHA -DNO_PSK -DNO_WOLFSSL_MEMORY" + enable_lowresource=yes fi AM_CONDITIONAL([BUILD_LEANTLS], [test "x$ENABLED_LEANTLS" = "xyes"]) +# low resource options to reduce flash and memory use +AC_ARG_ENABLE([lowresource], + [AS_HELP_STRING([--enable-lowresource],[Enable low resource options for memory/flash (default: disabled)])], + [ ENABLED_LOWRESOURCE=$enableval ], + [ ENABLED_LOWRESOURCE=no ] + ) + +if test "$ENABLED_LOWRESOURCE" = "yes" +then + # low memory / flash flags + AM_CFLAGS="$AM_CFLAGS -DNO_SESSION_CACHE -DRSA_LOW_MEM -DGCM_SMALL -DCURVE25519_SMALL -DED25519_SMALL" + + # low flash flags + AM_CFLAGS="$AM_CFLAGS -DUSE_SLOW_SHA -DUSE_SLOW_SHA256 -DUSE_SLOW_SHA512" +fi + +AM_CONDITIONAL([BUILD_LOWMEM], [test "x$ENABLED_LOWRESOURCE" = "xyes"]) + + # big cache AC_ARG_ENABLE([bigcache], [AS_HELP_STRING([--enable-bigcache],[Enable big session cache (default: disabled)])], @@ -686,6 +734,7 @@ AS_IF([ test "x$ENABLED_SNIFFER" = "xyes" ], AM_CONDITIONAL([BUILD_SNIFFER], [ test "x$ENABLED_SNIFFER" = "xyes" ]) AM_CONDITIONAL([BUILD_SNIFFTEST], [ test "x$ENABLED_SNIFFTEST" = "xyes" ]) + # AES-GCM AC_ARG_ENABLE([aesgcm], [AS_HELP_STRING([--enable-aesgcm],[Enable wolfSSL AES-GCM support (default: enabled)])], @@ -700,26 +749,26 @@ then ENABLED_AESGCM=no fi -if test "$ENABLED_AESGCM" = "word32" +if test "$ENABLED_AESGCM" != "no" then - AM_CFLAGS="$AM_CFLAGS -DGCM_WORD32" - ENABLED_AESGCM=yes -fi + if test "$ENABLED_AESGCM" = "word32" + then + AM_CFLAGS="$AM_CFLAGS -DGCM_WORD32" + ENABLED_AESGCM=yes + fi -if test "$ENABLED_AESGCM" = "small" -then - AM_CFLAGS="$AM_CFLAGS -DGCM_SMALL" - ENABLED_AESGCM=yes -fi + if test "$ENABLED_AESGCM" = "small" || test "$ENABLED_LOWRESOURCE" = "yes" + then + AM_CFLAGS="$AM_CFLAGS -DGCM_SMALL" + ENABLED_AESGCM=yes + fi -if test "$ENABLED_AESGCM" = "table" -then - AM_CFLAGS="$AM_CFLAGS -DGCM_TABLE" - ENABLED_AESGCM=yes -fi + if test "$ENABLED_AESGCM" = "table" + then + AM_CFLAGS="$AM_CFLAGS -DGCM_TABLE" + ENABLED_AESGCM=yes + fi -if test "$ENABLED_AESGCM" = "yes" -then AM_CFLAGS="$AM_CFLAGS -DHAVE_AESGCM" fi @@ -764,7 +813,7 @@ AC_ARG_ENABLE([armasm], [ ENABLED_ARMASM=$enableval ], [ ENABLED_ARMASM=no ] ) -if test "$ENABLED_ARMASM" = "yes" +if test "$ENABLED_ARMASM" = "yes" && test "$ENABLED_ASM" = "yes" then AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_ARMASM" #Check if mcpu and mfpu values already set if not use default @@ -813,31 +862,34 @@ AC_ARG_ENABLE([intelasm], [ ENABLED_INTELASM=no ] ) -if test "$ENABLED_AESNI" = "small" +if test "$ENABLED_ASM" = "yes" then - AM_CFLAGS="$AM_CFLAGS -DAES_GCM_AESNI_NO_UNROLL" - ENABLED_AESNI=yes -fi - -if test "$ENABLED_AESNI" = "yes" || test "$ENABLED_INTELASM" = "yes" -then - AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_AESNI" - if test "$GCC" = "yes" + if test "$ENABLED_AESNI" = "small" then - # GCC needs these flags, icc doesn't - # opt levels greater than 2 may cause problems on systems w/o aesni - if test "$CC" != "icc" - then - AM_CFLAGS="$AM_CFLAGS -maes -msse4 -mpclmul" - fi + AM_CFLAGS="$AM_CFLAGS -DAES_GCM_AESNI_NO_UNROLL" + ENABLED_AESNI=yes fi - AS_IF([test "x$ENABLED_AESGCM" != "xno"],[AM_CCASFLAGS="$AM_CCASFLAGS -DHAVE_AESGCM"]) -fi -if test "$ENABLED_INTELASM" = "yes" -then - AM_CFLAGS="$AM_CFLAGS -DHAVE_INTEL_RDSEED -DUSE_INTEL_SPEEDUP" - ENABLED_AESNI=yes + if test "$ENABLED_AESNI" = "yes" || test "$ENABLED_INTELASM" = "yes" + then + AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_AESNI" + if test "$GCC" = "yes" + then + # GCC needs these flags, icc doesn't + # opt levels greater than 2 may cause problems on systems w/o aesni + if test "$CC" != "icc" + then + AM_CFLAGS="$AM_CFLAGS -maes -msse4 -mpclmul" + fi + fi + AS_IF([test "x$ENABLED_AESGCM" != "xno"],[AM_CCASFLAGS="$AM_CCASFLAGS -DHAVE_AESGCM"]) + fi + + if test "$ENABLED_INTELASM" = "yes" + then + AM_CFLAGS="$AM_CFLAGS -DHAVE_INTEL_RDSEED -DUSE_INTEL_SPEEDUP" + ENABLED_AESNI=yes + fi fi # INTEL RDRAND @@ -956,28 +1008,23 @@ AC_ARG_ENABLE([sha512], [ ENABLED_SHA512=$SHA512_DEFAULT ] ) -# leanpsk and leantls don't need sha512 +# options that don't require sha512 if test "$ENABLED_LEANPSK" = "yes" || test "$ENABLED_LEANTLS" = "yes" then ENABLED_SHA512=no fi -if test "$ENABLED_OPENSSH" = "yes" +# options that require sha512 +if test "$ENABLED_OPENSSH" = "yes" || test "$ENABLED_WPAS" = "yes" || test "$ENABLED_FORTRESS" = "yes" then ENABLED_SHA512="yes" fi -if test "$ENABLED_SHA512" = "yes" +if test "$ENABLED_SHA512" = "yes" && test "$ENABLED_32BIT" = "no" then AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_SHA512 -DWOLFSSL_SHA384" fi -if test "$ENABLED_FORTRESS" = "yes" -then - ENABLED_SHA512="yes" -fi - - AM_CONDITIONAL([BUILD_SHA512], [test "x$ENABLED_SHA512" = "xyes"]) @@ -1221,21 +1268,21 @@ then ENABLED_CURVE25519="yes" fi -if test "$ENABLED_CURVE25519" = "small" +if test "$ENABLED_CURVE25519" != "no" then - AM_CFLAGS="$AM_CFLAGS -DCURVE25519_SMALL" - ENABLED_CURVE25519_SMALL=yes - ENABLED_CURVE25519=yes -fi + if test "$ENABLED_CURVE25519" = "small" || test "$ENABLED_LOWRESOURCE" = "yes" + then + AM_CFLAGS="$AM_CFLAGS -DCURVE25519_SMALL" + ENABLED_CURVE25519_SMALL=yes + ENABLED_CURVE25519=yes + fi -if test "$ENABLED_CURVE25519" = "no128bit" -then - AM_CFLAGS="$AM_CFLAGS -DNO_CURVED25519_128BIT" - ENABLED_CURVE25519=yes -fi + if test "$ENABLED_CURVE25519" = "no128bit" + then + AM_CFLAGS="$AM_CFLAGS -DNO_CURVED25519_128BIT" + ENABLED_CURVE25519=yes + fi -if test "$ENABLED_CURVE25519" = "yes" -then AM_CFLAGS="$AM_CFLAGS -DHAVE_CURVE25519" ENABLED_FEMATH=yes fi @@ -1257,15 +1304,15 @@ then ENABLED_ED25519="yes" fi -if test "$ENABLED_ED25519" = "small" +if test "$ENABLED_ED25519" != "no" && test "$ENABLED_32BIT" = "no" then - AM_CFLAGS="$AM_CFLAGS -DED25519_SMALL" - ENABLED_ED25519_SMALL=yes - ENABLED_ED25519=yes -fi + if test "$ENABLED_ED25519" = "small" || test "$ENABLED_LOWRESOURCE" = "yes" + then + AM_CFLAGS="$AM_CFLAGS -DED25519_SMALL" + ENABLED_ED25519_SMALL=yes + ENABLED_ED25519=yes + fi -if test "$ENABLED_ED25519" = "yes" -then if test "$ENABLED_SHA512" = "no" then AC_MSG_ERROR([cannot enable ed25519 without enabling sha512.]) @@ -1366,6 +1413,19 @@ else fi +# TLSv1.0 +AC_ARG_ENABLE([tlsv10], + [AS_HELP_STRING([--enable-tlsv10],[Enable old TLS versions 1.0 (default: disabled)])], + [ ENABLED_TLSV10=$enableval ], + [ ENABLED_TLSV10=no ] + ) + +if test "$ENABLED_TLSV10" = "yes" +then + AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_ALLOW_TLSV10" +fi + + # SSLv3 AC_ARG_ENABLE([sslv3], [AS_HELP_STRING([--enable-sslv3],[Enable SSL version 3.0 (default: disabled)])], @@ -1922,7 +1982,7 @@ then ENABLED_SHA3="yes" fi -if test "$ENABLED_SHA3" = "yes" +if test "$ENABLED_SHA3" = "yes" && test "$ENABLED_32BIT" = "no" then AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_SHA3" fi @@ -1951,7 +2011,7 @@ then ENABLED_POLY1305=no fi -if test "$ENABLED_POLY1305" = "yes" +if test "$ENABLED_POLY1305" = "yes" && test "$ENABLED_32BIT" = "no" then AM_CFLAGS="$AM_CFLAGS -DHAVE_POLY1305 -DHAVE_ONE_TIME_AUTH" fi @@ -1969,24 +2029,22 @@ fi # CHACHA AC_ARG_ENABLE([chacha], - [AS_HELP_STRING([--enable-chacha],[Enable CHACHA (default: enabled). Use `=noasm` to disable Intel AVX/AVX2 speedups])], + [AS_HELP_STRING([--enable-chacha],[Enable CHACHA (default: enabled). Use `=noasm` to disable ASM AVX/AVX2 speedups])], [ ENABLED_CHACHA=$enableval ], [ ENABLED_CHACHA=$CHACHA_DEFAULT] ) -if test "$ENABLED_AESNI" = "noasm" -then - AM_CFLAGS="$AM_CFLAGS -DNO_CHACHA_ASM" - ENABLED_AESNI=yes -fi - - # leanpsk and leantls don't need chacha if test "$ENABLED_LEANPSK" = "yes" || test "$ENABLED_LEANTLS" = "yes" then ENABLED_CHACHA=no fi +if test "$ENABLED_CHACHA" = "noasm" || test "$ENABLED_ASM" = "no" +then + AM_CFLAGS="$AM_CFLAGS -DNO_CHACHA_ASM" +fi + if test "$ENABLED_CHACHA" = "yes" then AM_CFLAGS="$AM_CFLAGS -DHAVE_CHACHA" @@ -3600,7 +3658,6 @@ then fi - # check if PSK was enabled for conditionally running psk.test script AM_CONDITIONAL([BUILD_PSK], [test "x$ENABLED_PSK" = "xyes"]) @@ -3664,6 +3721,16 @@ AS_IF([test "x$ENABLED_PKCS7" = "xyes" && \ test "x$ENABLED_SHA" = "xno"], [AC_MSG_ERROR([please enable sha if enabling pkcs7.])]) +AS_IF([test "x$ENABLED_PKCS7" = "xyes" && \ + test "x$ENABLED_AES" = "xno" && \ + test "x$ENABLED_DES3" = "xno"], + [AC_MSG_ERROR([please enable either AES or 3DES if enabling pkcs7.])]) + +AS_IF([test "x$ENABLED_WOLFSCEP" = "xyes" && \ + test "x$ENABLED_AES" = "xno" && \ + test "x$ENABLED_DES3" = "xno"], + [AC_MSG_ERROR([please enable either AES or 3DES if enabling scep.])]) + AS_IF([test "x$ENABLED_LEANTLS" = "xyes" && \ test "x$ENABLED_ECC" = "xno"], [AC_MSG_ERROR([please enable ecc if enabling leantls.])]) @@ -3733,8 +3800,6 @@ then ENABLED_X963KDF="yes" AM_CFLAGS="$AM_CFLAGS -DHAVE_X963_KDF" fi - AS_IF([test "x$ENABLED_DES3" = "xno"], - [ENABLED_DES3=yes]) fi if test "x$ENABLED_DES3" = "xno" @@ -4016,6 +4081,7 @@ echo " * Old Names: $ENABLED_OLDNAMES" echo " * Max Strength Build: $ENABLED_MAXSTRENGTH" echo " * Distro Build: $ENABLED_DISTRO" echo " * fastmath: $ENABLED_FASTMATH" +echo " * Assembly Allowed: $ENABLED_ASM" echo " * sniffer: $ENABLED_SNIFFER" echo " * snifftest: $ENABLED_SNIFFTEST" echo " * ARC4: $ENABLED_ARC4" @@ -4078,6 +4144,7 @@ echo " * SCTP: $ENABLED_SCTP" echo " * Multicast: $ENABLED_MCAST" echo " * Old TLS Versions: $ENABLED_OLD_TLS" echo " * SSL version 3.0: $ENABLED_SSLV3" +echo " * TLS v1.0: $ENABLED_TLSV10" echo " * TLS v1.3: $ENABLED_TLS13" echo " * TLS v1.3 Draft 18: $ENABLED_TLS13_DRAFT18" echo " * Post-handshake Auth: $ENABLED_TLS13_POST_AUTH" diff --git a/cyassl/ssl.h b/cyassl/ssl.h index 000221b1d2..c2100e11e6 100644 --- a/cyassl/ssl.h +++ b/cyassl/ssl.h @@ -445,7 +445,6 @@ #define CyaSSL_CTX_use_PrivateKey_file wolfSSL_CTX_use_PrivateKey_file #define CyaSSL_CTX_use_certificate_file wolfSSL_CTX_use_certificate_file #define CyaSSL_CTX_use_PrivateKey_buffer wolfSSL_CTX_use_PrivateKey_buffer -#define CyaSSL_CTX_use_PrivateKey_buffer wolfSSL_CTX_use_PrivateKey_buffer #define CyaSSL_CTX_use_certificate_buffer wolfSSL_CTX_use_certificate_buffer #define CyaSSL_CTX_use_NTRUPrivateKey_file wolfSSL_CTX_use_NTRUPrivateKey_file #define CyaSSL_use_certificate_chain_buffer wolfSSL_use_certificate_chain_buffer diff --git a/examples/client/client.c b/examples/client/client.c index e6a32d45a8..036c890cb9 100644 --- a/examples/client/client.c +++ b/examples/client/client.c @@ -170,12 +170,19 @@ static void ShowCiphers(void) static void ShowVersions(void) { #ifndef NO_OLD_TLS -#ifdef WOLFSSL_ALLOW_SSLV3 - printf("0:"); -#endif /* WOLFSSL_ALLOW_SSLV3 */ - printf("1:2:"); + #ifdef WOLFSSL_ALLOW_SSLV3 + printf("0:"); + #endif + #ifdef WOLFSSL_ALLOW_TLSV10 + printf("1:"); + #endif + printf("2:"); #endif /* NO_OLD_TLS */ - printf("3\n"); + printf("3:"); +#ifdef WOLFSSL_TLS13 + printf("4:"); +#endif + printf("\n"); } /* Measures average time to create, connect and disconnect a connection (TPS). @@ -684,7 +691,9 @@ static void Usage(void) #endif printf("-m Match domain name in cert\n"); printf("-N Use Non-blocking sockets\n"); +#ifndef NO_SESSION_CACHE printf("-r Resume session\n"); +#endif printf("-w Wait for bidirectional shutdown\n"); printf("-M Use STARTTLS, using protocol (smtp)\n"); #ifdef HAVE_SECURE_RENEGOTIATION @@ -1392,27 +1401,29 @@ THREAD_RETURN WOLFSSL_THREAD client_test(void* args) #endif #ifndef NO_TLS + #ifdef WOLFSSL_ALLOW_TLSV10 case 1: method = wolfTLSv1_client_method_ex; break; + #endif case 2: method = wolfTLSv1_1_client_method_ex; break; - #endif /* NO_TLS */ - -#endif /* NO_OLD_TLS */ + #endif /* !NO_TLS */ +#endif /* !NO_OLD_TLS */ #ifndef NO_TLS case 3: method = wolfTLSv1_2_client_method_ex; break; + #ifdef WOLFSSL_TLS13 case 4: method = wolfTLSv1_3_client_method_ex; break; #endif -#endif +#endif /* NO_TLS */ #ifdef WOLFSSL_DTLS #ifndef NO_OLD_TLS diff --git a/examples/echoclient/echoclient.c b/examples/echoclient/echoclient.c index 0cd781d95a..d4cfcf3541 100644 --- a/examples/echoclient/echoclient.c +++ b/examples/echoclient/echoclient.c @@ -124,7 +124,7 @@ void echoclient_test(void* args) #if defined(CYASSL_DTLS) method = DTLSv1_2_client_method(); -#elif !defined(NO_TLS) +#elif !defined(NO_TLS) method = CyaSSLv23_client_method(); #elif defined(WOLFSSL_ALLOW_SSLV3) method = SSLv3_client_method(); diff --git a/examples/echoserver/echoserver.c b/examples/echoserver/echoserver.c index ffa97f84ad..5bcb2014eb 100644 --- a/examples/echoserver/echoserver.c +++ b/examples/echoserver/echoserver.c @@ -143,7 +143,7 @@ THREAD_RETURN CYASSL_THREAD echoserver_test(void* args) #if defined(CYASSL_DTLS) method = CyaDTLSv1_2_server_method(); -#elif !defined(NO_TLS) +#elif !defined(NO_TLS) method = CyaSSLv23_server_method(); #elif defined(WOLFSSL_ALLOW_SSLV3) method = CyaSSLv3_server_method(); diff --git a/examples/server/server.c b/examples/server/server.c index 08c0cf0000..3be4008cd9 100644 --- a/examples/server/server.c +++ b/examples/server/server.c @@ -877,29 +877,29 @@ THREAD_RETURN CYASSL_THREAD server_test(void* args) #endif #ifndef NO_TLS + #ifdef WOLFSSL_ALLOW_TLSV10 case 1: method = wolfTLSv1_server_method_ex; break; - + #endif case 2: method = wolfTLSv1_1_server_method_ex; break; - - #endif -#endif + #endif /* !NO_TLS */ +#endif /* !NO_OLD_TLS */ #ifndef NO_TLS case 3: method = wolfTLSv1_2_server_method_ex; break; -#endif -#ifdef WOLFSSL_TLS13 + #ifdef WOLFSSL_TLS13 case 4: method = wolfTLSv1_3_server_method_ex; break; -#endif + #endif +#endif /* NO_TLS */ #ifdef CYASSL_DTLS #ifndef NO_OLD_TLS diff --git a/scripts/resume.test b/scripts/resume.test index 5badb4838b..f948fb5687 100755 --- a/scripts/resume.test +++ b/scripts/resume.test @@ -1,10 +1,11 @@ #!/bin/sh -#reusme.test +#resume.test # need a unique resume port since may run the same time as testsuite # use server port zero hack to get one resume_string="reused" +resume_sup_string="Resume session" ems_string="Extended\ Master\ Secret" resume_port=0 no_pid=-1 @@ -45,6 +46,18 @@ do_trap() { do_test() { echo -e "\nStarting example server for resume test...\n" + #make sure we support session resumption (!NO_SESSION_CACHE) + # Check the client for the extended master secret disable option. If + # present we need to run the test twice. + options_check=`./examples/client/client -?` + case "$options_check" in + *$resume_sup_string*) + echo -e "\nResume test supported";; + *) + echo -e "\nResume test not supported with build" + return;; + esac + remove_ready_file ./examples/server/server -r -R $ready_file -p $resume_port & server_pid=$! diff --git a/src/internal.c b/src/internal.c index c45aa2bb45..9d70b25088 100644 --- a/src/internal.c +++ b/src/internal.c @@ -11965,10 +11965,16 @@ int ProcessReply(WOLFSSL* ssl) ssl->keys.decryptedCur = 1; #ifdef WOLFSSL_TLS13 if (ssl->options.tls1_3) { + word16 i = ssl->buffers.inputBuffer.length - + ssl->keys.padSz; + /* Remove padding from end of plain text. */ + for (--i; i > ssl->buffers.inputBuffer.idx; i--) { + if (ssl->buffers.inputBuffer.buffer[i] != 0) + break; + } /* Get the real content type from the end of the data. */ - ssl->keys.padSz++; - ssl->curRL.type = ssl->buffers.inputBuffer.buffer[ - ssl->buffers.inputBuffer.length - ssl->keys.padSz]; + ssl->curRL.type = ssl->buffers.inputBuffer.buffer[i]; + ssl->keys.padSz = ssl->buffers.inputBuffer.length - i; } #endif } @@ -13237,7 +13243,7 @@ int SendCertificateRequest(WOLFSSL* ssl) /* supported hash/sig */ if (IsAtLeastTLSv1_2(ssl)) { c16toa(ssl->suites->hashSigAlgoSz, &output[i]); - i += LENGTH_SZ; + i += OPAQUE16_LEN; XMEMCPY(&output[i], ssl->suites->hashSigAlgo, ssl->suites->hashSigAlgoSz); @@ -14200,9 +14206,6 @@ const char* wolfSSL_ERR_reason_error_string(unsigned long e) case NOT_READY_ERROR : return "handshake layer not ready yet, complete first"; - case PMS_VERSION_ERROR : - return "premaster secret version mismatch error"; - case VERSION_ERROR : return "record layer version error"; @@ -14487,6 +14490,9 @@ const char* wolfSSL_ERR_reason_error_string(unsigned long e) case INVALID_PARAMETER: return "The security parameter is invalid"; + case UNSUPPORTED_EXTENSION: + return "TLS Extension not requested by the client"; + case KEY_SHARE_ERROR: return "Key share extension did not contain a valid named group"; @@ -18767,8 +18773,10 @@ int SendClientKeyExchange(WOLFSSL* ssl) #ifndef NO_RSA case rsa_kea: { + /* build PreMasterSecret with RNG data */ ret = wc_RNG_GenerateBlock(ssl->rng, - ssl->arrays->preMasterSecret, SECRET_LEN); + &ssl->arrays->preMasterSecret[VERSION_SZ], + SECRET_LEN - VERSION_SZ); if (ret != 0) { goto exit_scke; } @@ -22762,18 +22770,25 @@ static int DoSessionTicket(WOLFSSL* ssl, const byte* input, word32* inOutIdx, return BUFFER_ERROR; if (extId == HELLO_EXT_SIG_ALGO) { - ato16(&input[i], &clSuites.hashSigAlgoSz); + word16 hashSigAlgoSz; + + ato16(&input[i], &hashSigAlgoSz); i += OPAQUE16_LEN; - if (OPAQUE16_LEN + clSuites.hashSigAlgoSz > extSz) + if (OPAQUE16_LEN + hashSigAlgoSz > extSz) return BUFFER_ERROR; - XMEMCPY(clSuites.hashSigAlgo, &input[i], - min(clSuites.hashSigAlgoSz, HELLO_EXT_SIGALGO_MAX)); - i += clSuites.hashSigAlgoSz; + clSuites.hashSigAlgoSz = hashSigAlgoSz; + if (clSuites.hashSigAlgoSz > WOLFSSL_MAX_SIGALGO) { + WOLFSSL_MSG("ClientHello SigAlgo list exceeds max, " + "truncating"); + clSuites.hashSigAlgoSz = WOLFSSL_MAX_SIGALGO; + } - if (clSuites.hashSigAlgoSz > HELLO_EXT_SIGALGO_MAX) - clSuites.hashSigAlgoSz = HELLO_EXT_SIGALGO_MAX; + XMEMCPY(clSuites.hashSigAlgo, &input[i], + clSuites.hashSigAlgoSz); + + i += hashSigAlgoSz; } #ifdef HAVE_EXTENDED_MASTER else if (extId == HELLO_EXT_EXTMS) @@ -23261,6 +23276,9 @@ static int DoSessionTicket(WOLFSSL* ssl, const byte* input, word32* inOutIdx, #ifdef WOLFSSL_TLS13 word32 ageAdd; /* Obfuscation of age */ byte namedGroup; /* Named group used */ + #ifndef WOLFSSL_TLS13_DRAFT_18 + TicketNonce ticketNonce; /* Ticket nonce */ + #endif #ifdef WOLFSSL_EARLY_DATA word32 maxEarlyDataSz; /* Max size of early data */ #endif @@ -23316,6 +23334,10 @@ static int DoSessionTicket(WOLFSSL* ssl, const byte* input, word32* inOutIdx, it.timestamp = TimeNowInMilliseconds(); /* Resumption master secret. */ XMEMCPY(it.msecret, ssl->session.masterSecret, SECRET_LEN); + #ifndef WOLFSSL_TLS13_DRAFT_18 + XMEMCPY(&it.ticketNonce, &ssl->session.ticketNonce, + sizeof(TicketNonce)); + #endif #endif } @@ -23428,6 +23450,10 @@ static int DoSessionTicket(WOLFSSL* ssl, const byte* input, word32* inOutIdx, #endif /* Resumption master secret. */ XMEMCPY(ssl->session.masterSecret, it->msecret, SECRET_LEN); + #ifndef WOLFSSL_TLS13_DRAFT_18 + XMEMCPY(&ssl->session.ticketNonce, &it->ticketNonce, + sizeof(TicketNonce)); + #endif ssl->session.namedGroup = it->namedGroup; #endif } @@ -23559,6 +23585,9 @@ static int DoSessionTicket(WOLFSSL* ssl, const byte* input, word32* inOutIdx, word32 idx; word32 begin; word32 sigSz; + #ifndef NO_RSA + int lastErr; + #endif } DckeArgs; static void FreeDckeArgs(WOLFSSL* ssl, void* pArgs) @@ -23784,6 +23813,14 @@ static int DoSessionTicket(WOLFSSL* ssl, const byte* input, word32* inOutIdx, ERROR_OUT(BUFFER_ERROR, exit_dcke); } + /* pre-load PreMasterSecret with RNG data */ + ret = wc_RNG_GenerateBlock(ssl->rng, + &ssl->arrays->preMasterSecret[VERSION_SZ], + SECRET_LEN - VERSION_SZ); + if (ret != 0) { + goto exit_dcke; + } + args->output = NULL; break; } /* rsa_kea */ @@ -24248,6 +24285,20 @@ static int DoSessionTicket(WOLFSSL* ssl, const byte* input, word32* inOutIdx, NULL, 0, NULL #endif ); + + /* Errors that can occur here that should be + * indistinguishable: + * RSA_BUFFER_E, RSA_PAD_E and RSA_PRIVATE_ERROR + */ + if (ret < 0 && ret != BAD_FUNC_ARG) { + #ifdef WOLFSSL_ASYNC_CRYPT + if (ret == WC_PENDING_E) + goto exit_dcke; + #endif + /* store error code for handling below */ + args->lastErr = ret; + ret = 0; + } break; } /* rsa_kea */ #endif /* !NO_RSA */ @@ -24394,16 +24445,42 @@ static int DoSessionTicket(WOLFSSL* ssl, const byte* input, word32* inOutIdx, /* Add the signature length to idx */ args->idx += args->length; - if (args->sigSz == SECRET_LEN && args->output != NULL) { - XMEMCPY(ssl->arrays->preMasterSecret, args->output, SECRET_LEN); - if (ssl->arrays->preMasterSecret[0] != ssl->chVersion.major || - ssl->arrays->preMasterSecret[1] != ssl->chVersion.minor) { - ERROR_OUT(PMS_VERSION_ERROR, exit_dcke); + #ifdef DEBUG_WOLFSSL + /* check version (debug warning message only) */ + if (args->output != NULL) { + if (args->output[0] != ssl->chVersion.major || + args->output[1] != ssl->chVersion.minor) { + WOLFSSL_MSG("preMasterSecret version mismatch"); } } - else { - ERROR_OUT(RSA_PRIVATE_ERROR, exit_dcke); + #endif + + /* RFC5246 7.4.7.1: + * Treat incorrectly formatted message blocks and/or + * mismatched version numbers in a manner + * indistinguishable from correctly formatted RSA blocks + */ + + ret = args->lastErr; + args->lastErr = 0; /* reset */ + + /* build PreMasterSecret */ + ssl->arrays->preMasterSecret[0] = ssl->chVersion.major; + ssl->arrays->preMasterSecret[1] = ssl->chVersion.minor; + if (ret == 0 && args->sigSz == SECRET_LEN && + args->output != NULL) { + XMEMCPY(&ssl->arrays->preMasterSecret[VERSION_SZ], + &args->output[VERSION_SZ], + SECRET_LEN - VERSION_SZ); } + else { + /* preMasterSecret has RNG and version set */ + /* return proper length and ignore error */ + /* error will be caught as decryption error */ + args->sigSz = SECRET_LEN; + ret = 0; + } + break; } /* rsa_kea */ #endif /* !NO_RSA */ diff --git a/src/sniffer.c b/src/sniffer.c index 0ab7e282cd..3d8aa17e08 100644 --- a/src/sniffer.c +++ b/src/sniffer.c @@ -1288,7 +1288,7 @@ static int SetNamedPrivateKey(const char* name, const char* address, int port, sniffer->server = serverIp; sniffer->port = port; - sniffer->ctx = SSL_CTX_new(TLSv1_client_method()); + sniffer->ctx = SSL_CTX_new(TLSv1_2_client_method()); if (!sniffer->ctx) { SetError(MEMORY_STR, error, NULL, 0); #ifdef HAVE_SNI diff --git a/src/ssl.c b/src/ssl.c index 53678cb93f..e2be1bda8a 100644 --- a/src/ssl.c +++ b/src/ssl.c @@ -288,16 +288,17 @@ WOLFSSL_CTX* wolfSSL_CTX_new_ex(WOLFSSL_METHOD* method, void* heap) wolfSSL_CTX_free(ctx); ctx = NULL; } - #if defined(OPENSSL_EXTRA) && defined(WOLFCRYPT_HAVE_SRP) \ - && !defined(NO_SHA256) && !defined(WC_NO_RNG) - ctx->srp = (Srp*) XMALLOC(sizeof(Srp), heap, DYNAMIC_TYPE_SRP); - if (ctx->srp == NULL){ - WOLFSSL_MSG("Init CTX failed"); - wolfSSL_CTX_free(ctx); - return NULL; + && !defined(NO_SHA256) && !defined(WC_NO_RNG) + else { + ctx->srp = (Srp*) XMALLOC(sizeof(Srp), heap, DYNAMIC_TYPE_SRP); + if (ctx->srp == NULL){ + WOLFSSL_MSG("Init CTX failed"); + wolfSSL_CTX_free(ctx); + return NULL; + } + XMEMSET(ctx->srp, 0, sizeof(Srp)); } - XMEMSET(ctx->srp, 0, sizeof(Srp)); #endif } else { @@ -3614,6 +3615,32 @@ int wolfSSL_SetMinVersion(WOLFSSL* ssl, int version) } +/* Function to get version as WOLFSSL_ enum value for wolfSSL_SetVersion */ +int wolfSSL_GetVersion(WOLFSSL* ssl) +{ + if (ssl == NULL) + return BAD_FUNC_ARG; + + if (ssl->version.major == SSLv3_MAJOR) { + switch (ssl->version.minor) { + case SSLv3_MINOR : + return WOLFSSL_SSLV3; + case TLSv1_MINOR : + return WOLFSSL_TLSV1; + case TLSv1_1_MINOR : + return WOLFSSL_TLSV1_1; + case TLSv1_2_MINOR : + return WOLFSSL_TLSV1_2; + case TLSv1_3_MINOR : + return WOLFSSL_TLSV1_3; + default: + break; + } + } + + return VERSION_ERROR; +} + int wolfSSL_SetVersion(WOLFSSL* ssl, int version) { word16 haveRSA = 1; @@ -3636,9 +3663,11 @@ int wolfSSL_SetVersion(WOLFSSL* ssl, int version) #ifndef NO_TLS #ifndef NO_OLD_TLS + #ifdef WOLFSSL_ALLOW_TLSV10 case WOLFSSL_TLSV1: ssl->version = MakeTLSv1(); break; + #endif case WOLFSSL_TLSV1_1: ssl->version = MakeTLSv1_1(); @@ -3647,6 +3676,12 @@ int wolfSSL_SetVersion(WOLFSSL* ssl, int version) case WOLFSSL_TLSV1_2: ssl->version = MakeTLSv1_2(); break; +#endif +#ifdef WOLFSSL_TLS13 + case WOLFSSL_TLSV1_3: + ssl->version = MakeTLSv1_3(); + break; + #endif default: @@ -4562,9 +4597,9 @@ int PemToDer(const unsigned char* buff, long longSz, int type, #if defined(OPENSSL_EXTRA) || defined(HAVE_WEBSERVER) { /* remove encrypted header if there */ - char encHeader[] = "Proc-Type"; + const char* const encHeader = "Proc-Type"; word32 headerEndSz = (word32)(bufferEnd - headerEnd); - char* line = XSTRNSTR(headerEnd, encHeader, min(headerEndSz, + char* line = XSTRNSTR(headerEnd, encHeader, min(headerEndSz, PEM_LINE_LEN)); if (line != NULL) { word32 lineSz; @@ -6587,8 +6622,7 @@ int wolfSSL_PemCertToDer(const char* fileName, unsigned char* derBuf, int derSz) #endif /* WOLFSSL_CERT_GEN */ -#ifdef WOLFSSL_CERT_EXT -#ifndef NO_FILESYSTEM +#if defined(WOLFSSL_CERT_EXT) || defined(WOLFSSL_PUB_PEM_TO_DER) /* load pem public key from file into der buffer, return der size or error */ int wolfSSL_PemPubKeyToDer(const char* fileName, unsigned char* derBuf, int derSz) @@ -6655,42 +6689,7 @@ int wolfSSL_PemPubKeyToDer(const char* fileName, return ret; } -#endif /* NO_FILESYSTEM */ - -/* Return bytes written to buff or < 0 for error */ -int wolfSSL_PubKeyPemToDer(const unsigned char* pem, int pemSz, - unsigned char* buff, int buffSz) -{ - int ret; - DerBuffer* der = NULL; - - WOLFSSL_ENTER("wolfSSL_PubKeyPemToDer"); - - if (pem == NULL || buff == NULL || buffSz <= 0) { - WOLFSSL_MSG("Bad pem der args"); - return BAD_FUNC_ARG; - } - - ret = PemToDer(pem, pemSz, PUBLICKEY_TYPE, &der, NULL, NULL, NULL); - if (ret < 0) { - WOLFSSL_MSG("Bad Pem To Der"); - } - else { - if (der->length <= (word32)buffSz) { - XMEMCPY(buff, der->buffer, der->length); - ret = der->length; - } - else { - WOLFSSL_MSG("Bad der length"); - ret = BAD_FUNC_ARG; - } - } - - FreeDer(&der); - return ret; -} - -#endif /* WOLFSSL_CERT_EXT */ +#endif /* WOLFSSL_CERT_EXT || WOLFSSL_PUB_PEM_TO_DER */ int wolfSSL_CTX_use_certificate_file(WOLFSSL_CTX* ctx, const char* file, int format) @@ -7503,6 +7502,42 @@ int wolfSSL_CTX_use_NTRUPrivateKey_file(WOLFSSL_CTX* ctx, const char* file) #endif /* NO_FILESYSTEM */ +#if defined(WOLFSSL_CERT_EXT) || defined(WOLFSSL_PUB_PEM_TO_DER) +/* Return bytes written to buff or < 0 for error */ +int wolfSSL_PubKeyPemToDer(const unsigned char* pem, int pemSz, + unsigned char* buff, int buffSz) +{ + int ret; + DerBuffer* der = NULL; + + WOLFSSL_ENTER("wolfSSL_PubKeyPemToDer"); + + if (pem == NULL || buff == NULL || buffSz <= 0) { + WOLFSSL_MSG("Bad pem der args"); + return BAD_FUNC_ARG; + } + + ret = PemToDer(pem, pemSz, PUBLICKEY_TYPE, &der, NULL, NULL, NULL); + if (ret < 0) { + WOLFSSL_MSG("Bad Pem To Der"); + } + else { + if (der->length <= (word32)buffSz) { + XMEMCPY(buff, der->buffer, der->length); + ret = der->length; + } + else { + WOLFSSL_MSG("Bad der length"); + ret = BAD_FUNC_ARG; + } + } + + FreeDer(&der); + return ret; +} +#endif /* WOLFSSL_CERT_EXT || WOLFSSL_PUB_PEM_TO_DER */ + + void wolfSSL_CTX_set_verify(WOLFSSL_CTX* ctx, int mode, VerifyCallback vc) { WOLFSSL_ENTER("wolfSSL_CTX_set_verify"); @@ -9754,6 +9789,10 @@ static int GetDeepCopySession(WOLFSSL* ssl, WOLFSSL_SESSION* copyFrom) copyInto->namedGroup = copyFrom->namedGroup; copyInto->ticketSeen = copyFrom->ticketSeen; copyInto->ticketAdd = copyFrom->ticketAdd; +#ifndef WOLFSSL_TLS13_DRAFT_18 + XMEMCPY(©Into->ticketNonce, ©From->ticketNonce, + sizeof(TicketNonce)); +#endif #ifdef WOLFSSL_EARLY_DATA copyInto->maxEarlyDataSz = copyFrom->maxEarlyDataSz; #endif @@ -9986,6 +10025,10 @@ int AddSession(WOLFSSL* ssl) session->namedGroup = ssl->session.namedGroup; session->ticketSeen = ssl->session.ticketSeen; session->ticketAdd = ssl->session.ticketAdd; +#ifndef WOLFSSL_TLS13_DRAFT_18 + XMEMCPY(&session->ticketNonce, &ssl->session.ticketNonce, + sizeof(TicketNonce)); +#endif #ifdef WOLFSSL_EARLY_DATA session->maxEarlyDataSz = ssl->session.maxEarlyDataSz; #endif @@ -11177,7 +11220,7 @@ int wolfSSL_set_compression(WOLFSSL* ssl) int wolfSSL_CTX_set_srp_username(WOLFSSL_CTX* ctx, char* username) { int r = 0; - int srp_side = 0; + SrpSide srp_side = SRP_CLIENT_SIDE; WC_RNG rng; byte salt[SRP_SALT_SIZE]; @@ -11290,9 +11333,8 @@ int wolfSSL_set_compression(WOLFSSL* ssl) if (ctx->srp_password != NULL) XFREE(ctx->srp_password,ctx->heap, DYNAMIC_TYPE_SRP); - ctx->srp_password = XMALLOC(XSTRLEN(password) + 1, - ctx->heap, - DYNAMIC_TYPE_SRP); + ctx->srp_password = (byte*)XMALLOC(XSTRLEN(password) + 1, ctx->heap, + DYNAMIC_TYPE_SRP); if (ctx->srp_password == NULL){ WOLFSSL_MSG("memory allocation error"); return SSL_FAILURE; @@ -16574,7 +16616,7 @@ void wolfSSL_X509_STORE_CTX_set_time(WOLFSSL_X509_STORE_CTX* ctx, { (void)flags; - ctx->param->check_time = t; + ctx->param->check_time = t; ctx->param->flags |= WOLFSSL_USE_CHECK_TIME; } #endif @@ -18331,7 +18373,6 @@ void wolfSSL_BN_CTX_free(WOLFSSL_BN_CTX* ctx) static void InitwolfSSL_BigNum(WOLFSSL_BIGNUM* bn) { - WOLFSSL_MSG("InitwolfSSL_BigNum"); if (bn) { bn->neg = 0; bn->internal = NULL; @@ -18873,18 +18914,18 @@ char *wolfSSL_BN_bn2dec(const WOLFSSL_BIGNUM *bn) return NULL; } - if (mp_radix_size((mp_int*)bn->internal, 10, &len) != MP_OKAY) { + if (mp_radix_size((mp_int*)bn->internal, MP_RADIX_DEC, &len) != MP_OKAY) { WOLFSSL_MSG("mp_radix_size failure"); return NULL; } buf = (char*) XMALLOC(len, NULL, DYNAMIC_TYPE_ECC); if (buf == NULL) { - WOLFSSL_MSG("wolfSSL_BN_bn2hex malloc buffer failure"); + WOLFSSL_MSG("BN_bn2dec malloc buffer failure"); return NULL; } - if (mp_toradix((mp_int*)bn->internal, buf, 10) != MP_OKAY) { + if (mp_todecimal((mp_int*)bn->internal, buf) != MP_OKAY) { XFREE(buf, NULL, DYNAMIC_TYPE_ECC); return NULL; } @@ -19042,36 +19083,41 @@ WOLFSSL_BN_ULONG wolfSSL_BN_mod_word(const WOLFSSL_BIGNUM *bn, } #endif /* #ifdef WOLFSSL_KEY_GEN */ -#if defined(WOLFSSL_KEY_GEN) || defined(HAVE_COMP_KEY) char *wolfSSL_BN_bn2hex(const WOLFSSL_BIGNUM *bn) { +#if defined(WOLFSSL_KEY_GEN) || defined(HAVE_COMP_KEY) || defined(DEBUG_WOLFSSL) int len = 0; char *buf; - WOLFSSL_MSG("wolfSSL_BN_bn2hex"); + WOLFSSL_ENTER("wolfSSL_BN_bn2hex"); if (bn == NULL || bn->internal == NULL) { WOLFSSL_MSG("bn NULL error"); return NULL; } - if (mp_radix_size((mp_int*)bn->internal, 16, &len) != MP_OKAY) { + if (mp_radix_size((mp_int*)bn->internal, MP_RADIX_HEX, &len) != MP_OKAY) { WOLFSSL_MSG("mp_radix_size failure"); return NULL; } buf = (char*) XMALLOC(len, NULL, DYNAMIC_TYPE_ECC); if (buf == NULL) { - WOLFSSL_MSG("wolfSSL_BN_bn2hex malloc buffer failure"); + WOLFSSL_MSG("BN_bn2hex malloc buffer failure"); return NULL; } - if (mp_toradix((mp_int*)bn->internal, buf, 16) != MP_OKAY) { + if (mp_tohex((mp_int*)bn->internal, buf) != MP_OKAY) { XFREE(buf, NULL, DYNAMIC_TYPE_ECC); return NULL; } return buf; +#else + (void)bn; + WOLFSSL_MSG("wolfSSL_BN_bn2hex not compiled in"); + return (char*)""; +#endif } #ifndef NO_FILESYSTEM @@ -19080,9 +19126,10 @@ char *wolfSSL_BN_bn2hex(const WOLFSSL_BIGNUM *bn) */ int wolfSSL_BN_print_fp(FILE *fp, const WOLFSSL_BIGNUM *bn) { +#if defined(WOLFSSL_KEY_GEN) || defined(HAVE_COMP_KEY) || defined(DEBUG_WOLFSSL) char *buf; - WOLFSSL_MSG("wolfSSL_BN_print_fp"); + WOLFSSL_ENTER("wolfSSL_BN_print_fp"); if (fp == NULL || bn == NULL || bn->internal == NULL) { WOLFSSL_MSG("bn NULL error"); @@ -19099,36 +19146,17 @@ int wolfSSL_BN_print_fp(FILE *fp, const WOLFSSL_BIGNUM *bn) XFREE(buf, NULL, DYNAMIC_TYPE_ECC); return WOLFSSL_SUCCESS; -} -#endif /* !defined(NO_FILESYSTEM) */ - -#else /* defined(WOLFSSL_KEY_GEN) || defined(HAVE_COMP_KEY) */ - -char *wolfSSL_BN_bn2hex(const WOLFSSL_BIGNUM *bn) -{ - (void)bn; - - WOLFSSL_MSG("wolfSSL_BN_bn2hex need WOLFSSL_KEY_GEN or HAVE_COMP_KEY"); - - return (char*)""; -} - -#ifndef NO_FILESYSTEM -/* return code compliant with OpenSSL : - * 1 if success, 0 if error - */ -int wolfSSL_BN_print_fp(FILE *fp, const WOLFSSL_BIGNUM *bn) -{ +#else (void)fp; (void)bn; - WOLFSSL_MSG("wolfSSL_BN_print_fp not implemented"); + WOLFSSL_MSG("wolfSSL_BN_print_fp not compiled in"); return WOLFSSL_SUCCESS; +#endif } -#endif /* !defined(NO_FILESYSTEM) */ +#endif /* !NO_FILESYSTEM */ -#endif /* defined(WOLFSSL_KEY_GEN) || defined(HAVE_COMP_KEY) */ WOLFSSL_BIGNUM *wolfSSL_BN_CTX_get(WOLFSSL_BN_CTX *ctx) { @@ -20092,7 +20120,7 @@ int wolfSSL_RSA_generate_key_ex(WOLFSSL_RSA* rsa, int bits, WOLFSSL_BIGNUM* bn, if (wc_InitRng(rng) < 0) WOLFSSL_MSG("RNG init failed"); else if (wc_MakeRsaKey((RsaKey*)rsa->internal, - bits, 65537, rng) != MP_OKAY) + bits, WC_RSA_EXPONENT, rng) != MP_OKAY) WOLFSSL_MSG("wc_MakeRsaKey failed"); else if (SetRsaExternal(rsa) != WOLFSSL_SUCCESS) WOLFSSL_MSG("SetRsaExternal failed"); @@ -22182,7 +22210,7 @@ int wolfSSL_EC_GROUP_get_order(const WOLFSSL_EC_GROUP *group, } if (mp_read_radix((mp_int*)order->internal, - ecc_sets[group->curve_idx].order, 16) != MP_OKAY) { + ecc_sets[group->curve_idx].order, MP_RADIX_HEX) != MP_OKAY) { WOLFSSL_MSG("wolfSSL_EC_GROUP_get_order mp_read order failure"); mp_clear((mp_int*)order->internal); return WOLFSSL_FAILURE; @@ -22364,9 +22392,9 @@ int wolfSSL_EC_POINT_mul(const WOLFSSL_EC_GROUP *group, WOLFSSL_EC_POINT *r, return WOLFSSL_FAILURE; } - ret = mp_read_radix(&prime, ecc_sets[group->curve_idx].prime, 16); + ret = mp_read_radix(&prime, ecc_sets[group->curve_idx].prime, MP_RADIX_HEX); if (ret == MP_OKAY) { - ret = mp_read_radix(&a, ecc_sets[group->curve_idx].Af, 16); + ret = mp_read_radix(&a, ecc_sets[group->curve_idx].Af, MP_RADIX_HEX); } /* r = q * m % prime */ diff --git a/src/tls.c b/src/tls.c index 2177d9b652..f8b5a3f3de 100644 --- a/src/tls.c +++ b/src/tls.c @@ -428,6 +428,7 @@ int BuildTlsFinished(WOLFSSL* ssl, Hashes* hashes, const byte* sender) #ifndef NO_OLD_TLS +#ifdef WOLFSSL_ALLOW_TLSV10 ProtocolVersion MakeTLSv1(void) { ProtocolVersion pv; @@ -436,6 +437,7 @@ ProtocolVersion MakeTLSv1(void) return pv; } +#endif /* WOLFSSL_ALLOW_TLSV10 */ ProtocolVersion MakeTLSv1_1(void) @@ -447,7 +449,7 @@ ProtocolVersion MakeTLSv1_1(void) return pv; } -#endif +#endif /* !NO_OLD_TLS */ ProtocolVersion MakeTLSv1_2(void) @@ -952,8 +954,9 @@ static int TLSX_Push(TLSX** list, TLSX_Type type, void* data, void* heap) TLSX_FreeAll(next, heap); - /* there is no way to occur more than */ - /* two extensions of the same type. */ + /* there is no way to occur more than + * two extensions of the same type. + */ break; } } while ((extension = extension->next)); @@ -961,6 +964,35 @@ static int TLSX_Push(TLSX** list, TLSX_Type type, void* data, void* heap) return 0; } +#ifndef NO_WOLFSSL_CLIENT + +int TLSX_CheckUnsupportedExtension(WOLFSSL* ssl, TLSX_Type type); + +int TLSX_CheckUnsupportedExtension(WOLFSSL* ssl, TLSX_Type type) +{ + TLSX *extension = TLSX_Find(ssl->extensions, type); + + if (!extension) + extension = TLSX_Find(ssl->ctx->extensions, type); + + return extension == NULL; +} + +int TLSX_HandleUnsupportedExtension(WOLFSSL* ssl); + +int TLSX_HandleUnsupportedExtension(WOLFSSL* ssl) +{ + SendAlert(ssl, alert_fatal, unsupported_extension); + return UNSUPPORTED_EXTENSION; +} + +#else + +#define TLSX_CheckUnsupportedExtension(ssl, type) 0 +#define TLSX_HandleUnsupportedExtension(ssl) 0 + +#endif + #ifndef NO_WOLFSSL_SERVER /** Mark an extension to be sent back to the client. */ @@ -968,10 +1000,10 @@ void TLSX_SetResponse(WOLFSSL* ssl, TLSX_Type type); void TLSX_SetResponse(WOLFSSL* ssl, TLSX_Type type) { - TLSX *ext = TLSX_Find(ssl->extensions, type); + TLSX *extension = TLSX_Find(ssl->extensions, type); - if (ext) - ext->resp = 1; + if (extension) + extension->resp = 1; } #endif @@ -1161,7 +1193,7 @@ static int TLSX_ALPN_ParseAndSet(WOLFSSL *ssl, byte *input, word16 length, ssl->alpnSelectArg) == 0) { WOLFSSL_MSG("ALPN protocol match"); if (TLSX_UseALPN(&ssl->extensions, (char*)out, outLen, 0, ssl->heap) - == WOLFSSL_SUCCESS) { + == WOLFSSL_SUCCESS) { if (extension == NULL) { extension = TLSX_Find(ssl->extensions, TLSX_APPLICATION_LAYER_PROTOCOL); @@ -1172,9 +1204,8 @@ static int TLSX_ALPN_ParseAndSet(WOLFSSL *ssl, byte *input, word16 length, #endif if (extension == NULL || extension->data == NULL) { - WOLFSSL_MSG("No ALPN extensions not used or bad"); - return isRequest ? 0 /* not using ALPN */ - : BUFFER_ERROR; /* unexpected ALPN response */ + return isRequest ? 0 + : TLSX_HandleUnsupportedExtension(ssl); } /* validating alpn list length */ @@ -1475,12 +1506,10 @@ static word16 TLSX_SNI_Write(SNI* list, byte* output) return offset; } -#ifndef NO_WOLFSSL_SERVER - /** Finds a SNI object in the provided list. */ static SNI* TLSX_SNI_Find(SNI *list, byte type) { - SNI *sni = list; + SNI* sni = list; while (sni && sni->type != type) sni = sni->next; @@ -1488,7 +1517,6 @@ static SNI* TLSX_SNI_Find(SNI *list, byte type) return sni; } - /** Sets the status of a SNI object. */ static void TLSX_SNI_SetStatus(TLSX* extensions, byte type, byte status) { @@ -1511,8 +1539,6 @@ byte TLSX_SNI_Status(TLSX* extensions, byte type) return 0; } -#endif /* NO_WOLFSSL_SERVER */ - /** Parses a buffer of SNI extensions. */ static int TLSX_SNI_Parse(WOLFSSL* ssl, byte* input, word16 length, byte isRequest) @@ -1528,27 +1554,44 @@ static int TLSX_SNI_Parse(WOLFSSL* ssl, byte* input, word16 length, if (!extension) extension = TLSX_Find(ssl->ctx->extensions, TLSX_SERVER_NAME); - (void)isRequest; - (void)input; + if (!isRequest) { + #ifndef NO_WOLFSSL_CLIENT + if (!extension || !extension->data) + return TLSX_HandleUnsupportedExtension(ssl); - if (!extension || !extension->data) { -#if defined(WOLFSSL_ALWAYS_KEEP_SNI) && !defined(NO_WOLFSSL_SERVER) - /* This will keep SNI even though TLSX_UseSNI has not been called. - * Enable it so that the received sni is available to functions - * that use a custom callback when SNI is received */ - cacheOnly = 1; - WOLFSSL_MSG("Forcing SSL object to store SNI parameter"); -#else - return isRequest ? 0 /* not using SNI. */ - : BUFFER_ERROR; /* unexpected SNI response. */ -#endif + if (length > 0) + return BUFFER_ERROR; /* SNI response MUST be empty. */ + + /* This call enables wolfSSL_SNI_GetRequest() to be called in the + * client side to fetch the used SNI. It will only work if the SNI + * was set at the SSL object level. Right now we only support one + * name type, WOLFSSL_SNI_HOST_NAME, but in the future, the + * inclusion of other name types will turn this method inaccurate, + * as the extension response doesn't contains information of which + * name was accepted. + */ + TLSX_SNI_SetStatus(ssl->extensions, WOLFSSL_SNI_HOST_NAME, + WOLFSSL_SNI_REAL_MATCH); + + return 0; + #endif } - if (!isRequest) - return length ? BUFFER_ERROR /* SNI response MUST be empty. */ - : 0; /* nothing else to do. */ - #ifndef NO_WOLFSSL_SERVER + if (!extension || !extension->data) { + #if defined(WOLFSSL_ALWAYS_KEEP_SNI) && !defined(NO_WOLFSSL_SERVER) + /* This will keep SNI even though TLSX_UseSNI has not been called. + * Enable it so that the received sni is available to functions + * that use a custom callback when SNI is received. + */ + + cacheOnly = 1; + WOLFSSL_MSG("Forcing SSL object to store SNI parameter"); + #else + /* Skipping, SNI not enabled at server side. */ + return 0; + #endif + } if (OPAQUE16_LEN > length) return BUFFER_ERROR; @@ -1587,9 +1630,9 @@ static int TLSX_SNI_Parse(WOLFSSL* ssl, byte* input, word16 length, break; #endif byte matched = cacheOnly || - ((XSTRLEN(sni->data.host_name) == size) - && (XSTRNCMP(sni->data.host_name, - (const char*)input + offset, size) == 0)); + ((XSTRLEN(sni->data.host_name) == size) && + (XSTRNCMP(sni->data.host_name, + (const char*)input + offset, size) == 0)); if (matched || sni->options & WOLFSSL_SNI_ANSWER_ON_MISMATCH) { int r = TLSX_UseSNI(&ssl->extensions, @@ -1623,7 +1666,8 @@ static int TLSX_SNI_Parse(WOLFSSL* ssl, byte* input, word16 length, } } } - +#else + (void)input; #endif return 0; @@ -1678,7 +1722,7 @@ int TLSX_UseSNI(TLSX** extensions, byte type, const void* data, word16 size, void* heap) { TLSX* extension; - SNI* sni = NULL; + SNI* sni = NULL; if (extensions == NULL || data == NULL) return BAD_FUNC_ARG; @@ -1689,6 +1733,7 @@ int TLSX_UseSNI(TLSX** extensions, byte type, const void* data, word16 size, extension = TLSX_Find(*extensions, TLSX_SERVER_NAME); if (!extension) { int ret = TLSX_Push(extensions, TLSX_SERVER_NAME, (void*)sni, heap); + if (ret != 0) { TLSX_SNI_Free(sni, heap); return ret; @@ -1702,13 +1747,14 @@ int TLSX_UseSNI(TLSX** extensions, byte type, const void* data, word16 size, /* remove duplicate SNI, there should be only one of each type. */ do { if (sni->next && sni->next->type == type) { - SNI *next = sni->next; + SNI* next = sni->next; sni->next = next->next; TLSX_SNI_Free(next, heap); - /* there is no way to occur more than */ - /* two SNIs of the same type. */ + /* there is no way to occur more than + * two SNIs of the same type. + */ break; } } while ((sni = sni->next)); @@ -1753,8 +1799,8 @@ int TLSX_SNI_GetFromBuffer(const byte* clientHello, word32 helloSz, byte type, byte* sni, word32* inOutSz) { word32 offset = 0; - word32 len32 = 0; - word16 len16 = 0; + word32 len32 = 0; + word16 len16 = 0; if (helloSz < RECORD_HEADER_SZ + HANDSHAKE_HEADER_SZ + CLIENT_HELLO_FIRST) return INCOMPLETE_DATA; @@ -1933,11 +1979,17 @@ static word16 TLSX_MFL_Write(byte* data, byte* output) static int TLSX_MFL_Parse(WOLFSSL* ssl, byte* input, word16 length, byte isRequest) { - (void)isRequest; - if (length != ENUM_LEN) return BUFFER_ERROR; +#ifdef WOLFSSL_OLD_UNSUPPORTED_EXTENSION + (void) isRequest; +#else + if (!isRequest) + if (TLSX_CheckUnsupportedExtension(ssl, TLSX_MAX_FRAGMENT_LENGTH)) + return TLSX_HandleUnsupportedExtension(ssl); +#endif + switch (*input) { case WOLFSSL_MFL_2_9 : ssl->max_fragment = 512; break; case WOLFSSL_MFL_2_10: ssl->max_fragment = 1024; break; @@ -1953,9 +2005,10 @@ static int TLSX_MFL_Parse(WOLFSSL* ssl, byte* input, word16 length, #ifndef NO_WOLFSSL_SERVER if (isRequest) { - int r = TLSX_UseMaxFragment(&ssl->extensions, *input, ssl->heap); + int ret = TLSX_UseMaxFragment(&ssl->extensions, *input, ssl->heap); - if (r != WOLFSSL_SUCCESS) return r; /* throw error */ + if (ret != WOLFSSL_SUCCESS) + return ret; /* throw error */ TLSX_SetResponse(ssl, TLSX_MAX_FRAGMENT_LENGTH); } @@ -1967,22 +2020,19 @@ static int TLSX_MFL_Parse(WOLFSSL* ssl, byte* input, word16 length, int TLSX_UseMaxFragment(TLSX** extensions, byte mfl, void* heap) { byte* data = NULL; - int ret = 0; + int ret = 0; - if (extensions == NULL) + if (extensions == NULL || mfl < WOLFSSL_MFL_2_9 || WOLFSSL_MFL_2_13 < mfl) return BAD_FUNC_ARG; - if (mfl < WOLFSSL_MFL_2_9 || WOLFSSL_MFL_2_13 < mfl) - return BAD_FUNC_ARG; - - if ((data = (byte*)XMALLOC(ENUM_LEN, heap, DYNAMIC_TYPE_TLSX)) == NULL) + data = (byte*)XMALLOC(ENUM_LEN, heap, DYNAMIC_TYPE_TLSX); + if (data == NULL) return MEMORY_E; data[0] = mfl; - /* push new MFL extension. */ - if ((ret = TLSX_Push(extensions, TLSX_MAX_FRAGMENT_LENGTH, data, heap)) - != 0) { + ret = TLSX_Push(extensions, TLSX_MAX_FRAGMENT_LENGTH, data, heap); + if (ret != 0) { XFREE(data, heap, DYNAMIC_TYPE_TLSX); return ret; } @@ -2014,21 +2064,25 @@ int TLSX_UseMaxFragment(TLSX** extensions, byte mfl, void* heap) static int TLSX_THM_Parse(WOLFSSL* ssl, byte* input, word16 length, byte isRequest) { - (void)isRequest; - if (length != 0 || input == NULL) return BUFFER_ERROR; -#ifndef NO_WOLFSSL_SERVER - if (isRequest) { - int r = TLSX_UseTruncatedHMAC(&ssl->extensions, ssl->heap); + if (!isRequest) { + #ifndef WOLFSSL_OLD_UNSUPPORTED_EXTENSION + if (TLSX_CheckUnsupportedExtension(ssl, TLSX_TRUNCATED_HMAC)) + return TLSX_HandleUnsupportedExtension(ssl); + #endif + } + else { + #ifndef NO_WOLFSSL_SERVER + int ret = TLSX_UseTruncatedHMAC(&ssl->extensions, ssl->heap); - if (r != WOLFSSL_SUCCESS) - return r; /* throw error */ + if (ret != WOLFSSL_SUCCESS) + return ret; /* throw error */ - TLSX_SetResponse(ssl, TLSX_TRUNCATED_HMAC); + TLSX_SetResponse(ssl, TLSX_TRUNCATED_HMAC); + #endif } -#endif ssl->truncated_hmac = 1; @@ -2042,7 +2096,8 @@ int TLSX_UseTruncatedHMAC(TLSX** extensions, void* heap) if (extensions == NULL) return BAD_FUNC_ARG; - if ((ret = TLSX_Push(extensions, TLSX_TRUNCATED_HMAC, NULL, heap)) != 0) + ret = TLSX_Push(extensions, TLSX_TRUNCATED_HMAC, NULL, heap); + if (ret != 0) return ret; return WOLFSSL_SUCCESS; @@ -2156,8 +2211,8 @@ static int TLSX_CSR_Parse(WOLFSSL* ssl, byte* input, word16 length, extension = TLSX_Find(ssl->ctx->extensions, TLSX_STATUS_REQUEST); csr = extension ? (CertificateStatusRequest*)extension->data : NULL; - if (!csr) - return BUFFER_ERROR; /* unexpected extension */ + if (!csr) /* unexpected extension */ + return TLSX_HandleUnsupportedExtension(ssl); /* enable extension at ssl level */ ret = TLSX_UseCertificateStatusRequest(&ssl->extensions, @@ -2518,8 +2573,8 @@ static int TLSX_CSR2_Parse(WOLFSSL* ssl, byte* input, word16 length, csr2 = extension ? (CertificateStatusRequestItemV2*)extension->data : NULL; - if (!csr2) - return BUFFER_ERROR; /* unexpected extension */ + if (!csr2) /* unexpected extension */ + return TLSX_HandleUnsupportedExtension(ssl); /* enable extension at ssl level */ for (; csr2; csr2 = csr2->next) { @@ -2881,38 +2936,44 @@ static void TLSX_PointFormat_FreeAll(PointFormat* list, void* heap) static int TLSX_SupportedCurve_Append(SupportedCurve* list, word16 name, void* heap) { - if (list == NULL) - return BAD_FUNC_ARG; + int ret = BAD_FUNC_ARG; - while (1) { - if (list->name == name) - return 0; /* curve alreay in use */ + while (list) { + if (list->name == name) { + ret = 0; /* curve alreay in use */ + break; + } - if (list->next == NULL) - return TLSX_SupportedCurve_New(&list->next, name, heap); + if (list->next == NULL) { + ret = TLSX_SupportedCurve_New(&list->next, name, heap); + break; + } list = list->next; } - return 0; + return ret; } static int TLSX_PointFormat_Append(PointFormat* list, byte format, void* heap) { - if (list == NULL) - return BAD_FUNC_ARG; + int ret = BAD_FUNC_ARG; - while (1) { - if (list->format == format) - return 0; /* format already in use */ + while (list) { + if (list->format == format) { + ret = 0; /* format already in use */ + break; + } - if (list->next == NULL) - return TLSX_PointFormat_New(&list->next, format, heap); + if (list->next == NULL) { + ret = TLSX_PointFormat_New(&list->next, format, heap); + break; + } list = list->next; } - return 0; + return ret; } #ifndef NO_WOLFSSL_CLIENT @@ -3032,7 +3093,8 @@ static int TLSX_SupportedCurve_Parse(WOLFSSL* ssl, byte* input, word16 length, word16 name; int ret; - (void) isRequest; /* shut up compiler! */ + if(!isRequest && !IsAtLeastTLSv1_3(ssl->version)) + return BUFFER_ERROR; /* servers doesn't send this extension. */ if (OPAQUE16_LEN > length || length % OPAQUE16_LEN) return BUFFER_ERROR; @@ -3047,7 +3109,6 @@ static int TLSX_SupportedCurve_Parse(WOLFSSL* ssl, byte* input, word16 length, ato16(input + offset, &name); ret = TLSX_UseSupportedCurve(&ssl->extensions, name, ssl->heap); - if (ret != WOLFSSL_SUCCESS) return ret; /* throw error */ } @@ -3718,7 +3779,9 @@ static int TLSX_SessionTicket_Parse(WOLFSSL* ssl, byte* input, word16 length, (void) input; /* avoid unused parameter if NO_WOLFSSL_SERVER defined */ if (!isRequest) { - /* client side */ + if (TLSX_CheckUnsupportedExtension(ssl, TLSX_SESSION_TICKET)) + return TLSX_HandleUnsupportedExtension(ssl); + if (length != 0) return BUFFER_ERROR; @@ -4853,8 +4916,13 @@ static int TLSX_SignatureAlgorithms_Parse(WOLFSSL *ssl, byte* input, if (length != OPAQUE16_LEN + len) return BUFFER_ERROR; - XMEMCPY(suites->hashSigAlgo, input, len); + /* truncate hashSigAlgo list if too long */ suites->hashSigAlgoSz = len; + if (suites->hashSigAlgoSz > WOLFSSL_MAX_SIGALGO) { + WOLFSSL_MSG("TLSX SigAlgo list exceeds max, truncating"); + suites->hashSigAlgoSz = WOLFSSL_MAX_SIGALGO; + } + XMEMCPY(suites->hashSigAlgo, input, suites->hashSigAlgoSz); return TLSX_SignatureAlgorithms_MapPss(ssl, input, len); } @@ -6461,11 +6529,11 @@ static INLINE byte GetHmacLength(int hmac) case sha256_mac: return WC_SHA256_DIGEST_SIZE; #endif - #ifndef NO_SHA384 + #ifdef WOLFSSL_SHA384 case sha384_mac: return WC_SHA384_DIGEST_SIZE; #endif - #ifndef NO_SHA512 + #ifdef WOLFSSL_SHA512 case sha512_mac: return WC_SHA512_DIGEST_SIZE; #endif @@ -8100,10 +8168,11 @@ word16 TLSX_GetResponseSize(WOLFSSL* ssl, byte msgType) XMEMSET(semaphore, 0xff, SEMAPHORE_SIZE); TURN_OFF(semaphore, TLSX_ToSemaphore(TLSX_KEY_SHARE)); TURN_OFF(semaphore, TLSX_ToSemaphore(TLSX_COOKIE)); -#endif break; +#endif #ifdef WOLFSSL_TLS13 case encrypted_extensions: + TURN_ON(semaphore, TLSX_ToSemaphore(TLSX_EC_POINT_FORMATS)); TURN_ON(semaphore, TLSX_ToSemaphore(TLSX_SESSION_TICKET)); TURN_ON(semaphore, TLSX_ToSemaphore(TLSX_KEY_SHARE)); #if defined(HAVE_SESSION_TICKET) || !defined(NO_PSK) @@ -8186,10 +8255,11 @@ word16 TLSX_WriteResponse(WOLFSSL *ssl, byte* output, byte msgType) case hello_retry_request: XMEMSET(semaphore, 0xff, SEMAPHORE_SIZE); TURN_OFF(semaphore, TLSX_ToSemaphore(TLSX_KEY_SHARE)); -#endif break; +#endif #ifdef WOLFSSL_TLS13 case encrypted_extensions: + TURN_ON(semaphore, TLSX_ToSemaphore(TLSX_EC_POINT_FORMATS)); TURN_ON(semaphore, TLSX_ToSemaphore(TLSX_SESSION_TICKET)); TURN_ON(semaphore, TLSX_ToSemaphore(TLSX_KEY_SHARE)); #if defined(HAVE_SESSION_TICKET) || !defined(NO_PSK) @@ -8569,18 +8639,12 @@ int TLSX_Parse(WOLFSSL* ssl, byte* input, word16 length, byte msgType, #ifndef NO_WOLFSSL_CLIENT #ifndef NO_OLD_TLS - + #ifdef WOLFSSL_ALLOW_TLSV10 WOLFSSL_METHOD* wolfTLSv1_client_method(void) { return wolfTLSv1_client_method_ex(NULL); } - - WOLFSSL_METHOD* wolfTLSv1_1_client_method(void) - { - return wolfTLSv1_1_client_method_ex(NULL); - } - WOLFSSL_METHOD* wolfTLSv1_client_method_ex(void* heap) { WOLFSSL_METHOD* method = @@ -8590,7 +8654,12 @@ int TLSX_Parse(WOLFSSL* ssl, byte* input, word16 length, byte msgType, InitSSL_Method(method, MakeTLSv1()); return method; } + #endif /* WOLFSSL_ALLOW_TLSV10 */ + WOLFSSL_METHOD* wolfTLSv1_1_client_method(void) + { + return wolfTLSv1_1_client_method_ex(NULL); + } WOLFSSL_METHOD* wolfTLSv1_1_client_method_ex(void* heap) { @@ -8687,18 +8756,12 @@ int TLSX_Parse(WOLFSSL* ssl, byte* input, word16 length, byte msgType, #ifndef NO_WOLFSSL_SERVER #ifndef NO_OLD_TLS - + #ifdef WOLFSSL_ALLOW_TLSV10 WOLFSSL_METHOD* wolfTLSv1_server_method(void) { return wolfTLSv1_server_method_ex(NULL); } - - WOLFSSL_METHOD* wolfTLSv1_1_server_method(void) - { - return wolfTLSv1_1_server_method_ex(NULL); - } - WOLFSSL_METHOD* wolfTLSv1_server_method_ex(void* heap) { WOLFSSL_METHOD* method = @@ -8710,7 +8773,12 @@ int TLSX_Parse(WOLFSSL* ssl, byte* input, word16 length, byte msgType, } return method; } + #endif /* WOLFSSL_ALLOW_TLSV10 */ + WOLFSSL_METHOD* wolfTLSv1_1_server_method(void) + { + return wolfTLSv1_1_server_method_ex(NULL); + } WOLFSSL_METHOD* wolfTLSv1_1_server_method_ex(void* heap) { diff --git a/src/tls13.c b/src/tls13.c index 250c709605..cd07c9d4dd 100644 --- a/src/tls13.c +++ b/src/tls13.c @@ -169,6 +169,7 @@ static int Tls13_HKDF_Extract(byte* prk, const byte* salt, int saltLen, * TLS v1.3 defines this function. * * okm The generated pseudorandom key - output key material. + * okmLen The length of generated pseudorandom key - output key material. * prk The salt - pseudo-random key. * prkLen The length of the salt - pseudo-random key. * protocol The TLS protocol label. @@ -808,6 +809,62 @@ static int DeriveMasterSecret(WOLFSSL* ssl) #endif } +#ifndef WOLFSSL_TLS13_DRAFT_18 +#if defined(HAVE_SESSION_TICKET) +/* Length of the resumption label. */ +#define RESUMPTION_LABEL_SZ 10 +/* Resumption label for generating PSK assocated with the ticket. */ +static const byte resumptionLabel[RESUMPTION_LABEL_SZ+1] = "resumption"; +/* Derive the PSK assocated with the ticket. + * + * ssl The SSL/TLS object. + * nonce The nonce to derive with. + * nonceLen The length of the nonce to derive with. + * secret The derived secret. + * returns 0 on success, otherwise failure. + */ +static int DeriveResumptionPSK(WOLFSSL* ssl, byte* nonce, byte nonceLen, + byte* secret) +{ + int digestAlg; + /* Only one protocol version defined at this time. */ + const byte* protocol = tls13ProtocolLabel; + word32 protocolLen = TLS13_PROTOCOL_LABEL_SZ; + + WOLFSSL_MSG("Derive Resumption PSK"); + + switch (ssl->specs.mac_algorithm) { + #ifndef NO_SHA256 + case sha256_mac: + digestAlg = WC_SHA256; + break; + #endif + + #ifdef WOLFSSL_SHA384 + case sha384_mac: + digestAlg = WC_SHA256; + break; + #endif + + #ifdef WOLFSSL_TLS13_TLS13_SHA512 + case sha512_mac: + digestAlg = WC_SHA256; + break; + #endif + + default: + return BAD_FUNC_ARG; + } + + return HKDF_Expand_Label(secret, ssl->specs.hash_size, + ssl->session.masterSecret, ssl->specs.hash_size, + protocol, protocolLen, resumptionLabel, + RESUMPTION_LABEL_SZ, nonce, nonceLen, digestAlg); +} +#endif /* HAVE_SESSION_TICKET */ +#endif /* WOLFSSL_TLS13_DRAFT_18 */ + + /* Calculate the HMAC of message data to this point. * * ssl The SSL/TLS object. @@ -1430,7 +1487,7 @@ static INLINE void BuildTls13Nonce(WOLFSSL* ssl, byte* nonce, const byte* iv, nonce[i] ^= iv[i]; } -#ifdef HAVE_CHACHA +#if defined(HAVE_CHACHA) && defined(HAVE_POLY1305) /* Encrypt with ChaCha20 and create authenication tag with Poly1305. * * ssl The SSL/TLS object. @@ -1630,7 +1687,7 @@ static int EncryptTls13(WOLFSSL* ssl, byte* output, const byte* input, return ret; } -#ifdef HAVE_CHACHA +#if defined(HAVE_CHACHA) && defined(HAVE_POLY1305) /* Decrypt with ChaCha20 and check authenication tag with Poly1305. * * ssl The SSL/TLS object. @@ -2055,8 +2112,15 @@ static int SetupPskKey(WOLFSSL* ssl, PreSharedKey* psk) #endif /* Resumption PSK is master secret. */ ssl->arrays->psk_keySz = ssl->specs.hash_size; +#ifdef WOLFSSL_TLS13_DRAFT_18 XMEMCPY(ssl->arrays->psk_key, ssl->session.masterSecret, ssl->arrays->psk_keySz); +#else + if ((ret = DeriveResumptionPSK(ssl, ssl->session.ticketNonce.data, + ssl->session.ticketNonce.len, ssl->arrays->psk_key)) != 0) { + return ret; + } +#endif } #endif #ifndef NO_PSK @@ -2916,8 +2980,15 @@ static int DoPreSharedKeys(WOLFSSL* ssl, const byte* input, word32 helloSz, /* Resumption PSK is resumption master secret. */ ssl->arrays->psk_keySz = ssl->specs.hash_size; +#ifdef WOLFSSL_TLS13_DRAFT_18 XMEMCPY(ssl->arrays->psk_key, ssl->session.masterSecret, - ssl->specs.hash_size); + ssl->arrays->psk_keySz); +#else + if ((ret = DeriveResumptionPSK(ssl, ssl->session.ticketNonce.data, + ssl->session.ticketNonce.len, ssl->arrays->psk_key)) != 0) { + return ret; + } +#endif /* Derive the early secret using the PSK. */ ret = DeriveEarlySecret(ssl); @@ -5550,6 +5621,10 @@ static int DoTls13NewSessionTicket(WOLFSSL* ssl, const byte* input, word32 ageAdd; word16 length; word32 now; +#ifndef WOLFSSL_TLS13_DRAFT_18 + const byte* nonce; + byte nonceLength; +#endif WOLFSSL_ENTER("DoTls13NewSessionTicket"); @@ -5567,6 +5642,24 @@ static int DoTls13NewSessionTicket(WOLFSSL* ssl, const byte* input, ato32(input + *inOutIdx, &ageAdd); *inOutIdx += SESSION_ADD_SZ; +#ifndef WOLFSSL_TLS13_DRAFT_18 + /* Ticket nonce. */ + if ((*inOutIdx - begin) + 1 > size) + return BUFFER_ERROR; + nonceLength = input[*inOutIdx]; + if (nonceLength == 0) + return INVALID_PARAMETER; + if (nonceLength > MAX_TICKET_NONCE_SZ) { + WOLFSSL_MSG("Nonce length not supported"); + return INVALID_PARAMETER; + } + *inOutIdx += 1; + if ((*inOutIdx - begin) + nonceLength > size) + return BUFFER_ERROR; + nonce = input + *inOutIdx; + *inOutIdx += 1; +#endif + /* Ticket length. */ if ((*inOutIdx - begin) + LENGTH_SZ > size) return BUFFER_ERROR; @@ -5592,6 +5685,10 @@ static int DoTls13NewSessionTicket(WOLFSSL* ssl, const byte* input, #ifdef WOLFSSL_EARLY_DATA ssl->session.maxEarlyDataSz = ssl->options.maxEarlyDataSz; #endif +#ifndef WOLFSSL_TLS13_DRAFT_18 + ssl->session.ticketNonce.len = nonceLength; + XMEMCPY(&ssl->session.ticketNonce.data, nonce, nonceLength); +#endif if ((*inOutIdx - begin) + EXTS_SZ > size) return BUFFER_ERROR; @@ -5751,6 +5848,16 @@ static int SendTls13NewSessionTicket(WOLFSSL* ssl) } #endif +#ifndef WOLFSSL_TLS13_DRAFT_18 + /* Start ticket nonce at 0 and go up to 255. */ + if (ssl->session.ticketNonce.len == 0) { + ssl->session.ticketNonce.len = DEF_TICKET_NONCE_SZ; + ssl->session.ticketNonce.data[0] = 0; + } + else + ssl->session.ticketNonce.data[0]++; +#endif + if (!ssl->options.noTicketTls13) { if ((ret = CreateTicket(ssl)) != 0) return ret; @@ -5768,6 +5875,10 @@ static int SendTls13NewSessionTicket(WOLFSSL* ssl) /* Lifetime | Age Add | Ticket | Extensions */ length = SESSION_HINT_SZ + SESSION_ADD_SZ + LENGTH_SZ + ssl->session.ticketLen + extSz; +#ifndef WOLFSSL_TLS13_DRAFT_18 + /* Nonce */ + length += TICKET_NONCE_LEN_SZ + DEF_TICKET_NONCE_SZ; +#endif sendSz = idx + length + MAX_MSG_EXTRA; /* Check buffers are big enough and grow if needed. */ @@ -5788,6 +5899,11 @@ static int SendTls13NewSessionTicket(WOLFSSL* ssl) c32toa(ssl->session.ticketAdd, output + idx); idx += SESSION_ADD_SZ; +#ifndef WOLFSSL_TLS13_DRAFT_18 + output[idx++] = ssl->session.ticketNonce.len; + output[idx++] = ssl->session.ticketNonce.data[0]; +#endif + /* length */ c16toa(ssl->session.ticketLen, output + idx); idx += LENGTH_SZ; @@ -6003,8 +6119,8 @@ int DoTls13HandShakeMsgType(WOLFSSL* ssl, byte* input, word32* inOutIdx, byte type, word32 size, word32 totalSz) { int ret = 0; - (void)totalSz; word32 inIdx = *inOutIdx; + (void)totalSz; WOLFSSL_ENTER("DoTls13HandShakeMsgType"); diff --git a/tests/api.c b/tests/api.c index 07f438984b..c6b98dad27 100644 --- a/tests/api.c +++ b/tests/api.c @@ -271,8 +271,10 @@ static void test_wolfSSL_Method_Allocators(void) TEST_VALID_METHOD_ALLOCATOR(wolfSSLv3_server_method); TEST_VALID_METHOD_ALLOCATOR(wolfSSLv3_client_method); #endif - TEST_VALID_METHOD_ALLOCATOR(wolfTLSv1_server_method); - TEST_VALID_METHOD_ALLOCATOR(wolfTLSv1_client_method); + #ifdef WOLFSL_ALLOW_TLSV10 + TEST_VALID_METHOD_ALLOCATOR(wolfTLSv1_server_method); + TEST_VALID_METHOD_ALLOCATOR(wolfTLSv1_client_method); + #endif TEST_VALID_METHOD_ALLOCATOR(wolfTLSv1_1_server_method); TEST_VALID_METHOD_ALLOCATOR(wolfTLSv1_1_client_method); #endif @@ -756,7 +758,9 @@ static void test_wolfSSL_EC(void) EC_POINT *Gxy, *new_point; BIGNUM *k = NULL, *Gx = NULL, *Gy = NULL, *Gz = NULL; BIGNUM *X, *Y; - +#if defined(WOLFSSL_KEY_GEN) || defined(HAVE_COMP_KEY) || defined(DEBUG_WOLFSSL) + char* hexStr; +#endif const char* kTest = "F4F8338AFCC562C5C3F3E1E46A7EFECD17AF381913FF7A96314EA47055EA0FD0"; /* NISTP256R1 Gx/Gy */ const char* kGx = "6B17D1F2E12C4247F8BCE6E563A440F277037D812DEB33A0F4A13945D898C296"; @@ -792,6 +796,21 @@ static void test_wolfSSL_EC(void) /* check if point X coordinate is zero */ AssertIntEQ(BN_is_zero(X), WOLFSSL_FAILURE); + /* check bx2hex */ +#if defined(WOLFSSL_KEY_GEN) || defined(HAVE_COMP_KEY) || defined(DEBUG_WOLFSSL) + hexStr = BN_bn2hex(k); + AssertStrEQ(hexStr, kTest); + XFREE(hexStr, NULL, DYNAMIC_TYPE_ECC); + + hexStr = BN_bn2hex(Gx); + AssertStrEQ(hexStr, kGx); + XFREE(hexStr, NULL, DYNAMIC_TYPE_ECC); + + hexStr = BN_bn2hex(Gy); + AssertStrEQ(hexStr, kGy); + XFREE(hexStr, NULL, DYNAMIC_TYPE_ECC); +#endif + /* cleanup */ BN_free(X); BN_free(Y); @@ -1789,7 +1808,7 @@ static void test_wolfSSL_UseSNI_connection(void) {0, use_SNI_at_ctx, 0, verify_SNI_real_matching}, /* success case at ssl */ - {0, 0, use_SNI_at_ssl, 0}, + {0, 0, use_SNI_at_ssl, verify_SNI_real_matching}, {0, 0, use_SNI_at_ssl, verify_SNI_real_matching}, /* default missmatch behavior */ @@ -7782,7 +7801,7 @@ static int test_wc_MakeRsaKey (void) if (ret == 0) { ret = wc_InitRng(&rng); if (ret == 0) { - ret = wc_MakeRsaKey(&genKey, 1024, 65537, &rng); + ret = wc_MakeRsaKey(&genKey, 1024, WC_RSA_EXPONENT, &rng); if (ret == 0 && wc_FreeRsaKey(&genKey) != 0) { ret = WOLFSSL_FATAL_ERROR; } @@ -7791,9 +7810,9 @@ static int test_wc_MakeRsaKey (void) #ifndef HAVE_USER_RSA /* Test bad args. */ if (ret == 0) { - ret = wc_MakeRsaKey(NULL, 1024, 65537, &rng); + ret = wc_MakeRsaKey(NULL, 1024, WC_RSA_EXPONENT, &rng); if (ret == BAD_FUNC_ARG) { - ret = wc_MakeRsaKey(&genKey, 1024, 65537, NULL); + ret = wc_MakeRsaKey(&genKey, 1024, WC_RSA_EXPONENT, NULL); } if (ret == BAD_FUNC_ARG) { /* e < 3 */ @@ -7812,9 +7831,9 @@ static int test_wc_MakeRsaKey (void) #else /* Test bad args. */ if (ret == 0) { - ret = wc_MakeRsaKey(NULL, 1024, 65537, &rng); + ret = wc_MakeRsaKey(NULL, 1024, WC_RSA_EXPONENT, &rng); if (ret == USER_CRYPTO_ERROR) { - ret = wc_MakeRsaKey(&genKey, 1024, 65537, NULL); + ret = wc_MakeRsaKey(&genKey, 1024, WC_RSA_EXPONENT, NULL); } if (ret == USER_CRYPTO_ERROR) { /* e < 3 */ @@ -7925,7 +7944,7 @@ static int test_wc_RsaKeyToDer (void) } /* Make key. */ if (ret == 0) { - ret = wc_MakeRsaKey(&genKey, 1024, 65537, &rng); + ret = wc_MakeRsaKey(&genKey, 1024, WC_RSA_EXPONENT, &rng); if (ret != 0) { ret = WOLFSSL_FATAL_ERROR; } @@ -8018,7 +8037,7 @@ static int test_wc_RsaKeyToPublicDer (void) ret = wc_InitRng(&rng); } if (ret == 0) { - ret = wc_MakeRsaKey(&key, 1024, 65537, &rng); + ret = wc_MakeRsaKey(&key, 1024, WC_RSA_EXPONENT, &rng); } printf(testingFmt, "wc_RsaKeyToPublicDer()"); @@ -8106,7 +8125,7 @@ static int test_wc_RsaPublicEncryptDecrypt (void) ret = wc_InitRng(&rng); } if (ret == 0) { - ret = wc_MakeRsaKey(&key, 1024, 65537, &rng); + ret = wc_MakeRsaKey(&key, 1024, WC_RSA_EXPONENT, &rng); } /* Encrypt. */ printf(testingFmt, "wc_RsaPublicEncrypt()"); @@ -8194,7 +8213,7 @@ static int test_wc_RsaPublicEncryptDecrypt_ex (void) ret = wc_InitRng(&rng); } if (ret == 0) { - ret = wc_MakeRsaKey(&key, 1024, 65537, &rng); + ret = wc_MakeRsaKey(&key, 1024, WC_RSA_EXPONENT, &rng); } /* Encrypt */ printf(testingFmt, "wc_RsaPublicEncrypt_ex()"); @@ -8301,7 +8320,7 @@ static int test_wc_RsaSSL_SignVerify (void) } if (ret == 0) { - ret = wc_MakeRsaKey(&key, 1024, 65537, &rng); + ret = wc_MakeRsaKey(&key, 1024, WC_RSA_EXPONENT, &rng); } /* Sign. */ printf(testingFmt, "wc_RsaSSL_Sign()"); @@ -8447,7 +8466,7 @@ static int test_wc_RsaEncryptSize (void) printf(testingFmt, "wc_RsaEncryptSize()"); if (ret == 0) { - ret = wc_MakeRsaKey(&key, 1024, 65537, &rng); + ret = wc_MakeRsaKey(&key, 1024, WC_RSA_EXPONENT, &rng); if (ret == 0) { ret = wc_RsaEncryptSize(&key); } @@ -8464,7 +8483,7 @@ static int test_wc_RsaEncryptSize (void) } if (ret == 0) { - ret = wc_MakeRsaKey(&key, FOURK_BUF, 65537, &rng); + ret = wc_MakeRsaKey(&key, FOURK_BUF, WC_RSA_EXPONENT, &rng); if (ret == 0) { ret = wc_RsaEncryptSize(&key); } @@ -8521,7 +8540,7 @@ static int test_wc_RsaFlattenPublicKey (void) } if (ret == 0) { - ret = wc_MakeRsaKey(&key, 1024, 65537, &rng); + ret = wc_MakeRsaKey(&key, 1024, WC_RSA_EXPONENT, &rng); if (ret >= 0) { ret = 0; } else { diff --git a/tests/suites.c b/tests/suites.c index 11825656ad..f1eb22e4fe 100644 --- a/tests/suites.c +++ b/tests/suites.c @@ -37,7 +37,8 @@ #define MAX_COMMAND_SZ 240 #define MAX_SUITE_SZ 80 #define NOT_BUILT_IN -123 -#if defined(NO_OLD_TLS) || !defined(WOLFSSL_ALLOW_SSLV3) +#if defined(NO_OLD_TLS) || !defined(WOLFSSL_ALLOW_SSLV3) || \ + !defined(WOLFSSL_ALLOW_TLSV10) #define VERSION_TOO_OLD -124 #endif @@ -61,49 +62,48 @@ static char forceDefCipherListFlag[] = "-HdefCipherList"; #endif -#ifndef WOLFSSL_ALLOW_SSLV3 -/* if the protocol version is sslv3 return 1, else 0 */ -static int IsSslVersion(const char* line) +#ifdef VERSION_TOO_OLD +static int GetTlsVersion(const char* line) { + int version = -1; const char* find = "-v "; const char* begin = strstr(line, find); if (begin) { - int version = -1; - begin += 3; version = atoi(begin); - - if (version == 0) - return 1; } + return version; +} - return 0; +#ifndef WOLFSSL_ALLOW_SSLV3 +/* if the protocol version is sslv3 return 1, else 0 */ +static int IsSslVersion(const char* line) +{ + int version = GetTlsVersion(line); + return (version == 0) ? 1 : 0; } #endif /* !WOLFSSL_ALLOW_SSLV3 */ +#ifndef WOLFSSL_ALLOW_TLSV10 +/* if the protocol version is TLSv1.0 return 1, else 0 */ +static int IsTls10Version(const char* line) +{ + int version = GetTlsVersion(line); + return (version == 1) ? 1 : 0; +} +#endif /* !WOLFSSL_ALLOW_TLSV10 */ + #ifdef NO_OLD_TLS /* if the protocol version is less than tls 1.2 return 1, else 0 */ static int IsOldTlsVersion(const char* line) { - const char* find = "-v "; - const char* begin = strstr(line, find); - - if (begin) { - int version = -1; - - begin += 3; - - version = atoi(begin); - - if (version < 3) - return 1; - } - - return 0; + int version = GetTlsVersion(line); + return (version < 3) ? 1 : 0; } #endif /* NO_OLD_TLS */ +#endif /* VERSION_TOO_OLD */ /* if the cipher suite on line is valid store in suite and return 1, else 0 */ @@ -214,6 +214,14 @@ static int execute_test_case(int svr_argc, char** svr_argv, return VERSION_TOO_OLD; } #endif +#ifndef WOLFSSL_ALLOW_TLSV10 + if (IsTls10Version(commandLine) == 1) { + #ifdef DEBUG_SUITE_TESTS + printf("protocol version on line %s is too old\n", commandLine); + #endif + return VERSION_TOO_OLD; + } +#endif #ifdef NO_OLD_TLS if (IsOldTlsVersion(commandLine) == 1) { #ifdef DEBUG_SUITE_TESTS diff --git a/wolfcrypt/benchmark/benchmark.c b/wolfcrypt/benchmark/benchmark.c index 872cb9ecc3..5ad5fb8368 100644 --- a/wolfcrypt/benchmark/benchmark.c +++ b/wolfcrypt/benchmark/benchmark.c @@ -2944,7 +2944,7 @@ void bench_rsaKeyGen(int doAsync) int ret = 0, i, count = 0, times, pending = 0; int k, keySz; const int keySizes[2] = {1024, 2048}; - const long rsa_e_val = 65537; + const long rsa_e_val = WC_RSA_EXPONENT; /* clear for done cleanup */ XMEMSET(genKey, 0, sizeof(genKey)); diff --git a/wolfcrypt/src/aes.c b/wolfcrypt/src/aes.c index 2684d8a8dd..b3c5eda48d 100644 --- a/wolfcrypt/src/aes.c +++ b/wolfcrypt/src/aes.c @@ -2979,25 +2979,33 @@ int wc_AesSetIV(Aes* aes, const byte* iv) #ifdef HAVE_AES_ECB int wc_AesEcbEncrypt(Aes* aes, byte* out, const byte* in, word32 sz) { + word32 blocks = sz / AES_BLOCK_SIZE; + if ((in == NULL) || (out == NULL) || (aes == NULL)) return BAD_FUNC_ARG; - while (sz>0) { + while (blocks>0) { wc_AesEncryptDirect(aes, out, in); out += AES_BLOCK_SIZE; in += AES_BLOCK_SIZE; sz -= AES_BLOCK_SIZE; + blocks--; } return 0; } + + int wc_AesEcbDecrypt(Aes* aes, byte* out, const byte* in, word32 sz) { + word32 blocks = sz / AES_BLOCK_SIZE; + if ((in == NULL) || (out == NULL) || (aes == NULL)) return BAD_FUNC_ARG; - while (sz>0) { + while (blocks>0) { wc_AesDecryptDirect(aes, out, in); out += AES_BLOCK_SIZE; in += AES_BLOCK_SIZE; sz -= AES_BLOCK_SIZE; + blocks--; } return 0; } @@ -3420,39 +3428,6 @@ static const __m128i MOD2_128 = M128_INIT(0x1, 0xc200000000000000UL); static __m128i gfmul_sw(__m128i a, __m128i b) { __m128i r, t1, t2, t3, t4, t5, t6, t7; -#ifndef WOLFSSL_AES_GCM_SLOW_CLMUL - /* 128 x 128 Carryless Multiply */ - t3 = _mm_clmulepi64_si128(a, b, 0x10); - t2 = _mm_clmulepi64_si128(a, b, 0x01); - t1 = _mm_clmulepi64_si128(a, b, 0x00); - t4 = _mm_clmulepi64_si128(a, b, 0x11); - t3 = _mm_xor_si128(t3, t2); - t2 = _mm_slli_si128(t3, 8); - t3 = _mm_srli_si128(t3, 8); - t1 = _mm_xor_si128(t1, t2); - t4 = _mm_xor_si128(t4, t3); - - /* shift left 1 bit - bits reversed */ - t5 = _mm_srli_epi32(t1, 31); - t6 = _mm_srli_epi32(t4, 31); - t1 = _mm_slli_epi32(t1, 1); - t4 = _mm_slli_epi32(t4, 1); - t7 = _mm_srli_si128(t5, 12); - t5 = _mm_slli_si128(t5, 4); - t6 = _mm_slli_si128(t6, 4); - t4 = _mm_or_si128(t4, t7); - t1 = _mm_or_si128(t1, t5); - t4 = _mm_or_si128(t4, t6); - - /* Reduction */ - t2 = _mm_clmulepi64_si128(t1, MOD2_128, 0x10); - t3 = _mm_shuffle_epi32(t1, 78); - t3 = _mm_xor_si128(t3, t2); - t2 = _mm_clmulepi64_si128(t3, MOD2_128, 0x10); - t3 = _mm_shuffle_epi32(t3, 78); - t3 = _mm_xor_si128(t3, t2); - r = _mm_xor_si128(t4, t3); -#else t2 = _mm_shuffle_epi32(b, 78); t3 = _mm_shuffle_epi32(a, 78); t2 = _mm_xor_si128(t2, b); @@ -3496,27 +3471,16 @@ static __m128i gfmul_sw(__m128i a, __m128i b) t7 = _mm_xor_si128(t7, t6); t7 = _mm_xor_si128(t7, t1); r = _mm_xor_si128(t4, t7); -#endif return r; } + static void gfmul_only(__m128i a, __m128i b, __m128i* r0, __m128i* r1) { __m128i t1, t2, t3, t4; /* 128 x 128 Carryless Multiply */ -#ifndef WOLFSSL_AES_GCM_SLOW_CLMUL - t3 = _mm_clmulepi64_si128(a, b, 0x10); - t2 = _mm_clmulepi64_si128(a, b, 0x01); - t1 = _mm_clmulepi64_si128(a, b, 0x00); - t4 = _mm_clmulepi64_si128(a, b, 0x11); - t3 = _mm_xor_si128(t3, t2); - t2 = _mm_slli_si128(t3, 8); - t3 = _mm_srli_si128(t3, 8); - t1 = _mm_xor_si128(t1, t2); - t4 = _mm_xor_si128(t4, t3); -#else t2 = _mm_shuffle_epi32(b, 78); t3 = _mm_shuffle_epi32(a, 78); t2 = _mm_xor_si128(t2, b); @@ -3530,7 +3494,6 @@ static void gfmul_only(__m128i a, __m128i b, __m128i* r0, __m128i* r1) t2 = _mm_srli_si128(t2, 8); t1 = _mm_xor_si128(t1, t3); t4 = _mm_xor_si128(t4, t2); -#endif *r0 = _mm_xor_si128(t1, *r0); *r1 = _mm_xor_si128(t4, *r1); } @@ -3553,15 +3516,6 @@ static __m128i gfmul_shl1(__m128i a) static __m128i ghash_red(__m128i r0, __m128i r1) { __m128i t2, t3; -#ifndef WOLFSSL_AES_GCM_SLOW_CLMUL - t2 = _mm_clmulepi64_si128(r0, MOD2_128, 0x10); - t3 = _mm_shuffle_epi32(r0, 78); - t3 = _mm_xor_si128(t3, t2); - t2 = _mm_clmulepi64_si128(t3, MOD2_128, 0x10); - t3 = _mm_shuffle_epi32(t3, 78); - t3 = _mm_xor_si128(t3, t2); - return _mm_xor_si128(r1, t3); -#else __m128i t5, t6, t7; t5 = _mm_slli_epi32(r0, 31); @@ -3582,7 +3536,6 @@ static __m128i ghash_red(__m128i r0, __m128i r1) t7 = _mm_xor_si128(t7, t6); t7 = _mm_xor_si128(t7, r0); return _mm_xor_si128(r1, t7); -#endif } static __m128i gfmul_shifted(__m128i a, __m128i b) @@ -3611,6 +3564,103 @@ static __m128i gfmul8(__m128i a1, __m128i a2, __m128i a3, __m128i a4, } #endif +#ifdef HAVE_INTEL_AVX2 +static __m128i gfmul_sw_avx2(__m128i a, __m128i b) +{ + __m128i r, t1, t2, t3, t4, t5, t6, t7; + /* 128 x 128 Carryless Multiply */ + t3 = _mm_clmulepi64_si128(a, b, 0x10); + t2 = _mm_clmulepi64_si128(a, b, 0x01); + t1 = _mm_clmulepi64_si128(a, b, 0x00); + t4 = _mm_clmulepi64_si128(a, b, 0x11); + t3 = _mm_xor_si128(t3, t2); + t2 = _mm_slli_si128(t3, 8); + t3 = _mm_srli_si128(t3, 8); + t1 = _mm_xor_si128(t1, t2); + t4 = _mm_xor_si128(t4, t3); + + /* shift left 1 bit - bits reversed */ + t5 = _mm_srli_epi32(t1, 31); + t6 = _mm_srli_epi32(t4, 31); + t1 = _mm_slli_epi32(t1, 1); + t4 = _mm_slli_epi32(t4, 1); + t7 = _mm_srli_si128(t5, 12); + t5 = _mm_slli_si128(t5, 4); + t6 = _mm_slli_si128(t6, 4); + t4 = _mm_or_si128(t4, t7); + t1 = _mm_or_si128(t1, t5); + t4 = _mm_or_si128(t4, t6); + + /* Reduction */ + t2 = _mm_clmulepi64_si128(t1, MOD2_128, 0x10); + t3 = _mm_shuffle_epi32(t1, 78); + t3 = _mm_xor_si128(t3, t2); + t2 = _mm_clmulepi64_si128(t3, MOD2_128, 0x10); + t3 = _mm_shuffle_epi32(t3, 78); + t3 = _mm_xor_si128(t3, t2); + r = _mm_xor_si128(t4, t3); + + return r; +} + +static void gfmul_only_avx2(__m128i a, __m128i b, __m128i* r0, __m128i* r1) +{ + __m128i t1, t2, t3, t4; + + /* 128 x 128 Carryless Multiply */ + t3 = _mm_clmulepi64_si128(a, b, 0x10); + t2 = _mm_clmulepi64_si128(a, b, 0x01); + t1 = _mm_clmulepi64_si128(a, b, 0x00); + t4 = _mm_clmulepi64_si128(a, b, 0x11); + t3 = _mm_xor_si128(t3, t2); + t2 = _mm_slli_si128(t3, 8); + t3 = _mm_srli_si128(t3, 8); + t1 = _mm_xor_si128(t1, t2); + t4 = _mm_xor_si128(t4, t3); + *r0 = _mm_xor_si128(t1, *r0); + *r1 = _mm_xor_si128(t4, *r1); +} + +static __m128i ghash_red_avx2(__m128i r0, __m128i r1) +{ + __m128i t2, t3; + t2 = _mm_clmulepi64_si128(r0, MOD2_128, 0x10); + t3 = _mm_shuffle_epi32(r0, 78); + t3 = _mm_xor_si128(t3, t2); + t2 = _mm_clmulepi64_si128(t3, MOD2_128, 0x10); + t3 = _mm_shuffle_epi32(t3, 78); + t3 = _mm_xor_si128(t3, t2); + return _mm_xor_si128(r1, t3); +} + +static __m128i gfmul_shifted_avx2(__m128i a, __m128i b) +{ + __m128i t0 = _mm_setzero_si128(), t1 = _mm_setzero_si128(); + gfmul_only_avx2(a, b, &t0, &t1); + return ghash_red_avx2(t0, t1); +} + +#ifndef AES_GCM_AESNI_NO_UNROLL +static __m128i gfmul8_avx2(__m128i a1, __m128i a2, __m128i a3, __m128i a4, + __m128i a5, __m128i a6, __m128i a7, __m128i a8, + __m128i b1, __m128i b2, __m128i b3, __m128i b4, + __m128i b5, __m128i b6, __m128i b7, __m128i b8) +{ + __m128i t0 = _mm_setzero_si128(), t1 = _mm_setzero_si128(); + gfmul_only_avx2(a1, b8, &t0, &t1); + gfmul_only_avx2(a2, b7, &t0, &t1); + gfmul_only_avx2(a3, b6, &t0, &t1); + gfmul_only_avx2(a4, b5, &t0, &t1); + gfmul_only_avx2(a5, b4, &t0, &t1); + gfmul_only_avx2(a6, b3, &t0, &t1); + gfmul_only_avx2(a7, b2, &t0, &t1); + gfmul_only_avx2(a8, b1, &t0, &t1); + return ghash_red_avx2(t0, t1); +} +#endif /* AES_GCM_AESNI_NO_UNROLL */ +#endif /* HAVE_INTEL_AVX2 */ + + /* See Intel® Carry-Less Multiplication Instruction * and its Usage for Computing the GCM Mode White Paper * by Shay Gueron, Intel Mobility Group, Israel Development Center; @@ -3632,7 +3682,173 @@ static const __m128i EIGHT = M128_INIT(0x0, 0x8); static const __m128i BSWAP_EPI64 = M128_INIT(0x0001020304050607, 0x08090a0b0c0d0e0f); static const __m128i BSWAP_MASK = M128_INIT(0x08090a0b0c0d0e0f, 0x0001020304050607); -static void AES_GCM_encrypt(const unsigned char *in, unsigned char *out, +#define aes_gcm_calc_iv_12(KEY, ivec, nr, H, Y, T, X) \ +do \ +{ \ + Y = _mm_setzero_si128(); \ + for (j=0; j < 12; j++) \ + ((unsigned char*)&Y)[j] = ivec[j]; \ + Y = _mm_insert_epi32(Y, 0x1000000, 3); \ + \ + /* (Compute E[ZERO, KS] and E[Y0, KS] together */ \ + tmp1 = _mm_xor_si128(X, KEY[0]); \ + tmp2 = _mm_xor_si128(Y, KEY[0]); \ + tmp1 = _mm_aesenc_si128(tmp1, KEY[1]); \ + tmp2 = _mm_aesenc_si128(tmp2, KEY[1]); \ + tmp1 = _mm_aesenc_si128(tmp1, KEY[2]); \ + tmp2 = _mm_aesenc_si128(tmp2, KEY[2]); \ + tmp1 = _mm_aesenc_si128(tmp1, KEY[3]); \ + tmp2 = _mm_aesenc_si128(tmp2, KEY[3]); \ + tmp1 = _mm_aesenc_si128(tmp1, KEY[4]); \ + tmp2 = _mm_aesenc_si128(tmp2, KEY[4]); \ + tmp1 = _mm_aesenc_si128(tmp1, KEY[5]); \ + tmp2 = _mm_aesenc_si128(tmp2, KEY[5]); \ + tmp1 = _mm_aesenc_si128(tmp1, KEY[6]); \ + tmp2 = _mm_aesenc_si128(tmp2, KEY[6]); \ + tmp1 = _mm_aesenc_si128(tmp1, KEY[7]); \ + tmp2 = _mm_aesenc_si128(tmp2, KEY[7]); \ + tmp1 = _mm_aesenc_si128(tmp1, KEY[8]); \ + tmp2 = _mm_aesenc_si128(tmp2, KEY[8]); \ + tmp1 = _mm_aesenc_si128(tmp1, KEY[9]); \ + tmp2 = _mm_aesenc_si128(tmp2, KEY[9]); \ + lastKey = KEY[10]; \ + if (nr > 10) { \ + tmp1 = _mm_aesenc_si128(tmp1, lastKey); \ + tmp2 = _mm_aesenc_si128(tmp2, lastKey); \ + tmp1 = _mm_aesenc_si128(tmp1, KEY[11]); \ + tmp2 = _mm_aesenc_si128(tmp2, KEY[11]); \ + lastKey = KEY[12]; \ + if (nr > 12) { \ + tmp1 = _mm_aesenc_si128(tmp1, lastKey); \ + tmp2 = _mm_aesenc_si128(tmp2, lastKey); \ + tmp1 = _mm_aesenc_si128(tmp1, KEY[13]); \ + tmp2 = _mm_aesenc_si128(tmp2, KEY[13]); \ + lastKey = KEY[14]; \ + } \ + } \ + H = _mm_aesenclast_si128(tmp1, lastKey); \ + T = _mm_aesenclast_si128(tmp2, lastKey); \ + H = _mm_shuffle_epi8(H, BSWAP_MASK); \ +} \ +while (0) + +#define aes_gcm_calc_iv(KEY, ivec, ibytes, nr, H, Y, T, X) \ +do \ +{ \ + if (ibytes % 16) { \ + i = ibytes / 16; \ + for (j=0; j < (int)(ibytes%16); j++) \ + ((unsigned char*)&last_block)[j] = ivec[i*16+j]; \ + } \ + tmp1 = _mm_xor_si128(X, KEY[0]); \ + tmp1 = _mm_aesenc_si128(tmp1, KEY[1]); \ + tmp1 = _mm_aesenc_si128(tmp1, KEY[2]); \ + tmp1 = _mm_aesenc_si128(tmp1, KEY[3]); \ + tmp1 = _mm_aesenc_si128(tmp1, KEY[4]); \ + tmp1 = _mm_aesenc_si128(tmp1, KEY[5]); \ + tmp1 = _mm_aesenc_si128(tmp1, KEY[6]); \ + tmp1 = _mm_aesenc_si128(tmp1, KEY[7]); \ + tmp1 = _mm_aesenc_si128(tmp1, KEY[8]); \ + tmp1 = _mm_aesenc_si128(tmp1, KEY[9]); \ + lastKey = KEY[10]; \ + if (nr > 10) { \ + tmp1 = _mm_aesenc_si128(tmp1, lastKey); \ + tmp1 = _mm_aesenc_si128(tmp1, KEY[11]); \ + lastKey = KEY[12]; \ + if (nr > 12) { \ + tmp1 = _mm_aesenc_si128(tmp1, lastKey); \ + tmp1 = _mm_aesenc_si128(tmp1, KEY[13]); \ + lastKey = KEY[14]; \ + } \ + } \ + H = _mm_aesenclast_si128(tmp1, lastKey); \ + H = _mm_shuffle_epi8(H, BSWAP_MASK); \ + Y = _mm_setzero_si128(); \ + for (i=0; i < (int)(ibytes/16); i++) { \ + tmp1 = _mm_loadu_si128(&((__m128i*)ivec)[i]); \ + tmp1 = _mm_shuffle_epi8(tmp1, BSWAP_MASK); \ + Y = _mm_xor_si128(Y, tmp1); \ + Y = gfmul_sw(Y, H); \ + } \ + if (ibytes % 16) { \ + tmp1 = last_block; \ + tmp1 = _mm_shuffle_epi8(tmp1, BSWAP_MASK); \ + Y = _mm_xor_si128(Y, tmp1); \ + Y = gfmul_sw(Y, H); \ + } \ + tmp1 = _mm_insert_epi64(tmp1, ibytes*8, 0); \ + tmp1 = _mm_insert_epi64(tmp1, 0, 1); \ + Y = _mm_xor_si128(Y, tmp1); \ + Y = gfmul_sw(Y, H); \ + Y = _mm_shuffle_epi8(Y, BSWAP_MASK); /* Compute E(K, Y0) */ \ + tmp1 = _mm_xor_si128(Y, KEY[0]); \ + tmp1 = _mm_aesenc_si128(tmp1, KEY[1]); \ + tmp1 = _mm_aesenc_si128(tmp1, KEY[2]); \ + tmp1 = _mm_aesenc_si128(tmp1, KEY[3]); \ + tmp1 = _mm_aesenc_si128(tmp1, KEY[4]); \ + tmp1 = _mm_aesenc_si128(tmp1, KEY[5]); \ + tmp1 = _mm_aesenc_si128(tmp1, KEY[6]); \ + tmp1 = _mm_aesenc_si128(tmp1, KEY[7]); \ + tmp1 = _mm_aesenc_si128(tmp1, KEY[8]); \ + tmp1 = _mm_aesenc_si128(tmp1, KEY[9]); \ + lastKey = KEY[10]; \ + if (nr > 10) { \ + tmp1 = _mm_aesenc_si128(tmp1, lastKey); \ + tmp1 = _mm_aesenc_si128(tmp1, KEY[11]); \ + lastKey = KEY[12]; \ + if (nr > 12) { \ + tmp1 = _mm_aesenc_si128(tmp1, lastKey); \ + tmp1 = _mm_aesenc_si128(tmp1, KEY[13]); \ + lastKey = KEY[14]; \ + } \ + } \ + T = _mm_aesenclast_si128(tmp1, lastKey); \ +} \ +while (0) + +#define AES_ENC_8(j) \ + tmp1 = _mm_aesenc_si128(tmp1, KEY[j]); \ + tmp2 = _mm_aesenc_si128(tmp2, KEY[j]); \ + tmp3 = _mm_aesenc_si128(tmp3, KEY[j]); \ + tmp4 = _mm_aesenc_si128(tmp4, KEY[j]); \ + tmp5 = _mm_aesenc_si128(tmp5, KEY[j]); \ + tmp6 = _mm_aesenc_si128(tmp6, KEY[j]); \ + tmp7 = _mm_aesenc_si128(tmp7, KEY[j]); \ + tmp8 = _mm_aesenc_si128(tmp8, KEY[j]); + +#define AES_ENC_LAST_8() \ + tmp1 =_mm_aesenclast_si128(tmp1, lastKey); \ + tmp2 =_mm_aesenclast_si128(tmp2, lastKey); \ + tmp1 = _mm_xor_si128(tmp1, _mm_loadu_si128(&((__m128i*)in)[i*8+0])); \ + tmp2 = _mm_xor_si128(tmp2, _mm_loadu_si128(&((__m128i*)in)[i*8+1])); \ + _mm_storeu_si128(&((__m128i*)out)[i*8+0], tmp1); \ + _mm_storeu_si128(&((__m128i*)out)[i*8+1], tmp2); \ + tmp3 =_mm_aesenclast_si128(tmp3, lastKey); \ + tmp4 =_mm_aesenclast_si128(tmp4, lastKey); \ + tmp3 = _mm_xor_si128(tmp3, _mm_loadu_si128(&((__m128i*)in)[i*8+2])); \ + tmp4 = _mm_xor_si128(tmp4, _mm_loadu_si128(&((__m128i*)in)[i*8+3])); \ + _mm_storeu_si128(&((__m128i*)out)[i*8+2], tmp3); \ + _mm_storeu_si128(&((__m128i*)out)[i*8+3], tmp4); \ + tmp5 =_mm_aesenclast_si128(tmp5, lastKey); \ + tmp6 =_mm_aesenclast_si128(tmp6, lastKey); \ + tmp5 = _mm_xor_si128(tmp5, _mm_loadu_si128(&((__m128i*)in)[i*8+4])); \ + tmp6 = _mm_xor_si128(tmp6, _mm_loadu_si128(&((__m128i*)in)[i*8+5])); \ + _mm_storeu_si128(&((__m128i*)out)[i*8+4], tmp5); \ + _mm_storeu_si128(&((__m128i*)out)[i*8+5], tmp6); \ + tmp7 =_mm_aesenclast_si128(tmp7, lastKey); \ + tmp8 =_mm_aesenclast_si128(tmp8, lastKey); \ + tmp7 = _mm_xor_si128(tmp7, _mm_loadu_si128(&((__m128i*)in)[i*8+6])); \ + tmp8 = _mm_xor_si128(tmp8, _mm_loadu_si128(&((__m128i*)in)[i*8+7])); \ + _mm_storeu_si128(&((__m128i*)out)[i*8+6], tmp7); \ + _mm_storeu_si128(&((__m128i*)out)[i*8+7], tmp8); + +void AES_GCM_encrypt(const unsigned char *in, unsigned char *out, + const unsigned char* addt, + const unsigned char* ivec, + unsigned char *tag, unsigned int nbytes, + unsigned int abytes, unsigned int ibytes, + const unsigned char* key, int nr); +void AES_GCM_encrypt(const unsigned char *in, unsigned char *out, const unsigned char* addt, const unsigned char* ivec, unsigned char *tag, unsigned int nbytes, @@ -3653,121 +3869,10 @@ static void AES_GCM_encrypt(const unsigned char *in, unsigned char *out, __m128i tmp3, tmp4, tmp5, tmp6, tmp7, tmp8; #endif - if (ibytes == 12) { - Y = _mm_setzero_si128(); - for (j=0; j < 12; j++) - ((unsigned char*)&Y)[j] = ivec[j]; - Y = _mm_insert_epi32(Y, 0x1000000, 3); - /* (Compute E[ZERO, KS] and E[Y0, KS] together */ - tmp1 = _mm_xor_si128(X, KEY[0]); - tmp2 = _mm_xor_si128(Y, KEY[0]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[1]); - tmp2 = _mm_aesenc_si128(tmp2, KEY[1]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[2]); - tmp2 = _mm_aesenc_si128(tmp2, KEY[2]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[3]); - tmp2 = _mm_aesenc_si128(tmp2, KEY[3]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[4]); - tmp2 = _mm_aesenc_si128(tmp2, KEY[4]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[5]); - tmp2 = _mm_aesenc_si128(tmp2, KEY[5]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[6]); - tmp2 = _mm_aesenc_si128(tmp2, KEY[6]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[7]); - tmp2 = _mm_aesenc_si128(tmp2, KEY[7]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[8]); - tmp2 = _mm_aesenc_si128(tmp2, KEY[8]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[9]); - tmp2 = _mm_aesenc_si128(tmp2, KEY[9]); - lastKey = KEY[10]; - if (nr > 10) { - tmp1 = _mm_aesenc_si128(tmp1, lastKey); - tmp2 = _mm_aesenc_si128(tmp2, lastKey); - tmp1 = _mm_aesenc_si128(tmp1, KEY[11]); - tmp2 = _mm_aesenc_si128(tmp2, KEY[11]); - lastKey = KEY[12]; - if (nr > 12) { - tmp1 = _mm_aesenc_si128(tmp1, lastKey); - tmp2 = _mm_aesenc_si128(tmp2, lastKey); - tmp1 = _mm_aesenc_si128(tmp1, KEY[13]); - tmp2 = _mm_aesenc_si128(tmp2, KEY[13]); - lastKey = KEY[14]; - } - } - H = _mm_aesenclast_si128(tmp1, lastKey); - T = _mm_aesenclast_si128(tmp2, lastKey); - H = _mm_shuffle_epi8(H, BSWAP_MASK); - } - else { - if (ibytes % 16) { - i = ibytes / 16; - for (j=0; j < (int)(ibytes%16); j++) - ((unsigned char*)&last_block)[j] = ivec[i*16+j]; - } - tmp1 = _mm_xor_si128(X, KEY[0]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[1]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[2]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[3]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[4]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[5]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[6]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[7]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[8]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[9]); - lastKey = KEY[10]; - if (nr > 10) { - tmp1 = _mm_aesenc_si128(tmp1, lastKey); - tmp1 = _mm_aesenc_si128(tmp1, KEY[11]); - lastKey = KEY[12]; - if (nr > 12) { - tmp1 = _mm_aesenc_si128(tmp1, lastKey); - tmp1 = _mm_aesenc_si128(tmp1, KEY[13]); - lastKey = KEY[14]; - } - } - H = _mm_aesenclast_si128(tmp1, lastKey); - H = _mm_shuffle_epi8(H, BSWAP_MASK); - Y = _mm_setzero_si128(); - for (i=0; i < (int)(ibytes/16); i++) { - tmp1 = _mm_loadu_si128(&((__m128i*)ivec)[i]); - tmp1 = _mm_shuffle_epi8(tmp1, BSWAP_MASK); - Y = _mm_xor_si128(Y, tmp1); - Y = gfmul_sw(Y, H); - } - if (ibytes % 16) { - tmp1 = last_block; - tmp1 = _mm_shuffle_epi8(tmp1, BSWAP_MASK); - Y = _mm_xor_si128(Y, tmp1); - Y = gfmul_sw(Y, H); - } - tmp1 = _mm_insert_epi64(tmp1, ibytes*8, 0); - tmp1 = _mm_insert_epi64(tmp1, 0, 1); - Y = _mm_xor_si128(Y, tmp1); - Y = gfmul_sw(Y, H); - Y = _mm_shuffle_epi8(Y, BSWAP_MASK); /* Compute E(K, Y0) */ - tmp1 = _mm_xor_si128(Y, KEY[0]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[1]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[2]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[3]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[4]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[5]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[6]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[7]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[8]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[9]); - lastKey = KEY[10]; - if (nr > 10) { - tmp1 = _mm_aesenc_si128(tmp1, lastKey); - tmp1 = _mm_aesenc_si128(tmp1, KEY[11]); - lastKey = KEY[12]; - if (nr > 12) { - tmp1 = _mm_aesenc_si128(tmp1, lastKey); - tmp1 = _mm_aesenc_si128(tmp1, KEY[13]); - lastKey = KEY[14]; - } - } - T = _mm_aesenclast_si128(tmp1, lastKey); - } + if (ibytes == 12) + aes_gcm_calc_iv_12(KEY, ivec, nr, H, Y, T, X); + else + aes_gcm_calc_iv(KEY, ivec, ibytes, nr, H, Y, T, X); for (i=0; i < (int)(abytes/16); i++) { tmp1 = _mm_loadu_si128(&((__m128i*)addt)[i]); @@ -3825,141 +3930,27 @@ static void AES_GCM_encrypt(const unsigned char *in, unsigned char *out, tmp6 =_mm_xor_si128(tmp6, KEY[0]); tmp7 =_mm_xor_si128(tmp7, KEY[0]); tmp8 =_mm_xor_si128(tmp8, KEY[0]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[1]); - tmp2 = _mm_aesenc_si128(tmp2, KEY[1]); - tmp3 = _mm_aesenc_si128(tmp3, KEY[1]); - tmp4 = _mm_aesenc_si128(tmp4, KEY[1]); - tmp5 = _mm_aesenc_si128(tmp5, KEY[1]); - tmp6 = _mm_aesenc_si128(tmp6, KEY[1]); - tmp7 = _mm_aesenc_si128(tmp7, KEY[1]); - tmp8 = _mm_aesenc_si128(tmp8, KEY[1]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[2]); - tmp2 = _mm_aesenc_si128(tmp2, KEY[2]); - tmp3 = _mm_aesenc_si128(tmp3, KEY[2]); - tmp4 = _mm_aesenc_si128(tmp4, KEY[2]); - tmp5 = _mm_aesenc_si128(tmp5, KEY[2]); - tmp6 = _mm_aesenc_si128(tmp6, KEY[2]); - tmp7 = _mm_aesenc_si128(tmp7, KEY[2]); - tmp8 = _mm_aesenc_si128(tmp8, KEY[2]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[3]); - tmp2 = _mm_aesenc_si128(tmp2, KEY[3]); - tmp3 = _mm_aesenc_si128(tmp3, KEY[3]); - tmp4 = _mm_aesenc_si128(tmp4, KEY[3]); - tmp5 = _mm_aesenc_si128(tmp5, KEY[3]); - tmp6 = _mm_aesenc_si128(tmp6, KEY[3]); - tmp7 = _mm_aesenc_si128(tmp7, KEY[3]); - tmp8 = _mm_aesenc_si128(tmp8, KEY[3]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[4]); - tmp2 = _mm_aesenc_si128(tmp2, KEY[4]); - tmp3 = _mm_aesenc_si128(tmp3, KEY[4]); - tmp4 = _mm_aesenc_si128(tmp4, KEY[4]); - tmp5 = _mm_aesenc_si128(tmp5, KEY[4]); - tmp6 = _mm_aesenc_si128(tmp6, KEY[4]); - tmp7 = _mm_aesenc_si128(tmp7, KEY[4]); - tmp8 = _mm_aesenc_si128(tmp8, KEY[4]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[5]); - tmp2 = _mm_aesenc_si128(tmp2, KEY[5]); - tmp3 = _mm_aesenc_si128(tmp3, KEY[5]); - tmp4 = _mm_aesenc_si128(tmp4, KEY[5]); - tmp5 = _mm_aesenc_si128(tmp5, KEY[5]); - tmp6 = _mm_aesenc_si128(tmp6, KEY[5]); - tmp7 = _mm_aesenc_si128(tmp7, KEY[5]); - tmp8 = _mm_aesenc_si128(tmp8, KEY[5]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[6]); - tmp2 = _mm_aesenc_si128(tmp2, KEY[6]); - tmp3 = _mm_aesenc_si128(tmp3, KEY[6]); - tmp4 = _mm_aesenc_si128(tmp4, KEY[6]); - tmp5 = _mm_aesenc_si128(tmp5, KEY[6]); - tmp6 = _mm_aesenc_si128(tmp6, KEY[6]); - tmp7 = _mm_aesenc_si128(tmp7, KEY[6]); - tmp8 = _mm_aesenc_si128(tmp8, KEY[6]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[7]); - tmp2 = _mm_aesenc_si128(tmp2, KEY[7]); - tmp3 = _mm_aesenc_si128(tmp3, KEY[7]); - tmp4 = _mm_aesenc_si128(tmp4, KEY[7]); - tmp5 = _mm_aesenc_si128(tmp5, KEY[7]); - tmp6 = _mm_aesenc_si128(tmp6, KEY[7]); - tmp7 = _mm_aesenc_si128(tmp7, KEY[7]); - tmp8 = _mm_aesenc_si128(tmp8, KEY[7]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[8]); - tmp2 = _mm_aesenc_si128(tmp2, KEY[8]); - tmp3 = _mm_aesenc_si128(tmp3, KEY[8]); - tmp4 = _mm_aesenc_si128(tmp4, KEY[8]); - tmp5 = _mm_aesenc_si128(tmp5, KEY[8]); - tmp6 = _mm_aesenc_si128(tmp6, KEY[8]); - tmp7 = _mm_aesenc_si128(tmp7, KEY[8]); - tmp8 = _mm_aesenc_si128(tmp8, KEY[8]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[9]); - tmp2 = _mm_aesenc_si128(tmp2, KEY[9]); - tmp3 = _mm_aesenc_si128(tmp3, KEY[9]); - tmp4 = _mm_aesenc_si128(tmp4, KEY[9]); - tmp5 = _mm_aesenc_si128(tmp5, KEY[9]); - tmp6 = _mm_aesenc_si128(tmp6, KEY[9]); - tmp7 = _mm_aesenc_si128(tmp7, KEY[9]); - tmp8 = _mm_aesenc_si128(tmp8, KEY[9]); + AES_ENC_8(1); + AES_ENC_8(2); + AES_ENC_8(3); + AES_ENC_8(4); + AES_ENC_8(5); + AES_ENC_8(6); + AES_ENC_8(7); + AES_ENC_8(8); + AES_ENC_8(9); lastKey = KEY[10]; if (nr > 10) { - tmp1 = _mm_aesenc_si128(tmp1, KEY[10]); - tmp2 = _mm_aesenc_si128(tmp2, KEY[10]); - tmp3 = _mm_aesenc_si128(tmp3, KEY[10]); - tmp4 = _mm_aesenc_si128(tmp4, KEY[10]); - tmp5 = _mm_aesenc_si128(tmp5, KEY[10]); - tmp6 = _mm_aesenc_si128(tmp6, KEY[10]); - tmp7 = _mm_aesenc_si128(tmp7, KEY[10]); - tmp8 = _mm_aesenc_si128(tmp8, KEY[10]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[11]); - tmp2 = _mm_aesenc_si128(tmp2, KEY[11]); - tmp3 = _mm_aesenc_si128(tmp3, KEY[11]); - tmp4 = _mm_aesenc_si128(tmp4, KEY[11]); - tmp5 = _mm_aesenc_si128(tmp5, KEY[11]); - tmp6 = _mm_aesenc_si128(tmp6, KEY[11]); - tmp7 = _mm_aesenc_si128(tmp7, KEY[11]); - tmp8 = _mm_aesenc_si128(tmp8, KEY[11]); + AES_ENC_8(10); + AES_ENC_8(11); lastKey = KEY[12]; if (nr > 12) { - tmp1 = _mm_aesenc_si128(tmp1, KEY[12]); - tmp2 = _mm_aesenc_si128(tmp2, KEY[12]); - tmp3 = _mm_aesenc_si128(tmp3, KEY[12]); - tmp4 = _mm_aesenc_si128(tmp4, KEY[12]); - tmp5 = _mm_aesenc_si128(tmp5, KEY[12]); - tmp6 = _mm_aesenc_si128(tmp6, KEY[12]); - tmp7 = _mm_aesenc_si128(tmp7, KEY[12]); - tmp8 = _mm_aesenc_si128(tmp8, KEY[12]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[13]); - tmp2 = _mm_aesenc_si128(tmp2, KEY[13]); - tmp3 = _mm_aesenc_si128(tmp3, KEY[13]); - tmp4 = _mm_aesenc_si128(tmp4, KEY[13]); - tmp5 = _mm_aesenc_si128(tmp5, KEY[13]); - tmp6 = _mm_aesenc_si128(tmp6, KEY[13]); - tmp7 = _mm_aesenc_si128(tmp7, KEY[13]); - tmp8 = _mm_aesenc_si128(tmp8, KEY[13]); + AES_ENC_8(12); + AES_ENC_8(13); lastKey = KEY[14]; } } - tmp1 =_mm_aesenclast_si128(tmp1, lastKey); - tmp2 =_mm_aesenclast_si128(tmp2, lastKey); - tmp3 =_mm_aesenclast_si128(tmp3, lastKey); - tmp4 =_mm_aesenclast_si128(tmp4, lastKey); - tmp5 =_mm_aesenclast_si128(tmp5, lastKey); - tmp6 =_mm_aesenclast_si128(tmp6, lastKey); - tmp7 =_mm_aesenclast_si128(tmp7, lastKey); - tmp8 =_mm_aesenclast_si128(tmp8, lastKey); - tmp1 = _mm_xor_si128(tmp1, _mm_loadu_si128(&((__m128i*)in)[0])); - tmp2 = _mm_xor_si128(tmp2, _mm_loadu_si128(&((__m128i*)in)[1])); - tmp3 = _mm_xor_si128(tmp3, _mm_loadu_si128(&((__m128i*)in)[2])); - tmp4 = _mm_xor_si128(tmp4, _mm_loadu_si128(&((__m128i*)in)[3])); - tmp5 = _mm_xor_si128(tmp5, _mm_loadu_si128(&((__m128i*)in)[4])); - tmp6 = _mm_xor_si128(tmp6, _mm_loadu_si128(&((__m128i*)in)[5])); - tmp7 = _mm_xor_si128(tmp7, _mm_loadu_si128(&((__m128i*)in)[6])); - tmp8 = _mm_xor_si128(tmp8, _mm_loadu_si128(&((__m128i*)in)[7])); - _mm_storeu_si128(&((__m128i*)out)[0], tmp1); - _mm_storeu_si128(&((__m128i*)out)[1], tmp2); - _mm_storeu_si128(&((__m128i*)out)[2], tmp3); - _mm_storeu_si128(&((__m128i*)out)[3], tmp4); - _mm_storeu_si128(&((__m128i*)out)[4], tmp5); - _mm_storeu_si128(&((__m128i*)out)[5], tmp6); - _mm_storeu_si128(&((__m128i*)out)[6], tmp7); - _mm_storeu_si128(&((__m128i*)out)[7], tmp8); + AES_ENC_LAST_8(); for (i=1; i < (int)(nbytes/16/8); i++) { r0 = _mm_setzero_si128(); @@ -4134,30 +4125,7 @@ static void AES_GCM_encrypt(const unsigned char *in, unsigned char *out, lastKey = KEY[14]; } } - tmp1 =_mm_aesenclast_si128(tmp1, lastKey); - tmp2 =_mm_aesenclast_si128(tmp2, lastKey); - tmp3 =_mm_aesenclast_si128(tmp3, lastKey); - tmp4 =_mm_aesenclast_si128(tmp4, lastKey); - tmp5 =_mm_aesenclast_si128(tmp5, lastKey); - tmp6 =_mm_aesenclast_si128(tmp6, lastKey); - tmp7 =_mm_aesenclast_si128(tmp7, lastKey); - tmp8 =_mm_aesenclast_si128(tmp8, lastKey); - tmp1 = _mm_xor_si128(tmp1, _mm_loadu_si128(&((__m128i*)in)[i*8+0])); - tmp2 = _mm_xor_si128(tmp2, _mm_loadu_si128(&((__m128i*)in)[i*8+1])); - tmp3 = _mm_xor_si128(tmp3, _mm_loadu_si128(&((__m128i*)in)[i*8+2])); - tmp4 = _mm_xor_si128(tmp4, _mm_loadu_si128(&((__m128i*)in)[i*8+3])); - tmp5 = _mm_xor_si128(tmp5, _mm_loadu_si128(&((__m128i*)in)[i*8+4])); - tmp6 = _mm_xor_si128(tmp6, _mm_loadu_si128(&((__m128i*)in)[i*8+5])); - tmp7 = _mm_xor_si128(tmp7, _mm_loadu_si128(&((__m128i*)in)[i*8+6])); - tmp8 = _mm_xor_si128(tmp8, _mm_loadu_si128(&((__m128i*)in)[i*8+7])); - _mm_storeu_si128(&((__m128i*)out)[i*8+0], tmp1); - _mm_storeu_si128(&((__m128i*)out)[i*8+1], tmp2); - _mm_storeu_si128(&((__m128i*)out)[i*8+2], tmp3); - _mm_storeu_si128(&((__m128i*)out)[i*8+3], tmp4); - _mm_storeu_si128(&((__m128i*)out)[i*8+4], tmp5); - _mm_storeu_si128(&((__m128i*)out)[i*8+5], tmp6); - _mm_storeu_si128(&((__m128i*)out)[i*8+6], tmp7); - _mm_storeu_si128(&((__m128i*)out)[i*8+7], tmp8); + AES_ENC_LAST_8(); } tmp1 = _mm_shuffle_epi8(tmp1, BSWAP_MASK); @@ -4315,46 +4283,356 @@ static void AES_GCM_encrypt(const unsigned char *in, unsigned char *out, _mm_storeu_si128((__m128i*)tag, T); } -#ifdef HAVE_INTEL_AVX2 +#ifdef HAVE_INTEL_AVX1 /* Encrypt with key in xmm12. */ -#define VAESENC() \ - "vaesenc %%xmm12, %[tmp1], %[tmp1]\n\t" \ - "vaesenc %%xmm12, %[tmp2], %[tmp2]\n\t" \ - "vaesenc %%xmm12, %[tmp3], %[tmp3]\n\t" \ - "vaesenc %%xmm12, %[tmp4], %[tmp4]\n\t" \ - "vaesenc %%xmm12, %[tmp5], %[tmp5]\n\t" \ - "vaesenc %%xmm12, %[tmp6], %[tmp6]\n\t" \ - "vaesenc %%xmm12, %[tmp7], %[tmp7]\n\t" \ - "vaesenc %%xmm12, %[tmp8], %[tmp8]\n\t" +#define VAESENC() \ + "vaesenc %%xmm12, %[tmp1], %[tmp1]\n\t" \ + "vaesenc %%xmm12, %[tmp2], %[tmp2]\n\t" \ + "vaesenc %%xmm12, %[tmp3], %[tmp3]\n\t" \ + "vaesenc %%xmm12, %[tmp4], %[tmp4]\n\t" \ + "vaesenc %%xmm12, %[tmp5], %[tmp5]\n\t" \ + "vaesenc %%xmm12, %[tmp6], %[tmp6]\n\t" \ + "vaesenc %%xmm12, %[tmp7], %[tmp7]\n\t" \ + "vaesenc %%xmm12, %[tmp8], %[tmp8]\n\t" + +#define VAESENC_SET(o) \ + "vmovaps "#o"(%[KEY]), %%xmm12\n\t" \ + VAESENC() + +#define VAESENC_CTR() \ + "vmovaps (%[pctr1]), %%xmm0\n\t" \ + "vmovaps %[BSWAP_EPI64], %%xmm1\n\t" \ + "vpshufb %%xmm1, %%xmm0, %[tmp1]\n\t" \ + "vpaddd %[ONE], %%xmm0, %[tmp2]\n\t" \ + "vpshufb %%xmm1, %[tmp2], %[tmp2]\n\t" \ + "vpaddd %[TWO], %%xmm0, %[tmp3]\n\t" \ + "vpshufb %%xmm1, %[tmp3], %[tmp3]\n\t" \ + "vpaddd %[THREE], %%xmm0, %[tmp4]\n\t" \ + "vpshufb %%xmm1, %[tmp4], %[tmp4]\n\t" \ + "vpaddd %[FOUR], %%xmm0, %[tmp5]\n\t" \ + "vpshufb %%xmm1, %[tmp5], %[tmp5]\n\t" \ + "vpaddd %[FIVE], %%xmm0, %[tmp6]\n\t" \ + "vpshufb %%xmm1, %[tmp6], %[tmp6]\n\t" \ + "vpaddd %[SIX], %%xmm0, %[tmp7]\n\t" \ + "vpshufb %%xmm1, %[tmp7], %[tmp7]\n\t" \ + "vpaddd %[SEVEN], %%xmm0, %[tmp8]\n\t" \ + "vpshufb %%xmm1, %[tmp8], %[tmp8]\n\t" \ + "vpaddd %[EIGHT], %%xmm0, %%xmm0\n\t" + +#define VAESENC_XOR() \ + "vmovaps (%[KEY]), %%xmm12\n\t" \ + "vmovaps %%xmm0, (%[pctr1])\n\t" \ + "vpxor %%xmm12, %[tmp1], %[tmp1]\n\t" \ + "vpxor %%xmm12, %[tmp2], %[tmp2]\n\t" \ + "vpxor %%xmm12, %[tmp3], %[tmp3]\n\t" \ + "vpxor %%xmm12, %[tmp4], %[tmp4]\n\t" \ + "vpxor %%xmm12, %[tmp5], %[tmp5]\n\t" \ + "vpxor %%xmm12, %[tmp6], %[tmp6]\n\t" \ + "vpxor %%xmm12, %[tmp7], %[tmp7]\n\t" \ + "vpxor %%xmm12, %[tmp8], %[tmp8]\n\t" -/* Encrypt and GCM mul with the nth round key. */ -#define VAESENC_PCLMUL_N(o1, o2, o3) \ - "vmovaps "#o1"(%[KEY]), %%xmm12\n\t" \ - "vmovdqu "#o2"(%[out]), %%xmm1\n\t" \ - "vaesenc %%xmm12, %[tmp1], %[tmp1]\n\t" \ - "vmovaps "#o3"(%[HT]), %%xmm0\n\t" \ - "vpshufb %[BSWAP_MASK], %%xmm1, %%xmm1\n\t" \ - "vaesenc %%xmm12, %[tmp2], %[tmp2]\n\t" \ - "vpclmulqdq $16, %%xmm1, %%xmm0, %%xmm13\n\t" \ - "vaesenc %%xmm12, %[tmp3], %[tmp3]\n\t" \ - "vpclmulqdq $1, %%xmm1, %%xmm0, %%xmm14\n\t" \ - "vaesenc %%xmm12, %[tmp4], %[tmp4]\n\t" \ - "vpclmulqdq $0, %%xmm1, %%xmm0, %%xmm15\n\t" \ - "vaesenc %%xmm12, %[tmp5], %[tmp5]\n\t" \ - "vpclmulqdq $17, %%xmm1, %%xmm0, %%xmm1\n\t" \ - "vaesenc %%xmm12, %[tmp6], %[tmp6]\n\t" \ - "vpxor %%xmm14, %%xmm13, %%xmm13\n\t" \ - "vpslldq $8, %%xmm13, %%xmm14\n\t" \ - "vpsrldq $8, %%xmm13, %%xmm13\n\t" \ - "vaesenc %%xmm12, %[tmp7], %[tmp7]\n\t" \ - "vpxor %%xmm15, %%xmm2, %%xmm2\n\t" \ - "vpxor %%xmm1, %%xmm3, %%xmm3\n\t" \ - "vpxor %%xmm14, %%xmm2, %%xmm2\n\t" \ - "vpxor %%xmm13, %%xmm3, %%xmm3\n\t" \ - "vaesenc %%xmm12, %[tmp8], %[tmp8]\n\t" +/* Encrypt and carry-less multiply for AVX1. */ +#define VAESENC_PCLMUL_1(src, o1, o2, o3) \ + "vmovdqa "#o3"(%[HT]), %%xmm12\n\t" \ + "vmovdqu "#o2"("#src"), %%xmm0\n\t" \ + "vaesenc "#o1"(%[KEY]), %[tmp1], %[tmp1]\n\t" \ + "vpshufb %[BSWAP_MASK], %%xmm0, %%xmm0\n\t" \ + "vpxor %[XV], %%xmm0, %%xmm0\n\t" \ + "vpshufd $78, %%xmm12, %%xmm1\n\t" \ + "vpshufd $78, %%xmm0, %%xmm14\n\t" \ + "vpxor %%xmm12, %%xmm1, %%xmm1\n\t" \ + "vpxor %%xmm0, %%xmm14, %%xmm14\n\t" \ + "vpclmulqdq $0x11, %%xmm12, %%xmm0, %%xmm3\n\t" \ + "vaesenc "#o1"(%[KEY]), %[tmp2], %[tmp2]\n\t" \ + "vaesenc "#o1"(%[KEY]), %[tmp3], %[tmp3]\n\t" \ + "vpclmulqdq $0x00, %%xmm12, %%xmm0, %%xmm2\n\t" \ + "vaesenc "#o1"(%[KEY]), %[tmp4], %[tmp4]\n\t" \ + "vaesenc "#o1"(%[KEY]), %[tmp5], %[tmp5]\n\t" \ + "vpclmulqdq $0x00, %%xmm14, %%xmm1, %%xmm1\n\t" \ + "vaesenc "#o1"(%[KEY]), %[tmp6], %[tmp6]\n\t" \ + "vaesenc "#o1"(%[KEY]), %[tmp7], %[tmp7]\n\t" \ + "vaesenc "#o1"(%[KEY]), %[tmp8], %[tmp8]\n\t" \ + "vpxor %%xmm2, %%xmm1, %%xmm1\n\t" \ + "vpxor %%xmm3, %%xmm1, %%xmm1\n\t" \ + +#define VAESENC_PCLMUL_N(src, o1, o2, o3) \ + "vmovdqa "#o3"(%[HT]), %%xmm12\n\t" \ + "vmovdqu "#o2"("#src"), %%xmm0\n\t" \ + "vpshufd $78, %%xmm12, %%xmm13\n\t" \ + "vpshufb %[BSWAP_MASK], %%xmm0, %%xmm0\n\t" \ + "vaesenc "#o1"(%[KEY]), %[tmp1], %[tmp1]\n\t" \ + "vpxor %%xmm12, %%xmm13, %%xmm13\n\t" \ + "vpshufd $78, %%xmm0, %%xmm14\n\t" \ + "vpxor %%xmm0, %%xmm14, %%xmm14\n\t" \ + "vpclmulqdq $0x11, %%xmm12, %%xmm0, %%xmm15\n\t" \ + "vaesenc "#o1"(%[KEY]), %[tmp2], %[tmp2]\n\t" \ + "vaesenc "#o1"(%[KEY]), %[tmp3], %[tmp3]\n\t" \ + "vpclmulqdq $0x00, %%xmm12, %%xmm0, %%xmm12\n\t" \ + "vaesenc "#o1"(%[KEY]), %[tmp4], %[tmp4]\n\t" \ + "vaesenc "#o1"(%[KEY]), %[tmp5], %[tmp5]\n\t" \ + "vpclmulqdq $0x00, %%xmm14, %%xmm13, %%xmm13\n\t" \ + "vaesenc "#o1"(%[KEY]), %[tmp6], %[tmp6]\n\t" \ + "vaesenc "#o1"(%[KEY]), %[tmp7], %[tmp7]\n\t" \ + "vaesenc "#o1"(%[KEY]), %[tmp8], %[tmp8]\n\t" \ + "vpxor %%xmm12, %%xmm1, %%xmm1\n\t" \ + "vpxor %%xmm12, %%xmm2, %%xmm2\n\t" \ + "vpxor %%xmm15, %%xmm1, %%xmm1\n\t" \ + "vpxor %%xmm15, %%xmm3, %%xmm3\n\t" \ + "vpxor %%xmm13, %%xmm1, %%xmm1\n\t" \ -static void AES_GCM_encrypt_avx2(const unsigned char *in, unsigned char *out, +#define VAESENC_PCLMUL_L(o) \ + "vpslldq $8, %%xmm1, %%xmm14\n\t" \ + "vpsrldq $8, %%xmm1, %%xmm1\n\t" \ + "vaesenc "#o"(%[KEY]), %[tmp1], %[tmp1]\n\t" \ + "vpxor %%xmm14, %%xmm2, %%xmm2\n\t" \ + "vpxor %%xmm1, %%xmm3, %%xmm3\n\t" \ + "vaesenc "#o"(%[KEY]), %[tmp2], %[tmp2]\n\t" \ + "vpslld $31, %%xmm2, %%xmm12\n\t" \ + "vpslld $30, %%xmm2, %%xmm13\n\t" \ + "vpslld $25, %%xmm2, %%xmm14\n\t" \ + "vaesenc "#o"(%[KEY]), %[tmp3], %[tmp3]\n\t" \ + "vpxor %%xmm13, %%xmm12, %%xmm12\n\t" \ + "vpxor %%xmm14, %%xmm12, %%xmm12\n\t" \ + "vaesenc "#o"(%[KEY]), %[tmp4], %[tmp4]\n\t" \ + "vpsrldq $4, %%xmm12, %%xmm13\n\t" \ + "vpslldq $12, %%xmm12, %%xmm12\n\t" \ + "vaesenc "#o"(%[KEY]), %[tmp5], %[tmp5]\n\t" \ + "vpxor %%xmm12, %%xmm2, %%xmm2\n\t" \ + "vpsrld $1, %%xmm2, %%xmm14\n\t" \ + "vaesenc "#o"(%[KEY]), %[tmp6], %[tmp6]\n\t" \ + "vpsrld $2, %%xmm2, %%xmm1\n\t" \ + "vpsrld $7, %%xmm2, %%xmm0\n\t" \ + "vaesenc "#o"(%[KEY]), %[tmp7], %[tmp7]\n\t" \ + "vpxor %%xmm1, %%xmm14, %%xmm14\n\t" \ + "vpxor %%xmm0, %%xmm14, %%xmm14\n\t" \ + "vaesenc "#o"(%[KEY]), %[tmp8], %[tmp8]\n\t" \ + "vpxor %%xmm13, %%xmm14, %%xmm14\n\t" \ + "vpxor %%xmm14, %%xmm2, %%xmm2\n\t" \ + "vpxor %%xmm3, %%xmm2, %%xmm2\n\t" \ + + +/* Encrypt and carry-less multiply for AVX2. */ +#define VAESENC_PCLMUL_AVX2_1(src, o1, o2, o3) \ + "vmovdqu "#o2"("#src"), %%xmm12\n\t" \ + "vmovdqa "#o1"(%[KEY]), %%xmm0\n\t" \ + "vpshufb %[BSWAP_MASK], %%xmm12, %%xmm12\n\t" \ + "vpxor %[XV], %%xmm12, %%xmm12\n\t" \ + "vpclmulqdq $0x10, "#o3"(%[HT]), %%xmm12, %%xmm1\n\t" \ + "vaesenc %%xmm0, %[tmp1], %[tmp1]\n\t" \ + "vaesenc %%xmm0, %[tmp2], %[tmp2]\n\t" \ + "vpclmulqdq $0x01, "#o3"(%[HT]), %%xmm12, %%xmm14\n\t" \ + "vaesenc %%xmm0, %[tmp3], %[tmp3]\n\t" \ + "vaesenc %%xmm0, %[tmp4], %[tmp4]\n\t" \ + "vpclmulqdq $0x00, "#o3"(%[HT]), %%xmm12, %%xmm2\n\t" \ + "vaesenc %%xmm0, %[tmp5], %[tmp5]\n\t" \ + "vaesenc %%xmm0, %[tmp6], %[tmp6]\n\t" \ + "vpclmulqdq $0x11, "#o3"(%[HT]), %%xmm12, %%xmm3\n\t" \ + "vaesenc %%xmm0, %[tmp7], %[tmp7]\n\t" \ + "vaesenc %%xmm0, %[tmp8], %[tmp8]\n\t" \ + +#define VAESENC_PCLMUL_AVX2_2(src, o1, o2, o3) \ + "vmovdqu "#o2"("#src"), %%xmm12\n\t" \ + "vmovdqa "#o1"(%[KEY]), %%xmm0\n\t" \ + "vpshufb %[BSWAP_MASK], %%xmm12, %%xmm12\n\t" \ + "vpxor %%xmm14, %%xmm1, %%xmm1\n\t" \ + "vpclmulqdq $0x10, "#o3"(%[HT]), %%xmm12, %%xmm13\n\t" \ + "vaesenc %%xmm0, %[tmp1], %[tmp1]\n\t" \ + "vaesenc %%xmm0, %[tmp2], %[tmp2]\n\t" \ + "vpclmulqdq $0x01, "#o3"(%[HT]), %%xmm12, %%xmm14\n\t" \ + "vaesenc %%xmm0, %[tmp3], %[tmp3]\n\t" \ + "vaesenc %%xmm0, %[tmp4], %[tmp4]\n\t" \ + "vpclmulqdq $0x00, "#o3"(%[HT]), %%xmm12, %%xmm15\n\t" \ + "vaesenc %%xmm0, %[tmp5], %[tmp5]\n\t" \ + "vaesenc %%xmm0, %[tmp6], %[tmp6]\n\t" \ + "vpclmulqdq $0x11, "#o3"(%[HT]), %%xmm12, %%xmm12\n\t" \ + "vaesenc %%xmm0, %[tmp7], %[tmp7]\n\t" \ + "vaesenc %%xmm0, %[tmp8], %[tmp8]\n\t" \ + "vpxor %%xmm13, %%xmm1, %%xmm1\n\t" \ + "vpxor %%xmm12, %%xmm3, %%xmm3\n\t" \ + "vpxor %%xmm14, %%xmm1, %%xmm1\n\t" \ + +#define VAESENC_PCLMUL_AVX2_N(src, o1, o2, o3) \ + "vmovdqu "#o2"("#src"), %%xmm12\n\t" \ + "vmovdqa "#o1"(%[KEY]), %%xmm0\n\t" \ + "vpshufb %[BSWAP_MASK], %%xmm12, %%xmm12\n\t" \ + "vpxor %%xmm15, %%xmm2, %%xmm2\n\t" \ + "vpclmulqdq $0x10, "#o3"(%[HT]), %%xmm12, %%xmm13\n\t" \ + "vaesenc %%xmm0, %[tmp1], %[tmp1]\n\t" \ + "vaesenc %%xmm0, %[tmp2], %[tmp2]\n\t" \ + "vpclmulqdq $0x01, "#o3"(%[HT]), %%xmm12, %%xmm14\n\t" \ + "vaesenc %%xmm0, %[tmp3], %[tmp3]\n\t" \ + "vaesenc %%xmm0, %[tmp4], %[tmp4]\n\t" \ + "vpclmulqdq $0x00, "#o3"(%[HT]), %%xmm12, %%xmm15\n\t" \ + "vaesenc %%xmm0, %[tmp5], %[tmp5]\n\t" \ + "vaesenc %%xmm0, %[tmp6], %[tmp6]\n\t" \ + "vpclmulqdq $0x11, "#o3"(%[HT]), %%xmm12, %%xmm12\n\t" \ + "vaesenc %%xmm0, %[tmp7], %[tmp7]\n\t" \ + "vaesenc %%xmm0, %[tmp8], %[tmp8]\n\t" \ + "vpxor %%xmm13, %%xmm1, %%xmm1\n\t" \ + "vpxor %%xmm12, %%xmm3, %%xmm3\n\t" \ + "vpxor %%xmm14, %%xmm1, %%xmm1\n\t" \ + +#define VAESENC_PCLMUL_AVX2_L(o) \ + "vpxor %%xmm15, %%xmm2, %%xmm2\n\t" \ + "vpslldq $8, %%xmm1, %%xmm12\n\t" \ + "vpsrldq $8, %%xmm1, %%xmm1\n\t" \ + "vmovdqa "#o"(%[KEY]), %%xmm15\n\t" \ + "vpxor %%xmm12, %%xmm2, %%xmm2\n\t" \ + "vpxor %%xmm1, %%xmm3, %%xmm3\n\t" \ + "vaesenc %%xmm15, %[tmp1], %[tmp1]\n\t" \ + "vmovdqa %[MOD2_128], %%xmm0\n\t" \ + "vpclmulqdq $0x10, %%xmm0, %%xmm2, %%xmm14\n\t" \ + "vaesenc %%xmm15, %[tmp2], %[tmp2]\n\t" \ + "vaesenc %%xmm15, %[tmp3], %[tmp3]\n\t" \ + "vaesenc %%xmm15, %[tmp4], %[tmp4]\n\t" \ + "vpshufd $78, %%xmm2, %%xmm13\n\t" \ + "vpxor %%xmm14, %%xmm13, %%xmm13\n\t" \ + "vpclmulqdq $0x10, %%xmm0, %%xmm13, %%xmm14\n\t" \ + "vaesenc %%xmm15, %[tmp5], %[tmp5]\n\t" \ + "vaesenc %%xmm15, %[tmp6], %[tmp6]\n\t" \ + "vaesenc %%xmm15, %[tmp7], %[tmp7]\n\t" \ + "vpshufd $78, %%xmm13, %%xmm13\n\t" \ + "vpxor %%xmm14, %%xmm13, %%xmm13\n\t" \ + "vpxor %%xmm3, %%xmm13, %%xmm13\n\t" \ + "vmovdqa %%xmm13, %%xmm2\n\t" \ + "vaesenc %%xmm15, %[tmp8], %[tmp8]\n\t" + + +/* Encrypt and carry-less multiply with last key. */ +#define VAESENC_LAST() \ + "vaesenclast %%xmm12, %[tmp1], %[tmp1]\n\t" \ + "vaesenclast %%xmm12, %[tmp2], %[tmp2]\n\t" \ + "vpxor (%[in]), %[tmp1], %[tmp1]\n\t" \ + "vpxor 16(%[in]), %[tmp2], %[tmp2]\n\t" \ + "vmovdqu %[tmp1], (%[out])\n\t" \ + "vmovdqu %[tmp2], 16(%[out])\n\t" \ + "vaesenclast %%xmm12, %[tmp3], %[tmp3]\n\t" \ + "vaesenclast %%xmm12, %[tmp4], %[tmp4]\n\t" \ + "vpxor 32(%[in]), %[tmp3], %[tmp3]\n\t" \ + "vpxor 48(%[in]), %[tmp4], %[tmp4]\n\t" \ + "vmovdqu %[tmp3], 32(%[out])\n\t" \ + "vmovdqu %[tmp4], 48(%[out])\n\t" \ + "vaesenclast %%xmm12, %[tmp5], %[tmp5]\n\t" \ + "vaesenclast %%xmm12, %[tmp6], %[tmp6]\n\t" \ + "vpxor 64(%[in]), %[tmp5], %[tmp5]\n\t" \ + "vpxor 80(%[in]), %[tmp6], %[tmp6]\n\t" \ + "vmovdqu %[tmp5], 64(%[out])\n\t" \ + "vmovdqu %[tmp6], 80(%[out])\n\t" \ + "vaesenclast %%xmm12, %[tmp7], %[tmp7]\n\t" \ + "vaesenclast %%xmm12, %[tmp8], %[tmp8]\n\t" \ + "vpxor 96(%[in]), %[tmp7], %[tmp7]\n\t" \ + "vpxor 112(%[in]), %[tmp8], %[tmp8]\n\t" \ + "vmovdqu %[tmp7], 96(%[out])\n\t" \ + "vmovdqu %[tmp8], 112(%[out])\n\t" + +#define VAESENC_BLOCK() \ + "vpshufb %[BSWAP_EPI64], %[ctr1], %[tmp1]\n\t" \ + "vpaddd %[ONE], %[ctr1], %[ctr1]\n\t" \ + "vpxor (%[KEY]), %[tmp1], %[tmp1]\n\t" \ + "vaesenc 16(%[KEY]), %[tmp1], %[tmp1]\n\t" \ + "vaesenc 32(%[KEY]), %[tmp1], %[tmp1]\n\t" \ + "vaesenc 48(%[KEY]), %[tmp1], %[tmp1]\n\t" \ + "vaesenc 64(%[KEY]), %[tmp1], %[tmp1]\n\t" \ + "vaesenc 80(%[KEY]), %[tmp1], %[tmp1]\n\t" \ + "vaesenc 96(%[KEY]), %[tmp1], %[tmp1]\n\t" \ + "vaesenc 112(%[KEY]), %[tmp1], %[tmp1]\n\t" \ + "vaesenc 128(%[KEY]), %[tmp1], %[tmp1]\n\t" \ + "vaesenc 144(%[KEY]), %[tmp1], %[tmp1]\n\t" \ + "cmpl $11, %[nr]\n\t" \ + "vmovaps 160(%[KEY]), %[tmp2]\n\t" \ + "jl %=f\n\t" \ + "vaesenc %[tmp2], %[tmp1], %[tmp1]\n\t" \ + "vaesenc 176(%[KEY]), %[tmp1], %[tmp1]\n\t" \ + "cmpl $13, %[nr]\n\t" \ + "vmovaps 192(%[KEY]), %[tmp2]\n\t" \ + "jl %=f\n\t" \ + "vaesenc %[tmp2], %[tmp1], %[tmp1]\n\t" \ + "vaesenc 208(%[KEY]), %[tmp1], %[tmp1]\n\t" \ + "vmovaps 224(%[KEY]), %[tmp2]\n\t" \ + "%=:\n\t" \ + "vaesenclast %[tmp2], %[tmp1], %[tmp1]\n\t" \ + "vpxor (%[in]), %[tmp1], %[tmp1]\n\t" \ + "vmovdqu %[tmp1], (%[out])\n\t" \ + "vpshufb %[BSWAP_MASK], %[tmp1], %[tmp1]\n\t" \ + "vpxor %[tmp1], %[X], %[X]\n\t" + +#define aes_gcm_avx1_calc_iv_12(kKEY, ivec, nr, H, Y, T, X) \ +do \ +{ \ + for (j=0; j < 12; j++) \ + ((unsigned char*)&Y)[j] = ivec[j]; \ + Y = _mm_insert_epi32(Y, 0x1000000, 3); \ + \ + __asm__ __volatile__ ( \ + "vmovaps 0(%[KEY]), %%xmm5\n\t" \ + "vmovaps 16(%[KEY]), %%xmm6\n\t" \ + "vpxor %%xmm5, %[X], %[H]\n\t" \ + "vpxor %%xmm5, %[Y], %[T]\n\t" \ + "vaesenc %%xmm6, %[H], %[H]\n\t" \ + "vaesenc %%xmm6, %[T], %[T]\n\t" \ + "vmovaps 32(%[KEY]), %%xmm5\n\t" \ + "vmovaps 48(%[KEY]), %%xmm6\n\t" \ + "vaesenc %%xmm5, %[H], %[H]\n\t" \ + "vaesenc %%xmm5, %[T], %[T]\n\t" \ + "vaesenc %%xmm6, %[H], %[H]\n\t" \ + "vaesenc %%xmm6, %[T], %[T]\n\t" \ + "vmovaps 64(%[KEY]), %%xmm5\n\t" \ + "vmovaps 80(%[KEY]), %%xmm6\n\t" \ + "vaesenc %%xmm5, %[H], %[H]\n\t" \ + "vaesenc %%xmm5, %[T], %[T]\n\t" \ + "vaesenc %%xmm6, %[H], %[H]\n\t" \ + "vaesenc %%xmm6, %[T], %[T]\n\t" \ + "vmovaps 96(%[KEY]), %%xmm5\n\t" \ + "vmovaps 112(%[KEY]), %%xmm6\n\t" \ + "vaesenc %%xmm5, %[H], %[H]\n\t" \ + "vaesenc %%xmm5, %[T], %[T]\n\t" \ + "vaesenc %%xmm6, %[H], %[H]\n\t" \ + "vaesenc %%xmm6, %[T], %[T]\n\t" \ + "vmovaps 128(%[KEY]), %%xmm5\n\t" \ + "vmovaps 144(%[KEY]), %%xmm6\n\t" \ + "vaesenc %%xmm5, %[H], %[H]\n\t" \ + "vaesenc %%xmm5, %[T], %[T]\n\t" \ + "vaesenc %%xmm6, %[H], %[H]\n\t" \ + "vaesenc %%xmm6, %[T], %[T]\n\t" \ + "cmpl $11, %[nr]\n\t" \ + "vmovaps 160(%[KEY]), %%xmm5\n\t" \ + "jl %=f\n\t" \ + "vaesenc %%xmm5, %[H], %[H]\n\t" \ + "vaesenc %%xmm5, %[T], %[T]\n\t" \ + "vmovaps 176(%[KEY]), %%xmm6\n\t" \ + "vmovaps 192(%[KEY]), %%xmm5\n\t" \ + "vaesenc %%xmm6, %[H], %[H]\n\t" \ + "vaesenc %%xmm6, %[T], %[T]\n\t" \ + "cmpl $13, %[nr]\n\t" \ + "jl %=f\n\t" \ + "vaesenc %%xmm5, %[H], %[H]\n\t" \ + "vaesenc %%xmm5, %[T], %[T]\n\t" \ + "vmovaps 208(%[KEY]), %%xmm6\n\t" \ + "vmovaps 224(%[KEY]), %%xmm5\n\t" \ + "vaesenc %%xmm6, %[H], %[H]\n\t" \ + "vaesenc %%xmm6, %[T], %[T]\n\t" \ + "%=:\n\t" \ + "vaesenclast %%xmm5, %[H], %[H]\n\t" \ + "vaesenclast %%xmm5, %[T], %[T]\n\t" \ + "vpshufb %[BSWAP_MASK], %[H], %[H]\n\t" \ + \ + : [H] "=xr" (H), [Y] "+xr" (Y), [T] "=xr" (T), \ + [X] "+xr" (X) \ + : [KEY] "r" (KEY), [nr] "r" (nr), \ + [BSWAP_MASK] "m" (BSWAP_MASK) \ + : "memory", "xmm5", "xmm6" \ + ); \ +} \ +while (0) + +void AES_GCM_encrypt_avx1(const unsigned char *in, unsigned char *out, + const unsigned char* addt, + const unsigned char* ivec, + unsigned char *tag, unsigned int nbytes, + unsigned int abytes, unsigned int ibytes, + const unsigned char* key, int nr); +void AES_GCM_encrypt_avx1(const unsigned char *in, unsigned char *out, const unsigned char* addt, const unsigned char* ivec, unsigned char *tag, unsigned int nbytes, @@ -4363,11 +4641,12 @@ static void AES_GCM_encrypt_avx2(const unsigned char *in, unsigned char *out, { int i, j ,k; __m128i ctr1; - __m128i H, Y, T; + __m128i H, T; __m128i X = _mm_setzero_si128(); + __m128i Y = _mm_setzero_si128(); __m128i *KEY = (__m128i*)key, lastKey; __m128i last_block = _mm_setzero_si128(); -#if !defined(AES_GCM_AESNI_NO_UNROLL) && !defined(AES_GCM_AVX2_NO_UNROLL) +#if !defined(AES_GCM_AESNI_NO_UNROLL) && !defined(AES_GCM_AVX1_NO_UNROLL) __m128i HT[8]; register __m128i tmp1 asm("xmm4"); register __m128i tmp2 asm("xmm5"); @@ -4383,121 +4662,10 @@ static void AES_GCM_encrypt_avx2(const unsigned char *in, unsigned char *out, __m128i tmp1, tmp2; #endif - if (ibytes == 12) { - Y = _mm_setzero_si128(); - for (j=0; j < 12; j++) - ((unsigned char*)&Y)[j] = ivec[j]; - Y = _mm_insert_epi32(Y, 0x1000000, 3); - /* (Compute E[ZERO, KS] and E[Y0, KS] together */ - tmp1 = _mm_xor_si128(X, KEY[0]); - tmp2 = _mm_xor_si128(Y, KEY[0]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[1]); - tmp2 = _mm_aesenc_si128(tmp2, KEY[1]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[2]); - tmp2 = _mm_aesenc_si128(tmp2, KEY[2]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[3]); - tmp2 = _mm_aesenc_si128(tmp2, KEY[3]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[4]); - tmp2 = _mm_aesenc_si128(tmp2, KEY[4]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[5]); - tmp2 = _mm_aesenc_si128(tmp2, KEY[5]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[6]); - tmp2 = _mm_aesenc_si128(tmp2, KEY[6]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[7]); - tmp2 = _mm_aesenc_si128(tmp2, KEY[7]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[8]); - tmp2 = _mm_aesenc_si128(tmp2, KEY[8]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[9]); - tmp2 = _mm_aesenc_si128(tmp2, KEY[9]); - lastKey = KEY[10]; - if (nr > 10) { - tmp1 = _mm_aesenc_si128(tmp1, lastKey); - tmp2 = _mm_aesenc_si128(tmp2, lastKey); - tmp1 = _mm_aesenc_si128(tmp1, KEY[11]); - tmp2 = _mm_aesenc_si128(tmp2, KEY[11]); - lastKey = KEY[12]; - if (nr > 12) { - tmp1 = _mm_aesenc_si128(tmp1, lastKey); - tmp2 = _mm_aesenc_si128(tmp2, lastKey); - tmp1 = _mm_aesenc_si128(tmp1, KEY[13]); - tmp2 = _mm_aesenc_si128(tmp2, KEY[13]); - lastKey = KEY[14]; - } - } - H = _mm_aesenclast_si128(tmp1, lastKey); - T = _mm_aesenclast_si128(tmp2, lastKey); - H = _mm_shuffle_epi8(H, BSWAP_MASK); - } - else { - if (ibytes % 16) { - i = ibytes / 16; - for (j=0; j < (int)(ibytes%16); j++) - ((unsigned char*)&last_block)[j] = ivec[i*16+j]; - } - tmp1 = _mm_xor_si128(X, KEY[0]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[1]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[2]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[3]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[4]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[5]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[6]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[7]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[8]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[9]); - lastKey = KEY[10]; - if (nr > 10) { - tmp1 = _mm_aesenc_si128(tmp1, lastKey); - tmp1 = _mm_aesenc_si128(tmp1, KEY[11]); - lastKey = KEY[12]; - if (nr > 12) { - tmp1 = _mm_aesenc_si128(tmp1, lastKey); - tmp1 = _mm_aesenc_si128(tmp1, KEY[13]); - lastKey = KEY[14]; - } - } - H = _mm_aesenclast_si128(tmp1, lastKey); - H = _mm_shuffle_epi8(H, BSWAP_MASK); - Y = _mm_setzero_si128(); - for (i=0; i < (int)(ibytes/16); i++) { - tmp1 = _mm_loadu_si128(&((__m128i*)ivec)[i]); - tmp1 = _mm_shuffle_epi8(tmp1, BSWAP_MASK); - Y = _mm_xor_si128(Y, tmp1); - Y = gfmul_sw(Y, H); - } - if (ibytes % 16) { - tmp1 = last_block; - tmp1 = _mm_shuffle_epi8(tmp1, BSWAP_MASK); - Y = _mm_xor_si128(Y, tmp1); - Y = gfmul_sw(Y, H); - } - tmp1 = _mm_insert_epi64(tmp1, ibytes*8, 0); - tmp1 = _mm_insert_epi64(tmp1, 0, 1); - Y = _mm_xor_si128(Y, tmp1); - Y = gfmul_sw(Y, H); - Y = _mm_shuffle_epi8(Y, BSWAP_MASK); /* Compute E(K, Y0) */ - tmp1 = _mm_xor_si128(Y, KEY[0]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[1]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[2]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[3]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[4]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[5]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[6]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[7]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[8]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[9]); - lastKey = KEY[10]; - if (nr > 10) { - tmp1 = _mm_aesenc_si128(tmp1, lastKey); - tmp1 = _mm_aesenc_si128(tmp1, KEY[11]); - lastKey = KEY[12]; - if (nr > 12) { - tmp1 = _mm_aesenc_si128(tmp1, lastKey); - tmp1 = _mm_aesenc_si128(tmp1, KEY[13]); - lastKey = KEY[14]; - } - } - T = _mm_aesenclast_si128(tmp1, lastKey); - } + if (ibytes == 12) + aes_gcm_avx1_calc_iv_12(KEY, ivec, nr, H, Y, T, X); + else + aes_gcm_calc_iv(KEY, ivec, ibytes, nr, H, Y, T, X); for (i=0; i < (int)(abytes/16); i++) { tmp1 = _mm_loadu_si128(&((__m128i*)addt)[i]); @@ -4519,7 +4687,7 @@ static void AES_GCM_encrypt_avx2(const unsigned char *in, unsigned char *out, ctr1 = _mm_add_epi32(tmp1, ONE); H = gfmul_shl1(H); -#if !defined(AES_GCM_AESNI_NO_UNROLL) && !defined(AES_GCM_AVX2_NO_UNROLL) +#if !defined(AES_GCM_AESNI_NO_UNROLL) && !defined(AES_GCM_AVX1_NO_UNROLL) i = 0; if (nbytes >= 16*8) { HT[0] = H; @@ -4533,95 +4701,33 @@ static void AES_GCM_encrypt_avx2(const unsigned char *in, unsigned char *out, pctr1[0] = ctr1; __asm__ __volatile__ ( - "vmovaps (%[pctr1]), %%xmm0\n\t" - "vmovaps %[BSWAP_EPI64], %%xmm1\n\t" - "vpshufb %%xmm1, %%xmm0, %[tmp1]\n\t" - "vpaddd %[ONE], %%xmm0, %[tmp2]\n\t" - "vpshufb %%xmm1, %[tmp2], %[tmp2]\n\t" - "vpaddd %[TWO], %%xmm0, %[tmp3]\n\t" - "vpshufb %%xmm1, %[tmp3], %[tmp3]\n\t" - "vpaddd %[THREE], %%xmm0, %[tmp4]\n\t" - "vpshufb %%xmm1, %[tmp4], %[tmp4]\n\t" - "vpaddd %[FOUR], %%xmm0, %[tmp5]\n\t" - "vpshufb %%xmm1, %[tmp5], %[tmp5]\n\t" - "vpaddd %[FIVE], %%xmm0, %[tmp6]\n\t" - "vpshufb %%xmm1, %[tmp6], %[tmp6]\n\t" - "vpaddd %[SIX], %%xmm0, %[tmp7]\n\t" - "vpshufb %%xmm1, %[tmp7], %[tmp7]\n\t" - "vpaddd %[SEVEN], %%xmm0, %[tmp8]\n\t" - "vpshufb %%xmm1, %[tmp8], %[tmp8]\n\t" - "vpaddd %[EIGHT], %%xmm0, %%xmm0\n\t" - - "vmovaps (%[KEY]), %%xmm1\n\t" - "vmovaps %%xmm0, (%[pctr1])\n\t" - "vpxor %%xmm1, %[tmp1], %[tmp1]\n\t" - "vpxor %%xmm1, %[tmp2], %[tmp2]\n\t" - "vpxor %%xmm1, %[tmp3], %[tmp3]\n\t" - "vpxor %%xmm1, %[tmp4], %[tmp4]\n\t" - "vpxor %%xmm1, %[tmp5], %[tmp5]\n\t" - "vpxor %%xmm1, %[tmp6], %[tmp6]\n\t" - "vpxor %%xmm1, %[tmp7], %[tmp7]\n\t" - "vpxor %%xmm1, %[tmp8], %[tmp8]\n\t" - - "vmovaps 16(%[KEY]), %%xmm12\n\t" - VAESENC() - "vmovaps 32(%[KEY]), %%xmm12\n\t" - VAESENC() - "vmovaps 48(%[KEY]), %%xmm12\n\t" - VAESENC() - "vmovaps 64(%[KEY]), %%xmm12\n\t" - VAESENC() - "vmovaps 80(%[KEY]), %%xmm12\n\t" - VAESENC() - "vmovaps 96(%[KEY]), %%xmm12\n\t" - VAESENC() - "vmovaps 112(%[KEY]), %%xmm12\n\t" - VAESENC() - "vmovaps 128(%[KEY]), %%xmm12\n\t" - VAESENC() - "vmovaps 144(%[KEY]), %%xmm12\n\t" - VAESENC() + VAESENC_CTR() + VAESENC_XOR() + VAESENC_SET(16) + VAESENC_SET(32) + VAESENC_SET(48) + VAESENC_SET(64) + VAESENC_SET(80) + VAESENC_SET(96) + VAESENC_SET(112) + VAESENC_SET(128) + VAESENC_SET(144) "cmpl $11, %[nr]\n\t" "vmovaps 160(%[KEY]), %%xmm12\n\t" - "jl L_enc128_enclast\n\t" + "jl 1f\n\t" VAESENC() - "vmovaps 176(%[KEY]), %%xmm12\n\t" - VAESENC() + VAESENC_SET(176) "cmpl $13, %[nr]\n\t" "vmovaps 192(%[KEY]), %%xmm12\n\t" - "jl L_enc128_enclast\n\t" + "jl 1f\n\t" VAESENC() - "vmovaps 208(%[KEY]), %%xmm12\n\t" - VAESENC() + VAESENC_SET(208) "vmovaps 224(%[KEY]), %%xmm12\n\t" "\n" - "L_enc128_enclast:\n\t" - "vaesenclast %%xmm12, %[tmp1], %[tmp1]\n\t" - "vaesenclast %%xmm12, %[tmp2], %[tmp2]\n\t" - "vpxor (%[in]), %[tmp1], %[tmp1]\n\t" - "vpxor 16(%[in]), %[tmp2], %[tmp2]\n\t" - "vmovdqu %[tmp1], (%[out])\n\t" - "vmovdqu %[tmp2], 16(%[out])\n\t" - "vaesenclast %%xmm12, %[tmp3], %[tmp3]\n\t" - "vaesenclast %%xmm12, %[tmp4], %[tmp4]\n\t" - "vpxor 32(%[in]), %[tmp3], %[tmp3]\n\t" - "vpxor 48(%[in]), %[tmp4], %[tmp4]\n\t" - "vmovdqu %[tmp3], 32(%[out])\n\t" - "vmovdqu %[tmp4], 48(%[out])\n\t" - "vaesenclast %%xmm12, %[tmp5], %[tmp5]\n\t" - "vaesenclast %%xmm12, %[tmp6], %[tmp6]\n\t" - "vpxor 64(%[in]), %[tmp5], %[tmp5]\n\t" - "vpxor 80(%[in]), %[tmp6], %[tmp6]\n\t" - "vmovdqu %[tmp5], 64(%[out])\n\t" - "vmovdqu %[tmp6], 80(%[out])\n\t" - "vaesenclast %%xmm12, %[tmp7], %[tmp7]\n\t" - "vaesenclast %%xmm12, %[tmp8], %[tmp8]\n\t" - "vpxor 96(%[in]), %[tmp7], %[tmp7]\n\t" - "vpxor 112(%[in]), %[tmp8], %[tmp8]\n\t" - "vmovdqu %[tmp7], 96(%[out])\n\t" - "vmovdqu %[tmp8], 112(%[out])\n\t" + "1:\n\t" + VAESENC_LAST() : [tmp1] "=xr" (tmp1), [tmp2] "=xr" (tmp2), [tmp3] "=xr" (tmp3), [tmp4] "=xr" (tmp4), [tmp5] "=xr" (tmp5), [tmp6] "=xr" (tmp6), @@ -4640,126 +4746,34 @@ static void AES_GCM_encrypt_avx2(const unsigned char *in, unsigned char *out, XV = X; for (i=1; i < (int)(nbytes/16/8); i++) { __asm__ __volatile__ ( - "vmovaps (%[pctr1]), %%xmm0\n\t" - "vmovaps %[BSWAP_EPI64], %%xmm1\n\t" - "vpshufb %%xmm1, %%xmm0, %[tmp1]\n\t" - "vpaddd %[ONE], %%xmm0, %[tmp2]\n\t" - "vpshufb %%xmm1, %[tmp2], %[tmp2]\n\t" - "vpaddd %[TWO], %%xmm0, %[tmp3]\n\t" - "vpshufb %%xmm1, %[tmp3], %[tmp3]\n\t" - "vpaddd %[THREE], %%xmm0, %[tmp4]\n\t" - "vpshufb %%xmm1, %[tmp4], %[tmp4]\n\t" - "vpaddd %[FOUR], %%xmm0, %[tmp5]\n\t" - "vpshufb %%xmm1, %[tmp5], %[tmp5]\n\t" - "vpaddd %[FIVE], %%xmm0, %[tmp6]\n\t" - "vpshufb %%xmm1, %[tmp6], %[tmp6]\n\t" - "vpaddd %[SIX], %%xmm0, %[tmp7]\n\t" - "vpshufb %%xmm1, %[tmp7], %[tmp7]\n\t" - "vpaddd %[SEVEN], %%xmm0, %[tmp8]\n\t" - "vpshufb %%xmm1, %[tmp8], %[tmp8]\n\t" - "vpaddd %[EIGHT], %%xmm0, %%xmm0\n\t" + VAESENC_CTR() + VAESENC_XOR() + VAESENC_PCLMUL_1(%[out], 16, -128, 112) + VAESENC_PCLMUL_N(%[out], 32, -112, 96) + VAESENC_PCLMUL_N(%[out], 48, -96, 80) + VAESENC_PCLMUL_N(%[out], 64, -80, 64) + VAESENC_PCLMUL_N(%[out], 80, -64, 48) + VAESENC_PCLMUL_N(%[out], 96, -48, 32) + VAESENC_PCLMUL_N(%[out], 112, -32, 16) + VAESENC_PCLMUL_N(%[out], 128, -16, 0) + VAESENC_PCLMUL_L(144) - "vmovaps (%[KEY]), %%xmm1\n\t" - "vmovaps %%xmm0, (%[pctr1])\n\t" - "vpxor %%xmm1, %[tmp1], %[tmp1]\n\t" - "vpxor %%xmm1, %[tmp2], %[tmp2]\n\t" - "vpxor %%xmm1, %[tmp3], %[tmp3]\n\t" - "vpxor %%xmm1, %[tmp4], %[tmp4]\n\t" - "vpxor %%xmm1, %[tmp5], %[tmp5]\n\t" - "vpxor %%xmm1, %[tmp6], %[tmp6]\n\t" - "vpxor %%xmm1, %[tmp7], %[tmp7]\n\t" - "vpxor %%xmm1, %[tmp8], %[tmp8]\n\t" - - "vmovaps 16(%[KEY]), %%xmm12\n\t" - "vmovdqu -128(%[out]), %%xmm1\n\t" - "vaesenc %%xmm12, %[tmp1], %[tmp1]\n\t" - "vmovaps 112(%[HT]), %%xmm0\n\t" - "vpshufb %[BSWAP_MASK], %%xmm1, %%xmm1\n\t" - "vpxor %[XV], %%xmm1, %%xmm1\n\t" - "vaesenc %%xmm12, %[tmp2], %[tmp2]\n\t" - "vpclmulqdq $16, %%xmm1, %%xmm0, %%xmm13\n\t" - "vaesenc %%xmm12, %[tmp3], %[tmp3]\n\t" - "vpclmulqdq $1, %%xmm1, %%xmm0, %%xmm14\n\t" - "vaesenc %%xmm12, %[tmp4], %[tmp4]\n\t" - "vpclmulqdq $0, %%xmm1, %%xmm0, %%xmm15\n\t" - "vaesenc %%xmm12, %[tmp5], %[tmp5]\n\t" - "vpclmulqdq $17, %%xmm1, %%xmm0, %%xmm1\n\t" - "vaesenc %%xmm12, %[tmp6], %[tmp6]\n\t" - "vpxor %%xmm14, %%xmm13, %%xmm13\n\t" - "vpslldq $8, %%xmm13, %%xmm2\n\t" - "vpsrldq $8, %%xmm13, %%xmm13\n\t" - "vaesenc %%xmm12, %[tmp7], %[tmp7]\n\t" - "vpxor %%xmm15, %%xmm2, %%xmm2\n\t" - "vpxor %%xmm13, %%xmm1, %%xmm3\n\t" - "vaesenc %%xmm12, %[tmp8], %[tmp8]\n\t" - - VAESENC_PCLMUL_N( 32, -112, 96) - VAESENC_PCLMUL_N( 48, -96, 80) - VAESENC_PCLMUL_N( 64, -80, 64) - VAESENC_PCLMUL_N( 80, -64, 48) - VAESENC_PCLMUL_N( 96, -48, 32) - VAESENC_PCLMUL_N(112, -32, 16) - VAESENC_PCLMUL_N(128, -16, 0) - - "vmovaps 144(%[KEY]), %%xmm12\n\t" - "vaesenc %%xmm12, %[tmp1], %[tmp1]\n\t" - "vmovdqa %[MOD2_128], %%xmm0\n\t" - "vaesenc %%xmm12, %[tmp2], %[tmp2]\n\t" - "vpclmulqdq $16, %%xmm0, %%xmm2, %%xmm14\n\t" - "vaesenc %%xmm12, %[tmp3], %[tmp3]\n\t" - "vpshufd $78, %%xmm2, %%xmm13\n\t" - "vpxor %%xmm14, %%xmm13, %%xmm13\n\t" - "vaesenc %%xmm12, %[tmp4], %[tmp4]\n\t" - "vpclmulqdq $16, %%xmm0, %%xmm13, %%xmm14\n\t" - "vaesenc %%xmm12, %[tmp5], %[tmp5]\n\t" - "vpshufd $78, %%xmm13, %%xmm13\n\t" - "vpxor %%xmm14, %%xmm13, %%xmm13\n\t" - "vpxor %%xmm3, %%xmm13, %%xmm13\n\t" - "vaesenc %%xmm12, %[tmp6], %[tmp6]\n\t" - "vmovdqa %%xmm13, %%xmm2\n\t" - "vaesenc %%xmm12, %[tmp7], %[tmp7]\n\t" - "vaesenc %%xmm12, %[tmp8], %[tmp8]\n\t" "cmpl $11, %[nr]\n\t" "vmovaps 160(%[KEY]), %%xmm12\n\t" "jl %=f\n\t" VAESENC() - "vmovaps 176(%[KEY]), %%xmm12\n\t" - VAESENC() + VAESENC_SET(176) "cmpl $13, %[nr]\n\t" "vmovaps 192(%[KEY]), %%xmm12\n\t" "jl %=f\n\t" VAESENC() - "vmovaps 208(%[KEY]), %%xmm12\n\t" - VAESENC() + VAESENC_SET(208) "vmovaps 224(%[KEY]), %%xmm12\n\t" "%=:\n\t" - "vaesenclast %%xmm12, %[tmp1], %[tmp1]\n\t" - "vaesenclast %%xmm12, %[tmp2], %[tmp2]\n\t" - "vpxor (%[in]), %[tmp1], %[tmp1]\n\t" - "vpxor 16(%[in]), %[tmp2], %[tmp2]\n\t" - "vmovdqu %[tmp1], (%[out])\n\t" - "vmovdqu %[tmp2], 16(%[out])\n\t" - "vaesenclast %%xmm12, %[tmp3], %[tmp3]\n\t" - "vaesenclast %%xmm12, %[tmp4], %[tmp4]\n\t" - "vpxor 32(%[in]), %[tmp3], %[tmp3]\n\t" - "vpxor 48(%[in]), %[tmp4], %[tmp4]\n\t" - "vmovdqu %[tmp3], 32(%[out])\n\t" - "vmovdqu %[tmp4], 48(%[out])\n\t" - "vaesenclast %%xmm12, %[tmp5], %[tmp5]\n\t" - "vaesenclast %%xmm12, %[tmp6], %[tmp6]\n\t" - "vpxor 64(%[in]), %[tmp5], %[tmp5]\n\t" - "vpxor 80(%[in]), %[tmp6], %[tmp6]\n\t" - "vmovdqu %[tmp5], 64(%[out])\n\t" - "vmovdqu %[tmp6], 80(%[out])\n\t" - "vaesenclast %%xmm12, %[tmp7], %[tmp7]\n\t" - "vaesenclast %%xmm12, %[tmp8], %[tmp8]\n\t" - "vpxor 96(%[in]), %[tmp7], %[tmp7]\n\t" - "vpxor 112(%[in]), %[tmp8], %[tmp8]\n\t" - "vmovdqu %[tmp7], 96(%[out])\n\t" - "vmovdqu %[tmp8], 112(%[out])\n\t" + VAESENC_LAST() : [tmp1] "=xr" (tmp1), [tmp2] "=xr" (tmp2), [tmp3] "=xr" (tmp3), [tmp4] "=xr" (tmp4), [tmp5] "=xr" (tmp5), [tmp6] "=xr" (tmp6), @@ -4794,36 +4808,8 @@ static void AES_GCM_encrypt_avx2(const unsigned char *in, unsigned char *out, } for (k = i*8; k < (int)(nbytes/16); k++) { __asm__ __volatile__ ( - "vpshufb %[BSWAP_EPI64], %[ctr1], %[tmp1]\n\t" - "vpaddd %[ONE], %[ctr1], %[ctr1]\n\t" - "vpxor (%[KEY]), %[tmp1], %[tmp1]\n\t" - "vaesenc 16(%[KEY]), %[tmp1], %[tmp1]\n\t" - "vaesenc 32(%[KEY]), %[tmp1], %[tmp1]\n\t" - "vaesenc 48(%[KEY]), %[tmp1], %[tmp1]\n\t" - "vaesenc 64(%[KEY]), %[tmp1], %[tmp1]\n\t" - "vaesenc 80(%[KEY]), %[tmp1], %[tmp1]\n\t" - "vaesenc 96(%[KEY]), %[tmp1], %[tmp1]\n\t" - "vaesenc 112(%[KEY]), %[tmp1], %[tmp1]\n\t" - "vaesenc 128(%[KEY]), %[tmp1], %[tmp1]\n\t" - "vaesenc 144(%[KEY]), %[tmp1], %[tmp1]\n\t" - "cmpl $11, %[nr]\n\t" - "vmovaps 160(%[KEY]), %[tmp2]\n\t" - "jl %=f\n\t" - "vaesenc %[tmp2], %[tmp1], %[tmp1]\n\t" - "vaesenc 176(%[KEY]), %[tmp1], %[tmp1]\n\t" - "cmpl $13, %[nr]\n\t" - "vmovaps 192(%[KEY]), %[tmp2]\n\t" - "jl %=f\n\t" - "vaesenc %[tmp2], %[tmp1], %[tmp1]\n\t" - "vaesenc 208(%[KEY]), %[tmp1], %[tmp1]\n\t" - "vmovaps 224(%[KEY]), %[tmp2]\n\t" - "%=:\n\t" - "vaesenclast %[tmp2], %[tmp1], %[tmp1]\n\t" - "vpxor (%[in]), %[tmp1], %[tmp1]\n\t" - "vmovdqu %[tmp1], (%[out])\n\t" - "vpshufb %[BSWAP_MASK], %[tmp1], %[tmp1]\n\t" + VAESENC_BLOCK() - "vpxor %[tmp1], %[X], %[X]\n\t" "# Carryless Multiply X by H (128 x 128)\n\t" "vpclmulqdq $16, %[H], %[X], %%xmm13\n\t" "vpclmulqdq $1, %[H], %[X], %%xmm14\n\t" @@ -4861,35 +4847,7 @@ static void AES_GCM_encrypt_avx2(const unsigned char *in, unsigned char *out, #else for (k = 0; k < (int)(nbytes/16) && k < 1; k++) { __asm__ __volatile__ ( - "vpshufb %[BSWAP_EPI64], %[ctr1], %[tmp1]\n\t" - "vpaddd %[ONE], %[ctr1], %[ctr1]\n\t" - "vpxor (%[KEY]), %[tmp1], %[tmp1]\n\t" - "vaesenc 16(%[KEY]), %[tmp1], %[tmp1]\n\t" - "vaesenc 32(%[KEY]), %[tmp1], %[tmp1]\n\t" - "vaesenc 48(%[KEY]), %[tmp1], %[tmp1]\n\t" - "vaesenc 64(%[KEY]), %[tmp1], %[tmp1]\n\t" - "vaesenc 80(%[KEY]), %[tmp1], %[tmp1]\n\t" - "vaesenc 96(%[KEY]), %[tmp1], %[tmp1]\n\t" - "vaesenc 112(%[KEY]), %[tmp1], %[tmp1]\n\t" - "vaesenc 128(%[KEY]), %[tmp1], %[tmp1]\n\t" - "vaesenc 144(%[KEY]), %[tmp1], %[tmp1]\n\t" - "cmpl $11, %[nr]\n\t" - "vmovaps 160(%[KEY]), %[tmp2]\n\t" - "jl %=f\n\t" - "vaesenc %[tmp2], %[tmp1], %[tmp1]\n\t" - "vaesenc 176(%[KEY]), %[tmp1], %[tmp1]\n\t" - "cmpl $13, %[nr]\n\t" - "vmovaps 192(%[KEY]), %[tmp2]\n\t" - "jl %=f\n\t" - "vaesenc %[tmp2], %[tmp1], %[tmp1]\n\t" - "vaesenc 208(%[KEY]), %[tmp1], %[tmp1]\n\t" - "vmovaps 224(%[KEY]), %[tmp2]\n\t" - "%=:\n\t" - "vaesenclast %[tmp2], %[tmp1], %[tmp1]\n\t" - "vpxor (%[in]), %[tmp1], %[tmp1]\n\t" - "vmovdqu %[tmp1], (%[out])\n\t" - "vpshufb %[BSWAP_MASK], %[tmp1], %[tmp1]\n\t" - "vpxor %[tmp1], %[X], %[X]\n\t" + VAESENC_BLOCK() : [tmp1] "+xr" (tmp1), [tmp2] "=xr" (tmp2), [H] "+xr" (H), [X] "+xr" (X), [ctr1] "+xr" (ctr1) @@ -5013,7 +4971,348 @@ static void AES_GCM_encrypt_avx2(const unsigned char *in, unsigned char *out, T = _mm_xor_si128(X, T); _mm_storeu_si128((__m128i*)tag, T); } + +#ifdef HAVE_INTEL_AVX2 +static void AES_GCM_encrypt_avx2(const unsigned char *in, unsigned char *out, + const unsigned char* addt, + const unsigned char* ivec, + unsigned char *tag, unsigned int nbytes, + unsigned int abytes, unsigned int ibytes, + const unsigned char* key, int nr) +{ + int i, j ,k; + __m128i ctr1; + __m128i H, Y, T; + __m128i X = _mm_setzero_si128(); + __m128i *KEY = (__m128i*)key, lastKey; + __m128i last_block = _mm_setzero_si128(); +#if !defined(AES_GCM_AESNI_NO_UNROLL) && !defined(AES_GCM_AVX1_NO_UNROLL) + __m128i HT[8]; + register __m128i tmp1 asm("xmm4"); + register __m128i tmp2 asm("xmm5"); + register __m128i tmp3 asm("xmm6"); + register __m128i tmp4 asm("xmm7"); + register __m128i tmp5 asm("xmm8"); + register __m128i tmp6 asm("xmm9"); + register __m128i tmp7 asm("xmm10"); + register __m128i tmp8 asm("xmm11"); + __m128i pctr1[1]; + register __m128i XV asm("xmm2"); +#else + __m128i tmp1, tmp2; +#endif + + if (ibytes == 12) + aes_gcm_avx1_calc_iv_12(KEY, ivec, nr, H, Y, T, X); + else + aes_gcm_calc_iv(KEY, ivec, ibytes, nr, H, Y, T, X); + + for (i=0; i < (int)(abytes/16); i++) { + tmp1 = _mm_loadu_si128(&((__m128i*)addt)[i]); + tmp1 = _mm_shuffle_epi8(tmp1, BSWAP_MASK); + X = _mm_xor_si128(X, tmp1); + X = gfmul_sw_avx2(X, H); + } + if (abytes%16) { + last_block = _mm_setzero_si128(); + for (j=0; j < (int)(abytes%16); j++) + ((unsigned char*)&last_block)[j] = addt[i*16+j]; + tmp1 = last_block; + tmp1 = _mm_shuffle_epi8(tmp1, BSWAP_MASK); + X = _mm_xor_si128(X, tmp1); + X = gfmul_sw_avx2(X, H); + } + + tmp1 = _mm_shuffle_epi8(Y, BSWAP_EPI64); + ctr1 = _mm_add_epi32(tmp1, ONE); + H = gfmul_shl1(H); + +#if !defined(AES_GCM_AESNI_NO_UNROLL) && !defined(AES_GCM_AVX1_NO_UNROLL) + i = 0; + if (nbytes >= 16*8) { + HT[0] = H; + HT[1] = gfmul_shifted_avx2(H, H); + HT[2] = gfmul_shifted_avx2(H, HT[1]); + HT[3] = gfmul_shifted_avx2(HT[1], HT[1]); + HT[4] = gfmul_shifted_avx2(HT[1], HT[2]); + HT[5] = gfmul_shifted_avx2(HT[2], HT[2]); + HT[6] = gfmul_shifted_avx2(HT[2], HT[3]); + HT[7] = gfmul_shifted_avx2(HT[3], HT[3]); + + pctr1[0] = ctr1; + __asm__ __volatile__ ( + VAESENC_CTR() + VAESENC_XOR() + VAESENC_SET(16) + VAESENC_SET(32) + VAESENC_SET(48) + VAESENC_SET(64) + VAESENC_SET(80) + VAESENC_SET(96) + VAESENC_SET(112) + VAESENC_SET(128) + VAESENC_SET(144) + "cmpl $11, %[nr]\n\t" + "vmovaps 160(%[KEY]), %%xmm12\n\t" + "jl 1f\n\t" + + VAESENC() + VAESENC_SET(176) + "cmpl $13, %[nr]\n\t" + "vmovaps 192(%[KEY]), %%xmm12\n\t" + "jl 1f\n\t" + + VAESENC() + VAESENC_SET(208) + "vmovaps 224(%[KEY]), %%xmm12\n\t" + "\n" + "1:\n\t" + VAESENC_LAST() + + : [tmp1] "=xr" (tmp1), [tmp2] "=xr" (tmp2), [tmp3] "=xr" (tmp3), + [tmp4] "=xr" (tmp4), [tmp5] "=xr" (tmp5), [tmp6] "=xr" (tmp6), + [tmp7] "=xr" (tmp7), [tmp8] "=xr" (tmp8) + : [KEY] "r" (KEY), [pctr1] "r" (pctr1), + [in] "r" (&in[i*16*8]), [out] "r" (&out[i*16*8]), [nr] "r" (nr), + [BSWAP_EPI64] "m" (BSWAP_EPI64), + [ONE] "m" (ONE), [TWO] "m" (TWO), + [THREE] "m" (THREE), [FOUR] "m" (FOUR), + [FIVE] "m" (FIVE), [SIX] "m" (SIX), + [SEVEN] "m" (SEVEN), [EIGHT] "m" (EIGHT) + : "xmm15", "xmm14", "xmm13", "xmm12", + "xmm0", "xmm1", "xmm3", "memory" + ); + + XV = X; + for (i=1; i < (int)(nbytes/16/8); i++) { + __asm__ __volatile__ ( + VAESENC_CTR() + VAESENC_XOR() + VAESENC_PCLMUL_AVX2_1(%[out], 16, -128, 112) + VAESENC_PCLMUL_AVX2_2(%[out], 32, -112, 96) + VAESENC_PCLMUL_AVX2_N(%[out], 48, -96, 80) + VAESENC_PCLMUL_AVX2_N(%[out], 64, -80, 64) + VAESENC_PCLMUL_AVX2_N(%[out], 80, -64, 48) + VAESENC_PCLMUL_AVX2_N(%[out], 96, -48, 32) + VAESENC_PCLMUL_AVX2_N(%[out], 112, -32, 16) + VAESENC_PCLMUL_AVX2_N(%[out], 128, -16, 0) + VAESENC_PCLMUL_AVX2_L(144) + + "cmpl $11, %[nr]\n\t" + "vmovaps 160(%[KEY]), %%xmm12\n\t" + "jl %=f\n\t" + + VAESENC() + VAESENC_SET(176) + "cmpl $13, %[nr]\n\t" + "vmovaps 192(%[KEY]), %%xmm12\n\t" + "jl %=f\n\t" + + VAESENC() + VAESENC_SET(208) + "vmovaps 224(%[KEY]), %%xmm12\n\t" + + "%=:\n\t" + VAESENC_LAST() + + : [tmp1] "=xr" (tmp1), [tmp2] "=xr" (tmp2), [tmp3] "=xr" (tmp3), + [tmp4] "=xr" (tmp4), [tmp5] "=xr" (tmp5), [tmp6] "=xr" (tmp6), + [tmp7] "=xr" (tmp7), [tmp8] "=xr" (tmp8), + [XV] "+xr" (XV) + : [KEY] "r" (KEY), [HT] "r" (HT), [pctr1] "r" (pctr1), + [in] "r" (&in[i*16*8]), [out] "r" (&out[i*16*8]), [nr] "r" (nr), + [BSWAP_MASK] "m" (BSWAP_MASK), + [BSWAP_EPI64] "m" (BSWAP_EPI64), + [ONE] "m" (ONE), [TWO] "m" (TWO), + [THREE] "m" (THREE), [FOUR] "m" (FOUR), + [FIVE] "m" (FIVE), [SIX] "m" (SIX), + [SEVEN] "m" (SEVEN), [EIGHT] "m" (EIGHT), + [MOD2_128] "m" (MOD2_128) + : "xmm15", "xmm14", "xmm13", "xmm12", + "xmm0", "xmm1", "xmm3", "memory" + ); + } + X = XV; + ctr1 = pctr1[0]; + tmp1 = _mm_shuffle_epi8(tmp1, BSWAP_MASK); + tmp2 = _mm_shuffle_epi8(tmp2, BSWAP_MASK); + tmp3 = _mm_shuffle_epi8(tmp3, BSWAP_MASK); + tmp4 = _mm_shuffle_epi8(tmp4, BSWAP_MASK); + tmp5 = _mm_shuffle_epi8(tmp5, BSWAP_MASK); + tmp6 = _mm_shuffle_epi8(tmp6, BSWAP_MASK); + tmp7 = _mm_shuffle_epi8(tmp7, BSWAP_MASK); + tmp8 = _mm_shuffle_epi8(tmp8, BSWAP_MASK); + tmp1 = _mm_xor_si128(X, tmp1); + X = gfmul8_avx2(tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, tmp8, + HT[0], HT[1], HT[2], HT[3], HT[4], HT[5], HT[6], HT[7]); + } + for (k = i*8; k < (int)(nbytes/16); k++) { + __asm__ __volatile__ ( + VAESENC_BLOCK() + + "# Carryless Multiply X by H (128 x 128)\n\t" + "vpclmulqdq $16, %[H], %[X], %%xmm13\n\t" + "vpclmulqdq $1, %[H], %[X], %%xmm14\n\t" + "vpclmulqdq $0, %[H], %[X], %%xmm15\n\t" + "vpclmulqdq $17, %[H], %[X], %%xmm1\n\t" + "vpxor %%xmm14, %%xmm13, %%xmm13\n\t" + "vpslldq $8, %%xmm13, %%xmm2\n\t" + "vpsrldq $8, %%xmm13, %%xmm13\n\t" + "vpxor %%xmm15, %%xmm2, %%xmm2\n\t" + "vpxor %%xmm13, %%xmm1, %%xmm3\n\t" + "# Reduce\n\t" + "vmovdqa %[MOD2_128], %%xmm0\n\t" + "vpclmulqdq $16, %%xmm0, %%xmm2, %%xmm14\n\t" + "vpshufd $78, %%xmm2, %%xmm13\n\t" + "vpxor %%xmm14, %%xmm13, %%xmm13\n\t" + "vpclmulqdq $16, %%xmm0, %%xmm13, %%xmm14\n\t" + "vpshufd $78, %%xmm13, %%xmm13\n\t" + "vpxor %%xmm14, %%xmm13, %%xmm13\n\t" + "vpxor %%xmm3, %%xmm13, %%xmm13\n\t" + "vmovdqa %%xmm13, %[X]\n\t" + "# End Reduce\n\t" + + : [tmp1] "+xr" (tmp1), [tmp2] "=xr" (tmp2), + [H] "+xr" (H), [X] "+xr" (X), [ctr1] "+xr" (ctr1) + : [KEY] "r" (KEY), + [in] "r" (&in[k*16]), [out] "r" (&out[k*16]), [nr] "r" (nr), + [BSWAP_MASK] "m" (BSWAP_MASK), + [BSWAP_EPI64] "m" (BSWAP_EPI64), + [ONE] "m" (ONE), + [MOD2_128] "m" (MOD2_128) + : "xmm15", "xmm14", "xmm13", + "xmm0", "xmm1", "xmm2", "xmm3", "memory" + ); + } +#else + for (k = 0; k < (int)(nbytes/16) && k < 1; k++) { + __asm__ __volatile__ ( + VAESENC_BLOCK() + + : [tmp1] "+xr" (tmp1), [tmp2] "=xr" (tmp2), + [H] "+xr" (H), [X] "+xr" (X), [ctr1] "+xr" (ctr1) + : [KEY] "r" (KEY), + [in] "r" (&in[k*16]), [out] "r" (&out[k*16]), [nr] "r" (nr), + [BSWAP_MASK] "m" (BSWAP_MASK), + [BSWAP_EPI64] "m" (BSWAP_EPI64), + [ONE] "m" (ONE), + [MOD2_128] "m" (MOD2_128) + : "memory" + ); + } + for (; k < (int)(nbytes/16); k++) { + __asm__ __volatile__ ( + "vpshufb %[BSWAP_EPI64], %[ctr1], %[tmp1]\n\t" + "vpaddd %[ONE], %[ctr1], %[ctr1]\n\t" + "vpxor (%[KEY]), %[tmp1], %[tmp1]\n\t" + "vaesenc 16(%[KEY]), %[tmp1], %[tmp1]\n\t" + "vpclmulqdq $16, %[H], %[X], %%xmm13\n\t" + "vaesenc 32(%[KEY]), %[tmp1], %[tmp1]\n\t" + "vpclmulqdq $1, %[H], %[X], %%xmm14\n\t" + "vaesenc 48(%[KEY]), %[tmp1], %[tmp1]\n\t" + "vpclmulqdq $0, %[H], %[X], %%xmm15\n\t" + "vaesenc 64(%[KEY]), %[tmp1], %[tmp1]\n\t" + "vpclmulqdq $17, %[H], %[X], %%xmm1\n\t" + "vaesenc 80(%[KEY]), %[tmp1], %[tmp1]\n\t" + "vpxor %%xmm14, %%xmm13, %%xmm13\n\t" + "vpslldq $8, %%xmm13, %%xmm2\n\t" + "vpsrldq $8, %%xmm13, %%xmm13\n\t" + "vaesenc 96(%[KEY]), %[tmp1], %[tmp1]\n\t" + "vpxor %%xmm15, %%xmm2, %%xmm2\n\t" + "vpxor %%xmm13, %%xmm1, %%xmm3\n\t" + "vmovdqa %[MOD2_128], %%xmm0\n\t" + "vpclmulqdq $16, %%xmm0, %%xmm2, %%xmm14\n\t" + "vaesenc 112(%[KEY]), %[tmp1], %[tmp1]\n\t" + "vpshufd $78, %%xmm2, %%xmm13\n\t" + "vpxor %%xmm14, %%xmm13, %%xmm13\n\t" + "vpclmulqdq $16, %%xmm0, %%xmm13, %%xmm14\n\t" + "vaesenc 128(%[KEY]), %[tmp1], %[tmp1]\n\t" + "vpshufd $78, %%xmm13, %%xmm13\n\t" + "vpxor %%xmm14, %%xmm13, %%xmm13\n\t" + "vpxor %%xmm3, %%xmm13, %%xmm13\n\t" + "vaesenc 144(%[KEY]), %[tmp1], %[tmp1]\n\t" + "vmovdqa %%xmm13, %[X]\n\t" + "cmpl $11, %[nr]\n\t" + "vmovaps 160(%[KEY]), %[tmp2]\n\t" + "jl %=f\n\t" + "vaesenc %[tmp2], %[tmp1], %[tmp1]\n\t" + "vaesenc 176(%[KEY]), %[tmp1], %[tmp1]\n\t" + "cmpl $13, %[nr]\n\t" + "vmovaps 192(%[KEY]), %[tmp2]\n\t" + "jl %=f\n\t" + "vaesenc %[tmp2], %[tmp1], %[tmp1]\n\t" + "vaesenc 208(%[KEY]), %[tmp1], %[tmp1]\n\t" + "vmovaps 224(%[KEY]), %[tmp2]\n\t" + "%=:\n\t" + "vaesenclast %[tmp2], %[tmp1], %[tmp1]\n\t" + "vpxor (%[in]), %[tmp1], %[tmp1]\n\t" + "vmovdqu %[tmp1], (%[out])\n\t" + "vpshufb %[BSWAP_MASK], %[tmp1], %[tmp1]\n\t" + "vpxor %[tmp1], %[X], %[X]\n\t" + + : [tmp1] "+xr" (tmp1), [tmp2] "=xr" (tmp2), + [H] "+xr" (H), [X] "+xr" (X), [ctr1] "+xr" (ctr1) + : [KEY] "r" (KEY), + [in] "r" (&in[k*16]), [out] "r" (&out[k*16]), [nr] "r" (nr), + [BSWAP_MASK] "m" (BSWAP_MASK), + [BSWAP_EPI64] "m" (BSWAP_EPI64), + [ONE] "m" (ONE), + [MOD2_128] "m" (MOD2_128) + : "xmm15", "xmm14", "xmm13", + "xmm0", "xmm1", "xmm2", "xmm3", "memory" + ); + } + if (k > 0) { + X = gfmul_shifted_avx2(X, H); + } +#endif + /* If one partial block remains */ + if (nbytes % 16) { + tmp1 = _mm_shuffle_epi8(ctr1, BSWAP_EPI64); + tmp1 = _mm_xor_si128(tmp1, KEY[0]); + tmp1 = _mm_aesenc_si128(tmp1, KEY[1]); + tmp1 = _mm_aesenc_si128(tmp1, KEY[2]); + tmp1 = _mm_aesenc_si128(tmp1, KEY[3]); + tmp1 = _mm_aesenc_si128(tmp1, KEY[4]); + tmp1 = _mm_aesenc_si128(tmp1, KEY[5]); + tmp1 = _mm_aesenc_si128(tmp1, KEY[6]); + tmp1 = _mm_aesenc_si128(tmp1, KEY[7]); + tmp1 = _mm_aesenc_si128(tmp1, KEY[8]); + tmp1 = _mm_aesenc_si128(tmp1, KEY[9]); + lastKey = KEY[10]; + if (nr > 10) { + tmp1 = _mm_aesenc_si128(tmp1, lastKey); + tmp1 = _mm_aesenc_si128(tmp1, KEY[11]); + lastKey = KEY[12]; + if (nr > 12) { + tmp1 = _mm_aesenc_si128(tmp1, lastKey); + tmp1 = _mm_aesenc_si128(tmp1, KEY[13]); + lastKey = KEY[14]; + } + } + tmp1 = _mm_aesenclast_si128(tmp1, lastKey); + last_block = tmp1; + for (j=0; j < (int)(nbytes%16); j++) + ((unsigned char*)&last_block)[j] = in[k*16+j]; + tmp1 = _mm_xor_si128(tmp1, last_block); + last_block = tmp1; + for (j=0; j < (int)(nbytes%16); j++) + out[k*16+j] = ((unsigned char*)&last_block)[j]; + tmp1 = last_block; + tmp1 = _mm_shuffle_epi8(tmp1, BSWAP_MASK); + X =_mm_xor_si128(X, tmp1); + X = gfmul_shifted_avx2(X, H); + } + tmp1 = _mm_insert_epi64(tmp1, nbytes*8, 0); + tmp1 = _mm_insert_epi64(tmp1, abytes*8, 1); + X = _mm_xor_si128(X, tmp1); + X = gfmul_shifted_avx2(X, H); + X = _mm_shuffle_epi8(X, BSWAP_MASK); + T = _mm_xor_si128(X, T); + _mm_storeu_si128((__m128i*)tag, T); +} #endif /* HAVE_INTEL_AVX2 */ +#endif /* HAVE_INTEL_AVX1 */ #ifdef HAVE_AES_DECRYPT /* Figure 10. AES-GCM – Decrypt With Single Block Ghash at a Time */ @@ -5036,122 +5335,10 @@ static int AES_GCM_decrypt(const unsigned char *in, unsigned char *out, __m128i tmp3, tmp4, tmp5, tmp6, tmp7, tmp8; #endif - if (ibytes == 12) { - Y = _mm_setzero_si128(); - for (j=0; j < 12; j++) - ((unsigned char*)&Y)[j] = ivec[j]; - Y = _mm_insert_epi32(Y, 0x1000000, 3); - /* (Compute E[ZERO, KS] and E[Y0, KS] together */ - tmp1 = _mm_xor_si128(X, KEY[0]); - tmp2 = _mm_xor_si128(Y, KEY[0]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[1]); - tmp2 = _mm_aesenc_si128(tmp2, KEY[1]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[2]); - tmp2 = _mm_aesenc_si128(tmp2, KEY[2]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[3]); - tmp2 = _mm_aesenc_si128(tmp2, KEY[3]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[4]); - tmp2 = _mm_aesenc_si128(tmp2, KEY[4]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[5]); - tmp2 = _mm_aesenc_si128(tmp2, KEY[5]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[6]); - tmp2 = _mm_aesenc_si128(tmp2, KEY[6]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[7]); - tmp2 = _mm_aesenc_si128(tmp2, KEY[7]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[8]); - tmp2 = _mm_aesenc_si128(tmp2, KEY[8]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[9]); - tmp2 = _mm_aesenc_si128(tmp2, KEY[9]); - lastKey = KEY[10]; - if (nr > 10) { - tmp1 = _mm_aesenc_si128(tmp1, lastKey); - tmp2 = _mm_aesenc_si128(tmp2, lastKey); - tmp1 = _mm_aesenc_si128(tmp1, KEY[11]); - tmp2 = _mm_aesenc_si128(tmp2, KEY[11]); - lastKey = KEY[12]; - if (nr > 12) { - tmp1 = _mm_aesenc_si128(tmp1, lastKey); - tmp2 = _mm_aesenc_si128(tmp2, lastKey); - tmp1 = _mm_aesenc_si128(tmp1, KEY[13]); - tmp2 = _mm_aesenc_si128(tmp2, KEY[13]); - lastKey = KEY[14]; - } - } - H = _mm_aesenclast_si128(tmp1, lastKey); - T = _mm_aesenclast_si128(tmp2, lastKey); - H = _mm_shuffle_epi8(H, BSWAP_MASK); - } - else { - if (ibytes % 16) { - i = ibytes / 16; - for (j=0; j < ibytes%16; j++) - ((unsigned char*)&last_block)[j] = ivec[i*16+j]; - } - tmp1 = _mm_xor_si128(X, KEY[0]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[1]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[2]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[3]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[4]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[5]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[6]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[7]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[8]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[9]); - lastKey = KEY[10]; - if (nr > 10) { - tmp1 = _mm_aesenc_si128(tmp1, lastKey); - tmp1 = _mm_aesenc_si128(tmp1, KEY[11]); - lastKey = KEY[12]; - if (nr > 12) { - tmp1 = _mm_aesenc_si128(tmp1, lastKey); - tmp1 = _mm_aesenc_si128(tmp1, KEY[13]); - lastKey = KEY[14]; - } - } - H = _mm_aesenclast_si128(tmp1, lastKey); - H = _mm_shuffle_epi8(H, BSWAP_MASK); - - Y = _mm_setzero_si128(); - for (i=0; i < ibytes/16; i++) { - tmp1 = _mm_loadu_si128(&((__m128i*)ivec)[i]); - tmp1 = _mm_shuffle_epi8(tmp1, BSWAP_MASK); - Y = _mm_xor_si128(Y, tmp1); - Y = gfmul_sw(Y, H); - } - if (ibytes % 16) { - tmp1 = last_block; - tmp1 = _mm_shuffle_epi8(tmp1, BSWAP_MASK); - Y = _mm_xor_si128(Y, tmp1); - Y = gfmul_sw(Y, H); - } - tmp1 = _mm_insert_epi64(tmp1, ibytes*8, 0); - tmp1 = _mm_insert_epi64(tmp1, 0, 1); - Y = _mm_xor_si128(Y, tmp1); - Y = gfmul_sw(Y, H); - Y = _mm_shuffle_epi8(Y, BSWAP_MASK); /* Compute E(K, Y0) */ - tmp1 = _mm_xor_si128(Y, KEY[0]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[1]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[2]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[3]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[4]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[5]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[6]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[7]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[8]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[9]); - lastKey = KEY[10]; - if (nr > 10) { - tmp1 = _mm_aesenc_si128(tmp1, lastKey); - tmp1 = _mm_aesenc_si128(tmp1, KEY[11]); - lastKey = KEY[12]; - if (nr > 12) { - tmp1 = _mm_aesenc_si128(tmp1, lastKey); - tmp1 = _mm_aesenc_si128(tmp1, KEY[13]); - lastKey = KEY[14]; - } - } - T = _mm_aesenclast_si128(tmp1, lastKey); - } + if (ibytes == 12) + aes_gcm_calc_iv_12(KEY, ivec, nr, H, Y, T, X); + else + aes_gcm_calc_iv(KEY, ivec, ibytes, nr, H, Y, T, X); for (i=0; i 10) { - tmp1 = _mm_aesenc_si128(tmp1, lastKey); - tmp2 = _mm_aesenc_si128(tmp2, lastKey); - tmp1 = _mm_aesenc_si128(tmp1, KEY[11]); - tmp2 = _mm_aesenc_si128(tmp2, KEY[11]); - lastKey = KEY[12]; - if (nr > 12) { - tmp1 = _mm_aesenc_si128(tmp1, lastKey); - tmp2 = _mm_aesenc_si128(tmp2, lastKey); - tmp1 = _mm_aesenc_si128(tmp1, KEY[13]); - tmp2 = _mm_aesenc_si128(tmp2, KEY[13]); - lastKey = KEY[14]; - } - } - H = _mm_aesenclast_si128(tmp1, lastKey); - T = _mm_aesenclast_si128(tmp2, lastKey); - H = _mm_shuffle_epi8(H, BSWAP_MASK); - } - else { - if (ibytes % 16) { - i = ibytes / 16; - for (j=0; j < ibytes%16; j++) - ((unsigned char*)&last_block)[j] = ivec[i*16+j]; - } - tmp1 = _mm_xor_si128(X, KEY[0]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[1]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[2]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[3]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[4]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[5]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[6]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[7]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[8]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[9]); - lastKey = KEY[10]; - if (nr > 10) { - tmp1 = _mm_aesenc_si128(tmp1, lastKey); - tmp1 = _mm_aesenc_si128(tmp1, KEY[11]); - lastKey = KEY[12]; - if (nr > 12) { - tmp1 = _mm_aesenc_si128(tmp1, lastKey); - tmp1 = _mm_aesenc_si128(tmp1, KEY[13]); - lastKey = KEY[14]; - } - } - H = _mm_aesenclast_si128(tmp1, lastKey); - H = _mm_shuffle_epi8(H, BSWAP_MASK); - - Y = _mm_setzero_si128(); - for (i=0; i < ibytes/16; i++) { - tmp1 = _mm_loadu_si128(&((__m128i*)ivec)[i]); - tmp1 = _mm_shuffle_epi8(tmp1, BSWAP_MASK); - Y = _mm_xor_si128(Y, tmp1); - Y = gfmul_sw(Y, H); - } - if (ibytes % 16) { - tmp1 = last_block; - tmp1 = _mm_shuffle_epi8(tmp1, BSWAP_MASK); - Y = _mm_xor_si128(Y, tmp1); - Y = gfmul_sw(Y, H); - } - tmp1 = _mm_insert_epi64(tmp1, ibytes*8, 0); - tmp1 = _mm_insert_epi64(tmp1, 0, 1); - Y = _mm_xor_si128(Y, tmp1); - Y = gfmul_sw(Y, H); - Y = _mm_shuffle_epi8(Y, BSWAP_MASK); /* Compute E(K, Y0) */ - tmp1 = _mm_xor_si128(Y, KEY[0]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[1]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[2]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[3]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[4]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[5]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[6]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[7]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[8]); - tmp1 = _mm_aesenc_si128(tmp1, KEY[9]); - lastKey = KEY[10]; - if (nr > 10) { - tmp1 = _mm_aesenc_si128(tmp1, lastKey); - tmp1 = _mm_aesenc_si128(tmp1, KEY[11]); - lastKey = KEY[12]; - if (nr > 12) { - tmp1 = _mm_aesenc_si128(tmp1, lastKey); - tmp1 = _mm_aesenc_si128(tmp1, KEY[13]); - lastKey = KEY[14]; - } - } - T = _mm_aesenclast_si128(tmp1, lastKey); - } + if (ibytes == 12) + aes_gcm_calc_iv_12(KEY, ivec, nr, H, Y, T, X); + else + aes_gcm_calc_iv(KEY, ivec, ibytes, nr, H, Y, T, X); for (i=0; i 10) { + tmp1 = _mm_aesenc_si128(tmp1, lastKey); + tmp1 = _mm_aesenc_si128(tmp1, KEY[11]); + lastKey = KEY[12]; + if (nr > 12) { + tmp1 = _mm_aesenc_si128(tmp1, lastKey); + tmp1 = _mm_aesenc_si128(tmp1, KEY[13]); + lastKey = KEY[14]; + } + } + tmp1 = _mm_aesenclast_si128(tmp1, lastKey); + last_block = _mm_setzero_si128(); + for (j=0; j < nbytes%16; j++) + ((unsigned char*)&last_block)[j] = in[k*16+j]; + XV = last_block; + tmp1 = _mm_xor_si128(tmp1, last_block); + last_block = tmp1; + for (j=0; j < nbytes%16; j++) + out[k*16+j] = ((unsigned char*)&last_block)[j]; + XV = _mm_shuffle_epi8(XV, BSWAP_MASK); + XV = _mm_xor_si128(XV, X); + X = gfmul_shifted_avx2(XV, H); + } + + tmp1 = _mm_insert_epi64(tmp1, nbytes*8, 0); + tmp1 = _mm_insert_epi64(tmp1, abytes*8, 1); + /* 128 x 128 Carryless Multiply */ + X = _mm_xor_si128(X, tmp1); + X = gfmul_shifted_avx2(X, H); + X = _mm_shuffle_epi8(X, BSWAP_MASK); + T = _mm_xor_si128(X, T); + + if (0xffff != + _mm_movemask_epi8(_mm_cmpeq_epi8(T, _mm_loadu_si128((__m128i*)tag)))) + return 0; /* in case the authentication failed */ + + return 1; /* when successful returns 1 */ +} #endif /* HAVE_INTEL_AVX2 */ +#endif /* HAVE_INTEL_AVX1 */ #endif /* HAVE_AES_DECRYPT */ #endif /* WOLFSSL_AESNI */ @@ -6793,11 +6816,15 @@ int wc_AesGcmEncrypt(Aes* aes, byte* out, const byte* in, word32 sz, #ifdef WOLFSSL_AESNI if (haveAESNI) { - #ifdef HAVE_INTEL_AVX2 + #ifdef HAVE_INTEL_AVX1 if (IS_INTEL_AVX2(intel_flags)) { AES_GCM_encrypt_avx2(in, out, authIn, iv, authTag, sz, authInSz, ivSz, (const byte*)aes->key, aes->rounds); } + else if (IS_INTEL_AVX1(intel_flags)) { + AES_GCM_encrypt_avx1(in, out, authIn, iv, authTag, + sz, authInSz, ivSz, (const byte*)aes->key, aes->rounds); + } else #endif AES_GCM_encrypt(in, out, authIn, iv, authTag, @@ -7055,11 +7082,16 @@ int wc_AesGcmDecrypt(Aes* aes, byte* out, const byte* in, word32 sz, #ifdef WOLFSSL_AESNI if (haveAESNI) { - #ifdef HAVE_INTEL_AVX2 + #ifdef HAVE_INTEL_AVX1 if (IS_INTEL_AVX2(intel_flags)) { if (AES_GCM_decrypt_avx2(in, out, authIn, iv, authTag, sz, authInSz, ivSz, (byte*)aes->key, aes->rounds) == 0) - return AES_GCM_AUTH_E; + return AES_GCM_AUTH_E; + } + else if (IS_INTEL_AVX1(intel_flags)) { + if (AES_GCM_decrypt_avx1(in, out, authIn, iv, authTag, sz, authInSz, + ivSz, (byte*)aes->key, aes->rounds) == 0) + return AES_GCM_AUTH_E; } else #endif @@ -7855,7 +7887,7 @@ int wc_AesXtsEncrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz, word32 blocks = (sz / AES_BLOCK_SIZE); Aes *aes, *tweak; - if (xaes == NULL || out == NULL) { + if (xaes == NULL || out == NULL || in == NULL) { return BAD_FUNC_ARG; } @@ -7866,13 +7898,12 @@ int wc_AesXtsEncrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz, return BAD_FUNC_ARG; } - if (in == NULL && sz > 0) { - return BAD_FUNC_ARG; - } - if (blocks > 0) { byte tmp[AES_BLOCK_SIZE]; + XMEMSET(tmp, 0, AES_BLOCK_SIZE); /* set to 0's in case of improper AES + * key setup passed to encrypt direct*/ + wc_AesEncryptDirect(tweak, tmp, i); while (blocks > 0) { @@ -7896,7 +7927,6 @@ int wc_AesXtsEncrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz, if (carry) { tmp[0] ^= GF_XTS; } - carry = 0; in += AES_BLOCK_SIZE; out += AES_BLOCK_SIZE; @@ -7948,7 +7978,7 @@ int wc_AesXtsDecrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz, word32 blocks = (sz / AES_BLOCK_SIZE); Aes *aes, *tweak; - if (xaes == NULL || out == NULL) { + if (xaes == NULL || out == NULL || in == NULL) { return BAD_FUNC_ARG; } @@ -7959,16 +7989,15 @@ int wc_AesXtsDecrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz, return BAD_FUNC_ARG; } - if (in == NULL && sz > 0) { - return BAD_FUNC_ARG; - } - if (blocks > 0) { word32 j; byte carry = 0; byte tmp[AES_BLOCK_SIZE]; byte stl = (sz % AES_BLOCK_SIZE); + XMEMSET(tmp, 0, AES_BLOCK_SIZE); /* set to 0's in case of improper AES + * key setup passed to decrypt direct*/ + wc_AesEncryptDirect(tweak, tmp, i); /* if Stealing then break out of loop one block early to handle special diff --git a/wolfcrypt/src/asn.c b/wolfcrypt/src/asn.c index 9e628f7480..cc7352dcd4 100644 --- a/wolfcrypt/src/asn.c +++ b/wolfcrypt/src/asn.c @@ -6196,6 +6196,13 @@ int wc_DerToPemEx(const byte* der, word32 derSz, byte* output, word32 outSz, XSTRNCPY(footer, END_RSA_PRIV, footerLen); XSTRNCAT(footer, "\n", 1); } + else if (type == PUBLICKEY_TYPE) { + XSTRNCPY(header, BEGIN_PUB_KEY, headerLen); + XSTRNCAT(header, "\n", 1); + + XSTRNCPY(footer, END_PUB_KEY, footerLen); + XSTRNCAT(footer, "\n", 1); + } #ifndef NO_DSA else if (type == DSA_PRIVATEKEY_TYPE) { XSTRNCPY(header, BEGIN_DSA_PRIV, headerLen); diff --git a/wolfcrypt/src/cpuid.c b/wolfcrypt/src/cpuid.c index f65eac6009..6fc30979a9 100644 --- a/wolfcrypt/src/cpuid.c +++ b/wolfcrypt/src/cpuid.c @@ -28,8 +28,8 @@ #include -#if defined(WOLFSSL_X86_64_BUILD) || defined(USE_INTEL_SPEEDUP) || \ - defined(WOLFSSL_AESNI) +#if (defined(WOLFSSL_X86_64_BUILD) || defined(USE_INTEL_SPEEDUP) || \ + defined(WOLFSSL_AESNI)) && !defined(WOLFSSL_NO_ASM) /* Each platform needs to query info type 1 from cpuid to see if aesni is * supported. Also, let's setup a macro for proper linkage w/o ABI conflicts */ diff --git a/wolfcrypt/src/ecc.c b/wolfcrypt/src/ecc.c index d55bce4f59..1f1850a936 100644 --- a/wolfcrypt/src/ecc.c +++ b/wolfcrypt/src/ecc.c @@ -1099,7 +1099,7 @@ static int wc_ecc_curve_load_item(const char* src, mp_int** dst, if (err == MP_OKAY) { curve->load_mask |= mask; - err = mp_read_radix(*dst, src, 16); + err = mp_read_radix(*dst, src, MP_RADIX_HEX); #ifdef HAVE_WOLF_BIGINT if (err == MP_OKAY) @@ -2599,7 +2599,7 @@ static int wc_ecc_cmp_param(const char* curveParam, err = mp_read_unsigned_bin(&a, param, paramSz); if (err == MP_OKAY) - err = mp_read_radix(&b, curveParam, 16); + err = mp_read_radix(&b, curveParam, MP_RADIX_HEX); if (err == MP_OKAY) { if (mp_cmp(&a, &b) != MP_EQ) { @@ -4951,7 +4951,7 @@ int wc_ecc_check_key(ecc_key* key) if (err == MP_OKAY) err = mp_init(b); if (err == MP_OKAY) - err = mp_read_radix(b, key->dp->Bf, 16); + err = mp_read_radix(b, key->dp->Bf, MP_RADIX_HEX); #else b = curve->Bf; #endif @@ -5378,9 +5378,9 @@ int wc_ecc_rs_to_sig(const char* r, const char* s, byte* out, word32* outlen) if (err != MP_OKAY) return err; - err = mp_read_radix(&rtmp, r, 16); + err = mp_read_radix(&rtmp, r, MP_RADIX_HEX); if (err == MP_OKAY) - err = mp_read_radix(&stmp, s, 16); + err = mp_read_radix(&stmp, s, MP_RADIX_HEX); /* convert mp_ints to ECDSA sig, initializes rtmp and stmp internally */ if (err == MP_OKAY) @@ -5496,11 +5496,11 @@ static int wc_ecc_import_raw_private(ecc_key* key, const char* qx, /* read Qx */ if (err == MP_OKAY) - err = mp_read_radix(key->pubkey.x, qx, 16); + err = mp_read_radix(key->pubkey.x, qx, MP_RADIX_HEX); /* read Qy */ if (err == MP_OKAY) - err = mp_read_radix(key->pubkey.y, qy, 16); + err = mp_read_radix(key->pubkey.y, qy, MP_RADIX_HEX); if (err == MP_OKAY) err = mp_set(key->pubkey.z, 1); @@ -5509,7 +5509,7 @@ static int wc_ecc_import_raw_private(ecc_key* key, const char* qx, if (err == MP_OKAY) { if (d != NULL) { key->type = ECC_PRIVATEKEY; - err = mp_read_radix(&key->k, d, 16); + err = mp_read_radix(&key->k, d, MP_RADIX_HEX); } else { key->type = ECC_PUBLICKEY; } @@ -6440,7 +6440,8 @@ static int accel_fp_mul(int idx, mp_int* k, ecc_point *R, mp_int* a, /* back off if we are on the 521 bit curve */ if (y == 66) --x; - if ((err = mp_read_radix(&order, ecc_sets[x].order, 16)) != MP_OKAY) { + if ((err = mp_read_radix(&order, ecc_sets[x].order, + MP_RADIX_HEX)) != MP_OKAY) { goto done; } @@ -6591,7 +6592,8 @@ static int accel_fp_mul2add(int idx1, int idx2, /* back off if we are on the 521 bit curve */ if (y == 66) --x; - if ((err = mp_read_radix(&order, ecc_sets[x].order, 16)) != MP_OKAY) { + if ((err = mp_read_radix(&order, ecc_sets[x].order, + MP_RADIX_HEX)) != MP_OKAY) { goto done; } @@ -6622,7 +6624,8 @@ static int accel_fp_mul2add(int idx1, int idx2, /* back off if we are on the 521 bit curve */ if (y == 66) --x; - if ((err = mp_read_radix(&order, ecc_sets[x].order, 16)) != MP_OKAY) { + if ((err = mp_read_radix(&order, ecc_sets[x].order, + MP_RADIX_HEX)) != MP_OKAY) { goto done; } diff --git a/wolfcrypt/src/integer.c b/wolfcrypt/src/integer.c index 1c66245f96..cb5df1c58d 100644 --- a/wolfcrypt/src/integer.c +++ b/wolfcrypt/src/integer.c @@ -4169,7 +4169,8 @@ int mp_sub_d (mp_int * a, mp_digit b, mp_int * c) #endif /* defined(HAVE_ECC) || !defined(NO_PWDBASED) */ -#if defined(WOLFSSL_KEY_GEN) || defined(HAVE_COMP_KEY) || defined(HAVE_ECC) +#if defined(WOLFSSL_KEY_GEN) || defined(HAVE_COMP_KEY) || defined(HAVE_ECC) || \ + defined(DEBUG_WOLFSSL) static const int lnz[16] = { 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0 @@ -4315,7 +4316,7 @@ int mp_mod_d (mp_int * a, mp_digit b, mp_digit * c) return mp_div_d(a, b, NULL, c); } -#endif /* defined(WOLFSSL_KEY_GEN)||defined(HAVE_COMP_KEY)||defined(HAVE_ECC) */ +#endif /* WOLFSSL_KEY_GEN || HAVE_COMP_KEY || HAVE_ECC || DEBUG_WOLFSSL */ #ifdef WOLFSSL_KEY_GEN @@ -4746,7 +4747,7 @@ int mp_read_radix (mp_int * a, const char *str, int radix) mp_zero(a); /* make sure the radix is ok */ - if (radix < 2 || radix > 64) { + if (radix < MP_RADIX_BIN || radix > MP_RADIX_MAX) { return MP_VAL; } @@ -4808,7 +4809,7 @@ int mp_read_radix (mp_int * a, const char *str, int radix) #endif /* HAVE_ECC */ #if defined(WOLFSSL_KEY_GEN) || defined(HAVE_COMP_KEY) || \ - defined(WOLFSSL_DEBUG_MATH) + defined(WOLFSSL_DEBUG_MATH) || defined(DEBUG_WOLFSSL) /* returns size of ASCII representation */ int mp_radix_size (mp_int *a, int radix, int *size) @@ -4820,13 +4821,13 @@ int mp_radix_size (mp_int *a, int radix, int *size) *size = 0; /* special case for binary */ - if (radix == 2) { + if (radix == MP_RADIX_BIN) { *size = mp_count_bits (a) + (a->sign == MP_NEG ? 1 : 0) + 1; return MP_OKAY; } /* make sure the radix is in range */ - if (radix < 2 || radix > 64) { + if (radix < MP_RADIX_BIN || radix > MP_RADIX_MAX) { return MP_VAL; } @@ -4875,7 +4876,7 @@ int mp_toradix (mp_int *a, char *str, int radix) char *_s = str; /* check range of the radix */ - if (radix < 2 || radix > 64) { + if (radix < MP_RADIX_BIN || radix > MP_RADIX_MAX) { return MP_VAL; } @@ -4933,7 +4934,7 @@ void mp_dump(const char* desc, mp_int* a, byte verbose) printf("%s: ptr=%p, used=%d, sign=%d, size=%d, mpd=%d\n", desc, a, a->used, a->sign, size, (int)sizeof(mp_digit)); - mp_toradix(a, buffer, 16); + mp_tohex(a, buffer); printf(" %s\n ", buffer); if (verbose) { diff --git a/wolfcrypt/src/logging.c b/wolfcrypt/src/logging.c index c27abb6c8c..68c55f366e 100644 --- a/wolfcrypt/src/logging.c +++ b/wolfcrypt/src/logging.c @@ -207,8 +207,8 @@ void WOLFSSL_BUFFER(const byte* buffer, word32 length) void WOLFSSL_ENTER(const char* msg) { if (loggingEnabled) { - char buffer[80]; - sprintf(buffer, "wolfSSL Entering %s", msg); + char buffer[WOLFSSL_MAX_ERROR_SZ]; + XSNPRINTF(buffer, sizeof(buffer), "wolfSSL Entering %s", msg); wolfssl_log(ENTER_LOG , buffer); } } @@ -217,8 +217,9 @@ void WOLFSSL_ENTER(const char* msg) void WOLFSSL_LEAVE(const char* msg, int ret) { if (loggingEnabled) { - char buffer[80]; - sprintf(buffer, "wolfSSL Leaving %s, return %d", msg, ret); + char buffer[WOLFSSL_MAX_ERROR_SZ]; + XSNPRINTF(buffer, sizeof(buffer), "wolfSSL Leaving %s, return %d", + msg, ret); wolfssl_log(LEAVE_LOG , buffer); } } @@ -241,18 +242,20 @@ void WOLFSSL_ERROR(int error) if (loggingEnabled && error != WC_PENDING_E) #endif { - char buffer[80]; + char buffer[WOLFSSL_MAX_ERROR_SZ]; #if defined(OPENSSL_EXTRA) || defined(DEBUG_WOLFSSL_VERBOSE) (void)usrCtx; /* a user ctx for future flexibility */ (void)func; if (wc_LockMutex(&debug_mutex) != 0) { WOLFSSL_MSG("Lock debug mutex failed"); - sprintf(buffer, "wolfSSL error occurred, error = %d", error); + XSNPRINTF(buffer, sizeof(buffer), + "wolfSSL error occurred, error = %d", error); } else { if (error < 0) error = error - (2*error); /*get absolute value*/ - sprintf(buffer, "wolfSSL error occurred, error = %d line:%d file:%s", + XSNPRINTF(buffer, sizeof(buffer), + "wolfSSL error occurred, error = %d line:%d file:%s", error, line, file); if (wc_AddErrorNode(error, line, buffer, (char*)file) != 0) { WOLFSSL_MSG("Error creating logging node"); @@ -263,7 +266,8 @@ void WOLFSSL_ERROR(int error) wc_UnLockMutex(&debug_mutex); } #else - sprintf(buffer, "wolfSSL error occurred, error = %d", error); + XSNPRINTF(buffer, sizeof(buffer), + "wolfSSL error occurred, error = %d", error); #endif #ifdef DEBUG_WOLFSSL wolfssl_log(ERROR_LOG , buffer); diff --git a/wolfcrypt/src/misc.c b/wolfcrypt/src/misc.c index 36a5aa3cc4..32b2f67aac 100644 --- a/wolfcrypt/src/misc.c +++ b/wolfcrypt/src/misc.c @@ -140,8 +140,8 @@ STATIC INLINE word64 rotrFixed64(word64 x, word64 y) STATIC INLINE word64 ByteReverseWord64(word64 value) { #if defined(WOLFCRYPT_SLOW_WORD64) - return (word64)(ByteReverseWord32((word32)value)) << 32 | - ByteReverseWord32((word32)(value>>32)); + return (word64)((word64)ByteReverseWord32((word32) value)) << 32 | + (word64)ByteReverseWord32((word32)(value >> 32)); #else value = ((value & W64LIT(0xFF00FF00FF00FF00)) >> 8) | ((value & W64LIT(0x00FF00FF00FF00FF)) << 8); @@ -192,7 +192,7 @@ STATIC INLINE void xorbuf(void* buf, const void* mask, word32 count) STATIC INLINE void ForceZero(const void* mem, word32 len) { volatile byte* z = (volatile byte*)mem; -#ifdef WOLFSSL_X86_64_BUILD +#if defined(WOLFSSL_X86_64_BUILD) && defined(WORD64_AVAILABLE) volatile word64* w; for (w = (volatile word64*)z; len >= sizeof(*w); len -= sizeof(*w)) diff --git a/wolfcrypt/src/pkcs7.c b/wolfcrypt/src/pkcs7.c index a81ef77d18..a5f2205f56 100644 --- a/wolfcrypt/src/pkcs7.c +++ b/wolfcrypt/src/pkcs7.c @@ -73,8 +73,11 @@ static int wc_SetContentType(int pkcs7TypeOID, byte* output) 0x0D, 0x01, 0x07, 0x04 }; const byte digestedData[] = { 0x2A, 0x86, 0x48, 0x86, 0xF7, 0x0D, 0x01, 0x07, 0x05 }; + +#ifndef NO_PKCS7_ENCRYPTED_DATA const byte encryptedData[] = { 0x2A, 0x86, 0x48, 0x86, 0xF7, 0x0D, 0x01, 0x07, 0x06 }; +#endif int idSz; int typeSz = 0, idx = 0; @@ -112,10 +115,12 @@ static int wc_SetContentType(int pkcs7TypeOID, byte* output) typeName = digestedData; break; +#ifndef NO_PKCS7_ENCRYPTED_DATA case ENCRYPTED_DATA: typeSz = sizeof(encryptedData); typeName = encryptedData; break; +#endif default: WOLFSSL_MSG("Unknown PKCS#7 Type"); @@ -382,7 +387,7 @@ typedef struct ESD { enum wc_HashType hashType; byte contentDigest[WC_MAX_DIGEST_SIZE + 2]; /* content only + ASN.1 heading */ byte contentAttribsDigest[WC_MAX_DIGEST_SIZE]; - byte encContentDigest[512]; + byte encContentDigest[MAX_ENCRYPTED_KEY_SZ]; byte outerSeq[MAX_SEQ_SZ]; byte outerContent[MAX_EXP_SZ]; @@ -3217,7 +3222,7 @@ static int wc_PKCS7_DecodeKtri(PKCS7* pkcs7, byte* pkiMsg, word32 pkiMsgSz, int keySz; word32 encOID; word32 keyIdx; - byte issuerHash[WC_SHA_DIGEST_SIZE]; + byte issuerHash[KEYID_SIZE]; byte* outKey = NULL; #ifdef WC_RSA_BLINDING @@ -3245,7 +3250,7 @@ static int wc_PKCS7_DecodeKtri(PKCS7* pkcs7, byte* pkiMsg, word32 pkiMsgSz, return ASN_PARSE_E; /* if we found correct recipient, issuer hashes will match */ - if (XMEMCMP(issuerHash, pkcs7->issuerHash, WC_SHA_DIGEST_SIZE) == 0) { + if (XMEMCMP(issuerHash, pkcs7->issuerHash, KEYID_SIZE) == 0) { *recipFound = 1; } @@ -4153,6 +4158,8 @@ WOLFSSL_API int wc_PKCS7_DecodeEnvelopedData(PKCS7* pkcs7, byte* pkiMsg, } +#ifndef NO_PKCS7_ENCRYPTED_DATA + /* build PKCS#7 encryptedData content type, return encrypted size */ int wc_PKCS7_EncodeEncryptedData(PKCS7* pkcs7, byte* output, word32 outputSz) { @@ -4617,6 +4624,8 @@ int wc_PKCS7_DecodeEncryptedData(PKCS7* pkcs7, byte* pkiMsg, word32 pkiMsgSz, return encryptedContentSz - padLen; } +#endif /* NO_PKCS7_ENCRYPTED_DATA */ + #else /* HAVE_PKCS7 */ diff --git a/wolfcrypt/src/port/ti/ti-aes.c b/wolfcrypt/src/port/ti/ti-aes.c index cbb6d9de2e..9dabcc560b 100644 --- a/wolfcrypt/src/port/ti/ti-aes.c +++ b/wolfcrypt/src/port/ti/ti-aes.c @@ -545,6 +545,22 @@ WOLFSSL_API int wc_AesCcmDecrypt(Aes* aes, byte* out, const byte* in, word32 inS } #endif /* HAVE_AESCCM */ +WOLFSSL_API int wc_AesInit(Aes* aes, void* heap, int devId) +{ + if (aes == NULL) + return BAD_FUNC_ARG; + + aes->heap = heap; + (void)devId; + + return 0; +} + +WOLFSSL_API void wc_AesFree(Aes* aes) +{ + (void)aes; +} + #endif /* WOLFSSL_TI_CRYPT */ #endif /* NO_AES */ diff --git a/wolfcrypt/src/port/ti/ti-des3.c b/wolfcrypt/src/port/ti/ti-des3.c index 5904fc2fe4..9d3ad7896e 100644 --- a/wolfcrypt/src/port/ti/ti-des3.c +++ b/wolfcrypt/src/port/ti/ti-des3.c @@ -182,6 +182,22 @@ WOLFSSL_API int wc_Des3_CbcDecryptWithKey(byte* out, const byte* in, word32 sz, return -1 ; } +WOLFSSL_API int wc_Des3Init(Des3* des, void* heap, int devId) +{ + if (des == NULL) + return BAD_FUNC_ARG; + + des->heap = heap; + (void)devId; + + return 0; +} + +WOLFSSL_API void wc_Des3Free(Des3* des) +{ + (void)des; +} + #endif /* WOLFSSL_TI_CRYPT */ diff --git a/wolfcrypt/src/sha.c b/wolfcrypt/src/sha.c index 8f7255f261..16a38b02cf 100644 --- a/wolfcrypt/src/sha.c +++ b/wolfcrypt/src/sha.c @@ -418,7 +418,7 @@ static INLINE void AddLength(wc_Sha* sha, word32 len) t = e; e = d; d = c; c = b; b = a; a = t; } #else - /* nearly 1 K bigger in code size but 25% faster */ + /* nearly 1 K bigger in code size but 25% faster */ /* 4 rounds of 20 operations each. Loop unrolled. */ R0(a,b,c,d,e, 0); R0(e,a,b,c,d, 1); R0(d,e,a,b,c, 2); R0(c,d,e,a,b, 3); R0(b,c,d,e,a, 4); R0(a,b,c,d,e, 5); R0(e,a,b,c,d, 6); R0(d,e,a,b,c, 7); diff --git a/wolfcrypt/src/sha256.c b/wolfcrypt/src/sha256.c index 470914cf1d..6cdc7b5949 100644 --- a/wolfcrypt/src/sha256.c +++ b/wolfcrypt/src/sha256.c @@ -92,7 +92,20 @@ #if defined(USE_INTEL_SPEEDUP) #define HAVE_INTEL_AVX1 - #define HAVE_INTEL_AVX2 + + #if defined(__GNUC__) && ((__GNUC__ < 4) || \ + (__GNUC__ == 4 && __GNUC_MINOR__ <= 8)) + #define NO_AVX2_SUPPORT + #endif + #if defined(__clang__) && ((__clang_major__ < 3) || \ + (__clang_major__ == 3 && __clang_minor__ <= 5)) + #define NO_AVX2_SUPPORT + #endif + + #define HAVE_INTEL_AVX1 + #ifndef NO_AVX2_SUPPORT + #define HAVE_INTEL_AVX2 + #endif #endif /* USE_INTEL_SPEEDUP */ #if defined(HAVE_INTEL_AVX2) @@ -150,9 +163,9 @@ static int InitSha256(wc_Sha256* sha256) } #if defined(HAVE_INTEL_AVX1)|| defined(HAVE_INTEL_AVX2) - Transform(); Function prototype + Transform_Sha256(); Function prototype #else - Transform() { } + Transform_Sha256() { } int Sha256Final() { Save/Recover XMM, YMM ... @@ -171,7 +184,7 @@ static int InitSha256(wc_Sha256* sha256) #define XMM Instructions/inline asm - int Transform() { + int Transform_Sha256() { Stitched Message Sched/Round } @@ -179,7 +192,7 @@ static int InitSha256(wc_Sha256* sha256) #define YMM Instructions/inline asm - int Transform() { + int Transform_Sha256() { More granural Stitched Message Sched/Round } @@ -192,18 +205,29 @@ static int InitSha256(wc_Sha256* sha256) */ /* #if defined(HAVE_INTEL_AVX1/2) at the tail of sha256 */ - static int Transform(wc_Sha256* sha256); + static int Transform_Sha256(wc_Sha256* sha256); #if defined(HAVE_INTEL_AVX1) - static int Transform_AVX1(wc_Sha256 *sha256); + static int Transform_Sha256_AVX1(wc_Sha256 *sha256); + static int Transform_Sha256_AVX1_Len(wc_Sha256* sha256, word32 len); #endif #if defined(HAVE_INTEL_AVX2) - static int Transform_AVX2(wc_Sha256 *sha256); - static int Transform_AVX1_RORX(wc_Sha256 *sha256); + static int Transform_Sha256_AVX2(wc_Sha256 *sha256); + static int Transform_Sha256_AVX2_Len(wc_Sha256* sha256, word32 len); + #ifdef HAVE_INTEL_RORX + static int Transform_Sha256_AVX1_RORX(wc_Sha256 *sha256); + static int Transform_Sha256_AVX1_RORX_Len(wc_Sha256* sha256, word32 len); + static int Transform_Sha256_AVX2_RORX(wc_Sha256 *sha256); + static int Transform_Sha256_AVX2_RORX_Len(wc_Sha256* sha256, word32 len); + #endif #endif - static int (*Transform_p)(wc_Sha256* sha256) /* = _Transform */; + static int (*Transform_Sha256_p)(wc_Sha256* sha256); + /* = _Transform_Sha256 */ + static int (*Transform_Sha256_Len_p)(wc_Sha256* sha256, word32 len); + /* = NULL */ static int transform_check = 0; static word32 intel_flags; - #define XTRANSFORM(S, B) (*Transform_p)((S)) + #define XTRANSFORM(S) (*Transform_Sha256_p)((S)) + #define XTRANSFORM_LEN(S, L) (*Transform_Sha256_Len_p)((S),(L)) static void Sha256_SetTransform(void) { @@ -213,37 +237,44 @@ static int InitSha256(wc_Sha256* sha256) intel_flags = cpuid_get_flags(); - #if defined(HAVE_INTEL_AVX2) - if (IS_INTEL_AVX2(intel_flags) && IS_INTEL_BMI2(intel_flags)) { - if (1) - Transform_p = Transform_AVX1_RORX; + #ifdef HAVE_INTEL_AVX2 + if (IS_INTEL_AVX2(intel_flags)) { + #ifdef HAVE_INTEL_RORX + if (IS_INTEL_BMI2(intel_flags)) { + Transform_Sha256_p = Transform_Sha256_AVX2_RORX; + Transform_Sha256_Len_p = Transform_Sha256_AVX2_RORX_Len; + } else - Transform_p = Transform_AVX2; + #endif + if (1) + { + Transform_Sha256_p = Transform_Sha256_AVX2; + Transform_Sha256_Len_p = Transform_Sha256_AVX2_Len; + } + #ifdef HAVE_INTEL_RORX + else { + Transform_Sha256_p = Transform_Sha256_AVX1_RORX; + Transform_Sha256_Len_p = Transform_Sha256_AVX1_RORX_Len; + } + #endif } else #endif - #if defined(HAVE_INTEL_AVX1) - if (1) { - Transform_p = ((IS_INTEL_AVX1(intel_flags)) ? Transform_AVX1 : - Transform); + #ifdef HAVE_INTEL_AVX1 + if (IS_INTEL_AVX1(intel_flags)) { + Transform_Sha256_p = Transform_Sha256_AVX1; + Transform_Sha256_Len_p = Transform_Sha256_AVX1_Len; } else #endif - Transform_p = Transform; + { + Transform_Sha256_p = Transform_Sha256; + Transform_Sha256_Len_p = NULL; + } transform_check = 1; } - /* Dummy for saving MM_REGs on behalf of Transform */ - #if defined(HAVE_INTEL_AVX2) && !defined(HAVE_INTEL_AVX1) - #define SAVE_XMM_YMM __asm__ volatile("or %%r8d, %%r8d":::\ - "%ymm4","%ymm5","%ymm6","%ymm7","%ymm8","%ymm9","%ymm10","%ymm11","%ymm12","%ymm13","%ymm14","%ymm15") - #elif defined(HAVE_INTEL_AVX1) - #define SAVE_XMM_YMM __asm__ volatile("or %%r8d, %%r8d":::\ - "xmm0","xmm1","xmm2","xmm3","xmm4","xmm5","xmm6","xmm7","xmm8","xmm9","xmm10",\ - "xmm11","xmm12","xmm13","xmm14","xmm15") - #endif - int wc_InitSha256_ex(wc_Sha256* sha256, void* heap, int devId) { int ret = 0; @@ -288,7 +319,8 @@ static int InitSha256(wc_Sha256* sha256) #include "fsl_mmcau.h" #endif - #define XTRANSFORM(S, B) Transform((S), (B)) + #define XTRANSFORM(S) Transform_Sha256((S)) + #define XTRANSFORM_LEN(S,L) Transform_Sha256_Len((S),(L)) int wc_InitSha256_ex(wc_Sha256* sha256, void* heap, int devId) { @@ -315,7 +347,7 @@ static int InitSha256(wc_Sha256* sha256) return ret; } - static int Transform(wc_Sha256* sha256, byte* buf) + static int Transform_Sha256(wc_Sha256* sha256, byte* buf) { int ret = wolfSSL_CryptHwMutexLock(); if (ret == 0) { @@ -511,10 +543,6 @@ static int InitSha256(wc_Sha256* sha256) } #endif /* End Hardware Acceleration */ -#ifndef SAVE_XMM_YMM - #define SAVE_XMM_YMM -#endif - #ifdef NEED_SOFT_SHA256 static const ALIGN32 word32 K[64] = { @@ -538,22 +566,32 @@ static int InitSha256(wc_Sha256* sha256) #define R(x, n) (((x) & 0xFFFFFFFFU) >> (n)) #define S(x, n) rotrFixed(x, n) - #define Sigma0(x) (S(x, 2) ^ S(x, 13) ^ S(x, 22)) - #define Sigma1(x) (S(x, 6) ^ S(x, 11) ^ S(x, 25)) - #define Gamma0(x) (S(x, 7) ^ S(x, 18) ^ R(x, 3)) + #define Sigma0(x) (S(x, 2) ^ S(x, 13) ^ S(x, 22)) + #define Sigma1(x) (S(x, 6) ^ S(x, 11) ^ S(x, 25)) + #define Gamma0(x) (S(x, 7) ^ S(x, 18) ^ R(x, 3)) #define Gamma1(x) (S(x, 17) ^ S(x, 19) ^ R(x, 10)) - #define RND(a,b,c,d,e,f,g,h,i) \ - t0 = (h) + Sigma1((e)) + Ch((e), (f), (g)) + K[(i)] + W[(i)]; \ - t1 = Sigma0((a)) + Maj((a), (b), (c)); \ - (d) += t0; \ - (h) = t0 + t1; + #define a(i) S[(0-i) & 7] + #define b(i) S[(1-i) & 7] + #define c(i) S[(2-i) & 7] + #define d(i) S[(3-i) & 7] + #define e(i) S[(4-i) & 7] + #define f(i) S[(5-i) & 7] + #define g(i) S[(6-i) & 7] + #define h(i) S[(7-i) & 7] + + #define RND(j) \ + t0 = h(j) + Sigma1(e(j)) + Ch(e(j), f(j), g(j)) + K[i+j] + W[i+j]; \ + t1 = Sigma0(a(j)) + Maj(a(j), b(j), c(j)); \ + d(j) += t0; \ + h(j) = t0 + t1 #ifndef XTRANSFORM - #define XTRANSFORM(S, B) Transform((S)) - #endif + #define XTRANSFORM(S) Transform_Sha256((S)) + #define XTRANSFORM_LEN(S,L) Transform_Sha256_Len((S),(L)) + #endif - static int Transform(wc_Sha256* sha256) + static int Transform_Sha256(wc_Sha256* sha256) { word32 S[8], t0, t1; int i; @@ -579,16 +617,21 @@ static int InitSha256(wc_Sha256* sha256) for (i = 16; i < WC_SHA256_BLOCK_SIZE; i++) W[i] = Gamma1(W[i-2]) + W[i-7] + Gamma0(W[i-15]) + W[i-16]; + #ifdef USE_SLOW_SHA256 + /* not unrolled - ~2k smaller and ~25% slower */ for (i = 0; i < WC_SHA256_BLOCK_SIZE; i += 8) { - RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],i+0); - RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],i+1); - RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],i+2); - RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],i+3); - RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],i+4); - RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],i+5); - RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],i+6); - RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],i+7); + int j; + for (j = 0; j < 8; j++) { /* braces needed here for macros {} */ + RND(j); + } } + #else + /* partially loop unrolled */ + for (i = 0; i < WC_SHA256_BLOCK_SIZE; i += 8) { + RND(0); RND(1); RND(2); RND(3); + RND(4); RND(5); RND(6); RND(7); + } + #endif /* USE_SLOW_SHA256 */ /* Add the working vars back into digest state[] */ for (i = 0; i < 8; i++) { @@ -609,7 +652,7 @@ static int InitSha256(wc_Sha256* sha256) static INLINE void AddLength(wc_Sha256* sha256, word32 len) { word32 tmp = sha256->loLen; - if ( (sha256->loLen += len) < tmp) + if ((sha256->loLen += len) < tmp) sha256->hiLen++; /* carry low to high */ } #endif @@ -641,9 +684,7 @@ static INLINE void AddLength(wc_Sha256* sha256, word32 len) if (sha256->buffLen >= WC_SHA256_BLOCK_SIZE) return BUFFER_E; - SAVE_XMM_YMM; /* for Intel AVX */ - - while (len) { + if (sha256->buffLen > 0) { word32 add = min(len, WC_SHA256_BLOCK_SIZE - sha256->buffLen); XMEMCPY(&local[sha256->buffLen], data, add); @@ -658,19 +699,77 @@ static INLINE void AddLength(wc_Sha256* sha256, word32 len) #endif { ByteReverseWords(sha256->buffer, sha256->buffer, - WC_SHA256_BLOCK_SIZE); + WC_SHA256_BLOCK_SIZE); } #endif - ret = XTRANSFORM(sha256, local); - if (ret != 0) { - break; + ret = XTRANSFORM(sha256); + if (ret == 0) { + AddLength(sha256, WC_SHA256_BLOCK_SIZE); + sha256->buffLen = 0; } - - AddLength(sha256, WC_SHA256_BLOCK_SIZE); - sha256->buffLen = 0; + else + len = 0; } } + #if defined(HAVE_INTEL_AVX1) || defined(HAVE_INTEL_AVX2) + if (Transform_Sha256_Len_p != NULL) { + word32 blocksLen = len & ~(WC_SHA256_BLOCK_SIZE-1); + + if (blocksLen > 0) { + AddLength(sha256, blocksLen); + sha256->data = data; + /* Byte reversal performed in function if required. */ + XTRANSFORM_LEN(sha256, blocksLen); + data += blocksLen; + len -= blocksLen; + } + } + else + #endif + #if !defined(LITTLE_ENDIAN_ORDER) || defined(FREESCALE_MMCAU_SHA) || \ + defined(HAVE_INTEL_AVX1) || defined(HAVE_INTEL_AVX2) + { + word32 blocksLen = len & ~(WC_SHA256_BLOCK_SIZE-1); + + AddLength(sha256, blocksLen); + while (len >= WC_SHA256_BLOCK_SIZE) { + XMEMCPY(local, data, WC_SHA256_BLOCK_SIZE); + + data += WC_SHA256_BLOCK_SIZE; + len -= WC_SHA256_BLOCK_SIZE; + + /* Byte reversal performed in function if required. */ + ret = XTRANSFORM(sha256); + if (ret != 0) + break; + } + } + #else + { + word32 blocksLen = len & ~(WC_SHA256_BLOCK_SIZE-1); + + AddLength(sha256, blocksLen); + while (len >= WC_SHA256_BLOCK_SIZE) { + XMEMCPY(local, data, WC_SHA256_BLOCK_SIZE); + + data += WC_SHA256_BLOCK_SIZE; + len -= WC_SHA256_BLOCK_SIZE; + + ByteReverseWords(sha256->buffer, sha256->buffer, + WC_SHA256_BLOCK_SIZE); + ret = XTRANSFORM(sha256); + if (ret != 0) + break; + } + } + #endif + + if (len > 0) { + XMEMCPY(local, data, len); + sha256->buffLen = len; + } + return ret; } @@ -689,8 +788,6 @@ static INLINE void AddLength(wc_Sha256* sha256, word32 len) return BAD_FUNC_ARG; } - SAVE_XMM_YMM; /* for Intel AVX */ - AddLength(sha256, sha256->buffLen); /* before adding pads */ local[sha256->buffLen++] = 0x80; /* add 1 */ @@ -707,12 +804,12 @@ static INLINE void AddLength(wc_Sha256* sha256, word32 len) #endif { ByteReverseWords(sha256->buffer, sha256->buffer, - WC_SHA256_BLOCK_SIZE); + WC_SHA256_BLOCK_SIZE); } #endif } - ret = XTRANSFORM(sha256, local); + ret = XTRANSFORM(sha256); if (ret != 0) return ret; @@ -754,7 +851,7 @@ static INLINE void AddLength(wc_Sha256* sha256, word32 len) } #endif - return XTRANSFORM(sha256, local); + return XTRANSFORM(sha256); } int wc_Sha256Final(wc_Sha256* sha256, byte* hash) @@ -791,477 +888,812 @@ static INLINE void AddLength(wc_Sha256* sha256, word32 len) #if defined(HAVE_INTEL_AVX1) || defined(HAVE_INTEL_AVX2) -#define _DigestToReg(S0, S1, S2, S3, S4, S5, S6, S7) \ - "leaq %[digest], %%r8\n\t" \ - "movl (%%r8), %"#S0"\n\t" \ - "movl 4(%%r8), %"#S1"\n\t" \ - "movl 8(%%r8), %"#S2"\n\t" \ - "movl 12(%%r8), %"#S3"\n\t" \ - "movl 16(%%r8), %"#S4"\n\t" \ - "movl 20(%%r8), %"#S5"\n\t" \ - "movl 24(%%r8), %"#S6"\n\t" \ - "movl 28(%%r8), %"#S7"\n\t" +#define _LOAD_DIGEST() \ + "movl (%[sha256]), %%r8d \n\t" \ + "movl 4(%[sha256]), %%r9d \n\t" \ + "movl 8(%[sha256]), %%r10d\n\t" \ + "movl 12(%[sha256]), %%r11d\n\t" \ + "movl 16(%[sha256]), %%r12d\n\t" \ + "movl 20(%[sha256]), %%r13d\n\t" \ + "movl 24(%[sha256]), %%r14d\n\t" \ + "movl 28(%[sha256]), %%r15d\n\t" -#define _RegToDigest(S0, S1, S2, S3, S4, S5, S6, S7) \ - "leaq %[digest], %%r8\n\t" \ - "addl %"#S0", (%%r8)\n\t" \ - "addl %"#S1", 4(%%r8)\n\t" \ - "addl %"#S2", 8(%%r8)\n\t" \ - "addl %"#S3", 12(%%r8)\n\t" \ - "addl %"#S4", 16(%%r8)\n\t" \ - "addl %"#S5", 20(%%r8)\n\t" \ - "addl %"#S6", 24(%%r8)\n\t" \ - "addl %"#S7", 28(%%r8)\n\t" +#define _STORE_ADD_DIGEST() \ + "addl %%r8d , (%[sha256])\n\t" \ + "addl %%r9d , 4(%[sha256])\n\t" \ + "addl %%r10d, 8(%[sha256])\n\t" \ + "addl %%r11d, 12(%[sha256])\n\t" \ + "addl %%r12d, 16(%[sha256])\n\t" \ + "addl %%r13d, 20(%[sha256])\n\t" \ + "addl %%r14d, 24(%[sha256])\n\t" \ + "addl %%r15d, 28(%[sha256])\n\t" -#define DigestToReg(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7 )\ - _DigestToReg(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7 ) +#define _ADD_DIGEST() \ + "addl (%[sha256]), %%r8d \n\t" \ + "addl 4(%[sha256]), %%r9d \n\t" \ + "addl 8(%[sha256]), %%r10d\n\t" \ + "addl 12(%[sha256]), %%r11d\n\t" \ + "addl 16(%[sha256]), %%r12d\n\t" \ + "addl 20(%[sha256]), %%r13d\n\t" \ + "addl 24(%[sha256]), %%r14d\n\t" \ + "addl 28(%[sha256]), %%r15d\n\t" -#define RegToDigest(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7 )\ - _RegToDigest(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7 ) +#define _STORE_DIGEST() \ + "movl %%r8d , (%[sha256])\n\t" \ + "movl %%r9d , 4(%[sha256])\n\t" \ + "movl %%r10d, 8(%[sha256])\n\t" \ + "movl %%r11d, 12(%[sha256])\n\t" \ + "movl %%r12d, 16(%[sha256])\n\t" \ + "movl %%r13d, 20(%[sha256])\n\t" \ + "movl %%r14d, 24(%[sha256])\n\t" \ + "movl %%r15d, 28(%[sha256])\n\t" + +#define LOAD_DIGEST() \ + _LOAD_DIGEST() + +#define STORE_ADD_DIGEST() \ + _STORE_ADD_DIGEST() + +#define ADD_DIGEST() \ + _ADD_DIGEST() + +#define STORE_DIGEST() \ + _STORE_DIGEST() -#define S_0 %r15d -#define S_1 %r10d -#define S_2 %r11d -#define S_3 %r12d -#define S_4 %r13d -#define S_5 %r14d -#define S_6 %ebx -#define S_7 %r9d +#define S_0 %r8d +#define S_1 %r9d +#define S_2 %r10d +#define S_3 %r11d +#define S_4 %r12d +#define S_5 %r13d +#define S_6 %r14d +#define S_7 %r15d -#define SSE_REGs "%edi", "%esi", "%edx", "%ebx","%r8","%r9","%r10","%r11","%r12","%r13","%r14","%r15" +#define L1 "%%edx" +#define L2 "%%ecx" +#define L3 "%%eax" +#define L4 "%%ebx" +#define WK "%%rsp" + +#define WORK_REGS "eax", "ebx", "ecx", "edx" +#define STATE_REGS "r8","r9","r10","r11","r12","r13","r14","r15" +#define XMM_REGS "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", \ + "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13" #if defined(HAVE_INTEL_RORX) -#define RND_STEP_RORX_1(a,b,c,d,e,f,g,h,i) \ - "# edx = e>>>6\n\t" \ - "rorx $6, %"#e", %%edx\n\t" +#define RND_STEP_RORX_0_1(a, b, c, d, e, f, g, h, i) \ + /* L3 = f */ \ + "movl %"#f", "L3"\n\t" \ + /* L2 = e>>>11 */ \ + "rorx $11, %"#e", "L2"\n\t" \ + /* h += w_k */ \ + "addl ("#i")*4("WK"), %"#h"\n\t" \ -#define RND_STEP_RORX_2(a,b,c,d,e,f,g,h,i) \ - "# edi = e>>>11\n\t" \ - "rorx $11, %"#e",%%edi\n\t" \ - "# edi = (e>>11) ^ (e>>6)\n\t" \ - "xorl %%edx, %%edi\n\t" \ - "# edx = e>>>25\n\t" \ - "rorx $25, %"#e", %%edx\n\t" +#define RND_STEP_RORX_0_2(a, b, c, d, e, f, g, h, i) \ + /* L2 = (e>>>6) ^ (e>>>11) */ \ + "xorl "L1", "L2"\n\t" \ + /* L3 = f ^ g */ \ + "xorl %"#g", "L3"\n\t" \ + /* L1 = e>>>25 */ \ + "rorx $25, %"#e", "L1"\n\t" \ -#define RND_STEP_RORX_3(a,b,c,d,e,f,g,h,i) \ - "# esi = f\n\t" \ - "movl %"#f", %%esi\n\t" \ - "# esi = f ^ g\n\t" \ - "xorl %"#g", %%esi\n\t" \ - "# edx = Sigma1(e)\n\t" \ - "xorl %%edi, %%edx\n\t" \ - "# esi = (f ^ g) & e\n\t" \ - "andl %"#e", %%esi\n\t" \ - "# esi = Ch(e,f,g)\n\t" \ - "xorl %"#g", %%esi\n\t" +#define RND_STEP_RORX_0_3(a, b, c, d, e, f, g, h, i) \ + /* L3 = (f ^ g) & e */ \ + "andl %"#e", "L3"\n\t" \ + /* L1 = Sigma1(e) */ \ + "xorl "L2", "L1"\n\t" \ + /* L2 = a>>>13 */ \ + "rorx $13, %"#a", "L2"\n\t" \ -#define RND_STEP_RORX_4(a,b,c,d,e,f,g,h,i) \ - "# h += w_k\n\t" \ - "leaq %[W_K], %%r8\n\t" \ - "addl ("#i")*4(%%r8), %"#h"\n\t" \ - "# h = h + w_k + Sigma1(e)\n\t" \ - "addl %%edx, %"#h"\n\t" \ - "# r8d = a>>>2\n\t" \ - "rorx $2, %"#a", %%r8d\n\t" \ - "# edi = a>>>13\n\t" \ - "rorx $13, %"#a", %%edi\n\t" +#define RND_STEP_RORX_0_4(a, b, c, d, e, f, g, h, i) \ + /* h += Sigma1(e) */ \ + "addl "L1", %"#h"\n\t" \ + /* L1 = a>>>2 */ \ + "rorx $2, %"#a", "L1"\n\t" \ + /* L3 = Ch(e,f,g) */ \ + "xorl %"#g", "L3"\n\t" \ -#define RND_STEP_RORX_5(a,b,c,d,e,f,g,h,i) \ - "# edx = a>>22\n\t" \ - "rorx $22, %"#a", %%edx\n\t" \ - "# edi = (a>>>2) ^ (a>>>13)\n\t" \ - "xorl %%r8d, %%edi\n\t" \ - "# edx = Sigma0(a)\n\t" \ - "xorl %%edi, %%edx\n\t" +#define RND_STEP_RORX_0_5(a, b, c, d, e, f, g, h, i) \ + /* L2 = (a>>>2) ^ (a>>>13) */ \ + "xorl "L1", "L2"\n\t" \ + /* L1 = a>>>22 */ \ + "rorx $22, %"#a", "L1"\n\t" \ + /* h += Ch(e,f,g) */ \ + "addl "L3", %"#h"\n\t" \ -#define RND_STEP_RORX_6(a,b,c,d,e,f,g,h,i) \ - "# edi = b\n\t" \ - "movl %"#b", %%edi\n\t" \ - "# edi = a | b\n\t" \ - "orl %"#a", %%edi\n\t" \ - "# edi = (a | b) & c\n\t" \ - "andl %"#c", %%edi\n\t" \ - "# r8d = b\n\t" \ - "movl %"#b", %%r8d\n\t" +#define RND_STEP_RORX_0_6(a, b, c, d, e, f, g, h, i) \ + /* L1 = Sigma0(a) */ \ + "xorl "L2", "L1"\n\t" \ + /* L3 = b */ \ + "movl %"#b", "L3"\n\t" \ + /* d += h + w_k + Sigma1(e) + Ch(e,f,g) */ \ + "addl %"#h", %"#d"\n\t" \ -#define RND_STEP_RORX_7(a,b,c,d,e,f,g,h,i) \ - "# h += Ch(e,f,g)\n\t" \ - "addl %%esi, %"#h"\n\t" \ - "# r8d = b & a\n\t" \ - "andl %"#a", %%r8d\n\t" \ - "# r8d = Maj(a,b,c)\n\t" \ - "orl %%edi, %%r8d\n\t" +#define RND_STEP_RORX_0_7(a, b, c, d, e, f, g, h, i) \ + /* L3 = a ^ b */ \ + "xorl %"#a", "L3"\n\t" \ + /* h += Sigma0(a) */ \ + "addl "L1", %"#h"\n\t" \ + /* L4 = (a ^ b) & (b ^ c) */ \ + "andl "L3", "L4"\n\t" \ + +#define RND_STEP_RORX_0_8(a, b, c, d, e, f, g, h, i) \ + /* L4 = Maj(a,b,c) */ \ + "xorl %"#b", "L4"\n\t" \ + /* L1 = d>>>6 (= e>>>6 next RND) */ \ + "rorx $6, %"#d", "L1"\n\t" \ + /* h += Maj(a,b,c) */ \ + "addl "L4", %"#h"\n\t" \ + +#define RND_STEP_RORX_1_1(a, b, c, d, e, f, g, h, i) \ + /* L4 = f */ \ + "movl %"#f", "L4"\n\t" \ + /* L2 = e>>>11 */ \ + "rorx $11, %"#e", "L2"\n\t" \ + /* h += w_k */ \ + "addl ("#i")*4("WK"), %"#h"\n\t" \ + +#define RND_STEP_RORX_1_2(a, b, c, d, e, f, g, h, i) \ + /* L2 = (e>>>6) ^ (e>>>11) */ \ + "xorl "L1", "L2"\n\t" \ + /* L4 = f ^ g */ \ + "xorl %"#g", "L4"\n\t" \ + /* L1 = e>>>25 */ \ + "rorx $25, %"#e", "L1"\n\t" \ + +#define RND_STEP_RORX_1_3(a, b, c, d, e, f, g, h, i) \ + /* L4 = (f ^ g) & e */ \ + "andl %"#e", "L4"\n\t" \ + /* L1 = Sigma1(e) */ \ + "xorl "L2", "L1"\n\t" \ + /* L2 = a>>>13 */ \ + "rorx $13, %"#a", "L2"\n\t" \ + +#define RND_STEP_RORX_1_4(a, b, c, d, e, f, g, h, i) \ + /* h += Sigma1(e) */ \ + "addl "L1", %"#h"\n\t" \ + /* L1 = a>>>2 */ \ + "rorx $2, %"#a", "L1"\n\t" \ + /* L4 = Ch(e,f,g) */ \ + "xorl %"#g", "L4"\n\t" \ + +#define RND_STEP_RORX_1_5(a, b, c, d, e, f, g, h, i) \ + /* L2 = (a>>>2) ^ (a>>>13) */ \ + "xorl "L1", "L2"\n\t" \ + /* L1 = a>>>22 */ \ + "rorx $22, %"#a", "L1"\n\t" \ + /* h += Ch(e,f,g) */ \ + "addl "L4", %"#h"\n\t" \ + +#define RND_STEP_RORX_1_6(a, b, c, d, e, f, g, h, i) \ + /* L1 = Sigma0(a) */ \ + "xorl "L2", "L1"\n\t" \ + /* L4 = b */ \ + "movl %"#b", "L4"\n\t" \ + /* d += h + w_k + Sigma1(e) + Ch(e,f,g) */ \ + "addl %"#h", %"#d"\n\t" \ + +#define RND_STEP_RORX_1_7(a, b, c, d, e, f, g, h, i) \ + /* L4 = a ^ b */ \ + "xorl %"#a", "L4"\n\t" \ + /* h += Sigma0(a) */ \ + "addl "L1", %"#h"\n\t" \ + /* L3 = (a ^ b) & (b ^ c) */ \ + "andl "L4", "L3"\n\t" \ + +#define RND_STEP_RORX_1_8(a, b, c, d, e, f, g, h, i) \ + /* L3 = Maj(a,b,c) */ \ + "xorl %"#b", "L3"\n\t" \ + /* L1 = d>>>6 (= e>>>6 next RND) */ \ + "rorx $6, %"#d", "L1"\n\t" \ + /* h += Maj(a,b,c) */ \ + "addl "L3", %"#h"\n\t" \ + +#define _RND_RORX_X_0(a, b, c, d, e, f, g, h, i) \ + /* L1 = e>>>6 */ \ + "rorx $6, %"#e", "L1"\n\t" \ + /* L2 = e>>>11 */ \ + "rorx $11, %"#e", "L2"\n\t" \ + /* Prev RND: h += Maj(a,b,c) */ \ + "addl "L3", %"#a"\n\t" \ + /* h += w_k */ \ + "addl ("#i")*4("WK"), %"#h"\n\t" \ + /* L3 = f */ \ + "movl %"#f", "L3"\n\t" \ + /* L2 = (e>>>6) ^ (e>>>11) */ \ + "xorl "L1", "L2"\n\t" \ + /* L3 = f ^ g */ \ + "xorl %"#g", "L3"\n\t" \ + /* L1 = e>>>25 */ \ + "rorx $25, %"#e", "L1"\n\t" \ + /* L1 = Sigma1(e) */ \ + "xorl "L2", "L1"\n\t" \ + /* L3 = (f ^ g) & e */ \ + "andl %"#e", "L3"\n\t" \ + /* h += Sigma1(e) */ \ + "addl "L1", %"#h"\n\t" \ + /* L1 = a>>>2 */ \ + "rorx $2, %"#a", "L1"\n\t" \ + /* L2 = a>>>13 */ \ + "rorx $13, %"#a", "L2"\n\t" \ + /* L3 = Ch(e,f,g) */ \ + "xorl %"#g", "L3"\n\t" \ + /* L2 = (a>>>2) ^ (a>>>13) */ \ + "xorl "L1", "L2"\n\t" \ + /* L1 = a>>>22 */ \ + "rorx $22, %"#a", "L1"\n\t" \ + /* h += Ch(e,f,g) */ \ + "addl "L3", %"#h"\n\t" \ + /* L1 = Sigma0(a) */ \ + "xorl "L2", "L1"\n\t" \ + /* L3 = b */ \ + "movl %"#b", "L3"\n\t" \ + /* d += h + w_k + Sigma1(e) + Ch(e,f,g) */ \ + "addl %"#h", %"#d"\n\t" \ + /* L3 = a ^ b */ \ + "xorl %"#a", "L3"\n\t" \ + /* L4 = (a ^ b) & (b ^ c) */ \ + "andl "L3", "L4"\n\t" \ + /* h += Sigma0(a) */ \ + "addl "L1", %"#h"\n\t" \ + /* L4 = Maj(a,b,c) */ \ + "xorl %"#b", "L4"\n\t" \ + +#define _RND_RORX_X_1(a, b, c, d, e, f, g, h, i) \ + /* L1 = e>>>6 */ \ + "rorx $6, %"#e", "L1"\n\t" \ + /* L2 = e>>>11 */ \ + "rorx $11, %"#e", "L2"\n\t" \ + /* Prev RND: h += Maj(a,b,c) */ \ + "addl "L4", %"#a"\n\t" \ + /* h += w_k */ \ + "addl ("#i")*4("WK"), %"#h"\n\t" \ + /* L4 = f */ \ + "movl %"#f", "L4"\n\t" \ + /* L2 = (e>>>6) ^ (e>>>11) */ \ + "xorl "L1", "L2"\n\t" \ + /* L4 = f ^ g */ \ + "xorl %"#g", "L4"\n\t" \ + /* L1 = e>>>25 */ \ + "rorx $25, %"#e", "L1"\n\t" \ + /* L1 = Sigma1(e) */ \ + "xorl "L2", "L1"\n\t" \ + /* L4 = (f ^ g) & e */ \ + "andl %"#e", "L4"\n\t" \ + /* h += Sigma1(e) */ \ + "addl "L1", %"#h"\n\t" \ + /* L1 = a>>>2 */ \ + "rorx $2, %"#a", "L1"\n\t" \ + /* L2 = a>>>13 */ \ + "rorx $13, %"#a", "L2"\n\t" \ + /* L4 = Ch(e,f,g) */ \ + "xorl %"#g", "L4"\n\t" \ + /* L2 = (a>>>2) ^ (a>>>13) */ \ + "xorl "L1", "L2"\n\t" \ + /* L1 = a>>>22 */ \ + "rorx $22, %"#a", "L1"\n\t" \ + /* h += Ch(e,f,g) */ \ + "addl "L4", %"#h"\n\t" \ + /* L1 = Sigma0(a) */ \ + "xorl "L2", "L1"\n\t" \ + /* L4 = b */ \ + "movl %"#b", "L4"\n\t" \ + /* d += h + w_k + Sigma1(e) + Ch(e,f,g) */ \ + "addl %"#h", %"#d"\n\t" \ + /* L4 = a ^ b */ \ + "xorl %"#a", "L4"\n\t" \ + /* L2 = (a ^ b) & (b ^ c) */ \ + "andl "L4", "L3"\n\t" \ + /* h += Sigma0(a) */ \ + "addl "L1", %"#h"\n\t" \ + /* L3 = Maj(a,b,c) */ \ + "xorl %"#b", "L3"\n\t" \ + + +#define RND_RORX_X_0(a,b,c,d,e,f,g,h,i) \ + _RND_RORX_X_0(a,b,c,d,e,f,g,h,i) +#define RND_RORX_X_1(a,b,c,d,e,f,g,h,i) \ + _RND_RORX_X_1(a,b,c,d,e,f,g,h,i) + +#define RND_RORX_X4(a,b,c,d,e,f,g,h,i) \ + RND_RORX_X_0(a,b,c,d,e,f,g,h,i+0) \ + RND_RORX_X_1(h,a,b,c,d,e,f,g,i+1) \ + RND_RORX_X_0(g,h,a,b,c,d,e,f,i+2) \ + RND_RORX_X_1(f,g,h,a,b,c,d,e,i+3) -#define RND_STEP_RORX_8(a,b,c,d,e,f,g,h,i) \ - "# d += h + w_k + Sigma1(e) + Ch(e,f,g)\n\t" \ - "addl %"#h", %"#d"\n\t" \ - "addl %"#h", %%r8d\n\t" \ - "addl %%edx, %%r8d\n\t" \ - "movl %%r8d, %"#h"\n\t" #endif /* HAVE_INTEL_RORX */ -#define RND_STEP_1(a,b,c,d,e,f,g,h,i) \ - "movl %"#e", %%edx\n\t" \ - "# edx = e>>>6\n\t" \ - "roll $26, %%edx\n\t" \ - "movl %"#e", %%edi\n\t" +#define RND_STEP_0_1(a,b,c,d,e,f,g,h,i) \ + /* L1 = e>>>14 */ \ + "rorl $14, "L1"\n\t" \ -#define RND_STEP_2(a,b,c,d,e,f,g,h,i) \ - "# edi = e>>>11\n\t" \ - "roll $21, %%edi\n\t" \ - "# edi = (e>>11) ^ (e>>6)\n\t" \ - "xorl %%edx, %%edi\n\t" \ - "# edx = e\n\t" \ - "movl %"#e", %%edx\n\t" \ - "# edx = e>>>25\n\t" \ - "roll $7, %%edx\n\t" +#define RND_STEP_0_2(a,b,c,d,e,f,g,h,i) \ + /* L3 = b */ \ + "movl %"#b", "L3"\n\t" \ + /* L2 = f */ \ + "movl %"#f", "L2"\n\t" \ + /* h += w_k */ \ + "addl ("#i")*4("WK"), %"#h"\n\t" \ + /* L2 = f ^ g */ \ + "xorl %"#g", "L2"\n\t" \ -#define RND_STEP_3(a,b,c,d,e,f,g,h,i) \ - "# esi = f\n\t" \ - "movl %"#f", %%esi\n\t" \ - "# esi = f ^ g\n\t" \ - "xorl %"#g", %%esi\n\t" \ - "# edx = Sigma1(e)\n\t" \ - "xorl %%edi, %%edx\n\t" \ - "# esi = (f ^ g) & e\n\t" \ - "andl %"#e", %%esi\n\t" \ - "# esi = Ch(e,f,g)\n\t" \ - "xorl %"#g", %%esi\n\t" +#define RND_STEP_0_3(a,b,c,d,e,f,g,h,i) \ + /* L1 = (e>>>14) ^ e */ \ + "xorl %"#e", "L1"\n\t" \ + /* L2 = (f ^ g) & e */ \ + "andl %"#e", "L2"\n\t" \ -#define RND_STEP_4(a,b,c,d,e,f,g,h,i) \ - "# h += w_k\n\t" \ - "leaq %[W_K], %%r8\n\t" \ - "addl ("#i")*4(%%r8), %"#h"\n\t" \ - "# h = h + w_k + Sigma1(e)\n\t" \ - "addl %%edx, %"#h"\n\t" \ - "# r8d = a\n\t" \ - "movl %"#a", %%r8d\n\t" \ - "# r8d = a>>>2\n\t" \ - "roll $30, %%r8d\n\t" \ - "# edi = a\n\t" \ - "movl %"#a", %%edi\n\t" \ - "# edi = a>>>13\n\t" \ - "roll $19, %%edi\n\t" \ - "# edx = a\n\t" \ - "movl %"#a", %%edx\n\t" +#define RND_STEP_0_4(a,b,c,d,e,f,g,h,i) \ + /* L1 = ((e>>>14) ^ e) >>> 5 */ \ + "rorl $5, "L1"\n\t" \ + /* L2 = Ch(e,f,g) */ \ + "xorl %"#g", "L2"\n\t" \ + /* L1 = (((e>>>14) ^ e) >>> 5) ^ e */ \ + "xorl %"#e", "L1"\n\t" \ + /* h += Ch(e,f,g) */ \ + "addl "L2", %"#h"\n\t" \ -#define RND_STEP_5(a,b,c,d,e,f,g,h,i) \ - "# edx = a>>>22\n\t" \ - "roll $10, %%edx\n\t" \ - "# edi = (a>>>2) ^ (a>>>13)\n\t" \ - "xorl %%r8d, %%edi\n\t" \ - "# edx = Sigma0(a)\n\t" \ - "xorl %%edi, %%edx\n\t" +#define RND_STEP_0_5(a,b,c,d,e,f,g,h,i) \ + /* L1 = ((((e>>>14) ^ e) >>> 5) ^ e) >>> 6 */ \ + "rorl $6, "L1"\n\t" \ + /* L3 = a ^ b (= b ^ c of next RND) */ \ + "xorl %"#a", "L3"\n\t" \ + /* h = h + w_k + Sigma1(e) */ \ + "addl "L1", %"#h"\n\t" \ + /* L2 = a */ \ + "movl %"#a", "L2"\n\t" \ -#define RND_STEP_6(a,b,c,d,e,f,g,h,i) \ - "# edi = b\n\t" \ - "movl %"#b", %%edi\n\t" \ - "# edi = a | b\n\t" \ - "orl %"#a", %%edi\n\t" \ - "# edi = (a | b) & c\n\t" \ - "andl %"#c", %%edi\n\t" \ - "# r8d = b\n\t" \ - "movl %"#b", %%r8d\n\t" +#define RND_STEP_0_6(a,b,c,d,e,f,g,h,i) \ + /* L3 = (a ^ b) & (b ^ c) */ \ + "andl "L3", "L4"\n\t" \ + /* L2 = a>>>9 */ \ + "rorl $9, "L2"\n\t" \ + /* L2 = (a>>>9) ^ a */ \ + "xorl %"#a", "L2"\n\t" \ + /* L1 = Maj(a,b,c) */ \ + "xorl %"#b", "L4"\n\t" \ -#define RND_STEP_7(a,b,c,d,e,f,g,h,i) \ - "# h += Ch(e,f,g)\n\t" \ - "addl %%esi, %"#h"\n\t" \ - "#r8d = b & a\n\t" \ - "andl %"#a", %%r8d\n\t" \ - "# r8d = Maj(a,b,c)\n\t" \ - "orl %%edi, %%r8d\n\t" +#define RND_STEP_0_7(a,b,c,d,e,f,g,h,i) \ + /* L2 = ((a>>>9) ^ a) >>> 11 */ \ + "rorl $11, "L2"\n\t" \ + /* d += h + w_k + Sigma1(e) + Ch(e,f,g) */ \ + "addl %"#h", %"#d"\n\t" \ + /* L2 = (((a>>>9) ^ a) >>> 11) ^ a */ \ + "xorl %"#a", "L2"\n\t" \ + /* h = h + w_k + Sigma1(e) + Ch(e,f,g) + Maj(a,b,c) */ \ + "addl "L4", %"#h"\n\t" \ -#define RND_STEP_8(a,b,c,d,e,f,g,h,i) \ - "# d += h + w_k + Sigma1(e) + Ch(e,f,g)\n\t" \ - "addl %"#h", %"#d"\n\t" \ - "# r8b = h + w_k + Sigma1(e) + Ch(e,f,g) + Maj(a,b,c)\n\t" \ - "addl %"#h", %%r8d\n\t" \ - "# r8b = h + w_k + Sigma1(e) Sigma0(a) + Ch(e,f,g) + Maj(a,b,c)\n\t" \ - "addl %%edx, %%r8d\n\t" \ - "# h = h + w_k + Sigma1(e) + Sigma0(a) + Ch(e,f,g) + Maj(a,b,c)\n\t" \ - "movl %%r8d, %"#h"\n\t" +#define RND_STEP_0_8(a,b,c,d,e,f,g,h,i) \ + /* L2 = ((((a>>>9) ^ a) >>> 11) ^ a) >>> 2 */ \ + "rorl $2, "L2"\n\t" \ + /* L1 = d (e of next RND) */ \ + "movl %"#d", "L1"\n\t" \ + /* h = h + w_k + Sigma1(e) Sigma0(a) + Ch(e,f,g) + Maj(a,b,c) */ \ + "addl "L2", %"#h"\n\t" \ -#define RND_X(a,b,c,d,e,f,g,h,i) \ - RND_STEP_1(a,b,c,d,e,f,g,h,i) \ - RND_STEP_2(a,b,c,d,e,f,g,h,i) \ - RND_STEP_3(a,b,c,d,e,f,g,h,i) \ - RND_STEP_4(a,b,c,d,e,f,g,h,i) \ - RND_STEP_5(a,b,c,d,e,f,g,h,i) \ - RND_STEP_6(a,b,c,d,e,f,g,h,i) \ - RND_STEP_7(a,b,c,d,e,f,g,h,i) \ - RND_STEP_8(a,b,c,d,e,f,g,h,i) +#define RND_STEP_1_1(a,b,c,d,e,f,g,h,i) \ + /* L1 = e>>>14 */ \ + "rorl $14, "L1"\n\t" \ -#define RND_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,_i) RND_X(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,_i) -#define RND_7(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,_i) RND_X(S_7,S_0,S_1,S_2,S_3,S_4,S_5,S_6,_i) -#define RND_6(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,_i) RND_X(S_6,S_7,S_0,S_1,S_2,S_3,S_4,S_5,_i) -#define RND_5(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,_i) RND_X(S_5,S_6,S_7,S_0,S_1,S_2,S_3,S_4,_i) -#define RND_4(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,_i) RND_X(S_4,S_5,S_6,S_7,S_0,S_1,S_2,S_3,_i) -#define RND_3(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,_i) RND_X(S_3,S_4,S_5,S_6,S_7,S_0,S_1,S_2,_i) -#define RND_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,_i) RND_X(S_2,S_3,S_4,S_5,S_6,S_7,S_0,S_1,_i) -#define RND_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,_i) RND_X(S_1,S_2,S_3,S_4,S_5,S_6,S_7,S_0,_i) +#define RND_STEP_1_2(a,b,c,d,e,f,g,h,i) \ + /* L3 = b */ \ + "movl %"#b", "L4"\n\t" \ + /* L2 = f */ \ + "movl %"#f", "L2"\n\t" \ + /* h += w_k */ \ + "addl ("#i")*4("WK"), %"#h"\n\t" \ + /* L2 = f ^ g */ \ + "xorl %"#g", "L2"\n\t" \ + +#define RND_STEP_1_3(a,b,c,d,e,f,g,h,i) \ + /* L1 = (e>>>14) ^ e */ \ + "xorl %"#e", "L1"\n\t" \ + /* L2 = (f ^ g) & e */ \ + "andl %"#e", "L2"\n\t" \ + +#define RND_STEP_1_4(a,b,c,d,e,f,g,h,i) \ + /* L1 = ((e>>>14) ^ e) >>> 5 */ \ + "rorl $5, "L1"\n\t" \ + /* L2 = Ch(e,f,g) */ \ + "xorl %"#g", "L2"\n\t" \ + /* L1 = (((e>>>14) ^ e) >>> 5) ^ e */ \ + "xorl %"#e", "L1"\n\t" \ + /* h += Ch(e,f,g) */ \ + "addl "L2", %"#h"\n\t" \ + +#define RND_STEP_1_5(a,b,c,d,e,f,g,h,i) \ + /* L1 = ((((e>>>14) ^ e) >>> 5) ^ e) >>> 6 */ \ + "rorl $6, "L1"\n\t" \ + /* L4 = a ^ b (= b ^ c of next RND) */ \ + "xorl %"#a", "L4"\n\t" \ + /* h = h + w_k + Sigma1(e) */ \ + "addl "L1", %"#h"\n\t" \ + /* L2 = a */ \ + "movl %"#a", "L2"\n\t" \ + +#define RND_STEP_1_6(a,b,c,d,e,f,g,h,i) \ + /* L3 = (a ^ b) & (b ^ c) */ \ + "andl "L4", "L3"\n\t" \ + /* L2 = a>>>9 */ \ + "rorl $9, "L2"\n\t" \ + /* L2 = (a>>>9) ^ a */ \ + "xorl %"#a", "L2"\n\t" \ + /* L1 = Maj(a,b,c) */ \ + "xorl %"#b", "L3"\n\t" \ + +#define RND_STEP_1_7(a,b,c,d,e,f,g,h,i) \ + /* L2 = ((a>>>9) ^ a) >>> 11 */ \ + "rorl $11, "L2"\n\t" \ + /* d += h + w_k + Sigma1(e) + Ch(e,f,g) */ \ + "addl %"#h", %"#d"\n\t" \ + /* L2 = (((a>>>9) ^ a) >>> 11) ^ a */ \ + "xorl %"#a", "L2"\n\t" \ + /* h = h + w_k + Sigma1(e) + Ch(e,f,g) + Maj(a,b,c) */ \ + "addl "L3", %"#h"\n\t" \ + +#define RND_STEP_1_8(a,b,c,d,e,f,g,h,i) \ + /* L2 = ((((a>>>9) ^ a) >>> 11) ^ a) >>> 2 */ \ + "rorl $2, "L2"\n\t" \ + /* L1 = d (e of next RND) */ \ + "movl %"#d", "L1"\n\t" \ + /* h = h + w_k + Sigma1(e) Sigma0(a) + Ch(e,f,g) + Maj(a,b,c) */ \ + "addl "L2", %"#h"\n\t" \ + +#define _RND_ALL_0(a,b,c,d,e,f,g,h,i) \ + /* h += w_k */ \ + "addl ("#i")*4("WK"), %"#h"\n\t" \ + /* L2 = f */ \ + "movl %"#f", "L2"\n\t" \ + /* L3 = b */ \ + "movl %"#b", "L3"\n\t" \ + /* L2 = f ^ g */ \ + "xorl %"#g", "L2"\n\t" \ + /* L1 = e>>>14 */ \ + "rorl $14, "L1"\n\t" \ + /* L2 = (f ^ g) & e */ \ + "andl %"#e", "L2"\n\t" \ + /* L1 = (e>>>14) ^ e */ \ + "xorl %"#e", "L1"\n\t" \ + /* L2 = Ch(e,f,g) */ \ + "xorl %"#g", "L2"\n\t" \ + /* L1 = ((e>>>14) ^ e) >>> 5 */ \ + "rorl $5, "L1"\n\t" \ + /* h += Ch(e,f,g) */ \ + "addl "L2", %"#h"\n\t" \ + /* L1 = (((e>>>14) ^ e) >>> 5) ^ e */ \ + "xorl %"#e", "L1"\n\t" \ + /* L3 = a ^ b */ \ + "xorl %"#a", "L3"\n\t" \ + /* L1 = ((((e>>>14) ^ e) >>> 5) ^ e) >>> 6 */ \ + "rorl $6, "L1"\n\t" \ + /* L2 = a */ \ + "movl %"#a", "L2"\n\t" \ + /* h = h + w_k + Sigma1(e) */ \ + "addl "L1", %"#h"\n\t" \ + /* L2 = a>>>9 */ \ + "rorl $9, "L2"\n\t" \ + /* L3 = (a ^ b) & (b ^ c) */ \ + "andl "L3", "L4"\n\t" \ + /* L2 = (a>>>9) ^ a */ \ + "xorl %"#a", "L2"\n\t" \ + /* L1 = Maj(a,b,c) */ \ + "xorl %"#b", "L4"\n\t" \ + /* L2 = ((a>>>9) ^ a) >>> 11 */ \ + "rorl $11, "L2"\n\t" \ + /* d += h + w_k + Sigma1(e) + Ch(e,f,g) */ \ + "addl %"#h", %"#d"\n\t" \ + /* L2 = (((a>>>9) ^ a) >>> 11) ^ a */ \ + "xorl %"#a", "L2"\n\t" \ + /* h = h + w_k + Sigma1(e) + Ch(e,f,g) + Maj(a,b,c) */ \ + "addl "L4", %"#h"\n\t" \ + /* L2 = ((((a>>>9) ^ a) >>> 11) ^ a) >>> 2 */ \ + "rorl $2, "L2"\n\t" \ + /* L1 = d (e of next RND) */ \ + "movl %"#d", "L1"\n\t" \ + /* h = h + w_k + Sigma1(e) Sigma0(a) + Ch(e,f,g) + Maj(a,b,c) */ \ + "addl "L2", %"#h"\n\t" \ + +#define _RND_ALL_1(a,b,c,d,e,f,g,h,i) \ + /* h += w_k */ \ + "addl ("#i")*4("WK"), %"#h"\n\t" \ + /* L2 = f */ \ + "movl %"#f", "L2"\n\t" \ + /* L3 = b */ \ + "movl %"#b", "L4"\n\t" \ + /* L2 = f ^ g */ \ + "xorl %"#g", "L2"\n\t" \ + /* L1 = e>>>14 */ \ + "rorl $14, "L1"\n\t" \ + /* L2 = (f ^ g) & e */ \ + "andl %"#e", "L2"\n\t" \ + /* L1 = (e>>>14) ^ e */ \ + "xorl %"#e", "L1"\n\t" \ + /* L2 = Ch(e,f,g) */ \ + "xorl %"#g", "L2"\n\t" \ + /* L1 = ((e>>>14) ^ e) >>> 5 */ \ + "rorl $5, "L1"\n\t" \ + /* h += Ch(e,f,g) */ \ + "addl "L2", %"#h"\n\t" \ + /* L1 = (((e>>>14) ^ e) >>> 5) ^ e */ \ + "xorl %"#e", "L1"\n\t" \ + /* L3 = a ^ b */ \ + "xorl %"#a", "L4"\n\t" \ + /* L1 = ((((e>>>14) ^ e) >>> 5) ^ e) >>> 6 */ \ + "rorl $6, "L1"\n\t" \ + /* L2 = a */ \ + "movl %"#a", "L2"\n\t" \ + /* h = h + w_k + Sigma1(e) */ \ + "addl "L1", %"#h"\n\t" \ + /* L2 = a>>>9 */ \ + "rorl $9, "L2"\n\t" \ + /* L3 = (a ^ b) & (b ^ c) */ \ + "andl "L4", "L3"\n\t" \ + /* L2 = (a>>>9) ^ a */ \ + "xorl %"#a", "L2"\n\t" \ + /* L1 = Maj(a,b,c) */ \ + "xorl %"#b", "L3"\n\t" \ + /* L2 = ((a>>>9) ^ a) >>> 11 */ \ + "rorl $11, "L2"\n\t" \ + /* d += h + w_k + Sigma1(e) + Ch(e,f,g) */ \ + "addl %"#h", %"#d"\n\t" \ + /* L2 = (((a>>>9) ^ a) >>> 11) ^ a */ \ + "xorl %"#a", "L2"\n\t" \ + /* h = h + w_k + Sigma1(e) + Ch(e,f,g) + Maj(a,b,c) */ \ + "addl "L3", %"#h"\n\t" \ + /* L2 = ((((a>>>9) ^ a) >>> 11) ^ a) >>> 2 */ \ + "rorl $2, "L2"\n\t" \ + /* L1 = d (e of next RND) */ \ + "movl %"#d", "L1"\n\t" \ + /* h = h + w_k + Sigma1(e) Sigma0(a) + Ch(e,f,g) + Maj(a,b,c) */ \ + "addl "L2", %"#h"\n\t" \ -#define RND_1_3(a,b,c,d,e,f,g,h,i) \ - RND_STEP_1(a,b,c,d,e,f,g,h,i) \ - RND_STEP_2(a,b,c,d,e,f,g,h,i) \ - RND_STEP_3(a,b,c,d,e,f,g,h,i) +#define RND_ALL_0(a, b, c, d, e, f, g, h, i) \ + _RND_ALL_0(a, b, c, d, e, f, g, h, i) +#define RND_ALL_1(a, b, c, d, e, f, g, h, i) \ + _RND_ALL_1(a, b, c, d, e, f, g, h, i) -#define RND_4_6(a,b,c,d,e,f,g,h,i) \ - RND_STEP_4(a,b,c,d,e,f,g,h,i) \ - RND_STEP_5(a,b,c,d,e,f,g,h,i) \ - RND_STEP_6(a,b,c,d,e,f,g,h,i) - -#define RND_7_8(a,b,c,d,e,f,g,h,i) \ - RND_STEP_7(a,b,c,d,e,f,g,h,i) \ - RND_STEP_8(a,b,c,d,e,f,g,h,i) - -#define RND_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,_i) RND_X(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,_i) -#define RND_7(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,_i) RND_X(S_7,S_0,S_1,S_2,S_3,S_4,S_5,S_6,_i) -#define RND_6(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,_i) RND_X(S_6,S_7,S_0,S_1,S_2,S_3,S_4,S_5,_i) -#define RND_5(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,_i) RND_X(S_5,S_6,S_7,S_0,S_1,S_2,S_3,S_4,_i) -#define RND_4(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,_i) RND_X(S_4,S_5,S_6,S_7,S_0,S_1,S_2,S_3,_i) -#define RND_3(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,_i) RND_X(S_3,S_4,S_5,S_6,S_7,S_0,S_1,S_2,_i) -#define RND_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,_i) RND_X(S_2,S_3,S_4,S_5,S_6,S_7,S_0,S_1,_i) -#define RND_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,_i) RND_X(S_1,S_2,S_3,S_4,S_5,S_6,S_7,S_0,_i) - - -#define RND_0_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,_i) RND_1_3(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,_i) -#define RND_7_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,_i) RND_1_3(S_7,S_0,S_1,S_2,S_3,S_4,S_5,S_6,_i) -#define RND_6_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,_i) RND_1_3(S_6,S_7,S_0,S_1,S_2,S_3,S_4,S_5,_i) -#define RND_5_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,_i) RND_1_3(S_5,S_6,S_7,S_0,S_1,S_2,S_3,S_4,_i) -#define RND_4_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,_i) RND_1_3(S_4,S_5,S_6,S_7,S_0,S_1,S_2,S_3,_i) -#define RND_3_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,_i) RND_1_3(S_3,S_4,S_5,S_6,S_7,S_0,S_1,S_2,_i) -#define RND_2_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,_i) RND_1_3(S_2,S_3,S_4,S_5,S_6,S_7,S_0,S_1,_i) -#define RND_1_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,_i) RND_1_3(S_1,S_2,S_3,S_4,S_5,S_6,S_7,S_0,_i) - -#define RND_0_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,_i) RND_4_6(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,_i) -#define RND_7_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,_i) RND_4_6(S_7,S_0,S_1,S_2,S_3,S_4,S_5,S_6,_i) -#define RND_6_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,_i) RND_4_6(S_6,S_7,S_0,S_1,S_2,S_3,S_4,S_5,_i) -#define RND_5_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,_i) RND_4_6(S_5,S_6,S_7,S_0,S_1,S_2,S_3,S_4,_i) -#define RND_4_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,_i) RND_4_6(S_4,S_5,S_6,S_7,S_0,S_1,S_2,S_3,_i) -#define RND_3_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,_i) RND_4_6(S_3,S_4,S_5,S_6,S_7,S_0,S_1,S_2,_i) -#define RND_2_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,_i) RND_4_6(S_2,S_3,S_4,S_5,S_6,S_7,S_0,S_1,_i) -#define RND_1_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,_i) RND_4_6(S_1,S_2,S_3,S_4,S_5,S_6,S_7,S_0,_i) - -#define RND_0_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,_i) RND_7_8(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,_i) -#define RND_7_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,_i) RND_7_8(S_7,S_0,S_1,S_2,S_3,S_4,S_5,S_6,_i) -#define RND_6_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,_i) RND_7_8(S_6,S_7,S_0,S_1,S_2,S_3,S_4,S_5,_i) -#define RND_5_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,_i) RND_7_8(S_5,S_6,S_7,S_0,S_1,S_2,S_3,S_4,_i) -#define RND_4_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,_i) RND_7_8(S_4,S_5,S_6,S_7,S_0,S_1,S_2,S_3,_i) -#define RND_3_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,_i) RND_7_8(S_3,S_4,S_5,S_6,S_7,S_0,S_1,S_2,_i) -#define RND_2_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,_i) RND_7_8(S_2,S_3,S_4,S_5,S_6,S_7,S_0,S_1,_i) -#define RND_1_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,_i) RND_7_8(S_1,S_2,S_3,S_4,S_5,S_6,S_7,S_0,_i) - -#define FOR(cnt, init, max, inc, loop) \ - __asm__ volatile("movl $"#init", %0\n\t"#loop":"::"m"(cnt):) -#define END(cnt, init, max, inc, loop) \ - __asm__ volatile("addl $"#inc", %0\n\tcmpl $"#max", %0\n\tjle "#loop"\n\t":"=m"(cnt)::); +#define RND_ALL_4(a, b, c, d, e, f, g, h, i) \ + RND_ALL_0(a, b, c, d, e, f, g, h, i+0) \ + RND_ALL_1(h, a, b, c, d, e, f, g, i+1) \ + RND_ALL_0(g, h, a, b, c, d, e, f, i+2) \ + RND_ALL_1(f, g, h, a, b, c, d, e, i+3) #endif /* defined(HAVE_INTEL_AVX1) || defined(HAVE_INTEL_AVX2) */ #if defined(HAVE_INTEL_AVX1) /* inline Assember for Intel AVX1 instructions */ -#define VPALIGNR(op1,op2,op3,op4) \ +#define _VPALIGNR(op1, op2, op3, op4) \ "vpalignr $"#op4", %"#op3", %"#op2", %"#op1"\n\t" -#define VPADDD(op1,op2,op3) \ +#define VPALIGNR(op1, op2, op3, op4) \ + _VPALIGNR(op1, op2, op3, op4) +#define _VPADDD(op1, op2, op3) \ "vpaddd %"#op3", %"#op2", %"#op1"\n\t" -#define VPSRLD(op1,op2,op3) \ +#define VPADDD(op1, op2, op3) \ + _VPADDD(op1, op2, op3) +#define _VPSRLD(op1, op2, op3) \ "vpsrld $"#op3", %"#op2", %"#op1"\n\t" -#define VPSRLQ(op1,op2,op3) \ +#define VPSRLD(op1, op2, op3) \ + _VPSRLD(op1, op2, op3) +#define _VPSRLQ(op1, op2, op3) \ "vpsrlq $"#op3", %"#op2", %"#op1"\n\t" -#define VPSLLD(op1,op2,op3) \ +#define VPSRLQ(op1,op2,op3) \ + _VPSRLQ(op1,op2,op3) +#define _VPSLLD(op1,op2,op3) \ "vpslld $"#op3", %"#op2", %"#op1"\n\t" -#define VPOR(op1,op2,op3) \ +#define VPSLLD(op1,op2,op3) \ + _VPSLLD(op1,op2,op3) +#define _VPOR(op1,op2,op3) \ "vpor %"#op3", %"#op2", %"#op1"\n\t" -#define VPXOR(op1,op2,op3) \ +#define VPOR(op1,op2,op3) \ + _VPOR(op1,op2,op3) +#define _VPXOR(op1,op2,op3) \ "vpxor %"#op3", %"#op2", %"#op1"\n\t" -#define VPSHUFD(op1,op2,op3) \ +#define VPXOR(op1,op2,op3) \ + _VPXOR(op1,op2,op3) +#define _VPSHUFD(op1,op2,op3) \ "vpshufd $"#op3", %"#op2", %"#op1"\n\t" -#define VPSHUFB(op1,op2,op3) \ +#define VPSHUFD(op1,op2,op3) \ + _VPSHUFD(op1,op2,op3) +#define _VPSHUFB(op1,op2,op3) \ "vpshufb %"#op3", %"#op2", %"#op1"\n\t" +#define VPSHUFB(op1,op2,op3) \ + _VPSHUFB(op1,op2,op3) +#define _VPSLLDQ(op1,op2,op3) \ + "vpslldq $"#op3", %"#op2", %"#op1"\n\t" +#define VPSLLDQ(op1,op2,op3) \ + _VPSLLDQ(op1,op2,op3) -#define MessageSched(X0, X1, X2, X3, XTMP0, XTMP1, XTMP2, XTMP3, XTMP4, XTMP5, XFER, SHUF_00BA, SHUF_DC00,\ - a,b,c,d,e,f,g,h,_i)\ - RND_STEP_1(a,b,c,d,e,f,g,h,_i)\ - VPALIGNR (XTMP0, X3, X2, 4)\ - RND_STEP_2(a,b,c,d,e,f,g,h,_i)\ - VPADDD (XTMP0, XTMP0, X0)\ - RND_STEP_3(a,b,c,d,e,f,g,h,_i)\ - VPALIGNR (XTMP1, X1, X0, 4) /* XTMP1 = W[-15] */\ - RND_STEP_4(a,b,c,d,e,f,g,h,_i)\ - VPSRLD (XTMP2, XTMP1, 7)\ - RND_STEP_5(a,b,c,d,e,f,g,h,_i)\ - VPSLLD (XTMP3, XTMP1, 25) /* VPSLLD (XTMP3, XTMP1, (32-7)) */\ - RND_STEP_6(a,b,c,d,e,f,g,h,_i)\ - VPOR (XTMP3, XTMP3, XTMP2) /* XTMP1 = W[-15] MY_ROR 7 */\ - RND_STEP_7(a,b,c,d,e,f,g,h,_i)\ - VPSRLD (XTMP2, XTMP1,18)\ - RND_STEP_8(a,b,c,d,e,f,g,h,_i)\ -\ - RND_STEP_1(h,a,b,c,d,e,f,g,_i+1)\ - VPSRLD (XTMP4, XTMP1, 3) /* XTMP4 = W[-15] >> 3 */\ - RND_STEP_2(h,a,b,c,d,e,f,g,_i+1)\ - VPSLLD (XTMP1, XTMP1, 14) /* VPSLLD (XTMP1, XTMP1, (32-18)) */\ - RND_STEP_3(h,a,b,c,d,e,f,g,_i+1)\ - VPXOR (XTMP3, XTMP3, XTMP1)\ - RND_STEP_4(h,a,b,c,d,e,f,g,_i+1)\ - VPXOR (XTMP3, XTMP3, XTMP2) /* XTMP1 = W[-15] MY_ROR 7 ^ W[-15] MY_ROR 18 */\ - RND_STEP_5(h,a,b,c,d,e,f,g,_i+1)\ - VPXOR (XTMP1, XTMP3, XTMP4) /* XTMP1 = s0 */\ - RND_STEP_6(h,a,b,c,d,e,f,g,_i+1)\ - VPSHUFD(XTMP2, X3, 0b11111010) /* XTMP2 = W[-2] {BBAA}*/\ - RND_STEP_7(h,a,b,c,d,e,f,g,_i+1)\ - VPADDD (XTMP0, XTMP0, XTMP1) /* XTMP0 = W[-16] + W[-7] + s0 */\ - RND_STEP_8(h,a,b,c,d,e,f,g,_i+1)\ -\ - RND_STEP_1(g,h,a,b,c,d,e,f,_i+2)\ - VPSRLD (XTMP4, XTMP2, 10) /* XTMP4 = W[-2] >> 10 {BBAA} */\ - RND_STEP_2(g,h,a,b,c,d,e,f,_i+2)\ - VPSRLQ (XTMP3, XTMP2, 19) /* XTMP3 = W[-2] MY_ROR 19 {xBxA} */\ - RND_STEP_3(g,h,a,b,c,d,e,f,_i+2)\ - VPSRLQ (XTMP2, XTMP2, 17) /* XTMP2 = W[-2] MY_ROR 17 {xBxA} */\ - RND_STEP_4(g,h,a,b,c,d,e,f,_i+2)\ - VPXOR (XTMP2, XTMP2, XTMP3)\ - RND_STEP_5(g,h,a,b,c,d,e,f,_i+2)\ - VPXOR (XTMP4, XTMP4, XTMP2) /* XTMP4 = s1 {xBxA} */\ - RND_STEP_6(g,h,a,b,c,d,e,f,_i+2)\ - VPSHUFB (XTMP4, XTMP4, SHUF_00BA) /* XTMP4 = s1 {00BA} */\ - RND_STEP_7(g,h,a,b,c,d,e,f,_i+2)\ - VPADDD (XTMP0, XTMP0, XTMP4) /* XTMP0 = {..., ..., W[1], W[0]} */\ - RND_STEP_8(g,h,a,b,c,d,e,f,_i+2)\ -\ - RND_STEP_1(f,g,h,a,b,c,d,e,_i+3)\ - VPSHUFD (XTMP2, XTMP0, 0b01010000) /* XTMP2 = W[-2] {DDCC} */\ - RND_STEP_2(f,g,h,a,b,c,d,e,_i+3)\ - VPSRLD (XTMP5, XTMP2, 10) /* XTMP5 = W[-2] >> 10 {DDCC} */\ - RND_STEP_3(f,g,h,a,b,c,d,e,_i+3)\ - VPSRLQ (XTMP3, XTMP2, 19) /* XTMP3 = W[-2] MY_ROR 19 {xDxC} */\ - RND_STEP_4(f,g,h,a,b,c,d,e,_i+3)\ - VPSRLQ (XTMP2, XTMP2, 17) /* XTMP2 = W[-2] MY_ROR 17 {xDxC} */\ - RND_STEP_5(f,g,h,a,b,c,d,e,_i+3)\ - VPXOR (XTMP2, XTMP2, XTMP3)\ - RND_STEP_6(f,g,h,a,b,c,d,e,_i+3)\ - VPXOR (XTMP5, XTMP5, XTMP2) /* XTMP5 = s1 {xDxC} */\ - RND_STEP_7(f,g,h,a,b,c,d,e,_i+3)\ - VPSHUFB (XTMP5, XTMP5, SHUF_DC00) /* XTMP5 = s1 {DC00} */\ - RND_STEP_8(f,g,h,a,b,c,d,e,_i+3)\ - VPADDD (X0, XTMP5, XTMP0) /* X0 = {W[3], W[2], W[1], W[0]} */\ +#define MsgSched(X0,X1,X2,X3,a,b,c,d,e,f,g,h,_i) \ + RND_STEP_0_1(a,b,c,d,e,f,g,h,_i) \ + VPALIGNR (XTMP1, X1, X0, 4) /* XTMP1 = W[-15] */ \ + VPALIGNR (XTMP0, X3, X2, 4) /* XTMP0 = W[-7] */ \ + RND_STEP_0_2(a,b,c,d,e,f,g,h,_i) \ + RND_STEP_0_3(a,b,c,d,e,f,g,h,_i) \ + VPSRLD (XTMP2, XTMP1, 7) /* XTMP2 = W[-15] >> 7 */ \ + VPSLLD (XTMP3, XTMP1, 25) /* XTEMP3 = W[-15] << (32-7) */ \ + RND_STEP_0_4(a,b,c,d,e,f,g,h,_i) \ + RND_STEP_0_5(a,b,c,d,e,f,g,h,_i) \ + VPSRLD (XTMP4, XTMP1, 18) /* XTEMP4 = W[-15] >> 18 */ \ + VPSLLD (XTMP5, XTMP1, 14) /* XTEMP5 = W[-15] << (32-18) */ \ + RND_STEP_0_6(a,b,c,d,e,f,g,h,_i) \ + RND_STEP_0_7(a,b,c,d,e,f,g,h,_i) \ + VPOR (XTMP2, XTMP3, XTMP2) /* XTMP2 = W[-15] >>> 7 */ \ + VPOR (XTMP4, XTMP5, XTMP4) /* XTMP4 = W[-15] >>> 18 */ \ + RND_STEP_0_8(a,b,c,d,e,f,g,h,_i) \ + RND_STEP_1_1(h,a,b,c,d,e,f,g,_i+1) \ + RND_STEP_1_2(h,a,b,c,d,e,f,g,_i+1) \ + VPSRLD (XTMP5, XTMP1, 3) /* XTMP4 = W[-15] >> 3 */ \ + VPXOR (XTMP2, XTMP4, XTMP2) \ + /* XTMP2 = W[-15] MY_ROR 7 ^ W[-15] MY_ROR 18 */ \ + RND_STEP_1_3(h,a,b,c,d,e,f,g,_i+1) \ + RND_STEP_1_4(h,a,b,c,d,e,f,g,_i+1) \ + VPXOR (XTMP1, XTMP5, XTMP2) /* XTMP1 = s0 */ \ + VPSHUFD (XTMP2, X3, 0b11111010) /* XTMP2 = W[-2] {BBAA}*/ \ + RND_STEP_1_5(h,a,b,c,d,e,f,g,_i+1) \ + RND_STEP_1_6(h,a,b,c,d,e,f,g,_i+1) \ + VPSRLD (XTMP4, XTMP2, 10) /* XTMP4 = W[-2] >> 10 {BBAA} */ \ + VPSRLQ (XTMP3, XTMP2, 19) /* XTMP3 = W[-2] MY_ROR 19 {xBxA} */ \ + RND_STEP_1_7(h,a,b,c,d,e,f,g,_i+1) \ + RND_STEP_1_8(h,a,b,c,d,e,f,g,_i+1) \ + RND_STEP_0_1(g,h,a,b,c,d,e,f,_i+2) \ + VPSRLQ (XTMP2, XTMP2, 17) /* XTMP2 = W[-2] MY_ROR 17 {xBxA} */ \ + VPADDD (XTMP0, XTMP0, X0) \ + RND_STEP_0_2(g,h,a,b,c,d,e,f,_i+2) \ + RND_STEP_0_3(g,h,a,b,c,d,e,f,_i+2) \ + RND_STEP_0_4(g,h,a,b,c,d,e,f,_i+2) \ + VPXOR (XTMP2, XTMP3, XTMP2) \ + VPADDD (XTMP0, XTMP0, XTMP1) /* XTMP0 = W[-16] + W[-7] + s0 */ \ + RND_STEP_0_5(g,h,a,b,c,d,e,f,_i+2) \ + VPXOR (XTMP4, XTMP4, XTMP2) /* XTMP4 = s1 {xBxA} */ \ + RND_STEP_0_6(g,h,a,b,c,d,e,f,_i+2) \ + VPSHUFB (XTMP4, XTMP4, SHUF_00BA) /* XTMP4 = s1 {00BA} */ \ + RND_STEP_0_7(g,h,a,b,c,d,e,f,_i+2) \ + VPADDD (XTMP0, XTMP0, XTMP4) /* XTMP0 = {..., ..., W[1], W[0]} */ \ + RND_STEP_0_8(g,h,a,b,c,d,e,f,_i+2) \ + RND_STEP_1_1(f,g,h,a,b,c,d,e,_i+3) \ + VPSHUFD (XTMP2, XTMP0, 0b01010000) /* XTMP2 = W[-2] {DDCC} */ \ + RND_STEP_1_2(f,g,h,a,b,c,d,e,_i+3) \ + VPSRLQ (XTMP4, XTMP2, 17) /* XTMP4 = W[-2] MY_ROR 17 {xDxC} */ \ + VPSRLQ (XTMP3, XTMP2, 19) /* XTMP3 = W[-2] MY_ROR 19 {xDxC} */ \ + RND_STEP_1_3(f,g,h,a,b,c,d,e,_i+3) \ + RND_STEP_1_4(f,g,h,a,b,c,d,e,_i+3) \ + VPSRLD (XTMP5, XTMP2, 10) /* XTMP5 = W[-2] >> 10 {DDCC} */ \ + VPXOR (XTMP4, XTMP3, XTMP4) \ + RND_STEP_1_5(f,g,h,a,b,c,d,e,_i+3) \ + RND_STEP_1_6(f,g,h,a,b,c,d,e,_i+3) \ + VPXOR (XTMP5, XTMP4, XTMP5) /* XTMP5 = s1 {xDxC} */ \ + RND_STEP_1_7(f,g,h,a,b,c,d,e,_i+3) \ + VPSHUFB (XTMP5, XTMP5, SHUF_DC00) /* XTMP5 = s1 {DC00} */ \ + RND_STEP_1_8(f,g,h,a,b,c,d,e,_i+3) \ + VPADDD (X0, XTMP5, XTMP0) /* X0 = {W[3], W[2], W[1], W[0]} */ #if defined(HAVE_INTEL_RORX) -#define MessageSched_RORX(X0, X1, X2, X3, XTMP0, XTMP1, XTMP2, XTMP3, XTMP4, XTMP5, \ - XFER, SHUF_00BA, SHUF_DC00,a,b,c,d,e,f,g,h,_i)\ - RND_STEP_RORX_1(a,b,c,d,e,f,g,h,_i)\ - VPALIGNR (XTMP0, X3, X2, 4)\ - RND_STEP_RORX_2(a,b,c,d,e,f,g,h,_i)\ - VPADDD (XTMP0, XTMP0, X0)\ - RND_STEP_RORX_3(a,b,c,d,e,f,g,h,_i)\ - VPALIGNR (XTMP1, X1, X0, 4) /* XTMP1 = W[-15] */\ - RND_STEP_RORX_4(a,b,c,d,e,f,g,h,_i)\ - VPSRLD (XTMP2, XTMP1, 7)\ - RND_STEP_RORX_5(a,b,c,d,e,f,g,h,_i)\ - VPSLLD (XTMP3, XTMP1, 25) /* VPSLLD (XTMP3, XTMP1, (32-7)) */\ - RND_STEP_RORX_6(a,b,c,d,e,f,g,h,_i)\ - VPOR (XTMP3, XTMP3, XTMP2) /* XTMP1 = W[-15] MY_ROR 7 */\ - RND_STEP_RORX_7(a,b,c,d,e,f,g,h,_i)\ - VPSRLD (XTMP2, XTMP1,18)\ - RND_STEP_RORX_8(a,b,c,d,e,f,g,h,_i)\ -\ - RND_STEP_RORX_1(h,a,b,c,d,e,f,g,_i+1)\ - VPSRLD (XTMP4, XTMP1, 3) /* XTMP4 = W[-15] >> 3 */\ - RND_STEP_RORX_2(h,a,b,c,d,e,f,g,_i+1)\ - VPSLLD (XTMP1, XTMP1, 14) /* VPSLLD (XTMP1, XTMP1, (32-18)) */\ - RND_STEP_RORX_3(h,a,b,c,d,e,f,g,_i+1)\ - VPXOR (XTMP3, XTMP3, XTMP1)\ - RND_STEP_RORX_4(h,a,b,c,d,e,f,g,_i+1)\ - VPXOR (XTMP3, XTMP3, XTMP2) /* XTMP1 = W[-15] MY_ROR 7 ^ W[-15] MY_ROR 18 */\ - RND_STEP_RORX_5(h,a,b,c,d,e,f,g,_i+1)\ - VPXOR (XTMP1, XTMP3, XTMP4) /* XTMP1 = s0 */\ - RND_STEP_RORX_6(h,a,b,c,d,e,f,g,_i+1)\ - VPSHUFD(XTMP2, X3, 0b11111010) /* XTMP2 = W[-2] {BBAA}*/\ - RND_STEP_RORX_7(h,a,b,c,d,e,f,g,_i+1)\ - VPADDD (XTMP0, XTMP0, XTMP1) /* XTMP0 = W[-16] + W[-7] + s0 */\ - RND_STEP_RORX_8(h,a,b,c,d,e,f,g,_i+1)\ -\ - RND_STEP_RORX_1(g,h,a,b,c,d,e,f,_i+2)\ - VPSRLD (XTMP4, XTMP2, 10) /* XTMP4 = W[-2] >> 10 {BBAA} */\ - RND_STEP_RORX_2(g,h,a,b,c,d,e,f,_i+2)\ - VPSRLQ (XTMP3, XTMP2, 19) /* XTMP3 = W[-2] MY_ROR 19 {xBxA} */\ - RND_STEP_RORX_3(g,h,a,b,c,d,e,f,_i+2)\ - VPSRLQ (XTMP2, XTMP2, 17) /* XTMP2 = W[-2] MY_ROR 17 {xBxA} */\ - RND_STEP_RORX_4(g,h,a,b,c,d,e,f,_i+2)\ - VPXOR (XTMP2, XTMP2, XTMP3)\ - RND_STEP_RORX_5(g,h,a,b,c,d,e,f,_i+2)\ - VPXOR (XTMP4, XTMP4, XTMP2) /* XTMP4 = s1 {xBxA} */\ - RND_STEP_RORX_6(g,h,a,b,c,d,e,f,_i+2)\ - VPSHUFB (XTMP4, XTMP4, SHUF_00BA) /* XTMP4 = s1 {00BA} */\ - RND_STEP_RORX_7(g,h,a,b,c,d,e,f,_i+2)\ - VPADDD (XTMP0, XTMP0, XTMP4) /* XTMP0 = {..., ..., W[1], W[0]} */\ - RND_STEP_RORX_8(g,h,a,b,c,d,e,f,_i+2)\ -\ - RND_STEP_RORX_1(f,g,h,a,b,c,d,e,_i+3)\ - VPSHUFD (XTMP2, XTMP0, 0b01010000) /* XTMP2 = W[-2] {DDCC} */\ - RND_STEP_RORX_2(f,g,h,a,b,c,d,e,_i+3)\ - VPSRLD (XTMP5, XTMP2, 10) /* XTMP5 = W[-2] >> 10 {DDCC} */\ - RND_STEP_RORX_3(f,g,h,a,b,c,d,e,_i+3)\ - VPSRLQ (XTMP3, XTMP2, 19) /* XTMP3 = W[-2] MY_ROR 19 {xDxC} */\ - RND_STEP_RORX_4(f,g,h,a,b,c,d,e,_i+3)\ - VPSRLQ (XTMP2, XTMP2, 17) /* XTMP2 = W[-2] MY_ROR 17 {xDxC} */\ - RND_STEP_RORX_5(f,g,h,a,b,c,d,e,_i+3)\ - VPXOR (XTMP2, XTMP2, XTMP3)\ - RND_STEP_RORX_6(f,g,h,a,b,c,d,e,_i+3)\ - VPXOR (XTMP5, XTMP5, XTMP2) /* XTMP5 = s1 {xDxC} */\ - RND_STEP_RORX_7(f,g,h,a,b,c,d,e,_i+3)\ - VPSHUFB (XTMP5, XTMP5, SHUF_DC00) /* XTMP5 = s1 {DC00} */\ - RND_STEP_RORX_8(f,g,h,a,b,c,d,e,_i+3)\ - VPADDD (X0, XTMP5, XTMP0) /* X0 = {W[3], W[2], W[1], W[0]} */\ +#define MsgSched_RORX(X0,X1,X2,X3,a,b,c,d,e,f,g,h,_i) \ + RND_STEP_RORX_0_1(a,b,c,d,e,f,g,h,_i) \ + VPALIGNR (XTMP0, X3, X2, 4) \ + VPALIGNR (XTMP1, X1, X0, 4) /* XTMP1 = W[-15] */ \ + RND_STEP_RORX_0_2(a,b,c,d,e,f,g,h,_i) \ + RND_STEP_RORX_0_3(a,b,c,d,e,f,g,h,_i) \ + VPSRLD (XTMP2, XTMP1, 7) \ + VPSLLD (XTMP3, XTMP1, 25) /* VPSLLD (XTMP3, XTMP1, (32-7)) */ \ + RND_STEP_RORX_0_4(a,b,c,d,e,f,g,h,_i) \ + RND_STEP_RORX_0_5(a,b,c,d,e,f,g,h,_i) \ + VPSRLD (XTMP4, XTMP1, 3) /* XTMP4 = W[-15] >> 3 */ \ + VPOR (XTMP3, XTMP3, XTMP2) /* XTMP1 = W[-15] MY_ROR 7 */ \ + RND_STEP_RORX_0_6(a,b,c,d,e,f,g,h,_i) \ + RND_STEP_RORX_0_7(a,b,c,d,e,f,g,h,_i) \ + RND_STEP_RORX_0_8(a,b,c,d,e,f,g,h,_i) \ + \ + RND_STEP_RORX_1_1(h,a,b,c,d,e,f,g,_i+1) \ + VPSRLD (XTMP2, XTMP1,18) \ + RND_STEP_RORX_1_2(h,a,b,c,d,e,f,g,_i+1) \ + VPSLLD (XTMP1, XTMP1, 14) /* VPSLLD (XTMP1, XTMP1, (32-18)) */ \ + RND_STEP_RORX_1_3(h,a,b,c,d,e,f,g,_i+1) \ + VPXOR (XTMP3, XTMP3, XTMP1) \ + RND_STEP_RORX_1_4(h,a,b,c,d,e,f,g,_i+1) \ + VPXOR (XTMP3, XTMP3, XTMP2) \ + /* XTMP1 = W[-15] MY_ROR 7 ^ W[-15] MY_ROR 18 */ \ + RND_STEP_RORX_1_5(h,a,b,c,d,e,f,g,_i+1) \ + VPSHUFD (XTMP2, X3, 0b11111010) /* XTMP2 = W[-2] {BBAA}*/ \ + RND_STEP_RORX_1_6(h,a,b,c,d,e,f,g,_i+1) \ + VPXOR (XTMP1, XTMP3, XTMP4) /* XTMP1 = s0 */ \ + RND_STEP_RORX_1_7(h,a,b,c,d,e,f,g,_i+1) \ + VPSRLD (XTMP4, XTMP2, 10) /* XTMP4 = W[-2] >> 10 {BBAA} */ \ + RND_STEP_RORX_1_8(h,a,b,c,d,e,f,g,_i+1) \ + \ + RND_STEP_RORX_0_1(g,h,a,b,c,d,e,f,_i+2) \ + VPSRLQ (XTMP3, XTMP2, 19) /* XTMP3 = W[-2] MY_ROR 19 {xBxA} */ \ + RND_STEP_RORX_0_2(g,h,a,b,c,d,e,f,_i+2) \ + VPSRLQ (XTMP2, XTMP2, 17) /* XTMP2 = W[-2] MY_ROR 17 {xBxA} */ \ + VPADDD (XTMP0, XTMP0, X0) \ + RND_STEP_RORX_0_3(g,h,a,b,c,d,e,f,_i+2) \ + VPADDD (XTMP0, XTMP0, XTMP1) /* XTMP0 = W[-16] + W[-7] + s0 */ \ + RND_STEP_RORX_0_4(g,h,a,b,c,d,e,f,_i+2) \ + VPXOR (XTMP2, XTMP2, XTMP3) \ + RND_STEP_RORX_0_5(g,h,a,b,c,d,e,f,_i+2) \ + VPXOR (XTMP4, XTMP4, XTMP2) /* XTMP4 = s1 {xBxA} */ \ + RND_STEP_RORX_0_6(g,h,a,b,c,d,e,f,_i+2) \ + VPSHUFB (XTMP4, XTMP4, SHUF_00BA) /* XTMP4 = s1 {00BA} */ \ + RND_STEP_RORX_0_7(g,h,a,b,c,d,e,f,_i+2) \ + VPADDD (XTMP0, XTMP0, XTMP4) /* XTMP0 = {..., ..., W[1], W[0]} */ \ + RND_STEP_RORX_0_8(g,h,a,b,c,d,e,f,_i+2) \ + \ + RND_STEP_RORX_1_1(f,g,h,a,b,c,d,e,_i+3) \ + VPSHUFD (XTMP2, XTMP0, 0b01010000) /* XTMP2 = W[-2] {DDCC} */ \ + RND_STEP_RORX_1_2(f,g,h,a,b,c,d,e,_i+3) \ + VPSRLD (XTMP5, XTMP2, 10) /* XTMP5 = W[-2] >> 10 {DDCC} */ \ + RND_STEP_RORX_1_3(f,g,h,a,b,c,d,e,_i+3) \ + VPSRLQ (XTMP3, XTMP2, 19) /* XTMP3 = W[-2] MY_ROR 19 {xDxC} */ \ + RND_STEP_RORX_1_4(f,g,h,a,b,c,d,e,_i+3) \ + VPSRLQ (XTMP2, XTMP2, 17) /* XTMP2 = W[-2] MY_ROR 17 {xDxC} */ \ + RND_STEP_RORX_1_5(f,g,h,a,b,c,d,e,_i+3) \ + VPXOR (XTMP2, XTMP2, XTMP3) \ + RND_STEP_RORX_1_6(f,g,h,a,b,c,d,e,_i+3) \ + VPXOR (XTMP5, XTMP5, XTMP2) /* XTMP5 = s1 {xDxC} */ \ + RND_STEP_RORX_1_7(f,g,h,a,b,c,d,e,_i+3) \ + VPSHUFB (XTMP5, XTMP5, SHUF_DC00) /* XTMP5 = s1 {DC00} */ \ + RND_STEP_RORX_1_8(f,g,h,a,b,c,d,e,_i+3) \ + VPADDD (X0, XTMP5, XTMP0) /* X0 = {W[3], W[2], W[1], W[0]} */ #endif /* HAVE_INTEL_RORX */ -#define W_K_from_buff() \ - "leaq %[buf], %%r8\n\t" \ - "vmovdqu (%%r8), %%xmm4\n\t" \ - "vpshufb %%xmm13, %%xmm4, %%xmm4\n\t" \ - "vmovdqu 16(%%r8), %%xmm5\n\t" \ - "vpshufb %%xmm13, %%xmm5, %%xmm5\n\t" \ - "vmovdqu 32(%%r8), %%xmm6\n\t" \ - "vpshufb %%xmm13, %%xmm6, %%xmm6\n\t" \ - "vmovdqu 48(%%r8), %%xmm7\n\t" \ - "vpshufb %%xmm13, %%xmm7, %%xmm7\n\t" +#define _W_K_from_buff(X0, X1, X2, X3, BYTE_FLIP_MASK) \ + "# X0, X1, X2, X3 = W[0..15]\n\t" \ + "vmovdqu (%%rax), %"#X0"\n\t" \ + "vmovdqu 16(%%rax), %"#X1"\n\t" \ + VPSHUFB(X0, X0, BYTE_FLIP_MASK) \ + VPSHUFB(X1, X1, BYTE_FLIP_MASK) \ + "vmovdqu 32(%%rax), %"#X2"\n\t" \ + "vmovdqu 48(%%rax), %"#X3"\n\t" \ + VPSHUFB(X2, X2, BYTE_FLIP_MASK) \ + VPSHUFB(X3, X3, BYTE_FLIP_MASK) -#define _SET_W_K_XFER(reg, i) \ - "leaq %[K], %%r8\n\t" \ - "vpaddd ("#i")*4(%%r8), %"#reg", %%xmm9\n\t" \ - "leaq %[W_K], %%r8\n\t" \ - "vmovdqa %%xmm9, ("#i")*4(%%r8)\n\t" +#define W_K_from_buff(X0, X1, X2, X3, BYTE_FLIP_MASK) \ + _W_K_from_buff(X0, X1, X2, X3, BYTE_FLIP_MASK) -#define SET_W_K_XFER(reg, i) _SET_W_K_XFER(reg, i) -static const ALIGN32 word64 mSHUF_00BA[] = { 0x0b0a090803020100, 0xFFFFFFFFFFFFFFFF }; /* shuffle xBxA -> 00BA */ -static const ALIGN32 word64 mSHUF_DC00[] = { 0xFFFFFFFFFFFFFFFF, 0x0b0a090803020100 }; /* shuffle xDxC -> DC00 */ -static const ALIGN32 word64 mBYTE_FLIP_MASK[] = { 0x0405060700010203, 0x0c0d0e0f08090a0b }; +#define _SET_W_K_XFER_4(i) \ + "vpaddd ("#i"*4)+ 0+%[K], %%xmm0, %%xmm4\n\t" \ + "vpaddd ("#i"*4)+16+%[K], %%xmm1, %%xmm5\n\t" \ + "vmovdqu %%xmm4, ("WK")\n\t" \ + "vmovdqu %%xmm5, 16("WK")\n\t" \ + "vpaddd ("#i"*4)+32+%[K], %%xmm2, %%xmm6\n\t" \ + "vpaddd ("#i"*4)+48+%[K], %%xmm3, %%xmm7\n\t" \ + "vmovdqu %%xmm6, 32("WK")\n\t" \ + "vmovdqu %%xmm7, 48("WK")\n\t" +#define SET_W_K_XFER_4(i) \ + _SET_W_K_XFER_4(i) + + +static const ALIGN32 word64 mSHUF_00BA[] = + { 0x0b0a090803020100, 0xFFFFFFFFFFFFFFFF }; /* shuffle xBxA -> 00BA */ +static const ALIGN32 word64 mSHUF_DC00[] = + { 0xFFFFFFFFFFFFFFFF, 0x0b0a090803020100 }; /* shuffle xDxC -> DC00 */ +static const ALIGN32 word64 mBYTE_FLIP_MASK[] = + { 0x0405060700010203, 0x0c0d0e0f08090a0b }; #define _Init_Masks(mask1, mask2, mask3) \ - "vmovdqu %[FLIP], %"#mask1"\n\t" \ - "vmovdqu %[SHUF00BA], %"#mask2"\n\t" \ - "vmovdqu %[SHUFDC00], %"#mask3"\n\t" + "vmovdqa %[FLIP], %"#mask1"\n\t" \ + "vmovdqa %[SHUF00BA], %"#mask2"\n\t" \ + "vmovdqa %[SHUFDC00], %"#mask3"\n\t" -#define Init_Masks(BYTE_FLIP_MASK, SHUF_00BA, SHUF_DC00)\ - _Init_Masks(BYTE_FLIP_MASK, SHUF_00BA, SHUF_DC00) +#define Init_Masks(BYTE_FLIP_MASK, SHUF_00BA, SHUF_DC00) \ + _Init_Masks(BYTE_FLIP_MASK, SHUF_00BA, SHUF_DC00) -#define X0 %xmm4 -#define X1 %xmm5 -#define X2 %xmm6 -#define X3 %xmm7 -#define X_ X0 +#define X0 %xmm0 +#define X1 %xmm1 +#define X2 %xmm2 +#define X3 %xmm3 -#define XTMP0 %xmm0 -#define XTMP1 %xmm1 -#define XTMP2 %xmm2 -#define XTMP3 %xmm3 +#define XTMP0 %xmm4 +#define XTMP1 %xmm5 +#define XTMP2 %xmm6 +#define XTMP3 %xmm7 #define XTMP4 %xmm8 #define XTMP5 %xmm9 #define XFER %xmm10 @@ -1271,755 +1703,858 @@ static const ALIGN32 word64 mBYTE_FLIP_MASK[] = { 0x0405060700010203, 0x0c0d0e0 #define BYTE_FLIP_MASK %xmm13 -static int Transform_AVX1(wc_Sha256* sha256) +SHA256_NOINLINE static int Transform_Sha256_AVX1(wc_Sha256* sha256) { - ALIGN32 word32 W_K[64]; /* temp for W+K */ - __asm__ __volatile__ ( + "subq $64, %%rsp\n\t" + + "leaq 32(%[sha256]), %%rax\n\t" Init_Masks(BYTE_FLIP_MASK, SHUF_00BA, SHUF_DC00) - "# X0, X1, X2, X3 = W[0..15]; \n\t" - W_K_from_buff() + LOAD_DIGEST() - DigestToReg(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7) + W_K_from_buff(X0, X1, X2, X3, BYTE_FLIP_MASK) - SET_W_K_XFER(X0, 0) - MessageSched(X0, X1, X2, X3, XTMP0, XTMP1, XTMP2, XTMP3, XTMP4, XTMP5, XFER, - SHUF_00BA, SHUF_DC00, S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,0) - SET_W_K_XFER(X1, 4) - MessageSched(X1, X2, X3, X0, XTMP0, XTMP1, XTMP2, XTMP3, XTMP4, XTMP5, XFER, - SHUF_00BA, SHUF_DC00, S_4,S_5,S_6,S_7,S_0,S_1,S_2,S_3,4) - SET_W_K_XFER(X2, 8) - MessageSched(X2, X3, X0, X1, XTMP0, XTMP1, XTMP2, XTMP3, XTMP4, XTMP5, XFER, - SHUF_00BA, SHUF_DC00, S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,8) - SET_W_K_XFER(X3, 12) - MessageSched(X3, X0, X1, X2, XTMP0, XTMP1, XTMP2, XTMP3, XTMP4, XTMP5, XFER, - SHUF_00BA, SHUF_DC00, S_4,S_5,S_6,S_7,S_0,S_1,S_2,S_3,12) - SET_W_K_XFER(X0, 16) - MessageSched(X0, X1, X2, X3, XTMP0, XTMP1, XTMP2, XTMP3, XTMP4, XTMP5, XFER, - SHUF_00BA, SHUF_DC00, S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,16) - SET_W_K_XFER(X1, 20) - MessageSched(X1, X2, X3, X0, XTMP0, XTMP1, XTMP2, XTMP3, XTMP4, XTMP5, XFER, - SHUF_00BA, SHUF_DC00, S_4,S_5,S_6,S_7,S_0,S_1,S_2,S_3,20) - SET_W_K_XFER(X2, 24) - MessageSched(X2, X3, X0, X1, XTMP0, XTMP1, XTMP2, XTMP3, XTMP4, XTMP5, XFER, - SHUF_00BA, SHUF_DC00, S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,24) - SET_W_K_XFER(X3, 28) - MessageSched(X3, X0, X1, X2, XTMP0, XTMP1, XTMP2, XTMP3, XTMP4, XTMP5, XFER, - SHUF_00BA, SHUF_DC00, S_4,S_5,S_6,S_7,S_0,S_1,S_2,S_3,28) - SET_W_K_XFER(X0, 32) - MessageSched(X0, X1, X2, X3, XTMP0, XTMP1, XTMP2, XTMP3, XTMP4, XTMP5, XFER, - SHUF_00BA, SHUF_DC00, S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,32) - SET_W_K_XFER(X1, 36) - MessageSched(X1, X2, X3, X0, XTMP0, XTMP1, XTMP2, XTMP3, XTMP4, XTMP5, XFER, - SHUF_00BA, SHUF_DC00, S_4,S_5,S_6,S_7,S_0,S_1,S_2,S_3,36) - SET_W_K_XFER(X2, 40) - MessageSched(X2, X3, X0, X1, XTMP0, XTMP1, XTMP2, XTMP3, XTMP4, XTMP5, XFER, - SHUF_00BA, SHUF_DC00, S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,40) - SET_W_K_XFER(X3, 44) - MessageSched(X3, X0, X1, X2, XTMP0, XTMP1, XTMP2, XTMP3, XTMP4, XTMP5, XFER, - SHUF_00BA, SHUF_DC00, S_4,S_5,S_6,S_7,S_0,S_1,S_2,S_3,44) + "movl %%r9d, "L4"\n\t" + "movl %%r12d, "L1"\n\t" + "xorl %%r10d, "L4"\n\t" - SET_W_K_XFER(X0, 48) - SET_W_K_XFER(X1, 52) - SET_W_K_XFER(X2, 56) - SET_W_K_XFER(X3, 60) + SET_W_K_XFER_4(0) + MsgSched(X0, X1, X2, X3, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 0) + MsgSched(X1, X2, X3, X0, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 4) + MsgSched(X2, X3, X0, X1, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 8) + MsgSched(X3, X0, X1, X2, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 12) - RND_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,48) - RND_7(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,49) - RND_6(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,50) - RND_5(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,51) + SET_W_K_XFER_4(16) + MsgSched(X0, X1, X2, X3, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 0) + MsgSched(X1, X2, X3, X0, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 4) + MsgSched(X2, X3, X0, X1, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 8) + MsgSched(X3, X0, X1, X2, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 12) - RND_4(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,52) - RND_3(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,53) - RND_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,54) - RND_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,55) + SET_W_K_XFER_4(32) + MsgSched(X0, X1, X2, X3, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 0) + MsgSched(X1, X2, X3, X0, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 4) + MsgSched(X2, X3, X0, X1, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 8) + MsgSched(X3, X0, X1, X2, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 12) - RND_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,56) - RND_7(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,57) - RND_6(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,58) - RND_5(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,59) + SET_W_K_XFER_4(48) + RND_ALL_4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 0) + RND_ALL_4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 4) + RND_ALL_4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 8) + RND_ALL_4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 12) - RND_4(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,60) - RND_3(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,61) - RND_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,62) - RND_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,63) + STORE_ADD_DIGEST() - RegToDigest(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7) + "addq $64, %%rsp\n\t" : : [FLIP] "m" (mBYTE_FLIP_MASK[0]), [SHUF00BA] "m" (mSHUF_00BA[0]), [SHUFDC00] "m" (mSHUF_DC00[0]), - [digest] "m" (sha256->digest), - [buf] "m" (sha256->buffer), - [K] "m" (K), - [W_K] "m" (W_K) - : SSE_REGs, "memory" + [sha256] "r" (sha256), + [K] "m" (K) + : WORK_REGS, STATE_REGS, XMM_REGS, "memory" + ); + + return 0; +} + +SHA256_NOINLINE static int Transform_Sha256_AVX1_Len(wc_Sha256* sha256, + word32 len) +{ + __asm__ __volatile__ ( + + "subq $64, %%rsp\n\t" + "movq 120(%[sha256]), %%rax\n\t" + + Init_Masks(BYTE_FLIP_MASK, SHUF_00BA, SHUF_DC00) + LOAD_DIGEST() + + "# Start of loop processing a block\n" + "1:\n\t" + + W_K_from_buff(X0, X1, X2, X3, BYTE_FLIP_MASK) + + "movl %%r9d, "L4"\n\t" + "movl %%r12d, "L1"\n\t" + "xorl %%r10d, "L4"\n\t" + + SET_W_K_XFER_4(0) + MsgSched(X0, X1, X2, X3, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 0) + MsgSched(X1, X2, X3, X0, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 4) + MsgSched(X2, X3, X0, X1, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 8) + MsgSched(X3, X0, X1, X2, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 12) + + SET_W_K_XFER_4(16) + MsgSched(X0, X1, X2, X3, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 0) + MsgSched(X1, X2, X3, X0, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 4) + MsgSched(X2, X3, X0, X1, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 8) + MsgSched(X3, X0, X1, X2, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 12) + + SET_W_K_XFER_4(32) + MsgSched(X0, X1, X2, X3, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 0) + MsgSched(X1, X2, X3, X0, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 4) + MsgSched(X2, X3, X0, X1, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 8) + MsgSched(X3, X0, X1, X2, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 12) + + SET_W_K_XFER_4(48) + RND_ALL_4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 0) + RND_ALL_4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 4) + RND_ALL_4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 8) + RND_ALL_4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 12) + "movq 120(%[sha256]), %%rax\n\t" + + ADD_DIGEST() + + "addq $64, %%rax\n\t" + "subl $64, %[len]\n\t" + + STORE_DIGEST() + + "movq %%rax, 120(%[sha256])\n\t" + "jnz 1b\n\t" + + "addq $64, %%rsp\n\t" + + : + : [FLIP] "m" (mBYTE_FLIP_MASK[0]), + [SHUF00BA] "m" (mSHUF_00BA[0]), + [SHUFDC00] "m" (mSHUF_DC00[0]), + [sha256] "r" (sha256), + [len] "r" (len), + [K] "m" (K) + : WORK_REGS, STATE_REGS, XMM_REGS, "memory" + ); + + return 0; +} +#endif /* HAVE_INTEL_AVX1 */ + +#if defined(HAVE_INTEL_AVX2) && defined(HAVE_INTEL_RORX) +SHA256_NOINLINE static int Transform_Sha256_AVX1_RORX(wc_Sha256* sha256) +{ + __asm__ __volatile__ ( + + "subq $64, %%rsp\n\t" + + Init_Masks(BYTE_FLIP_MASK, SHUF_00BA, SHUF_DC00) + "leaq 32(%[sha256]), %%rax\n\t" + W_K_from_buff(X0, X1, X2, X3, BYTE_FLIP_MASK) + + LOAD_DIGEST() + + SET_W_K_XFER_4(0) + "movl %%r9d, "L4"\n\t" + "rorx $6, %%r12d, "L1"\n\t" + "xorl %%r10d, "L4"\n\t" + MsgSched_RORX(X0, X1, X2, X3, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 0) + MsgSched_RORX(X1, X2, X3, X0, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 4) + MsgSched_RORX(X2, X3, X0, X1, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 8) + MsgSched_RORX(X3, X0, X1, X2, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 12) + + SET_W_K_XFER_4(16) + MsgSched_RORX(X0, X1, X2, X3, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 0) + MsgSched_RORX(X1, X2, X3, X0, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 4) + MsgSched_RORX(X2, X3, X0, X1, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 8) + MsgSched_RORX(X3, X0, X1, X2, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 12) + + SET_W_K_XFER_4(32) + MsgSched_RORX(X0, X1, X2, X3, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 0) + MsgSched_RORX(X1, X2, X3, X0, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 4) + MsgSched_RORX(X2, X3, X0, X1, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 8) + MsgSched_RORX(X3, X0, X1, X2, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 12) + + SET_W_K_XFER_4(48) + "xorl "L3", "L3"\n\t" + RND_RORX_X4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 0) + RND_RORX_X4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 4) + RND_RORX_X4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 8) + RND_RORX_X4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 12) + /* Prev RND: h += Maj(a,b,c) */ + "addl "L3", %%r8d\n\t" + + STORE_ADD_DIGEST() + + "addq $64, %%rsp\n\t" + + : + : [FLIP] "m" (mBYTE_FLIP_MASK[0]), + [SHUF00BA] "m" (mSHUF_00BA[0]), + [SHUFDC00] "m" (mSHUF_DC00[0]), + [sha256] "r" (sha256), + [K] "m" (K) + : WORK_REGS, STATE_REGS, XMM_REGS, "memory" + ); + + return 0; +} + +SHA256_NOINLINE static int Transform_Sha256_AVX1_RORX_Len(wc_Sha256* sha256, + word32 len) +{ + __asm__ __volatile__ ( + + "subq $64, %%rsp\n\t" + "movq 120(%[sha256]), %%rax\n\t" + + Init_Masks(BYTE_FLIP_MASK, SHUF_00BA, SHUF_DC00) + LOAD_DIGEST() + + "# Start of loop processing a block\n" + "1:\n\t" + + W_K_from_buff(X0, X1, X2, X3, BYTE_FLIP_MASK) + + SET_W_K_XFER_4(0) + "movl %%r9d, "L4"\n\t" + "rorx $6, %%r12d, "L1"\n\t" + "xorl %%r10d, "L4"\n\t" + MsgSched_RORX(X0, X1, X2, X3, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 0) + MsgSched_RORX(X1, X2, X3, X0, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 4) + MsgSched_RORX(X2, X3, X0, X1, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 8) + MsgSched_RORX(X3, X0, X1, X2, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 12) + + SET_W_K_XFER_4(16) + MsgSched_RORX(X0, X1, X2, X3, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 0) + MsgSched_RORX(X1, X2, X3, X0, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 4) + MsgSched_RORX(X2, X3, X0, X1, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 8) + MsgSched_RORX(X3, X0, X1, X2, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 12) + + SET_W_K_XFER_4(32) + MsgSched_RORX(X0, X1, X2, X3, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 0) + MsgSched_RORX(X1, X2, X3, X0, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 4) + MsgSched_RORX(X2, X3, X0, X1, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 8) + MsgSched_RORX(X3, X0, X1, X2, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 12) + + SET_W_K_XFER_4(48) + "xorl "L3", "L3"\n\t" + "xorl "L2", "L2"\n\t" + RND_RORX_X4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 0) + RND_RORX_X4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 4) + RND_RORX_X4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 8) + RND_RORX_X4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 12) + /* Prev RND: h += Maj(a,b,c) */ + "addl "L3", %%r8d\n\t" + "movq 120(%[sha256]), %%rax\n\t" + + ADD_DIGEST() + + "addq $64, %%rax\n\t" + "subl $64, %[len]\n\t" + + STORE_DIGEST() + + "movq %%rax, 120(%[sha256])\n\t" + "jnz 1b\n\t" + + "addq $64, %%rsp\n\t" + + : + : [FLIP] "m" (mBYTE_FLIP_MASK[0]), + [SHUF00BA] "m" (mSHUF_00BA[0]), + [SHUFDC00] "m" (mSHUF_DC00[0]), + [sha256] "r" (sha256), + [len] "r" (len), + [K] "m" (K) + : WORK_REGS, STATE_REGS, XMM_REGS, "memory" + ); + + return 0; +} +#endif /* HAVE_INTEL_AVX2 && HAVE_INTEL_RORX */ + + +#if defined(HAVE_INTEL_AVX2) +#define Y0 %ymm0 +#define Y1 %ymm1 +#define Y2 %ymm2 +#define Y3 %ymm3 + +#define YTMP0 %ymm4 +#define YTMP1 %ymm5 +#define YTMP2 %ymm6 +#define YTMP3 %ymm7 +#define YTMP4 %ymm8 +#define YTMP5 %ymm9 +#define YXFER %ymm10 + +#define SHUF_Y_00BA %ymm11 /* shuffle xBxA -> 00BA */ +#define SHUF_Y_DC00 %ymm12 /* shuffle xDxC -> DC00 */ +#define BYTE_FLIP_Y_MASK %ymm13 + +#define YMM_REGS "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", \ + "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13" + +#define MsgSched_Y(Y0,Y1,Y2,Y3,a,b,c,d,e,f,g,h,_i) \ + RND_STEP_0_1(a,b,c,d,e,f,g,h,_i) \ + VPALIGNR (YTMP1, Y1, Y0, 4) /* YTMP1 = W[-15] */ \ + VPALIGNR (YTMP0, Y3, Y2, 4) /* YTMP0 = W[-7] */ \ + RND_STEP_0_2(a,b,c,d,e,f,g,h,_i) \ + RND_STEP_0_3(a,b,c,d,e,f,g,h,_i) \ + VPSRLD (YTMP2, YTMP1, 7) /* YTMP2 = W[-15] >> 7 */ \ + VPSLLD (YTMP3, YTMP1, 25) /* YTEMP3 = W[-15] << (32-7) */ \ + RND_STEP_0_4(a,b,c,d,e,f,g,h,_i) \ + RND_STEP_0_5(a,b,c,d,e,f,g,h,_i) \ + VPSRLD (YTMP4, YTMP1, 18) /* YTEMP4 = W[-15] >> 18 */ \ + VPSLLD (YTMP5, YTMP1, 14) /* YTEMP5 = W[-15] << (32-18) */ \ + RND_STEP_0_6(a,b,c,d,e,f,g,h,_i) \ + RND_STEP_0_7(a,b,c,d,e,f,g,h,_i) \ + VPOR (YTMP2, YTMP3, YTMP2) /* YTMP2 = W[-15] >>> 7 */ \ + VPOR (YTMP4, YTMP5, YTMP4) /* YTMP4 = W[-15] >>> 18 */ \ + RND_STEP_0_8(a,b,c,d,e,f,g,h,_i) \ + RND_STEP_1_1(h,a,b,c,d,e,f,g,_i+1) \ + RND_STEP_1_2(h,a,b,c,d,e,f,g,_i+1) \ + VPSRLD (YTMP5, YTMP1, 3) /* YTMP4 = W[-15] >> 3 */ \ + VPXOR (YTMP2, YTMP4, YTMP2) /* YTMP2 = W[-15] >>> 7 ^ W[-15] >>> 18 */ \ + RND_STEP_1_3(h,a,b,c,d,e,f,g,_i+1) \ + RND_STEP_1_4(h,a,b,c,d,e,f,g,_i+1) \ + VPXOR (YTMP1, YTMP5, YTMP2) /* YTMP1 = s0 */ \ + VPSHUFD (YTMP2, Y3, 0b11111010) /* YTMP2 = W[-2] {BBAA}*/ \ + RND_STEP_1_5(h,a,b,c,d,e,f,g,_i+1) \ + RND_STEP_1_6(h,a,b,c,d,e,f,g,_i+1) \ + VPSRLD (YTMP4, YTMP2, 10) /* YTMP4 = W[-2] >> 10 {BBAA} */ \ + VPSRLQ (YTMP3, YTMP2, 19) /* YTMP3 = W[-2] MY_ROR 19 {xBxA} */ \ + RND_STEP_1_7(h,a,b,c,d,e,f,g,_i+1) \ + RND_STEP_1_8(h,a,b,c,d,e,f,g,_i+1) \ + RND_STEP_0_1(g,h,a,b,c,d,e,f,_i+2) \ + VPSRLQ (YTMP2, YTMP2, 17) /* YTMP2 = W[-2] MY_ROR 17 {xBxA} */ \ + VPADDD (YTMP0, YTMP0, Y0) \ + RND_STEP_0_2(g,h,a,b,c,d,e,f,_i+2) \ + RND_STEP_0_3(g,h,a,b,c,d,e,f,_i+2) \ + RND_STEP_0_4(g,h,a,b,c,d,e,f,_i+2) \ + VPXOR (YTMP2, YTMP3, YTMP2) \ + VPADDD (YTMP0, YTMP0, YTMP1) /* YTMP0 = W[-16] + W[-7] + s0 */ \ + RND_STEP_0_5(g,h,a,b,c,d,e,f,_i+2) \ + VPXOR (YTMP4, YTMP4, YTMP2) /* YTMP4 = s1 {xBxA} */ \ + RND_STEP_0_6(g,h,a,b,c,d,e,f,_i+2) \ + VPSHUFB (YTMP4, YTMP4, SHUF_Y_00BA) /* YTMP4 = s1 {00BA} */ \ + RND_STEP_0_7(g,h,a,b,c,d,e,f,_i+2) \ + VPADDD (YTMP0, YTMP0, YTMP4) /* YTMP0 = {..., ..., W[1], W[0]} */ \ + RND_STEP_0_8(g,h,a,b,c,d,e,f,_i+2) \ + RND_STEP_1_1(f,g,h,a,b,c,d,e,_i+3) \ + VPSHUFD (YTMP2, YTMP0, 0b01010000) /* YTMP2 = W[-2] {DDCC} */ \ + RND_STEP_1_2(f,g,h,a,b,c,d,e,_i+3) \ + VPSRLQ (YTMP4, YTMP2, 17) /* YTMP4 = W[-2] MY_ROR 17 {xDxC} */ \ + VPSRLQ (YTMP3, YTMP2, 19) /* YTMP3 = W[-2] MY_ROR 19 {xDxC} */ \ + RND_STEP_1_3(f,g,h,a,b,c,d,e,_i+3) \ + RND_STEP_1_4(f,g,h,a,b,c,d,e,_i+3) \ + VPSRLD (YTMP5, YTMP2, 10) /* YTMP5 = W[-2] >> 10 {DDCC} */ \ + VPXOR (YTMP4, YTMP3, YTMP4) \ + RND_STEP_1_5(f,g,h,a,b,c,d,e,_i+3) \ + RND_STEP_1_6(f,g,h,a,b,c,d,e,_i+3) \ + VPXOR (YTMP5, YTMP4, YTMP5) /* YTMP5 = s1 {xDxC} */ \ + RND_STEP_1_7(f,g,h,a,b,c,d,e,_i+3) \ + VPSHUFB (YTMP5, YTMP5, SHUF_Y_DC00) /* YTMP5 = s1 {DC00} */ \ + RND_STEP_1_8(f,g,h,a,b,c,d,e,_i+3) \ + VPADDD (Y0, YTMP5, YTMP0) /* Y0 = {W[3], W[2], W[1], W[0]} */ + +#if defined(HAVE_INTEL_RORX) + +#define MsgSched_Y_RORX(Y0,Y1,Y2,Y3,a,b,c,d,e,f,g,h,_i) \ + RND_STEP_RORX_0_1(a,b,c,d,e,f,g,h,_i) \ + VPALIGNR (YTMP1, Y1, Y0, 4) /* YTMP1 = W[-15] */ \ + RND_STEP_RORX_0_2(a,b,c,d,e,f,g,h,_i) \ + VPALIGNR (YTMP0, Y3, Y2, 4) /* YTMP0 = W[-7] */ \ + RND_STEP_RORX_0_3(a,b,c,d,e,f,g,h,_i) \ + VPSRLD (YTMP2, YTMP1, 7) /* YTMP2 = W[-15] >> 7 */ \ + RND_STEP_RORX_0_4(a,b,c,d,e,f,g,h,_i) \ + VPSLLD (YTMP3, YTMP1, 25) /* YTEMP3 = W[-15] << (32-7) */ \ + RND_STEP_RORX_0_5(a,b,c,d,e,f,g,h,_i) \ + VPSRLD (YTMP4, YTMP1, 18) /* YTEMP4 = W[-15] >> 18 */ \ + RND_STEP_RORX_0_6(a,b,c,d,e,f,g,h,_i) \ + VPSLLD (YTMP5, YTMP1, 14) /* YTEMP5 = W[-15] << (32-18) */ \ + RND_STEP_RORX_0_7(a,b,c,d,e,f,g,h,_i) \ + VPOR (YTMP2, YTMP2, YTMP3) /* YTMP2 = W[-15] >>> 7 */ \ + RND_STEP_RORX_0_8(a,b,c,d,e,f,g,h,_i) \ + VPOR (YTMP4, YTMP4, YTMP5) /* YTMP4 = W[-15] >>> 18 */ \ + RND_STEP_RORX_1_1(h,a,b,c,d,e,f,g,_i+1) \ + VPSRLD (YTMP5, YTMP1, 3) /* YTMP4 = W[-15] >> 3 */ \ + RND_STEP_RORX_1_2(h,a,b,c,d,e,f,g,_i+1) \ + VPXOR (YTMP2, YTMP2, YTMP4) /* YTMP2 = W[-15] >>> 7 ^ W[-15] >>> 18 */ \ + RND_STEP_RORX_1_3(h,a,b,c,d,e,f,g,_i+1) \ + VPSHUFD (YTMP3, Y3, 0b11111010) /* YTMP2 = W[-2] {BBAA}*/ \ + RND_STEP_RORX_1_4(h,a,b,c,d,e,f,g,_i+1) \ + VPXOR (YTMP1, YTMP5, YTMP2) /* YTMP1 = s0 */ \ + RND_STEP_RORX_1_5(h,a,b,c,d,e,f,g,_i+1) \ + VPSRLD (YTMP4, YTMP3, 10) /* YTMP4 = W[-2] >> 10 {BBAA} */ \ + RND_STEP_RORX_1_6(h,a,b,c,d,e,f,g,_i+1) \ + VPSRLQ (YTMP2, YTMP3, 19) /* YTMP3 = W[-2] MY_ROR 19 {xBxA} */ \ + RND_STEP_RORX_1_7(h,a,b,c,d,e,f,g,_i+1) \ + VPSRLQ (YTMP3, YTMP3, 17) /* YTMP2 = W[-2] MY_ROR 17 {xBxA} */ \ + RND_STEP_RORX_1_8(h,a,b,c,d,e,f,g,_i+1) \ + VPADDD (YTMP0, YTMP0, Y0) \ + RND_STEP_RORX_0_1(g,h,a,b,c,d,e,f,_i+2) \ + VPXOR (YTMP2, YTMP2, YTMP3) \ + RND_STEP_RORX_0_2(g,h,a,b,c,d,e,f,_i+2) \ + VPXOR (YTMP4, YTMP4, YTMP2) /* YTMP4 = s1 {xBxA} */ \ + RND_STEP_RORX_0_3(g,h,a,b,c,d,e,f,_i+2) \ + VPADDD (YTMP0, YTMP0, YTMP1) /* YTMP0 = W[-16] + W[-7] + s0 */ \ + RND_STEP_RORX_0_4(g,h,a,b,c,d,e,f,_i+2) \ + VPSHUFB (YTMP4, YTMP4, SHUF_Y_00BA) /* YTMP4 = s1 {00BA} */ \ + RND_STEP_RORX_0_5(g,h,a,b,c,d,e,f,_i+2) \ + VPADDD (YTMP0, YTMP0, YTMP4) /* YTMP0 = {..., ..., W[1], W[0]} */ \ + RND_STEP_RORX_0_6(g,h,a,b,c,d,e,f,_i+2) \ + VPSHUFD (YTMP2, YTMP0, 0b01010000) /* YTMP2 = W[-2] {DDCC} */ \ + RND_STEP_RORX_0_7(g,h,a,b,c,d,e,f,_i+2) \ + RND_STEP_RORX_0_8(g,h,a,b,c,d,e,f,_i+2) \ + VPSRLQ (YTMP4, YTMP2, 17) /* YTMP4 = W[-2] MY_ROR 17 {xDxC} */ \ + RND_STEP_RORX_1_1(f,g,h,a,b,c,d,e,_i+3) \ + VPSRLQ (YTMP3, YTMP2, 19) /* YTMP3 = W[-2] MY_ROR 19 {xDxC} */ \ + RND_STEP_RORX_1_2(f,g,h,a,b,c,d,e,_i+3) \ + VPSRLD (YTMP5, YTMP2, 10) /* YTMP5 = W[-2] >> 10 {DDCC} */ \ + RND_STEP_RORX_1_3(f,g,h,a,b,c,d,e,_i+3) \ + VPXOR (YTMP4, YTMP4, YTMP3) \ + RND_STEP_RORX_1_4(f,g,h,a,b,c,d,e,_i+3) \ + VPXOR (YTMP5, YTMP5, YTMP4) /* YTMP5 = s1 {xDxC} */ \ + RND_STEP_RORX_1_5(f,g,h,a,b,c,d,e,_i+3) \ + RND_STEP_RORX_1_6(f,g,h,a,b,c,d,e,_i+3) \ + VPSHUFB (YTMP5, YTMP5, SHUF_Y_DC00) /* YTMP5 = s1 {DC00} */ \ + RND_STEP_RORX_1_7(f,g,h,a,b,c,d,e,_i+3) \ + RND_STEP_RORX_1_8(f,g,h,a,b,c,d,e,_i+3) \ + VPADDD (Y0, YTMP5, YTMP0) /* Y0 = {W[3], W[2], W[1], W[0]} */ \ + +#endif /* HAVE_INTEL_RORX */ + +#define _VINSERTI128(op1,op2,op3,op4) \ + "vinserti128 $"#op4", %"#op3", %"#op2", %"#op1"\n\t" +#define VINSERTI128(op1,op2,op3,op4) \ + _VINSERTI128(op1,op2,op3,op4) + + +#define _LOAD_W_K_LOW(BYTE_FLIP_MASK, reg) \ + "# X0, X1, X2, X3 = W[0..15]\n\t" \ + "vmovdqu (%%"#reg"), %%xmm0\n\t" \ + "vmovdqu 16(%%"#reg"), %%xmm1\n\t" \ + VPSHUFB(X0, X0, BYTE_FLIP_MASK) \ + VPSHUFB(X1, X1, BYTE_FLIP_MASK) \ + "vmovdqu 32(%%"#reg"), %%xmm2\n\t" \ + "vmovdqu 48(%%"#reg"), %%xmm3\n\t" \ + VPSHUFB(X2, X2, BYTE_FLIP_MASK) \ + VPSHUFB(X3, X3, BYTE_FLIP_MASK) + +#define LOAD_W_K_LOW(BYTE_FLIP_MASK, reg) \ + _LOAD_W_K_LOW(BYTE_FLIP_MASK, reg) + + +#define _LOAD_W_K(BYTE_FLIP_Y_MASK, reg) \ + "# X0, X1, X2, X3 = W[0..15]\n\t" \ + "vmovdqu (%%"#reg"), %%xmm0\n\t" \ + "vmovdqu 16(%%"#reg"), %%xmm1\n\t" \ + "vmovdqu 64(%%"#reg"), %%xmm4\n\t" \ + "vmovdqu 80(%%"#reg"), %%xmm5\n\t" \ + VINSERTI128(Y0, Y0, XTMP0, 1) \ + VINSERTI128(Y1, Y1, XTMP1, 1) \ + VPSHUFB(Y0, Y0, BYTE_FLIP_Y_MASK) \ + VPSHUFB(Y1, Y1, BYTE_FLIP_Y_MASK) \ + "vmovdqu 32(%%"#reg"), %%xmm2\n\t" \ + "vmovdqu 48(%%"#reg"), %%xmm3\n\t" \ + "vmovdqu 96(%%"#reg"), %%xmm6\n\t" \ + "vmovdqu 112(%%"#reg"), %%xmm7\n\t" \ + VINSERTI128(Y2, Y2, XTMP2, 1) \ + VINSERTI128(Y3, Y3, XTMP3, 1) \ + VPSHUFB(Y2, Y2, BYTE_FLIP_Y_MASK) \ + VPSHUFB(Y3, Y3, BYTE_FLIP_Y_MASK) + +#define LOAD_W_K(BYTE_FLIP_Y_MASK, reg) \ + _LOAD_W_K(BYTE_FLIP_Y_MASK, reg) + + +#define _SET_W_Y_4(i) \ + "vpaddd ("#i"*8)+ 0+%[K], %%ymm0, %%ymm4\n\t" \ + "vpaddd ("#i"*8)+32+%[K], %%ymm1, %%ymm5\n\t" \ + "vmovdqu %%ymm4, ("#i"*8)+ 0("WK")\n\t" \ + "vmovdqu %%ymm5, ("#i"*8)+32("WK")\n\t" \ + "vpaddd ("#i"*8)+64+%[K], %%ymm2, %%ymm4\n\t" \ + "vpaddd ("#i"*8)+96+%[K], %%ymm3, %%ymm5\n\t" \ + "vmovdqu %%ymm4, ("#i"*8)+64("WK")\n\t" \ + "vmovdqu %%ymm5, ("#i"*8)+96("WK")\n\t" + +#define SET_W_Y_4(i) \ + _SET_W_Y_4(i) + + +static const ALIGN32 word64 mSHUF_Y_00BA[] = + { 0x0b0a090803020100, 0xFFFFFFFFFFFFFFFF, + 0x0b0a090803020100, 0xFFFFFFFFFFFFFFFF }; /* shuffle xBxA -> 00BA */ +static const ALIGN32 word64 mSHUF_Y_DC00[] = + { 0xFFFFFFFFFFFFFFFF, 0x0b0a090803020100, + 0xFFFFFFFFFFFFFFFF, 0x0b0a090803020100 }; /* shuffle xDxC -> DC00 */ +static const ALIGN32 word64 mBYTE_FLIP_Y_MASK[] = + { 0x0405060700010203, 0x0c0d0e0f08090a0b, + 0x0405060700010203, 0x0c0d0e0f08090a0b }; + +#define _INIT_MASKS_Y(BYTE_FLIP_MASK, SHUF_00BA, SHUF_DC00) \ + "vmovdqa %[FLIP], %"#BYTE_FLIP_MASK"\n\t" \ + "vmovdqa %[SHUF00BA], %"#SHUF_00BA"\n\t" \ + "vmovdqa %[SHUFDC00], %"#SHUF_DC00"\n\t" + +#define INIT_MASKS_Y(BYTE_FLIP_MASK, SHUF_00BA, SHUF_DC00) \ + _INIT_MASKS_Y(BYTE_FLIP_MASK, SHUF_00BA, SHUF_DC00) + +static const ALIGN32 word32 K256[128] = { + 0x428A2F98L, 0x71374491L, 0xB5C0FBCFL, 0xE9B5DBA5L, + 0x428A2F98L, 0x71374491L, 0xB5C0FBCFL, 0xE9B5DBA5L, + 0x3956C25BL, 0x59F111F1L, 0x923F82A4L, 0xAB1C5ED5L, + 0x3956C25BL, 0x59F111F1L, 0x923F82A4L, 0xAB1C5ED5L, + 0xD807AA98L, 0x12835B01L, 0x243185BEL, 0x550C7DC3L, + 0xD807AA98L, 0x12835B01L, 0x243185BEL, 0x550C7DC3L, + 0x72BE5D74L, 0x80DEB1FEL, 0x9BDC06A7L, 0xC19BF174L, + 0x72BE5D74L, 0x80DEB1FEL, 0x9BDC06A7L, 0xC19BF174L, + 0xE49B69C1L, 0xEFBE4786L, 0x0FC19DC6L, 0x240CA1CCL, + 0xE49B69C1L, 0xEFBE4786L, 0x0FC19DC6L, 0x240CA1CCL, + 0x2DE92C6FL, 0x4A7484AAL, 0x5CB0A9DCL, 0x76F988DAL, + 0x2DE92C6FL, 0x4A7484AAL, 0x5CB0A9DCL, 0x76F988DAL, + 0x983E5152L, 0xA831C66DL, 0xB00327C8L, 0xBF597FC7L, + 0x983E5152L, 0xA831C66DL, 0xB00327C8L, 0xBF597FC7L, + 0xC6E00BF3L, 0xD5A79147L, 0x06CA6351L, 0x14292967L, + 0xC6E00BF3L, 0xD5A79147L, 0x06CA6351L, 0x14292967L, + 0x27B70A85L, 0x2E1B2138L, 0x4D2C6DFCL, 0x53380D13L, + 0x27B70A85L, 0x2E1B2138L, 0x4D2C6DFCL, 0x53380D13L, + 0x650A7354L, 0x766A0ABBL, 0x81C2C92EL, 0x92722C85L, + 0x650A7354L, 0x766A0ABBL, 0x81C2C92EL, 0x92722C85L, + 0xA2BFE8A1L, 0xA81A664BL, 0xC24B8B70L, 0xC76C51A3L, + 0xA2BFE8A1L, 0xA81A664BL, 0xC24B8B70L, 0xC76C51A3L, + 0xD192E819L, 0xD6990624L, 0xF40E3585L, 0x106AA070L, + 0xD192E819L, 0xD6990624L, 0xF40E3585L, 0x106AA070L, + 0x19A4C116L, 0x1E376C08L, 0x2748774CL, 0x34B0BCB5L, + 0x19A4C116L, 0x1E376C08L, 0x2748774CL, 0x34B0BCB5L, + 0x391C0CB3L, 0x4ED8AA4AL, 0x5B9CCA4FL, 0x682E6FF3L, + 0x391C0CB3L, 0x4ED8AA4AL, 0x5B9CCA4FL, 0x682E6FF3L, + 0x748F82EEL, 0x78A5636FL, 0x84C87814L, 0x8CC70208L, + 0x748F82EEL, 0x78A5636FL, 0x84C87814L, 0x8CC70208L, + 0x90BEFFFAL, 0xA4506CEBL, 0xBEF9A3F7L, 0xC67178F2L, + 0x90BEFFFAL, 0xA4506CEBL, 0xBEF9A3F7L, 0xC67178F2L +}; + +SHA256_NOINLINE static int Transform_Sha256_AVX2(wc_Sha256* sha256) +{ + __asm__ __volatile__ ( + + "subq $512, %%rsp\n\t" + "leaq 32(%[sha256]), %%rax\n\t" + + INIT_MASKS_Y(BYTE_FLIP_MASK, SHUF_Y_00BA, SHUF_Y_DC00) + LOAD_DIGEST() + + LOAD_W_K_LOW(BYTE_FLIP_MASK, rax) + + "movl %%r9d, "L4"\n\t" + "movl %%r12d, "L1"\n\t" + "xorl %%r10d, "L4"\n\t" + + SET_W_Y_4(0) + MsgSched_Y(Y0, Y1, Y2, Y3, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 0) + MsgSched_Y(Y1, Y2, Y3, Y0, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 8) + MsgSched_Y(Y2, Y3, Y0, Y1, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 16) + MsgSched_Y(Y3, Y0, Y1, Y2, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 24) + + SET_W_Y_4(16) + MsgSched_Y(Y0, Y1, Y2, Y3, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 32) + MsgSched_Y(Y1, Y2, Y3, Y0, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 40) + MsgSched_Y(Y2, Y3, Y0, Y1, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 48) + MsgSched_Y(Y3, Y0, Y1, Y2, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 56) + + SET_W_Y_4(32) + MsgSched_Y(Y0, Y1, Y2, Y3, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 64) + MsgSched_Y(Y1, Y2, Y3, Y0, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 72) + MsgSched_Y(Y2, Y3, Y0, Y1, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 80) + MsgSched_Y(Y3, Y0, Y1, Y2, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 88) + + SET_W_Y_4(48) + RND_ALL_4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 96) + RND_ALL_4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 104) + RND_ALL_4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 112) + RND_ALL_4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 120) + + STORE_ADD_DIGEST() + + "addq $512, %%rsp\n\t" + + : + : [FLIP] "m" (mBYTE_FLIP_MASK[0]), + [SHUF00BA] "m" (mSHUF_Y_00BA[0]), + [SHUFDC00] "m" (mSHUF_Y_DC00[0]), + [sha256] "r" (sha256), + [K] "m" (K256) + : WORK_REGS, STATE_REGS, YMM_REGS, "memory" + ); + + return 0; +} + +SHA256_NOINLINE static int Transform_Sha256_AVX2_Len(wc_Sha256* sha256, + word32 len) +{ + if ((len & WC_SHA256_BLOCK_SIZE) != 0) { + Transform_Sha256_AVX2(sha256); + sha256->data += WC_SHA256_BLOCK_SIZE; + len -= WC_SHA256_BLOCK_SIZE; + if (len == 0) + return 0; + } + + __asm__ __volatile__ ( + + "subq $512, %%rsp\n\t" + "movq 120(%[sha256]), %%rax\n\t" + + INIT_MASKS_Y(BYTE_FLIP_Y_MASK, SHUF_Y_00BA, SHUF_Y_DC00) + LOAD_DIGEST() + + "# Start of loop processing two blocks\n" + "1:\n\t" + + LOAD_W_K(BYTE_FLIP_Y_MASK, rax) + + "movl %%r9d, "L4"\n\t" + "movl %%r12d, "L1"\n\t" + "xorl %%r10d, "L4"\n\t" + + SET_W_Y_4(0) + MsgSched_Y(Y0, Y1, Y2, Y3, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 0) + MsgSched_Y(Y1, Y2, Y3, Y0, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 8) + MsgSched_Y(Y2, Y3, Y0, Y1, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 16) + MsgSched_Y(Y3, Y0, Y1, Y2, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 24) + + SET_W_Y_4(16) + MsgSched_Y(Y0, Y1, Y2, Y3, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 32) + MsgSched_Y(Y1, Y2, Y3, Y0, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 40) + MsgSched_Y(Y2, Y3, Y0, Y1, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 48) + MsgSched_Y(Y3, Y0, Y1, Y2, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 56) + + SET_W_Y_4(32) + MsgSched_Y(Y0, Y1, Y2, Y3, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 64) + MsgSched_Y(Y1, Y2, Y3, Y0, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 72) + MsgSched_Y(Y2, Y3, Y0, Y1, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 80) + MsgSched_Y(Y3, Y0, Y1, Y2, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 88) + + SET_W_Y_4(48) + RND_ALL_4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 96) + RND_ALL_4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 104) + RND_ALL_4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 112) + RND_ALL_4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 120) + + ADD_DIGEST() + STORE_DIGEST() + + "movl %%r9d, "L4"\n\t" + "movl %%r12d, "L1"\n\t" + "xorl %%r10d, "L4"\n\t" + + RND_ALL_4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 4) + RND_ALL_4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 12) + RND_ALL_4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 20) + RND_ALL_4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 28) + RND_ALL_4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 36) + RND_ALL_4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 44) + RND_ALL_4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 52) + RND_ALL_4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 60) + RND_ALL_4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 68) + RND_ALL_4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 76) + RND_ALL_4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 84) + RND_ALL_4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 92) + RND_ALL_4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 100) + RND_ALL_4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 108) + RND_ALL_4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 116) + RND_ALL_4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 124) + + ADD_DIGEST() + + "movq 120(%[sha256]), %%rax\n\t" + "addq $128, %%rax\n\t" + "subl $128, %[len]\n\t" + + STORE_DIGEST() + + "movq %%rax, 120(%[sha256])\n\t" + "jnz 1b\n\t" + + "addq $512, %%rsp\n\t" + + : + : [FLIP] "m" (mBYTE_FLIP_Y_MASK[0]), + [SHUF00BA] "m" (mSHUF_Y_00BA[0]), + [SHUFDC00] "m" (mSHUF_Y_DC00[0]), + [sha256] "r" (sha256), + [len] "r" (len), + [K] "m" (K256) + : WORK_REGS, STATE_REGS, YMM_REGS, "memory" ); return 0; } #if defined(HAVE_INTEL_RORX) -static int Transform_AVX1_RORX(wc_Sha256* sha256) +SHA256_NOINLINE static int Transform_Sha256_AVX2_RORX(wc_Sha256* sha256) { - ALIGN32 word32 W_K[64]; /* temp for W+K */ - __asm__ __volatile__ ( - Init_Masks(BYTE_FLIP_MASK, SHUF_00BA, SHUF_DC00) - "# X0, X1, X2, X3 = W[0..15]; \n\t" - W_K_from_buff() + "subq $512, %%rsp\n\t" + "leaq 32(%[sha256]), %%rax\n\t" - DigestToReg(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7) + INIT_MASKS_Y(BYTE_FLIP_MASK, SHUF_Y_00BA, SHUF_Y_DC00) + LOAD_W_K_LOW(BYTE_FLIP_MASK, rax) - SET_W_K_XFER(X0, 0) - MessageSched(X0, X1, X2, X3, XTMP0, XTMP1, XTMP2, XTMP3, XTMP4, XTMP5, XFER, - SHUF_00BA, SHUF_DC00, S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,0) - SET_W_K_XFER(X1, 4) - MessageSched_RORX(X1, X2, X3, X0, XTMP0, XTMP1, XTMP2, XTMP3, XTMP4, XTMP5, - XFER, SHUF_00BA, SHUF_DC00, S_4,S_5,S_6,S_7,S_0,S_1,S_2,S_3,4) - SET_W_K_XFER(X2, 8) - MessageSched_RORX(X2, X3, X0, X1, XTMP0, XTMP1, XTMP2, XTMP3, XTMP4, XTMP5, - XFER, SHUF_00BA, SHUF_DC00, S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,8) - SET_W_K_XFER(X3, 12) - MessageSched_RORX(X3, X0, X1, X2, XTMP0, XTMP1, XTMP2, XTMP3, XTMP4, XTMP5, - XFER, SHUF_00BA, SHUF_DC00, S_4,S_5,S_6,S_7,S_0,S_1,S_2,S_3,12) - SET_W_K_XFER(X0, 16) - MessageSched_RORX(X0, X1, X2, X3, XTMP0, XTMP1, XTMP2, XTMP3, XTMP4, XTMP5, - XFER, SHUF_00BA, SHUF_DC00, S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,16) - SET_W_K_XFER(X1, 20) - MessageSched_RORX(X1, X2, X3, X0, XTMP0, XTMP1, XTMP2, XTMP3, XTMP4, XTMP5, - XFER, SHUF_00BA, SHUF_DC00, S_4,S_5,S_6,S_7,S_0,S_1,S_2,S_3,20) - SET_W_K_XFER(X2, 24) - MessageSched_RORX(X2, X3, X0, X1, XTMP0, XTMP1, XTMP2, XTMP3, XTMP4, XTMP5, - XFER, SHUF_00BA, SHUF_DC00, S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,24) - SET_W_K_XFER(X3, 28) - MessageSched_RORX(X3, X0, X1, X2, XTMP0, XTMP1, XTMP2, XTMP3, XTMP4, XTMP5, - XFER, SHUF_00BA, SHUF_DC00, S_4,S_5,S_6,S_7,S_0,S_1,S_2,S_3,28) - SET_W_K_XFER(X0, 32) - MessageSched_RORX(X0, X1, X2, X3, XTMP0, XTMP1, XTMP2, XTMP3, XTMP4, XTMP5, - XFER, SHUF_00BA, SHUF_DC00, S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,32) - SET_W_K_XFER(X1, 36) - MessageSched_RORX(X1, X2, X3, X0, XTMP0, XTMP1, XTMP2, XTMP3, XTMP4, XTMP5, - XFER, SHUF_00BA, SHUF_DC00, S_4,S_5,S_6,S_7,S_0,S_1,S_2,S_3,36) - SET_W_K_XFER(X2, 40) - MessageSched_RORX(X2, X3, X0, X1, XTMP0, XTMP1, XTMP2, XTMP3, XTMP4, XTMP5, - XFER, SHUF_00BA, SHUF_DC00, S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,40) - SET_W_K_XFER(X3, 44) - MessageSched_RORX(X3, X0, X1, X2, XTMP0, XTMP1, XTMP2, XTMP3, XTMP4, XTMP5, - XFER, SHUF_00BA, SHUF_DC00, S_4,S_5,S_6,S_7,S_0,S_1,S_2,S_3,44) + LOAD_DIGEST() - SET_W_K_XFER(X0, 48) - SET_W_K_XFER(X1, 52) - SET_W_K_XFER(X2, 56) - SET_W_K_XFER(X3, 60) + "movl %%r9d, "L4"\n\t" + "rorx $6, %%r12d, "L1"\n\t" + "xorl %%r10d, "L4"\n\t" - RND_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,48) - RND_7(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,49) - RND_6(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,50) - RND_5(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,51) + SET_W_Y_4(0) + MsgSched_Y_RORX(Y0, Y1, Y2, Y3, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 0) + MsgSched_Y_RORX(Y1, Y2, Y3, Y0, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 8) + MsgSched_Y_RORX(Y2, Y3, Y0, Y1, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 16) + MsgSched_Y_RORX(Y3, Y0, Y1, Y2, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 24) - RND_4(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,52) - RND_3(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,53) - RND_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,54) - RND_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,55) + SET_W_Y_4(16) + MsgSched_Y_RORX(Y0, Y1, Y2, Y3, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 32) + MsgSched_Y_RORX(Y1, Y2, Y3, Y0, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 40) + MsgSched_Y_RORX(Y2, Y3, Y0, Y1, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 48) + MsgSched_Y_RORX(Y3, Y0, Y1, Y2, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 56) - RND_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,56) - RND_7(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,57) - RND_6(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,58) - RND_5(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,59) + SET_W_Y_4(32) + MsgSched_Y_RORX(Y0, Y1, Y2, Y3, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 64) + MsgSched_Y_RORX(Y1, Y2, Y3, Y0, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 72) + MsgSched_Y_RORX(Y2, Y3, Y0, Y1, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 80) + MsgSched_Y_RORX(Y3, Y0, Y1, Y2, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 88) - RND_4(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,60) - RND_3(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,61) - RND_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,62) - RND_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,63) + SET_W_Y_4(48) + "xorl "L3", "L3"\n\t" + "xorl "L2", "L2"\n\t" + RND_RORX_X4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 96) + RND_RORX_X4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 104) + RND_RORX_X4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 112) + RND_RORX_X4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 120) + /* Prev RND: h += Maj(a,b,c) */ + "addl "L3", %%r8d\n\t" - RegToDigest(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7) + STORE_ADD_DIGEST() + + "addq $512, %%rsp\n\t" : : [FLIP] "m" (mBYTE_FLIP_MASK[0]), - [SHUF00BA] "m" (mSHUF_00BA[0]), - [SHUFDC00] "m" (mSHUF_DC00[0]), - [digest] "m" (sha256->digest), - [buf] "m" (sha256->buffer), - [K] "m" (K), - [W_K] "m" (W_K) - : SSE_REGs, "memory" + [SHUF00BA] "m" (mSHUF_Y_00BA[0]), + [SHUFDC00] "m" (mSHUF_Y_DC00[0]), + [sha256] "r" (sha256), + [K] "m" (K256) + : WORK_REGS, STATE_REGS, YMM_REGS, "memory" + ); + + return 0; +} + +SHA256_NOINLINE static int Transform_Sha256_AVX2_RORX_Len(wc_Sha256* sha256, + word32 len) +{ + if ((len & WC_SHA256_BLOCK_SIZE) != 0) { + Transform_Sha256_AVX2_RORX(sha256); + sha256->data += WC_SHA256_BLOCK_SIZE; + len -= WC_SHA256_BLOCK_SIZE; + if (len == 0) + return 0; + } + + __asm__ __volatile__ ( + + "subq $512, %%rsp\n\t" + "movq 120(%[sha256]), %%rax\n\t" + + INIT_MASKS_Y(BYTE_FLIP_Y_MASK, SHUF_Y_00BA, SHUF_Y_DC00) + LOAD_DIGEST() + + "# Start of loop processing two blocks\n" + "1:\n\t" + + LOAD_W_K(BYTE_FLIP_Y_MASK, rax) + + "movl %%r9d, "L4"\n\t" + "rorx $6, %%r12d, "L1"\n\t" + "xorl %%r10d, "L4"\n\t" + + SET_W_Y_4(0) + MsgSched_Y_RORX(Y0, Y1, Y2, Y3, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 0) + MsgSched_Y_RORX(Y1, Y2, Y3, Y0, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 8) + MsgSched_Y_RORX(Y2, Y3, Y0, Y1, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 16) + MsgSched_Y_RORX(Y3, Y0, Y1, Y2, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 24) + + SET_W_Y_4(16) + MsgSched_Y_RORX(Y0, Y1, Y2, Y3, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 32) + MsgSched_Y_RORX(Y1, Y2, Y3, Y0, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 40) + MsgSched_Y_RORX(Y2, Y3, Y0, Y1, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 48) + MsgSched_Y_RORX(Y3, Y0, Y1, Y2, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 56) + + SET_W_Y_4(32) + MsgSched_Y_RORX(Y0, Y1, Y2, Y3, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 64) + MsgSched_Y_RORX(Y1, Y2, Y3, Y0, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 72) + MsgSched_Y_RORX(Y2, Y3, Y0, Y1, S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 80) + MsgSched_Y_RORX(Y3, Y0, Y1, Y2, S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 88) + + SET_W_Y_4(48) + "xorl "L3", "L3"\n\t" + "xorl "L2", "L2"\n\t" + RND_RORX_X4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 96) + RND_RORX_X4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 104) + RND_RORX_X4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 112) + RND_RORX_X4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 120) + /* Prev RND: h += Maj(a,b,c) */ + "addl "L3", %%r8d\n\t" + "xorl "L2", "L2"\n\t" + + ADD_DIGEST() + STORE_DIGEST() + + "movl %%r9d, "L4"\n\t" + "xorl "L3", "L3"\n\t" + "xorl %%r10d, "L4"\n\t" + + RND_RORX_X4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 4) + RND_RORX_X4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 12) + RND_RORX_X4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 20) + RND_RORX_X4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 28) + RND_RORX_X4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 36) + RND_RORX_X4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 44) + RND_RORX_X4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 52) + RND_RORX_X4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 60) + RND_RORX_X4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 68) + RND_RORX_X4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 76) + RND_RORX_X4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 84) + RND_RORX_X4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 92) + RND_RORX_X4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 100) + RND_RORX_X4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 108) + RND_RORX_X4(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7, 116) + RND_RORX_X4(S_4, S_5, S_6, S_7, S_0, S_1, S_2, S_3, 124) + /* Prev RND: h += Maj(a,b,c) */ + "addl "L3", %%r8d\n\t" + "movq 120(%[sha256]), %%rax\n\t" + + ADD_DIGEST() + + "addq $128, %%rax\n\t" + "subl $128, %[len]\n\t" + + STORE_DIGEST() + + "movq %%rax, 120(%[sha256])\n\t" + "jnz 1b\n\t" + + "addq $512, %%rsp\n\t" + + : + : [FLIP] "m" (mBYTE_FLIP_Y_MASK[0]), + [SHUF00BA] "m" (mSHUF_Y_00BA[0]), + [SHUFDC00] "m" (mSHUF_Y_DC00[0]), + [sha256] "r" (sha256), + [len] "r" (len), + [K] "m" (K256) + : WORK_REGS, STATE_REGS, YMM_REGS, "memory" ); return 0; } #endif /* HAVE_INTEL_RORX */ -#endif /* HAVE_INTEL_AVX1 */ - - -#if defined(HAVE_INTEL_AVX2) - -#define _MOVE_to_REG(ymm, mem, i) \ - "leaq %["#mem"], %%r8\n\t" \ - "vmovdqu ("#i")*4(%%r8), %%"#ymm"\n\t" -#define _MOVE_to_MEM(mem, i, ymm) \ - "leaq %["#mem"], %%r8\n\t" \ - "vmovdqu %%"#ymm", "#i"*4(%%r8)\n\t" -#define _BYTE_SWAP(ymm, map) \ - "vpshufb %["#map"], %%"#ymm", %%"#ymm"\n\t" -#define _MOVE_128(ymm0, ymm1, ymm2, map) \ - "vperm2i128 $"#map", %%"#ymm2", %%"#ymm1", %%"#ymm0"\n\t" -#define _MOVE_BYTE(ymm0, ymm1, map) \ - "vpshufb %["#map"], %%"#ymm1", %%"#ymm0"\n\t" -#define _S_TEMP(dest, src, bits, temp) \ - "vpsrld $"#bits", %%"#src", %%"#dest"\n\t" \ - "vpslld $32-"#bits", %%"#src", %%"#temp"\n\t" \ - "vpor %%"#temp",%%"#dest", %%"#dest"\n\t" -#define _AVX2_R(dest, src, bits) \ - "vpsrld $"#bits", %%"#src", %%"#dest"\n\t" -#define _XOR(dest, src1, src2) \ - "vpxor %%"#src1", %%"#src2", %%"#dest"\n\t" -#define _OR(dest, src1, src2) \ - "vpor %%"#src1", %%"#src2", %%"#dest"\n\t" -#define _ADD(dest, src1, src2) \ - "vpaddd %%"#src1", %%"#src2", %%"#dest"\n\t" -#define _ADD_MEM(dest, src1, mem, i) \ - "leaq %["#mem"], %%r8\n\t" \ - "vpaddd "#i"*4(%%r8), %%"#src1", %%"#dest"\n\t" -#define _BLEND(map, dest, src1, src2) \ - "vpblendd $"#map", %%"#src1", %%"#src2", %%"#dest"\n\t" - -#define _EXTRACT_XMM_0(xmm, mem) \ - "vpextrd $0, %%"#xmm", %["#mem"]\n\t" -#define _EXTRACT_XMM_1(xmm, mem) \ - "vpextrd $1, %%"#xmm", %["#mem"]\n\t" -#define _EXTRACT_XMM_2(xmm, mem) \ - "vpextrd $2, %%"#xmm", %["#mem"]\n\t" -#define _EXTRACT_XMM_3(xmm, mem) \ - "vpextrd $3, %%"#xmm", %["#mem"]\n\t" -#define _EXTRACT_XMM_4(ymm, xmm, mem) \ - "vperm2i128 $0x1, %%"#ymm", %%"#ymm", %%"#ymm"\n\t" \ - "vpextrd $0, %%"#xmm", %["#mem"]\n\t" -#define _EXTRACT_XMM_5(xmm, mem) \ - "vpextrd $1, %%"#xmm", %["#mem"]\n\t" -#define _EXTRACT_XMM_6(xmm, mem) \ - "vpextrd $2, %%"#xmm", %["#mem"]\n\t" -#define _EXTRACT_XMM_7(xmm, mem) \ - "vpextrd $3, %%"#xmm", %["#mem"]\n\t" - -#define _SWAP_YMM_HL(ymm) \ - "vperm2i128 $0x1, %%"#ymm", %%"#ymm", %%"#ymm"\n\t" -#define SWAP_YMM_HL(ymm) _SWAP_YMM_HL(ymm) - -#define MOVE_to_REG(ymm, mem, i) _MOVE_to_REG(ymm, mem, i) -#define MOVE_to_MEM(mem, i, ymm) _MOVE_to_MEM(mem, i, ymm) -#define BYTE_SWAP(ymm, map) _BYTE_SWAP(ymm, map) -#define MOVE_128(ymm0, ymm1, ymm2, map) _MOVE_128(ymm0, ymm1, ymm2, map) -#define MOVE_BYTE(ymm0, ymm1, map) _MOVE_BYTE(ymm0, ymm1, map) -#define XOR(dest, src1, src2) _XOR(dest, src1, src2) -#define OR(dest, src1, src2) _OR(dest, src1, src2) -#define ADD(dest, src1, src2) _ADD(dest, src1, src2) -#define ADD_MEM(dest, src1, mem, i) _ADD_MEM(dest, src1, mem, i) -#define BLEND(map, dest, src1, src2) _BLEND(map, dest, src1, src2) - -#define S_TMP(dest, src, bits, temp) _S_TEMP(dest, src, bits, temp) -#define AVX2_S(dest, src, bits) S_TMP(dest, src, bits, S_TEMP) -#define AVX2_R(dest, src, bits) _AVX2_R(dest, src, bits) - -#define GAMMA0(dest, src) AVX2_S(dest, src, 7) AVX2_S(G_TEMP, src, 18) \ - XOR(dest, G_TEMP, dest) AVX2_R(G_TEMP, src, 3) XOR(dest, G_TEMP, dest) -#define GAMMA0_1(dest, src) AVX2_S(dest, src, 7) AVX2_S(G_TEMP, src, 18) -#define GAMMA0_2(dest, src) XOR(dest, G_TEMP, dest) AVX2_R(G_TEMP, src, 3) \ - XOR(dest, G_TEMP, dest) - -#define GAMMA1(dest, src) AVX2_S(dest, src, 17) AVX2_S(G_TEMP, src, 19) \ - XOR(dest, G_TEMP, dest) AVX2_R(G_TEMP, src, 10) XOR(dest, G_TEMP, dest) -#define GAMMA1_1(dest, src) AVX2_S(dest, src, 17) AVX2_S(G_TEMP, src, 19) -#define GAMMA1_2(dest, src) XOR(dest, G_TEMP, dest) AVX2_R(G_TEMP, src, 10) \ - XOR(dest, G_TEMP, dest) - -#define FEEDBACK1_to_W_I_2 MOVE_BYTE(YMM_TEMP0, W_I, MAP1W_2) \ - BLEND(0x0c, W_I_2, YMM_TEMP0, W_I_2) -#define FEEDBACK2_to_W_I_2 MOVE_128(YMM_TEMP0, W_I, W_I, 0x08) \ - MOVE_BYTE(YMM_TEMP0, YMM_TEMP0, MAP2W_2) BLEND(0x30, W_I_2, YMM_TEMP0, W_I_2) -#define FEEDBACK3_to_W_I_2 MOVE_BYTE(YMM_TEMP0, W_I, MAP3W_2) \ - BLEND(0xc0, W_I_2, YMM_TEMP0, W_I_2) - -#define FEEDBACK_to_W_I_7 MOVE_128(YMM_TEMP0, W_I, W_I, 0x08)\ - MOVE_BYTE(YMM_TEMP0, YMM_TEMP0, MAPW_7) BLEND(0x80, W_I_7, YMM_TEMP0, W_I_7) - -#undef voitle - -#define W_I_16 ymm8 -#define W_I_15 ymm9 -#define W_I_7 ymm10 -#define W_I_2 ymm11 -#define W_I ymm12 -#define G_TEMP ymm13 -#define S_TEMP ymm14 -#define YMM_TEMP0 ymm15 -#define YMM_TEMP0x xmm15 -#define W_I_TEMP ymm7 -#define W_K_TEMP ymm15 -#define W_K_TEMPx xmm15 - -#define MOVE_15_to_16(w_i_16, w_i_15, w_i_7)\ - "vperm2i128 $0x01, %%"#w_i_15", %%"#w_i_15", %%"#w_i_15"\n\t" \ - "vpblendd $0x08, %%"#w_i_15", %%"#w_i_7", %%"#w_i_16"\n\t" \ - "vperm2i128 $0x01, %%"#w_i_7", %%"#w_i_7", %%"#w_i_15"\n\t" \ - "vpblendd $0x80, %%"#w_i_15", %%"#w_i_16", %%"#w_i_16"\n\t" \ - "vpshufd $0x93, %%"#w_i_16", %%"#w_i_16"\n\t" - -#define MOVE_7_to_15(w_i_15, w_i_7)\ - "vmovdqu %%"#w_i_7", %%"#w_i_15"\n\t" - -#define MOVE_I_to_7(w_i_7, w_i)\ - "vperm2i128 $0x01, %%"#w_i", %%"#w_i", %%"#w_i_7"\n\t" \ - "vpblendd $0x01, %%"#w_i_7", %%"#w_i", %%"#w_i_7"\n\t" \ - "vpshufd $0x39, %%"#w_i_7", %%"#w_i_7"\n\t" - -#define MOVE_I_to_2(w_i_2, w_i)\ - "vperm2i128 $0x01, %%"#w_i", %%"#w_i", %%"#w_i_2"\n\t" \ - "vpshufd $0x0e, %%"#w_i_2", %%"#w_i_2"\n\t" - -#define ROTATE_W(w_i_16, w_i_15, w_i_7, w_i_2, w_i)\ - MOVE_15_to_16(w_i_16, w_i_15, w_i_7) \ - MOVE_7_to_15(w_i_15, w_i_7) \ - MOVE_I_to_7(w_i_7, w_i) \ - MOVE_I_to_2(w_i_2, w_i) - -#define _DumpS(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7 )\ - { word32 d[8];\ - __asm__ volatile("movl %"#S_0", %0":"=r"(d[0])::SSE_REGs);\ - __asm__ volatile("movl %"#S_1", %0":"=r"(d[1])::SSE_REGs);\ - __asm__ volatile("movl %"#S_2", %0":"=r"(d[2])::SSE_REGs);\ - __asm__ volatile("movl %"#S_3", %0":"=r"(d[3])::SSE_REGs);\ - __asm__ volatile("movl %"#S_4", %0":"=r"(d[4])::SSE_REGs);\ - __asm__ volatile("movl %"#S_5", %0":"=r"(d[5])::SSE_REGs);\ - __asm__ volatile("movl %"#S_6", %0":"=r"(d[6])::SSE_REGs);\ - __asm__ volatile("movl %"#S_7", %0":"=r"(d[7])::SSE_REGs);\ - printf("S[0..7]=%08x,%08x,%08x,%08x,%08x,%08x,%08x,%08x\n", d[0],d[1],d[2],d[3],d[4],d[5],d[6],d[7]);\ - __asm__ volatile("movl %0, %"#S_0::"r"(d[0]):SSE_REGs);\ - __asm__ volatile("movl %0, %"#S_1::"r"(d[1]):SSE_REGs);\ - __asm__ volatile("movl %0, %"#S_2::"r"(d[2]):SSE_REGs);\ - __asm__ volatile("movl %0, %"#S_3::"r"(d[3]):SSE_REGs);\ - __asm__ volatile("movl %0, %"#S_4::"r"(d[4]):SSE_REGs);\ - __asm__ volatile("movl %0, %"#S_5::"r"(d[5]):SSE_REGs);\ - __asm__ volatile("movl %0, %"#S_6::"r"(d[6]):SSE_REGs);\ - __asm__ volatile("movl %0, %"#S_7::"r"(d[7]):SSE_REGs);\ -} - - -#define DigestToReg(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7 )\ - _DigestToReg(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7 ) - -#define RegToDigest(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7 )\ - _RegToDigest(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7 ) - -#define DumS(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7 )\ - _DumpS(S_0, S_1, S_2, S_3, S_4, S_5, S_6, S_7 ) - - - /* Byte swap Masks to ensure that rest of the words are filled with zero's. */ - static const unsigned long mBYTE_FLIP_MASK_16[] = - { 0x0405060700010203, 0x0c0d0e0f08090a0b, 0x0405060700010203, 0x0c0d0e0f08090a0b }; - static const unsigned long mBYTE_FLIP_MASK_15[] = - { 0x0405060700010203, 0x0c0d0e0f08090a0b, 0x0405060700010203, 0x0c0d0e0f08090a0b }; - static const unsigned long mBYTE_FLIP_MASK_7 [] = - { 0x0405060700010203, 0x0c0d0e0f08090a0b, 0x0405060700010203, 0x8080808008090a0b }; - static const unsigned long mBYTE_FLIP_MASK_2 [] = - { 0x0405060700010203, 0x8080808080808080, 0x8080808080808080, 0x8080808080808080 }; - - static const unsigned long mMAPtoW_I_7[] = - { 0x8080808080808080, 0x8080808080808080, 0x8080808080808080, 0x0302010080808080 }; - static const unsigned long mMAP1toW_I_2[] = - { 0x8080808080808080, 0x0706050403020100, 0x8080808080808080, 0x8080808080808080 }; - static const unsigned long mMAP2toW_I_2[] = - { 0x8080808080808080, 0x8080808080808080, 0x0f0e0d0c0b0a0908, 0x8080808080808080 }; - static const unsigned long mMAP3toW_I_2[] = - { 0x8080808080808080, 0x8080808080808080, 0x8080808080808080, 0x0706050403020100 }; - -static int Transform_AVX2(wc_Sha256* sha256) -{ -#ifdef WOLFSSL_SMALL_STACK - word32* W_K; - W_K = (word32*) XMALLOC(sizeof(word32) * 64, NULL, DYNAMIC_TYPE_TMP_BUFFER); - if (W_K == NULL) - return MEMORY_E; -#else - word32 W_K[64]; -#endif - - __asm__ __volatile__ ( - - MOVE_to_REG(W_I_16, buf, 0) BYTE_SWAP(W_I_16, FLIP_16) - MOVE_to_REG(W_I_15, buf, 1) BYTE_SWAP(W_I_15, FLIP_15) - MOVE_to_REG(W_I, buf, 8) BYTE_SWAP(W_I, FLIP_16) - MOVE_to_REG(W_I_7, buf, 16-7) BYTE_SWAP(W_I_7, FLIP_7) - MOVE_to_REG(W_I_2, buf, 16-2) BYTE_SWAP(W_I_2, FLIP_2) - - DigestToReg(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7) - - ADD_MEM(W_K_TEMP, W_I_16, K, 0) - MOVE_to_MEM(W_K, 0, W_K_TEMP) - - RND_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,0) - RND_7(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,1) - RND_6(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,2) - RND_5(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,3) - RND_4(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,4) - RND_3(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,5) - RND_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,6) - RND_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,7) - - ADD_MEM(YMM_TEMP0, W_I, K, 8) - MOVE_to_MEM(W_K, 8, YMM_TEMP0) - - /* W[i] = Gamma1(W[i-2]) + W[i-7] + Gamma0(W[i-15] + W[i-16]) */ - RND_0_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,8) - GAMMA0_1(W_I_TEMP, W_I_15) - RND_0_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,8) - GAMMA0_2(W_I_TEMP, W_I_15) - RND_0_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,8) - ADD(W_I_TEMP, W_I_16, W_I_TEMP)/* for saving W_I before adding incomplete W_I_7 */ - RND_7_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,9) - ADD(W_I, W_I_7, W_I_TEMP) - RND_7_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,9) - GAMMA1_1(YMM_TEMP0, W_I_2) - RND_7_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,9) - GAMMA1_2(YMM_TEMP0, W_I_2) - RND_6_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,10) - ADD(W_I, W_I, YMM_TEMP0)/* now W[16..17] are completed */ - RND_6_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,10) - FEEDBACK1_to_W_I_2 - RND_6_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,10) - FEEDBACK_to_W_I_7 - RND_5_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,11) - ADD(W_I_TEMP, W_I_7, W_I_TEMP) - RND_5_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,11) - GAMMA1_1(YMM_TEMP0, W_I_2) - RND_5_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,11) - GAMMA1_2(YMM_TEMP0, W_I_2) - RND_4_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,12) - ADD(W_I, W_I_TEMP, YMM_TEMP0)/* now W[16..19] are completed */ - RND_4_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,12) - FEEDBACK2_to_W_I_2 - RND_4_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,12) - GAMMA1_1(YMM_TEMP0, W_I_2) - RND_3_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,13) - GAMMA1_2(YMM_TEMP0, W_I_2) - RND_3_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,13) - ADD(W_I, W_I_TEMP, YMM_TEMP0) /* now W[16..21] are completed */ - RND_3_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,13) - FEEDBACK3_to_W_I_2 - RND_2_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,14) - GAMMA1(YMM_TEMP0, W_I_2) - RND_2_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,14) - RND_2_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,14) - ADD(W_I, W_I_TEMP, YMM_TEMP0) /* now W[16..23] are completed */ - RND_1_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,15) - - MOVE_to_REG(YMM_TEMP0, K, 16) - RND_1_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,15) - ROTATE_W(W_I_16, W_I_15, W_I_7, W_I_2, W_I) - RND_1_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,15) - ADD(YMM_TEMP0, YMM_TEMP0, W_I) - MOVE_to_MEM(W_K, 16, YMM_TEMP0) - - /* W[i] = Gamma1(W[i-2]) + W[i-7] + Gamma0(W[i-15] + W[i-16]) */ - RND_0_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,16) - GAMMA0_1(W_I_TEMP, W_I_15) - RND_0_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,16) - GAMMA0_2(W_I_TEMP, W_I_15) - RND_0_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,16) - ADD(W_I_TEMP, W_I_16, W_I_TEMP)/* for saving W_I before adding incomplete W_I_7 */ - RND_7_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,17) - ADD(W_I, W_I_7, W_I_TEMP) - RND_7_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,17) - GAMMA1_1(YMM_TEMP0, W_I_2) - RND_7_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,17) - GAMMA1_2(YMM_TEMP0, W_I_2) - RND_6_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,18) - ADD(W_I, W_I, YMM_TEMP0)/* now W[16..17] are completed */ - RND_6_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,18) - FEEDBACK1_to_W_I_2 - RND_6_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,18) - FEEDBACK_to_W_I_7 - RND_5_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,19) - ADD(W_I_TEMP, W_I_7, W_I_TEMP) - RND_5_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,19) - GAMMA1(YMM_TEMP0, W_I_2) - RND_5_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,19) - GAMMA1_2(YMM_TEMP0, W_I_2) - RND_4_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,20) - ADD(W_I, W_I_TEMP, YMM_TEMP0)/* now W[16..19] are completed */ - RND_4_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,20) - FEEDBACK2_to_W_I_2 - RND_4_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,20) - GAMMA1_1(YMM_TEMP0, W_I_2) - RND_3_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,21) - GAMMA1_2(YMM_TEMP0, W_I_2) - RND_3_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,21) - ADD(W_I, W_I_TEMP, YMM_TEMP0) /* now W[16..21] are completed */ - RND_3_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,21) - FEEDBACK3_to_W_I_2 - RND_2_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,22) - GAMMA1_1(YMM_TEMP0, W_I_2) - RND_2_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,22) - GAMMA1_2(YMM_TEMP0, W_I_2) - RND_2_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,22) - ADD(W_I, W_I_TEMP, YMM_TEMP0) /* now W[16..23] are completed */ - RND_1_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,23) - - MOVE_to_REG(YMM_TEMP0, K, 24) - RND_1_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,23) - ROTATE_W(W_I_16, W_I_15, W_I_7, W_I_2, W_I) - RND_1_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,23) - ADD(YMM_TEMP0, YMM_TEMP0, W_I) - MOVE_to_MEM(W_K, 24, YMM_TEMP0) - - /* W[i] = Gamma1(W[i-2]) + W[i-7] + Gamma0(W[i-15] + W[i-16]) */ - RND_0_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,24) - GAMMA0_1(W_I_TEMP, W_I_15) - RND_0_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,24) - GAMMA0_2(W_I_TEMP, W_I_15) - RND_0_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,24) - ADD(W_I_TEMP, W_I_16, W_I_TEMP)/* for saving W_I before adding incomplete W_I_7 */ - RND_7_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,25) - ADD(W_I, W_I_7, W_I_TEMP) - RND_7_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,25) - GAMMA1_1(YMM_TEMP0, W_I_2) - RND_7_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,25) - GAMMA1_2(YMM_TEMP0, W_I_2) - RND_6_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,26) - ADD(W_I, W_I, YMM_TEMP0)/* now W[16..17] are completed */ - RND_6_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,26) - FEEDBACK1_to_W_I_2 - RND_6_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,26) - FEEDBACK_to_W_I_7 - RND_5_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,27) - ADD(W_I_TEMP, W_I_7, W_I_TEMP) - RND_5_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,27) - GAMMA1_1(YMM_TEMP0, W_I_2) - RND_5_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,27) - GAMMA1_2(YMM_TEMP0, W_I_2) - RND_4_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,28) - ADD(W_I, W_I_TEMP, YMM_TEMP0)/* now W[16..19] are completed */ - RND_4_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,28) - FEEDBACK2_to_W_I_2 - RND_4_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,28) - GAMMA1_1(YMM_TEMP0, W_I_2) - RND_3_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,29) - GAMMA1_2(YMM_TEMP0, W_I_2) - RND_3_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,29) - ADD(W_I, W_I_TEMP, YMM_TEMP0) /* now W[16..21] are completed */ - RND_3_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,29) - FEEDBACK3_to_W_I_2 - RND_2_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,30) - GAMMA1(YMM_TEMP0, W_I_2) - RND_2_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,30) - RND_2_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,30) - ADD(W_I, W_I_TEMP, YMM_TEMP0) /* now W[16..23] are completed */ - RND_1_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,31) - - MOVE_to_REG(YMM_TEMP0, K, 32) - RND_1_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,31) - ROTATE_W(W_I_16, W_I_15, W_I_7, W_I_2, W_I) - RND_1_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,31) - ADD(YMM_TEMP0, YMM_TEMP0, W_I) - MOVE_to_MEM(W_K, 32, YMM_TEMP0) - - - /* W[i] = Gamma1(W[i-2]) + W[i-7] + Gamma0(W[i-15] + W[i-16]) */ - RND_0_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,32) - GAMMA0_1(W_I_TEMP, W_I_15) - RND_0_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,32) - GAMMA0_2(W_I_TEMP, W_I_15) - RND_0_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,32) - ADD(W_I_TEMP, W_I_16, W_I_TEMP)/* for saving W_I before adding incomplete W_I_7 */ - RND_7_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,33) - ADD(W_I, W_I_7, W_I_TEMP) - RND_7_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,33) - GAMMA1_1(YMM_TEMP0, W_I_2) - RND_7_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,33) - GAMMA1_2(YMM_TEMP0, W_I_2) - RND_6_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,34) - ADD(W_I, W_I, YMM_TEMP0)/* now W[16..17] are completed */ - RND_6_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,34) - FEEDBACK1_to_W_I_2 - RND_6_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,34) - FEEDBACK_to_W_I_7 - RND_5_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,35) - ADD(W_I_TEMP, W_I_7, W_I_TEMP) - RND_5_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,35) - GAMMA1_1(YMM_TEMP0, W_I_2) - RND_5_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,35) - GAMMA1_2(YMM_TEMP0, W_I_2) - RND_4_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,36) - ADD(W_I, W_I_TEMP, YMM_TEMP0)/* now W[16..19] are completed */ - RND_4_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,36) - FEEDBACK2_to_W_I_2 - RND_4_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,36) - GAMMA1_1(YMM_TEMP0, W_I_2) - RND_3_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,37) - GAMMA1_2(YMM_TEMP0, W_I_2) - RND_3_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,37) - ADD(W_I, W_I_TEMP, YMM_TEMP0) /* now W[16..21] are completed */ - RND_3_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,37) - FEEDBACK3_to_W_I_2 - RND_2_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,38) - GAMMA1_1(YMM_TEMP0, W_I_2) - RND_2_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,38) - GAMMA1_2(YMM_TEMP0, W_I_2) - RND_2_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,38) - ADD(W_I, W_I_TEMP, YMM_TEMP0) /* now W[16..23] are completed */ - RND_1_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,39) - - MOVE_to_REG(YMM_TEMP0, K, 40) - RND_1_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,39) - ROTATE_W(W_I_16, W_I_15, W_I_7, W_I_2, W_I) - RND_1_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,39) - ADD(YMM_TEMP0, YMM_TEMP0, W_I) - MOVE_to_MEM(W_K, 40, YMM_TEMP0) - - /* W[i] = Gamma1(W[i-2]) + W[i-7] + Gamma0(W[i-15] + W[i-16]) */ - RND_0_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,40) - GAMMA0_1(W_I_TEMP, W_I_15) - RND_0_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,40) - GAMMA0_2(W_I_TEMP, W_I_15) - RND_0_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,40) - ADD(W_I_TEMP, W_I_16, W_I_TEMP)/* for saving W_I before adding incomplete W_I_7 */ - RND_7_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,41) - ADD(W_I, W_I_7, W_I_TEMP) - RND_7_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,41) - GAMMA1_1(YMM_TEMP0, W_I_2) - RND_7_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,41) - GAMMA1_2(YMM_TEMP0, W_I_2) - RND_6_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,42) - ADD(W_I, W_I, YMM_TEMP0)/* now W[16..17] are completed */ - RND_6_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,42) - FEEDBACK1_to_W_I_2 - RND_6_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,42) - FEEDBACK_to_W_I_7 - RND_5_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,43) - ADD(W_I_TEMP, W_I_7, W_I_TEMP) - RND_5_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,43) - GAMMA1_1(YMM_TEMP0, W_I_2) - RND_5_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,43) - GAMMA1_2(YMM_TEMP0, W_I_2) - RND_4_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,44) - ADD(W_I, W_I_TEMP, YMM_TEMP0)/* now W[16..19] are completed */ - RND_4_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,44) - FEEDBACK2_to_W_I_2 - RND_4_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,44) - GAMMA1_1(YMM_TEMP0, W_I_2) - RND_3_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,45) - GAMMA1_2(YMM_TEMP0, W_I_2) - RND_3_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,45) - ADD(W_I, W_I_TEMP, YMM_TEMP0) /* now W[16..21] are completed */ - RND_3_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,45) - FEEDBACK3_to_W_I_2 - RND_2_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,46) - GAMMA1_1(YMM_TEMP0, W_I_2) - RND_2_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,46) - GAMMA1_2(YMM_TEMP0, W_I_2) - RND_2_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,46) - ADD(W_I, W_I_TEMP, YMM_TEMP0) /* now W[16..23] are completed */ - RND_1_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,47) - - MOVE_to_REG(YMM_TEMP0, K, 48) - RND_1_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,47) - ROTATE_W(W_I_16, W_I_15, W_I_7, W_I_2, W_I) - RND_1_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,47) - ADD(YMM_TEMP0, YMM_TEMP0, W_I) - MOVE_to_MEM(W_K, 48, YMM_TEMP0) - - /* W[i] = Gamma1(W[i-2]) + W[i-7] + Gamma0(W[i-15] + W[i-16]) */ - RND_0_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,48) - GAMMA0_1(W_I_TEMP, W_I_15) - RND_0_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,48) - GAMMA0_2(W_I_TEMP, W_I_15) - RND_0_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,48) - ADD(W_I_TEMP, W_I_16, W_I_TEMP)/* for saving W_I before adding incomplete W_I_7 */ - RND_7_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,49) - ADD(W_I, W_I_7, W_I_TEMP) - RND_7_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,49) - GAMMA1_1(YMM_TEMP0, W_I_2) - RND_7_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,49) - GAMMA1_2(YMM_TEMP0, W_I_2) - RND_6_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,50) - ADD(W_I, W_I, YMM_TEMP0)/* now W[16..17] are completed */ - RND_6_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,50) - FEEDBACK1_to_W_I_2 - RND_6_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,50) - FEEDBACK_to_W_I_7 - RND_5_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,51) - ADD(W_I_TEMP, W_I_7, W_I_TEMP) - RND_5_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,51) - GAMMA1_1(YMM_TEMP0, W_I_2) - RND_5_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,51) - GAMMA1_2(YMM_TEMP0, W_I_2) - RND_4_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,52) - ADD(W_I, W_I_TEMP, YMM_TEMP0)/* now W[16..19] are completed */ - RND_4_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,52) - FEEDBACK2_to_W_I_2 - RND_4_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,52) - GAMMA1_1(YMM_TEMP0, W_I_2) - RND_3_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,53) - GAMMA1_2(YMM_TEMP0, W_I_2) - RND_3_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,53) - ADD(W_I, W_I_TEMP, YMM_TEMP0) /* now W[16..21] are completed */ - RND_3_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,53) - FEEDBACK3_to_W_I_2 - RND_2_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,54) - GAMMA1_1(YMM_TEMP0, W_I_2) - RND_2_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,54) - GAMMA1_2(YMM_TEMP0, W_I_2) - RND_2_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,54) - ADD(W_I, W_I_TEMP, YMM_TEMP0) /* now W[16..23] are completed */ - RND_1_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,55) - - MOVE_to_REG(YMM_TEMP0, K, 56) - RND_1_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,55) - ROTATE_W(W_I_16, W_I_15, W_I_7, W_I_2, W_I) - RND_1_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,55) - ADD(YMM_TEMP0, YMM_TEMP0, W_I) - MOVE_to_MEM(W_K, 56, YMM_TEMP0) - - RND_0(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,56) - RND_7(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,57) - RND_6(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,58) - RND_5(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,59) - - RND_4(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,60) - RND_3(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,61) - RND_2(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,62) - RND_1(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7,63) - - RegToDigest(S_0,S_1,S_2,S_3,S_4,S_5,S_6,S_7) - - : - : [FLIP_16] "m" (mBYTE_FLIP_MASK_16[0]), - [FLIP_15] "m" (mBYTE_FLIP_MASK_15[0]), - [FLIP_7] "m" (mBYTE_FLIP_MASK_7[0]), - [FLIP_2] "m" (mBYTE_FLIP_MASK_2), - [MAPW_7] "m" (mMAPtoW_I_7[0]), - [MAP1W_2] "m" (mMAP1toW_I_2[0]), - [MAP2W_2] "m" (mMAP2toW_I_2[0]), - [MAP3W_2] "m" (mMAP3toW_I_2[0]), - [digest] "m" (sha256->digest), - [buf] "m" (sha256->buffer), - [K] "m" (K), - [W_K] "m" (W_K) - : SSE_REGs, "memory" - ); - -#ifdef WOLFSSL_SMALL_STACK - XFREE(W_K, NULL, DYNAMIC_TYPE_TMP_BUFFER); -#endif - - return 0; -} - -#endif /* HAVE_INTEL_AVX2 */ +#endif /* HAVE_INTEL_AVX2 */ #ifdef WOLFSSL_SHA224 diff --git a/wolfcrypt/src/sha3.c b/wolfcrypt/src/sha3.c index 2d51eaa908..d1f47f140f 100644 --- a/wolfcrypt/src/sha3.c +++ b/wolfcrypt/src/sha3.c @@ -327,7 +327,7 @@ do \ } \ } \ while (0) -#endif +#endif /* SHA3_BY_SPEC */ /* The block operation performed on the state. * @@ -376,7 +376,7 @@ static void BlockSha3(word64 *s) } #else #include "sha3_long.i" -#endif +#endif /* WOLFSSL_SHA3_SMALL */ /* Convert the array of bytes, in little-endian order, to a 64-bit integer. * diff --git a/wolfcrypt/src/sha512.c b/wolfcrypt/src/sha512.c index 7bb1bf33fa..15cfeeb827 100644 --- a/wolfcrypt/src/sha512.c +++ b/wolfcrypt/src/sha512.c @@ -31,6 +31,11 @@ #include #include +/* deprecated USE_SLOW_SHA2 (replaced with USE_SLOW_SHA512) */ +#if defined(USE_SLOW_SHA2) && !defined(USE_SLOW_SHA512) + #define USE_SLOW_SHA512 +#endif + /* fips wrapper calls, user can call direct */ #ifdef HAVE_FIPS int wc_InitSha512(wc_Sha512* sha) @@ -124,7 +129,20 @@ #if defined(USE_INTEL_SPEEDUP) #define HAVE_INTEL_AVX1 - #define HAVE_INTEL_AVX2 + + #if defined(__GNUC__) && ((__GNUC__ < 4) || \ + (__GNUC__ == 4 && __GNUC_MINOR__ <= 8)) + #define NO_AVX2_SUPPORT + #endif + #if defined(__clang__) && ((__clang_major__ < 3) || \ + (__clang_major__ == 3 && __clang_minor__ <= 5)) + #define NO_AVX2_SUPPORT + #endif + + #define HAVE_INTEL_AVX1 + #ifndef NO_AVX2_SUPPORT + #define HAVE_INTEL_AVX2 + #endif #endif #if defined(HAVE_INTEL_AVX1) @@ -137,24 +155,6 @@ #endif -#if defined(HAVE_INTEL_RORX) - #define ROTR(func, bits, x) \ - word64 func(word64 x) { word64 ret ;\ - __asm__ ("rorx $"#bits", %1, %0\n\t":"=r"(ret):"r"(x)) ;\ - return ret ;\ - } - - static INLINE ROTR(rotrFixed64_28, 28, x); - static INLINE ROTR(rotrFixed64_34, 34, x); - static INLINE ROTR(rotrFixed64_39, 39, x); - static INLINE ROTR(rotrFixed64_14, 14, x); - static INLINE ROTR(rotrFixed64_18, 18, x); - static INLINE ROTR(rotrFixed64_41, 41, x); - - #define S0_RORX(x) (rotrFixed64_28(x)^rotrFixed64_34(x)^rotrFixed64_39(x)) - #define S1_RORX(x) (rotrFixed64_14(x)^rotrFixed64_18(x)^rotrFixed64_41(x)) -#endif /* HAVE_INTEL_RORX */ - #if defined(HAVE_BYTEREVERSE64) && \ !defined(HAVE_INTEL_AVX1) && !defined(HAVE_INTEL_AVX2) #define ByteReverseWords64(out, in, size) ByteReverseWords64_1(out, size) @@ -207,11 +207,11 @@ static int InitSha512(wc_Sha512* sha512) } #if defined(HAVE_INTEL_AVX1)|| defined(HAVE_INTEL_AVX2) - Transform_AVX1(); # Function prototype - Transform_AVX2(); # + Transform_Sha512_AVX1(); # Function prototype + Transform_Sha512_AVX2(); # #endif - _Transform() { # Native Transform Function body + _Transform_Sha512() { # Native Transform Function body } @@ -240,7 +240,7 @@ static int InitSha512(wc_Sha512* sha512) #if defnied(HAVE_INTEL_AVX1) - int Transform_AVX1() { + int Transform_Sha512_AVX1() { Stitched Message Sched/Round } @@ -248,7 +248,7 @@ static int InitSha512(wc_Sha512* sha512) #if defnied(HAVE_INTEL_AVX2) - int Transform_AVX2() { + int Transform_Sha512_AVX2() { Stitched Message Sched/Round } #endif @@ -261,30 +261,29 @@ static int InitSha512(wc_Sha512* sha512) */ #if defined(HAVE_INTEL_AVX1) - static int Transform_AVX1(wc_Sha512 *sha512); + static int Transform_Sha512_AVX1(wc_Sha512 *sha512); + static int Transform_Sha512_AVX1_Len(wc_Sha512 *sha512, word32 len); #endif #if defined(HAVE_INTEL_AVX2) - static int Transform_AVX2(wc_Sha512 *sha512); - #if defined(HAVE_INTEL_AVX1) && defined(HAVE_INTEL_AVX2) && defined(HAVE_INTEL_RORX) - static int Transform_AVX1_RORX(wc_Sha512 *sha512); + static int Transform_Sha512_AVX2(wc_Sha512 *sha512); + static int Transform_Sha512_AVX2_Len(wc_Sha512 *sha512, word32 len); + #if defined(HAVE_INTEL_RORX) + static int Transform_Sha512_AVX1_RORX(wc_Sha512 *sha512); + static int Transform_Sha512_AVX1_RORX_Len(wc_Sha512 *sha512, + word32 len); + static int Transform_Sha512_AVX2_RORX(wc_Sha512 *sha512); + static int Transform_Sha512_AVX2_RORX_Len(wc_Sha512 *sha512, + word32 len); #endif #endif - static int _Transform(wc_Sha512 *sha512); - static int (*Transform_p)(wc_Sha512* sha512) = _Transform; + static int _Transform_Sha512(wc_Sha512 *sha512); + static int (*Transform_Sha512_p)(wc_Sha512* sha512) = _Transform_Sha512; + static int (*Transform_Sha512_Len_p)(wc_Sha512* sha512, word32 len) = NULL; static int transform_check = 0; static int intel_flags; - #define Transform(sha512) (*Transform_p)(sha512) - - /* Dummy for saving MM_REGs on behalf of Transform */ - /* #if defined(HAVE_INTEL_AVX2) - #define SAVE_XMM_YMM __asm__ volatile("orq %%r8, %%r8":::\ - "%ymm0","%ymm1","%ymm2","%ymm3","%ymm4","%ymm5","%ymm6","%ymm7","%ymm8","%ymm9","%ymm10","%ymm11",\ - "%ymm12","%ymm13","%ymm14","%ymm15") - */ - #if defined(HAVE_INTEL_AVX1) - #define SAVE_XMM_YMM __asm__ volatile("orq %%r8, %%r8":::\ - "xmm0","xmm1","xmm2","xmm3","xmm4","xmm5","xmm6","xmm7","xmm8","xmm9","xmm10","xmm11","xmm12","xmm13","xmm14","xmm15") - #endif + #define Transform_Sha512(sha512) (*Transform_Sha512_p)(sha512) + #define Transform_Sha512_Len(sha512, len) \ + (*Transform_Sha512_Len_p)(sha512, len) static void Sha512_SetTransform() { @@ -294,22 +293,35 @@ static int InitSha512(wc_Sha512* sha512) intel_flags = cpuid_get_flags(); #if defined(HAVE_INTEL_AVX2) - if (IS_INTEL_AVX2(intel_flags) && IS_INTEL_BMI2(intel_flags)) { - if (1) - Transform_p = Transform_AVX1_RORX; + if (IS_INTEL_AVX2(intel_flags)) { + #ifdef HAVE_INTEL_RORX + if (IS_INTEL_BMI2(intel_flags)) { + Transform_Sha512_p = Transform_Sha512_AVX2_RORX; + Transform_Sha512_Len_p = Transform_Sha512_AVX2_RORX_Len; + } else - Transform_p = Transform_AVX2; + #endif + if (1) { + Transform_Sha512_p = Transform_Sha512_AVX2; + Transform_Sha512_Len_p = Transform_Sha512_AVX2_Len; + } + #ifdef HAVE_INTEL_RORX + else { + Transform_Sha512_p = Transform_Sha512_AVX1_RORX; + Transform_Sha512_Len_p = Transform_Sha512_AVX1_RORX_Len; + } + #endif } else #endif #if defined(HAVE_INTEL_AVX1) - if (1) { - Transform_p = ((IS_INTEL_AVX1(intel_flags)) ? Transform_AVX1 : - _Transform); + if (IS_INTEL_AVX1(intel_flags)) { + Transform_Sha512_p = Transform_Sha512_AVX1; + Transform_Sha512_Len_p = Transform_Sha512_AVX1_Len; } else #endif - Transform_p = _Transform; + Transform_Sha512_p = _Transform_Sha512; transform_check = 1; } @@ -327,7 +339,7 @@ static int InitSha512(wc_Sha512* sha512) } #else - #define Transform(sha512) _Transform(sha512) + #define Transform_Sha512(sha512) _Transform_Sha512(sha512) int wc_InitSha512_ex(wc_Sha512* sha512, void* heap, int devId) { @@ -354,10 +366,6 @@ static int InitSha512(wc_Sha512* sha512) #endif /* Hardware Acceleration */ -#ifndef SAVE_XMM_YMM - #define SAVE_XMM_YMM -#endif - static const word64 K512[80] = { W64LIT(0x428a2f98d728ae22), W64LIT(0x7137449123ef65cd), W64LIT(0xb5c0fbcfec4d3b2f), W64LIT(0xe9b5dba58189dbbc), @@ -401,40 +409,43 @@ static const word64 K512[80] = { W64LIT(0x5fcb6fab3ad6faec), W64LIT(0x6c44198c4a475817) }; - - #define blk0(i) (W[i] = sha512->buffer[i]) -#define blk2(i) (W[i&15]+=s1(W[(i-2)&15])+W[(i-7)&15]+s0(W[(i-15)&15])) +#define blk2(i) (\ + W[ i & 15] += \ + s1(W[(i-2) & 15])+ \ + W[(i-7) & 15] + \ + s0(W[(i-15) & 15]) \ + ) -#define Ch(x,y,z) (z^(x&(y^z))) -#define Maj(x,y,z) ((x&y)|(z&(x|y))) +#define Ch(x,y,z) (z ^ (x & (y ^ z))) +#define Maj(x,y,z) ((x & y) | (z & (x | y))) -#define a(i) T[(0-i)&7] -#define b(i) T[(1-i)&7] -#define c(i) T[(2-i)&7] -#define d(i) T[(3-i)&7] -#define e(i) T[(4-i)&7] -#define f(i) T[(5-i)&7] -#define g(i) T[(6-i)&7] -#define h(i) T[(7-i)&7] +#define a(i) T[(0-i) & 7] +#define b(i) T[(1-i) & 7] +#define c(i) T[(2-i) & 7] +#define d(i) T[(3-i) & 7] +#define e(i) T[(4-i) & 7] +#define f(i) T[(5-i) & 7] +#define g(i) T[(6-i) & 7] +#define h(i) T[(7-i) & 7] -#define S0(x) (rotrFixed64(x,28)^rotrFixed64(x,34)^rotrFixed64(x,39)) -#define S1(x) (rotrFixed64(x,14)^rotrFixed64(x,18)^rotrFixed64(x,41)) -#define s0(x) (rotrFixed64(x,1)^rotrFixed64(x,8)^(x>>7)) -#define s1(x) (rotrFixed64(x,19)^rotrFixed64(x,61)^(x>>6)) +#define S0(x) (rotrFixed64(x,28) ^ rotrFixed64(x,34) ^ rotrFixed64(x,39)) +#define S1(x) (rotrFixed64(x,14) ^ rotrFixed64(x,18) ^ rotrFixed64(x,41)) +#define s0(x) (rotrFixed64(x,1) ^ rotrFixed64(x,8) ^ (x>>7)) +#define s1(x) (rotrFixed64(x,19) ^ rotrFixed64(x,61) ^ (x>>6)) -#define R(i) h(i)+=S1(e(i))+Ch(e(i),f(i),g(i))+K[i+j]+(j?blk2(i):blk0(i));\ - d(i)+=h(i);h(i)+=S0(a(i))+Maj(a(i),b(i),c(i)) +#define R(i) \ + h(i) += S1(e(i)) + Ch(e(i),f(i),g(i)) + K[i+j] + (j ? blk2(i) : blk0(i)); \ + d(i) += h(i); \ + h(i) += S0(a(i)) + Maj(a(i),b(i),c(i)) -static int _Transform(wc_Sha512* sha512) +static int _Transform_Sha512(wc_Sha512* sha512) { const word64* K = K512; - word32 j; word64 T[8]; - #ifdef WOLFSSL_SMALL_STACK word64* W; W = (word64*) XMALLOC(sizeof(word64) * 16, NULL, DYNAMIC_TYPE_TMP_BUFFER); @@ -447,7 +458,7 @@ static int _Transform(wc_Sha512* sha512) /* Copy digest to working vars */ XMEMCPY(T, sha512->digest, sizeof(T)); -#ifdef USE_SLOW_SHA2 +#ifdef USE_SLOW_SHA512 /* over twice as small, but 50% slower */ /* 80 operations, not unrolled */ for (j = 0; j < 80; j += 16) { @@ -464,10 +475,9 @@ static int _Transform(wc_Sha512* sha512) R( 8); R( 9); R(10); R(11); R(12); R(13); R(14); R(15); } -#endif /* USE_SLOW_SHA2 */ +#endif /* USE_SLOW_SHA512 */ /* Add the working vars back into digest */ - sha512->digest[0] += a(0); sha512->digest[1] += b(0); sha512->digest[2] += c(0); @@ -506,9 +516,7 @@ static INLINE int Sha512Update(wc_Sha512* sha512, const byte* data, word32 len) if (sha512->buffLen >= WC_SHA512_BLOCK_SIZE) return BUFFER_E; - SAVE_XMM_YMM; /* for Intel AVX */ - - while (len) { + if (sha512->buffLen > 0) { word32 add = min(len, WC_SHA512_BLOCK_SIZE - sha512->buffLen); XMEMCPY(&local[sha512->buffLen], data, add); @@ -523,17 +531,76 @@ static INLINE int Sha512Update(wc_Sha512* sha512, const byte* data, word32 len) #endif { ByteReverseWords64(sha512->buffer, sha512->buffer, - WC_SHA512_BLOCK_SIZE); + WC_SHA512_BLOCK_SIZE); } #endif - ret = Transform(sha512); + ret = Transform_Sha512(sha512); + if (ret == 0) { + AddLength(sha512, WC_SHA512_BLOCK_SIZE); + sha512->buffLen = 0; + } + else + len = 0; + } + } + +#if defined(HAVE_INTEL_AVX1) || defined(HAVE_INTEL_AVX2) + if (Transform_Sha512_Len_p != NULL) { + word32 blocksLen = len & ~(WC_SHA512_BLOCK_SIZE-1); + + if (blocksLen > 0) { + AddLength(sha512, blocksLen); + sha512->data = data; + /* Byte reversal performed in function if required. */ + Transform_Sha512_Len(sha512, blocksLen); + data += blocksLen; + len -= blocksLen; + } + } + else +#endif +#if !defined(LITTLE_ENDIAN_ORDER) || defined(FREESCALE_MMCAU_SHA) || \ + defined(HAVE_INTEL_AVX1) || defined(HAVE_INTEL_AVX2) + { + word32 blocksLen = len & ~(WC_SHA512_BLOCK_SIZE-1); + + AddLength(sha512, blocksLen); + while (len >= WC_SHA512_BLOCK_SIZE) { + XMEMCPY(local, data, WC_SHA512_BLOCK_SIZE); + + data += WC_SHA512_BLOCK_SIZE; + len -= WC_SHA512_BLOCK_SIZE; + + /* Byte reversal performed in function if required. */ + ret = Transform_Sha512(sha512); if (ret != 0) break; - - AddLength(sha512, WC_SHA512_BLOCK_SIZE); - sha512->buffLen = 0; } } +#else + { + word32 blocksLen = len & ~(WC_SHA512_BLOCK_SIZE-1); + + AddLength(sha512, blocksLen); + while (len >= WC_SHA512_BLOCK_SIZE) { + XMEMCPY(local, data, WC_SHA512_BLOCK_SIZE); + + data += WC_SHA512_BLOCK_SIZE; + len -= WC_SHA512_BLOCK_SIZE; + + ByteReverseWords64(sha512->buffer, sha512->buffer, + WC_SHA512_BLOCK_SIZE); + ret = Transform_Sha512(sha512); + if (ret != 0) + break; + } + } +#endif + + if (len > 0) { + XMEMCPY(local, data, len); + sha512->buffLen = len; + } return ret; } @@ -565,7 +632,6 @@ static INLINE int Sha512Final(wc_Sha512* sha512) return BAD_FUNC_ARG; } - SAVE_XMM_YMM ; /* for Intel AVX */ AddLength(sha512, sha512->buffLen); /* before adding pads */ local[sha512->buffLen++] = 0x80; /* add 1 */ @@ -583,7 +649,7 @@ static INLINE int Sha512Final(wc_Sha512* sha512) WC_SHA512_BLOCK_SIZE); } #endif /* LITTLE_ENDIAN_ORDER */ - ret = Transform(sha512); + ret = Transform_Sha512(sha512); if (ret != 0) return ret; @@ -613,7 +679,7 @@ static INLINE int Sha512Final(wc_Sha512* sha512) &(sha512->buffer[WC_SHA512_BLOCK_SIZE / sizeof(word64) - 2]), WC_SHA512_BLOCK_SIZE - WC_SHA512_PAD_SIZE); #endif - ret = Transform(sha512); + ret = Transform_Sha512(sha512); if (ret != 0) return ret; @@ -669,674 +735,1799 @@ void wc_Sha512Free(wc_Sha512* sha512) #if defined(HAVE_INTEL_AVX1) -#define Rx_1(i) h(i)+=S1(e(i))+Ch(e(i),f(i),g(i))+K[i+j] + W_X[i]; -#define Rx_2(i) d(i)+=h(i); -#define Rx_3(i) h(i)+=S0(a(i))+Maj(a(i),b(i),c(i)); +static word64 mBYTE_FLIP_MASK[] = { 0x0001020304050607, 0x08090a0b0c0d0e0f }; + +#define W_0 xmm0 +#define W_2 xmm1 +#define W_4 xmm2 +#define W_6 xmm3 +#define W_8 xmm4 +#define W_10 xmm5 +#define W_12 xmm6 +#define W_14 xmm7 + +#define W_M15 xmm12 +#define W_M7 xmm13 +#define MASK xmm14 + +#define XTMP1 xmm8 +#define XTMP2 xmm9 +#define XTMP3 xmm10 +#define XTMP4 xmm11 + +#define XMM_REGS \ + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", \ + "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", "xmm15" + +#define _VPALIGNR(dest, src1, src2, bits) \ + "vpalignr $"#bits", %%"#src2", %%"#src1", %%"#dest"\n\t" +#define VPALIGNR(dest, src1, src2, bits) \ + _VPALIGNR(dest, src1, src2, bits) + +#define _V_SHIFT_R(dest, src, bits) \ + "vpsrlq $"#bits", %%"#src", %%"#dest"\n\t" +#define V_SHIFT_R(dest, src, bits) \ + _V_SHIFT_R(dest, src, bits) + +#define _V_SHIFT_L(dest, src, bits) \ + "vpsllq $"#bits", %%"#src", %%"#dest"\n\t" +#define V_SHIFT_L(dest, src, bits) \ + _V_SHIFT_L(dest, src, bits) + +#define _V_ADD(dest, src1, src2) \ + "vpaddq %%"#src1", %%"#src2", %%"#dest"\n\t" +#define V_ADD(dest, src1, src2) \ + _V_ADD(dest, src1, src2) + +#define _V_XOR(dest, src1, src2) \ + "vpxor %%"#src1", %%"#src2", %%"#dest"\n\t" +#define V_XOR(dest, src1, src2) \ + _V_XOR(dest, src1, src2) + +#define _V_OR(dest, src1, src2) \ + "vpor %%"#src1", %%"#src2", %%"#dest"\n\t" +#define V_OR(dest, src1, src2) \ + _V_OR(dest, src1, src2) + +#define RA %%r8 +#define RB %%r9 +#define RC %%r10 +#define RD %%r11 +#define RE %%r12 +#define RF %%r13 +#define RG %%r14 +#define RH %%r15 + +#define STATE_REGS "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15" + +#define L1 "%%rax" +#define L2 "%%rcx" +#define L3 "%%rdx" +#define L4 "%%rbx" +#define WX "%%rsp" + +#define WORK_REGS "rax", "rbx", "rcx", "rdx" + +#define RND_0_1(a,b,c,d,e,f,g,h,i) \ + /* L1 = e >>> 23 */ \ + "rorq $23, "L1"\n\t" \ + +#define RND_0_2(a,b,c,d,e,f,g,h,i) \ + /* L3 = a */ \ + "movq "#a", "L3"\n\t" \ + /* L2 = f */ \ + "movq "#f", "L2"\n\t" \ + /* h += W_X[i] */ \ + "addq ("#i")*8("WX"), "#h"\n\t" \ + /* L2 = f ^ g */ \ + "xorq "#g", "L2"\n\t" \ + +#define RND_0_2_A(a,b,c,d,e,f,g,h,i) \ + /* L3 = a */ \ + "movq "#a", "L3"\n\t" \ + /* L2 = f */ \ + "movq "#f", "L2"\n\t" \ + +#define RND_0_2_B(a,b,c,d,e,f,g,h,i) \ + /* h += W_X[i] */ \ + "addq ("#i")*8("WX"), "#h"\n\t" \ + /* L2 = f ^ g */ \ + "xorq "#g", "L2"\n\t" \ + +#define RND_0_3(a,b,c,d,e,f,g,h,i) \ + /* L1 = (e >>> 23) ^ e */ \ + "xorq "#e", "L1"\n\t" \ + /* L2 = (f ^ g) & e */ \ + "andq "#e", "L2"\n\t" \ + +#define RND_0_4(a,b,c,d,e,f,g,h,i) \ + /* L1 = ((e >>> 23) ^ e) >>> 4 */ \ + "rorq $4, "L1"\n\t" \ + /* L2 = ((f ^ g) & e) ^ g */ \ + "xorq "#g", "L2"\n\t" \ + +#define RND_0_5(a,b,c,d,e,f,g,h,i) \ + /* L1 = (((e >>> 23) ^ e) >>> 4) ^ e */ \ + "xorq "#e", "L1"\n\t" \ + /* h += Ch(e,f,g) */ \ + "addq "L2", "#h"\n\t" \ + +#define RND_0_6(a,b,c,d,e,f,g,h,i) \ + /* L1 = ((((e >>> 23) ^ e) >>> 4) ^ e) >>> 14 */ \ + "rorq $14, "L1"\n\t" \ + /* L3 = a ^ b */ \ + "xorq "#b", "L3"\n\t" \ + +#define RND_0_7(a,b,c,d,e,f,g,h,i) \ + /* h += Sigma1(e) */ \ + "addq "L1", "#h"\n\t" \ + /* L2 = a */ \ + "movq "#a", "L2"\n\t" \ + +#define RND_0_8(a,b,c,d,e,f,g,h,i) \ + /* L4 = (a ^ b) & (b ^ c) */ \ + "andq "L3", "L4"\n\t" \ + /* L2 = a >>> 5 */ \ + "rorq $5, "L2"\n\t" \ + +#define RND_0_9(a,b,c,d,e,f,g,h,i) \ + /* L2 = (a >>> 5) ^ a */ \ + "xorq "#a", "L2"\n\t" \ + /* L4 = ((a ^ b) & (b ^ c) ^ b */ \ + "xorq "#b", "L4"\n\t" \ + +#define RND_0_10(a,b,c,d,e,f,g,h,i) \ + /* L2 = ((a >>> 5) ^ a) >>> 6 */ \ + "rorq $6, "L2"\n\t" \ + /* d += h */ \ + "addq "#h", "#d"\n\t" \ + +#define RND_0_11(a,b,c,d,e,f,g,h,i) \ + /* L2 = (((a >>> 5) ^ a) >>> 6) ^ a */ \ + "xorq "#a", "L2"\n\t" \ + /* h += Sigma0(a) */ \ + "addq "L4", "#h"\n\t" \ + +#define RND_0_12(a,b,c,d,e,f,g,h,i) \ + /* L2 = ((((a >>> 5) ^ a) >>> 6) ^ a) >>> 28 */ \ + "rorq $28, "L2"\n\t" \ + /* d (= e next RND) */ \ + "movq "#d", "L1"\n\t" \ + /* h += Maj(a,b,c) */ \ + "addq "L2", "#h"\n\t" \ + +#define RND_1_1(a,b,c,d,e,f,g,h,i) \ + /* L1 = e >>> 23 */ \ + "rorq $23, "L1"\n\t" \ + +#define RND_1_2(a,b,c,d,e,f,g,h,i) \ + /* L4 = a */ \ + "movq "#a", "L4"\n\t" \ + /* L2 = f */ \ + "movq "#f", "L2"\n\t" \ + /* h += W_X[i] */ \ + "addq ("#i")*8("WX"), "#h"\n\t" \ + /* L2 = f ^ g */ \ + "xorq "#g", "L2"\n\t" \ + +#define RND_1_2_A(a,b,c,d,e,f,g,h,i) \ + /* L4 = a */ \ + "movq "#a", "L4"\n\t" \ + /* L2 = f */ \ + "movq "#f", "L2"\n\t" \ + +#define RND_1_2_B(a,b,c,d,e,f,g,h,i) \ + /* h += W_X[i] */ \ + "addq ("#i")*8("WX"), "#h"\n\t" \ + /* L2 = f ^ g */ \ + "xorq "#g", "L2"\n\t" \ + +#define RND_1_3(a,b,c,d,e,f,g,h,i) \ + /* L1 = (e >>> 23) ^ e */ \ + "xorq "#e", "L1"\n\t" \ + /* L2 = (f ^ g) & e */ \ + "andq "#e", "L2"\n\t" \ + +#define RND_1_4(a,b,c,d,e,f,g,h,i) \ + /* ((e >>> 23) ^ e) >>> 4 */ \ + "rorq $4, "L1"\n\t" \ + /* ((f ^ g) & e) ^ g */ \ + "xorq "#g", "L2"\n\t" \ + +#define RND_1_5(a,b,c,d,e,f,g,h,i) \ + /* (((e >>> 23) ^ e) >>> 4) ^ e */ \ + "xorq "#e", "L1"\n\t" \ + /* h += Ch(e,f,g) */ \ + "addq "L2", "#h"\n\t" \ + +#define RND_1_6(a,b,c,d,e,f,g,h,i) \ + /* L1 = ((((e >>> 23) ^ e) >>> 4) ^ e) >>> 14 */ \ + "rorq $14, "L1"\n\t" \ + /* L4 = a ^ b */ \ + "xorq "#b", "L4"\n\t" \ + +#define RND_1_7(a,b,c,d,e,f,g,h,i) \ + /* h += Sigma1(e) */ \ + "addq "L1", "#h"\n\t" \ + /* L2 = a */ \ + "movq "#a", "L2"\n\t" \ + +#define RND_1_8(a,b,c,d,e,f,g,h,i) \ + /* L3 = (a ^ b) & (b ^ c) */ \ + "andq "L4", "L3"\n\t" \ + /* L2 = a >>> 5 */ \ + "rorq $5, "L2"\n\t" \ + +#define RND_1_9(a,b,c,d,e,f,g,h,i) \ + /* L2 = (a >>> 5) ^ a */ \ + "xorq "#a", "L2"\n\t" \ + /* L3 = ((a ^ b) & (b ^ c) ^ b */ \ + "xorq "#b", "L3"\n\t" \ + +#define RND_1_10(a,b,c,d,e,f,g,h,i) \ + /* L2 = ((a >>> 5) ^ a) >>> 6 */ \ + "rorq $6, "L2"\n\t" \ + /* d += h */ \ + "addq "#h", "#d"\n\t" \ + +#define RND_1_11(a,b,c,d,e,f,g,h,i) \ + /* L2 = (((a >>> 5) ^ a) >>> 6) ^ a */ \ + "xorq "#a", "L2"\n\t" \ + /* h += Sigma0(a) */ \ + "addq "L3", "#h"\n\t" \ + +#define RND_1_12(a,b,c,d,e,f,g,h,i) \ + /* L2 = ((((a >>> 5) ^ a) >>> 6) ^ a) >>> 28 */ \ + "rorq $28, "L2"\n\t" \ + /* d (= e next RND) */ \ + "movq "#d", "L1"\n\t" \ + /* h += Maj(a,b,c) */ \ + "addq "L2", "#h"\n\t" \ + + +#define MsgSched2(W_0,W_2,W_4,W_6,W_8,W_10,W_12,W_14,a,b,c,d,e,f,g,h,i) \ + RND_0_1(a,b,c,d,e,f,g,h,i) \ + VPALIGNR(W_M15, W_2, W_0, 8) \ + VPALIGNR(W_M7, W_10, W_8, 8) \ + RND_0_2(a,b,c,d,e,f,g,h,i) \ + V_SHIFT_R(XTMP1, W_M15, 1) \ + V_SHIFT_L(XTMP2, W_M15, 63) \ + RND_0_3(a,b,c,d,e,f,g,h,i) \ + RND_0_4(a,b,c,d,e,f,g,h,i) \ + V_SHIFT_R(XTMP3, W_M15, 8) \ + V_SHIFT_L(XTMP4, W_M15, 56) \ + RND_0_5(a,b,c,d,e,f,g,h,i) \ + RND_0_6(a,b,c,d,e,f,g,h,i) \ + V_OR(XTMP1, XTMP2, XTMP1) \ + V_OR(XTMP3, XTMP4, XTMP3) \ + RND_0_7(a,b,c,d,e,f,g,h,i) \ + RND_0_8(a,b,c,d,e,f,g,h,i) \ + V_SHIFT_R(XTMP4, W_M15, 7) \ + V_XOR(XTMP1, XTMP3, XTMP1) \ + RND_0_9(a,b,c,d,e,f,g,h,i) \ + RND_0_10(a,b,c,d,e,f,g,h,i) \ + V_XOR(XTMP1, XTMP4, XTMP1) \ + V_ADD(W_0, W_0, W_M7) \ + RND_0_11(a,b,c,d,e,f,g,h,i) \ + RND_0_12(a,b,c,d,e,f,g,h,i) \ + RND_1_1(h,a,b,c,d,e,f,g,i+1) \ + V_ADD(W_0, W_0, XTMP1) \ + RND_1_2(h,a,b,c,d,e,f,g,i+1) \ + V_SHIFT_R(XTMP1, W_14, 19) \ + V_SHIFT_L(XTMP2, W_14, 45) \ + RND_1_3(h,a,b,c,d,e,f,g,i+1) \ + RND_1_4(h,a,b,c,d,e,f,g,i+1) \ + V_SHIFT_R(XTMP3, W_14, 61) \ + V_SHIFT_L(XTMP4, W_14, 3) \ + RND_1_5(h,a,b,c,d,e,f,g,i+1) \ + RND_1_6(h,a,b,c,d,e,f,g,i+1) \ + RND_1_7(h,a,b,c,d,e,f,g,i+1) \ + V_OR(XTMP1, XTMP2, XTMP1) \ + V_OR(XTMP3, XTMP4, XTMP3) \ + RND_1_8(h,a,b,c,d,e,f,g,i+1) \ + RND_1_9(h,a,b,c,d,e,f,g,i+1) \ + V_XOR(XTMP1, XTMP3, XTMP1) \ + V_SHIFT_R(XTMP4, W_14, 6) \ + RND_1_10(h,a,b,c,d,e,f,g,i+1) \ + RND_1_11(h,a,b,c,d,e,f,g,i+1) \ + V_XOR(XTMP1, XTMP4, XTMP1) \ + RND_1_12(h,a,b,c,d,e,f,g,i+1) \ + V_ADD(W_0, W_0, XTMP1) \ + +#define RND_ALL_2(a, b, c, d, e, f, g, h, i) \ + RND_0_1 (a, b, c, d, e, f, g, h, i ) \ + RND_0_2 (a, b, c, d, e, f, g, h, i ) \ + RND_0_3 (a, b, c, d, e, f, g, h, i ) \ + RND_0_4 (a, b, c, d, e, f, g, h, i ) \ + RND_0_5 (a, b, c, d, e, f, g, h, i ) \ + RND_0_6 (a, b, c, d, e, f, g, h, i ) \ + RND_0_7 (a, b, c, d, e, f, g, h, i ) \ + RND_0_8 (a, b, c, d, e, f, g, h, i ) \ + RND_0_9 (a, b, c, d, e, f, g, h, i ) \ + RND_0_10(a, b, c, d, e, f, g, h, i ) \ + RND_0_11(a, b, c, d, e, f, g, h, i ) \ + RND_0_12(a, b, c, d, e, f, g, h, i ) \ + RND_1_1 (h, a, b, c, d, e, f, g, i+1) \ + RND_1_2 (h, a, b, c, d, e, f, g, i+1) \ + RND_1_3 (h, a, b, c, d, e, f, g, i+1) \ + RND_1_4 (h, a, b, c, d, e, f, g, i+1) \ + RND_1_5 (h, a, b, c, d, e, f, g, i+1) \ + RND_1_6 (h, a, b, c, d, e, f, g, i+1) \ + RND_1_7 (h, a, b, c, d, e, f, g, i+1) \ + RND_1_8 (h, a, b, c, d, e, f, g, i+1) \ + RND_1_9 (h, a, b, c, d, e, f, g, i+1) \ + RND_1_10(h, a, b, c, d, e, f, g, i+1) \ + RND_1_11(h, a, b, c, d, e, f, g, i+1) \ + RND_1_12(h, a, b, c, d, e, f, g, i+1) + #if defined(HAVE_INTEL_RORX) - #define Rx_RORX_1(i) h(i)+=S1_RORX(e(i))+Ch(e(i),f(i),g(i))+K[i+j] + W_X[i]; - #define Rx_RORX_2(i) d(i)+=h(i); - #define Rx_RORX_3(i) h(i)+=S0_RORX(a(i))+Maj(a(i),b(i),c(i)); -#endif /* HAVE_INTEL_RORX */ +#define RND_RORX_0_1(a, b, c, d, e, f, g, h, i) \ + /* L1 = e>>>14 */ \ + "rorxq $14, "#e", "L1"\n\t" \ + /* L2 = e>>>18 */ \ + "rorxq $18, "#e", "L2"\n\t" \ + /* Prev RND: h += Maj(a,b,c) */ \ + "addq "L3", "#a"\n\t" \ + +#define RND_RORX_0_2(a, b, c, d, e, f, g, h, i) \ + /* h += w_k */ \ + "addq ("#i")*8("WX"), "#h"\n\t" \ + /* L3 = f */ \ + "movq "#f", "L3"\n\t" \ + /* L2 = (e>>>14) ^ (e>>>18) */ \ + "xorq "L1", "L2"\n\t" \ + +#define RND_RORX_0_3(a, b, c, d, e, f, g, h, i) \ + /* L3 = f ^ g */ \ + "xorq "#g", "L3"\n\t" \ + /* L1 = e>>>41 */ \ + "rorxq $41, "#e", "L1"\n\t" \ + /* L1 = Sigma1(e) */ \ + "xorq "L2", "L1"\n\t" \ + +#define RND_RORX_0_4(a, b, c, d, e, f, g, h, i) \ + /* L3 = (f ^ g) & e */ \ + "andq "#e", "L3"\n\t" \ + /* h += Sigma1(e) */ \ + "addq "L1", "#h"\n\t" \ + /* L1 = a>>>28 */ \ + "rorxq $28, "#a", "L1"\n\t" \ + +#define RND_RORX_0_5(a, b, c, d, e, f, g, h, i) \ + /* L2 = a>>>34 */ \ + "rorxq $34, "#a", "L2"\n\t" \ + /* L3 = Ch(e,f,g) */ \ + "xorq "#g", "L3"\n\t" \ + /* L2 = (a>>>28) ^ (a>>>34) */ \ + "xorq "L1", "L2"\n\t" \ + +#define RND_RORX_0_6(a, b, c, d, e, f, g, h, i) \ + /* L1 = a>>>39 */ \ + "rorxq $39, "#a", "L1"\n\t" \ + /* h += Ch(e,f,g) */ \ + "addq "L3", "#h"\n\t" \ + /* L1 = Sigma0(a) */ \ + "xorq "L2", "L1"\n\t" \ + +#define RND_RORX_0_7(a, b, c, d, e, f, g, h, i) \ + /* L3 = b */ \ + "movq "#b", "L3"\n\t" \ + /* d += h + w_k + Sigma1(e) + Ch(e,f,g) */ \ + "addq "#h", "#d"\n\t" \ + /* L3 = a ^ b */ \ + "xorq "#a", "L3"\n\t" \ + +#define RND_RORX_0_8(a, b, c, d, e, f, g, h, i) \ + /* L4 = (a ^ b) & (b ^ c) */ \ + "andq "L3", "L4"\n\t" \ + /* h += Sigma0(a) */ \ + "addq "L1", "#h"\n\t" \ + /* L4 = Maj(a,b,c) */ \ + "xorq "#b", "L4"\n\t" \ + +#define RND_RORX_1_1(a, b, c, d, e, f, g, h, i) \ + /* L1 = e>>>14 */ \ + "rorxq $14, "#e", "L1"\n\t" \ + /* L2 = e>>>18 */ \ + "rorxq $18, "#e", "L2"\n\t" \ + /* Prev RND: h += Maj(a,b,c) */ \ + "addq "L4", "#a"\n\t" \ + +#define RND_RORX_1_2(a, b, c, d, e, f, g, h, i) \ + /* h += w_k */ \ + "addq ("#i")*8("WX"), "#h"\n\t" \ + /* L4 = f */ \ + "movq "#f", "L4"\n\t" \ + /* L2 = (e>>>14) ^ (e>>>18) */ \ + "xorq "L1", "L2"\n\t" \ + +#define RND_RORX_1_3(a, b, c, d, e, f, g, h, i) \ + /* L4 = f ^ g */ \ + "xorq "#g", "L4"\n\t" \ + /* L1 = e>>>41 */ \ + "rorxq $41, "#e", "L1"\n\t" \ + /* L1 = Sigma1(e) */ \ + "xorq "L2", "L1"\n\t" \ + +#define RND_RORX_1_4(a, b, c, d, e, f, g, h, i) \ + /* L4 = (f ^ g) & e */ \ + "andq "#e", "L4"\n\t" \ + /* h += Sigma1(e) */ \ + "addq "L1", "#h"\n\t" \ + /* L1 = a>>>28 */ \ + "rorxq $28, "#a", "L1"\n\t" \ + +#define RND_RORX_1_5(a, b, c, d, e, f, g, h, i) \ + /* L2 = a>>>34 */ \ + "rorxq $34, "#a", "L2"\n\t" \ + /* L4 = Ch(e,f,g) */ \ + "xorq "#g", "L4"\n\t" \ + /* L2 = (a>>>28) ^ (a>>>34) */ \ + "xorq "L1", "L2"\n\t" \ + +#define RND_RORX_1_6(a, b, c, d, e, f, g, h, i) \ + /* L1 = a>>>39 */ \ + "rorxq $39, "#a", "L1"\n\t" \ + /* h += Ch(e,f,g) */ \ + "addq "L4", "#h"\n\t" \ + /* L1 = Sigma0(a) */ \ + "xorq "L2", "L1"\n\t" \ + +#define RND_RORX_1_7(a, b, c, d, e, f, g, h, i) \ + /* L4 = b */ \ + "movq "#b", "L4"\n\t" \ + /* d += h + w_k + Sigma1(e) + Ch(e,f,g) */ \ + "addq "#h", "#d"\n\t" \ + /* L4 = a ^ b */ \ + "xorq "#a", "L4"\n\t" \ + +#define RND_RORX_1_8(a, b, c, d, e, f, g, h, i) \ + /* L2 = (a ^ b) & (b ^ c) */ \ + "andq "L4", "L3"\n\t" \ + /* h += Sigma0(a) */ \ + "addq "L1", "#h"\n\t" \ + /* L3 = Maj(a,b,c) */ \ + "xorq "#b", "L3"\n\t" \ + +#define RND_RORX_ALL_2(a, b, c, d, e, f, g, h, i) \ + RND_RORX_0_1(a, b, c, d, e, f, g, h, i+0) \ + RND_RORX_0_2(a, b, c, d, e, f, g, h, i+0) \ + RND_RORX_0_3(a, b, c, d, e, f, g, h, i+0) \ + RND_RORX_0_4(a, b, c, d, e, f, g, h, i+0) \ + RND_RORX_0_5(a, b, c, d, e, f, g, h, i+0) \ + RND_RORX_0_6(a, b, c, d, e, f, g, h, i+0) \ + RND_RORX_0_7(a, b, c, d, e, f, g, h, i+0) \ + RND_RORX_0_8(a, b, c, d, e, f, g, h, i+0) \ + RND_RORX_1_1(h, a, b, c, d, e, f, g, i+1) \ + RND_RORX_1_2(h, a, b, c, d, e, f, g, i+1) \ + RND_RORX_1_3(h, a, b, c, d, e, f, g, i+1) \ + RND_RORX_1_4(h, a, b, c, d, e, f, g, i+1) \ + RND_RORX_1_5(h, a, b, c, d, e, f, g, i+1) \ + RND_RORX_1_6(h, a, b, c, d, e, f, g, i+1) \ + RND_RORX_1_7(h, a, b, c, d, e, f, g, i+1) \ + RND_RORX_1_8(h, a, b, c, d, e, f, g, i+1) \ + +#define RND_RORX_ALL_4(a, b, c, d, e, f, g, h, i) \ + RND_RORX_ALL_2(a, b, c, d, e, f, g, h, i+0) \ + RND_RORX_ALL_2(g, h, a, b, c, d, e, f, i+2) + +#define MsgSched_RORX(W_0,W_2,W_4,W_6,W_8,W_10,W_12,W_14,a,b,c,d,e,f,g,h,i) \ + RND_RORX_0_1(a,b,c,d,e,f,g,h,i) \ + VPALIGNR(W_M15, W_2, W_0, 8) \ + VPALIGNR(W_M7, W_10, W_8, 8) \ + RND_RORX_0_2(a,b,c,d,e,f,g,h,i) \ + V_SHIFT_R(XTMP1, W_M15, 1) \ + V_SHIFT_L(XTMP2, W_M15, 63) \ + RND_RORX_0_3(a,b,c,d,e,f,g,h,i) \ + V_SHIFT_R(XTMP3, W_M15, 8) \ + V_SHIFT_L(XTMP4, W_M15, 56) \ + RND_RORX_0_4(a,b,c,d,e,f,g,h,i) \ + V_OR(XTMP1, XTMP2, XTMP1) \ + V_OR(XTMP3, XTMP4, XTMP3) \ + RND_RORX_0_5(a,b,c,d,e,f,g,h,i) \ + V_SHIFT_R(XTMP4, W_M15, 7) \ + V_XOR(XTMP1, XTMP3, XTMP1) \ + RND_RORX_0_6(a,b,c,d,e,f,g,h,i) \ + V_XOR(XTMP1, XTMP4, XTMP1) \ + V_ADD(W_0, W_0, W_M7) \ + RND_RORX_0_7(a,b,c,d,e,f,g,h,i) \ + RND_RORX_0_8(a,b,c,d,e,f,g,h,i) \ + V_ADD(W_0, W_0, XTMP1) \ + RND_RORX_1_1(h,a,b,c,d,e,f,g,i+1) \ + V_SHIFT_R(XTMP1, W_14, 19) \ + V_SHIFT_L(XTMP2, W_14, 45) \ + RND_RORX_1_2(h,a,b,c,d,e,f,g,i+1) \ + V_SHIFT_R(XTMP3, W_14, 61) \ + V_SHIFT_L(XTMP4, W_14, 3) \ + RND_RORX_1_3(h,a,b,c,d,e,f,g,i+1) \ + V_OR(XTMP1, XTMP2, XTMP1) \ + V_OR(XTMP3, XTMP4, XTMP3) \ + RND_RORX_1_4(h,a,b,c,d,e,f,g,i+1) \ + RND_RORX_1_5(h,a,b,c,d,e,f,g,i+1) \ + V_XOR(XTMP1, XTMP3, XTMP1) \ + V_SHIFT_R(XTMP4, W_14, 6) \ + RND_RORX_1_6(h,a,b,c,d,e,f,g,i+1) \ + RND_RORX_1_7(h,a,b,c,d,e,f,g,i+1) \ + V_XOR(XTMP1, XTMP4, XTMP1) \ + RND_RORX_1_8(h,a,b,c,d,e,f,g,i+1) \ + V_ADD(W_0, W_0, XTMP1) \ + +#endif + +#define _INIT_MASK(mask) \ + "vmovdqu %[mask], %%"#mask"\n\t" +#define INIT_MASK(mask) \ + _INIT_MASK(mask) + +#define _LOAD_W_2(i1, i2, xmm1, xmm2, mask, reg) \ + "vmovdqu "#i1"*16(%%"#reg"), %%"#xmm1"\n\t" \ + "vmovdqu "#i2"*16(%%"#reg"), %%"#xmm2"\n\t" \ + "vpshufb %%"#mask", %%"#xmm1", %%"#xmm1"\n\t" \ + "vpshufb %%"#mask", %%"#xmm2", %%"#xmm2"\n\t" +#define LOAD_W_2(i1, i2, xmm1, xmm2, mask, reg) \ + _LOAD_W_2(i1, i2, xmm1, xmm2, mask, reg) + +#define LOAD_W(mask, reg) \ + /* X0..3(xmm4..7), W[0..15] = buffer[0.15]; */ \ + LOAD_W_2(0, 1, W_0 , W_2 , mask, reg) \ + LOAD_W_2(2, 3, W_4 , W_6 , mask, reg) \ + LOAD_W_2(4, 5, W_8 , W_10, mask, reg) \ + LOAD_W_2(6, 7, W_12, W_14, mask, reg) + +#define _SET_W_X_2(xmm0, xmm1, reg, i) \ + "vpaddq "#i"+ 0(%%"#reg"), %%"#xmm0", %%xmm8\n\t" \ + "vpaddq "#i"+16(%%"#reg"), %%"#xmm1", %%xmm9\n\t" \ + "vmovdqu %%xmm8, "#i"+ 0("WX")\n\t" \ + "vmovdqu %%xmm9, "#i"+16("WX")\n\t" \ + +#define SET_W_X_2(xmm0, xmm1, reg, i) \ + _SET_W_X_2(xmm0, xmm1, reg, i) + +#define SET_W_X(reg) \ + SET_W_X_2(W_0 , W_2 , reg, 0) \ + SET_W_X_2(W_4 , W_6 , reg, 32) \ + SET_W_X_2(W_8 , W_10, reg, 64) \ + SET_W_X_2(W_12, W_14, reg, 96) + +#define LOAD_DIGEST() \ + "movq (%[sha512]), %%r8 \n\t" \ + "movq 8(%[sha512]), %%r9 \n\t" \ + "movq 16(%[sha512]), %%r10\n\t" \ + "movq 24(%[sha512]), %%r11\n\t" \ + "movq 32(%[sha512]), %%r12\n\t" \ + "movq 40(%[sha512]), %%r13\n\t" \ + "movq 48(%[sha512]), %%r14\n\t" \ + "movq 56(%[sha512]), %%r15\n\t" + +#define STORE_ADD_DIGEST() \ + "addq %%r8, (%[sha512])\n\t" \ + "addq %%r9, 8(%[sha512])\n\t" \ + "addq %%r10, 16(%[sha512])\n\t" \ + "addq %%r11, 24(%[sha512])\n\t" \ + "addq %%r12, 32(%[sha512])\n\t" \ + "addq %%r13, 40(%[sha512])\n\t" \ + "addq %%r14, 48(%[sha512])\n\t" \ + "addq %%r15, 56(%[sha512])\n\t" + +#define ADD_DIGEST() \ + "addq (%[sha512]), %%r8 \n\t" \ + "addq 8(%[sha512]), %%r9 \n\t" \ + "addq 16(%[sha512]), %%r10\n\t" \ + "addq 24(%[sha512]), %%r11\n\t" \ + "addq 32(%[sha512]), %%r12\n\t" \ + "addq 40(%[sha512]), %%r13\n\t" \ + "addq 48(%[sha512]), %%r14\n\t" \ + "addq 56(%[sha512]), %%r15\n\t" + +#define STORE_DIGEST() \ + "movq %%r8, (%[sha512])\n\t" \ + "movq %%r9, 8(%[sha512])\n\t" \ + "movq %%r10, 16(%[sha512])\n\t" \ + "movq %%r11, 24(%[sha512])\n\t" \ + "movq %%r12, 32(%[sha512])\n\t" \ + "movq %%r13, 40(%[sha512])\n\t" \ + "movq %%r14, 48(%[sha512])\n\t" \ + "movq %%r15, 56(%[sha512])\n\t" #endif /* HAVE_INTEL_AVX1 */ -#if defined(HAVE_INTEL_AVX2) -#define Ry_1(i, w) h(i)+=S1(e(i))+Ch(e(i),f(i),g(i))+K[i+j] + w; -#define Ry_2(i, w) d(i)+=h(i); -#define Ry_3(i, w) h(i)+=S0(a(i))+Maj(a(i),b(i),c(i)); -#endif /* HAVE_INTEL_AVX2 */ - -/* INLINE Assember for Intel AVX1 instructions */ -#if defined(HAVE_INTEL_AVX1) -#if defined(DEBUG_XMM) - #define SAVE_REG(i) __asm__ volatile("vmovdqu %%xmm"#i", %0 \n\t":"=m"(reg[i][0]):); - #define RECV_REG(i) __asm__ volatile("vmovdqu %0, %%xmm"#i" \n\t"::"m"(reg[i][0])); - - #define _DUMP_REG(REG, name)\ - { word64 buf[16];word64 reg[16][2];int k;\ - SAVE_REG(0); SAVE_REG(1); SAVE_REG(2); SAVE_REG(3); SAVE_REG(4); \ - SAVE_REG(5); SAVE_REG(6); SAVE_REG(7);SAVE_REG(8); SAVE_REG(9); SAVE_REG(10);\ - SAVE_REG(11); SAVE_REG(12); SAVE_REG(13); SAVE_REG(14); SAVE_REG(15); \ - __asm__ volatile("vmovdqu %%"#REG", %0 \n\t":"=m"(buf[0]):);\ - printf(" "#name":\t"); for(k=0; k<2; k++) printf("%016lx.", (word64)(buf[k])); printf("\n"); \ - RECV_REG(0); RECV_REG(1); RECV_REG(2); RECV_REG(3); RECV_REG(4);\ - RECV_REG(5); RECV_REG(6); RECV_REG(7); RECV_REG(8); RECV_REG(9);\ - RECV_REG(10); RECV_REG(11); RECV_REG(12); RECV_REG(13); RECV_REG(14); RECV_REG(15);\ - } - - #define DUMP_REG(REG) _DUMP_REG(REG, #REG) - #define PRINTF(fmt, ...) -#else - #define DUMP_REG(REG) - #define PRINTF(fmt, ...) -#endif /* DEBUG_XMM */ - -#define _MOVE_to_REG(xymm, mem) __asm__ volatile("vmovdqu %0, %%"#xymm" "\ - :: "m"(mem)); -#define _MOVE_to_MEM(mem,i, xymm) __asm__ volatile("vmovdqu %%"#xymm", %0" :\ - "=m"(mem[i]),"=m"(mem[i+1]),"=m"(mem[i+2]),"=m"(mem[i+3]):); -#define _MOVE(dest, src) __asm__ volatile("vmovdqu %%"#src", %%"\ - #dest" "::); - -#define _S_TEMP(dest, src, bits, temp) __asm__ volatile("vpsrlq $"#bits", %%"\ - #src", %%"#dest"\n\tvpsllq $64-"#bits", %%"#src", %%"#temp"\n\tvpor %%"\ - #temp",%%"#dest", %%"#dest" "::); -#define _AVX1_R(dest, src, bits) __asm__ volatile("vpsrlq $"#bits", %%"\ - #src", %%"#dest" "::); -#define _XOR(dest, src1, src2) __asm__ volatile("vpxor %%"#src1", %%"\ - #src2", %%"#dest" "::); -#define _OR(dest, src1, src2) __asm__ volatile("vpor %%"#src1", %%"\ - #src2", %%"#dest" "::); -#define _ADD(dest, src1, src2) __asm__ volatile("vpaddq %%"#src1", %%"\ - #src2", %%"#dest" "::); -#define _ADD_MEM(dest, src1, mem) __asm__ volatile("vpaddq %0, %%"#src1", %%"\ - #dest" "::"m"(mem)); - -#define MOVE_to_REG(xymm, mem) _MOVE_to_REG(xymm, mem) -#define MOVE_to_MEM(mem, i, xymm) _MOVE_to_MEM(mem, i, xymm) -#define MOVE(dest, src) _MOVE(dest, src) - -#define XOR(dest, src1, src2) _XOR(dest, src1, src2) -#define OR(dest, src1, src2) _OR(dest, src1, src2) -#define ADD(dest, src1, src2) _ADD(dest, src1, src2) - -#define S_TMP(dest, src, bits, temp) _S_TEMP(dest, src, bits, temp); -#define AVX1_S(dest, src, bits) S_TMP(dest, src, bits, S_TEMP) -#define AVX1_R(dest, src, bits) _AVX1_R(dest, src, bits) - -#define Init_Mask(mask) \ - __asm__ volatile("vmovdqu %0, %%xmm1\n\t"::"m"(mask):"%xmm1"); - -#define _W_from_buff1(w, buff, xmm) \ - /* X0..3(xmm4..7), W[0..15] = sha512->buffer[0.15]; */\ - __asm__ volatile("vmovdqu %1, %%"#xmm"\n\t"\ - "vpshufb %%xmm1, %%"#xmm", %%"#xmm"\n\t"\ - "vmovdqu %%"#xmm", %0"\ - :"=m"(w): "m"(buff):"%xmm0"); - -#define W_from_buff1(w, buff, xmm) _W_from_buff1(w, buff, xmm) - -#define W_from_buff(w, buff)\ - Init_Mask(mBYTE_FLIP_MASK[0]);\ - W_from_buff1(w[0], buff[0], W_0);\ - W_from_buff1(w[2], buff[2], W_2);\ - W_from_buff1(w[4], buff[4], W_4);\ - W_from_buff1(w[6], buff[6], W_6);\ - W_from_buff1(w[8], buff[8], W_8);\ - W_from_buff1(w[10],buff[10],W_10);\ - W_from_buff1(w[12],buff[12],W_12);\ - W_from_buff1(w[14],buff[14],W_14); - -static word64 mBYTE_FLIP_MASK[] = { 0x0001020304050607, 0x08090a0b0c0d0e0f }; - -#define W_I_15 xmm14 -#define W_I_7 xmm11 -#define W_I_2 xmm13 -#define W_I xmm12 -#define G_TEMP xmm0 -#define S_TEMP xmm1 -#define XMM_TEMP0 xmm2 - -#define W_0 xmm12 -#define W_2 xmm3 -#define W_4 xmm4 -#define W_6 xmm5 -#define W_8 xmm6 -#define W_10 xmm7 -#define W_12 xmm8 -#define W_14 xmm9 - -#define s0_1(dest, src) AVX1_S(dest, src, 1); -#define s0_2(dest, src) AVX1_S(G_TEMP, src, 8); XOR(dest, G_TEMP, dest); -#define s0_3(dest, src) AVX1_R(G_TEMP, src, 7); XOR(dest, G_TEMP, dest); - -#define s1_1(dest, src) AVX1_S(dest, src, 19); -#define s1_2(dest, src) AVX1_S(G_TEMP, src, 61); XOR(dest, G_TEMP, dest); -#define s1_3(dest, src) AVX1_R(G_TEMP, src, 6); XOR(dest, G_TEMP, dest); - -#define s0_(dest, src) s0_1(dest, src); s0_2(dest, src); s0_3(dest, src) -#define s1_(dest, src) s1_1(dest, src); s1_2(dest, src); s1_3(dest, src) - -#define Block_xx_1(i) \ - MOVE_to_REG(W_I_15, W_X[(i-15)&15]);\ - MOVE_to_REG(W_I_7, W_X[(i- 7)&15]);\ - -#define Block_xx_2(i) \ - MOVE_to_REG(W_I_2, W_X[(i- 2)&15]);\ - MOVE_to_REG(W_I, W_X[(i)]);\ - -#define Block_xx_3(i) \ - s0_ (XMM_TEMP0, W_I_15);\ - -#define Block_xx_4(i) \ - ADD(W_I, W_I, XMM_TEMP0);\ - ADD(W_I, W_I, W_I_7);\ - -#define Block_xx_5(i) \ - s1_ (XMM_TEMP0, W_I_2);\ - -#define Block_xx_6(i) \ - ADD(W_I, W_I, XMM_TEMP0);\ - MOVE_to_MEM(W_X,i, W_I);\ - if (i==0)\ - MOVE_to_MEM(W_X,16, W_I);\ - -#define Block_xx_7(i) \ - MOVE_to_REG(W_I_15, W_X[(i-15)&15]);\ - MOVE_to_REG(W_I_7, W_X[(i- 7)&15]);\ - -#define Block_xx_8(i) \ - MOVE_to_REG(W_I_2, W_X[(i- 2)&15]);\ - MOVE_to_REG(W_I, W_X[(i)]);\ - -#define Block_xx_9(i) \ - s0_ (XMM_TEMP0, W_I_15);\ - -#define Block_xx_10(i) \ - ADD(W_I, W_I, XMM_TEMP0);\ - ADD(W_I, W_I, W_I_7);\ - -#define Block_xx_11(i) \ - s1_ (XMM_TEMP0, W_I_2);\ - -#define Block_xx_12(i) \ - ADD(W_I, W_I, XMM_TEMP0);\ - MOVE_to_MEM(W_X,i, W_I);\ - if ((i)==0)\ - MOVE_to_MEM(W_X,16, W_I);\ - -static INLINE void Block_0_1(word64 *W_X) { Block_xx_1(0); } -static INLINE void Block_0_2(word64 *W_X) { Block_xx_2(0); } -static INLINE void Block_0_3(void) { Block_xx_3(0); } -static INLINE void Block_0_4(void) { Block_xx_4(0); } -static INLINE void Block_0_5(void) { Block_xx_5(0); } -static INLINE void Block_0_6(word64 *W_X) { Block_xx_6(0); } -static INLINE void Block_0_7(word64 *W_X) { Block_xx_7(2); } -static INLINE void Block_0_8(word64 *W_X) { Block_xx_8(2); } -static INLINE void Block_0_9(void) { Block_xx_9(2); } -static INLINE void Block_0_10(void){ Block_xx_10(2); } -static INLINE void Block_0_11(void){ Block_xx_11(2); } -static INLINE void Block_0_12(word64 *W_X){ Block_xx_12(2); } - -static INLINE void Block_4_1(word64 *W_X) { Block_xx_1(4); } -static INLINE void Block_4_2(word64 *W_X) { Block_xx_2(4); } -static INLINE void Block_4_3(void) { Block_xx_3(4); } -static INLINE void Block_4_4(void) { Block_xx_4(4); } -static INLINE void Block_4_5(void) { Block_xx_5(4); } -static INLINE void Block_4_6(word64 *W_X) { Block_xx_6(4); } -static INLINE void Block_4_7(word64 *W_X) { Block_xx_7(6); } -static INLINE void Block_4_8(word64 *W_X) { Block_xx_8(6); } -static INLINE void Block_4_9(void) { Block_xx_9(6); } -static INLINE void Block_4_10(void){ Block_xx_10(6); } -static INLINE void Block_4_11(void){ Block_xx_11(6); } -static INLINE void Block_4_12(word64 *W_X){ Block_xx_12(6); } - -static INLINE void Block_8_1(word64 *W_X) { Block_xx_1(8); } -static INLINE void Block_8_2(word64 *W_X) { Block_xx_2(8); } -static INLINE void Block_8_3(void) { Block_xx_3(8); } -static INLINE void Block_8_4(void) { Block_xx_4(8); } -static INLINE void Block_8_5(void) { Block_xx_5(8); } -static INLINE void Block_8_6(word64 *W_X) { Block_xx_6(8); } -static INLINE void Block_8_7(word64 *W_X) { Block_xx_7(10); } -static INLINE void Block_8_8(word64 *W_X) { Block_xx_8(10); } -static INLINE void Block_8_9(void) { Block_xx_9(10); } -static INLINE void Block_8_10(void){ Block_xx_10(10); } -static INLINE void Block_8_11(void){ Block_xx_11(10); } -static INLINE void Block_8_12(word64 *W_X){ Block_xx_12(10); } - -static INLINE void Block_12_1(word64 *W_X) { Block_xx_1(12); } -static INLINE void Block_12_2(word64 *W_X) { Block_xx_2(12); } -static INLINE void Block_12_3(void) { Block_xx_3(12); } -static INLINE void Block_12_4(void) { Block_xx_4(12); } -static INLINE void Block_12_5(void) { Block_xx_5(12); } -static INLINE void Block_12_6(word64 *W_X) { Block_xx_6(12); } -static INLINE void Block_12_7(word64 *W_X) { Block_xx_7(14); } -static INLINE void Block_12_8(word64 *W_X) { Block_xx_8(14); } -static INLINE void Block_12_9(void) { Block_xx_9(14); } -static INLINE void Block_12_10(void){ Block_xx_10(14); } -static INLINE void Block_12_11(void){ Block_xx_11(14); } -static INLINE void Block_12_12(word64 *W_X){ Block_xx_12(14); } - -#endif /* HAVE_INTEL_AVX1 */ - -#if defined(HAVE_INTEL_AVX2) -static const unsigned long mBYTE_FLIP_MASK_Y[] = - { 0x0001020304050607, 0x08090a0b0c0d0e0f, 0x0001020304050607, 0x08090a0b0c0d0e0f }; - -#define W_from_buff_Y(buff)\ - { /* X0..3(ymm9..12), W_X[0..15] = sha512->buffer[0.15]; */\ - __asm__ volatile("vmovdqu %0, %%ymm8\n\t"::"m"(mBYTE_FLIP_MASK_Y[0]));\ - __asm__ volatile("vmovdqu %0, %%ymm12\n\t"\ - "vmovdqu %1, %%ymm4\n\t"\ - "vpshufb %%ymm8, %%ymm12, %%ymm12\n\t"\ - "vpshufb %%ymm8, %%ymm4, %%ymm4\n\t"\ - :: "m"(buff[0]), "m"(buff[4]));\ - __asm__ volatile("vmovdqu %0, %%ymm5\n\t"\ - "vmovdqu %1, %%ymm6\n\t"\ - "vpshufb %%ymm8, %%ymm5, %%ymm5\n\t"\ - "vpshufb %%ymm8, %%ymm6, %%ymm6\n\t"\ - :: "m"(buff[8]), "m"(buff[12]));\ - } - -#if defined(DEBUG_YMM) - #define SAVE_REG_Y(i) __asm__ volatile("vmovdqu %%ymm"#i", %0 \n\t":"=m"(reg[i-4][0]):); - #define RECV_REG_Y(i) __asm__ volatile("vmovdqu %0, %%ymm"#i" \n\t"::"m"(reg[i-4][0])); - - #define _DUMP_REG_Y(REG, name)\ - { word64 buf[16];word64 reg[16][2];int k;\ - SAVE_REG_Y(4); SAVE_REG_Y(5); SAVE_REG_Y(6); SAVE_REG_Y(7); \ - SAVE_REG_Y(8); SAVE_REG_Y(9); SAVE_REG_Y(10); SAVE_REG_Y(11); SAVE_REG_Y(12);\ - SAVE_REG_Y(13); SAVE_REG_Y(14); SAVE_REG_Y(15); \ - __asm__ volatile("vmovdqu %%"#REG", %0 \n\t":"=m"(buf[0]):);\ - printf(" "#name":\t"); for(k=0; k<4; k++) printf("%016lx.", (word64)buf[k]); printf("\n"); \ - RECV_REG_Y(4); RECV_REG_Y(5); RECV_REG_Y(6); RECV_REG_Y(7); \ - RECV_REG_Y(8); RECV_REG_Y(9); RECV_REG_Y(10); RECV_REG_Y(11); RECV_REG_Y(12); \ - RECV_REG_Y(13); RECV_REG_Y(14); RECV_REG_Y(15);\ - } - - #define DUMP_REG_Y(REG) _DUMP_REG_Y(REG, #REG) - #define DUMP_REG2_Y(REG) _DUMP_REG_Y(REG, #REG) - #define PRINTF_Y(fmt, ...) -#else - #define DUMP_REG_Y(REG) - #define DUMP_REG2_Y(REG) - #define PRINTF_Y(fmt, ...) -#endif /* DEBUG_YMM */ - -#define _MOVE_to_REGy(ymm, mem) __asm__ volatile("vmovdqu %0, %%"#ymm" "\ - :: "m"(mem)); -#define _MOVE_to_MEMy(mem,i, ymm) __asm__ volatile("vmovdqu %%"#ymm", %0" \ - : "=m"(mem[i]),"=m"(mem[i+1]),"=m"(mem[i+2]),"=m"(mem[i+3]):); -#define _MOVE_128y(ymm0, ymm1, ymm2, map) __asm__ volatile("vperm2i128 $"\ - #map", %%"#ymm2", %%"#ymm1", %%"#ymm0" "::); -#define _S_TEMPy(dest, src, bits, temp) \ - __asm__ volatile("vpsrlq $"#bits", %%"#src", %%"#dest"\n\tvpsllq $64-"#bits\ - ", %%"#src", %%"#temp"\n\tvpor %%"#temp",%%"#dest", %%"#dest" "::); -#define _AVX2_R(dest, src, bits) __asm__ volatile("vpsrlq $"#bits", %%"\ - #src", %%"#dest" "::); -#define _XORy(dest, src1, src2) __asm__ volatile("vpxor %%"#src1", %%"\ - #src2", %%"#dest" "::); -#define _ADDy(dest, src1, src2) __asm__ volatile("vpaddq %%"#src1", %%"\ - #src2", %%"#dest" "::); -#define _BLENDy(map, dest, src1, src2) __asm__ volatile("vpblendd $"#map", %%"\ - #src1", %%"#src2", %%"#dest" "::); -#define _BLENDQy(map, dest, src1, src2) __asm__ volatile("vblendpd $"#map", %%"\ - #src1", %%"#src2", %%"#dest" "::); -#define _PERMQy(map, dest, src) __asm__ volatile("vpermq $"#map", %%"\ - #src", %%"#dest" "::); - -#define MOVE_to_REGy(ymm, mem) _MOVE_to_REGy(ymm, mem) -#define MOVE_to_MEMy(mem, i, ymm) _MOVE_to_MEMy(mem, i, ymm) - -#define MOVE_128y(ymm0, ymm1, ymm2, map) _MOVE_128y(ymm0, ymm1, ymm2, map) -#define XORy(dest, src1, src2) _XORy(dest, src1, src2) -#define ADDy(dest, src1, src2) _ADDy(dest, src1, src2) -#define BLENDy(map, dest, src1, src2) _BLENDy(map, dest, src1, src2) -#define BLENDQy(map, dest, src1, src2) _BLENDQy(map, dest, src1, src2) -#define PERMQy(map, dest, src) _PERMQy(map, dest, src) - - -#define S_TMPy(dest, src, bits, temp) _S_TEMPy(dest, src, bits, temp); -#define AVX2_S(dest, src, bits) S_TMPy(dest, src, bits, S_TEMPy) -#define AVX2_R(dest, src, bits) _AVX2_R(dest, src, bits) - - -#define FEEDBACK1_to_W_I_2(w_i_2, w_i) MOVE_128y(YMM_TEMP0, w_i, w_i, 0x08);\ - BLENDy(0xf0, w_i_2, YMM_TEMP0, w_i_2); - -#define MOVE_W_to_W_I_15(w_i_15, w_0, w_4) BLENDQy(0x1, w_i_15, w_4, w_0);\ - PERMQy(0x39, w_i_15, w_i_15); -#define MOVE_W_to_W_I_7(w_i_7, w_8, w_12) BLENDQy(0x1, w_i_7, w_12, w_8);\ - PERMQy(0x39, w_i_7, w_i_7); -#define MOVE_W_to_W_I_2(w_i_2, w_12) BLENDQy(0xc, w_i_2, w_12, w_i_2);\ - PERMQy(0x0e, w_i_2, w_i_2); - - -#define W_I_16y ymm8 -#define W_I_15y ymm9 -#define W_I_7y ymm10 -#define W_I_2y ymm11 -#define W_Iy ymm12 -#define G_TEMPy ymm13 -#define S_TEMPy ymm14 -#define YMM_TEMP0 ymm15 -#define YMM_TEMP0x xmm15 -#define W_I_TEMPy ymm7 -#define W_K_TEMPy ymm15 -#define W_K_TEMPx xmm15 -#define W_0y ymm12 -#define W_4y ymm4 -#define W_8y ymm5 -#define W_12y ymm6 - - -#define MOVE_15_to_16(w_i_16, w_i_15, w_i_7)\ - __asm__ volatile("vperm2i128 $0x01, %%"#w_i_15", %%"#w_i_15", %%"#w_i_15" "::);\ - __asm__ volatile("vpblendd $0x08, %%"#w_i_15", %%"#w_i_7", %%"#w_i_16" "::);\ - __asm__ volatile("vperm2i128 $0x01, %%"#w_i_7", %%"#w_i_7", %%"#w_i_15" "::);\ - __asm__ volatile("vpblendd $0x80, %%"#w_i_15", %%"#w_i_16", %%"#w_i_16" "::);\ - __asm__ volatile("vpshufd $0x93, %%"#w_i_16", %%"#w_i_16" "::);\ - -#define MOVE_7_to_15(w_i_15, w_i_7)\ - __asm__ volatile("vmovdqu %%"#w_i_7", %%"#w_i_15" "::);\ - -#define MOVE_I_to_7(w_i_7, w_i)\ - __asm__ volatile("vperm2i128 $0x01, %%"#w_i", %%"#w_i", %%"#w_i_7" "::);\ - __asm__ volatile("vpblendd $0x01, %%"#w_i_7", %%"#w_i", %%"#w_i_7" "::);\ - __asm__ volatile("vpshufd $0x39, %%"#w_i_7", %%"#w_i_7" "::);\ - -#define MOVE_I_to_2(w_i_2, w_i)\ - __asm__ volatile("vperm2i128 $0x01, %%"#w_i", %%"#w_i", %%"#w_i_2" "::);\ - __asm__ volatile("vpshufd $0x0e, %%"#w_i_2", %%"#w_i_2" "::);\ - -#endif /* HAVE_INTEL_AVX2 */ - /*** Transform Body ***/ #if defined(HAVE_INTEL_AVX1) -static int Transform_AVX1(wc_Sha512* sha512) +static int Transform_Sha512_AVX1(wc_Sha512* sha512) { - const word64* K = K512; - word64 W_X[16+4] = {0}; - word32 j; - word64 T[8]; + __asm__ __volatile__ ( - /* Copy digest to working vars */ - XMEMCPY(T, sha512->digest, sizeof(T)); + /* 16 Ws plus loop counter. */ + "subq $136, %%rsp\n\t" + "leaq 64(%[sha512]), %%rax\n\t" - W_from_buff(W_X, sha512->buffer); - for (j = 0; j < 80; j += 16) { - Rx_1( 0); Block_0_1(W_X); Rx_2( 0); Block_0_2(W_X); Rx_3( 0); Block_0_3(); - Rx_1( 1); Block_0_4(); Rx_2( 1); Block_0_5(); Rx_3( 1); Block_0_6(W_X); - Rx_1( 2); Block_0_7(W_X); Rx_2( 2); Block_0_8(W_X); Rx_3( 2); Block_0_9(); - Rx_1( 3); Block_0_10();Rx_2( 3); Block_0_11();Rx_3( 3); Block_0_12(W_X); + INIT_MASK(MASK) + LOAD_DIGEST() - Rx_1( 4); Block_4_1(W_X); Rx_2( 4); Block_4_2(W_X); Rx_3( 4); Block_4_3(); - Rx_1( 5); Block_4_4(); Rx_2( 5); Block_4_5(); Rx_3( 5); Block_4_6(W_X); - Rx_1( 6); Block_4_7(W_X); Rx_2( 6); Block_4_8(W_X); Rx_3( 6); Block_4_9(); - Rx_1( 7); Block_4_10();Rx_2( 7); Block_4_11();Rx_3( 7); Block_4_12(W_X); + LOAD_W(MASK, rax) - Rx_1( 8); Block_8_1(W_X); Rx_2( 8); Block_8_2(W_X); Rx_3( 8); Block_8_3(); - Rx_1( 9); Block_8_4(); Rx_2( 9); Block_8_5(); Rx_3( 9); Block_8_6(W_X); - Rx_1(10); Block_8_7(W_X); Rx_2(10); Block_8_8(W_X); Rx_3(10); Block_8_9(); - Rx_1(11); Block_8_10();Rx_2(11); Block_8_11();Rx_3(11); Block_8_12(W_X); + "movl $4, 16*8("WX")\n\t" + "leaq %[K512], %%rsi\n\t" + /* b */ + "movq %%r9, "L4"\n\t" + /* e */ + "movq %%r12, "L1"\n\t" + /* b ^ c */ + "xorq %%r10, "L4"\n\t" - Rx_1(12); Block_12_1(W_X); Rx_2(12); Block_12_2(W_X); Rx_3(12); Block_12_3(); - Rx_1(13); Block_12_4(); Rx_2(13); Block_12_5(); Rx_3(13); Block_12_6(W_X); - Rx_1(14); Block_12_7(W_X); Rx_2(14); Block_12_8(W_X); Rx_3(14); Block_12_9(); - Rx_1(15); Block_12_10();Rx_2(15); Block_12_11();Rx_3(15); Block_12_12(W_X); - } + "# Start of 16 rounds\n" + "1:\n\t" - /* Add the working vars back into digest */ - sha512->digest[0] += a(0); - sha512->digest[1] += b(0); - sha512->digest[2] += c(0); - sha512->digest[3] += d(0); - sha512->digest[4] += e(0); - sha512->digest[5] += f(0); - sha512->digest[6] += g(0); - sha512->digest[7] += h(0); + SET_W_X(rsi) - /* Wipe variables */ -#if !defined(HAVE_INTEL_AVX1) && !defined(HAVE_INTEL_AVX2) - XMEMSET(W_X, 0, sizeof(word64) * 16); -#endif - XMEMSET(T, 0, sizeof(T)); + "addq $128, %%rsi\n\t" + + MsgSched2(W_0,W_2,W_4,W_6,W_8,W_10,W_12,W_14,RA,RB,RC,RD,RE,RF,RG,RH, 0) + MsgSched2(W_2,W_4,W_6,W_8,W_10,W_12,W_14,W_0,RG,RH,RA,RB,RC,RD,RE,RF, 2) + MsgSched2(W_4,W_6,W_8,W_10,W_12,W_14,W_0,W_2,RE,RF,RG,RH,RA,RB,RC,RD, 4) + MsgSched2(W_6,W_8,W_10,W_12,W_14,W_0,W_2,W_4,RC,RD,RE,RF,RG,RH,RA,RB, 6) + MsgSched2(W_8,W_10,W_12,W_14,W_0,W_2,W_4,W_6,RA,RB,RC,RD,RE,RF,RG,RH, 8) + MsgSched2(W_10,W_12,W_14,W_0,W_2,W_4,W_6,W_8,RG,RH,RA,RB,RC,RD,RE,RF,10) + MsgSched2(W_12,W_14,W_0,W_2,W_4,W_6,W_8,W_10,RE,RF,RG,RH,RA,RB,RC,RD,12) + MsgSched2(W_14,W_0,W_2,W_4,W_6,W_8,W_10,W_12,RC,RD,RE,RF,RG,RH,RA,RB,14) + + "subl $1, 16*8("WX")\n\t" + "jne 1b\n\t" + + SET_W_X(rsi) + + RND_ALL_2(RA,RB,RC,RD,RE,RF,RG,RH, 0) + RND_ALL_2(RG,RH,RA,RB,RC,RD,RE,RF, 2) + RND_ALL_2(RE,RF,RG,RH,RA,RB,RC,RD, 4) + RND_ALL_2(RC,RD,RE,RF,RG,RH,RA,RB, 6) + + RND_ALL_2(RA,RB,RC,RD,RE,RF,RG,RH, 8) + RND_ALL_2(RG,RH,RA,RB,RC,RD,RE,RF,10) + RND_ALL_2(RE,RF,RG,RH,RA,RB,RC,RD,12) + RND_ALL_2(RC,RD,RE,RF,RG,RH,RA,RB,14) + + STORE_ADD_DIGEST() + + "addq $136, %%rsp\n\t" + + : + : [mask] "m" (mBYTE_FLIP_MASK), + [sha512] "r" (sha512), + [K512] "m" (K512) + : WORK_REGS, STATE_REGS, XMM_REGS, "memory", "rsi" + ); + + return 0; +} + +static int Transform_Sha512_AVX1_Len(wc_Sha512* sha512, word32 len) +{ + __asm__ __volatile__ ( + + "movq 224(%[sha512]), %%rsi\n\t" + "leaq %[K512], %%rdx\n\t" + + INIT_MASK(MASK) + LOAD_DIGEST() + + "# Start of processing a block\n" + "2:\n\t" + + /* 16 Ws plus loop counter and K512. len goes into -4(%rsp). + * Debug needs more stack space. */ + "subq $256, %%rsp\n\t" + + LOAD_W(MASK, rsi) + + "movl $4, 16*8("WX")\n\t" + /* b */ + "movq %%r9, "L4"\n\t" + /* e */ + "movq %%r12, "L1"\n\t" + /* b ^ c */ + "xorq %%r10, "L4"\n\t" + + SET_W_X(rdx) + + "# Start of 16 rounds\n" + "1:\n\t" + + "addq $128, %%rdx\n\t" + "movq %%rdx, 17*8(%%rsp)\n\t" + + MsgSched2(W_0,W_2,W_4,W_6,W_8,W_10,W_12,W_14,RA,RB,RC,RD,RE,RF,RG,RH, 0) + MsgSched2(W_2,W_4,W_6,W_8,W_10,W_12,W_14,W_0,RG,RH,RA,RB,RC,RD,RE,RF, 2) + MsgSched2(W_4,W_6,W_8,W_10,W_12,W_14,W_0,W_2,RE,RF,RG,RH,RA,RB,RC,RD, 4) + MsgSched2(W_6,W_8,W_10,W_12,W_14,W_0,W_2,W_4,RC,RD,RE,RF,RG,RH,RA,RB, 6) + MsgSched2(W_8,W_10,W_12,W_14,W_0,W_2,W_4,W_6,RA,RB,RC,RD,RE,RF,RG,RH, 8) + MsgSched2(W_10,W_12,W_14,W_0,W_2,W_4,W_6,W_8,RG,RH,RA,RB,RC,RD,RE,RF,10) + MsgSched2(W_12,W_14,W_0,W_2,W_4,W_6,W_8,W_10,RE,RF,RG,RH,RA,RB,RC,RD,12) + MsgSched2(W_14,W_0,W_2,W_4,W_6,W_8,W_10,W_12,RC,RD,RE,RF,RG,RH,RA,RB,14) + + "movq 17*8(%%rsp), %%rdx\n\t" + + SET_W_X(rdx) + + "subl $1, 16*8("WX")\n\t" + "jne 1b\n\t" + + RND_ALL_2(RA,RB,RC,RD,RE,RF,RG,RH, 0) + RND_ALL_2(RG,RH,RA,RB,RC,RD,RE,RF, 2) + RND_ALL_2(RE,RF,RG,RH,RA,RB,RC,RD, 4) + RND_ALL_2(RC,RD,RE,RF,RG,RH,RA,RB, 6) + + RND_ALL_2(RA,RB,RC,RD,RE,RF,RG,RH, 8) + RND_ALL_2(RG,RH,RA,RB,RC,RD,RE,RF,10) + RND_ALL_2(RE,RF,RG,RH,RA,RB,RC,RD,12) + RND_ALL_2(RC,RD,RE,RF,RG,RH,RA,RB,14) + + ADD_DIGEST() + + "addq $256, %%rsp\n\t" + "leaq %[K512], %%rdx\n\t" + "addq $128, %%rsi\n\t" + "subl $128, %[len]\n\t" + + STORE_DIGEST() + + "jnz 2b\n\t" + + : + : [mask] "m" (mBYTE_FLIP_MASK), + [len] "m" (len), + [sha512] "r" (sha512), + [K512] "m" (K512) + : WORK_REGS, STATE_REGS, XMM_REGS, "memory", "rsi" + ); return 0; } #endif /* HAVE_INTEL_AVX1 */ -#if defined(HAVE_INTEL_AVX2) && defined(HAVE_INTEL_AVX1) && defined(HAVE_INTEL_RORX) -static int Transform_AVX1_RORX(wc_Sha512* sha512) +#if defined(HAVE_INTEL_AVX2) && defined(HAVE_INTEL_RORX) +static int Transform_Sha512_AVX1_RORX(wc_Sha512* sha512) { - const word64* K = K512; - word64 W_X[16+4] = {0}; - word32 j; - word64 T[8]; + __asm__ __volatile__ ( - /* Copy digest to working vars */ - XMEMCPY(T, sha512->digest, sizeof(T)); + /* 16 Ws plus loop counter and K512. */ + "subq $144, %%rsp\n\t" + "leaq 64(%[sha512]), %%rax\n\t" - W_from_buff(W_X, sha512->buffer); - for (j = 0; j < 80; j += 16) { - Rx_RORX_1( 0); Block_0_1(W_X); Rx_RORX_2( 0); Block_0_2(W_X); - Rx_RORX_3( 0); Block_0_3(); - Rx_RORX_1( 1); Block_0_4(); Rx_RORX_2( 1); Block_0_5(); - Rx_RORX_3( 1); Block_0_6(W_X); - Rx_RORX_1( 2); Block_0_7(W_X); Rx_RORX_2( 2); Block_0_8(W_X); - Rx_RORX_3( 2); Block_0_9(); - Rx_RORX_1( 3); Block_0_10();Rx_RORX_2( 3); Block_0_11(); - Rx_RORX_3( 3); Block_0_12(W_X); + INIT_MASK(MASK) + LOAD_DIGEST() - Rx_RORX_1( 4); Block_4_1(W_X); Rx_RORX_2( 4); Block_4_2(W_X); - Rx_RORX_3( 4); Block_4_3(); - Rx_RORX_1( 5); Block_4_4(); Rx_RORX_2( 5); Block_4_5(); - Rx_RORX_3( 5); Block_4_6(W_X); - Rx_RORX_1( 6); Block_4_7(W_X); Rx_RORX_2( 6); Block_4_8(W_X); - Rx_RORX_3( 6); Block_4_9(); - Rx_RORX_1( 7); Block_4_10();Rx_RORX_2( 7); Block_4_11(); - Rx_RORX_3( 7); Block_4_12(W_X); + LOAD_W(MASK, rax) - Rx_RORX_1( 8); Block_8_1(W_X); Rx_RORX_2( 8); Block_8_2(W_X); - Rx_RORX_3( 8); Block_8_3(); - Rx_RORX_1( 9); Block_8_4(); Rx_RORX_2( 9); Block_8_5(); - Rx_RORX_3( 9); Block_8_6(W_X); - Rx_RORX_1(10); Block_8_7(W_X); Rx_RORX_2(10); Block_8_8(W_X); - Rx_RORX_3(10); Block_8_9(); - Rx_RORX_1(11); Block_8_10();Rx_RORX_2(11); Block_8_11(); - Rx_RORX_3(11); Block_8_12(W_X); + "movl $4, 16*8("WX")\n\t" + "leaq %[K512], %%rsi\n\t" + /* L4 = b */ + "movq %%r9, "L4"\n\t" + /* L3 = 0 (add to prev h) */ + "xorq "L3", "L3"\n\t" + /* L4 = b ^ c */ + "xorq %%r10, "L4"\n\t" - Rx_RORX_1(12); Block_12_1(W_X); Rx_RORX_2(12); Block_12_2(W_X); - Rx_RORX_3(12); Block_12_3(); - Rx_RORX_1(13); Block_12_4(); Rx_RORX_2(13); Block_12_5(); - Rx_RORX_3(13); Block_12_6(W_X); - Rx_RORX_1(14); Block_12_7(W_X); Rx_RORX_2(14); Block_12_8(W_X); - Rx_RORX_3(14); Block_12_9(); - Rx_RORX_1(15); Block_12_10();Rx_RORX_2(15); Block_12_11(); - Rx_RORX_3(15); Block_12_12(W_X); - } + SET_W_X(rsi) - /* Add the working vars back into digest */ - sha512->digest[0] += a(0); - sha512->digest[1] += b(0); - sha512->digest[2] += c(0); - sha512->digest[3] += d(0); - sha512->digest[4] += e(0); - sha512->digest[5] += f(0); - sha512->digest[6] += g(0); - sha512->digest[7] += h(0); + "# Start of 16 rounds\n" + "1:\n\t" - /* Wipe variables */ -#if !defined(HAVE_INTEL_AVX1)&&!defined(HAVE_INTEL_AVX2) - XMEMSET(W_X, 0, sizeof(word64) * 16); -#endif - XMEMSET(T, 0, sizeof(T)); + "addq $128, %%rsi\n\t" + + MsgSched_RORX(W_0,W_2,W_4,W_6,W_8,W_10,W_12,W_14,RA,RB,RC,RD,RE,RF,RG,RH, 0) + MsgSched_RORX(W_2,W_4,W_6,W_8,W_10,W_12,W_14,W_0,RG,RH,RA,RB,RC,RD,RE,RF, 2) + MsgSched_RORX(W_4,W_6,W_8,W_10,W_12,W_14,W_0,W_2,RE,RF,RG,RH,RA,RB,RC,RD, 4) + MsgSched_RORX(W_6,W_8,W_10,W_12,W_14,W_0,W_2,W_4,RC,RD,RE,RF,RG,RH,RA,RB, 6) + MsgSched_RORX(W_8,W_10,W_12,W_14,W_0,W_2,W_4,W_6,RA,RB,RC,RD,RE,RF,RG,RH, 8) + MsgSched_RORX(W_10,W_12,W_14,W_0,W_2,W_4,W_6,W_8,RG,RH,RA,RB,RC,RD,RE,RF,10) + MsgSched_RORX(W_12,W_14,W_0,W_2,W_4,W_6,W_8,W_10,RE,RF,RG,RH,RA,RB,RC,RD,12) + MsgSched_RORX(W_14,W_0,W_2,W_4,W_6,W_8,W_10,W_12,RC,RD,RE,RF,RG,RH,RA,RB,14) + + SET_W_X(rsi) + + "subl $1, 16*8("WX")\n\t" + "jne 1b\n\t" + + RND_RORX_ALL_2(RA,RB,RC,RD,RE,RF,RG,RH, 0) + RND_RORX_ALL_2(RG,RH,RA,RB,RC,RD,RE,RF, 2) + RND_RORX_ALL_2(RE,RF,RG,RH,RA,RB,RC,RD, 4) + RND_RORX_ALL_2(RC,RD,RE,RF,RG,RH,RA,RB, 6) + + RND_RORX_ALL_2(RA,RB,RC,RD,RE,RF,RG,RH, 8) + RND_RORX_ALL_2(RG,RH,RA,RB,RC,RD,RE,RF,10) + RND_RORX_ALL_2(RE,RF,RG,RH,RA,RB,RC,RD,12) + RND_RORX_ALL_2(RC,RD,RE,RF,RG,RH,RA,RB,14) + + /* Prev RND: h += Maj(a,b,c) */ + "addq "L3", %%r8\n\t" + "addq $144, %%rsp\n\t" + + STORE_ADD_DIGEST() + + : + : [mask] "m" (mBYTE_FLIP_MASK), + [sha512] "r" (sha512), + [K512] "m" (K512) + : WORK_REGS, STATE_REGS, XMM_REGS, "memory", "rsi" + ); return 0; } -#endif /* HAVE_INTEL_AVX2 && HAVE_INTEL_AVX1 && HAVE_INTEL_RORX */ + +static int Transform_Sha512_AVX1_RORX_Len(wc_Sha512* sha512, word32 len) +{ + __asm__ __volatile__ ( + + "movq 224(%[sha512]), %%rsi\n\t" + "leaq %[K512], %%rcx\n\t" + + INIT_MASK(MASK) + LOAD_DIGEST() + + "# Start of processing a block\n" + "2:\n\t" + + /* 16 Ws plus loop counter and K512. len goes into -4(%rsp). + * Debug needs more stack space. */ + "subq $256, %%rsp\n\t" + + LOAD_W(MASK, rsi) + + "movl $4, 16*8("WX")\n\t" + /* L4 = b */ + "movq %%r9, "L4"\n\t" + /* L3 = 0 (add to prev h) */ + "xorq "L3", "L3"\n\t" + /* L4 = b ^ c */ + "xorq %%r10, "L4"\n\t" + + SET_W_X(rcx) + + "# Start of 16 rounds\n" + "1:\n\t" + + "addq $128, %%rcx\n\t" + "movq %%rcx, 17*8(%%rsp)\n\t" + + MsgSched_RORX(W_0,W_2,W_4,W_6,W_8,W_10,W_12,W_14,RA,RB,RC,RD,RE,RF,RG,RH, 0) + MsgSched_RORX(W_2,W_4,W_6,W_8,W_10,W_12,W_14,W_0,RG,RH,RA,RB,RC,RD,RE,RF, 2) + MsgSched_RORX(W_4,W_6,W_8,W_10,W_12,W_14,W_0,W_2,RE,RF,RG,RH,RA,RB,RC,RD, 4) + MsgSched_RORX(W_6,W_8,W_10,W_12,W_14,W_0,W_2,W_4,RC,RD,RE,RF,RG,RH,RA,RB, 6) + MsgSched_RORX(W_8,W_10,W_12,W_14,W_0,W_2,W_4,W_6,RA,RB,RC,RD,RE,RF,RG,RH, 8) + MsgSched_RORX(W_10,W_12,W_14,W_0,W_2,W_4,W_6,W_8,RG,RH,RA,RB,RC,RD,RE,RF,10) + MsgSched_RORX(W_12,W_14,W_0,W_2,W_4,W_6,W_8,W_10,RE,RF,RG,RH,RA,RB,RC,RD,12) + MsgSched_RORX(W_14,W_0,W_2,W_4,W_6,W_8,W_10,W_12,RC,RD,RE,RF,RG,RH,RA,RB,14) + + "movq 17*8(%%rsp), %%rcx\n\t" + + SET_W_X(rcx) + + "subl $1, 16*8("WX")\n\t" + "jne 1b\n\t" + + SET_W_X(rcx) + + RND_RORX_ALL_2(RA,RB,RC,RD,RE,RF,RG,RH, 0) + RND_RORX_ALL_2(RG,RH,RA,RB,RC,RD,RE,RF, 2) + RND_RORX_ALL_2(RE,RF,RG,RH,RA,RB,RC,RD, 4) + RND_RORX_ALL_2(RC,RD,RE,RF,RG,RH,RA,RB, 6) + + RND_RORX_ALL_2(RA,RB,RC,RD,RE,RF,RG,RH, 8) + RND_RORX_ALL_2(RG,RH,RA,RB,RC,RD,RE,RF,10) + RND_RORX_ALL_2(RE,RF,RG,RH,RA,RB,RC,RD,12) + RND_RORX_ALL_2(RC,RD,RE,RF,RG,RH,RA,RB,14) + + /* Prev RND: h += Maj(a,b,c) */ + "addq "L3", %%r8\n\t" + "addq $256, %%rsp\n\t" + + ADD_DIGEST() + + "leaq %[K512], %%rcx\n\t" + "addq $128, %%rsi\n\t" + "subl $128, %[len]\n\t" + + STORE_DIGEST() + + "jnz 2b\n\t" + + : + : [mask] "m" (mBYTE_FLIP_MASK), + [len] "m" (len), + [sha512] "r" (sha512), + [K512] "m" (K512) + : WORK_REGS, STATE_REGS, XMM_REGS, "memory", "rsi" + ); + + return 0; +} +#endif /* HAVE_INTEL_AVX2 && HAVE_INTEL_RORX */ #if defined(HAVE_INTEL_AVX2) +static const unsigned long mBYTE_FLIP_MASK_Y[] = + { 0x0001020304050607, 0x08090a0b0c0d0e0f, + 0x0001020304050607, 0x08090a0b0c0d0e0f }; -#define s0_1y(dest, src) AVX2_S(dest, src, 1); -#define s0_2y(dest, src) AVX2_S(G_TEMPy, src, 8); XORy(dest, G_TEMPy, dest); -#define s0_3y(dest, src) AVX2_R(G_TEMPy, src, 7); XORy(dest, G_TEMPy, dest); +#define W_Y_0 ymm0 +#define W_Y_4 ymm1 +#define W_Y_8 ymm2 +#define W_Y_12 ymm3 -#define s1_1y(dest, src) AVX2_S(dest, src, 19); -#define s1_2y(dest, src) AVX2_S(G_TEMPy, src, 61); XORy(dest, G_TEMPy, dest); -#define s1_3y(dest, src) AVX2_R(G_TEMPy, src, 6); XORy(dest, G_TEMPy, dest); +#define X0 xmm0 +#define X1 xmm1 +#define X2 xmm2 +#define X3 xmm3 +#define X4 xmm4 +#define X5 xmm5 +#define X6 xmm6 +#define X7 xmm7 +#define X8 xmm8 +#define X9 xmm9 +#define Y0 ymm0 +#define Y1 ymm1 +#define Y2 ymm2 +#define Y3 ymm3 +#define Y4 ymm4 +#define Y5 ymm5 +#define Y6 ymm6 +#define Y7 ymm7 -#define s0_y(dest, src) s0_1y(dest, src); s0_2y(dest, src); s0_3y(dest, src) -#define s1_y(dest, src) s1_1y(dest, src); s1_2y(dest, src); s1_3y(dest, src) +#define W_Y_M15 ymm12 +#define W_Y_M7 ymm13 +#define W_Y_M2 ymm14 +#define MASK_Y ymm15 + +#define YTMP1 ymm8 +#define YTMP2 ymm9 +#define YTMP3 ymm10 +#define YTMP4 ymm11 + +#define YMM_REGS \ + "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", "ymm7", \ + "xmm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", "ymm14", "ymm15" + +#define _VPERM2I128(dest, src1, src2, sel) \ + "vperm2I128 $"#sel", %%"#src2", %%"#src1", %%"#dest"\n\t" +#define VPERM2I128(dest, src1, src2, sel) \ + _VPERM2I128(dest, src1, src2, sel) + +#define _VPERMQ(dest, src, sel) \ + "vpermq $"#sel", %%"#src", %%"#dest"\n\t" +#define VPERMQ(dest, src, sel) \ + _VPERMQ(dest, src, sel) + +#define _VPBLENDD(dest, src1, src2, sel) \ + "vpblendd $"#sel", %%"#src2", %%"#src1", %%"#dest"\n\t" +#define VPBLENDD(dest, src1, src2, sel) \ + _VPBLENDD(dest, src1, src2, sel) + +#define _V_ADD_I(dest, src1, addr, i) \ + "vpaddq "#i"*8(%%"#addr"), %%"#src1", %%"#dest"\n\t" +#define V_ADD_I(dest, src1, addr, i) \ + _V_ADD_I(dest, src1, addr, i) + +#define _VMOVDQU_I(addr, i, src) \ + "vmovdqu %%"#src", "#i"*8(%%"#addr")\n\t" +#define VMOVDQU_I(addr, i, src) \ + _VMOVDQU_I(addr, i, src) + +#define MsgSched4_AVX2(W_Y_0,W_Y_4,W_Y_8,W_Y_12,a,b,c,d,e,f,g,h,i) \ + RND_0_1(a,b,c,d,e,f,g,h,i) \ + /* W[-13]..W[-15], W[-12] */ \ + VPBLENDD(W_Y_M15, W_Y_0, W_Y_4, 0x03) \ + /* W[-5]..W[-7], W[-4] */ \ + VPBLENDD(W_Y_M7, W_Y_8, W_Y_12, 0x03) \ + RND_0_2(a,b,c,d,e,f,g,h,i) \ + RND_0_3(a,b,c,d,e,f,g,h,i) \ + /* W_Y_M15 = W[-12]..W[-15] */ \ + VPERMQ(W_Y_M15, W_Y_M15, 0x39) \ + RND_0_4(a,b,c,d,e,f,g,h,i) \ + /* W_Y_M7 = W[-4]..W[-7] */ \ + VPERMQ(W_Y_M7, W_Y_M7, 0x39) \ + RND_0_5(a,b,c,d,e,f,g,h,i) \ + RND_0_6(a,b,c,d,e,f,g,h,i) \ + /* W[-15] >> 1 */ \ + V_SHIFT_R(YTMP1, W_Y_M15, 1) \ + RND_0_7(a,b,c,d,e,f,g,h,i) \ + /* W[-15] << 63 */ \ + V_SHIFT_L(YTMP2, W_Y_M15, 63) \ + RND_0_8(a,b,c,d,e,f,g,h,i) \ + /* W[-15] >> 8 */ \ + V_SHIFT_R(YTMP3, W_Y_M15, 8) \ + RND_0_9(a,b,c,d,e,f,g,h,i) \ + /* W[-15] << 56 */ \ + V_SHIFT_L(YTMP4, W_Y_M15, 56) \ + RND_0_10(a,b,c,d,e,f,g,h,i) \ + /* W[-15] >>> 1 */ \ + V_OR(YTMP1, YTMP2, YTMP1) \ + RND_0_11(a,b,c,d,e,f,g,h,i) \ + /* W[-15] >>> 8 */ \ + V_OR(YTMP3, YTMP4, YTMP3) \ + RND_0_12(a,b,c,d,e,f,g,h,i) \ + RND_1_1(h,a,b,c,d,e,f,g,i+1) \ + /* W[-15] >> 7 */ \ + V_SHIFT_R(YTMP4, W_Y_M15, 7) \ + RND_1_2_A(h,a,b,c,d,e,f,g,i+1) \ + /* (W[-15] >>> 1) ^ (W[-15] >>> 8) */ \ + V_XOR(YTMP1, YTMP3, YTMP1) \ + RND_1_2_B(h,a,b,c,d,e,f,g,i+1) \ + /* (W[-15] >>> 1) ^ (W[-15] >>> 8) ^ (W[-15] >> 7) */ \ + V_XOR(YTMP1, YTMP4, YTMP1) \ + RND_1_3(h,a,b,c,d,e,f,g,i+1) \ + /* W[0] = W[-16] + W[-7] */ \ + V_ADD(W_Y_0, W_Y_0, W_Y_M7) \ + RND_1_4(h,a,b,c,d,e,f,g,i+1) \ + /* W[0] = W[-16] + W[-7] + s0(W[-15]) */ \ + V_ADD(W_Y_0, W_Y_0, YTMP1) \ + RND_1_5(h,a,b,c,d,e,f,g,i+1) \ + /* 0, 0, W[-1], W[-2] */ \ + VPERM2I128(W_Y_M2, W_Y_12, W_Y_12, 0x81) \ + RND_1_6(h,a,b,c,d,e,f,g,i+1) \ + RND_1_7(h,a,b,c,d,e,f,g,i+1) \ + RND_1_8(h,a,b,c,d,e,f,g,i+1) \ + /* W[-2] >> 19 */ \ + V_SHIFT_R(YTMP1, W_Y_M2, 19) \ + RND_1_9(h,a,b,c,d,e,f,g,i+1) \ + /* W[-2] << 45 */ \ + V_SHIFT_L(YTMP2, W_Y_M2, 45) \ + RND_1_10(h,a,b,c,d,e,f,g,i+1) \ + /* W[-2] >> 61 */ \ + V_SHIFT_R(YTMP3, W_Y_M2, 61) \ + RND_1_11(h,a,b,c,d,e,f,g,i+1) \ + /* W[-2] << 3 */ \ + V_SHIFT_L(YTMP4, W_Y_M2, 3) \ + RND_1_12(h,a,b,c,d,e,f,g,i+1) \ + RND_0_1(g,h,a,b,c,d,e,f,i+2) \ + /* W[-2] >>> 19 */ \ + V_OR(YTMP1, YTMP2, YTMP1) \ + RND_0_2(g,h,a,b,c,d,e,f,i+2) \ + /* W[-2] >>> 61 */ \ + V_OR(YTMP3, YTMP4, YTMP3) \ + RND_0_3(g,h,a,b,c,d,e,f,i+2) \ + /* (W[-2] >>> 19) ^ (W[-2] >>> 61) */ \ + V_XOR(YTMP1, YTMP3, YTMP1) \ + RND_0_4(g,h,a,b,c,d,e,f,i+2) \ + /* W[-2] >> 6 */ \ + V_SHIFT_R(YTMP4, W_Y_M2, 6) \ + RND_0_5(g,h,a,b,c,d,e,f,i+2) \ + /* (W[-2] >>> 19) ^ (W[-2] >>> 61) ^ (W[-2] >> 6) */ \ + V_XOR(YTMP1, YTMP4, YTMP1) \ + RND_0_6(g,h,a,b,c,d,e,f,i+2) \ + /* W[0] = W[-16] + W[-7] + s0(W[-15]) + s1(W[-2]) */ \ + V_ADD(W_Y_0, W_Y_0, YTMP1) \ + RND_0_7(g,h,a,b,c,d,e,f,i+2) \ + RND_0_8(g,h,a,b,c,d,e,f,i+2) \ + /* W[1], W[0], 0, 0 */ \ + VPERM2I128(W_Y_M2, W_Y_0, W_Y_0, 0x08) \ + RND_0_9(g,h,a,b,c,d,e,f,i+2) \ + RND_0_10(g,h,a,b,c,d,e,f,i+2) \ + /* W[-2] >> 19 */ \ + V_SHIFT_R(YTMP1, W_Y_M2, 19) \ + RND_0_11(g,h,a,b,c,d,e,f,i+2) \ + /* W[-2] << 45 */ \ + V_SHIFT_L(YTMP2, W_Y_M2, 45) \ + RND_0_12(g,h,a,b,c,d,e,f,i+2) \ + RND_1_1(f,g,h,a,b,c,d,e,i+3) \ + /* W[-2] >> 61 */ \ + V_SHIFT_R(YTMP3, W_Y_M2, 61) \ + RND_1_2(f,g,h,a,b,c,d,e,i+3) \ + /* W[-2] << 3 */ \ + V_SHIFT_L(YTMP4, W_Y_M2, 3) \ + RND_1_3(f,g,h,a,b,c,d,e,i+3) \ + /* W[-2] >>> 19 */ \ + V_OR(YTMP1, YTMP2, YTMP1) \ + RND_1_4(f,g,h,a,b,c,d,e,i+3) \ + /* W[-2] >>> 61 */ \ + V_OR(YTMP3, YTMP4, YTMP3) \ + RND_1_5(f,g,h,a,b,c,d,e,i+3) \ + /* (W[-2] >>> 19) ^ (W[-2] >>> 61) */ \ + V_XOR(YTMP1, YTMP3, YTMP1) \ + RND_1_6(f,g,h,a,b,c,d,e,i+3) \ + /* W[-2] >> 6 */ \ + V_SHIFT_R(YTMP4, W_Y_M2, 6) \ + RND_1_7(f,g,h,a,b,c,d,e,i+3) \ + /* (W[-2] >>> 19) ^ (W[-2] >>> 61) ^ (W[-2] >> 6) */ \ + V_XOR(YTMP1, YTMP4, YTMP1) \ + RND_1_8(f,g,h,a,b,c,d,e,i+3) \ + /* W[0] = W[-16] + W[-7] + s0(W[-15]) + s1(W[-2]) */ \ + V_ADD(W_Y_0, W_Y_0, YTMP1) \ + RND_1_9(f,g,h,a,b,c,d,e,i+3) \ + RND_1_10(f,g,h,a,b,c,d,e,i+3) \ + RND_1_11(f,g,h,a,b,c,d,e,i+3) \ + RND_1_12(f,g,h,a,b,c,d,e,i+3) \ + +#define MsgSched2_AVX2(W_0,W_2,W_4,W_6,W_8,W_10,W_12,W_14,a,b,c,d,e,f,g,h,i) \ + RND_0_1(a,b,c,d,e,f,g,h,i) \ + VPALIGNR(W_Y_M15, W_2, W_0, 8) \ + VPALIGNR(W_Y_M7, W_10, W_8, 8) \ + RND_0_2(a,b,c,d,e,f,g,h,i) \ + V_SHIFT_R(YTMP1, W_Y_M15, 1) \ + V_SHIFT_L(YTMP2, W_Y_M15, 63) \ + RND_0_3(a,b,c,d,e,f,g,h,i) \ + RND_0_4(a,b,c,d,e,f,g,h,i) \ + V_SHIFT_R(YTMP3, W_Y_M15, 8) \ + V_SHIFT_L(YTMP4, W_Y_M15, 56) \ + RND_0_5(a,b,c,d,e,f,g,h,i) \ + RND_0_6(a,b,c,d,e,f,g,h,i) \ + V_OR(YTMP1, YTMP2, YTMP1) \ + V_OR(YTMP3, YTMP4, YTMP3) \ + RND_0_7(a,b,c,d,e,f,g,h,i) \ + RND_0_8(a,b,c,d,e,f,g,h,i) \ + V_SHIFT_R(YTMP4, W_Y_M15, 7) \ + V_XOR(YTMP1, YTMP3, YTMP1) \ + RND_0_9(a,b,c,d,e,f,g,h,i) \ + RND_0_10(a,b,c,d,e,f,g,h,i) \ + V_XOR(YTMP1, YTMP4, YTMP1) \ + V_ADD(W_0, W_0, W_Y_M7) \ + RND_0_11(a,b,c,d,e,f,g,h,i) \ + RND_0_12(a,b,c,d,e,f,g,h,i) \ + RND_1_1(h,a,b,c,d,e,f,g,i+1) \ + V_ADD(W_0, W_0, YTMP1) \ + RND_1_2(h,a,b,c,d,e,f,g,i+1) \ + V_SHIFT_R(YTMP1, W_14, 19) \ + V_SHIFT_L(YTMP2, W_14, 45) \ + RND_1_3(h,a,b,c,d,e,f,g,i+1) \ + RND_1_4(h,a,b,c,d,e,f,g,i+1) \ + V_SHIFT_R(YTMP3, W_14, 61) \ + V_SHIFT_L(YTMP4, W_14, 3) \ + RND_1_5(h,a,b,c,d,e,f,g,i+1) \ + RND_1_6(h,a,b,c,d,e,f,g,i+1) \ + RND_1_7(h,a,b,c,d,e,f,g,i+1) \ + V_OR(YTMP1, YTMP2, YTMP1) \ + V_OR(YTMP3, YTMP4, YTMP3) \ + RND_1_8(h,a,b,c,d,e,f,g,i+1) \ + RND_1_9(h,a,b,c,d,e,f,g,i+1) \ + V_XOR(YTMP1, YTMP3, YTMP1) \ + V_SHIFT_R(YTMP4, W_14, 6) \ + RND_1_10(h,a,b,c,d,e,f,g,i+1) \ + RND_1_11(h,a,b,c,d,e,f,g,i+1) \ + V_XOR(YTMP1, YTMP4, YTMP1) \ + RND_1_12(h,a,b,c,d,e,f,g,i+1) \ + V_ADD(W_0, W_0, YTMP1) \ + +#define MsgSched4_AVX2_RORX_SET(W_Y_0,W_Y_4,W_Y_8,W_Y_12,a,b,c,d,e,f,g,h,i) \ + RND_RORX_0_1(a,b,c,d,e,f,g,h,i) \ + /* W[-13]..W[-15], W[-12] */ \ + VPBLENDD(W_Y_M15, W_Y_0, W_Y_4, 0x03) \ + /* W[-5]..W[-7], W[-4] */ \ + VPBLENDD(W_Y_M7, W_Y_8, W_Y_12, 0x03) \ + RND_RORX_0_2(a,b,c,d,e,f,g,h,i) \ + /* W_Y_M15 = W[-12]..W[-15] */ \ + VPERMQ(W_Y_M15, W_Y_M15, 0x39) \ + RND_RORX_0_3(a,b,c,d,e,f,g,h,i) \ + /* W_Y_M7 = W[-4]..W[-7] */ \ + VPERMQ(W_Y_M7, W_Y_M7, 0x39) \ + RND_RORX_0_4(a,b,c,d,e,f,g,h,i) \ + /* W[-15] >> 1 */ \ + V_SHIFT_R(YTMP1, W_Y_M15, 1) \ + /* W[-15] << 63 */ \ + V_SHIFT_L(YTMP2, W_Y_M15, 63) \ + RND_RORX_0_5(a,b,c,d,e,f,g,h,i) \ + /* W[-15] >> 8 */ \ + V_SHIFT_R(YTMP3, W_Y_M15, 8) \ + /* W[-15] << 56 */ \ + V_SHIFT_L(YTMP4, W_Y_M15, 56) \ + /* W[-15] >>> 1 */ \ + V_OR(YTMP1, YTMP2, YTMP1) \ + /* W[-15] >>> 8 */ \ + V_OR(YTMP3, YTMP4, YTMP3) \ + RND_RORX_0_6(a,b,c,d,e,f,g,h,i) \ + /* W[-15] >> 7 */ \ + V_SHIFT_R(YTMP4, W_Y_M15, 7) \ + RND_RORX_0_7(a,b,c,d,e,f,g,h,i) \ + /* 0, 0, W[-1], W[-2] */ \ + VPERM2I128(W_Y_M2, W_Y_12, W_Y_12, 0x81) \ + RND_RORX_0_8(a,b,c,d,e,f,g,h,i) \ + RND_RORX_1_1(h,a,b,c,d,e,f,g,i+1) \ + /* (W[-15] >>> 1) ^ (W[-15] >>> 8) */ \ + V_XOR(YTMP1, YTMP3, YTMP1) \ + RND_RORX_1_2(h,a,b,c,d,e,f,g,i+1) \ + /* (W[-15] >>> 1) ^ (W[-15] >>> 8) ^ (W[-15] >> 7) */ \ + V_XOR(YTMP1, YTMP4, YTMP1) \ + RND_RORX_1_3(h,a,b,c,d,e,f,g,i+1) \ + /* W[0] = W[-16] + W[-7] */ \ + V_ADD(W_Y_0, W_Y_0, W_Y_M7) \ + /* W[0] = W[-16] + W[-7] + s0(W[-15]) */ \ + V_ADD(W_Y_0, W_Y_0, YTMP1) \ + RND_RORX_1_4(h,a,b,c,d,e,f,g,i+1) \ + /* W[-2] >> 19 */ \ + V_SHIFT_R(YTMP1, W_Y_M2, 19) \ + /* W[-2] << 45 */ \ + V_SHIFT_L(YTMP2, W_Y_M2, 45) \ + RND_RORX_1_5(h,a,b,c,d,e,f,g,i+1) \ + /* W[-2] >> 61 */ \ + V_SHIFT_R(YTMP3, W_Y_M2, 61) \ + /* W[-2] << 3 */ \ + V_SHIFT_L(YTMP4, W_Y_M2, 3) \ + /* W[-2] >>> 19 */ \ + V_OR(YTMP1, YTMP2, YTMP1) \ + RND_RORX_1_6(h,a,b,c,d,e,f,g,i+1) \ + /* W[-2] >>> 61 */ \ + V_OR(YTMP3, YTMP4, YTMP3) \ + RND_RORX_1_7(h,a,b,c,d,e,f,g,i+1) \ + /* (W[-2] >>> 19) ^ (W[-2] >>> 61) */ \ + V_XOR(YTMP1, YTMP3, YTMP1) \ + RND_RORX_1_8(h,a,b,c,d,e,f,g,i+1) \ + /* W[-2] >> 6 */ \ + V_SHIFT_R(YTMP4, W_Y_M2, 6) \ + RND_RORX_0_1(g,h,a,b,c,d,e,f,i+2) \ + /* (W[-2] >>> 19) ^ (W[-2] >>> 61) ^ (W[-2] >> 6) */ \ + V_XOR(YTMP1, YTMP4, YTMP1) \ + RND_RORX_0_2(g,h,a,b,c,d,e,f,i+2) \ + /* W[0] = W[-16] + W[-7] + s0(W[-15]) + s1(W[-2]) */ \ + V_ADD(W_Y_0, W_Y_0, YTMP1) \ + RND_RORX_0_3(g,h,a,b,c,d,e,f,i+2) \ + /* W[1], W[0], 0, 0 */ \ + VPERM2I128(W_Y_M2, W_Y_0, W_Y_0, 0x08) \ + RND_RORX_0_4(g,h,a,b,c,d,e,f,i+2) \ + RND_RORX_0_5(g,h,a,b,c,d,e,f,i+2) \ + /* W[-2] >> 19 */ \ + V_SHIFT_R(YTMP1, W_Y_M2, 19) \ + /* W[-2] << 45 */ \ + V_SHIFT_L(YTMP2, W_Y_M2, 45) \ + RND_RORX_0_6(g,h,a,b,c,d,e,f,i+2) \ + /* W[-2] >> 61 */ \ + V_SHIFT_R(YTMP3, W_Y_M2, 61) \ + /* W[-2] << 3 */ \ + V_SHIFT_L(YTMP4, W_Y_M2, 3) \ + /* W[-2] >>> 19 */ \ + V_OR(YTMP1, YTMP2, YTMP1) \ + RND_RORX_0_7(g,h,a,b,c,d,e,f,i+2) \ + /* W[-2] >>> 61 */ \ + V_OR(YTMP3, YTMP4, YTMP3) \ + RND_RORX_0_8(g,h,a,b,c,d,e,f,i+2) \ + /* (W[-2] >>> 19) ^ (W[-2] >>> 61) */ \ + V_XOR(YTMP1, YTMP3, YTMP1) \ + RND_RORX_1_1(f,g,h,a,b,c,d,e,i+3) \ + /* W[-2] >> 6 */ \ + V_SHIFT_R(YTMP4, W_Y_M2, 6) \ + RND_RORX_1_2(f,g,h,a,b,c,d,e,i+3) \ + RND_RORX_1_3(f,g,h,a,b,c,d,e,i+3) \ + /* (W[-2] >>> 19) ^ (W[-2] >>> 61) ^ (W[-2] >> 6) */ \ + V_XOR(YTMP1, YTMP4, YTMP1) \ + RND_RORX_1_4(f,g,h,a,b,c,d,e,i+3) \ + RND_RORX_1_5(f,g,h,a,b,c,d,e,i+3) \ + /* W[0] = W[-16] + W[-7] + s0(W[-15]) + s1(W[-2]) */ \ + V_ADD(W_Y_0, W_Y_0, YTMP1) \ + RND_RORX_1_6(f,g,h,a,b,c,d,e,i+3) \ + V_ADD_I(YTMP1, W_Y_0, rsi, i) \ + RND_RORX_1_7(f,g,h,a,b,c,d,e,i+3) \ + RND_RORX_1_8(f,g,h,a,b,c,d,e,i+3) \ + VMOVDQU_I(rsp, i, YTMP1) \ + +#define MsgSched2_AVX2_RORX(W_0,W_2,W_4,W_6,W_8,W_10,W_12,W_14,a,b,c,d,e, \ + f,g,h,i) \ + RND_RORX_0_1(a,b,c,d,e,f,g,h,i) \ + VPALIGNR(W_Y_M15, W_2, W_0, 8) \ + VPALIGNR(W_Y_M7, W_10, W_8, 8) \ + RND_RORX_0_2(a,b,c,d,e,f,g,h,i) \ + V_SHIFT_R(YTMP1, W_Y_M15, 1) \ + V_SHIFT_L(YTMP2, W_Y_M15, 63) \ + RND_RORX_0_3(a,b,c,d,e,f,g,h,i) \ + V_SHIFT_R(YTMP3, W_Y_M15, 8) \ + V_SHIFT_L(YTMP4, W_Y_M15, 56) \ + RND_RORX_0_4(a,b,c,d,e,f,g,h,i) \ + V_OR(YTMP1, YTMP2, YTMP1) \ + V_OR(YTMP3, YTMP4, YTMP3) \ + RND_RORX_0_5(a,b,c,d,e,f,g,h,i) \ + V_SHIFT_R(YTMP4, W_Y_M15, 7) \ + V_XOR(YTMP1, YTMP3, YTMP1) \ + RND_RORX_0_6(a,b,c,d,e,f,g,h,i) \ + V_XOR(YTMP1, YTMP4, YTMP1) \ + V_ADD(W_0, W_0, W_Y_M7) \ + RND_RORX_0_7(a,b,c,d,e,f,g,h,i) \ + RND_RORX_0_8(a,b,c,d,e,f,g,h,i) \ + V_ADD(W_0, W_0, YTMP1) \ + RND_RORX_1_1(h,a,b,c,d,e,f,g,i+1) \ + V_SHIFT_R(YTMP1, W_14, 19) \ + V_SHIFT_L(YTMP2, W_14, 45) \ + RND_RORX_1_2(h,a,b,c,d,e,f,g,i+1) \ + V_SHIFT_R(YTMP3, W_14, 61) \ + V_SHIFT_L(YTMP4, W_14, 3) \ + RND_RORX_1_3(h,a,b,c,d,e,f,g,i+1) \ + V_OR(YTMP1, YTMP2, YTMP1) \ + V_OR(YTMP3, YTMP4, YTMP3) \ + RND_RORX_1_4(h,a,b,c,d,e,f,g,i+1) \ + RND_RORX_1_5(h,a,b,c,d,e,f,g,i+1) \ + V_XOR(YTMP1, YTMP3, YTMP1) \ + V_SHIFT_R(YTMP4, W_14, 6) \ + RND_RORX_1_6(h,a,b,c,d,e,f,g,i+1) \ + RND_RORX_1_7(h,a,b,c,d,e,f,g,i+1) \ + V_XOR(YTMP1, YTMP4, YTMP1) \ + RND_RORX_1_8(h,a,b,c,d,e,f,g,i+1) \ + V_ADD(W_0, W_0, YTMP1) \ -#define Block_Y_xx_1(i, w_0, w_4, w_8, w_12)\ - MOVE_W_to_W_I_15(W_I_15y, w_0, w_4);\ - MOVE_W_to_W_I_7 (W_I_7y, w_8, w_12);\ - MOVE_W_to_W_I_2 (W_I_2y, w_12);\ +#define _INIT_MASK_Y(mask) \ + "vmovdqu %[mask], %%"#mask"\n\t" +#define INIT_MASK_Y(mask) \ + _INIT_MASK_Y(mask) -#define Block_Y_xx_2(i, w_0, w_4, w_8, w_12)\ - s0_1y (YMM_TEMP0, W_I_15y);\ +/* Load into YMM registers and swap endian. */ +#define _LOAD_BLOCK_W_Y_2(mask, ymm0, ymm1, reg, i) \ + /* buffer[0..15] => ymm0..ymm3; */ \ + "vmovdqu "#i"+ 0(%%"#reg"), %%"#ymm0"\n\t" \ + "vmovdqu "#i"+32(%%"#reg"), %%"#ymm1"\n\t" \ + "vpshufb %%"#mask", %%"#ymm0", %%"#ymm0"\n\t" \ + "vpshufb %%"#mask", %%"#ymm1", %%"#ymm1"\n\t" -#define Block_Y_xx_3(i, w_0, w_4, w_8, w_12)\ - s0_2y (YMM_TEMP0, W_I_15y);\ +#define LOAD_BLOCK_W_Y_2(mask, ymm1, ymm2, reg, i) \ + _LOAD_BLOCK_W_Y_2(mask, ymm1, ymm2, reg, i) -#define Block_Y_xx_4(i, w_0, w_4, w_8, w_12)\ - s0_3y (YMM_TEMP0, W_I_15y);\ +#define LOAD_BLOCK_W_Y(mask, reg) \ + LOAD_BLOCK_W_Y_2(mask, W_Y_0, W_Y_4 , reg, 0) \ + LOAD_BLOCK_W_Y_2(mask, W_Y_8, W_Y_12, reg, 64) -#define Block_Y_xx_5(i, w_0, w_4, w_8, w_12)\ - ADDy(W_I_TEMPy, w_0, YMM_TEMP0);\ +#define _SET_W_Y_2(ymm0, ymm1, ymm2, ymm3, reg, i) \ + "vpaddq "#i"+ 0(%%"#reg"), %%"#ymm0", %%"#ymm2"\n\t" \ + "vpaddq "#i"+32(%%"#reg"), %%"#ymm1", %%"#ymm3"\n\t" \ + "vmovdqu %%"#ymm2", "#i"+ 0("WX")\n\t" \ + "vmovdqu %%"#ymm3", "#i"+32("WX")\n\t" -#define Block_Y_xx_6(i, w_0, w_4, w_8, w_12)\ - ADDy(W_I_TEMPy, W_I_TEMPy, W_I_7y);\ - s1_1y (YMM_TEMP0, W_I_2y);\ +#define SET_W_Y_2(ymm0, ymm1, ymm2, ymm3, reg, i) \ + _SET_W_Y_2(ymm0, ymm1, ymm2, ymm3, reg, i) -#define Block_Y_xx_7(i, w_0, w_4, w_8, w_12)\ - s1_2y (YMM_TEMP0, W_I_2y);\ +#define SET_BLOCK_W_Y(reg) \ + SET_W_Y_2(W_Y_0, W_Y_4 , YTMP1, YTMP2, reg, 0) \ + SET_W_Y_2(W_Y_8, W_Y_12, YTMP1, YTMP2, reg, 64) -#define Block_Y_xx_8(i, w_0, w_4, w_8, w_12)\ - s1_3y (YMM_TEMP0, W_I_2y);\ - ADDy(w_0, W_I_TEMPy, YMM_TEMP0);\ +/* Load into YMM registers and swap endian. */ +#define _LOAD_BLOCK2_W_Y_2(mask, Y0, Y1, X0, X1, X8, X9, reg, i) \ + "vmovdqu "#i"+ 0(%%"#reg"), %%"#X0"\n\t" \ + "vmovdqu "#i"+ 16(%%"#reg"), %%"#X1"\n\t" \ + "vmovdqu "#i"+128(%%"#reg"), %%"#X8"\n\t" \ + "vmovdqu "#i"+144(%%"#reg"), %%"#X9"\n\t" \ + "vinserti128 $1, %%"#X8", %%"#Y0", %%"#Y0"\n\t" \ + "vinserti128 $1, %%"#X9", %%"#Y1", %%"#Y1"\n\t" \ + "vpshufb %%"#mask", %%"#Y0", %%"#Y0"\n\t" \ + "vpshufb %%"#mask", %%"#Y1", %%"#Y1"\n\t" -#define Block_Y_xx_9(i, w_0, w_4, w_8, w_12)\ - FEEDBACK1_to_W_I_2(W_I_2y, w_0);\ +#define LOAD_BLOCK2_W_Y_2(mask, Y0, Y1, X0, X1, X8, X9, reg, i) \ + _LOAD_BLOCK2_W_Y_2(mask, Y0, Y1, X0, X1, X8, X9, reg, i) -#define Block_Y_xx_10(i, w_0, w_4, w_8, w_12) \ - s1_1y (YMM_TEMP0, W_I_2y);\ +#define LOAD_BLOCK2_W_Y(mask, reg) \ + LOAD_BLOCK2_W_Y_2(mask, Y0, Y1, X0, X1, X8, X9, reg, 0) \ + LOAD_BLOCK2_W_Y_2(mask, Y2, Y3, X2, X3, X8, X9, reg, 32) \ + LOAD_BLOCK2_W_Y_2(mask, Y4, Y5, X4, X5, X8, X9, reg, 64) \ + LOAD_BLOCK2_W_Y_2(mask, Y6, Y7, X6, X7, X8, X9, reg, 96) \ -#define Block_Y_xx_11(i, w_0, w_4, w_8, w_12) \ - s1_2y (YMM_TEMP0, W_I_2y);\ +#define SET_BLOCK2_W_Y(reg) \ + SET_W_Y_2(Y0, Y1, YTMP1, YTMP2, reg, 0) \ + SET_W_Y_2(Y2, Y3, YTMP1, YTMP2, reg, 64) \ + SET_W_Y_2(Y4, Y5, YTMP1, YTMP2, reg, 128) \ + SET_W_Y_2(Y6, Y7, YTMP1, YTMP2, reg, 192) -#define Block_Y_xx_12(i, w_0, w_4, w_8, w_12)\ - s1_3y (YMM_TEMP0, W_I_2y);\ - ADDy(w_0, W_I_TEMPy, YMM_TEMP0);\ - MOVE_to_MEMy(w,0, w_4);\ +static const word64 K512_AVX2[160] = { + W64LIT(0x428a2f98d728ae22), W64LIT(0x7137449123ef65cd), + W64LIT(0x428a2f98d728ae22), W64LIT(0x7137449123ef65cd), + W64LIT(0xb5c0fbcfec4d3b2f), W64LIT(0xe9b5dba58189dbbc), + W64LIT(0xb5c0fbcfec4d3b2f), W64LIT(0xe9b5dba58189dbbc), + W64LIT(0x3956c25bf348b538), W64LIT(0x59f111f1b605d019), + W64LIT(0x3956c25bf348b538), W64LIT(0x59f111f1b605d019), + W64LIT(0x923f82a4af194f9b), W64LIT(0xab1c5ed5da6d8118), + W64LIT(0x923f82a4af194f9b), W64LIT(0xab1c5ed5da6d8118), + W64LIT(0xd807aa98a3030242), W64LIT(0x12835b0145706fbe), + W64LIT(0xd807aa98a3030242), W64LIT(0x12835b0145706fbe), + W64LIT(0x243185be4ee4b28c), W64LIT(0x550c7dc3d5ffb4e2), + W64LIT(0x243185be4ee4b28c), W64LIT(0x550c7dc3d5ffb4e2), + W64LIT(0x72be5d74f27b896f), W64LIT(0x80deb1fe3b1696b1), + W64LIT(0x72be5d74f27b896f), W64LIT(0x80deb1fe3b1696b1), + W64LIT(0x9bdc06a725c71235), W64LIT(0xc19bf174cf692694), + W64LIT(0x9bdc06a725c71235), W64LIT(0xc19bf174cf692694), + W64LIT(0xe49b69c19ef14ad2), W64LIT(0xefbe4786384f25e3), + W64LIT(0xe49b69c19ef14ad2), W64LIT(0xefbe4786384f25e3), + W64LIT(0x0fc19dc68b8cd5b5), W64LIT(0x240ca1cc77ac9c65), + W64LIT(0x0fc19dc68b8cd5b5), W64LIT(0x240ca1cc77ac9c65), + W64LIT(0x2de92c6f592b0275), W64LIT(0x4a7484aa6ea6e483), + W64LIT(0x2de92c6f592b0275), W64LIT(0x4a7484aa6ea6e483), + W64LIT(0x5cb0a9dcbd41fbd4), W64LIT(0x76f988da831153b5), + W64LIT(0x5cb0a9dcbd41fbd4), W64LIT(0x76f988da831153b5), + W64LIT(0x983e5152ee66dfab), W64LIT(0xa831c66d2db43210), + W64LIT(0x983e5152ee66dfab), W64LIT(0xa831c66d2db43210), + W64LIT(0xb00327c898fb213f), W64LIT(0xbf597fc7beef0ee4), + W64LIT(0xb00327c898fb213f), W64LIT(0xbf597fc7beef0ee4), + W64LIT(0xc6e00bf33da88fc2), W64LIT(0xd5a79147930aa725), + W64LIT(0xc6e00bf33da88fc2), W64LIT(0xd5a79147930aa725), + W64LIT(0x06ca6351e003826f), W64LIT(0x142929670a0e6e70), + W64LIT(0x06ca6351e003826f), W64LIT(0x142929670a0e6e70), + W64LIT(0x27b70a8546d22ffc), W64LIT(0x2e1b21385c26c926), + W64LIT(0x27b70a8546d22ffc), W64LIT(0x2e1b21385c26c926), + W64LIT(0x4d2c6dfc5ac42aed), W64LIT(0x53380d139d95b3df), + W64LIT(0x4d2c6dfc5ac42aed), W64LIT(0x53380d139d95b3df), + W64LIT(0x650a73548baf63de), W64LIT(0x766a0abb3c77b2a8), + W64LIT(0x650a73548baf63de), W64LIT(0x766a0abb3c77b2a8), + W64LIT(0x81c2c92e47edaee6), W64LIT(0x92722c851482353b), + W64LIT(0x81c2c92e47edaee6), W64LIT(0x92722c851482353b), + W64LIT(0xa2bfe8a14cf10364), W64LIT(0xa81a664bbc423001), + W64LIT(0xa2bfe8a14cf10364), W64LIT(0xa81a664bbc423001), + W64LIT(0xc24b8b70d0f89791), W64LIT(0xc76c51a30654be30), + W64LIT(0xc24b8b70d0f89791), W64LIT(0xc76c51a30654be30), + W64LIT(0xd192e819d6ef5218), W64LIT(0xd69906245565a910), + W64LIT(0xd192e819d6ef5218), W64LIT(0xd69906245565a910), + W64LIT(0xf40e35855771202a), W64LIT(0x106aa07032bbd1b8), + W64LIT(0xf40e35855771202a), W64LIT(0x106aa07032bbd1b8), + W64LIT(0x19a4c116b8d2d0c8), W64LIT(0x1e376c085141ab53), + W64LIT(0x19a4c116b8d2d0c8), W64LIT(0x1e376c085141ab53), + W64LIT(0x2748774cdf8eeb99), W64LIT(0x34b0bcb5e19b48a8), + W64LIT(0x2748774cdf8eeb99), W64LIT(0x34b0bcb5e19b48a8), + W64LIT(0x391c0cb3c5c95a63), W64LIT(0x4ed8aa4ae3418acb), + W64LIT(0x391c0cb3c5c95a63), W64LIT(0x4ed8aa4ae3418acb), + W64LIT(0x5b9cca4f7763e373), W64LIT(0x682e6ff3d6b2b8a3), + W64LIT(0x5b9cca4f7763e373), W64LIT(0x682e6ff3d6b2b8a3), + W64LIT(0x748f82ee5defb2fc), W64LIT(0x78a5636f43172f60), + W64LIT(0x748f82ee5defb2fc), W64LIT(0x78a5636f43172f60), + W64LIT(0x84c87814a1f0ab72), W64LIT(0x8cc702081a6439ec), + W64LIT(0x84c87814a1f0ab72), W64LIT(0x8cc702081a6439ec), + W64LIT(0x90befffa23631e28), W64LIT(0xa4506cebde82bde9), + W64LIT(0x90befffa23631e28), W64LIT(0xa4506cebde82bde9), + W64LIT(0xbef9a3f7b2c67915), W64LIT(0xc67178f2e372532b), + W64LIT(0xbef9a3f7b2c67915), W64LIT(0xc67178f2e372532b), + W64LIT(0xca273eceea26619c), W64LIT(0xd186b8c721c0c207), + W64LIT(0xca273eceea26619c), W64LIT(0xd186b8c721c0c207), + W64LIT(0xeada7dd6cde0eb1e), W64LIT(0xf57d4f7fee6ed178), + W64LIT(0xeada7dd6cde0eb1e), W64LIT(0xf57d4f7fee6ed178), + W64LIT(0x06f067aa72176fba), W64LIT(0x0a637dc5a2c898a6), + W64LIT(0x06f067aa72176fba), W64LIT(0x0a637dc5a2c898a6), + W64LIT(0x113f9804bef90dae), W64LIT(0x1b710b35131c471b), + W64LIT(0x113f9804bef90dae), W64LIT(0x1b710b35131c471b), + W64LIT(0x28db77f523047d84), W64LIT(0x32caab7b40c72493), + W64LIT(0x28db77f523047d84), W64LIT(0x32caab7b40c72493), + W64LIT(0x3c9ebe0a15c9bebc), W64LIT(0x431d67c49c100d4c), + W64LIT(0x3c9ebe0a15c9bebc), W64LIT(0x431d67c49c100d4c), + W64LIT(0x4cc5d4becb3e42b6), W64LIT(0x597f299cfc657e2a), + W64LIT(0x4cc5d4becb3e42b6), W64LIT(0x597f299cfc657e2a), + W64LIT(0x5fcb6fab3ad6faec), W64LIT(0x6c44198c4a475817), + W64LIT(0x5fcb6fab3ad6faec), W64LIT(0x6c44198c4a475817) +}; +static const word64* K512_AVX2_END = &K512_AVX2[128]; - -static INLINE void Block_Y_0_1(void) { Block_Y_xx_1(0, W_0y, W_4y, W_8y, W_12y); } -static INLINE void Block_Y_0_2(void) { Block_Y_xx_2(0, W_0y, W_4y, W_8y, W_12y); } -static INLINE void Block_Y_0_3(void) { Block_Y_xx_3(0, W_0y, W_4y, W_8y, W_12y); } -static INLINE void Block_Y_0_4(void) { Block_Y_xx_4(0, W_0y, W_4y, W_8y, W_12y); } -static INLINE void Block_Y_0_5(void) { Block_Y_xx_5(0, W_0y, W_4y, W_8y, W_12y); } -static INLINE void Block_Y_0_6(void) { Block_Y_xx_6(0, W_0y, W_4y, W_8y, W_12y); } -static INLINE void Block_Y_0_7(void) { Block_Y_xx_7(0, W_0y, W_4y, W_8y, W_12y); } -static INLINE void Block_Y_0_8(void) { Block_Y_xx_8(0, W_0y, W_4y, W_8y, W_12y); } -static INLINE void Block_Y_0_9(void) { Block_Y_xx_9(0, W_0y, W_4y, W_8y, W_12y); } -static INLINE void Block_Y_0_10(void){ Block_Y_xx_10(0, W_0y, W_4y, W_8y, W_12y); } -static INLINE void Block_Y_0_11(void){ Block_Y_xx_11(0, W_0y, W_4y, W_8y, W_12y); } -static INLINE void Block_Y_0_12(word64 *w){ Block_Y_xx_12(0, W_0y, W_4y, W_8y, W_12y); } - -static INLINE void Block_Y_4_1(void) { Block_Y_xx_1(4, W_4y, W_8y, W_12y, W_0y); } -static INLINE void Block_Y_4_2(void) { Block_Y_xx_2(4, W_4y, W_8y, W_12y, W_0y); } -static INLINE void Block_Y_4_3(void) { Block_Y_xx_3(4, W_4y, W_8y, W_12y, W_0y); } -static INLINE void Block_Y_4_4(void) { Block_Y_xx_4(4, W_4y, W_8y, W_12y, W_0y); } -static INLINE void Block_Y_4_5(void) { Block_Y_xx_5(4, W_4y, W_8y, W_12y, W_0y); } -static INLINE void Block_Y_4_6(void) { Block_Y_xx_6(4, W_4y, W_8y, W_12y, W_0y); } -static INLINE void Block_Y_4_7(void) { Block_Y_xx_7(4, W_4y, W_8y, W_12y, W_0y); } -static INLINE void Block_Y_4_8(void) { Block_Y_xx_8(4, W_4y, W_8y, W_12y, W_0y); } -static INLINE void Block_Y_4_9(void) { Block_Y_xx_9(4, W_4y, W_8y, W_12y, W_0y); } -static INLINE void Block_Y_4_10(void) { Block_Y_xx_10(4, W_4y, W_8y, W_12y, W_0y); } -static INLINE void Block_Y_4_11(void) { Block_Y_xx_11(4, W_4y, W_8y, W_12y, W_0y); } -static INLINE void Block_Y_4_12(word64 *w) { Block_Y_xx_12(4, W_4y, W_8y, W_12y, W_0y); } - -static INLINE void Block_Y_8_1(void) { Block_Y_xx_1(8, W_8y, W_12y, W_0y, W_4y); } -static INLINE void Block_Y_8_2(void) { Block_Y_xx_2(8, W_8y, W_12y, W_0y, W_4y); } -static INLINE void Block_Y_8_3(void) { Block_Y_xx_3(8, W_8y, W_12y, W_0y, W_4y); } -static INLINE void Block_Y_8_4(void) { Block_Y_xx_4(8, W_8y, W_12y, W_0y, W_4y); } -static INLINE void Block_Y_8_5(void) { Block_Y_xx_5(8, W_8y, W_12y, W_0y, W_4y); } -static INLINE void Block_Y_8_6(void) { Block_Y_xx_6(8, W_8y, W_12y, W_0y, W_4y); } -static INLINE void Block_Y_8_7(void) { Block_Y_xx_7(8, W_8y, W_12y, W_0y, W_4y); } -static INLINE void Block_Y_8_8(void) { Block_Y_xx_8(8, W_8y, W_12y, W_0y, W_4y); } -static INLINE void Block_Y_8_9(void) { Block_Y_xx_9(8, W_8y, W_12y, W_0y, W_4y); } -static INLINE void Block_Y_8_10(void) { Block_Y_xx_10(8, W_8y, W_12y, W_0y, W_4y); } -static INLINE void Block_Y_8_11(void) { Block_Y_xx_11(8, W_8y, W_12y, W_0y, W_4y); } -static INLINE void Block_Y_8_12(word64 *w) { Block_Y_xx_12(8, W_8y, W_12y, W_0y, W_4y); } - -static INLINE void Block_Y_12_1(void) { Block_Y_xx_1(12, W_12y, W_0y, W_4y, W_8y); } -static INLINE void Block_Y_12_2(void) { Block_Y_xx_2(12, W_12y, W_0y, W_4y, W_8y); } -static INLINE void Block_Y_12_3(void) { Block_Y_xx_3(12, W_12y, W_0y, W_4y, W_8y); } -static INLINE void Block_Y_12_4(void) { Block_Y_xx_4(12, W_12y, W_0y, W_4y, W_8y); } -static INLINE void Block_Y_12_5(void) { Block_Y_xx_5(12, W_12y, W_0y, W_4y, W_8y); } -static INLINE void Block_Y_12_6(void) { Block_Y_xx_6(12, W_12y, W_0y, W_4y, W_8y); } -static INLINE void Block_Y_12_7(void) { Block_Y_xx_7(12, W_12y, W_0y, W_4y, W_8y); } -static INLINE void Block_Y_12_8(void) { Block_Y_xx_8(12, W_12y, W_0y, W_4y, W_8y); } -static INLINE void Block_Y_12_9(void) { Block_Y_xx_9(12, W_12y, W_0y, W_4y, W_8y); } -static INLINE void Block_Y_12_10(void) { Block_Y_xx_10(12, W_12y, W_0y, W_4y, W_8y); } -static INLINE void Block_Y_12_11(void) { Block_Y_xx_11(12, W_12y, W_0y, W_4y, W_8y); } -static INLINE void Block_Y_12_12(word64 *w) { Block_Y_xx_12(12, W_12y, W_0y, W_4y, W_8y); } - - -static int Transform_AVX2(wc_Sha512* sha512) +static int Transform_Sha512_AVX2(wc_Sha512* sha512) { - const word64* K = K512; - word64 w[4]; - word32 j; - word64 T[8]; + __asm__ __volatile__ ( - /* Copy digest to working vars */ - XMEMCPY(T, sha512->digest, sizeof(T)); + /* 16 Ws plus loop counter and K512. */ + "subq $136, %%rsp\n\t" + "leaq 64(%[sha512]), %%rax\n\t" - W_from_buff_Y(sha512->buffer); - MOVE_to_MEMy(w,0, W_0y); - for (j = 0; j < 80; j += 16) { - Ry_1( 0, w[0]); Block_Y_0_1(); Ry_2( 0, w[0]); Block_Y_0_2(); - Ry_3( 0, w[0]); Block_Y_0_3(); - Ry_1( 1, w[1]); Block_Y_0_4(); Ry_2( 1, w[1]); Block_Y_0_5(); - Ry_3( 1, w[1]); Block_Y_0_6(); - Ry_1( 2, w[2]); Block_Y_0_7(); Ry_2( 2, w[2]); Block_Y_0_8(); - Ry_3( 2, w[2]); Block_Y_0_9(); - Ry_1( 3, w[3]); Block_Y_0_10();Ry_2( 3, w[3]); Block_Y_0_11(); - Ry_3( 3, w[3]); Block_Y_0_12(w); + INIT_MASK(MASK_Y) + LOAD_DIGEST() - Ry_1( 4, w[0]); Block_Y_4_1(); Ry_2( 4, w[0]); Block_Y_4_2(); - Ry_3( 4, w[0]); Block_Y_4_3(); - Ry_1( 5, w[1]); Block_Y_4_4(); Ry_2( 5, w[1]); Block_Y_4_5(); - Ry_3( 5, w[1]); Block_Y_4_6(); - Ry_1( 6, w[2]); Block_Y_4_7(); Ry_2( 6, w[2]); Block_Y_4_8(); - Ry_3( 6, w[2]); Block_Y_4_9(); - Ry_1( 7, w[3]); Block_Y_4_10(); Ry_2( 7, w[3]);Block_Y_4_11(); - Ry_3( 7, w[3]);Block_Y_4_12(w); + LOAD_BLOCK_W_Y(MASK_Y, rax) - Ry_1( 8, w[0]); Block_Y_8_1(); Ry_2( 8, w[0]); Block_Y_8_2(); - Ry_3( 8, w[0]); Block_Y_8_3(); - Ry_1( 9, w[1]); Block_Y_8_4(); Ry_2( 9, w[1]); Block_Y_8_5(); - Ry_3( 9, w[1]); Block_Y_8_6(); - Ry_1(10, w[2]); Block_Y_8_7(); Ry_2(10, w[2]); Block_Y_8_8(); - Ry_3(10, w[2]); Block_Y_8_9(); - Ry_1(11, w[3]); Block_Y_8_10();Ry_2(11, w[3]); Block_Y_8_11(); - Ry_3(11, w[3]); Block_Y_8_12(w); + "movl $4, 16*8("WX")\n\t" + "leaq %[K512], %%rsi\n\t" + /* b */ + "movq %%r9, "L4"\n\t" + /* e */ + "movq %%r12, "L1"\n\t" + /* b ^ c */ + "xorq %%r10, "L4"\n\t" - Ry_1(12, w[0]); Block_Y_12_1(); Ry_2(12, w[0]); Block_Y_12_2(); - Ry_3(12, w[0]); Block_Y_12_3(); - Ry_1(13, w[1]); Block_Y_12_4(); Ry_2(13, w[1]); Block_Y_12_5(); - Ry_3(13, w[1]); Block_Y_12_6(); - Ry_1(14, w[2]); Block_Y_12_7(); Ry_2(14, w[2]); Block_Y_12_8(); - Ry_3(14, w[2]); Block_Y_12_9(); - Ry_1(15, w[3]); Block_Y_12_10();Ry_2(15, w[3]); Block_Y_12_11(); - Ry_3(15, w[3]);Block_Y_12_12(w); - } + SET_BLOCK_W_Y(rsi) - /* Add the working vars back into digest */ - sha512->digest[0] += a(0); - sha512->digest[1] += b(0); - sha512->digest[2] += c(0); - sha512->digest[3] += d(0); - sha512->digest[4] += e(0); - sha512->digest[5] += f(0); - sha512->digest[6] += g(0); - sha512->digest[7] += h(0); + "# Start of 16 rounds\n" + "1:\n\t" - /* Wipe variables */ -#if !defined(HAVE_INTEL_AVX1) && !defined(HAVE_INTEL_AVX2) - XMEMSET(W, 0, sizeof(word64) * 16); -#endif - XMEMSET(T, 0, sizeof(T)); + "addq $128, %%rsi\n\t" + + MsgSched4_AVX2(W_Y_0,W_Y_4,W_Y_8,W_Y_12,RA,RB,RC,RD,RE,RF,RG,RH, 0) + MsgSched4_AVX2(W_Y_4,W_Y_8,W_Y_12,W_Y_0,RE,RF,RG,RH,RA,RB,RC,RD, 4) + MsgSched4_AVX2(W_Y_8,W_Y_12,W_Y_0,W_Y_4,RA,RB,RC,RD,RE,RF,RG,RH, 8) + MsgSched4_AVX2(W_Y_12,W_Y_0,W_Y_4,W_Y_8,RE,RF,RG,RH,RA,RB,RC,RD,12) + + SET_BLOCK_W_Y(rsi) + + "subl $1, 16*8("WX")\n\t" + "jne 1b\n\t" + + RND_ALL_2(RA,RB,RC,RD,RE,RF,RG,RH, 0) + RND_ALL_2(RG,RH,RA,RB,RC,RD,RE,RF, 2) + RND_ALL_2(RE,RF,RG,RH,RA,RB,RC,RD, 4) + RND_ALL_2(RC,RD,RE,RF,RG,RH,RA,RB, 6) + + RND_ALL_2(RA,RB,RC,RD,RE,RF,RG,RH, 8) + RND_ALL_2(RG,RH,RA,RB,RC,RD,RE,RF,10) + RND_ALL_2(RE,RF,RG,RH,RA,RB,RC,RD,12) + RND_ALL_2(RC,RD,RE,RF,RG,RH,RA,RB,14) + + STORE_ADD_DIGEST() + + "addq $136, %%rsp\n\t" + + : + : [mask] "m" (mBYTE_FLIP_MASK_Y), + [sha512] "r" (sha512), + [K512] "m" (K512) + : WORK_REGS, STATE_REGS, YMM_REGS, "memory", "rsi" + ); return 0; } + +static int Transform_Sha512_AVX2_Len(wc_Sha512* sha512, word32 len) +{ + if ((len & WC_SHA512_BLOCK_SIZE) != 0) { + Transform_Sha512_AVX2(sha512); + sha512->data += WC_SHA512_BLOCK_SIZE; + len -= WC_SHA512_BLOCK_SIZE; + if (len == 0) + return 0; + } + + __asm__ __volatile__ ( + + "movq 224(%[sha512]), %%rcx\n\t" + + INIT_MASK(MASK_Y) + LOAD_DIGEST() + + "# Start of processing two blocks\n" + "2:\n\t" + + "subq $1344, %%rsp\n\t" + "leaq %[K512], %%rsi\n\t" + + /* L4 = b */ + "movq %%r9, "L4"\n\t" + /* e */ + "movq %%r12, "L1"\n\t" + + LOAD_BLOCK2_W_Y(MASK_Y, rcx) + + /* L4 = b ^ c */ + "xorq %%r10, "L4"\n\t" + "\n" + "1:\n\t" + SET_BLOCK2_W_Y(rsi) + MsgSched2_AVX2(Y0,Y1,Y2,Y3,Y4,Y5,Y6,Y7,RA,RB,RC,RD,RE,RF,RG,RH, 0) + MsgSched2_AVX2(Y1,Y2,Y3,Y4,Y5,Y6,Y7,Y0,RG,RH,RA,RB,RC,RD,RE,RF, 4) + MsgSched2_AVX2(Y2,Y3,Y4,Y5,Y6,Y7,Y0,Y1,RE,RF,RG,RH,RA,RB,RC,RD, 8) + MsgSched2_AVX2(Y3,Y4,Y5,Y6,Y7,Y0,Y1,Y2,RC,RD,RE,RF,RG,RH,RA,RB,12) + MsgSched2_AVX2(Y4,Y5,Y6,Y7,Y0,Y1,Y2,Y3,RA,RB,RC,RD,RE,RF,RG,RH,16) + MsgSched2_AVX2(Y5,Y6,Y7,Y0,Y1,Y2,Y3,Y4,RG,RH,RA,RB,RC,RD,RE,RF,20) + MsgSched2_AVX2(Y6,Y7,Y0,Y1,Y2,Y3,Y4,Y5,RE,RF,RG,RH,RA,RB,RC,RD,24) + MsgSched2_AVX2(Y7,Y0,Y1,Y2,Y3,Y4,Y5,Y6,RC,RD,RE,RF,RG,RH,RA,RB,28) + "addq $256, %%rsi\n\t" + "addq $256, %%rsp\n\t" + "cmpq %[K512_END], %%rsi\n\t" + "jne 1b\n\t" + + SET_BLOCK2_W_Y(rsi) + RND_ALL_2(RA,RB,RC,RD,RE,RF,RG,RH, 0) + RND_ALL_2(RG,RH,RA,RB,RC,RD,RE,RF, 4) + RND_ALL_2(RE,RF,RG,RH,RA,RB,RC,RD, 8) + RND_ALL_2(RC,RD,RE,RF,RG,RH,RA,RB,12) + + RND_ALL_2(RA,RB,RC,RD,RE,RF,RG,RH,16) + RND_ALL_2(RG,RH,RA,RB,RC,RD,RE,RF,20) + RND_ALL_2(RE,RF,RG,RH,RA,RB,RC,RD,24) + RND_ALL_2(RC,RD,RE,RF,RG,RH,RA,RB,28) + "subq $1024, %%rsp\n\t" + + ADD_DIGEST() + STORE_DIGEST() + + /* L4 = b */ + "movq %%r9, "L4"\n\t" + /* e */ + "movq %%r12, "L1"\n\t" + /* L4 = b ^ c */ + "xorq %%r10, "L4"\n\t" + + "movq $5, %%rsi\n\t" + "\n" + "3:\n\t" + RND_ALL_2(RA,RB,RC,RD,RE,RF,RG,RH, 2) + RND_ALL_2(RG,RH,RA,RB,RC,RD,RE,RF, 6) + RND_ALL_2(RE,RF,RG,RH,RA,RB,RC,RD,10) + RND_ALL_2(RC,RD,RE,RF,RG,RH,RA,RB,14) + + RND_ALL_2(RA,RB,RC,RD,RE,RF,RG,RH,18) + RND_ALL_2(RG,RH,RA,RB,RC,RD,RE,RF,22) + RND_ALL_2(RE,RF,RG,RH,RA,RB,RC,RD,26) + RND_ALL_2(RC,RD,RE,RF,RG,RH,RA,RB,30) + "addq $256, %%rsp\n\t" + "subq $1, %%rsi\n\t" + "jnz 3b\n\t" + + ADD_DIGEST() + + "movq 224(%[sha512]), %%rcx\n\t" + "addq $64, %%rsp\n\t" + "addq $256, %%rcx\n\t" + "subl $256, %[len]\n\t" + "movq %%rcx, 224(%[sha512])\n\t" + + STORE_DIGEST() + + "jnz 2b\n\t" + + : + : [mask] "m" (mBYTE_FLIP_MASK_Y), + [len] "m" (len), + [sha512] "r" (sha512), + [K512] "m" (K512_AVX2), + [K512_END] "m" (K512_AVX2_END) + : WORK_REGS, STATE_REGS, YMM_REGS, "memory", "rsi" + ); + + return 0; +} + +#ifdef HAVE_INTEL_RORX +static int Transform_Sha512_AVX2_RORX(wc_Sha512* sha512) +{ + __asm__ __volatile__ ( + + /* 16 Ws plus loop counter. */ + "subq $136, %%rsp\n\t" + "leaq 64(%[sha512]), "L2"\n\t" + + INIT_MASK(MASK_Y) + LOAD_DIGEST() + + LOAD_BLOCK_W_Y(MASK_Y, rcx) + + "movl $4, 16*8("WX")\n\t" + "leaq %[K512], %%rsi\n\t" + /* b */ + "movq %%r9, "L4"\n\t" + /* L3 = 0 (add to prev h) */ + "xorq "L3", "L3"\n\t" + /* b ^ c */ + "xorq %%r10, "L4"\n\t" + + SET_BLOCK_W_Y(rsi) + + "# Start of 16 rounds\n" + "1:\n\t" + + "addq $128, %%rsi\n\t" + + MsgSched4_AVX2_RORX_SET(W_Y_0,W_Y_4,W_Y_8,W_Y_12,RA,RB,RC,RD,RE,RF,RG,RH, 0) + MsgSched4_AVX2_RORX_SET(W_Y_4,W_Y_8,W_Y_12,W_Y_0,RE,RF,RG,RH,RA,RB,RC,RD, 4) + MsgSched4_AVX2_RORX_SET(W_Y_8,W_Y_12,W_Y_0,W_Y_4,RA,RB,RC,RD,RE,RF,RG,RH, 8) + MsgSched4_AVX2_RORX_SET(W_Y_12,W_Y_0,W_Y_4,W_Y_8,RE,RF,RG,RH,RA,RB,RC,RD,12) + + "subl $1, 16*8(%%rsp)\n\t" + "jnz 1b\n\t" + + RND_RORX_ALL_4(RA,RB,RC,RD,RE,RF,RG,RH, 0) + RND_RORX_ALL_4(RE,RF,RG,RH,RA,RB,RC,RD, 4) + RND_RORX_ALL_4(RA,RB,RC,RD,RE,RF,RG,RH, 8) + RND_RORX_ALL_4(RE,RF,RG,RH,RA,RB,RC,RD,12) + /* Prev RND: h += Maj(a,b,c) */ + "addq "L3", %%r8\n\t" + "addq $136, %%rsp\n\t" + + STORE_ADD_DIGEST() + + : + : [mask] "m" (mBYTE_FLIP_MASK_Y), + [sha512] "r" (sha512), + [K512] "m" (K512) + : WORK_REGS, STATE_REGS, YMM_REGS, "memory", "rsi" + ); + + return 0; +} + +static int Transform_Sha512_AVX2_RORX_Len(wc_Sha512* sha512, word32 len) +{ + if ((len & WC_SHA512_BLOCK_SIZE) != 0) { + Transform_Sha512_AVX2_RORX(sha512); + sha512->data += WC_SHA512_BLOCK_SIZE; + len -= WC_SHA512_BLOCK_SIZE; + if (len == 0) + return 0; + } + + __asm__ __volatile__ ( + + "movq 224(%[sha512]), %%rax\n\t" + + INIT_MASK(MASK_Y) + LOAD_DIGEST() + + "# Start of processing two blocks\n" + "2:\n\t" + + "subq $1344, %%rsp\n\t" + "leaq %[K512], %%rsi\n\t" + + /* L4 = b */ + "movq %%r9, "L4"\n\t" + /* L3 = 0 (add to prev h) */ + "xorq "L3", "L3"\n\t" + + LOAD_BLOCK2_W_Y(MASK_Y, rax) + + /* L4 = b ^ c */ + "xorq %%r10, "L4"\n\t" + "\n" + "1:\n\t" + SET_BLOCK2_W_Y(rsi) + MsgSched2_AVX2_RORX(Y0,Y1,Y2,Y3,Y4,Y5,Y6,Y7,RA,RB,RC,RD,RE,RF,RG,RH, 0) + MsgSched2_AVX2_RORX(Y1,Y2,Y3,Y4,Y5,Y6,Y7,Y0,RG,RH,RA,RB,RC,RD,RE,RF, 4) + MsgSched2_AVX2_RORX(Y2,Y3,Y4,Y5,Y6,Y7,Y0,Y1,RE,RF,RG,RH,RA,RB,RC,RD, 8) + MsgSched2_AVX2_RORX(Y3,Y4,Y5,Y6,Y7,Y0,Y1,Y2,RC,RD,RE,RF,RG,RH,RA,RB,12) + MsgSched2_AVX2_RORX(Y4,Y5,Y6,Y7,Y0,Y1,Y2,Y3,RA,RB,RC,RD,RE,RF,RG,RH,16) + MsgSched2_AVX2_RORX(Y5,Y6,Y7,Y0,Y1,Y2,Y3,Y4,RG,RH,RA,RB,RC,RD,RE,RF,20) + MsgSched2_AVX2_RORX(Y6,Y7,Y0,Y1,Y2,Y3,Y4,Y5,RE,RF,RG,RH,RA,RB,RC,RD,24) + MsgSched2_AVX2_RORX(Y7,Y0,Y1,Y2,Y3,Y4,Y5,Y6,RC,RD,RE,RF,RG,RH,RA,RB,28) + "addq $256, %%rsi\n\t" + "addq $256, %%rsp\n\t" + "cmpq %[K512_END], %%rsi\n\t" + "jne 1b\n\t" + + SET_BLOCK2_W_Y(rsi) + RND_RORX_ALL_2(RA,RB,RC,RD,RE,RF,RG,RH, 0) + RND_RORX_ALL_2(RG,RH,RA,RB,RC,RD,RE,RF, 4) + RND_RORX_ALL_2(RE,RF,RG,RH,RA,RB,RC,RD, 8) + RND_RORX_ALL_2(RC,RD,RE,RF,RG,RH,RA,RB,12) + + RND_RORX_ALL_2(RA,RB,RC,RD,RE,RF,RG,RH,16) + RND_RORX_ALL_2(RG,RH,RA,RB,RC,RD,RE,RF,20) + RND_RORX_ALL_2(RE,RF,RG,RH,RA,RB,RC,RD,24) + RND_RORX_ALL_2(RC,RD,RE,RF,RG,RH,RA,RB,28) + "addq "L3", %%r8\n\t" + "subq $1024, %%rsp\n\t" + + ADD_DIGEST() + STORE_DIGEST() + + /* L4 = b */ + "movq %%r9, "L4"\n\t" + /* L3 = 0 (add to prev h) */ + "xorq "L3", "L3"\n\t" + /* L4 = b ^ c */ + "xorq %%r10, "L4"\n\t" + + "movq $5, %%rsi\n\t" + "\n" + "3:\n\t" + RND_RORX_ALL_2(RA,RB,RC,RD,RE,RF,RG,RH, 2) + RND_RORX_ALL_2(RG,RH,RA,RB,RC,RD,RE,RF, 6) + RND_RORX_ALL_2(RE,RF,RG,RH,RA,RB,RC,RD,10) + RND_RORX_ALL_2(RC,RD,RE,RF,RG,RH,RA,RB,14) + + RND_RORX_ALL_2(RA,RB,RC,RD,RE,RF,RG,RH,18) + RND_RORX_ALL_2(RG,RH,RA,RB,RC,RD,RE,RF,22) + RND_RORX_ALL_2(RE,RF,RG,RH,RA,RB,RC,RD,26) + RND_RORX_ALL_2(RC,RD,RE,RF,RG,RH,RA,RB,30) + "addq $256, %%rsp\n\t" + "subq $1, %%rsi\n\t" + "jnz 3b\n\t" + + "addq "L3", %%r8\n\t" + + ADD_DIGEST() + + "movq 224(%[sha512]), %%rax\n\t" + "addq $64, %%rsp\n\t" + "addq $256, %%rax\n\t" + "subl $256, %[len]\n\t" + "movq %%rax, 224(%[sha512])\n\t" + + STORE_DIGEST() + + "jnz 2b\n\t" + + : + : [mask] "m" (mBYTE_FLIP_MASK_Y), + [len] "m" (len), + [sha512] "r" (sha512), + [K512] "m" (K512_AVX2), + [K512_END] "m" (K512_AVX2_END) + : WORK_REGS, STATE_REGS, YMM_REGS, "memory", "rsi" + ); + + return 0; +} +#endif /* HAVE_INTEL_RORX */ #endif /* HAVE_INTEL_AVX2 */ diff --git a/wolfcrypt/src/tfm.c b/wolfcrypt/src/tfm.c index 159b8e6ca2..2044102993 100644 --- a/wolfcrypt/src/tfm.c +++ b/wolfcrypt/src/tfm.c @@ -2600,7 +2600,7 @@ int mp_montgomery_calc_normalization(mp_int *a, mp_int *b) #if defined(WOLFSSL_KEY_GEN) || defined(HAVE_COMP_KEY) || \ - defined(WOLFSSL_DEBUG_MATH) + defined(WOLFSSL_DEBUG_MATH) || defined(DEBUG_WOLFSSL) #ifdef WOLFSSL_KEY_GEN /* swap the elements of two integers, for cases where you can't simply swap the @@ -3252,7 +3252,7 @@ int mp_set(fp_int *a, fp_digit b) #endif #if defined(WOLFSSL_KEY_GEN) || defined(HAVE_COMP_KEY) || \ - defined(WOLFSSL_DEBUG_MATH) + defined(WOLFSSL_DEBUG_MATH) || defined(DEBUG_WOLFSSL) /* returns size of ASCII representation */ int mp_radix_size (mp_int *a, int radix, int *size) @@ -3373,7 +3373,7 @@ void mp_dump(const char* desc, mp_int* a, byte verbose) printf("%s: ptr=%p, used=%d, sign=%d, size=%d, fpd=%d\n", desc, a, a->used, a->sign, size, (int)sizeof(fp_digit)); - mp_toradix(a, buffer, 16); + mp_tohex(a, buffer); printf(" %s\n ", buffer); if (verbose) { diff --git a/wolfcrypt/test/test.c b/wolfcrypt/test/test.c index 63c80fb3a1..87cbf081a1 100644 --- a/wolfcrypt/test/test.c +++ b/wolfcrypt/test/test.c @@ -293,7 +293,9 @@ int scrypt_test(void); #ifdef HAVE_PKCS7 int pkcs7enveloped_test(void); int pkcs7signed_test(void); - int pkcs7encrypted_test(void); + #ifndef NO_PKCS7_ENCRYPTED_DATA + int pkcs7encrypted_test(void); + #endif #endif #if !defined(NO_ASN_TIME) && !defined(NO_RSA) && defined(WOLFSSL_TEST_CERT) int cert_test(void); @@ -838,10 +840,12 @@ int wolfcrypt_test(void* args) else printf( "PKCS7signed test passed!\n"); - if ( (ret = pkcs7encrypted_test()) != 0) - return err_sys("PKCS7encrypted test failed!\n", ret); - else - printf( "PKCS7encrypted test passed!\n"); + #ifndef NO_PKCS7_ENCRYPTED_DATA + if ( (ret = pkcs7encrypted_test()) != 0) + return err_sys("PKCS7encrypted test failed!\n", ret); + else + printf( "PKCS7encrypted test passed!\n"); + #endif #endif #ifdef HAVE_VALGRIND @@ -6916,8 +6920,11 @@ static const CertName certDefaultName = { }; #ifdef WOLFSSL_CERT_EXT + #if (defined(HAVE_ED25519) && defined(WOLFSSL_TEST_CERT)) || \ + defined(HAVE_ECC) static const char certKeyUsage[] = "digitalSignature,nonRepudiation"; + #endif #if defined(WOLFSSL_CERT_REQ) || defined(HAVE_NTRU) static const char certKeyUsage2[] = "digitalSignature,nonRepudiation,keyEncipherment,keyAgreement"; @@ -8220,7 +8227,7 @@ int rsa_test(void) if (ret != 0) { ERROR_OUT(-5550, exit_rsa); } - ret = wc_MakeRsaKey(&genKey, 1024, 65537, &rng); + ret = wc_MakeRsaKey(&genKey, 1024, WC_RSA_EXPONENT, &rng); if (ret != 0) { ERROR_OUT(-5551, exit_rsa); } @@ -10988,6 +10995,9 @@ static int ecc_test_make_pub(WC_RNG* rng) #ifdef HAVE_ECC_VERIFY int verify = 0; #endif +#ifndef USE_CERT_BUFFERS_256 + FILE* file; +#endif tmp = (byte*)XMALLOC(FOURK_BUF, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); if (tmp == NULL) { @@ -11003,7 +11013,6 @@ static int ecc_test_make_pub(WC_RNG* rng) XMEMCPY(tmp, ecc_key_der_256, (size_t)sizeof_ecc_key_der_256); tmpSz = (size_t)sizeof_ecc_key_der_256; #else - FILE* file; file = fopen(eccKeyDerFile, "rb"); if (!file) { ERROR_OUT(-6812, done); @@ -14073,7 +14082,7 @@ static int pkcs7enveloped_run_vectors(byte* rsaCert, word32 rsaCertSz, 0x72,0x6c,0x64 }; -#if !defined(NO_AES) && defined(HAVE_ECC) +#if !defined(NO_AES) && defined(HAVE_ECC) && defined(WOLFSSL_SHA512) byte optionalUkm[] = { 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07 }; @@ -14083,8 +14092,10 @@ static int pkcs7enveloped_run_vectors(byte* rsaCert, word32 rsaCertSz, { /* key transport key encryption technique */ #ifndef NO_RSA + #ifndef NO_DES3 {data, (word32)sizeof(data), DATA, DES3b, 0, 0, rsaCert, rsaCertSz, rsaPrivKey, rsaPrivKeySz, NULL, 0, "pkcs7envelopedDataDES3.der"}, + #endif #ifndef NO_AES {data, (word32)sizeof(data), DATA, AES128CBCb, 0, 0, rsaCert, rsaCertSz, @@ -14294,6 +14305,8 @@ int pkcs7enveloped_test(void) } +#ifndef NO_PKCS7_ENCRYPTED_DATA + typedef struct { const byte* content; word32 contentSz; @@ -14495,6 +14508,8 @@ int pkcs7encrypted_test(void) return ret; } +#endif /* NO_PKCS7_ENCRYPTED_DATA */ + typedef struct { const byte* content; @@ -14540,7 +14555,11 @@ static int pkcs7signed_run_vectors(byte* rsaCert, word32 rsaCertSz, static byte senderNonceOid[] = { 0x06, 0x0a, 0x60, 0x86, 0x48, 0x01, 0x86, 0xF8, 0x45, 0x01, 0x09, 0x05 }; +#ifndef NO_SHA static byte transId[(WC_SHA_DIGEST_SIZE + 1) * 2 + 1]; +#else + static byte transId[(WC_SHA256_DIGEST_SIZE + 1) * 2 + 1]; +#endif static byte messageType[] = { 0x13, 2, '1', '9' }; static byte senderNonce[PKCS7_NONCE_SZ + 2]; @@ -14684,15 +14703,21 @@ static int pkcs7signed_run_vectors(byte* rsaCert, word32 rsaCertSz, } } - /* generate trans ID */ + /* generate transactionID (used with SCEP) */ { + #ifndef NO_SHA wc_Sha sha; byte digest[WC_SHA_DIGEST_SIZE]; + #else + wc_Sha256 sha; + byte digest[WC_SHA256_DIGEST_SIZE]; + #endif int j,k; transId[0] = 0x13; - transId[1] = WC_SHA_DIGEST_SIZE * 2; + transId[1] = sizeof(digest) * 2; + #ifndef NO_SHA ret = wc_InitSha_ex(&sha, HEAP_HINT, devId); if (ret != 0) { XFREE(out, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); @@ -14702,8 +14727,19 @@ static int pkcs7signed_run_vectors(byte* rsaCert, word32 rsaCertSz, wc_ShaUpdate(&sha, pkcs7.publicKey, pkcs7.publicKeySz); wc_ShaFinal(&sha, digest); wc_ShaFree(&sha); + #else + ret = wc_InitSha256_ex(&sha, HEAP_HINT, devId); + if (ret != 0) { + XFREE(out, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + wc_PKCS7_Free(&pkcs7); + return -7704; + } + wc_Sha256Update(&sha, pkcs7.publicKey, pkcs7.publicKeySz); + wc_Sha256Final(&sha, digest); + wc_Sha256Free(&sha); + #endif - for (j = 0, k = 2; j < WC_SHA_DIGEST_SIZE; j++, k += 2) { + for (j = 0, k = 2; j < (int)sizeof(digest); j++, k += 2) { XSNPRINTF((char*)&transId[k], 3, "%02x", digest[j]); } } diff --git a/wolfcrypt/user-crypto/include/user_rsa.h b/wolfcrypt/user-crypto/include/user_rsa.h index bf15fa9f5b..bb962ba9e6 100644 --- a/wolfcrypt/user-crypto/include/user_rsa.h +++ b/wolfcrypt/user-crypto/include/user_rsa.h @@ -50,12 +50,12 @@ #define WOLFSSL_RSA_TYPE_DEFINED #endif + enum { RSA_PUBLIC = 0, RSA_PRIVATE = 1, }; - /* RSA */ struct RsaKey { IppsBigNumState* n; diff --git a/wolfssl/error-ssl.h b/wolfssl/error-ssl.h index cef6ed9454..facab69a92 100644 --- a/wolfssl/error-ssl.h +++ b/wolfssl/error-ssl.h @@ -57,7 +57,7 @@ enum wolfSSL_ErrorCodes { DOMAIN_NAME_MISMATCH = -322, /* peer subject name mismatch */ WANT_READ = -323, /* want read, call again */ NOT_READY_ERROR = -324, /* handshake layer not ready */ - PMS_VERSION_ERROR = -325, /* pre m secret version error */ + VERSION_ERROR = -326, /* record layer version error */ WANT_WRITE = -327, /* want write, call again */ BUFFER_ERROR = -328, /* malformed buffer input */ @@ -164,6 +164,7 @@ enum wolfSSL_ErrorCodes { MCAST_HIGHWATER_CB_E = -426, /* Multicast highwater cb err */ ALERT_COUNT_E = -427, /* Alert Count exceeded err */ EXT_MISSING = -428, /* Required extension not found */ + UNSUPPORTED_EXTENSION = -429, /* TLSX not requested by client */ /* add strings to wolfSSL_ERR_reason_error_string in internal.c !!!!! */ /* begin negotiation parameter errors */ diff --git a/wolfssl/internal.h b/wolfssl/internal.h index 47d6e1c23c..9c77120a19 100644 --- a/wolfssl/internal.h +++ b/wolfssl/internal.h @@ -646,7 +646,7 @@ #endif #endif - #ifdef HAVE_CHACHA + #if defined(HAVE_CHACHA) && defined(HAVE_POLY1305) #ifndef NO_SHA256 #define BUILD_TLS_CHACHA20_POLY1305_SHA256 #endif @@ -955,7 +955,7 @@ enum Misc { #ifdef WOLFSSL_TLS13_DRAFT_18 TLS_DRAFT_MINOR = 0x12, /* Minor version number of TLS draft */ #else - TLS_DRAFT_MINOR = 0x14, /* Minor version number of TLS draft */ + TLS_DRAFT_MINOR = 0x15, /* Minor version number of TLS draft */ #endif OLD_HELLO_ID = 0x01, /* SSLv2 Client Hello Indicator */ INVALID_BYTE = 0xff, /* Used to initialize cipher specs values */ @@ -1003,6 +1003,9 @@ enum Misc { NAMED_DH_MASK = 0x100, /* Named group mask for DH parameters */ SESSION_HINT_SZ = 4, /* session timeout hint */ SESSION_ADD_SZ = 4, /* session age add */ + TICKET_NONCE_LEN_SZ = 1, /* Ticket nonce length size */ + DEF_TICKET_NONCE_SZ = 1, /* Default ticket nonce size */ + MAX_TICKET_NONCE_SZ = 4, /* maximum ticket nonce size */ MAX_LIFETIME = 604800, /* maximum ticket lifetime */ MAX_EARLY_DATA_SZ = 4096, /* maximum early data size */ @@ -1034,7 +1037,6 @@ enum Misc { HELLO_EXT_TYPE_SZ = 2, /* length of a hello extension type */ HELLO_EXT_SZ_SZ = 2, /* length of a hello extension size */ HELLO_EXT_SIGALGO_SZ = 2, /* length of number of items in sigalgo list */ - HELLO_EXT_SIGALGO_MAX = 32, /* number of items in the signature algo list */ DTLS_HANDSHAKE_HEADER_SZ = 12, /* normal + seq(2) + offset(3) + length(3) */ DTLS_RECORD_HEADER_SZ = 13, /* normal + epoch(2) + seq_num(6) */ @@ -1192,6 +1194,12 @@ enum Misc { /* 150 suites for now! */ #endif +/* number of items in the signature algo list */ +#ifndef WOLFSSL_MAX_SIGALGO + #define WOLFSSL_MAX_SIGALGO 32 +#endif + + /* set minimum ECC key size allowed */ #ifndef WOLFSSL_MIN_ECC_BITS #ifdef WOLFSSL_MAX_STRENGTH @@ -1527,7 +1535,7 @@ typedef struct Suites { word16 suiteSz; /* suite length in bytes */ word16 hashSigAlgoSz; /* SigAlgo extension length in bytes */ byte suites[WOLFSSL_MAX_SUITE_SZ]; - byte hashSigAlgo[HELLO_EXT_SIGALGO_MAX]; /* sig/algo to offer */ + byte hashSigAlgo[WOLFSSL_MAX_SIGALGO]; /* sig/algo to offer */ byte setSuites; /* user set suites from default */ byte hashAlgo; /* selected hash algorithm */ byte sigAlgo; /* selected sig algorithm */ @@ -1896,24 +1904,24 @@ WOLFSSL_LOCAL int TLSX_Parse(WOLFSSL* ssl, byte* input, word16 length, #ifdef HAVE_SNI typedef struct SNI { - byte type; /* SNI Type */ - union { char* host_name; } data; /* SNI Data */ - struct SNI* next; /* List Behavior */ + byte type; /* SNI Type */ + union { char* host_name; } data; /* SNI Data */ + struct SNI* next; /* List Behavior */ + byte status; /* Matching result */ #ifndef NO_WOLFSSL_SERVER byte options; /* Behavior options */ - byte status; /* Matching result */ #endif } SNI; WOLFSSL_LOCAL int TLSX_UseSNI(TLSX** extensions, byte type, const void* data, word16 size, void* heap); +WOLFSSL_LOCAL byte TLSX_SNI_Status(TLSX* extensions, byte type); +WOLFSSL_LOCAL word16 TLSX_SNI_GetRequest(TLSX* extensions, byte type, + void** data); #ifndef NO_WOLFSSL_SERVER WOLFSSL_LOCAL void TLSX_SNI_SetOptions(TLSX* extensions, byte type, byte options); -WOLFSSL_LOCAL byte TLSX_SNI_Status(TLSX* extensions, byte type); -WOLFSSL_LOCAL word16 TLSX_SNI_GetRequest(TLSX* extensions, byte type, - void** data); WOLFSSL_LOCAL int TLSX_SNI_GetFromBuffer(const byte* buffer, word32 bufferSz, byte type, byte* sni, word32* inOutSz); #endif @@ -2150,6 +2158,16 @@ WOLFSSL_LOCAL int TLSX_KeyShare_Establish(WOLFSSL* ssl); #if defined(HAVE_SESSION_TICKET) || !defined(NO_PSK) +#ifndef WOLFSSL_TLS13_DRAFT_18 +/* Ticket nonce - for deriving PSK. + * Length allowed to be: 1..255. Only support 4 bytes. + */ +typedef struct TicketNonce { + byte len; + byte data[MAX_TICKET_NONCE_SZ]; +} TicketNonce; +#endif + /* The PreSharedKey extension information - entry in a linked list. */ typedef struct PreSharedKey { word16 identityLen; /* Length of identity */ @@ -2670,6 +2688,9 @@ struct WOLFSSL_SESSION { byte namedGroup; word32 ticketSeen; /* Time ticket seen (ms) */ word32 ticketAdd; /* Added by client */ + #ifndef WOLFSSL_TLS13_DRAFT_18 + TicketNonce ticketNonce; /* Nonce used to derive PSK */ + #endif #endif #ifdef WOLFSSL_EARLY_DATA word32 maxEarlyDataSz; diff --git a/wolfssl/ssl.h b/wolfssl/ssl.h index 912d507112..7aa6eaf106 100644 --- a/wolfssl/ssl.h +++ b/wolfssl/ssl.h @@ -301,6 +301,7 @@ enum AlertDescription { protocol_version = 70, #endif no_renegotiation = 100, + unsupported_extension = 110, /**< RFC 5246, section 7.2.2 */ unrecognized_name = 112, /**< RFC 6066, section 3 */ bad_certificate_status_response = 113, /**< RFC 6066, section 8 */ no_application_protocol = 120 @@ -1536,6 +1537,7 @@ enum { WOLFSSL_TLSV1 = 1, WOLFSSL_TLSV1_1 = 2, WOLFSSL_TLSV1_2 = 3, + WOLFSSL_TLSV1_3 = 4, WOLFSSL_USER_CA = 1, /* user added as trusted */ WOLFSSL_CHAIN_CA = 2 /* added to cache from trusted chain */ }; @@ -1549,6 +1551,7 @@ WOLFSSL_API int wolfSSL_CTX_GetObjectSize(void); WOLFSSL_API int wolfSSL_METHOD_GetObjectSize(void); WOLFSSL_API int wolfSSL_GetOutputSize(WOLFSSL*, int); WOLFSSL_API int wolfSSL_GetMaxOutputSize(WOLFSSL*); +WOLFSSL_API int wolfSSL_GetVersion(WOLFSSL* ssl); WOLFSSL_API int wolfSSL_SetVersion(WOLFSSL* ssl, int version); WOLFSSL_API int wolfSSL_KeyPemToDer(const unsigned char*, int, unsigned char*, int, const char*); @@ -1918,6 +1921,11 @@ WOLFSSL_API void wolfSSL_SNI_SetOptions(WOLFSSL* ssl, unsigned char type, unsigned char options); WOLFSSL_API void wolfSSL_CTX_SNI_SetOptions(WOLFSSL_CTX* ctx, unsigned char type, unsigned char options); +WOLFSSL_API int wolfSSL_SNI_GetFromBuffer( + const unsigned char* clientHello, unsigned int helloSz, + unsigned char type, unsigned char* sni, unsigned int* inOutSz); + +#endif /* NO_WOLFSSL_SERVER */ /* SNI status */ enum { @@ -1931,12 +1939,8 @@ WOLFSSL_API unsigned char wolfSSL_SNI_Status(WOLFSSL* ssl, unsigned char type); WOLFSSL_API unsigned short wolfSSL_SNI_GetRequest(WOLFSSL *ssl, unsigned char type, void** data); -WOLFSSL_API int wolfSSL_SNI_GetFromBuffer( - const unsigned char* clientHello, unsigned int helloSz, - unsigned char type, unsigned char* sni, unsigned int* inOutSz); -#endif -#endif +#endif /* HAVE_SNI */ /* Application-Layer Protocol Negotiation */ #ifdef HAVE_ALPN diff --git a/wolfssl/wolfcrypt/integer.h b/wolfssl/wolfcrypt/integer.h index f3f7410749..cf8069bc76 100644 --- a/wolfssl/wolfcrypt/integer.h +++ b/wolfssl/wolfcrypt/integer.h @@ -248,10 +248,16 @@ typedef int ltm_prime_callback(unsigned char *dst, int len, void *dat); #define mp_mag_size(mp) mp_unsigned_bin_size(mp) #define mp_tomag(mp, str) mp_to_unsigned_bin((mp), (str)) -#define mp_tobinary(M, S) mp_toradix((M), (S), 2) -#define mp_tooctal(M, S) mp_toradix((M), (S), 8) -#define mp_todecimal(M, S) mp_toradix((M), (S), 10) -#define mp_tohex(M, S) mp_toradix((M), (S), 16) +#define MP_RADIX_BIN 2 +#define MP_RADIX_OCT 8 +#define MP_RADIX_DEC 10 +#define MP_RADIX_HEX 16 +#define MP_RADIX_MAX 64 + +#define mp_tobinary(M, S) mp_toradix((M), (S), MP_RADIX_BIN) +#define mp_tooctal(M, S) mp_toradix((M), (S), MP_RADIX_OCT) +#define mp_todecimal(M, S) mp_toradix((M), (S), MP_RADIX_DEC) +#define mp_tohex(M, S) mp_toradix((M), (S), MP_RADIX_HEX) #define s_mp_mul(a, b, c) s_mp_mul_digs(a, b, c, (a)->used + (b)->used + 1) diff --git a/wolfssl/wolfcrypt/pkcs7.h b/wolfssl/wolfcrypt/pkcs7.h index 05ed997f87..96af4f0eec 100644 --- a/wolfssl/wolfcrypt/pkcs7.h +++ b/wolfssl/wolfcrypt/pkcs7.h @@ -133,6 +133,7 @@ typedef struct PKCS7 { WOLFSSL_API int wc_PKCS7_Init(PKCS7* pkcs7, void* heap, int devId); WOLFSSL_API int wc_PKCS7_InitWithCert(PKCS7* pkcs7, byte* cert, word32 certSz); WOLFSSL_API void wc_PKCS7_Free(PKCS7* pkcs7); + WOLFSSL_API int wc_PKCS7_EncodeData(PKCS7* pkcs7, byte* output, word32 outputSz); WOLFSSL_API int wc_PKCS7_EncodeSignedData(PKCS7* pkcs7, @@ -144,11 +145,15 @@ WOLFSSL_API int wc_PKCS7_EncodeEnvelopedData(PKCS7* pkcs7, WOLFSSL_API int wc_PKCS7_DecodeEnvelopedData(PKCS7* pkcs7, byte* pkiMsg, word32 pkiMsgSz, byte* output, word32 outputSz); + +#ifndef NO_PKCS7_ENCRYPTED_DATA WOLFSSL_API int wc_PKCS7_EncodeEncryptedData(PKCS7* pkcs7, byte* output, word32 outputSz); WOLFSSL_API int wc_PKCS7_DecodeEncryptedData(PKCS7* pkcs7, byte* pkiMsg, word32 pkiMsgSz, byte* output, word32 outputSz); +#endif /* NO_PKCS7_ENCRYPTED_DATA */ + #ifdef __cplusplus } /* extern "C" */ #endif diff --git a/wolfssl/wolfcrypt/port/ti/ti-hash.h b/wolfssl/wolfcrypt/port/ti/ti-hash.h index 081f4971ba..361993896b 100644 --- a/wolfssl/wolfcrypt/port/ti/ti-hash.h +++ b/wolfssl/wolfcrypt/port/ti/ti-hash.h @@ -44,16 +44,16 @@ typedef struct { #ifndef TI_HASH_TEST #if !defined(NO_MD5) - typedef wolfssl_TI_Hash Md5; + typedef wolfssl_TI_Hash wc_Md5; #endif #if !defined(NO_SHA) - typedef wolfssl_TI_Hash Sha; + typedef wolfssl_TI_Hash wc_Sha; #endif #if !defined(NO_SHA256) - typedef wolfssl_TI_Hash Sha256; + typedef wolfssl_TI_Hash wc_Sha256; #endif #if defined(WOLFSSL_SHA224) - typedef wolfssl_TI_Hash Sha224; + typedef wolfssl_TI_Hash wc_Sha224; #endif #endif /* !TI_HASH_TEST */ diff --git a/wolfssl/wolfcrypt/rsa.h b/wolfssl/wolfcrypt/rsa.h index 931fc8163c..3dafb8a343 100644 --- a/wolfssl/wolfcrypt/rsa.h +++ b/wolfssl/wolfcrypt/rsa.h @@ -27,6 +27,13 @@ #ifndef NO_RSA + +/* RSA default exponent */ +#ifndef WC_RSA_EXPONENT + #define WC_RSA_EXPONENT 65537L +#endif + + /* allow for user to plug in own crypto */ #if !defined(HAVE_FIPS) && (defined(HAVE_USER_RSA) || defined(HAVE_FAST_RSA)) #include "user_rsa.h" @@ -90,7 +97,6 @@ enum { #endif }; - /* RSA */ struct RsaKey { mp_int n, e, d, p, q, dP, dQ, u; diff --git a/wolfssl/wolfcrypt/settings.h b/wolfssl/wolfcrypt/settings.h index 5d5f49410b..41bab112c4 100644 --- a/wolfssl/wolfcrypt/settings.h +++ b/wolfssl/wolfcrypt/settings.h @@ -596,6 +596,8 @@ extern void uITRON4_free(void *p) ; #define NO_WOLFSSL_DIR #define USE_FAST_MATH #define TFM_TIMING_RESISTANT + #define ECC_TIMING_RESISTANT + #define WC_RSA_BLINDING #define NO_DEV_RANDOM #define NO_FILESYSTEM #define USE_CERT_BUFFERS_2048 @@ -603,6 +605,7 @@ extern void uITRON4_free(void *p) ; #define USER_TIME #define HAVE_ECC #define HAVE_ALPN + #define USE_WOLF_STRTOK /* use with HAVE_ALPN */ #define HAVE_TLS_EXTENSIONS #define HAVE_AESGCM #define HAVE_SUPPORTED_CURVES @@ -1495,6 +1498,9 @@ extern void uITRON4_free(void *p) ; #endif #ifdef HAVE_PKCS7 + #if defined(NO_AES) && defined(NO_DES3) + #error PKCS7 needs either AES or 3DES enabled, please enable one + #endif #ifndef HAVE_AES_KEYWRAP #error PKCS7 requires AES key wrap please define HAVE_AES_KEYWRAP #endif diff --git a/wolfssl/wolfcrypt/sha.h b/wolfssl/wolfcrypt/sha.h index 0831e056b5..97ce2e63cb 100644 --- a/wolfssl/wolfcrypt/sha.h +++ b/wolfssl/wolfcrypt/sha.h @@ -91,6 +91,9 @@ typedef struct wc_Sha { #ifdef WOLFSSL_PIC32MZ_HASH hashUpdCache cache; /* cache for updates */ #endif + #if defined(STM32_HASH) && defined(WOLFSSL_STM32_CUBEMX) + HASH_HandleTypeDef hashHandle; + #endif #ifdef WOLFSSL_ASYNC_CRYPT WC_ASYNC_DEV asyncDev; #endif /* WOLFSSL_ASYNC_CRYPT */ diff --git a/wolfssl/wolfcrypt/sha256.h b/wolfssl/wolfcrypt/sha256.h index c71dca32a3..0cc9beb3cf 100644 --- a/wolfssl/wolfcrypt/sha256.h +++ b/wolfssl/wolfcrypt/sha256.h @@ -66,6 +66,14 @@ #include #endif +#if defined(_MSC_VER) + #define SHA256_NOINLINE __declspec(noinline) +#elif defined(__GNUC__) + #define SHA256_NOINLINE __attribute__((noinline)) +#else + #define SHA256_NOINLINE +#endif + #ifndef NO_OLD_WC_NAMES #define Sha256 wc_Sha256 #define SHA256 WC_SHA256 @@ -96,6 +104,9 @@ typedef struct wc_Sha256 { word32 loLen; /* length in bytes */ word32 hiLen; /* length in bytes */ void* heap; +#ifdef USE_INTEL_SPEEDUP + const byte* data; +#endif #ifdef WOLFSSL_PIC32MZ_HASH hashUpdCache cache; /* cache for updates */ #endif diff --git a/wolfssl/wolfcrypt/sha512.h b/wolfssl/wolfcrypt/sha512.h index f8982850a1..277f1a6874 100644 --- a/wolfssl/wolfcrypt/sha512.h +++ b/wolfssl/wolfcrypt/sha512.h @@ -59,6 +59,14 @@ #include #endif +#if defined(_MSC_VER) + #define SHA512_NOINLINE __declspec(noinline) +#elif defined(__GNUC__) + #define SHA512_NOINLINE __attribute__((noinline)) +#else + #define SHA512_NOINLINE +#endif + #ifndef NO_OLD_WC_NAMES #define Sha512 wc_Sha512 #define SHA512 WC_SHA512 @@ -78,12 +86,15 @@ enum { /* wc_Sha512 digest */ typedef struct wc_Sha512 { + word64 digest[WC_SHA512_DIGEST_SIZE / sizeof(word64)]; + word64 buffer[WC_SHA512_BLOCK_SIZE / sizeof(word64)]; word32 buffLen; /* in bytes */ word64 loLen; /* length in bytes */ word64 hiLen; /* length in bytes */ - word64 digest[WC_SHA512_DIGEST_SIZE / sizeof(word64)]; - word64 buffer[WC_SHA512_BLOCK_SIZE / sizeof(word64)]; void* heap; +#ifdef USE_INTEL_SPEEDUP + const byte* data; +#endif #ifdef WOLFSSL_ASYNC_CRYPT WC_ASYNC_DEV asyncDev; #endif /* WOLFSSL_ASYNC_CRYPT */ diff --git a/wolfssl/wolfcrypt/tfm.h b/wolfssl/wolfcrypt/tfm.h index 27c884a251..e0432311b0 100644 --- a/wolfssl/wolfcrypt/tfm.h +++ b/wolfssl/wolfcrypt/tfm.h @@ -64,6 +64,10 @@ #define MAX(x,y) ((x)>(y)?(x):(y)) #endif +#ifdef WOLFSSL_NO_ASM + #undef TFM_NO_ASM + #define TFM_NO_ASM +#endif #ifndef NO_64BIT /* autodetect x86-64 and make sure we are using 64-bit digits with x86-64 asm */ @@ -632,10 +636,22 @@ typedef fp_int mp_int; #define MP_MASK FP_MASK /* Prototypes */ -#define mp_zero(a) fp_zero(a) +#define mp_zero(a) fp_zero(a) #define mp_isone(a) fp_isone(a) -#define mp_iseven(a) fp_iseven(a) -#define mp_isneg(a) fp_isneg(a) +#define mp_iseven(a) fp_iseven(a) +#define mp_isneg(a) fp_isneg(a) + +#define MP_RADIX_BIN 2 +#define MP_RADIX_OCT 8 +#define MP_RADIX_DEC 10 +#define MP_RADIX_HEX 16 +#define MP_RADIX_MAX 64 + +#define mp_tobinary(M, S) mp_toradix((M), (S), MP_RADIX_BIN) +#define mp_tooctal(M, S) mp_toradix((M), (S), MP_RADIX_OCT) +#define mp_todecimal(M, S) mp_toradix((M), (S), MP_RADIX_DEC) +#define mp_tohex(M, S) mp_toradix((M), (S), MP_RADIX_HEX) + MP_API int mp_init (mp_int * a); MP_API void mp_clear (mp_int * a); MP_API void mp_free (mp_int * a); diff --git a/wolfssl/wolfcrypt/types.h b/wolfssl/wolfcrypt/types.h index e75a689a51..f7766d49ca 100644 --- a/wolfssl/wolfcrypt/types.h +++ b/wolfssl/wolfcrypt/types.h @@ -51,26 +51,30 @@ /* try to set SIZEOF_LONG or LONG_LONG if user didn't */ - #if !defined(_MSC_VER) && !defined(__BCPLUSPLUS__) + #if !defined(_MSC_VER) && !defined(__BCPLUSPLUS__) && !defined(__EMSCRIPTEN__) #if !defined(SIZEOF_LONG_LONG) && !defined(SIZEOF_LONG) - #if (defined(__alpha__) || defined(__ia64__) || defined(_ARCH_PPC64) \ - || defined(__mips64) || defined(__x86_64__) || \ - ((defined(sun) || defined(__sun)) && \ - (defined(LP64) || defined(_LP64)))) + #if (defined(__alpha__) || defined(__ia64__) || \ + defined(_ARCH_PPC64) || defined(__mips64) || \ + defined(__x86_64__) || \ + ((defined(sun) || defined(__sun)) && \ + (defined(LP64) || defined(_LP64)))) /* long should be 64bit */ #define SIZEOF_LONG 8 #elif defined(__i386__) || defined(__CORTEX_M3__) /* long long should be 64bit */ #define SIZEOF_LONG_LONG 8 #endif - #endif + #endif #endif - #if defined(_MSC_VER) || defined(__BCPLUSPLUS__) #define WORD64_AVAILABLE #define W64LIT(x) x##ui64 typedef unsigned __int64 word64; + #elif defined(__EMSCRIPTEN__) + #define WORD64_AVAILABLE + #define W64LIT(x) x##ull + typedef unsigned long long word64; #elif defined(SIZEOF_LONG) && SIZEOF_LONG == 8 #define WORD64_AVAILABLE #define W64LIT(x) x##LL @@ -83,12 +87,9 @@ #define WORD64_AVAILABLE #define W64LIT(x) x##LL typedef unsigned long long word64; - #else - #define MP_16BIT /* for mp_int, mp_word needs to be twice as big as - mp_digit, no 64 bit type so make mp_digit 16 bit */ #endif - +#if !defined(NO_64BIT) && defined(WORD64_AVAILABLE) /* These platforms have 64-bit CPU registers. */ #if (defined(__alpha__) || defined(__ia64__) || defined(_ARCH_PPC64) || \ defined(__mips64) || defined(__x86_64__) || defined(_M_X64)) || \ @@ -109,7 +110,12 @@ #define WOLFCRYPT_SLOW_WORD64 #endif #endif - +#else + #undef WORD64_AVAILABLE + typedef word32 wolfssl_word; + #define MP_16BIT /* for mp_int, mp_word needs to be twice as big as + mp_digit, no 64 bit type so make mp_digit 16 bit */ +#endif enum { WOLFSSL_WORD_SIZE = sizeof(wolfssl_word), @@ -309,7 +315,7 @@ #define XSTRNCMP(s1,s2,n) strncmp((s1),(s2),(n)) #define XSTRNCAT(s1,s2,n) strncat((s1),(s2),(n)) - #ifdef MICROCHIP_PIC32 + #if defined(MICROCHIP_PIC32) || defined(WOLFSSL_TIRTOS) /* XC32 does not support strncasecmp, so use case sensitive one */ #define XSTRNCASECMP(s1,s2,n) strncmp((s1),(s2),(n)) #elif defined(USE_WINDOWS_API) @@ -318,7 +324,8 @@ #define XSTRNCASECMP(s1,s2,n) strncasecmp((s1),(s2),(n)) #endif - /* snprintf is used in asn.c for GetTimeString and PKCS7 test */ + /* snprintf is used in asn.c for GetTimeString, PKCS7 test, and when + debugging is turned on */ #ifndef USE_WINDOWS_API #define XSNPRINTF snprintf #else @@ -327,16 +334,16 @@ #if defined(WOLFSSL_CERT_EXT) || defined(HAVE_ALPN) /* use only Thread Safe version of strtok */ - #if !defined(USE_WINDOWS_API) && !defined(INTIME_RTOS) - #define XSTRTOK strtok_r - #elif defined(__MINGW32__) || defined(WOLFSSL_TIRTOS) || \ + #if defined(__MINGW32__) || defined(WOLFSSL_TIRTOS) || \ defined(USE_WOLF_STRTOK) #ifndef USE_WOLF_STRTOK #define USE_WOLF_STRTOK #endif #define XSTRTOK wc_strtok - #else + #elif defined(USE_WINDOWS_API) || defined(INTIME_RTOS) #define XSTRTOK strtok_s + #else + #define XSTRTOK strtok_r #endif #endif #endif @@ -445,9 +452,9 @@ }; /* max error buffer string size */ - enum { - WOLFSSL_MAX_ERROR_SZ = 80 - }; + #ifndef WOLFSSL_MAX_ERROR_SZ + #define WOLFSSL_MAX_ERROR_SZ 80 + #endif /* stack protection */ enum {