wolfSSL/wolfssl
1
0
Fork 1
mirror of https://github.com/wolfSSL/wolfssl.git synced 2026-07-08 16:30:48 +02:00
Code Issues Packages Projects Releases Wiki Activity

Wycheproof: fixes for various platforms

Browse Source 
Fixes for C, Intel x64, Aarch64, ARM32, Thumb2, Thumb.
This commit is contained in:
Sean Parkinson
2026-07-06 20:04:54 +10:00
parent 38a4143a48
commit 649054f7b8
20 changed files with 5650 additions and 1224 deletions
Download Patch File Download Diff File Expand all files Collapse all files
wolfcrypt/src/ecc.c
+4 -2
View File
@@ -9345,9 +9345,11 @@ static int ecc_verify_hash(mp_int *r, mp_int *s, const byte* hash,
if (err == MP_OKAY)
err = mp_montgomery_setup(curve->prime, &mp);
/* add them */
/* add them - pass mG as A (first) not B (second): the safe-add
* recovery for a missed-double doubles B into R, which requires
* B and R to be different points. */
if (err == MP_OKAY)
err = ecc_projective_add_point_safe(mQ, mG, mG, curve->Af,
err = ecc_projective_add_point_safe(mG, mQ, mG, curve->Af,
curve->prime, mp, NULL);
#ifdef WOLFSSL_CHECK_VER_FAULTS
if (err == MP_OKAY && wc_ecc_cmp_point(mG, &mG1) == MP_EQ) {
wolfcrypt/src/fe_448.c
+1
View File
@@ -2302,6 +2302,7 @@ int curve448(byte* r, const byte* n, const byte* a)
fe448_reduce(x3);
fe448_sqr(x3, x3);
fe448_sub(z3, z3, t1);
fe448_reduce(z3);
fe448_sqr(z3, z3);
fe448_mul(z3, z3, x1);
fe448_sub(t1, t0, x2);
wolfcrypt/src/fe_x25519_asm.S
+18
View File
@@ -5738,6 +5738,15 @@ L_curve25519_x64_bits:
adcq %r12, %r9
adcq %r13, %r10
adcq %r14, %r11
movq $0x7fffffffffffffff, %rbp
movq %r11, %rax
sarq $63, %rax
andq $19, %rax
andq %rbp, %r11
addq %rax, %rcx
adcq $0x00, %r9
adcq $0x00, %r10
adcq $0x00, %r11
# Store
movq %rcx, 64(%rsp)
movq %r9, 72(%rsp)
@@ -16022,6 +16031,15 @@ L_curve25519_avx2_bits:
adcxq %rbx, %r11
adoxq %r15, %r12
adcxq %rcx, %r12
movq $0x7fffffffffffffff, %rcx
movq %r12, %rdx
sarq $63, %rdx
andq $19, %rdx
andq %rcx, %r12
addq %rdx, %r9
adcq $0x00, %r10
adcq $0x00, %r11
adcq $0x00, %r12
# Store
movq %r9, 64(%rsp)
movq %r10, 72(%rsp)
wolfcrypt/src/port/arm/armv8-32-curve25519.S
+1075 -16
View File
File diff suppressed because it is too large Load Diff
wolfcrypt/src/port/arm/armv8-32-curve25519_c.c
+1143 -32
View File
File diff suppressed because it is too large Load Diff
wolfcrypt/src/port/arm/armv8-aes-asm.S
+46 -38
View File
@@ -5736,7 +5736,7 @@ L_aes_gcm_encrypt_arm64_crypto_nonce_end_bytes:
# Done GHASH
L_aes_gcm_encrypt_arm64_crypto_nonce_partial_done:
eor x14, x14, x14
lsl x24, x4, #3
ubfiz x24, x4, #3, #32
mov v28.d[0], x14
mov v28.d[1], x24
rev64 v28.16b, v28.16b
@@ -7099,10 +7099,10 @@ L_aes_gcm_encrypt_arm64_crypto_192_start_zero:
# Done GHASH
L_aes_gcm_encrypt_arm64_crypto_192_partial_done:
ld1 {v14.2d}, [x12]
lsl x8, x8, #3
ubfiz x8, x8, #3, #32
rbit x8, x8
mov v28.d[0], x8
lsl x2, x2, #3
ubfiz x2, x2, #3, #32
rbit x2, x2
mov v28.d[1], x2
eor v26.16b, v26.16b, v28.16b
@@ -8637,10 +8637,10 @@ L_aes_gcm_encrypt_arm64_crypto_256_start_zero:
# Done GHASH
L_aes_gcm_encrypt_arm64_crypto_256_partial_done:
ld1 {v14.2d}, [x12]
lsl x8, x8, #3
ubfiz x8, x8, #3, #32
rbit x8, x8
mov v28.d[0], x8
lsl x2, x2, #3
ubfiz x2, x2, #3, #32
rbit x2, x2
mov v28.d[1], x2
aese v14.16b, v0.16b
@@ -9944,10 +9944,10 @@ L_aes_gcm_encrypt_arm64_crypto_128_start_zero:
# Done GHASH
L_aes_gcm_encrypt_arm64_crypto_128_partial_done:
ld1 {v14.2d}, [x12]
lsl x8, x8, #3
ubfiz x8, x8, #3, #32
rbit x8, x8
mov v28.d[0], x8
lsl x2, x2, #3
ubfiz x2, x2, #3, #32
rbit x2, x2
mov v28.d[1], x2
eor v26.16b, v26.16b, v28.16b
@@ -10537,7 +10537,7 @@ L_aes_gcm_decrypt_arm64_crypto_nonce_end_bytes:
# Done GHASH
L_aes_gcm_decrypt_arm64_crypto_nonce_partial_done:
eor x14, x14, x14
lsl x24, x4, #3
ubfiz x24, x4, #3, #32
mov v28.d[0], x14
mov v28.d[1], x24
rev64 v28.16b, v28.16b
@@ -11893,10 +11893,10 @@ L_aes_gcm_decrypt_arm64_crypto_192_out_start_byte:
L_aes_gcm_decrypt_arm64_crypto_192_out_end_bytes:
L_aes_gcm_decrypt_arm64_crypto_192_partial_done:
ld1 {v14.2d}, [x12]
lsl x8, x8, #3
ubfiz x8, x8, #3, #32
rbit x8, x8
mov v28.d[0], x8
lsl x2, x2, #3
ubfiz x2, x2, #3, #32
rbit x2, x2
mov v28.d[1], x2
eor v26.16b, v26.16b, v28.16b
@@ -11945,6 +11945,7 @@ L_aes_gcm_decrypt_arm64_crypto_192_partial_done:
ld1 {v28.16b}, [x5]
b L_aes_gcm_decrypt_arm64_crypto_192_tag_loaded
L_aes_gcm_decrypt_arm64_crypto_192_part_tag:
ubfiz x6, x6, #0, #32
eor v28.16b, v28.16b, v28.16b
mov x17, x6
st1 {v28.2d}, [x11]
@@ -13446,10 +13447,10 @@ L_aes_gcm_decrypt_arm64_crypto_256_out_start_byte:
L_aes_gcm_decrypt_arm64_crypto_256_out_end_bytes:
L_aes_gcm_decrypt_arm64_crypto_256_partial_done:
ld1 {v14.2d}, [x12]
lsl x8, x8, #3
ubfiz x8, x8, #3, #32
rbit x8, x8
mov v28.d[0], x8
lsl x2, x2, #3
ubfiz x2, x2, #3, #32
rbit x2, x2
mov v28.d[1], x2
aese v14.16b, v0.16b
@@ -13506,6 +13507,7 @@ L_aes_gcm_decrypt_arm64_crypto_256_partial_done:
ld1 {v28.16b}, [x5]
b L_aes_gcm_decrypt_arm64_crypto_256_tag_loaded
L_aes_gcm_decrypt_arm64_crypto_256_part_tag:
ubfiz x6, x6, #0, #32
eor v28.16b, v28.16b, v28.16b
mov x17, x6
st1 {v28.2d}, [x11]
@@ -14768,10 +14770,10 @@ L_aes_gcm_decrypt_arm64_crypto_128_out_start_byte:
L_aes_gcm_decrypt_arm64_crypto_128_out_end_bytes:
L_aes_gcm_decrypt_arm64_crypto_128_partial_done:
ld1 {v14.2d}, [x12]
lsl x8, x8, #3
ubfiz x8, x8, #3, #32
rbit x8, x8
mov v28.d[0], x8
lsl x2, x2, #3
ubfiz x2, x2, #3, #32
rbit x2, x2
mov v28.d[1], x2
eor v26.16b, v26.16b, v28.16b
@@ -14816,6 +14818,7 @@ L_aes_gcm_decrypt_arm64_crypto_128_partial_done:
ld1 {v28.16b}, [x5]
b L_aes_gcm_decrypt_arm64_crypto_128_tag_loaded
L_aes_gcm_decrypt_arm64_crypto_128_part_tag:
ubfiz x6, x6, #0, #32
eor v28.16b, v28.16b, v28.16b
mov x17, x6
st1 {v28.2d}, [x11]
@@ -15364,7 +15367,7 @@ L_aes_gcm_encrypt_arm64_crypto_eor3_nonce_end_bytes:
# Done GHASH
L_aes_gcm_encrypt_arm64_crypto_eor3_nonce_partial_done:
eor x14, x14, x14
lsl x24, x4, #3
ubfiz x24, x4, #3, #32
mov v28.d[0], x14
mov v28.d[1], x24
rev64 v28.16b, v28.16b
@@ -16698,10 +16701,10 @@ L_aes_gcm_encrypt_arm64_crypto_eor3_192_start_zero:
# Done GHASH
L_aes_gcm_encrypt_arm64_crypto_eor3_192_partial_done:
ld1 {v14.2d}, [x12]
lsl x8, x8, #3
ubfiz x8, x8, #3, #32
rbit x8, x8
mov v28.d[0], x8
lsl x2, x2, #3
ubfiz x2, x2, #3, #32
rbit x2, x2
mov v28.d[1], x2
eor v26.16b, v26.16b, v28.16b
@@ -18207,10 +18210,10 @@ L_aes_gcm_encrypt_arm64_crypto_eor3_256_start_zero:
# Done GHASH
L_aes_gcm_encrypt_arm64_crypto_eor3_256_partial_done:
ld1 {v14.2d}, [x12]
lsl x8, x8, #3
ubfiz x8, x8, #3, #32
rbit x8, x8
mov v28.d[0], x8
lsl x2, x2, #3
ubfiz x2, x2, #3, #32
rbit x2, x2
mov v28.d[1], x2
aese v14.16b, v0.16b
@@ -19485,10 +19488,10 @@ L_aes_gcm_encrypt_arm64_crypto_eor3_128_start_zero:
# Done GHASH
L_aes_gcm_encrypt_arm64_crypto_eor3_128_partial_done:
ld1 {v14.2d}, [x12]
lsl x8, x8, #3
ubfiz x8, x8, #3, #32
rbit x8, x8
mov v28.d[0], x8
lsl x2, x2, #3
ubfiz x2, x2, #3, #32
rbit x2, x2
mov v28.d[1], x2
eor v26.16b, v26.16b, v28.16b
@@ -20056,7 +20059,7 @@ L_aes_gcm_decrypt_arm64_crypto_eor3_nonce_end_bytes:
# Done GHASH
L_aes_gcm_decrypt_arm64_crypto_eor3_nonce_partial_done:
eor x14, x14, x14
lsl x24, x4, #3
ubfiz x24, x4, #3, #32
mov v28.d[0], x14
mov v28.d[1], x24
rev64 v28.16b, v28.16b
@@ -21383,10 +21386,10 @@ L_aes_gcm_decrypt_arm64_crypto_eor3_192_out_start_byte:
L_aes_gcm_decrypt_arm64_crypto_eor3_192_out_end_bytes:
L_aes_gcm_decrypt_arm64_crypto_eor3_192_partial_done:
ld1 {v14.2d}, [x12]
lsl x8, x8, #3
ubfiz x8, x8, #3, #32
rbit x8, x8
mov v28.d[0], x8
lsl x2, x2, #3
ubfiz x2, x2, #3, #32
rbit x2, x2
mov v28.d[1], x2
eor v26.16b, v26.16b, v28.16b
@@ -21434,6 +21437,7 @@ L_aes_gcm_decrypt_arm64_crypto_eor3_192_partial_done:
ld1 {v28.16b}, [x5]
b L_aes_gcm_decrypt_arm64_crypto_eor3_192_tag_loaded
L_aes_gcm_decrypt_arm64_crypto_eor3_192_part_tag:
ubfiz x6, x6, #0, #32
eor v28.16b, v28.16b, v28.16b
mov x17, x6
st1 {v28.2d}, [x11]
@@ -22907,10 +22911,10 @@ L_aes_gcm_decrypt_arm64_crypto_eor3_256_out_start_byte:
L_aes_gcm_decrypt_arm64_crypto_eor3_256_out_end_bytes:
L_aes_gcm_decrypt_arm64_crypto_eor3_256_partial_done:
ld1 {v14.2d}, [x12]
lsl x8, x8, #3
ubfiz x8, x8, #3, #32
rbit x8, x8
mov v28.d[0], x8
lsl x2, x2, #3
ubfiz x2, x2, #3, #32
rbit x2, x2
mov v28.d[1], x2
aese v14.16b, v0.16b
@@ -22966,6 +22970,7 @@ L_aes_gcm_decrypt_arm64_crypto_eor3_256_partial_done:
ld1 {v28.16b}, [x5]
b L_aes_gcm_decrypt_arm64_crypto_eor3_256_tag_loaded
L_aes_gcm_decrypt_arm64_crypto_eor3_256_part_tag:
ubfiz x6, x6, #0, #32
eor v28.16b, v28.16b, v28.16b
mov x17, x6
st1 {v28.2d}, [x11]
@@ -24200,10 +24205,10 @@ L_aes_gcm_decrypt_arm64_crypto_eor3_128_out_start_byte:
L_aes_gcm_decrypt_arm64_crypto_eor3_128_out_end_bytes:
L_aes_gcm_decrypt_arm64_crypto_eor3_128_partial_done:
ld1 {v14.2d}, [x12]
lsl x8, x8, #3
ubfiz x8, x8, #3, #32
rbit x8, x8
mov v28.d[0], x8
lsl x2, x2, #3
ubfiz x2, x2, #3, #32
rbit x2, x2
mov v28.d[1], x2
eor v26.16b, v26.16b, v28.16b
@@ -24247,6 +24252,7 @@ L_aes_gcm_decrypt_arm64_crypto_eor3_128_partial_done:
ld1 {v28.16b}, [x5]
b L_aes_gcm_decrypt_arm64_crypto_eor3_128_tag_loaded
L_aes_gcm_decrypt_arm64_crypto_eor3_128_part_tag:
ubfiz x6, x6, #0, #32
eor v28.16b, v28.16b, v28.16b
mov x17, x6
st1 {v28.2d}, [x11]
@@ -24420,7 +24426,7 @@ L_aes_gcm_init_arm64_crypto_end_bytes:
# Done GHASH
L_aes_gcm_init_arm64_crypto_partial_done:
eor x7, x7, x7
lsl x13, x3, #3
ubfiz x13, x3, #3, #32
mov v7.d[0], x7
mov v7.d[1], x13
rev64 v7.16b, v7.16b
@@ -28782,10 +28788,10 @@ _AES_GCM_encrypt_final_AARCH64:
ld1 {v4.2d}, [x5]
ushr v6.2d, v6.2d, #56
ld1 {v7.2d}, [x6]
lsl x4, x4, #3
ubfiz x4, x4, #3, #32
rbit x4, x4
mov v0.d[0], x4
lsl x3, x3, #3
ubfiz x3, x3, #3, #32
rbit x3, x3
mov v0.d[1], x3
eor v5.16b, v5.16b, v0.16b
@@ -32668,10 +32674,10 @@ _AES_GCM_decrypt_final_AARCH64:
ld1 {v4.2d}, [x5]
ushr v6.2d, v6.2d, #56
ld1 {v7.2d}, [x6]
lsl x4, x4, #3
ubfiz x4, x4, #3, #32
rbit x4, x4
mov v0.d[0], x4
lsl x3, x3, #3
ubfiz x3, x3, #3, #32
rbit x3, x3
mov v0.d[1], x3
eor v5.16b, v5.16b, v0.16b
@@ -32696,6 +32702,7 @@ _AES_GCM_decrypt_final_AARCH64:
ld1 {v0.16b}, [x1]
b L_aes_gcm_decrypt_final_arm64_crypto_tag_loaded
L_aes_gcm_decrypt_final_arm64_crypto_part_tag:
ubfiz x2, x2, #0, #32
eor v0.16b, v0.16b, v0.16b
mov x10, x2
st1 {v0.2d}, [x0]
@@ -32856,7 +32863,7 @@ L_aes_gcm_init_arm64_crypto_eor3_end_bytes:
# Done GHASH
L_aes_gcm_init_arm64_crypto_eor3_partial_done:
eor x7, x7, x7
lsl x13, x3, #3
ubfiz x13, x3, #3, #32
mov v7.d[0], x7
mov v7.d[1], x13
rev64 v7.16b, v7.16b
@@ -37114,10 +37121,10 @@ _AES_GCM_encrypt_final_AARCH64_EOR3:
ld1 {v4.2d}, [x5]
ushr v6.2d, v6.2d, #56
ld1 {v7.2d}, [x6]
lsl x4, x4, #3
ubfiz x4, x4, #3, #32
rbit x4, x4
mov v0.d[0], x4
lsl x3, x3, #3
ubfiz x3, x3, #3, #32
rbit x3, x3
mov v0.d[1], x3
eor v5.16b, v5.16b, v0.16b
@@ -40915,10 +40922,10 @@ _AES_GCM_decrypt_final_AARCH64_EOR3:
ld1 {v4.2d}, [x5]
ushr v6.2d, v6.2d, #56
ld1 {v7.2d}, [x6]
lsl x4, x4, #3
ubfiz x4, x4, #3, #32
rbit x4, x4
mov v0.d[0], x4
lsl x3, x3, #3
ubfiz x3, x3, #3, #32
rbit x3, x3
mov v0.d[1], x3
eor v5.16b, v5.16b, v0.16b
@@ -40942,6 +40949,7 @@ _AES_GCM_decrypt_final_AARCH64_EOR3:
ld1 {v0.16b}, [x1]
b L_aes_gcm_decrypt_final_arm64_crypto_eor3_tag_loaded
L_aes_gcm_decrypt_final_arm64_crypto_eor3_part_tag:
ubfiz x2, x2, #0, #32
eor v0.16b, v0.16b, v0.16b
mov x10, x2
st1 {v0.2d}, [x0]
wolfcrypt/src/port/arm/armv8-aes-asm_c.c
+872 -939
View File
File diff suppressed because it is too large Load Diff
wolfcrypt/src/port/arm/armv8-curve25519.S
+8
View File
@@ -5401,6 +5401,14 @@ L_curve25519_bits:
adcs x7, x7, x25
adcs x8, x8, x26
adc x9, x9, x27
# Reduce if top bit set
mov x3, #19
and x4, x3, x9, asr 63
adds x6, x6, x4
adcs x7, x7, xzr
and x9, x9, #0x7fffffffffffffff
adcs x8, x8, xzr
adc x9, x9, xzr
# Square
# A[0] * A[1]
umulh x16, x19, x20
wolfcrypt/src/port/arm/armv8-curve25519_c.c
+13 -5
View File
@@ -38,7 +38,7 @@
#if !defined(CURVE25519_SMALL) || !defined(ED25519_SMALL)
#include <wolfssl/wolfcrypt/fe_operations.h>
void fe_init()
void fe_init(void)
{
__asm__ __volatile__ (
"\n\t"
@@ -229,8 +229,8 @@ int fe_isnonzero(const fe a)
"orr %x[a], x1, x2\n\t"
"orr x3, x3, x4\n\t"
"orr %x[a], %x[a], x3\n\t"
: [a] "+r" (a)
:
: [a] "r" (a)
: "memory", "cc", "x1", "x2", "x3", "x4", "x5", "x6"
);
return (word32)(size_t)a;
@@ -248,8 +248,8 @@ int fe_isnegative(const fe a)
"adc x5, x4, xzr\n\t"
"and %x[a], x1, #1\n\t"
"eor %x[a], %x[a], x5, lsr 63\n\t"
: [a] "+r" (a)
:
: [a] "r" (a)
: "memory", "cc", "x1", "x2", "x3", "x4", "x5", "x6"
);
return (word32)(size_t)a;
@@ -4362,7 +4362,7 @@ int curve25519_base(byte* r, const byte* n)
/* Store */
"stp x14, x15, [%x[r]]\n\t"
"stp x16, x17, [%x[r], #16]\n\t"
"mov x0, xzr\n\t"
"mov %x[r], xzr\n\t"
"ldp x29, x30, [sp], #0xb0\n\t"
: [r] "+r" (r)
: [n] "r" (n), [x2] "r" (x2)
@@ -5271,6 +5271,14 @@ int curve25519(byte* r, const byte* n, const byte* a)
"adcs x7, x7, x25\n\t"
"adcs x8, x8, x26\n\t"
"adc x9, x9, x27\n\t"
/* Reduce if top bit set */
"mov x3, #19\n\t"
"and x4, x3, x9, asr 63\n\t"
"adds x6, x6, x4\n\t"
"adcs x7, x7, xzr\n\t"
"and x9, x9, #0x7fffffffffffffff\n\t"
"adcs x8, x8, xzr\n\t"
"adc x9, x9, xzr\n\t"
/* Square */
/* A[0] * A[1] */
"umulh x16, x19, x20\n\t"
@@ -6969,7 +6977,7 @@ int curve25519(byte* r, const byte* n, const byte* a)
/* Store */
"stp x14, x15, [%x[r]]\n\t"
"stp x16, x17, [%x[r], #16]\n\t"
"mov x0, xzr\n\t"
"mov %x[r], xzr\n\t"
"ldp x29, x30, [sp], #0xc0\n\t"
: [r] "+r" (r)
: [n] "r" (n), [a] "r" (a)
wolfcrypt/src/port/arm/armv8-mlkem-asm_c.c
+6 -6
View File
@@ -8406,11 +8406,11 @@ int mlkem_cmp_neon(const byte* a, const byte* b, int sz)
"orr v8.16b, v8.16b, v10.16b\n\t"
"ext v9.16b, v8.16b, v8.16b, #8\n\t"
"orr v8.16b, v8.16b, v9.16b\n\t"
"mov x0, v8.d[0]\n\t"
"subs x0, x0, xzr\n\t"
"csetm w0, ne\n\t"
: [sz] "+r" (sz)
: [a] "r" (a), [b] "r" (b)
"mov %x[a], v8.d[0]\n\t"
"subs %x[a], %x[a], xzr\n\t"
"csetm %w[a], ne\n\t"
: [a] "+r" (a), [sz] "+r" (sz)
: [b] "r" (b)
: "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8",
"v9", "v10", "v11"
);
@@ -9089,7 +9089,7 @@ unsigned int mlkem_rej_uniform_neon(sword16* p, unsigned int len, const byte* r,
"b L_mlkem_rej_uniform_loop_lt_4_%=\n\t"
"\n"
"L_mlkem_rej_uniform_done_%=:\n\t"
"mov x0, x12\n\t"
"mov %x[p], x12\n\t"
: [p] "+r" (p), [len] "+r" (len), [rLen] "+r" (rLen)
: [r] "r" (r), [mask] "r" (mask), [q] "r" (q), [bits] "r" (bits),
[indices] "r" (indices)
wolfcrypt/src/port/arm/thumb2-curve25519.S
+1031 -18
View File
File diff suppressed because it is too large Load Diff
wolfcrypt/src/port/arm/thumb2-curve25519_c.c
+1093 -32
View File
File diff suppressed because it is too large Load Diff
wolfcrypt/src/sp_arm32.c
+18 -27
View File
@@ -11951,9 +11951,8 @@ WC_OMIT_FRAME_POINTER static sp_digit div_2048_word_32(sp_digit d1, sp_digit d0,
"subs r7, %[d0], r4\n\t"
"sbc r8, %[d1], r5\n\t"
"add r3, r3, r8\n\t"
"subs r6, %[div], r7\n\t"
"sbc r6, r6, r6\n\t"
"sub %[d1], r3, r6\n\t"
"subs r6, r7, %[div]\n\t"
"adc %[d1], r3, #0\n\t"
#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
: [d1] "+r" (d1), [d0] "+r" (d0), [div] "+r" (div)
:
@@ -16382,9 +16381,8 @@ WC_OMIT_FRAME_POINTER static sp_digit div_2048_word_64(sp_digit d1, sp_digit d0,
"subs r7, %[d0], r4\n\t"
"sbc r8, %[d1], r5\n\t"
"add r3, r3, r8\n\t"
"subs r6, %[div], r7\n\t"
"sbc r6, r6, r6\n\t"
"sub %[d1], r3, r6\n\t"
"subs r6, r7, %[div]\n\t"
"adc %[d1], r3, #0\n\t"
#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
: [d1] "+r" (d1), [d0] "+r" (d0), [div] "+r" (div)
:
@@ -37688,9 +37686,8 @@ WC_OMIT_FRAME_POINTER static sp_digit div_3072_word_48(sp_digit d1, sp_digit d0,
"subs r7, %[d0], r4\n\t"
"sbc r8, %[d1], r5\n\t"
"add r3, r3, r8\n\t"
"subs r6, %[div], r7\n\t"
"sbc r6, r6, r6\n\t"
"sub %[d1], r3, r6\n\t"
"subs r6, r7, %[div]\n\t"
"adc %[d1], r3, #0\n\t"
#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
: [d1] "+r" (d1), [d0] "+r" (d0), [div] "+r" (div)
:
@@ -43807,9 +43804,8 @@ WC_OMIT_FRAME_POINTER static sp_digit div_3072_word_96(sp_digit d1, sp_digit d0,
"subs r7, %[d0], r4\n\t"
"sbc r8, %[d1], r5\n\t"
"add r3, r3, r8\n\t"
"subs r6, %[div], r7\n\t"
"sbc r6, r6, r6\n\t"
"sub %[d1], r3, r6\n\t"
"subs r6, r7, %[div]\n\t"
"adc %[d1], r3, #0\n\t"
#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
: [d1] "+r" (d1), [d0] "+r" (d0), [div] "+r" (div)
:
@@ -59105,9 +59101,8 @@ WC_OMIT_FRAME_POINTER static sp_digit div_4096_word_128(sp_digit d1,
"subs r7, %[d0], r4\n\t"
"sbc r8, %[d1], r5\n\t"
"add r3, r3, r8\n\t"
"subs r6, %[div], r7\n\t"
"sbc r6, r6, r6\n\t"
"sub %[d1], r3, r6\n\t"
"subs r6, r7, %[div]\n\t"
"adc %[d1], r3, #0\n\t"
#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
: [d1] "+r" (d1), [d0] "+r" (d0), [div] "+r" (div)
:
@@ -79263,9 +79258,8 @@ WC_OMIT_FRAME_POINTER static sp_digit div_256_word_8(sp_digit d1, sp_digit d0,
"subs r7, %[d0], r4\n\t"
"sbc r8, %[d1], r5\n\t"
"add r3, r3, r8\n\t"
"subs r6, %[div], r7\n\t"
"sbc r6, r6, r6\n\t"
"sub %[d1], r3, r6\n\t"
"subs r6, r7, %[div]\n\t"
"adc %[d1], r3, #0\n\t"
#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
: [d1] "+r" (d1), [d0] "+r" (d0), [div] "+r" (div)
:
@@ -97707,9 +97701,8 @@ WC_OMIT_FRAME_POINTER static sp_digit div_384_word_12(sp_digit d1, sp_digit d0,
"subs r7, %[d0], r4\n\t"
"sbc r8, %[d1], r5\n\t"
"add r3, r3, r8\n\t"
"subs r6, %[div], r7\n\t"
"sbc r6, r6, r6\n\t"
"sub %[d1], r3, r6\n\t"
"subs r6, r7, %[div]\n\t"
"adc %[d1], r3, #0\n\t"
#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
: [d1] "+r" (d1), [d0] "+r" (d0), [div] "+r" (div)
:
@@ -126361,9 +126354,8 @@ WC_OMIT_FRAME_POINTER static sp_digit div_521_word_17(sp_digit d1, sp_digit d0,
"subs r7, %[d0], r4\n\t"
"sbc r8, %[d1], r5\n\t"
"add r3, r3, r8\n\t"
"subs r6, %[div], r7\n\t"
"sbc r6, r6, r6\n\t"
"sub %[d1], r3, r6\n\t"
"subs r6, r7, %[div]\n\t"
"adc %[d1], r3, #0\n\t"
#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
: [d1] "+r" (d1), [d0] "+r" (d0), [div] "+r" (div)
:
@@ -146927,9 +146919,8 @@ WC_OMIT_FRAME_POINTER static sp_digit div_1024_word_32(sp_digit d1, sp_digit d0,
"subs r7, %[d0], r4\n\t"
"sbc r8, %[d1], r5\n\t"
"add r3, r3, r8\n\t"
"subs r6, %[div], r7\n\t"
"sbc r6, r6, r6\n\t"
"sub %[d1], r3, r6\n\t"
"subs r6, r7, %[div]\n\t"
"adc %[d1], r3, #0\n\t"
#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
: [d1] "+r" (d1), [d0] "+r" (d0), [div] "+r" (div)
:
wolfcrypt/src/sp_armthumb.c
+90 -45
View File
@@ -23915,11 +23915,11 @@ WC_OMIT_FRAME_POINTER static SP_NOINLINE sp_digit div_2048_word_32(sp_digit d1,
#else
"add r3, r3, r5\n\t"
#endif
"movs r6, %[div]\n\t"
"movs r6, r4\n\t"
#if defined(__clang__) || defined(WOLFSSL_KEIL)
"subs r6, r6, r4\n\t"
"subs r6, r6, %[div]\n\t"
#else
"sub r6, r6, r4\n\t"
"sub r6, r6, %[div]\n\t"
#endif
#ifdef WOLFSSL_KEIL
"sbcs r6, r6, r6\n\t"
@@ -23929,9 +23929,14 @@ WC_OMIT_FRAME_POINTER static SP_NOINLINE sp_digit div_2048_word_32(sp_digit d1,
"sbc r6, r6\n\t"
#endif
#if defined(__clang__) || defined(WOLFSSL_KEIL)
"subs r3, r3, r6\n\t"
"adds r3, r3, #1\n\t"
#else
"sub r3, r3, r6\n\t"
"add r3, r3, #1\n\t"
#endif
#if defined(__clang__) || defined(WOLFSSL_KEIL)
"adds r3, r3, r6\n\t"
#else
"add r3, r3, r6\n\t"
#endif
"movs %[d1], r3\n\t"
: [d1] "+l" (d1), [d0] "+l" (d0), [div] "+l" (div)
@@ -27389,11 +27394,11 @@ WC_OMIT_FRAME_POINTER static SP_NOINLINE sp_digit div_2048_word_64(sp_digit d1,
#else
"add r3, r3, r5\n\t"
#endif
"movs r6, %[div]\n\t"
"movs r6, r4\n\t"
#if defined(__clang__) || defined(WOLFSSL_KEIL)
"subs r6, r6, r4\n\t"
"subs r6, r6, %[div]\n\t"
#else
"sub r6, r6, r4\n\t"
"sub r6, r6, %[div]\n\t"
#endif
#ifdef WOLFSSL_KEIL
"sbcs r6, r6, r6\n\t"
@@ -27403,9 +27408,14 @@ WC_OMIT_FRAME_POINTER static SP_NOINLINE sp_digit div_2048_word_64(sp_digit d1,
"sbc r6, r6\n\t"
#endif
#if defined(__clang__) || defined(WOLFSSL_KEIL)
"subs r3, r3, r6\n\t"
"adds r3, r3, #1\n\t"
#else
"sub r3, r3, r6\n\t"
"add r3, r3, #1\n\t"
#endif
#if defined(__clang__) || defined(WOLFSSL_KEIL)
"adds r3, r3, r6\n\t"
#else
"add r3, r3, r6\n\t"
#endif
"movs %[d1], r3\n\t"
: [d1] "+l" (d1), [d0] "+l" (d0), [div] "+l" (div)
@@ -75363,11 +75373,11 @@ WC_OMIT_FRAME_POINTER static SP_NOINLINE sp_digit div_3072_word_48(sp_digit d1,
#else
"add r3, r3, r5\n\t"
#endif
"movs r6, %[div]\n\t"
"movs r6, r4\n\t"
#if defined(__clang__) || defined(WOLFSSL_KEIL)
"subs r6, r6, r4\n\t"
"subs r6, r6, %[div]\n\t"
#else
"sub r6, r6, r4\n\t"
"sub r6, r6, %[div]\n\t"
#endif
#ifdef WOLFSSL_KEIL
"sbcs r6, r6, r6\n\t"
@@ -75377,9 +75387,14 @@ WC_OMIT_FRAME_POINTER static SP_NOINLINE sp_digit div_3072_word_48(sp_digit d1,
"sbc r6, r6\n\t"
#endif
#if defined(__clang__) || defined(WOLFSSL_KEIL)
"subs r3, r3, r6\n\t"
"adds r3, r3, #1\n\t"
#else
"sub r3, r3, r6\n\t"
"add r3, r3, #1\n\t"
#endif
#if defined(__clang__) || defined(WOLFSSL_KEIL)
"adds r3, r3, r6\n\t"
#else
"add r3, r3, r6\n\t"
#endif
"movs %[d1], r3\n\t"
: [d1] "+l" (d1), [d0] "+l" (d0), [div] "+l" (div)
@@ -79663,11 +79678,11 @@ WC_OMIT_FRAME_POINTER static SP_NOINLINE sp_digit div_3072_word_96(sp_digit d1,
#else
"add r3, r3, r5\n\t"
#endif
"movs r6, %[div]\n\t"
"movs r6, r4\n\t"
#if defined(__clang__) || defined(WOLFSSL_KEIL)
"subs r6, r6, r4\n\t"
"subs r6, r6, %[div]\n\t"
#else
"sub r6, r6, r4\n\t"
"sub r6, r6, %[div]\n\t"
#endif
#ifdef WOLFSSL_KEIL
"sbcs r6, r6, r6\n\t"
@@ -79677,9 +79692,14 @@ WC_OMIT_FRAME_POINTER static SP_NOINLINE sp_digit div_3072_word_96(sp_digit d1,
"sbc r6, r6\n\t"
#endif
#if defined(__clang__) || defined(WOLFSSL_KEIL)
"subs r3, r3, r6\n\t"
"adds r3, r3, #1\n\t"
#else
"sub r3, r3, r6\n\t"
"add r3, r3, #1\n\t"
#endif
#if defined(__clang__) || defined(WOLFSSL_KEIL)
"adds r3, r3, r6\n\t"
#else
"add r3, r3, r6\n\t"
#endif
"movs %[d1], r3\n\t"
: [d1] "+l" (d1), [d0] "+l" (d0), [div] "+l" (div)
@@ -92180,11 +92200,11 @@ WC_OMIT_FRAME_POINTER static SP_NOINLINE sp_digit div_4096_word_128(sp_digit d1,
#else
"add r3, r3, r5\n\t"
#endif
"movs r6, %[div]\n\t"
"movs r6, r4\n\t"
#if defined(__clang__) || defined(WOLFSSL_KEIL)
"subs r6, r6, r4\n\t"
"subs r6, r6, %[div]\n\t"
#else
"sub r6, r6, r4\n\t"
"sub r6, r6, %[div]\n\t"
#endif
#ifdef WOLFSSL_KEIL
"sbcs r6, r6, r6\n\t"
@@ -92194,9 +92214,14 @@ WC_OMIT_FRAME_POINTER static SP_NOINLINE sp_digit div_4096_word_128(sp_digit d1,
"sbc r6, r6\n\t"
#endif
#if defined(__clang__) || defined(WOLFSSL_KEIL)
"subs r3, r3, r6\n\t"
"adds r3, r3, #1\n\t"
#else
"sub r3, r3, r6\n\t"
"add r3, r3, #1\n\t"
#endif
#if defined(__clang__) || defined(WOLFSSL_KEIL)
"adds r3, r3, r6\n\t"
#else
"add r3, r3, r6\n\t"
#endif
"movs %[d1], r3\n\t"
: [d1] "+l" (d1), [d0] "+l" (d0), [div] "+l" (div)
@@ -105145,11 +105170,11 @@ WC_OMIT_FRAME_POINTER static SP_NOINLINE sp_digit div_256_word_8(sp_digit d1,
#else
"add r3, r3, r5\n\t"
#endif
"movs r6, %[div]\n\t"
"movs r6, r4\n\t"
#if defined(__clang__) || defined(WOLFSSL_KEIL)
"subs r6, r6, r4\n\t"
"subs r6, r6, %[div]\n\t"
#else
"sub r6, r6, r4\n\t"
"sub r6, r6, %[div]\n\t"
#endif
#ifdef WOLFSSL_KEIL
"sbcs r6, r6, r6\n\t"
@@ -105159,9 +105184,14 @@ WC_OMIT_FRAME_POINTER static SP_NOINLINE sp_digit div_256_word_8(sp_digit d1,
"sbc r6, r6\n\t"
#endif
#if defined(__clang__) || defined(WOLFSSL_KEIL)
"subs r3, r3, r6\n\t"
"adds r3, r3, #1\n\t"
#else
"sub r3, r3, r6\n\t"
"add r3, r3, #1\n\t"
#endif
#if defined(__clang__) || defined(WOLFSSL_KEIL)
"adds r3, r3, r6\n\t"
#else
"add r3, r3, r6\n\t"
#endif
"movs %[d1], r3\n\t"
: [d1] "+l" (d1), [d0] "+l" (d0), [div] "+l" (div)
@@ -115944,11 +115974,11 @@ WC_OMIT_FRAME_POINTER static SP_NOINLINE sp_digit div_384_word_12(sp_digit d1,
#else
"add r3, r3, r5\n\t"
#endif
"movs r6, %[div]\n\t"
"movs r6, r4\n\t"
#if defined(__clang__) || defined(WOLFSSL_KEIL)
"subs r6, r6, r4\n\t"
"subs r6, r6, %[div]\n\t"
#else
"sub r6, r6, r4\n\t"
"sub r6, r6, %[div]\n\t"
#endif
#ifdef WOLFSSL_KEIL
"sbcs r6, r6, r6\n\t"
@@ -115958,9 +115988,14 @@ WC_OMIT_FRAME_POINTER static SP_NOINLINE sp_digit div_384_word_12(sp_digit d1,
"sbc r6, r6\n\t"
#endif
#if defined(__clang__) || defined(WOLFSSL_KEIL)
"subs r3, r3, r6\n\t"
"adds r3, r3, #1\n\t"
#else
"sub r3, r3, r6\n\t"
"add r3, r3, #1\n\t"
#endif
#if defined(__clang__) || defined(WOLFSSL_KEIL)
"adds r3, r3, r6\n\t"
#else
"add r3, r3, r6\n\t"
#endif
"movs %[d1], r3\n\t"
: [d1] "+l" (d1), [d0] "+l" (d0), [div] "+l" (div)
@@ -131764,11 +131799,11 @@ WC_OMIT_FRAME_POINTER static SP_NOINLINE sp_digit div_521_word_17(sp_digit d1,
#else
"add r3, r3, r5\n\t"
#endif
"movs r6, %[div]\n\t"
"movs r6, r4\n\t"
#if defined(__clang__) || defined(WOLFSSL_KEIL)
"subs r6, r6, r4\n\t"
"subs r6, r6, %[div]\n\t"
#else
"sub r6, r6, r4\n\t"
"sub r6, r6, %[div]\n\t"
#endif
#ifdef WOLFSSL_KEIL
"sbcs r6, r6, r6\n\t"
@@ -131778,9 +131813,14 @@ WC_OMIT_FRAME_POINTER static SP_NOINLINE sp_digit div_521_word_17(sp_digit d1,
"sbc r6, r6\n\t"
#endif
#if defined(__clang__) || defined(WOLFSSL_KEIL)
"subs r3, r3, r6\n\t"
"adds r3, r3, #1\n\t"
#else
"sub r3, r3, r6\n\t"
"add r3, r3, #1\n\t"
#endif
#if defined(__clang__) || defined(WOLFSSL_KEIL)
"adds r3, r3, r6\n\t"
#else
"add r3, r3, r6\n\t"
#endif
"movs %[d1], r3\n\t"
: [d1] "+l" (d1), [d0] "+l" (d0), [div] "+l" (div)
@@ -201799,11 +201839,11 @@ WC_OMIT_FRAME_POINTER static SP_NOINLINE sp_digit div_1024_word_32(sp_digit d1,
#else
"add r3, r3, r5\n\t"
#endif
"movs r6, %[div]\n\t"
"movs r6, r4\n\t"
#if defined(__clang__) || defined(WOLFSSL_KEIL)
"subs r6, r6, r4\n\t"
"subs r6, r6, %[div]\n\t"
#else
"sub r6, r6, r4\n\t"
"sub r6, r6, %[div]\n\t"
#endif
#ifdef WOLFSSL_KEIL
"sbcs r6, r6, r6\n\t"
@@ -201813,9 +201853,14 @@ WC_OMIT_FRAME_POINTER static SP_NOINLINE sp_digit div_1024_word_32(sp_digit d1,
"sbc r6, r6\n\t"
#endif
#if defined(__clang__) || defined(WOLFSSL_KEIL)
"subs r3, r3, r6\n\t"
"adds r3, r3, #1\n\t"
#else
"sub r3, r3, r6\n\t"
"add r3, r3, #1\n\t"
#endif
#if defined(__clang__) || defined(WOLFSSL_KEIL)
"adds r3, r3, r6\n\t"
#else
"add r3, r3, r6\n\t"
#endif
"movs %[d1], r3\n\t"
: [d1] "+l" (d1), [d0] "+l" (d0), [div] "+l" (div)
wolfcrypt/src/sp_c32.c
+6
View File
@@ -26258,6 +26258,8 @@ static int sp_256_div_9(const sp_digit* a, const sp_digit* d,
sp_256_norm_9(&t1[i + 1]);
}
sp_256_norm_9(t1);
sp_256_cond_add_9(t1, t1, sd, t1[8] >> 31);
sp_256_norm_9(t1);
sp_256_rshift_9(r, t1, 5);
}
@@ -34192,6 +34194,8 @@ static int sp_384_div_15(const sp_digit* a, const sp_digit* d,
sp_384_norm_15(&t1[i + 1]);
}
sp_384_norm_15(t1);
sp_384_cond_add_15(t1, t1, sd, t1[14] >> 31);
sp_384_norm_15(t1);
sp_384_rshift_15(r, t1, 6);
}
@@ -42165,6 +42169,8 @@ static int sp_521_div_21(const sp_digit* a, const sp_digit* d,
sp_521_norm_21(&t1[i + 1]);
}
sp_521_norm_21(t1);
sp_521_cond_add_21(t1, t1, sd, t1[20] >> 31);
sp_521_norm_21(t1);
sp_521_rshift_21(r, t1, 4);
}
wolfcrypt/src/sp_c64.c
+4
View File
@@ -34293,6 +34293,8 @@ static int sp_384_div_7(const sp_digit* a, const sp_digit* d,
sp_384_norm_7(&t1[i + 1]);
}
sp_384_norm_7(t1);
sp_384_cond_add_7(t1, t1, sd, t1[6] >> 63);
sp_384_norm_7(t1);
sp_384_rshift_7(r, t1, 1);
}
@@ -41580,6 +41582,8 @@ static int sp_521_div_9(const sp_digit* a, const sp_digit* d,
sp_521_norm_9(&t1[i + 1]);
}
sp_521_norm_9(t1);
sp_521_cond_add_9(t1, t1, sd, t1[8] >> 63);
sp_521_norm_9(t1);
sp_521_rshift_9(r, t1, 1);
}
wolfcrypt/src/sp_cortexm.c
+18 -27
View File
@@ -5047,9 +5047,8 @@ WC_OMIT_FRAME_POINTER static SP_NOINLINE sp_digit div_2048_word_32(sp_digit d1,
"SUBS r9, %[d0], r6\n\t"
"SBC r10, %[d1], r7\n\t"
"ADD r3, r3, r10\n\t"
"SUBS r8, %[div], r9\n\t"
"SBC r8, r8, r8\n\t"
"SUB %[d1], r3, r8\n\t"
"SUBS r8, r9, %[div]\n\t"
"ADC %[d1], r3, #0\n\t"
#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
: [d1] "+r" (d1), [d0] "+r" (d0), [div] "+r" (div)
:
@@ -7786,9 +7785,8 @@ WC_OMIT_FRAME_POINTER static SP_NOINLINE sp_digit div_2048_word_64(sp_digit d1,
"SUBS r9, %[d0], r6\n\t"
"SBC r10, %[d1], r7\n\t"
"ADD r3, r3, r10\n\t"
"SUBS r8, %[div], r9\n\t"
"SBC r8, r8, r8\n\t"
"SUB %[d1], r3, r8\n\t"
"SUBS r8, r9, %[div]\n\t"
"ADC %[d1], r3, #0\n\t"
#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
: [d1] "+r" (d1), [d0] "+r" (d0), [div] "+r" (div)
:
@@ -16836,9 +16834,8 @@ WC_OMIT_FRAME_POINTER static SP_NOINLINE sp_digit div_3072_word_48(sp_digit d1,
"SUBS r9, %[d0], r6\n\t"
"SBC r10, %[d1], r7\n\t"
"ADD r3, r3, r10\n\t"
"SUBS r8, %[div], r9\n\t"
"SBC r8, r8, r8\n\t"
"SUB %[d1], r3, r8\n\t"
"SUBS r8, r9, %[div]\n\t"
"ADC %[d1], r3, #0\n\t"
#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
: [d1] "+r" (d1), [d0] "+r" (d0), [div] "+r" (div)
:
@@ -20335,9 +20332,8 @@ WC_OMIT_FRAME_POINTER static SP_NOINLINE sp_digit div_3072_word_96(sp_digit d1,
"SUBS r9, %[d0], r6\n\t"
"SBC r10, %[d1], r7\n\t"
"ADD r3, r3, r10\n\t"
"SUBS r8, %[div], r9\n\t"
"SBC r8, r8, r8\n\t"
"SUB %[d1], r3, r8\n\t"
"SUBS r8, r9, %[div]\n\t"
"ADC %[d1], r3, #0\n\t"
#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
: [d1] "+r" (d1), [d0] "+r" (d0), [div] "+r" (div)
:
@@ -28485,9 +28481,8 @@ WC_OMIT_FRAME_POINTER static SP_NOINLINE sp_digit div_4096_word_128(sp_digit d1,
"SUBS r9, %[d0], r6\n\t"
"SBC r10, %[d1], r7\n\t"
"ADD r3, r3, r10\n\t"
"SUBS r8, %[div], r9\n\t"
"SBC r8, r8, r8\n\t"
"SUB %[d1], r3, r8\n\t"
"SUBS r8, r9, %[div]\n\t"
"ADC %[d1], r3, #0\n\t"
#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
: [d1] "+r" (d1), [d0] "+r" (d0), [div] "+r" (div)
:
@@ -41062,9 +41057,8 @@ WC_OMIT_FRAME_POINTER static SP_NOINLINE sp_digit div_256_word_8(sp_digit d1,
"SUBS r9, %[d0], r6\n\t"
"SBC r10, %[d1], r7\n\t"
"ADD r3, r3, r10\n\t"
"SUBS r8, %[div], r9\n\t"
"SBC r8, r8, r8\n\t"
"SUB %[d1], r3, r8\n\t"
"SUBS r8, r9, %[div]\n\t"
"ADC %[d1], r3, #0\n\t"
#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
: [d1] "+r" (d1), [d0] "+r" (d0), [div] "+r" (div)
:
@@ -51466,9 +51460,8 @@ WC_OMIT_FRAME_POINTER static SP_NOINLINE sp_digit div_384_word_12(sp_digit d1,
"SUBS r9, %[d0], r6\n\t"
"SBC r10, %[d1], r7\n\t"
"ADD r3, r3, r10\n\t"
"SUBS r8, %[div], r9\n\t"
"SBC r8, r8, r8\n\t"
"SUB %[d1], r3, r8\n\t"
"SUBS r8, r9, %[div]\n\t"
"ADC %[d1], r3, #0\n\t"
#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
: [d1] "+r" (d1), [d0] "+r" (d0), [div] "+r" (div)
:
@@ -64807,9 +64800,8 @@ WC_OMIT_FRAME_POINTER static SP_NOINLINE sp_digit div_521_word_17(sp_digit d1,
"SUBS r9, %[d0], r6\n\t"
"SBC r10, %[d1], r7\n\t"
"ADD r3, r3, r10\n\t"
"SUBS r8, %[div], r9\n\t"
"SBC r8, r8, r8\n\t"
"SUB %[d1], r3, r8\n\t"
"SUBS r8, r9, %[div]\n\t"
"ADC %[d1], r3, #0\n\t"
#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
: [d1] "+r" (d1), [d0] "+r" (d0), [div] "+r" (div)
:
@@ -71609,9 +71601,8 @@ WC_OMIT_FRAME_POINTER static SP_NOINLINE sp_digit div_1024_word_32(sp_digit d1,
"SUBS r9, %[d0], r6\n\t"
"SBC r10, %[d1], r7\n\t"
"ADD r3, r3, r10\n\t"
"SUBS r8, %[div], r9\n\t"
"SBC r8, r8, r8\n\t"
"SUB %[d1], r3, r8\n\t"
"SUBS r8, r9, %[div]\n\t"
"ADC %[d1], r3, #0\n\t"
#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
: [d1] "+r" (d1), [d0] "+r" (d0), [div] "+r" (div)
:
wolfcrypt/src/sp_int.c
+32 -29
View File
@@ -5929,6 +5929,35 @@ int sp_cmp_ct(const sp_int* a, const sp_int* b, unsigned int n)
}
#endif /* HAVE_ECC && !WC_NO_RNG && WOLFSSL_ECC_GEN_REJECT_SAMPLING */
/* Constant time clamping.
*
* @param [in, out] a SP integer to clamp.
*/
static void sp_clamp_ct(sp_int* a)
{
int i;
sp_size_t used = a->used;
volatile sp_size_t mask = (sp_size_t)-1;
for (i = (int)a->used - 1; i >= 0; i--) {
#if ((SP_WORD_SIZE == 64) && \
(defined(_WIN64) || !defined(WOLFSSL_UINT128_T_DEFINED))) || \
((SP_WORD_SIZE == 32) && defined(NO_64BIT))
sp_int_digit negVal = ~a->dp[i];
sp_int_digit minusOne = a->dp[i] - 1;
sp_int_digit zeroMask =
(sp_int_digit)((sp_int_sdigit)(negVal & minusOne) >>
(SP_WORD_SIZE - 1));
#else
sp_size_t zeroMask =
(sp_size_t)((((sp_int_sword)a->dp[i]) - 1) >> SP_WORD_SIZE);
#endif
mask &= (sp_size_t)zeroMask;
used = (sp_size_t)(used + mask);
}
a->used = used;
}
/*************************
* Bit check/set functions
*************************/
@@ -7550,6 +7579,9 @@ int sp_div_2_mod_ct(const sp_int* a, const sp_int* m, sp_int* r)
/* Divide conditional sum by 2. */
_sp_div_2(r, r);
/* Remove leading zeros. */
sp_clamp_ct(r);
#if 0
sp_print(r, "rd2");
#endif
@@ -8133,35 +8165,6 @@ int sp_submod(const sp_int* a, const sp_int* b, const sp_int* m, sp_int* r)
}
#endif /* WOLFSSL_SP_MATH_ALL */
/* Constant time clamping.
*
* @param [in, out] a SP integer to clamp.
*/
static void sp_clamp_ct(sp_int* a)
{
int i;
sp_size_t used = a->used;
volatile sp_size_t mask = (sp_size_t)-1;
for (i = (int)a->used - 1; i >= 0; i--) {
#if ((SP_WORD_SIZE == 64) && \
(defined(_WIN64) || !defined(WOLFSSL_UINT128_T_DEFINED))) || \
((SP_WORD_SIZE == 32) && defined(NO_64BIT))
sp_int_digit negVal = ~a->dp[i];
sp_int_digit minusOne = a->dp[i] - 1;
sp_int_digit zeroMask =
(sp_int_digit)((sp_int_sdigit)(negVal & minusOne) >>
(SP_WORD_SIZE - 1));
#else
sp_size_t zeroMask =
(sp_size_t)((((sp_int_sword)a->dp[i]) - 1) >> SP_WORD_SIZE);
#endif
mask &= (sp_size_t)zeroMask;
used = (sp_size_t)(used + mask);
}
a->used = used;
}
#if defined(WOLFSSL_SP_MATH_ALL) && defined(HAVE_ECC)
/* Add two values and reduce: r = (a + b) % m
*
wolfcrypt/src/wc_mldsa.c
+12 -8
View File
@@ -5441,8 +5441,9 @@ static void mldsa_use_hint_88(sword32* w1, const byte* h, unsigned int i,
r = w1[j] + ((0 - (((word32)w1[j]) >> 31)) & MLDSA_Q);
/* Decompose value into low and high parts. */
mldsa_decompose_q88(r, &r0, &r1);
/* Make hint positive or negative based on sign of r0. */
hint = (1 - (2 * (((word32)r0) >> 31))) & (0 - hint);
/* Make hint positive or negative: r0 > 0 adds, r0 <= 0 subtracts
* (FIPS 204). (r0 - 1) >> 31 is 1 when r0 <= 0. */
hint = (1 - (2 * (((word32)(r0 - 1)) >> 31))) & (0 - hint);
/* Make w1 only the top part plus the hint. */
w1[j] = r1 + hint;
@@ -5457,8 +5458,9 @@ static void mldsa_use_hint_88(sword32* w1, const byte* h, unsigned int i,
mldsa_decompose_q88(r, &r0, &r1);
/* Check for hint. */
if ((o < h[PARAMS_ML_DSA_44_OMEGA + i]) && (h[o] == (byte)j)) {
/* Add or subtract hint based on sign of r0. */
r1 += (sword32)(1U - (2U * (((word32)r0) >> 31)));
/* Add or subtract hint: r0 > 0 adds, r0 <= 0 subtracts
* (FIPS 204). (r0 - 1) >> 31 is 1 when r0 <= 0. */
r1 += (sword32)(1U - (2U * (((word32)(r0 - 1)) >> 31)));
/* Go to next hint offset. */
o++;
}
@@ -5511,8 +5513,9 @@ static void mldsa_use_hint_32(sword32* w1, const byte* h, byte omega,
r = w1[j] + (sword32)((0 - (((word32)w1[j]) >> 31)) & MLDSA_Q);
/* Decompose value into low and high parts. */
mldsa_decompose_q32(r, &r0, &r1);
/* Make hint positive or negative based on sign of r0. */
hint = (sword32)((1 - (2 * (((word32)r0) >> 31))) & (0 - hint));
/* Make hint positive or negative: r0 > 0 adds, r0 <= 0 subtracts
* (FIPS 204). (r0 - 1) >> 31 is 1 when r0 <= 0. */
hint = (sword32)((1 - (2 * (((word32)(r0 - 1)) >> 31))) & (0 - hint));
/* Make w1 only the top part plus the hint. */
w1[j] = r1 + hint;
@@ -5525,8 +5528,9 @@ static void mldsa_use_hint_32(sword32* w1, const byte* h, byte omega,
mldsa_decompose_q32(r, &r0, &r1);
/* Check for hint. */
if ((o < h[omega + i]) && (h[o] == (byte)j)) {
/* Add or subtract hint based on sign of r0. */
r1 += (sword32)(1 - (2 * (((word32)r0) >> 31)));
/* Add or subtract hint: r0 > 0 adds, r0 <= 0 subtracts
* (FIPS 204). (r0 - 1) >> 31 is 1 when r0 <= 0. */
r1 += (sword32)(1 - (2 * (((word32)(r0 - 1)) >> 31)));
/* Go to next hint offset. */
o++;
}
wolfcrypt/src/wc_mldsa_asm.S
+160
View File
@@ -27987,6 +27987,8 @@ L_mldsa_use_hint_88_avx2_hints_done_0_0:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -28071,6 +28073,8 @@ L_mldsa_use_hint_88_avx2_hints_done_0_1:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -28155,6 +28159,8 @@ L_mldsa_use_hint_88_avx2_hints_done_0_2:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -28239,6 +28245,8 @@ L_mldsa_use_hint_88_avx2_hints_done_0_3:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -28323,6 +28331,8 @@ L_mldsa_use_hint_88_avx2_hints_done_0_4:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -28407,6 +28417,8 @@ L_mldsa_use_hint_88_avx2_hints_done_0_5:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -28491,6 +28503,8 @@ L_mldsa_use_hint_88_avx2_hints_done_0_6:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -28575,6 +28589,8 @@ L_mldsa_use_hint_88_avx2_hints_done_0_7:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -28659,6 +28675,8 @@ L_mldsa_use_hint_88_avx2_hints_done_0_8:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -28743,6 +28761,8 @@ L_mldsa_use_hint_88_avx2_hints_done_0_9:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -28827,6 +28847,8 @@ L_mldsa_use_hint_88_avx2_hints_done_0_10:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -28911,6 +28933,8 @@ L_mldsa_use_hint_88_avx2_hints_done_0_11:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -28995,6 +29019,8 @@ L_mldsa_use_hint_88_avx2_hints_done_0_12:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -29079,6 +29105,8 @@ L_mldsa_use_hint_88_avx2_hints_done_0_13:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -29163,6 +29191,8 @@ L_mldsa_use_hint_88_avx2_hints_done_0_14:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -29247,6 +29277,8 @@ L_mldsa_use_hint_88_avx2_hints_done_0_15:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -29332,6 +29364,8 @@ L_mldsa_use_hint_88_avx2_hints_done_1_0:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -29416,6 +29450,8 @@ L_mldsa_use_hint_88_avx2_hints_done_1_1:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -29500,6 +29536,8 @@ L_mldsa_use_hint_88_avx2_hints_done_1_2:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -29584,6 +29622,8 @@ L_mldsa_use_hint_88_avx2_hints_done_1_3:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -29668,6 +29708,8 @@ L_mldsa_use_hint_88_avx2_hints_done_1_4:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -29752,6 +29794,8 @@ L_mldsa_use_hint_88_avx2_hints_done_1_5:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -29836,6 +29880,8 @@ L_mldsa_use_hint_88_avx2_hints_done_1_6:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -29920,6 +29966,8 @@ L_mldsa_use_hint_88_avx2_hints_done_1_7:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -30004,6 +30052,8 @@ L_mldsa_use_hint_88_avx2_hints_done_1_8:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -30088,6 +30138,8 @@ L_mldsa_use_hint_88_avx2_hints_done_1_9:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -30172,6 +30224,8 @@ L_mldsa_use_hint_88_avx2_hints_done_1_10:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -30256,6 +30310,8 @@ L_mldsa_use_hint_88_avx2_hints_done_1_11:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -30340,6 +30396,8 @@ L_mldsa_use_hint_88_avx2_hints_done_1_12:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -30424,6 +30482,8 @@ L_mldsa_use_hint_88_avx2_hints_done_1_13:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -30508,6 +30568,8 @@ L_mldsa_use_hint_88_avx2_hints_done_1_14:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -30592,6 +30654,8 @@ L_mldsa_use_hint_88_avx2_hints_done_1_15:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -30677,6 +30741,8 @@ L_mldsa_use_hint_88_avx2_hints_done_2_0:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -30761,6 +30827,8 @@ L_mldsa_use_hint_88_avx2_hints_done_2_1:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -30845,6 +30913,8 @@ L_mldsa_use_hint_88_avx2_hints_done_2_2:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -30929,6 +30999,8 @@ L_mldsa_use_hint_88_avx2_hints_done_2_3:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -31013,6 +31085,8 @@ L_mldsa_use_hint_88_avx2_hints_done_2_4:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -31097,6 +31171,8 @@ L_mldsa_use_hint_88_avx2_hints_done_2_5:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -31181,6 +31257,8 @@ L_mldsa_use_hint_88_avx2_hints_done_2_6:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -31265,6 +31343,8 @@ L_mldsa_use_hint_88_avx2_hints_done_2_7:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -31349,6 +31429,8 @@ L_mldsa_use_hint_88_avx2_hints_done_2_8:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -31433,6 +31515,8 @@ L_mldsa_use_hint_88_avx2_hints_done_2_9:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -31517,6 +31601,8 @@ L_mldsa_use_hint_88_avx2_hints_done_2_10:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -31601,6 +31687,8 @@ L_mldsa_use_hint_88_avx2_hints_done_2_11:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -31685,6 +31773,8 @@ L_mldsa_use_hint_88_avx2_hints_done_2_12:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -31769,6 +31859,8 @@ L_mldsa_use_hint_88_avx2_hints_done_2_13:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -31853,6 +31945,8 @@ L_mldsa_use_hint_88_avx2_hints_done_2_14:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -31937,6 +32031,8 @@ L_mldsa_use_hint_88_avx2_hints_done_2_15:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -32022,6 +32118,8 @@ L_mldsa_use_hint_88_avx2_hints_done_3_0:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -32106,6 +32204,8 @@ L_mldsa_use_hint_88_avx2_hints_done_3_1:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -32190,6 +32290,8 @@ L_mldsa_use_hint_88_avx2_hints_done_3_2:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -32274,6 +32376,8 @@ L_mldsa_use_hint_88_avx2_hints_done_3_3:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -32358,6 +32462,8 @@ L_mldsa_use_hint_88_avx2_hints_done_3_4:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -32442,6 +32548,8 @@ L_mldsa_use_hint_88_avx2_hints_done_3_5:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -32526,6 +32634,8 @@ L_mldsa_use_hint_88_avx2_hints_done_3_6:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -32610,6 +32720,8 @@ L_mldsa_use_hint_88_avx2_hints_done_3_7:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -32694,6 +32806,8 @@ L_mldsa_use_hint_88_avx2_hints_done_3_8:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -32778,6 +32892,8 @@ L_mldsa_use_hint_88_avx2_hints_done_3_9:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -32862,6 +32978,8 @@ L_mldsa_use_hint_88_avx2_hints_done_3_10:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -32946,6 +33064,8 @@ L_mldsa_use_hint_88_avx2_hints_done_3_11:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -33030,6 +33150,8 @@ L_mldsa_use_hint_88_avx2_hints_done_3_12:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -33114,6 +33236,8 @@ L_mldsa_use_hint_88_avx2_hints_done_3_13:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -33198,6 +33322,8 @@ L_mldsa_use_hint_88_avx2_hints_done_3_14:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -33282,6 +33408,8 @@ L_mldsa_use_hint_88_avx2_hints_done_3_15:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -33486,6 +33614,8 @@ L_mldsa_use_hint_32_avx2_hints_done__0:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -33558,6 +33688,8 @@ L_mldsa_use_hint_32_avx2_hints_done__1:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -33630,6 +33762,8 @@ L_mldsa_use_hint_32_avx2_hints_done__2:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -33702,6 +33836,8 @@ L_mldsa_use_hint_32_avx2_hints_done__3:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -33774,6 +33910,8 @@ L_mldsa_use_hint_32_avx2_hints_done__4:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -33846,6 +33984,8 @@ L_mldsa_use_hint_32_avx2_hints_done__5:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -33918,6 +34058,8 @@ L_mldsa_use_hint_32_avx2_hints_done__6:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -33990,6 +34132,8 @@ L_mldsa_use_hint_32_avx2_hints_done__7:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -34062,6 +34206,8 @@ L_mldsa_use_hint_32_avx2_hints_done__8:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -34134,6 +34280,8 @@ L_mldsa_use_hint_32_avx2_hints_done__9:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -34206,6 +34354,8 @@ L_mldsa_use_hint_32_avx2_hints_done__10:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -34278,6 +34428,8 @@ L_mldsa_use_hint_32_avx2_hints_done__11:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -34350,6 +34502,8 @@ L_mldsa_use_hint_32_avx2_hints_done__12:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -34422,6 +34576,8 @@ L_mldsa_use_hint_32_avx2_hints_done__13:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -34494,6 +34650,8 @@ L_mldsa_use_hint_32_avx2_hints_done__14:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2
@@ -34566,6 +34724,8 @@ L_mldsa_use_hint_32_avx2_hints_done__15:
vpsllvd %ymm13, %ymm7, %ymm7
vpsrad $31, %ymm6, %ymm6
vpsrad $31, %ymm7, %ymm7
vpsubd %ymm14, %ymm2, %ymm2
vpsubd %ymm14, %ymm3, %ymm3
vpsrld $31, %ymm2, %ymm2
vpsrld $31, %ymm3, %ymm3
vpslld $0x01, %ymm2, %ymm2