diff --git a/wolfcrypt/src/port/arm/armv8-mlkem-asm.S b/wolfcrypt/src/port/arm/armv8-mlkem-asm.S index 31eda1e0ca..14010de903 100644 --- a/wolfcrypt/src/port/arm/armv8-mlkem-asm.S +++ b/wolfcrypt/src/port/arm/armv8-mlkem-asm.S @@ -9767,17 +9767,17 @@ L_sha3_aarch64_r: #ifdef WOLFSSL_ARMASM_CRYPTO_SHA3 #ifndef __APPLE__ .text -.globl mlkem_sha3_blocksx3_neon -WC_ASM_ATT_HIDDEN(mlkem_sha3_blocksx3_neon) -.type mlkem_sha3_blocksx3_neon,@function +.globl mlkem_sha3_blocksx3_crypto +WC_ASM_ATT_HIDDEN(mlkem_sha3_blocksx3_crypto) +.type mlkem_sha3_blocksx3_crypto,@function .align 2 -mlkem_sha3_blocksx3_neon: +mlkem_sha3_blocksx3_crypto: #else .section __TEXT,__text -.globl _mlkem_sha3_blocksx3_neon -WC_ASM_ATT_HIDDEN(_mlkem_sha3_blocksx3_neon) +.globl _mlkem_sha3_blocksx3_crypto +WC_ASM_ATT_HIDDEN(_mlkem_sha3_blocksx3_crypto) .p2align 2 -_mlkem_sha3_blocksx3_neon: +_mlkem_sha3_blocksx3_crypto: #endif /* __APPLE__ */ stp x29, x30, [sp, #-224]! add x29, sp, #0 @@ -9830,7 +9830,7 @@ _mlkem_sha3_blocksx3_neon: ldr x26, [x0, #192] mov x28, #24 # Start of 24 rounds -L_SHA3_transform_blocksx3_neon_begin: +L_SHA3_transform_blocksx3_crypto_begin: stp x27, x28, [x29, #48] # Col Mix eor3 v31.16b, v0.16b, v5.16b, v10.16b @@ -10046,7 +10046,7 @@ L_SHA3_transform_blocksx3_neon_begin: mov v30.d[1], x0 eor x1, x1, x0 eor v0.16b, v0.16b, v30.16b - bne L_SHA3_transform_blocksx3_neon_begin + bne L_SHA3_transform_blocksx3_crypto_begin ldr x0, [x29, #40] st4 {v0.d, v1.d, v2.d, v3.d}[0], [x0], #32 st4 {v4.d, v5.d, v6.d, v7.d}[0], [x0], #32 @@ -10090,21 +10090,21 @@ L_SHA3_transform_blocksx3_neon_begin: ldp x29, x30, [sp], #0xe0 ret #ifndef __APPLE__ - .size mlkem_sha3_blocksx3_neon,.-mlkem_sha3_blocksx3_neon + .size mlkem_sha3_blocksx3_crypto,.-mlkem_sha3_blocksx3_crypto #endif /* __APPLE__ */ #ifndef __APPLE__ .text -.globl mlkem_shake128_blocksx3_seed_neon -WC_ASM_ATT_HIDDEN(mlkem_shake128_blocksx3_seed_neon) -.type mlkem_shake128_blocksx3_seed_neon,@function +.globl mlkem_shake128_blocksx3_seed_crypto +WC_ASM_ATT_HIDDEN(mlkem_shake128_blocksx3_seed_crypto) +.type mlkem_shake128_blocksx3_seed_crypto,@function .align 2 -mlkem_shake128_blocksx3_seed_neon: +mlkem_shake128_blocksx3_seed_crypto: #else .section __TEXT,__text -.globl _mlkem_shake128_blocksx3_seed_neon -WC_ASM_ATT_HIDDEN(_mlkem_shake128_blocksx3_seed_neon) +.globl _mlkem_shake128_blocksx3_seed_crypto +WC_ASM_ATT_HIDDEN(_mlkem_shake128_blocksx3_seed_crypto) .p2align 2 -_mlkem_shake128_blocksx3_seed_neon: +_mlkem_shake128_blocksx3_seed_crypto: #endif /* __APPLE__ */ stp x29, x30, [sp, #-224]! add x29, sp, #0 @@ -10179,7 +10179,7 @@ _mlkem_shake128_blocksx3_seed_neon: dup v20.2d, x23 mov x1, #24 # Start of 24 rounds -L_SHA3_shake128_blocksx3_seed_neon_begin: +L_SHA3_shake128_blocksx3_seed_crypto_begin: stp x28, x1, [x29, #48] # Col Mix eor3 v31.16b, v0.16b, v5.16b, v10.16b @@ -10395,7 +10395,7 @@ L_SHA3_shake128_blocksx3_seed_neon_begin: mov v30.d[1], x0 eor x2, x2, x0 eor v0.16b, v0.16b, v30.16b - bne L_SHA3_shake128_blocksx3_seed_neon_begin + bne L_SHA3_shake128_blocksx3_seed_crypto_begin ldr x0, [x29, #40] st4 {v0.d, v1.d, v2.d, v3.d}[0], [x0], #32 st4 {v4.d, v5.d, v6.d, v7.d}[0], [x0], #32 @@ -10439,21 +10439,21 @@ L_SHA3_shake128_blocksx3_seed_neon_begin: ldp x29, x30, [sp], #0xe0 ret #ifndef __APPLE__ - .size mlkem_shake128_blocksx3_seed_neon,.-mlkem_shake128_blocksx3_seed_neon + .size mlkem_shake128_blocksx3_seed_crypto,.-mlkem_shake128_blocksx3_seed_crypto #endif /* __APPLE__ */ #ifndef __APPLE__ .text -.globl mlkem_shake256_blocksx3_seed_neon -WC_ASM_ATT_HIDDEN(mlkem_shake256_blocksx3_seed_neon) -.type mlkem_shake256_blocksx3_seed_neon,@function +.globl mlkem_shake256_blocksx3_seed_crypto +WC_ASM_ATT_HIDDEN(mlkem_shake256_blocksx3_seed_crypto) +.type mlkem_shake256_blocksx3_seed_crypto,@function .align 2 -mlkem_shake256_blocksx3_seed_neon: +mlkem_shake256_blocksx3_seed_crypto: #else .section __TEXT,__text -.globl _mlkem_shake256_blocksx3_seed_neon -WC_ASM_ATT_HIDDEN(_mlkem_shake256_blocksx3_seed_neon) +.globl _mlkem_shake256_blocksx3_seed_crypto +WC_ASM_ATT_HIDDEN(_mlkem_shake256_blocksx3_seed_crypto) .p2align 2 -_mlkem_shake256_blocksx3_seed_neon: +_mlkem_shake256_blocksx3_seed_crypto: #endif /* __APPLE__ */ stp x29, x30, [sp, #-224]! add x29, sp, #0 @@ -10528,7 +10528,7 @@ _mlkem_shake256_blocksx3_seed_neon: dup v16.2d, x19 mov x1, #24 # Start of 24 rounds -L_SHA3_shake256_blocksx3_seed_neon_begin: +L_SHA3_shake256_blocksx3_seed_crypto_begin: stp x28, x1, [x29, #48] # Col Mix eor3 v31.16b, v0.16b, v5.16b, v10.16b @@ -10744,7 +10744,7 @@ L_SHA3_shake256_blocksx3_seed_neon_begin: mov v30.d[1], x0 eor x2, x2, x0 eor v0.16b, v0.16b, v30.16b - bne L_SHA3_shake256_blocksx3_seed_neon_begin + bne L_SHA3_shake256_blocksx3_seed_crypto_begin ldr x0, [x29, #40] st4 {v0.d, v1.d, v2.d, v3.d}[0], [x0], #32 st4 {v4.d, v5.d, v6.d, v7.d}[0], [x0], #32 @@ -10788,1290 +10788,1289 @@ L_SHA3_shake256_blocksx3_seed_neon_begin: ldp x29, x30, [sp], #0xe0 ret #ifndef __APPLE__ - .size mlkem_shake256_blocksx3_seed_neon,.-mlkem_shake256_blocksx3_seed_neon -#endif /* __APPLE__ */ -#else -#ifndef __APPLE__ -.text -.globl mlkem_sha3_blocksx3_neon -WC_ASM_ATT_HIDDEN(mlkem_sha3_blocksx3_neon) -.type mlkem_sha3_blocksx3_neon,@function -.align 2 -mlkem_sha3_blocksx3_neon: -#else -.section __TEXT,__text -.globl _mlkem_sha3_blocksx3_neon -WC_ASM_ATT_HIDDEN(_mlkem_sha3_blocksx3_neon) -.p2align 2 -_mlkem_sha3_blocksx3_neon: -#endif /* __APPLE__ */ - stp x29, x30, [sp, #-224]! - add x29, sp, #0 - stp x17, x19, [x29, #72] - stp x20, x21, [x29, #88] - stp x22, x23, [x29, #104] - stp x24, x25, [x29, #120] - stp x26, x27, [x29, #136] - str x28, [x29, #152] - stp d8, d9, [x29, #160] - stp d10, d11, [x29, #176] - stp d12, d13, [x29, #192] - stp d14, d15, [x29, #208] -#ifndef __APPLE__ - adrp x27, L_sha3_aarch64_r - add x27, x27, :lo12:L_sha3_aarch64_r -#else - adrp x27, L_sha3_aarch64_r@PAGE - add x27, x27, L_sha3_aarch64_r@PAGEOFF -#endif /* __APPLE__ */ - str x0, [x29, #40] - ld4 {v0.d, v1.d, v2.d, v3.d}[0], [x0], #32 - ld4 {v4.d, v5.d, v6.d, v7.d}[0], [x0], #32 - ld4 {v8.d, v9.d, v10.d, v11.d}[0], [x0], #32 - ld4 {v12.d, v13.d, v14.d, v15.d}[0], [x0], #32 - ld4 {v16.d, v17.d, v18.d, v19.d}[0], [x0], #32 - ld4 {v20.d, v21.d, v22.d, v23.d}[0], [x0], #32 - ld1 {v24.d}[0], [x0] - add x0, x0, #8 - ld4 {v0.d, v1.d, v2.d, v3.d}[1], [x0], #32 - ld4 {v4.d, v5.d, v6.d, v7.d}[1], [x0], #32 - ld4 {v8.d, v9.d, v10.d, v11.d}[1], [x0], #32 - ld4 {v12.d, v13.d, v14.d, v15.d}[1], [x0], #32 - ld4 {v16.d, v17.d, v18.d, v19.d}[1], [x0], #32 - ld4 {v20.d, v21.d, v22.d, v23.d}[1], [x0], #32 - ld1 {v24.d}[1], [x0] - add x0, x0, #8 - ldp x1, x2, [x0] - ldp x3, x4, [x0, #16] - ldp x5, x6, [x0, #32] - ldp x7, x8, [x0, #48] - ldp x9, x10, [x0, #64] - ldp x11, x12, [x0, #80] - ldp x13, x14, [x0, #96] - ldp x15, x16, [x0, #112] - ldp x17, x19, [x0, #128] - ldp x20, x21, [x0, #144] - ldp x22, x23, [x0, #160] - ldp x24, x25, [x0, #176] - ldr x26, [x0, #192] - mov x28, #24 - # Start of 24 rounds -L_SHA3_transform_blocksx3_neon_begin: - stp x27, x28, [x29, #48] - # Col Mix NEON - eor v30.16b, v4.16b, v9.16b - eor x0, x5, x10 - eor v27.16b, v1.16b, v6.16b - eor x30, x1, x6 - eor v30.16b, v30.16b, v14.16b - eor x28, x3, x8 - eor v27.16b, v27.16b, v11.16b - eor x0, x0, x15 - eor v30.16b, v30.16b, v19.16b - eor x30, x30, x11 - eor v27.16b, v27.16b, v16.16b - eor x28, x28, x13 - eor v30.16b, v30.16b, v24.16b - eor x0, x0, x21 - eor v27.16b, v27.16b, v21.16b - eor x30, x30, x16 - ushr v25.2d, v27.2d, #63 - eor x28, x28, x19 - sli v25.2d, v27.2d, #1 - eor x0, x0, x26 - eor v25.16b, v25.16b, v30.16b - eor x30, x30, x22 - eor v31.16b, v0.16b, v5.16b - eor x28, x28, x24 - eor v28.16b, v2.16b, v7.16b - str x0, [x29, #32] - eor v31.16b, v31.16b, v10.16b - str x28, [x29, #24] - eor v28.16b, v28.16b, v12.16b - eor x27, x2, x7 - eor v31.16b, v31.16b, v15.16b - eor x28, x4, x9 - eor v28.16b, v28.16b, v17.16b - eor x27, x27, x12 - eor v31.16b, v31.16b, v20.16b - eor x28, x28, x14 - eor v28.16b, v28.16b, v22.16b - eor x27, x27, x17 - ushr v29.2d, v30.2d, #63 - eor x28, x28, x20 - ushr v26.2d, v28.2d, #63 - eor x27, x27, x23 - sli v29.2d, v30.2d, #1 - eor x28, x28, x25 - sli v26.2d, v28.2d, #1 - eor x0, x0, x27, ror 63 - eor v28.16b, v28.16b, v29.16b - eor x27, x27, x28, ror 63 - eor v29.16b, v3.16b, v8.16b - eor x1, x1, x0 - eor v26.16b, v26.16b, v31.16b - eor x6, x6, x0 - eor v29.16b, v29.16b, v13.16b - eor x11, x11, x0 - eor v29.16b, v29.16b, v18.16b - eor x16, x16, x0 - eor v29.16b, v29.16b, v23.16b - eor x22, x22, x0 - ushr v30.2d, v29.2d, #63 - eor x3, x3, x27 - sli v30.2d, v29.2d, #1 - eor x8, x8, x27 - eor v27.16b, v27.16b, v30.16b - eor x13, x13, x27 - ushr v30.2d, v31.2d, #63 - eor x19, x19, x27 - sli v30.2d, v31.2d, #1 - eor x24, x24, x27 - eor v29.16b, v29.16b, v30.16b - ldr x0, [x29, #32] - # Swap Rotate NEON - eor v0.16b, v0.16b, v25.16b - eor v31.16b, v1.16b, v26.16b - ldr x27, [x29, #24] - eor v6.16b, v6.16b, v26.16b - eor x28, x28, x30, ror 63 - ushr v30.2d, v31.2d, #63 - eor x30, x30, x27, ror 63 - ushr v1.2d, v6.2d, #20 - eor x27, x27, x0, ror 63 - sli v30.2d, v31.2d, #1 - eor x5, x5, x28 - sli v1.2d, v6.2d, #44 - eor x10, x10, x28 - eor v31.16b, v9.16b, v29.16b - eor x15, x15, x28 - eor v22.16b, v22.16b, v27.16b - eor x21, x21, x28 - ushr v6.2d, v31.2d, #44 - eor x26, x26, x28 - ushr v9.2d, v22.2d, #3 - eor x2, x2, x30 - sli v6.2d, v31.2d, #20 - eor x7, x7, x30 - sli v9.2d, v22.2d, #61 - eor x12, x12, x30 - eor v31.16b, v14.16b, v29.16b - eor x17, x17, x30 - eor v20.16b, v20.16b, v25.16b - eor x23, x23, x30 - ushr v22.2d, v31.2d, #25 - eor x4, x4, x27 - ushr v14.2d, v20.2d, #46 - eor x9, x9, x27 - sli v22.2d, v31.2d, #39 - eor x14, x14, x27 - sli v14.2d, v20.2d, #18 - eor x20, x20, x27 - eor v31.16b, v2.16b, v27.16b - eor x25, x25, x27 - # Swap Rotate Base - eor v12.16b, v12.16b, v27.16b - ror x0, x2, #63 - ushr v20.2d, v31.2d, #2 - ror x2, x7, #20 - ushr v2.2d, v12.2d, #21 - ror x7, x10, #44 - sli v20.2d, v31.2d, #62 - ror x10, x24, #3 - sli v2.2d, v12.2d, #43 - ror x24, x15, #25 - eor v31.16b, v13.16b, v28.16b - ror x15, x22, #46 - eor v19.16b, v19.16b, v29.16b - ror x22, x3, #2 - ushr v12.2d, v31.2d, #39 - ror x3, x13, #21 - ushr v13.2d, v19.2d, #56 - ror x13, x14, #39 - sli v12.2d, v31.2d, #25 - ror x14, x21, #56 - sli v13.2d, v19.2d, #8 - ror x21, x25, #8 - eor v31.16b, v23.16b, v28.16b - ror x25, x16, #23 - eor v15.16b, v15.16b, v25.16b - ror x16, x5, #37 - ushr v19.2d, v31.2d, #8 - ror x5, x26, #50 - ushr v23.2d, v15.2d, #23 - ror x26, x23, #62 - sli v19.2d, v31.2d, #56 - ror x23, x9, #9 - sli v23.2d, v15.2d, #41 - ror x9, x17, #19 - eor v31.16b, v4.16b, v29.16b - ror x17, x6, #28 - eor v24.16b, v24.16b, v29.16b - ror x6, x4, #36 - ushr v15.2d, v31.2d, #37 - ror x4, x20, #43 - ushr v4.2d, v24.2d, #50 - ror x20, x19, #49 - sli v15.2d, v31.2d, #27 - ror x19, x12, #54 - sli v4.2d, v24.2d, #14 - ror x12, x8, #58 - eor v31.16b, v21.16b, v26.16b - ror x8, x11, #61 - # Row Mix Base - eor v8.16b, v8.16b, v28.16b - bic x11, x3, x2 - ushr v24.2d, v31.2d, #62 - bic x27, x4, x3 - ushr v21.2d, v8.2d, #9 - bic x28, x1, x5 - sli v24.2d, v31.2d, #2 - bic x30, x2, x1 - sli v21.2d, v8.2d, #55 - eor x1, x1, x11 - eor v31.16b, v16.16b, v26.16b - eor x2, x2, x27 - eor v5.16b, v5.16b, v25.16b - bic x11, x5, x4 - ushr v8.2d, v31.2d, #19 - eor x4, x4, x28 - ushr v16.2d, v5.2d, #28 - eor x3, x3, x11 - sli v8.2d, v31.2d, #45 - eor x5, x5, x30 - sli v16.2d, v5.2d, #36 - bic x11, x8, x7 - eor v31.16b, v3.16b, v28.16b - bic x27, x9, x8 - eor v18.16b, v18.16b, v28.16b - bic x28, x6, x10 - ushr v5.2d, v31.2d, #36 - bic x30, x7, x6 - ushr v3.2d, v18.2d, #43 - eor x6, x6, x11 - sli v5.2d, v31.2d, #28 - eor x7, x7, x27 - sli v3.2d, v18.2d, #21 - bic x11, x10, x9 - eor v31.16b, v17.16b, v27.16b - eor x9, x9, x28 - eor v11.16b, v11.16b, v26.16b - eor x8, x8, x11 - ushr v18.2d, v31.2d, #49 - eor x10, x10, x30 - ushr v17.2d, v11.2d, #54 - bic x11, x13, x12 - sli v18.2d, v31.2d, #15 - bic x27, x14, x13 - sli v17.2d, v11.2d, #10 - bic x28, x0, x15 - eor v31.16b, v7.16b, v27.16b - bic x30, x12, x0 - eor v10.16b, v10.16b, v25.16b - eor x11, x0, x11 - ushr v11.2d, v31.2d, #58 - eor x12, x12, x27 - ushr v7.2d, v10.2d, #61 - bic x0, x15, x14 - sli v11.2d, v31.2d, #6 - eor x14, x14, x28 - sli v7.2d, v10.2d, #3 - eor x13, x13, x0 - # Row Mix NEON - bic v25.16b, v2.16b, v1.16b - eor x15, x15, x30 - bic v26.16b, v3.16b, v2.16b - bic x0, x19, x17 - bic v27.16b, v4.16b, v3.16b - bic x27, x20, x19 - bic v28.16b, v0.16b, v4.16b - bic x28, x16, x21 - bic v29.16b, v1.16b, v0.16b - bic x30, x17, x16 - eor v0.16b, v0.16b, v25.16b - eor x16, x16, x0 - eor v1.16b, v1.16b, v26.16b - eor x17, x17, x27 - eor v2.16b, v2.16b, v27.16b - bic x0, x21, x20 - eor v3.16b, v3.16b, v28.16b - eor x20, x20, x28 - eor v4.16b, v4.16b, v29.16b - eor x19, x19, x0 - bic v25.16b, v7.16b, v6.16b - eor x21, x21, x30 - bic v26.16b, v8.16b, v7.16b - bic x0, x24, x23 - bic v27.16b, v9.16b, v8.16b - bic x27, x25, x24 - bic v28.16b, v5.16b, v9.16b - bic x28, x22, x26 - bic v29.16b, v6.16b, v5.16b - bic x30, x23, x22 - eor v5.16b, v5.16b, v25.16b - eor x22, x22, x0 - eor v6.16b, v6.16b, v26.16b - eor x23, x23, x27 - eor v7.16b, v7.16b, v27.16b - bic x0, x26, x25 - eor v8.16b, v8.16b, v28.16b - eor x25, x25, x28 - eor v9.16b, v9.16b, v29.16b - eor x24, x24, x0 - bic v25.16b, v12.16b, v11.16b - eor x26, x26, x30 - bic v26.16b, v13.16b, v12.16b - bic v27.16b, v14.16b, v13.16b - bic v28.16b, v30.16b, v14.16b - bic v29.16b, v11.16b, v30.16b - eor v10.16b, v30.16b, v25.16b - eor v11.16b, v11.16b, v26.16b - eor v12.16b, v12.16b, v27.16b - eor v13.16b, v13.16b, v28.16b - eor v14.16b, v14.16b, v29.16b - bic v25.16b, v17.16b, v16.16b - bic v26.16b, v18.16b, v17.16b - bic v27.16b, v19.16b, v18.16b - bic v28.16b, v15.16b, v19.16b - bic v29.16b, v16.16b, v15.16b - eor v15.16b, v15.16b, v25.16b - eor v16.16b, v16.16b, v26.16b - eor v17.16b, v17.16b, v27.16b - eor v18.16b, v18.16b, v28.16b - eor v19.16b, v19.16b, v29.16b - bic v25.16b, v22.16b, v21.16b - bic v26.16b, v23.16b, v22.16b - bic v27.16b, v24.16b, v23.16b - bic v28.16b, v20.16b, v24.16b - bic v29.16b, v21.16b, v20.16b - eor v20.16b, v20.16b, v25.16b - eor v21.16b, v21.16b, v26.16b - eor v22.16b, v22.16b, v27.16b - eor v23.16b, v23.16b, v28.16b - eor v24.16b, v24.16b, v29.16b - # Done transforming - ldp x27, x28, [x29, #48] - ldr x0, [x27], #8 - subs x28, x28, #1 - mov v30.d[0], x0 - mov v30.d[1], x0 - eor x1, x1, x0 - eor v0.16b, v0.16b, v30.16b - bne L_SHA3_transform_blocksx3_neon_begin - ldr x0, [x29, #40] - st4 {v0.d, v1.d, v2.d, v3.d}[0], [x0], #32 - st4 {v4.d, v5.d, v6.d, v7.d}[0], [x0], #32 - st4 {v8.d, v9.d, v10.d, v11.d}[0], [x0], #32 - st4 {v12.d, v13.d, v14.d, v15.d}[0], [x0], #32 - st4 {v16.d, v17.d, v18.d, v19.d}[0], [x0], #32 - st4 {v20.d, v21.d, v22.d, v23.d}[0], [x0], #32 - st1 {v24.d}[0], [x0] - add x0, x0, #8 - st4 {v0.d, v1.d, v2.d, v3.d}[1], [x0], #32 - st4 {v4.d, v5.d, v6.d, v7.d}[1], [x0], #32 - st4 {v8.d, v9.d, v10.d, v11.d}[1], [x0], #32 - st4 {v12.d, v13.d, v14.d, v15.d}[1], [x0], #32 - st4 {v16.d, v17.d, v18.d, v19.d}[1], [x0], #32 - st4 {v20.d, v21.d, v22.d, v23.d}[1], [x0], #32 - st1 {v24.d}[1], [x0] - add x0, x0, #8 - stp x1, x2, [x0] - stp x3, x4, [x0, #16] - stp x5, x6, [x0, #32] - stp x7, x8, [x0, #48] - stp x9, x10, [x0, #64] - stp x11, x12, [x0, #80] - stp x13, x14, [x0, #96] - stp x15, x16, [x0, #112] - stp x17, x19, [x0, #128] - stp x20, x21, [x0, #144] - stp x22, x23, [x0, #160] - stp x24, x25, [x0, #176] - str x26, [x0, #192] - ldp x17, x19, [x29, #72] - ldp x20, x21, [x29, #88] - ldp x22, x23, [x29, #104] - ldp x24, x25, [x29, #120] - ldp x26, x27, [x29, #136] - ldr x28, [x29, #152] - ldp d8, d9, [x29, #160] - ldp d10, d11, [x29, #176] - ldp d12, d13, [x29, #192] - ldp d14, d15, [x29, #208] - ldp x29, x30, [sp], #0xe0 - ret -#ifndef __APPLE__ - .size mlkem_sha3_blocksx3_neon,.-mlkem_sha3_blocksx3_neon -#endif /* __APPLE__ */ -#ifndef __APPLE__ -.text -.globl mlkem_shake128_blocksx3_seed_neon -WC_ASM_ATT_HIDDEN(mlkem_shake128_blocksx3_seed_neon) -.type mlkem_shake128_blocksx3_seed_neon,@function -.align 2 -mlkem_shake128_blocksx3_seed_neon: -#else -.section __TEXT,__text -.globl _mlkem_shake128_blocksx3_seed_neon -WC_ASM_ATT_HIDDEN(_mlkem_shake128_blocksx3_seed_neon) -.p2align 2 -_mlkem_shake128_blocksx3_seed_neon: -#endif /* __APPLE__ */ - stp x29, x30, [sp, #-224]! - add x29, sp, #0 - stp x17, x19, [x29, #72] - stp x20, x21, [x29, #88] - stp x22, x23, [x29, #104] - stp x24, x25, [x29, #120] - stp x26, x27, [x29, #136] - str x28, [x29, #152] - stp d8, d9, [x29, #160] - stp d10, d11, [x29, #176] - stp d12, d13, [x29, #192] - stp d14, d15, [x29, #208] -#ifndef __APPLE__ - adrp x28, L_sha3_aarch64_r - add x28, x28, :lo12:L_sha3_aarch64_r -#else - adrp x28, L_sha3_aarch64_r@PAGE - add x28, x28, L_sha3_aarch64_r@PAGEOFF -#endif /* __APPLE__ */ - str x0, [x29, #40] - add x0, x0, #32 - ld1 {v4.d}[0], [x0] - ldp x2, x3, [x1], #16 - add x0, x0, #0xc8 - ld1 {v4.d}[1], [x0] - ldp x4, x5, [x1], #16 - ldr x6, [x0, #200] - eor v5.16b, v5.16b, v5.16b - eor x7, x7, x7 - eor v6.16b, v6.16b, v6.16b - eor x8, x8, x8 - eor v7.16b, v7.16b, v7.16b - eor x9, x9, x9 - eor v8.16b, v8.16b, v8.16b - eor x10, x10, x10 - eor v9.16b, v9.16b, v9.16b - eor x11, x11, x11 - eor v10.16b, v10.16b, v10.16b - eor x12, x12, x12 - eor v11.16b, v11.16b, v11.16b - eor x13, x13, x13 - eor v12.16b, v12.16b, v12.16b - eor x14, x14, x14 - eor v13.16b, v13.16b, v13.16b - eor x15, x15, x15 - eor v14.16b, v14.16b, v14.16b - eor x16, x16, x16 - eor v15.16b, v15.16b, v15.16b - eor x17, x17, x17 - eor v16.16b, v16.16b, v16.16b - eor x19, x19, x19 - eor v17.16b, v17.16b, v17.16b - eor x20, x20, x20 - eor v18.16b, v18.16b, v18.16b - eor x21, x21, x21 - eor v19.16b, v19.16b, v19.16b - eor x22, x22, x22 - movz x23, #0x8000, lsl 48 - eor v21.16b, v21.16b, v21.16b - eor x24, x24, x24 - eor v22.16b, v22.16b, v22.16b - eor x25, x25, x25 - eor v23.16b, v23.16b, v23.16b - eor x26, x26, x26 - eor v24.16b, v24.16b, v24.16b - eor x27, x27, x27 - dup v0.2d, x2 - dup v1.2d, x3 - dup v2.2d, x4 - dup v3.2d, x5 - dup v20.2d, x23 - mov x1, #24 - # Start of 24 rounds -L_SHA3_shake128_blocksx3_seed_neon_begin: - stp x28, x1, [x29, #48] - # Col Mix NEON - eor v30.16b, v4.16b, v9.16b - eor x0, x6, x11 - eor v27.16b, v1.16b, v6.16b - eor x30, x2, x7 - eor v30.16b, v30.16b, v14.16b - eor x28, x4, x9 - eor v27.16b, v27.16b, v11.16b - eor x0, x0, x16 - eor v30.16b, v30.16b, v19.16b - eor x30, x30, x12 - eor v27.16b, v27.16b, v16.16b - eor x28, x28, x14 - eor v30.16b, v30.16b, v24.16b - eor x0, x0, x22 - eor v27.16b, v27.16b, v21.16b - eor x30, x30, x17 - ushr v25.2d, v27.2d, #63 - eor x28, x28, x20 - sli v25.2d, v27.2d, #1 - eor x0, x0, x27 - eor v25.16b, v25.16b, v30.16b - eor x30, x30, x23 - eor v31.16b, v0.16b, v5.16b - eor x28, x28, x25 - eor v28.16b, v2.16b, v7.16b - str x0, [x29, #32] - eor v31.16b, v31.16b, v10.16b - str x28, [x29, #24] - eor v28.16b, v28.16b, v12.16b - eor x1, x3, x8 - eor v31.16b, v31.16b, v15.16b - eor x28, x5, x10 - eor v28.16b, v28.16b, v17.16b - eor x1, x1, x13 - eor v31.16b, v31.16b, v20.16b - eor x28, x28, x15 - eor v28.16b, v28.16b, v22.16b - eor x1, x1, x19 - ushr v29.2d, v30.2d, #63 - eor x28, x28, x21 - ushr v26.2d, v28.2d, #63 - eor x1, x1, x24 - sli v29.2d, v30.2d, #1 - eor x28, x28, x26 - sli v26.2d, v28.2d, #1 - eor x0, x0, x1, ror 63 - eor v28.16b, v28.16b, v29.16b - eor x1, x1, x28, ror 63 - eor v29.16b, v3.16b, v8.16b - eor x2, x2, x0 - eor v26.16b, v26.16b, v31.16b - eor x7, x7, x0 - eor v29.16b, v29.16b, v13.16b - eor x12, x12, x0 - eor v29.16b, v29.16b, v18.16b - eor x17, x17, x0 - eor v29.16b, v29.16b, v23.16b - eor x23, x23, x0 - ushr v30.2d, v29.2d, #63 - eor x4, x4, x1 - sli v30.2d, v29.2d, #1 - eor x9, x9, x1 - eor v27.16b, v27.16b, v30.16b - eor x14, x14, x1 - ushr v30.2d, v31.2d, #63 - eor x20, x20, x1 - sli v30.2d, v31.2d, #1 - eor x25, x25, x1 - eor v29.16b, v29.16b, v30.16b - ldr x0, [x29, #32] - # Swap Rotate NEON - eor v0.16b, v0.16b, v25.16b - eor v31.16b, v1.16b, v26.16b - ldr x1, [x29, #24] - eor v6.16b, v6.16b, v26.16b - eor x28, x28, x30, ror 63 - ushr v30.2d, v31.2d, #63 - eor x30, x30, x1, ror 63 - ushr v1.2d, v6.2d, #20 - eor x1, x1, x0, ror 63 - sli v30.2d, v31.2d, #1 - eor x6, x6, x28 - sli v1.2d, v6.2d, #44 - eor x11, x11, x28 - eor v31.16b, v9.16b, v29.16b - eor x16, x16, x28 - eor v22.16b, v22.16b, v27.16b - eor x22, x22, x28 - ushr v6.2d, v31.2d, #44 - eor x27, x27, x28 - ushr v9.2d, v22.2d, #3 - eor x3, x3, x30 - sli v6.2d, v31.2d, #20 - eor x8, x8, x30 - sli v9.2d, v22.2d, #61 - eor x13, x13, x30 - eor v31.16b, v14.16b, v29.16b - eor x19, x19, x30 - eor v20.16b, v20.16b, v25.16b - eor x24, x24, x30 - ushr v22.2d, v31.2d, #25 - eor x5, x5, x1 - ushr v14.2d, v20.2d, #46 - eor x10, x10, x1 - sli v22.2d, v31.2d, #39 - eor x15, x15, x1 - sli v14.2d, v20.2d, #18 - eor x21, x21, x1 - eor v31.16b, v2.16b, v27.16b - eor x26, x26, x1 - # Swap Rotate Base - eor v12.16b, v12.16b, v27.16b - ror x0, x3, #63 - ushr v20.2d, v31.2d, #2 - ror x3, x8, #20 - ushr v2.2d, v12.2d, #21 - ror x8, x11, #44 - sli v20.2d, v31.2d, #62 - ror x11, x25, #3 - sli v2.2d, v12.2d, #43 - ror x25, x16, #25 - eor v31.16b, v13.16b, v28.16b - ror x16, x23, #46 - eor v19.16b, v19.16b, v29.16b - ror x23, x4, #2 - ushr v12.2d, v31.2d, #39 - ror x4, x14, #21 - ushr v13.2d, v19.2d, #56 - ror x14, x15, #39 - sli v12.2d, v31.2d, #25 - ror x15, x22, #56 - sli v13.2d, v19.2d, #8 - ror x22, x26, #8 - eor v31.16b, v23.16b, v28.16b - ror x26, x17, #23 - eor v15.16b, v15.16b, v25.16b - ror x17, x6, #37 - ushr v19.2d, v31.2d, #8 - ror x6, x27, #50 - ushr v23.2d, v15.2d, #23 - ror x27, x24, #62 - sli v19.2d, v31.2d, #56 - ror x24, x10, #9 - sli v23.2d, v15.2d, #41 - ror x10, x19, #19 - eor v31.16b, v4.16b, v29.16b - ror x19, x7, #28 - eor v24.16b, v24.16b, v29.16b - ror x7, x5, #36 - ushr v15.2d, v31.2d, #37 - ror x5, x21, #43 - ushr v4.2d, v24.2d, #50 - ror x21, x20, #49 - sli v15.2d, v31.2d, #27 - ror x20, x13, #54 - sli v4.2d, v24.2d, #14 - ror x13, x9, #58 - eor v31.16b, v21.16b, v26.16b - ror x9, x12, #61 - # Row Mix Base - eor v8.16b, v8.16b, v28.16b - bic x12, x4, x3 - ushr v24.2d, v31.2d, #62 - bic x1, x5, x4 - ushr v21.2d, v8.2d, #9 - bic x28, x2, x6 - sli v24.2d, v31.2d, #2 - bic x30, x3, x2 - sli v21.2d, v8.2d, #55 - eor x2, x2, x12 - eor v31.16b, v16.16b, v26.16b - eor x3, x3, x1 - eor v5.16b, v5.16b, v25.16b - bic x12, x6, x5 - ushr v8.2d, v31.2d, #19 - eor x5, x5, x28 - ushr v16.2d, v5.2d, #28 - eor x4, x4, x12 - sli v8.2d, v31.2d, #45 - eor x6, x6, x30 - sli v16.2d, v5.2d, #36 - bic x12, x9, x8 - eor v31.16b, v3.16b, v28.16b - bic x1, x10, x9 - eor v18.16b, v18.16b, v28.16b - bic x28, x7, x11 - ushr v5.2d, v31.2d, #36 - bic x30, x8, x7 - ushr v3.2d, v18.2d, #43 - eor x7, x7, x12 - sli v5.2d, v31.2d, #28 - eor x8, x8, x1 - sli v3.2d, v18.2d, #21 - bic x12, x11, x10 - eor v31.16b, v17.16b, v27.16b - eor x10, x10, x28 - eor v11.16b, v11.16b, v26.16b - eor x9, x9, x12 - ushr v18.2d, v31.2d, #49 - eor x11, x11, x30 - ushr v17.2d, v11.2d, #54 - bic x12, x14, x13 - sli v18.2d, v31.2d, #15 - bic x1, x15, x14 - sli v17.2d, v11.2d, #10 - bic x28, x0, x16 - eor v31.16b, v7.16b, v27.16b - bic x30, x13, x0 - eor v10.16b, v10.16b, v25.16b - eor x12, x0, x12 - ushr v11.2d, v31.2d, #58 - eor x13, x13, x1 - ushr v7.2d, v10.2d, #61 - bic x0, x16, x15 - sli v11.2d, v31.2d, #6 - eor x15, x15, x28 - sli v7.2d, v10.2d, #3 - eor x14, x14, x0 - # Row Mix NEON - bic v25.16b, v2.16b, v1.16b - eor x16, x16, x30 - bic v26.16b, v3.16b, v2.16b - bic x0, x20, x19 - bic v27.16b, v4.16b, v3.16b - bic x1, x21, x20 - bic v28.16b, v0.16b, v4.16b - bic x28, x17, x22 - bic v29.16b, v1.16b, v0.16b - bic x30, x19, x17 - eor v0.16b, v0.16b, v25.16b - eor x17, x17, x0 - eor v1.16b, v1.16b, v26.16b - eor x19, x19, x1 - eor v2.16b, v2.16b, v27.16b - bic x0, x22, x21 - eor v3.16b, v3.16b, v28.16b - eor x21, x21, x28 - eor v4.16b, v4.16b, v29.16b - eor x20, x20, x0 - bic v25.16b, v7.16b, v6.16b - eor x22, x22, x30 - bic v26.16b, v8.16b, v7.16b - bic x0, x25, x24 - bic v27.16b, v9.16b, v8.16b - bic x1, x26, x25 - bic v28.16b, v5.16b, v9.16b - bic x28, x23, x27 - bic v29.16b, v6.16b, v5.16b - bic x30, x24, x23 - eor v5.16b, v5.16b, v25.16b - eor x23, x23, x0 - eor v6.16b, v6.16b, v26.16b - eor x24, x24, x1 - eor v7.16b, v7.16b, v27.16b - bic x0, x27, x26 - eor v8.16b, v8.16b, v28.16b - eor x26, x26, x28 - eor v9.16b, v9.16b, v29.16b - eor x25, x25, x0 - bic v25.16b, v12.16b, v11.16b - eor x27, x27, x30 - bic v26.16b, v13.16b, v12.16b - bic v27.16b, v14.16b, v13.16b - bic v28.16b, v30.16b, v14.16b - bic v29.16b, v11.16b, v30.16b - eor v10.16b, v30.16b, v25.16b - eor v11.16b, v11.16b, v26.16b - eor v12.16b, v12.16b, v27.16b - eor v13.16b, v13.16b, v28.16b - eor v14.16b, v14.16b, v29.16b - bic v25.16b, v17.16b, v16.16b - bic v26.16b, v18.16b, v17.16b - bic v27.16b, v19.16b, v18.16b - bic v28.16b, v15.16b, v19.16b - bic v29.16b, v16.16b, v15.16b - eor v15.16b, v15.16b, v25.16b - eor v16.16b, v16.16b, v26.16b - eor v17.16b, v17.16b, v27.16b - eor v18.16b, v18.16b, v28.16b - eor v19.16b, v19.16b, v29.16b - bic v25.16b, v22.16b, v21.16b - bic v26.16b, v23.16b, v22.16b - bic v27.16b, v24.16b, v23.16b - bic v28.16b, v20.16b, v24.16b - bic v29.16b, v21.16b, v20.16b - eor v20.16b, v20.16b, v25.16b - eor v21.16b, v21.16b, v26.16b - eor v22.16b, v22.16b, v27.16b - eor v23.16b, v23.16b, v28.16b - eor v24.16b, v24.16b, v29.16b - # Done transforming - ldp x28, x1, [x29, #48] - ldr x0, [x28], #8 - subs x1, x1, #1 - mov v30.d[0], x0 - mov v30.d[1], x0 - eor x2, x2, x0 - eor v0.16b, v0.16b, v30.16b - bne L_SHA3_shake128_blocksx3_seed_neon_begin - ldr x0, [x29, #40] - st4 {v0.d, v1.d, v2.d, v3.d}[0], [x0], #32 - st4 {v4.d, v5.d, v6.d, v7.d}[0], [x0], #32 - st4 {v8.d, v9.d, v10.d, v11.d}[0], [x0], #32 - st4 {v12.d, v13.d, v14.d, v15.d}[0], [x0], #32 - st4 {v16.d, v17.d, v18.d, v19.d}[0], [x0], #32 - st4 {v20.d, v21.d, v22.d, v23.d}[0], [x0], #32 - st1 {v24.d}[0], [x0] - add x0, x0, #8 - st4 {v0.d, v1.d, v2.d, v3.d}[1], [x0], #32 - st4 {v4.d, v5.d, v6.d, v7.d}[1], [x0], #32 - st4 {v8.d, v9.d, v10.d, v11.d}[1], [x0], #32 - st4 {v12.d, v13.d, v14.d, v15.d}[1], [x0], #32 - st4 {v16.d, v17.d, v18.d, v19.d}[1], [x0], #32 - st4 {v20.d, v21.d, v22.d, v23.d}[1], [x0], #32 - st1 {v24.d}[1], [x0] - add x0, x0, #8 - stp x2, x3, [x0] - stp x4, x5, [x0, #16] - stp x6, x7, [x0, #32] - stp x8, x9, [x0, #48] - stp x10, x11, [x0, #64] - stp x12, x13, [x0, #80] - stp x14, x15, [x0, #96] - stp x16, x17, [x0, #112] - stp x19, x20, [x0, #128] - stp x21, x22, [x0, #144] - stp x23, x24, [x0, #160] - stp x25, x26, [x0, #176] - str x27, [x0, #192] - ldp x17, x19, [x29, #72] - ldp x20, x21, [x29, #88] - ldp x22, x23, [x29, #104] - ldp x24, x25, [x29, #120] - ldp x26, x27, [x29, #136] - ldr x28, [x29, #152] - ldp d8, d9, [x29, #160] - ldp d10, d11, [x29, #176] - ldp d12, d13, [x29, #192] - ldp d14, d15, [x29, #208] - ldp x29, x30, [sp], #0xe0 - ret -#ifndef __APPLE__ - .size mlkem_shake128_blocksx3_seed_neon,.-mlkem_shake128_blocksx3_seed_neon -#endif /* __APPLE__ */ -#ifndef __APPLE__ -.text -.globl mlkem_shake256_blocksx3_seed_neon -WC_ASM_ATT_HIDDEN(mlkem_shake256_blocksx3_seed_neon) -.type mlkem_shake256_blocksx3_seed_neon,@function -.align 2 -mlkem_shake256_blocksx3_seed_neon: -#else -.section __TEXT,__text -.globl _mlkem_shake256_blocksx3_seed_neon -WC_ASM_ATT_HIDDEN(_mlkem_shake256_blocksx3_seed_neon) -.p2align 2 -_mlkem_shake256_blocksx3_seed_neon: -#endif /* __APPLE__ */ - stp x29, x30, [sp, #-224]! - add x29, sp, #0 - stp x17, x19, [x29, #72] - stp x20, x21, [x29, #88] - stp x22, x23, [x29, #104] - stp x24, x25, [x29, #120] - stp x26, x27, [x29, #136] - str x28, [x29, #152] - stp d8, d9, [x29, #160] - stp d10, d11, [x29, #176] - stp d12, d13, [x29, #192] - stp d14, d15, [x29, #208] -#ifndef __APPLE__ - adrp x28, L_sha3_aarch64_r - add x28, x28, :lo12:L_sha3_aarch64_r -#else - adrp x28, L_sha3_aarch64_r@PAGE - add x28, x28, L_sha3_aarch64_r@PAGEOFF -#endif /* __APPLE__ */ - str x0, [x29, #40] - add x0, x0, #32 - ld1 {v4.d}[0], [x0] - ldp x2, x3, [x1], #16 - add x0, x0, #0xc8 - ld1 {v4.d}[1], [x0] - ldp x4, x5, [x1], #16 - ldr x6, [x0, #200] - eor v5.16b, v5.16b, v5.16b - eor x7, x7, x7 - eor v6.16b, v6.16b, v6.16b - eor x8, x8, x8 - eor v7.16b, v7.16b, v7.16b - eor x9, x9, x9 - eor v8.16b, v8.16b, v8.16b - eor x10, x10, x10 - eor v9.16b, v9.16b, v9.16b - eor x11, x11, x11 - eor v10.16b, v10.16b, v10.16b - eor x12, x12, x12 - eor v11.16b, v11.16b, v11.16b - eor x13, x13, x13 - eor v12.16b, v12.16b, v12.16b - eor x14, x14, x14 - eor v13.16b, v13.16b, v13.16b - eor x15, x15, x15 - eor v14.16b, v14.16b, v14.16b - eor x16, x16, x16 - eor v15.16b, v15.16b, v15.16b - eor x17, x17, x17 - movz x19, #0x8000, lsl 48 - eor v17.16b, v17.16b, v17.16b - eor x20, x20, x20 - eor v18.16b, v18.16b, v18.16b - eor x21, x21, x21 - eor v19.16b, v19.16b, v19.16b - eor x22, x22, x22 - eor v20.16b, v20.16b, v20.16b - eor x23, x23, x23 - eor v21.16b, v21.16b, v21.16b - eor x24, x24, x24 - eor v22.16b, v22.16b, v22.16b - eor x25, x25, x25 - eor v23.16b, v23.16b, v23.16b - eor x26, x26, x26 - eor v24.16b, v24.16b, v24.16b - eor x27, x27, x27 - dup v0.2d, x2 - dup v1.2d, x3 - dup v2.2d, x4 - dup v3.2d, x5 - dup v16.2d, x19 - mov x1, #24 - # Start of 24 rounds -L_SHA3_shake256_blocksx3_seed_neon_begin: - stp x28, x1, [x29, #48] - # Col Mix NEON - eor v30.16b, v4.16b, v9.16b - eor x0, x6, x11 - eor v27.16b, v1.16b, v6.16b - eor x30, x2, x7 - eor v30.16b, v30.16b, v14.16b - eor x28, x4, x9 - eor v27.16b, v27.16b, v11.16b - eor x0, x0, x16 - eor v30.16b, v30.16b, v19.16b - eor x30, x30, x12 - eor v27.16b, v27.16b, v16.16b - eor x28, x28, x14 - eor v30.16b, v30.16b, v24.16b - eor x0, x0, x22 - eor v27.16b, v27.16b, v21.16b - eor x30, x30, x17 - ushr v25.2d, v27.2d, #63 - eor x28, x28, x20 - sli v25.2d, v27.2d, #1 - eor x0, x0, x27 - eor v25.16b, v25.16b, v30.16b - eor x30, x30, x23 - eor v31.16b, v0.16b, v5.16b - eor x28, x28, x25 - eor v28.16b, v2.16b, v7.16b - str x0, [x29, #32] - eor v31.16b, v31.16b, v10.16b - str x28, [x29, #24] - eor v28.16b, v28.16b, v12.16b - eor x1, x3, x8 - eor v31.16b, v31.16b, v15.16b - eor x28, x5, x10 - eor v28.16b, v28.16b, v17.16b - eor x1, x1, x13 - eor v31.16b, v31.16b, v20.16b - eor x28, x28, x15 - eor v28.16b, v28.16b, v22.16b - eor x1, x1, x19 - ushr v29.2d, v30.2d, #63 - eor x28, x28, x21 - ushr v26.2d, v28.2d, #63 - eor x1, x1, x24 - sli v29.2d, v30.2d, #1 - eor x28, x28, x26 - sli v26.2d, v28.2d, #1 - eor x0, x0, x1, ror 63 - eor v28.16b, v28.16b, v29.16b - eor x1, x1, x28, ror 63 - eor v29.16b, v3.16b, v8.16b - eor x2, x2, x0 - eor v26.16b, v26.16b, v31.16b - eor x7, x7, x0 - eor v29.16b, v29.16b, v13.16b - eor x12, x12, x0 - eor v29.16b, v29.16b, v18.16b - eor x17, x17, x0 - eor v29.16b, v29.16b, v23.16b - eor x23, x23, x0 - ushr v30.2d, v29.2d, #63 - eor x4, x4, x1 - sli v30.2d, v29.2d, #1 - eor x9, x9, x1 - eor v27.16b, v27.16b, v30.16b - eor x14, x14, x1 - ushr v30.2d, v31.2d, #63 - eor x20, x20, x1 - sli v30.2d, v31.2d, #1 - eor x25, x25, x1 - eor v29.16b, v29.16b, v30.16b - ldr x0, [x29, #32] - # Swap Rotate NEON - eor v0.16b, v0.16b, v25.16b - eor v31.16b, v1.16b, v26.16b - ldr x1, [x29, #24] - eor v6.16b, v6.16b, v26.16b - eor x28, x28, x30, ror 63 - ushr v30.2d, v31.2d, #63 - eor x30, x30, x1, ror 63 - ushr v1.2d, v6.2d, #20 - eor x1, x1, x0, ror 63 - sli v30.2d, v31.2d, #1 - eor x6, x6, x28 - sli v1.2d, v6.2d, #44 - eor x11, x11, x28 - eor v31.16b, v9.16b, v29.16b - eor x16, x16, x28 - eor v22.16b, v22.16b, v27.16b - eor x22, x22, x28 - ushr v6.2d, v31.2d, #44 - eor x27, x27, x28 - ushr v9.2d, v22.2d, #3 - eor x3, x3, x30 - sli v6.2d, v31.2d, #20 - eor x8, x8, x30 - sli v9.2d, v22.2d, #61 - eor x13, x13, x30 - eor v31.16b, v14.16b, v29.16b - eor x19, x19, x30 - eor v20.16b, v20.16b, v25.16b - eor x24, x24, x30 - ushr v22.2d, v31.2d, #25 - eor x5, x5, x1 - ushr v14.2d, v20.2d, #46 - eor x10, x10, x1 - sli v22.2d, v31.2d, #39 - eor x15, x15, x1 - sli v14.2d, v20.2d, #18 - eor x21, x21, x1 - eor v31.16b, v2.16b, v27.16b - eor x26, x26, x1 - # Swap Rotate Base - eor v12.16b, v12.16b, v27.16b - ror x0, x3, #63 - ushr v20.2d, v31.2d, #2 - ror x3, x8, #20 - ushr v2.2d, v12.2d, #21 - ror x8, x11, #44 - sli v20.2d, v31.2d, #62 - ror x11, x25, #3 - sli v2.2d, v12.2d, #43 - ror x25, x16, #25 - eor v31.16b, v13.16b, v28.16b - ror x16, x23, #46 - eor v19.16b, v19.16b, v29.16b - ror x23, x4, #2 - ushr v12.2d, v31.2d, #39 - ror x4, x14, #21 - ushr v13.2d, v19.2d, #56 - ror x14, x15, #39 - sli v12.2d, v31.2d, #25 - ror x15, x22, #56 - sli v13.2d, v19.2d, #8 - ror x22, x26, #8 - eor v31.16b, v23.16b, v28.16b - ror x26, x17, #23 - eor v15.16b, v15.16b, v25.16b - ror x17, x6, #37 - ushr v19.2d, v31.2d, #8 - ror x6, x27, #50 - ushr v23.2d, v15.2d, #23 - ror x27, x24, #62 - sli v19.2d, v31.2d, #56 - ror x24, x10, #9 - sli v23.2d, v15.2d, #41 - ror x10, x19, #19 - eor v31.16b, v4.16b, v29.16b - ror x19, x7, #28 - eor v24.16b, v24.16b, v29.16b - ror x7, x5, #36 - ushr v15.2d, v31.2d, #37 - ror x5, x21, #43 - ushr v4.2d, v24.2d, #50 - ror x21, x20, #49 - sli v15.2d, v31.2d, #27 - ror x20, x13, #54 - sli v4.2d, v24.2d, #14 - ror x13, x9, #58 - eor v31.16b, v21.16b, v26.16b - ror x9, x12, #61 - # Row Mix Base - eor v8.16b, v8.16b, v28.16b - bic x12, x4, x3 - ushr v24.2d, v31.2d, #62 - bic x1, x5, x4 - ushr v21.2d, v8.2d, #9 - bic x28, x2, x6 - sli v24.2d, v31.2d, #2 - bic x30, x3, x2 - sli v21.2d, v8.2d, #55 - eor x2, x2, x12 - eor v31.16b, v16.16b, v26.16b - eor x3, x3, x1 - eor v5.16b, v5.16b, v25.16b - bic x12, x6, x5 - ushr v8.2d, v31.2d, #19 - eor x5, x5, x28 - ushr v16.2d, v5.2d, #28 - eor x4, x4, x12 - sli v8.2d, v31.2d, #45 - eor x6, x6, x30 - sli v16.2d, v5.2d, #36 - bic x12, x9, x8 - eor v31.16b, v3.16b, v28.16b - bic x1, x10, x9 - eor v18.16b, v18.16b, v28.16b - bic x28, x7, x11 - ushr v5.2d, v31.2d, #36 - bic x30, x8, x7 - ushr v3.2d, v18.2d, #43 - eor x7, x7, x12 - sli v5.2d, v31.2d, #28 - eor x8, x8, x1 - sli v3.2d, v18.2d, #21 - bic x12, x11, x10 - eor v31.16b, v17.16b, v27.16b - eor x10, x10, x28 - eor v11.16b, v11.16b, v26.16b - eor x9, x9, x12 - ushr v18.2d, v31.2d, #49 - eor x11, x11, x30 - ushr v17.2d, v11.2d, #54 - bic x12, x14, x13 - sli v18.2d, v31.2d, #15 - bic x1, x15, x14 - sli v17.2d, v11.2d, #10 - bic x28, x0, x16 - eor v31.16b, v7.16b, v27.16b - bic x30, x13, x0 - eor v10.16b, v10.16b, v25.16b - eor x12, x0, x12 - ushr v11.2d, v31.2d, #58 - eor x13, x13, x1 - ushr v7.2d, v10.2d, #61 - bic x0, x16, x15 - sli v11.2d, v31.2d, #6 - eor x15, x15, x28 - sli v7.2d, v10.2d, #3 - eor x14, x14, x0 - # Row Mix NEON - bic v25.16b, v2.16b, v1.16b - eor x16, x16, x30 - bic v26.16b, v3.16b, v2.16b - bic x0, x20, x19 - bic v27.16b, v4.16b, v3.16b - bic x1, x21, x20 - bic v28.16b, v0.16b, v4.16b - bic x28, x17, x22 - bic v29.16b, v1.16b, v0.16b - bic x30, x19, x17 - eor v0.16b, v0.16b, v25.16b - eor x17, x17, x0 - eor v1.16b, v1.16b, v26.16b - eor x19, x19, x1 - eor v2.16b, v2.16b, v27.16b - bic x0, x22, x21 - eor v3.16b, v3.16b, v28.16b - eor x21, x21, x28 - eor v4.16b, v4.16b, v29.16b - eor x20, x20, x0 - bic v25.16b, v7.16b, v6.16b - eor x22, x22, x30 - bic v26.16b, v8.16b, v7.16b - bic x0, x25, x24 - bic v27.16b, v9.16b, v8.16b - bic x1, x26, x25 - bic v28.16b, v5.16b, v9.16b - bic x28, x23, x27 - bic v29.16b, v6.16b, v5.16b - bic x30, x24, x23 - eor v5.16b, v5.16b, v25.16b - eor x23, x23, x0 - eor v6.16b, v6.16b, v26.16b - eor x24, x24, x1 - eor v7.16b, v7.16b, v27.16b - bic x0, x27, x26 - eor v8.16b, v8.16b, v28.16b - eor x26, x26, x28 - eor v9.16b, v9.16b, v29.16b - eor x25, x25, x0 - bic v25.16b, v12.16b, v11.16b - eor x27, x27, x30 - bic v26.16b, v13.16b, v12.16b - bic v27.16b, v14.16b, v13.16b - bic v28.16b, v30.16b, v14.16b - bic v29.16b, v11.16b, v30.16b - eor v10.16b, v30.16b, v25.16b - eor v11.16b, v11.16b, v26.16b - eor v12.16b, v12.16b, v27.16b - eor v13.16b, v13.16b, v28.16b - eor v14.16b, v14.16b, v29.16b - bic v25.16b, v17.16b, v16.16b - bic v26.16b, v18.16b, v17.16b - bic v27.16b, v19.16b, v18.16b - bic v28.16b, v15.16b, v19.16b - bic v29.16b, v16.16b, v15.16b - eor v15.16b, v15.16b, v25.16b - eor v16.16b, v16.16b, v26.16b - eor v17.16b, v17.16b, v27.16b - eor v18.16b, v18.16b, v28.16b - eor v19.16b, v19.16b, v29.16b - bic v25.16b, v22.16b, v21.16b - bic v26.16b, v23.16b, v22.16b - bic v27.16b, v24.16b, v23.16b - bic v28.16b, v20.16b, v24.16b - bic v29.16b, v21.16b, v20.16b - eor v20.16b, v20.16b, v25.16b - eor v21.16b, v21.16b, v26.16b - eor v22.16b, v22.16b, v27.16b - eor v23.16b, v23.16b, v28.16b - eor v24.16b, v24.16b, v29.16b - # Done transforming - ldp x28, x1, [x29, #48] - ldr x0, [x28], #8 - subs x1, x1, #1 - mov v30.d[0], x0 - mov v30.d[1], x0 - eor x2, x2, x0 - eor v0.16b, v0.16b, v30.16b - bne L_SHA3_shake256_blocksx3_seed_neon_begin - ldr x0, [x29, #40] - st4 {v0.d, v1.d, v2.d, v3.d}[0], [x0], #32 - st4 {v4.d, v5.d, v6.d, v7.d}[0], [x0], #32 - st4 {v8.d, v9.d, v10.d, v11.d}[0], [x0], #32 - st4 {v12.d, v13.d, v14.d, v15.d}[0], [x0], #32 - st4 {v16.d, v17.d, v18.d, v19.d}[0], [x0], #32 - st4 {v20.d, v21.d, v22.d, v23.d}[0], [x0], #32 - st1 {v24.d}[0], [x0] - add x0, x0, #8 - st4 {v0.d, v1.d, v2.d, v3.d}[1], [x0], #32 - st4 {v4.d, v5.d, v6.d, v7.d}[1], [x0], #32 - st4 {v8.d, v9.d, v10.d, v11.d}[1], [x0], #32 - st4 {v12.d, v13.d, v14.d, v15.d}[1], [x0], #32 - st4 {v16.d, v17.d, v18.d, v19.d}[1], [x0], #32 - st4 {v20.d, v21.d, v22.d, v23.d}[1], [x0], #32 - st1 {v24.d}[1], [x0] - add x0, x0, #8 - stp x2, x3, [x0] - stp x4, x5, [x0, #16] - stp x6, x7, [x0, #32] - stp x8, x9, [x0, #48] - stp x10, x11, [x0, #64] - stp x12, x13, [x0, #80] - stp x14, x15, [x0, #96] - stp x16, x17, [x0, #112] - stp x19, x20, [x0, #128] - stp x21, x22, [x0, #144] - stp x23, x24, [x0, #160] - stp x25, x26, [x0, #176] - str x27, [x0, #192] - ldp x17, x19, [x29, #72] - ldp x20, x21, [x29, #88] - ldp x22, x23, [x29, #104] - ldp x24, x25, [x29, #120] - ldp x26, x27, [x29, #136] - ldr x28, [x29, #152] - ldp d8, d9, [x29, #160] - ldp d10, d11, [x29, #176] - ldp d12, d13, [x29, #192] - ldp d14, d15, [x29, #208] - ldp x29, x30, [sp], #0xe0 - ret -#ifndef __APPLE__ - .size mlkem_shake256_blocksx3_seed_neon,.-mlkem_shake256_blocksx3_seed_neon + .size mlkem_shake256_blocksx3_seed_crypto,.-mlkem_shake256_blocksx3_seed_crypto #endif /* __APPLE__ */ #endif /* WOLFSSL_ARMASM_CRYPTO_SHA3 */ +#ifndef __APPLE__ +.text +.globl mlkem_sha3_blocksx3_neon +WC_ASM_ATT_HIDDEN(mlkem_sha3_blocksx3_neon) +.type mlkem_sha3_blocksx3_neon,@function +.align 2 +mlkem_sha3_blocksx3_neon: +#else +.section __TEXT,__text +.globl _mlkem_sha3_blocksx3_neon +WC_ASM_ATT_HIDDEN(_mlkem_sha3_blocksx3_neon) +.p2align 2 +_mlkem_sha3_blocksx3_neon: +#endif /* __APPLE__ */ + stp x29, x30, [sp, #-224]! + add x29, sp, #0 + stp x17, x19, [x29, #72] + stp x20, x21, [x29, #88] + stp x22, x23, [x29, #104] + stp x24, x25, [x29, #120] + stp x26, x27, [x29, #136] + str x28, [x29, #152] + stp d8, d9, [x29, #160] + stp d10, d11, [x29, #176] + stp d12, d13, [x29, #192] + stp d14, d15, [x29, #208] +#ifndef __APPLE__ + adrp x27, L_sha3_aarch64_r + add x27, x27, :lo12:L_sha3_aarch64_r +#else + adrp x27, L_sha3_aarch64_r@PAGE + add x27, x27, L_sha3_aarch64_r@PAGEOFF +#endif /* __APPLE__ */ + str x0, [x29, #40] + ld4 {v0.d, v1.d, v2.d, v3.d}[0], [x0], #32 + ld4 {v4.d, v5.d, v6.d, v7.d}[0], [x0], #32 + ld4 {v8.d, v9.d, v10.d, v11.d}[0], [x0], #32 + ld4 {v12.d, v13.d, v14.d, v15.d}[0], [x0], #32 + ld4 {v16.d, v17.d, v18.d, v19.d}[0], [x0], #32 + ld4 {v20.d, v21.d, v22.d, v23.d}[0], [x0], #32 + ld1 {v24.d}[0], [x0] + add x0, x0, #8 + ld4 {v0.d, v1.d, v2.d, v3.d}[1], [x0], #32 + ld4 {v4.d, v5.d, v6.d, v7.d}[1], [x0], #32 + ld4 {v8.d, v9.d, v10.d, v11.d}[1], [x0], #32 + ld4 {v12.d, v13.d, v14.d, v15.d}[1], [x0], #32 + ld4 {v16.d, v17.d, v18.d, v19.d}[1], [x0], #32 + ld4 {v20.d, v21.d, v22.d, v23.d}[1], [x0], #32 + ld1 {v24.d}[1], [x0] + add x0, x0, #8 + ldp x1, x2, [x0] + ldp x3, x4, [x0, #16] + ldp x5, x6, [x0, #32] + ldp x7, x8, [x0, #48] + ldp x9, x10, [x0, #64] + ldp x11, x12, [x0, #80] + ldp x13, x14, [x0, #96] + ldp x15, x16, [x0, #112] + ldp x17, x19, [x0, #128] + ldp x20, x21, [x0, #144] + ldp x22, x23, [x0, #160] + ldp x24, x25, [x0, #176] + ldr x26, [x0, #192] + mov x28, #24 + # Start of 24 rounds +L_SHA3_transform_blocksx3_neon_begin: + stp x27, x28, [x29, #48] + # Col Mix NEON + eor v30.16b, v4.16b, v9.16b + eor x0, x5, x10 + eor v27.16b, v1.16b, v6.16b + eor x30, x1, x6 + eor v30.16b, v30.16b, v14.16b + eor x28, x3, x8 + eor v27.16b, v27.16b, v11.16b + eor x0, x0, x15 + eor v30.16b, v30.16b, v19.16b + eor x30, x30, x11 + eor v27.16b, v27.16b, v16.16b + eor x28, x28, x13 + eor v30.16b, v30.16b, v24.16b + eor x0, x0, x21 + eor v27.16b, v27.16b, v21.16b + eor x30, x30, x16 + ushr v25.2d, v27.2d, #63 + eor x28, x28, x19 + sli v25.2d, v27.2d, #1 + eor x0, x0, x26 + eor v25.16b, v25.16b, v30.16b + eor x30, x30, x22 + eor v31.16b, v0.16b, v5.16b + eor x28, x28, x24 + eor v28.16b, v2.16b, v7.16b + str x0, [x29, #32] + eor v31.16b, v31.16b, v10.16b + str x28, [x29, #24] + eor v28.16b, v28.16b, v12.16b + eor x27, x2, x7 + eor v31.16b, v31.16b, v15.16b + eor x28, x4, x9 + eor v28.16b, v28.16b, v17.16b + eor x27, x27, x12 + eor v31.16b, v31.16b, v20.16b + eor x28, x28, x14 + eor v28.16b, v28.16b, v22.16b + eor x27, x27, x17 + ushr v29.2d, v30.2d, #63 + eor x28, x28, x20 + ushr v26.2d, v28.2d, #63 + eor x27, x27, x23 + sli v29.2d, v30.2d, #1 + eor x28, x28, x25 + sli v26.2d, v28.2d, #1 + eor x0, x0, x27, ror 63 + eor v28.16b, v28.16b, v29.16b + eor x27, x27, x28, ror 63 + eor v29.16b, v3.16b, v8.16b + eor x1, x1, x0 + eor v26.16b, v26.16b, v31.16b + eor x6, x6, x0 + eor v29.16b, v29.16b, v13.16b + eor x11, x11, x0 + eor v29.16b, v29.16b, v18.16b + eor x16, x16, x0 + eor v29.16b, v29.16b, v23.16b + eor x22, x22, x0 + ushr v30.2d, v29.2d, #63 + eor x3, x3, x27 + sli v30.2d, v29.2d, #1 + eor x8, x8, x27 + eor v27.16b, v27.16b, v30.16b + eor x13, x13, x27 + ushr v30.2d, v31.2d, #63 + eor x19, x19, x27 + sli v30.2d, v31.2d, #1 + eor x24, x24, x27 + eor v29.16b, v29.16b, v30.16b + ldr x0, [x29, #32] + # Swap Rotate NEON + eor v0.16b, v0.16b, v25.16b + eor v31.16b, v1.16b, v26.16b + ldr x27, [x29, #24] + eor v6.16b, v6.16b, v26.16b + eor x28, x28, x30, ror 63 + ushr v30.2d, v31.2d, #63 + eor x30, x30, x27, ror 63 + ushr v1.2d, v6.2d, #20 + eor x27, x27, x0, ror 63 + sli v30.2d, v31.2d, #1 + eor x5, x5, x28 + sli v1.2d, v6.2d, #44 + eor x10, x10, x28 + eor v31.16b, v9.16b, v29.16b + eor x15, x15, x28 + eor v22.16b, v22.16b, v27.16b + eor x21, x21, x28 + ushr v6.2d, v31.2d, #44 + eor x26, x26, x28 + ushr v9.2d, v22.2d, #3 + eor x2, x2, x30 + sli v6.2d, v31.2d, #20 + eor x7, x7, x30 + sli v9.2d, v22.2d, #61 + eor x12, x12, x30 + eor v31.16b, v14.16b, v29.16b + eor x17, x17, x30 + eor v20.16b, v20.16b, v25.16b + eor x23, x23, x30 + ushr v22.2d, v31.2d, #25 + eor x4, x4, x27 + ushr v14.2d, v20.2d, #46 + eor x9, x9, x27 + sli v22.2d, v31.2d, #39 + eor x14, x14, x27 + sli v14.2d, v20.2d, #18 + eor x20, x20, x27 + eor v31.16b, v2.16b, v27.16b + eor x25, x25, x27 + # Swap Rotate Base + eor v12.16b, v12.16b, v27.16b + ror x0, x2, #63 + ushr v20.2d, v31.2d, #2 + ror x2, x7, #20 + ushr v2.2d, v12.2d, #21 + ror x7, x10, #44 + sli v20.2d, v31.2d, #62 + ror x10, x24, #3 + sli v2.2d, v12.2d, #43 + ror x24, x15, #25 + eor v31.16b, v13.16b, v28.16b + ror x15, x22, #46 + eor v19.16b, v19.16b, v29.16b + ror x22, x3, #2 + ushr v12.2d, v31.2d, #39 + ror x3, x13, #21 + ushr v13.2d, v19.2d, #56 + ror x13, x14, #39 + sli v12.2d, v31.2d, #25 + ror x14, x21, #56 + sli v13.2d, v19.2d, #8 + ror x21, x25, #8 + eor v31.16b, v23.16b, v28.16b + ror x25, x16, #23 + eor v15.16b, v15.16b, v25.16b + ror x16, x5, #37 + ushr v19.2d, v31.2d, #8 + ror x5, x26, #50 + ushr v23.2d, v15.2d, #23 + ror x26, x23, #62 + sli v19.2d, v31.2d, #56 + ror x23, x9, #9 + sli v23.2d, v15.2d, #41 + ror x9, x17, #19 + eor v31.16b, v4.16b, v29.16b + ror x17, x6, #28 + eor v24.16b, v24.16b, v29.16b + ror x6, x4, #36 + ushr v15.2d, v31.2d, #37 + ror x4, x20, #43 + ushr v4.2d, v24.2d, #50 + ror x20, x19, #49 + sli v15.2d, v31.2d, #27 + ror x19, x12, #54 + sli v4.2d, v24.2d, #14 + ror x12, x8, #58 + eor v31.16b, v21.16b, v26.16b + ror x8, x11, #61 + # Row Mix Base + eor v8.16b, v8.16b, v28.16b + bic x11, x3, x2 + ushr v24.2d, v31.2d, #62 + bic x27, x4, x3 + ushr v21.2d, v8.2d, #9 + bic x28, x1, x5 + sli v24.2d, v31.2d, #2 + bic x30, x2, x1 + sli v21.2d, v8.2d, #55 + eor x1, x1, x11 + eor v31.16b, v16.16b, v26.16b + eor x2, x2, x27 + eor v5.16b, v5.16b, v25.16b + bic x11, x5, x4 + ushr v8.2d, v31.2d, #19 + eor x4, x4, x28 + ushr v16.2d, v5.2d, #28 + eor x3, x3, x11 + sli v8.2d, v31.2d, #45 + eor x5, x5, x30 + sli v16.2d, v5.2d, #36 + bic x11, x8, x7 + eor v31.16b, v3.16b, v28.16b + bic x27, x9, x8 + eor v18.16b, v18.16b, v28.16b + bic x28, x6, x10 + ushr v5.2d, v31.2d, #36 + bic x30, x7, x6 + ushr v3.2d, v18.2d, #43 + eor x6, x6, x11 + sli v5.2d, v31.2d, #28 + eor x7, x7, x27 + sli v3.2d, v18.2d, #21 + bic x11, x10, x9 + eor v31.16b, v17.16b, v27.16b + eor x9, x9, x28 + eor v11.16b, v11.16b, v26.16b + eor x8, x8, x11 + ushr v18.2d, v31.2d, #49 + eor x10, x10, x30 + ushr v17.2d, v11.2d, #54 + bic x11, x13, x12 + sli v18.2d, v31.2d, #15 + bic x27, x14, x13 + sli v17.2d, v11.2d, #10 + bic x28, x0, x15 + eor v31.16b, v7.16b, v27.16b + bic x30, x12, x0 + eor v10.16b, v10.16b, v25.16b + eor x11, x0, x11 + ushr v11.2d, v31.2d, #58 + eor x12, x12, x27 + ushr v7.2d, v10.2d, #61 + bic x0, x15, x14 + sli v11.2d, v31.2d, #6 + eor x14, x14, x28 + sli v7.2d, v10.2d, #3 + eor x13, x13, x0 + # Row Mix NEON + bic v25.16b, v2.16b, v1.16b + eor x15, x15, x30 + bic v26.16b, v3.16b, v2.16b + bic x0, x19, x17 + bic v27.16b, v4.16b, v3.16b + bic x27, x20, x19 + bic v28.16b, v0.16b, v4.16b + bic x28, x16, x21 + bic v29.16b, v1.16b, v0.16b + bic x30, x17, x16 + eor v0.16b, v0.16b, v25.16b + eor x16, x16, x0 + eor v1.16b, v1.16b, v26.16b + eor x17, x17, x27 + eor v2.16b, v2.16b, v27.16b + bic x0, x21, x20 + eor v3.16b, v3.16b, v28.16b + eor x20, x20, x28 + eor v4.16b, v4.16b, v29.16b + eor x19, x19, x0 + bic v25.16b, v7.16b, v6.16b + eor x21, x21, x30 + bic v26.16b, v8.16b, v7.16b + bic x0, x24, x23 + bic v27.16b, v9.16b, v8.16b + bic x27, x25, x24 + bic v28.16b, v5.16b, v9.16b + bic x28, x22, x26 + bic v29.16b, v6.16b, v5.16b + bic x30, x23, x22 + eor v5.16b, v5.16b, v25.16b + eor x22, x22, x0 + eor v6.16b, v6.16b, v26.16b + eor x23, x23, x27 + eor v7.16b, v7.16b, v27.16b + bic x0, x26, x25 + eor v8.16b, v8.16b, v28.16b + eor x25, x25, x28 + eor v9.16b, v9.16b, v29.16b + eor x24, x24, x0 + bic v25.16b, v12.16b, v11.16b + eor x26, x26, x30 + bic v26.16b, v13.16b, v12.16b + bic v27.16b, v14.16b, v13.16b + bic v28.16b, v30.16b, v14.16b + bic v29.16b, v11.16b, v30.16b + eor v10.16b, v30.16b, v25.16b + eor v11.16b, v11.16b, v26.16b + eor v12.16b, v12.16b, v27.16b + eor v13.16b, v13.16b, v28.16b + eor v14.16b, v14.16b, v29.16b + bic v25.16b, v17.16b, v16.16b + bic v26.16b, v18.16b, v17.16b + bic v27.16b, v19.16b, v18.16b + bic v28.16b, v15.16b, v19.16b + bic v29.16b, v16.16b, v15.16b + eor v15.16b, v15.16b, v25.16b + eor v16.16b, v16.16b, v26.16b + eor v17.16b, v17.16b, v27.16b + eor v18.16b, v18.16b, v28.16b + eor v19.16b, v19.16b, v29.16b + bic v25.16b, v22.16b, v21.16b + bic v26.16b, v23.16b, v22.16b + bic v27.16b, v24.16b, v23.16b + bic v28.16b, v20.16b, v24.16b + bic v29.16b, v21.16b, v20.16b + eor v20.16b, v20.16b, v25.16b + eor v21.16b, v21.16b, v26.16b + eor v22.16b, v22.16b, v27.16b + eor v23.16b, v23.16b, v28.16b + eor v24.16b, v24.16b, v29.16b + # Done transforming + ldp x27, x28, [x29, #48] + ldr x0, [x27], #8 + subs x28, x28, #1 + mov v30.d[0], x0 + mov v30.d[1], x0 + eor x1, x1, x0 + eor v0.16b, v0.16b, v30.16b + bne L_SHA3_transform_blocksx3_neon_begin + ldr x0, [x29, #40] + st4 {v0.d, v1.d, v2.d, v3.d}[0], [x0], #32 + st4 {v4.d, v5.d, v6.d, v7.d}[0], [x0], #32 + st4 {v8.d, v9.d, v10.d, v11.d}[0], [x0], #32 + st4 {v12.d, v13.d, v14.d, v15.d}[0], [x0], #32 + st4 {v16.d, v17.d, v18.d, v19.d}[0], [x0], #32 + st4 {v20.d, v21.d, v22.d, v23.d}[0], [x0], #32 + st1 {v24.d}[0], [x0] + add x0, x0, #8 + st4 {v0.d, v1.d, v2.d, v3.d}[1], [x0], #32 + st4 {v4.d, v5.d, v6.d, v7.d}[1], [x0], #32 + st4 {v8.d, v9.d, v10.d, v11.d}[1], [x0], #32 + st4 {v12.d, v13.d, v14.d, v15.d}[1], [x0], #32 + st4 {v16.d, v17.d, v18.d, v19.d}[1], [x0], #32 + st4 {v20.d, v21.d, v22.d, v23.d}[1], [x0], #32 + st1 {v24.d}[1], [x0] + add x0, x0, #8 + stp x1, x2, [x0] + stp x3, x4, [x0, #16] + stp x5, x6, [x0, #32] + stp x7, x8, [x0, #48] + stp x9, x10, [x0, #64] + stp x11, x12, [x0, #80] + stp x13, x14, [x0, #96] + stp x15, x16, [x0, #112] + stp x17, x19, [x0, #128] + stp x20, x21, [x0, #144] + stp x22, x23, [x0, #160] + stp x24, x25, [x0, #176] + str x26, [x0, #192] + ldp x17, x19, [x29, #72] + ldp x20, x21, [x29, #88] + ldp x22, x23, [x29, #104] + ldp x24, x25, [x29, #120] + ldp x26, x27, [x29, #136] + ldr x28, [x29, #152] + ldp d8, d9, [x29, #160] + ldp d10, d11, [x29, #176] + ldp d12, d13, [x29, #192] + ldp d14, d15, [x29, #208] + ldp x29, x30, [sp], #0xe0 + ret +#ifndef __APPLE__ + .size mlkem_sha3_blocksx3_neon,.-mlkem_sha3_blocksx3_neon +#endif /* __APPLE__ */ +#ifndef __APPLE__ +.text +.globl mlkem_shake128_blocksx3_seed_neon +WC_ASM_ATT_HIDDEN(mlkem_shake128_blocksx3_seed_neon) +.type mlkem_shake128_blocksx3_seed_neon,@function +.align 2 +mlkem_shake128_blocksx3_seed_neon: +#else +.section __TEXT,__text +.globl _mlkem_shake128_blocksx3_seed_neon +WC_ASM_ATT_HIDDEN(_mlkem_shake128_blocksx3_seed_neon) +.p2align 2 +_mlkem_shake128_blocksx3_seed_neon: +#endif /* __APPLE__ */ + stp x29, x30, [sp, #-224]! + add x29, sp, #0 + stp x17, x19, [x29, #72] + stp x20, x21, [x29, #88] + stp x22, x23, [x29, #104] + stp x24, x25, [x29, #120] + stp x26, x27, [x29, #136] + str x28, [x29, #152] + stp d8, d9, [x29, #160] + stp d10, d11, [x29, #176] + stp d12, d13, [x29, #192] + stp d14, d15, [x29, #208] +#ifndef __APPLE__ + adrp x28, L_sha3_aarch64_r + add x28, x28, :lo12:L_sha3_aarch64_r +#else + adrp x28, L_sha3_aarch64_r@PAGE + add x28, x28, L_sha3_aarch64_r@PAGEOFF +#endif /* __APPLE__ */ + str x0, [x29, #40] + add x0, x0, #32 + ld1 {v4.d}[0], [x0] + ldp x2, x3, [x1], #16 + add x0, x0, #0xc8 + ld1 {v4.d}[1], [x0] + ldp x4, x5, [x1], #16 + ldr x6, [x0, #200] + eor v5.16b, v5.16b, v5.16b + eor x7, x7, x7 + eor v6.16b, v6.16b, v6.16b + eor x8, x8, x8 + eor v7.16b, v7.16b, v7.16b + eor x9, x9, x9 + eor v8.16b, v8.16b, v8.16b + eor x10, x10, x10 + eor v9.16b, v9.16b, v9.16b + eor x11, x11, x11 + eor v10.16b, v10.16b, v10.16b + eor x12, x12, x12 + eor v11.16b, v11.16b, v11.16b + eor x13, x13, x13 + eor v12.16b, v12.16b, v12.16b + eor x14, x14, x14 + eor v13.16b, v13.16b, v13.16b + eor x15, x15, x15 + eor v14.16b, v14.16b, v14.16b + eor x16, x16, x16 + eor v15.16b, v15.16b, v15.16b + eor x17, x17, x17 + eor v16.16b, v16.16b, v16.16b + eor x19, x19, x19 + eor v17.16b, v17.16b, v17.16b + eor x20, x20, x20 + eor v18.16b, v18.16b, v18.16b + eor x21, x21, x21 + eor v19.16b, v19.16b, v19.16b + eor x22, x22, x22 + movz x23, #0x8000, lsl 48 + eor v21.16b, v21.16b, v21.16b + eor x24, x24, x24 + eor v22.16b, v22.16b, v22.16b + eor x25, x25, x25 + eor v23.16b, v23.16b, v23.16b + eor x26, x26, x26 + eor v24.16b, v24.16b, v24.16b + eor x27, x27, x27 + dup v0.2d, x2 + dup v1.2d, x3 + dup v2.2d, x4 + dup v3.2d, x5 + dup v20.2d, x23 + mov x1, #24 + # Start of 24 rounds +L_SHA3_shake128_blocksx3_seed_neon_begin: + stp x28, x1, [x29, #48] + # Col Mix NEON + eor v30.16b, v4.16b, v9.16b + eor x0, x6, x11 + eor v27.16b, v1.16b, v6.16b + eor x30, x2, x7 + eor v30.16b, v30.16b, v14.16b + eor x28, x4, x9 + eor v27.16b, v27.16b, v11.16b + eor x0, x0, x16 + eor v30.16b, v30.16b, v19.16b + eor x30, x30, x12 + eor v27.16b, v27.16b, v16.16b + eor x28, x28, x14 + eor v30.16b, v30.16b, v24.16b + eor x0, x0, x22 + eor v27.16b, v27.16b, v21.16b + eor x30, x30, x17 + ushr v25.2d, v27.2d, #63 + eor x28, x28, x20 + sli v25.2d, v27.2d, #1 + eor x0, x0, x27 + eor v25.16b, v25.16b, v30.16b + eor x30, x30, x23 + eor v31.16b, v0.16b, v5.16b + eor x28, x28, x25 + eor v28.16b, v2.16b, v7.16b + str x0, [x29, #32] + eor v31.16b, v31.16b, v10.16b + str x28, [x29, #24] + eor v28.16b, v28.16b, v12.16b + eor x1, x3, x8 + eor v31.16b, v31.16b, v15.16b + eor x28, x5, x10 + eor v28.16b, v28.16b, v17.16b + eor x1, x1, x13 + eor v31.16b, v31.16b, v20.16b + eor x28, x28, x15 + eor v28.16b, v28.16b, v22.16b + eor x1, x1, x19 + ushr v29.2d, v30.2d, #63 + eor x28, x28, x21 + ushr v26.2d, v28.2d, #63 + eor x1, x1, x24 + sli v29.2d, v30.2d, #1 + eor x28, x28, x26 + sli v26.2d, v28.2d, #1 + eor x0, x0, x1, ror 63 + eor v28.16b, v28.16b, v29.16b + eor x1, x1, x28, ror 63 + eor v29.16b, v3.16b, v8.16b + eor x2, x2, x0 + eor v26.16b, v26.16b, v31.16b + eor x7, x7, x0 + eor v29.16b, v29.16b, v13.16b + eor x12, x12, x0 + eor v29.16b, v29.16b, v18.16b + eor x17, x17, x0 + eor v29.16b, v29.16b, v23.16b + eor x23, x23, x0 + ushr v30.2d, v29.2d, #63 + eor x4, x4, x1 + sli v30.2d, v29.2d, #1 + eor x9, x9, x1 + eor v27.16b, v27.16b, v30.16b + eor x14, x14, x1 + ushr v30.2d, v31.2d, #63 + eor x20, x20, x1 + sli v30.2d, v31.2d, #1 + eor x25, x25, x1 + eor v29.16b, v29.16b, v30.16b + ldr x0, [x29, #32] + # Swap Rotate NEON + eor v0.16b, v0.16b, v25.16b + eor v31.16b, v1.16b, v26.16b + ldr x1, [x29, #24] + eor v6.16b, v6.16b, v26.16b + eor x28, x28, x30, ror 63 + ushr v30.2d, v31.2d, #63 + eor x30, x30, x1, ror 63 + ushr v1.2d, v6.2d, #20 + eor x1, x1, x0, ror 63 + sli v30.2d, v31.2d, #1 + eor x6, x6, x28 + sli v1.2d, v6.2d, #44 + eor x11, x11, x28 + eor v31.16b, v9.16b, v29.16b + eor x16, x16, x28 + eor v22.16b, v22.16b, v27.16b + eor x22, x22, x28 + ushr v6.2d, v31.2d, #44 + eor x27, x27, x28 + ushr v9.2d, v22.2d, #3 + eor x3, x3, x30 + sli v6.2d, v31.2d, #20 + eor x8, x8, x30 + sli v9.2d, v22.2d, #61 + eor x13, x13, x30 + eor v31.16b, v14.16b, v29.16b + eor x19, x19, x30 + eor v20.16b, v20.16b, v25.16b + eor x24, x24, x30 + ushr v22.2d, v31.2d, #25 + eor x5, x5, x1 + ushr v14.2d, v20.2d, #46 + eor x10, x10, x1 + sli v22.2d, v31.2d, #39 + eor x15, x15, x1 + sli v14.2d, v20.2d, #18 + eor x21, x21, x1 + eor v31.16b, v2.16b, v27.16b + eor x26, x26, x1 + # Swap Rotate Base + eor v12.16b, v12.16b, v27.16b + ror x0, x3, #63 + ushr v20.2d, v31.2d, #2 + ror x3, x8, #20 + ushr v2.2d, v12.2d, #21 + ror x8, x11, #44 + sli v20.2d, v31.2d, #62 + ror x11, x25, #3 + sli v2.2d, v12.2d, #43 + ror x25, x16, #25 + eor v31.16b, v13.16b, v28.16b + ror x16, x23, #46 + eor v19.16b, v19.16b, v29.16b + ror x23, x4, #2 + ushr v12.2d, v31.2d, #39 + ror x4, x14, #21 + ushr v13.2d, v19.2d, #56 + ror x14, x15, #39 + sli v12.2d, v31.2d, #25 + ror x15, x22, #56 + sli v13.2d, v19.2d, #8 + ror x22, x26, #8 + eor v31.16b, v23.16b, v28.16b + ror x26, x17, #23 + eor v15.16b, v15.16b, v25.16b + ror x17, x6, #37 + ushr v19.2d, v31.2d, #8 + ror x6, x27, #50 + ushr v23.2d, v15.2d, #23 + ror x27, x24, #62 + sli v19.2d, v31.2d, #56 + ror x24, x10, #9 + sli v23.2d, v15.2d, #41 + ror x10, x19, #19 + eor v31.16b, v4.16b, v29.16b + ror x19, x7, #28 + eor v24.16b, v24.16b, v29.16b + ror x7, x5, #36 + ushr v15.2d, v31.2d, #37 + ror x5, x21, #43 + ushr v4.2d, v24.2d, #50 + ror x21, x20, #49 + sli v15.2d, v31.2d, #27 + ror x20, x13, #54 + sli v4.2d, v24.2d, #14 + ror x13, x9, #58 + eor v31.16b, v21.16b, v26.16b + ror x9, x12, #61 + # Row Mix Base + eor v8.16b, v8.16b, v28.16b + bic x12, x4, x3 + ushr v24.2d, v31.2d, #62 + bic x1, x5, x4 + ushr v21.2d, v8.2d, #9 + bic x28, x2, x6 + sli v24.2d, v31.2d, #2 + bic x30, x3, x2 + sli v21.2d, v8.2d, #55 + eor x2, x2, x12 + eor v31.16b, v16.16b, v26.16b + eor x3, x3, x1 + eor v5.16b, v5.16b, v25.16b + bic x12, x6, x5 + ushr v8.2d, v31.2d, #19 + eor x5, x5, x28 + ushr v16.2d, v5.2d, #28 + eor x4, x4, x12 + sli v8.2d, v31.2d, #45 + eor x6, x6, x30 + sli v16.2d, v5.2d, #36 + bic x12, x9, x8 + eor v31.16b, v3.16b, v28.16b + bic x1, x10, x9 + eor v18.16b, v18.16b, v28.16b + bic x28, x7, x11 + ushr v5.2d, v31.2d, #36 + bic x30, x8, x7 + ushr v3.2d, v18.2d, #43 + eor x7, x7, x12 + sli v5.2d, v31.2d, #28 + eor x8, x8, x1 + sli v3.2d, v18.2d, #21 + bic x12, x11, x10 + eor v31.16b, v17.16b, v27.16b + eor x10, x10, x28 + eor v11.16b, v11.16b, v26.16b + eor x9, x9, x12 + ushr v18.2d, v31.2d, #49 + eor x11, x11, x30 + ushr v17.2d, v11.2d, #54 + bic x12, x14, x13 + sli v18.2d, v31.2d, #15 + bic x1, x15, x14 + sli v17.2d, v11.2d, #10 + bic x28, x0, x16 + eor v31.16b, v7.16b, v27.16b + bic x30, x13, x0 + eor v10.16b, v10.16b, v25.16b + eor x12, x0, x12 + ushr v11.2d, v31.2d, #58 + eor x13, x13, x1 + ushr v7.2d, v10.2d, #61 + bic x0, x16, x15 + sli v11.2d, v31.2d, #6 + eor x15, x15, x28 + sli v7.2d, v10.2d, #3 + eor x14, x14, x0 + # Row Mix NEON + bic v25.16b, v2.16b, v1.16b + eor x16, x16, x30 + bic v26.16b, v3.16b, v2.16b + bic x0, x20, x19 + bic v27.16b, v4.16b, v3.16b + bic x1, x21, x20 + bic v28.16b, v0.16b, v4.16b + bic x28, x17, x22 + bic v29.16b, v1.16b, v0.16b + bic x30, x19, x17 + eor v0.16b, v0.16b, v25.16b + eor x17, x17, x0 + eor v1.16b, v1.16b, v26.16b + eor x19, x19, x1 + eor v2.16b, v2.16b, v27.16b + bic x0, x22, x21 + eor v3.16b, v3.16b, v28.16b + eor x21, x21, x28 + eor v4.16b, v4.16b, v29.16b + eor x20, x20, x0 + bic v25.16b, v7.16b, v6.16b + eor x22, x22, x30 + bic v26.16b, v8.16b, v7.16b + bic x0, x25, x24 + bic v27.16b, v9.16b, v8.16b + bic x1, x26, x25 + bic v28.16b, v5.16b, v9.16b + bic x28, x23, x27 + bic v29.16b, v6.16b, v5.16b + bic x30, x24, x23 + eor v5.16b, v5.16b, v25.16b + eor x23, x23, x0 + eor v6.16b, v6.16b, v26.16b + eor x24, x24, x1 + eor v7.16b, v7.16b, v27.16b + bic x0, x27, x26 + eor v8.16b, v8.16b, v28.16b + eor x26, x26, x28 + eor v9.16b, v9.16b, v29.16b + eor x25, x25, x0 + bic v25.16b, v12.16b, v11.16b + eor x27, x27, x30 + bic v26.16b, v13.16b, v12.16b + bic v27.16b, v14.16b, v13.16b + bic v28.16b, v30.16b, v14.16b + bic v29.16b, v11.16b, v30.16b + eor v10.16b, v30.16b, v25.16b + eor v11.16b, v11.16b, v26.16b + eor v12.16b, v12.16b, v27.16b + eor v13.16b, v13.16b, v28.16b + eor v14.16b, v14.16b, v29.16b + bic v25.16b, v17.16b, v16.16b + bic v26.16b, v18.16b, v17.16b + bic v27.16b, v19.16b, v18.16b + bic v28.16b, v15.16b, v19.16b + bic v29.16b, v16.16b, v15.16b + eor v15.16b, v15.16b, v25.16b + eor v16.16b, v16.16b, v26.16b + eor v17.16b, v17.16b, v27.16b + eor v18.16b, v18.16b, v28.16b + eor v19.16b, v19.16b, v29.16b + bic v25.16b, v22.16b, v21.16b + bic v26.16b, v23.16b, v22.16b + bic v27.16b, v24.16b, v23.16b + bic v28.16b, v20.16b, v24.16b + bic v29.16b, v21.16b, v20.16b + eor v20.16b, v20.16b, v25.16b + eor v21.16b, v21.16b, v26.16b + eor v22.16b, v22.16b, v27.16b + eor v23.16b, v23.16b, v28.16b + eor v24.16b, v24.16b, v29.16b + # Done transforming + ldp x28, x1, [x29, #48] + ldr x0, [x28], #8 + subs x1, x1, #1 + mov v30.d[0], x0 + mov v30.d[1], x0 + eor x2, x2, x0 + eor v0.16b, v0.16b, v30.16b + bne L_SHA3_shake128_blocksx3_seed_neon_begin + ldr x0, [x29, #40] + st4 {v0.d, v1.d, v2.d, v3.d}[0], [x0], #32 + st4 {v4.d, v5.d, v6.d, v7.d}[0], [x0], #32 + st4 {v8.d, v9.d, v10.d, v11.d}[0], [x0], #32 + st4 {v12.d, v13.d, v14.d, v15.d}[0], [x0], #32 + st4 {v16.d, v17.d, v18.d, v19.d}[0], [x0], #32 + st4 {v20.d, v21.d, v22.d, v23.d}[0], [x0], #32 + st1 {v24.d}[0], [x0] + add x0, x0, #8 + st4 {v0.d, v1.d, v2.d, v3.d}[1], [x0], #32 + st4 {v4.d, v5.d, v6.d, v7.d}[1], [x0], #32 + st4 {v8.d, v9.d, v10.d, v11.d}[1], [x0], #32 + st4 {v12.d, v13.d, v14.d, v15.d}[1], [x0], #32 + st4 {v16.d, v17.d, v18.d, v19.d}[1], [x0], #32 + st4 {v20.d, v21.d, v22.d, v23.d}[1], [x0], #32 + st1 {v24.d}[1], [x0] + add x0, x0, #8 + stp x2, x3, [x0] + stp x4, x5, [x0, #16] + stp x6, x7, [x0, #32] + stp x8, x9, [x0, #48] + stp x10, x11, [x0, #64] + stp x12, x13, [x0, #80] + stp x14, x15, [x0, #96] + stp x16, x17, [x0, #112] + stp x19, x20, [x0, #128] + stp x21, x22, [x0, #144] + stp x23, x24, [x0, #160] + stp x25, x26, [x0, #176] + str x27, [x0, #192] + ldp x17, x19, [x29, #72] + ldp x20, x21, [x29, #88] + ldp x22, x23, [x29, #104] + ldp x24, x25, [x29, #120] + ldp x26, x27, [x29, #136] + ldr x28, [x29, #152] + ldp d8, d9, [x29, #160] + ldp d10, d11, [x29, #176] + ldp d12, d13, [x29, #192] + ldp d14, d15, [x29, #208] + ldp x29, x30, [sp], #0xe0 + ret +#ifndef __APPLE__ + .size mlkem_shake128_blocksx3_seed_neon,.-mlkem_shake128_blocksx3_seed_neon +#endif /* __APPLE__ */ +#ifndef __APPLE__ +.text +.globl mlkem_shake256_blocksx3_seed_neon +WC_ASM_ATT_HIDDEN(mlkem_shake256_blocksx3_seed_neon) +.type mlkem_shake256_blocksx3_seed_neon,@function +.align 2 +mlkem_shake256_blocksx3_seed_neon: +#else +.section __TEXT,__text +.globl _mlkem_shake256_blocksx3_seed_neon +WC_ASM_ATT_HIDDEN(_mlkem_shake256_blocksx3_seed_neon) +.p2align 2 +_mlkem_shake256_blocksx3_seed_neon: +#endif /* __APPLE__ */ + stp x29, x30, [sp, #-224]! + add x29, sp, #0 + stp x17, x19, [x29, #72] + stp x20, x21, [x29, #88] + stp x22, x23, [x29, #104] + stp x24, x25, [x29, #120] + stp x26, x27, [x29, #136] + str x28, [x29, #152] + stp d8, d9, [x29, #160] + stp d10, d11, [x29, #176] + stp d12, d13, [x29, #192] + stp d14, d15, [x29, #208] +#ifndef __APPLE__ + adrp x28, L_sha3_aarch64_r + add x28, x28, :lo12:L_sha3_aarch64_r +#else + adrp x28, L_sha3_aarch64_r@PAGE + add x28, x28, L_sha3_aarch64_r@PAGEOFF +#endif /* __APPLE__ */ + str x0, [x29, #40] + add x0, x0, #32 + ld1 {v4.d}[0], [x0] + ldp x2, x3, [x1], #16 + add x0, x0, #0xc8 + ld1 {v4.d}[1], [x0] + ldp x4, x5, [x1], #16 + ldr x6, [x0, #200] + eor v5.16b, v5.16b, v5.16b + eor x7, x7, x7 + eor v6.16b, v6.16b, v6.16b + eor x8, x8, x8 + eor v7.16b, v7.16b, v7.16b + eor x9, x9, x9 + eor v8.16b, v8.16b, v8.16b + eor x10, x10, x10 + eor v9.16b, v9.16b, v9.16b + eor x11, x11, x11 + eor v10.16b, v10.16b, v10.16b + eor x12, x12, x12 + eor v11.16b, v11.16b, v11.16b + eor x13, x13, x13 + eor v12.16b, v12.16b, v12.16b + eor x14, x14, x14 + eor v13.16b, v13.16b, v13.16b + eor x15, x15, x15 + eor v14.16b, v14.16b, v14.16b + eor x16, x16, x16 + eor v15.16b, v15.16b, v15.16b + eor x17, x17, x17 + movz x19, #0x8000, lsl 48 + eor v17.16b, v17.16b, v17.16b + eor x20, x20, x20 + eor v18.16b, v18.16b, v18.16b + eor x21, x21, x21 + eor v19.16b, v19.16b, v19.16b + eor x22, x22, x22 + eor v20.16b, v20.16b, v20.16b + eor x23, x23, x23 + eor v21.16b, v21.16b, v21.16b + eor x24, x24, x24 + eor v22.16b, v22.16b, v22.16b + eor x25, x25, x25 + eor v23.16b, v23.16b, v23.16b + eor x26, x26, x26 + eor v24.16b, v24.16b, v24.16b + eor x27, x27, x27 + dup v0.2d, x2 + dup v1.2d, x3 + dup v2.2d, x4 + dup v3.2d, x5 + dup v16.2d, x19 + mov x1, #24 + # Start of 24 rounds +L_SHA3_shake256_blocksx3_seed_neon_begin: + stp x28, x1, [x29, #48] + # Col Mix NEON + eor v30.16b, v4.16b, v9.16b + eor x0, x6, x11 + eor v27.16b, v1.16b, v6.16b + eor x30, x2, x7 + eor v30.16b, v30.16b, v14.16b + eor x28, x4, x9 + eor v27.16b, v27.16b, v11.16b + eor x0, x0, x16 + eor v30.16b, v30.16b, v19.16b + eor x30, x30, x12 + eor v27.16b, v27.16b, v16.16b + eor x28, x28, x14 + eor v30.16b, v30.16b, v24.16b + eor x0, x0, x22 + eor v27.16b, v27.16b, v21.16b + eor x30, x30, x17 + ushr v25.2d, v27.2d, #63 + eor x28, x28, x20 + sli v25.2d, v27.2d, #1 + eor x0, x0, x27 + eor v25.16b, v25.16b, v30.16b + eor x30, x30, x23 + eor v31.16b, v0.16b, v5.16b + eor x28, x28, x25 + eor v28.16b, v2.16b, v7.16b + str x0, [x29, #32] + eor v31.16b, v31.16b, v10.16b + str x28, [x29, #24] + eor v28.16b, v28.16b, v12.16b + eor x1, x3, x8 + eor v31.16b, v31.16b, v15.16b + eor x28, x5, x10 + eor v28.16b, v28.16b, v17.16b + eor x1, x1, x13 + eor v31.16b, v31.16b, v20.16b + eor x28, x28, x15 + eor v28.16b, v28.16b, v22.16b + eor x1, x1, x19 + ushr v29.2d, v30.2d, #63 + eor x28, x28, x21 + ushr v26.2d, v28.2d, #63 + eor x1, x1, x24 + sli v29.2d, v30.2d, #1 + eor x28, x28, x26 + sli v26.2d, v28.2d, #1 + eor x0, x0, x1, ror 63 + eor v28.16b, v28.16b, v29.16b + eor x1, x1, x28, ror 63 + eor v29.16b, v3.16b, v8.16b + eor x2, x2, x0 + eor v26.16b, v26.16b, v31.16b + eor x7, x7, x0 + eor v29.16b, v29.16b, v13.16b + eor x12, x12, x0 + eor v29.16b, v29.16b, v18.16b + eor x17, x17, x0 + eor v29.16b, v29.16b, v23.16b + eor x23, x23, x0 + ushr v30.2d, v29.2d, #63 + eor x4, x4, x1 + sli v30.2d, v29.2d, #1 + eor x9, x9, x1 + eor v27.16b, v27.16b, v30.16b + eor x14, x14, x1 + ushr v30.2d, v31.2d, #63 + eor x20, x20, x1 + sli v30.2d, v31.2d, #1 + eor x25, x25, x1 + eor v29.16b, v29.16b, v30.16b + ldr x0, [x29, #32] + # Swap Rotate NEON + eor v0.16b, v0.16b, v25.16b + eor v31.16b, v1.16b, v26.16b + ldr x1, [x29, #24] + eor v6.16b, v6.16b, v26.16b + eor x28, x28, x30, ror 63 + ushr v30.2d, v31.2d, #63 + eor x30, x30, x1, ror 63 + ushr v1.2d, v6.2d, #20 + eor x1, x1, x0, ror 63 + sli v30.2d, v31.2d, #1 + eor x6, x6, x28 + sli v1.2d, v6.2d, #44 + eor x11, x11, x28 + eor v31.16b, v9.16b, v29.16b + eor x16, x16, x28 + eor v22.16b, v22.16b, v27.16b + eor x22, x22, x28 + ushr v6.2d, v31.2d, #44 + eor x27, x27, x28 + ushr v9.2d, v22.2d, #3 + eor x3, x3, x30 + sli v6.2d, v31.2d, #20 + eor x8, x8, x30 + sli v9.2d, v22.2d, #61 + eor x13, x13, x30 + eor v31.16b, v14.16b, v29.16b + eor x19, x19, x30 + eor v20.16b, v20.16b, v25.16b + eor x24, x24, x30 + ushr v22.2d, v31.2d, #25 + eor x5, x5, x1 + ushr v14.2d, v20.2d, #46 + eor x10, x10, x1 + sli v22.2d, v31.2d, #39 + eor x15, x15, x1 + sli v14.2d, v20.2d, #18 + eor x21, x21, x1 + eor v31.16b, v2.16b, v27.16b + eor x26, x26, x1 + # Swap Rotate Base + eor v12.16b, v12.16b, v27.16b + ror x0, x3, #63 + ushr v20.2d, v31.2d, #2 + ror x3, x8, #20 + ushr v2.2d, v12.2d, #21 + ror x8, x11, #44 + sli v20.2d, v31.2d, #62 + ror x11, x25, #3 + sli v2.2d, v12.2d, #43 + ror x25, x16, #25 + eor v31.16b, v13.16b, v28.16b + ror x16, x23, #46 + eor v19.16b, v19.16b, v29.16b + ror x23, x4, #2 + ushr v12.2d, v31.2d, #39 + ror x4, x14, #21 + ushr v13.2d, v19.2d, #56 + ror x14, x15, #39 + sli v12.2d, v31.2d, #25 + ror x15, x22, #56 + sli v13.2d, v19.2d, #8 + ror x22, x26, #8 + eor v31.16b, v23.16b, v28.16b + ror x26, x17, #23 + eor v15.16b, v15.16b, v25.16b + ror x17, x6, #37 + ushr v19.2d, v31.2d, #8 + ror x6, x27, #50 + ushr v23.2d, v15.2d, #23 + ror x27, x24, #62 + sli v19.2d, v31.2d, #56 + ror x24, x10, #9 + sli v23.2d, v15.2d, #41 + ror x10, x19, #19 + eor v31.16b, v4.16b, v29.16b + ror x19, x7, #28 + eor v24.16b, v24.16b, v29.16b + ror x7, x5, #36 + ushr v15.2d, v31.2d, #37 + ror x5, x21, #43 + ushr v4.2d, v24.2d, #50 + ror x21, x20, #49 + sli v15.2d, v31.2d, #27 + ror x20, x13, #54 + sli v4.2d, v24.2d, #14 + ror x13, x9, #58 + eor v31.16b, v21.16b, v26.16b + ror x9, x12, #61 + # Row Mix Base + eor v8.16b, v8.16b, v28.16b + bic x12, x4, x3 + ushr v24.2d, v31.2d, #62 + bic x1, x5, x4 + ushr v21.2d, v8.2d, #9 + bic x28, x2, x6 + sli v24.2d, v31.2d, #2 + bic x30, x3, x2 + sli v21.2d, v8.2d, #55 + eor x2, x2, x12 + eor v31.16b, v16.16b, v26.16b + eor x3, x3, x1 + eor v5.16b, v5.16b, v25.16b + bic x12, x6, x5 + ushr v8.2d, v31.2d, #19 + eor x5, x5, x28 + ushr v16.2d, v5.2d, #28 + eor x4, x4, x12 + sli v8.2d, v31.2d, #45 + eor x6, x6, x30 + sli v16.2d, v5.2d, #36 + bic x12, x9, x8 + eor v31.16b, v3.16b, v28.16b + bic x1, x10, x9 + eor v18.16b, v18.16b, v28.16b + bic x28, x7, x11 + ushr v5.2d, v31.2d, #36 + bic x30, x8, x7 + ushr v3.2d, v18.2d, #43 + eor x7, x7, x12 + sli v5.2d, v31.2d, #28 + eor x8, x8, x1 + sli v3.2d, v18.2d, #21 + bic x12, x11, x10 + eor v31.16b, v17.16b, v27.16b + eor x10, x10, x28 + eor v11.16b, v11.16b, v26.16b + eor x9, x9, x12 + ushr v18.2d, v31.2d, #49 + eor x11, x11, x30 + ushr v17.2d, v11.2d, #54 + bic x12, x14, x13 + sli v18.2d, v31.2d, #15 + bic x1, x15, x14 + sli v17.2d, v11.2d, #10 + bic x28, x0, x16 + eor v31.16b, v7.16b, v27.16b + bic x30, x13, x0 + eor v10.16b, v10.16b, v25.16b + eor x12, x0, x12 + ushr v11.2d, v31.2d, #58 + eor x13, x13, x1 + ushr v7.2d, v10.2d, #61 + bic x0, x16, x15 + sli v11.2d, v31.2d, #6 + eor x15, x15, x28 + sli v7.2d, v10.2d, #3 + eor x14, x14, x0 + # Row Mix NEON + bic v25.16b, v2.16b, v1.16b + eor x16, x16, x30 + bic v26.16b, v3.16b, v2.16b + bic x0, x20, x19 + bic v27.16b, v4.16b, v3.16b + bic x1, x21, x20 + bic v28.16b, v0.16b, v4.16b + bic x28, x17, x22 + bic v29.16b, v1.16b, v0.16b + bic x30, x19, x17 + eor v0.16b, v0.16b, v25.16b + eor x17, x17, x0 + eor v1.16b, v1.16b, v26.16b + eor x19, x19, x1 + eor v2.16b, v2.16b, v27.16b + bic x0, x22, x21 + eor v3.16b, v3.16b, v28.16b + eor x21, x21, x28 + eor v4.16b, v4.16b, v29.16b + eor x20, x20, x0 + bic v25.16b, v7.16b, v6.16b + eor x22, x22, x30 + bic v26.16b, v8.16b, v7.16b + bic x0, x25, x24 + bic v27.16b, v9.16b, v8.16b + bic x1, x26, x25 + bic v28.16b, v5.16b, v9.16b + bic x28, x23, x27 + bic v29.16b, v6.16b, v5.16b + bic x30, x24, x23 + eor v5.16b, v5.16b, v25.16b + eor x23, x23, x0 + eor v6.16b, v6.16b, v26.16b + eor x24, x24, x1 + eor v7.16b, v7.16b, v27.16b + bic x0, x27, x26 + eor v8.16b, v8.16b, v28.16b + eor x26, x26, x28 + eor v9.16b, v9.16b, v29.16b + eor x25, x25, x0 + bic v25.16b, v12.16b, v11.16b + eor x27, x27, x30 + bic v26.16b, v13.16b, v12.16b + bic v27.16b, v14.16b, v13.16b + bic v28.16b, v30.16b, v14.16b + bic v29.16b, v11.16b, v30.16b + eor v10.16b, v30.16b, v25.16b + eor v11.16b, v11.16b, v26.16b + eor v12.16b, v12.16b, v27.16b + eor v13.16b, v13.16b, v28.16b + eor v14.16b, v14.16b, v29.16b + bic v25.16b, v17.16b, v16.16b + bic v26.16b, v18.16b, v17.16b + bic v27.16b, v19.16b, v18.16b + bic v28.16b, v15.16b, v19.16b + bic v29.16b, v16.16b, v15.16b + eor v15.16b, v15.16b, v25.16b + eor v16.16b, v16.16b, v26.16b + eor v17.16b, v17.16b, v27.16b + eor v18.16b, v18.16b, v28.16b + eor v19.16b, v19.16b, v29.16b + bic v25.16b, v22.16b, v21.16b + bic v26.16b, v23.16b, v22.16b + bic v27.16b, v24.16b, v23.16b + bic v28.16b, v20.16b, v24.16b + bic v29.16b, v21.16b, v20.16b + eor v20.16b, v20.16b, v25.16b + eor v21.16b, v21.16b, v26.16b + eor v22.16b, v22.16b, v27.16b + eor v23.16b, v23.16b, v28.16b + eor v24.16b, v24.16b, v29.16b + # Done transforming + ldp x28, x1, [x29, #48] + ldr x0, [x28], #8 + subs x1, x1, #1 + mov v30.d[0], x0 + mov v30.d[1], x0 + eor x2, x2, x0 + eor v0.16b, v0.16b, v30.16b + bne L_SHA3_shake256_blocksx3_seed_neon_begin + ldr x0, [x29, #40] + st4 {v0.d, v1.d, v2.d, v3.d}[0], [x0], #32 + st4 {v4.d, v5.d, v6.d, v7.d}[0], [x0], #32 + st4 {v8.d, v9.d, v10.d, v11.d}[0], [x0], #32 + st4 {v12.d, v13.d, v14.d, v15.d}[0], [x0], #32 + st4 {v16.d, v17.d, v18.d, v19.d}[0], [x0], #32 + st4 {v20.d, v21.d, v22.d, v23.d}[0], [x0], #32 + st1 {v24.d}[0], [x0] + add x0, x0, #8 + st4 {v0.d, v1.d, v2.d, v3.d}[1], [x0], #32 + st4 {v4.d, v5.d, v6.d, v7.d}[1], [x0], #32 + st4 {v8.d, v9.d, v10.d, v11.d}[1], [x0], #32 + st4 {v12.d, v13.d, v14.d, v15.d}[1], [x0], #32 + st4 {v16.d, v17.d, v18.d, v19.d}[1], [x0], #32 + st4 {v20.d, v21.d, v22.d, v23.d}[1], [x0], #32 + st1 {v24.d}[1], [x0] + add x0, x0, #8 + stp x2, x3, [x0] + stp x4, x5, [x0, #16] + stp x6, x7, [x0, #32] + stp x8, x9, [x0, #48] + stp x10, x11, [x0, #64] + stp x12, x13, [x0, #80] + stp x14, x15, [x0, #96] + stp x16, x17, [x0, #112] + stp x19, x20, [x0, #128] + stp x21, x22, [x0, #144] + stp x23, x24, [x0, #160] + stp x25, x26, [x0, #176] + str x27, [x0, #192] + ldp x17, x19, [x29, #72] + ldp x20, x21, [x29, #88] + ldp x22, x23, [x29, #104] + ldp x24, x25, [x29, #120] + ldp x26, x27, [x29, #136] + ldr x28, [x29, #152] + ldp d8, d9, [x29, #160] + ldp d10, d11, [x29, #176] + ldp d12, d13, [x29, #192] + ldp d14, d15, [x29, #208] + ldp x29, x30, [sp], #0xe0 + ret +#ifndef __APPLE__ + .size mlkem_shake256_blocksx3_seed_neon,.-mlkem_shake256_blocksx3_seed_neon +#endif /* __APPLE__ */ #endif /* WOLFSSL_HAVE_MLKEM */ #endif /* __aarch64__ */ #endif /* WOLFSSL_ARMASM */ diff --git a/wolfcrypt/src/port/arm/armv8-mlkem-asm.asm b/wolfcrypt/src/port/arm/armv8-mlkem-asm.asm index e7a4700c1f..d72216929d 100644 --- a/wolfcrypt/src/port/arm/armv8-mlkem-asm.asm +++ b/wolfcrypt/src/port/arm/armv8-mlkem-asm.asm @@ -9218,8 +9218,8 @@ L_sha3_aarch64_r IF :DEF:WOLFSSL_ARMASM_CRYPTO_SHA3 AREA |.text|, CODE, READONLY ALIGN 4 - EXPORT mlkem_sha3_blocksx3_neon -mlkem_sha3_blocksx3_neon PROC + EXPORT mlkem_sha3_blocksx3_crypto +mlkem_sha3_blocksx3_crypto PROC stp x29, x30, [sp, #-224]! add x29, sp, #0 stp x17, x19, [x29, #72] @@ -9266,7 +9266,7 @@ mlkem_sha3_blocksx3_neon PROC ldr x26, [x0, #192] mov x28, #24 ; Start of 24 rounds -L_SHA3_transform_blocksx3_neon_begin +L_SHA3_transform_blocksx3_crypto_begin stp x27, x28, [x29, #48] ; Col Mix eor3 V31.16B, V0.16B, V5.16B, V10.16B @@ -9482,7 +9482,7 @@ L_SHA3_transform_blocksx3_neon_begin mov V30.D[1], x0 eor x1, x1, x0 eor V0.16B, V0.16B, V30.16B - bne L_SHA3_transform_blocksx3_neon_begin + bne L_SHA3_transform_blocksx3_crypto_begin ldr x0, [x29, #40] st4 {V0.D, V1.D, V2.D, V3.D}[0], [x0], #32 st4 {V4.D, V5.D, V6.D, V7.D}[0], [x0], #32 @@ -9528,8 +9528,8 @@ L_SHA3_transform_blocksx3_neon_begin ENDP AREA |.text|, CODE, READONLY ALIGN 4 - EXPORT mlkem_shake128_blocksx3_seed_neon -mlkem_shake128_blocksx3_seed_neon PROC + EXPORT mlkem_shake128_blocksx3_seed_crypto +mlkem_shake128_blocksx3_seed_crypto PROC stp x29, x30, [sp, #-224]! add x29, sp, #0 stp x17, x19, [x29, #72] @@ -9598,7 +9598,7 @@ mlkem_shake128_blocksx3_seed_neon PROC dup V20.2D, x23 mov x1, #24 ; Start of 24 rounds -L_SHA3_shake128_blocksx3_seed_neon_begin +L_SHA3_shake128_blocksx3_seed_crypto_begin stp x28, x1, [x29, #48] ; Col Mix eor3 V31.16B, V0.16B, V5.16B, V10.16B @@ -9814,7 +9814,7 @@ L_SHA3_shake128_blocksx3_seed_neon_begin mov V30.D[1], x0 eor x2, x2, x0 eor V0.16B, V0.16B, V30.16B - bne L_SHA3_shake128_blocksx3_seed_neon_begin + bne L_SHA3_shake128_blocksx3_seed_crypto_begin ldr x0, [x29, #40] st4 {V0.D, V1.D, V2.D, V3.D}[0], [x0], #32 st4 {V4.D, V5.D, V6.D, V7.D}[0], [x0], #32 @@ -9860,8 +9860,8 @@ L_SHA3_shake128_blocksx3_seed_neon_begin ENDP AREA |.text|, CODE, READONLY ALIGN 4 - EXPORT mlkem_shake256_blocksx3_seed_neon -mlkem_shake256_blocksx3_seed_neon PROC + EXPORT mlkem_shake256_blocksx3_seed_crypto +mlkem_shake256_blocksx3_seed_crypto PROC stp x29, x30, [sp, #-224]! add x29, sp, #0 stp x17, x19, [x29, #72] @@ -9930,7 +9930,7 @@ mlkem_shake256_blocksx3_seed_neon PROC dup V16.2D, x19 mov x1, #24 ; Start of 24 rounds -L_SHA3_shake256_blocksx3_seed_neon_begin +L_SHA3_shake256_blocksx3_seed_crypto_begin stp x28, x1, [x29, #48] ; Col Mix eor3 V31.16B, V0.16B, V5.16B, V10.16B @@ -10146,1237 +10146,7 @@ L_SHA3_shake256_blocksx3_seed_neon_begin mov V30.D[1], x0 eor x2, x2, x0 eor V0.16B, V0.16B, V30.16B - bne L_SHA3_shake256_blocksx3_seed_neon_begin - ldr x0, [x29, #40] - st4 {V0.D, V1.D, V2.D, V3.D}[0], [x0], #32 - st4 {V4.D, V5.D, V6.D, V7.D}[0], [x0], #32 - st4 {V8.D, V9.D, V10.D, V11.D}[0], [x0], #32 - st4 {V12.D, V13.D, V14.D, V15.D}[0], [x0], #32 - st4 {V16.D, V17.D, V18.D, V19.D}[0], [x0], #32 - st4 {V20.D, V21.D, V22.D, V23.D}[0], [x0], #32 - st1 {V24.D}[0], [x0] - add x0, x0, #8 - st4 {V0.D, V1.D, V2.D, V3.D}[1], [x0], #32 - st4 {V4.D, V5.D, V6.D, V7.D}[1], [x0], #32 - st4 {V8.D, V9.D, V10.D, V11.D}[1], [x0], #32 - st4 {V12.D, V13.D, V14.D, V15.D}[1], [x0], #32 - st4 {V16.D, V17.D, V18.D, V19.D}[1], [x0], #32 - st4 {V20.D, V21.D, V22.D, V23.D}[1], [x0], #32 - st1 {V24.D}[1], [x0] - add x0, x0, #8 - stp x2, x3, [x0] - stp x4, x5, [x0, #16] - stp x6, x7, [x0, #32] - stp x8, x9, [x0, #48] - stp x10, x11, [x0, #64] - stp x12, x13, [x0, #80] - stp x14, x15, [x0, #96] - stp x16, x17, [x0, #112] - stp x19, x20, [x0, #128] - stp x21, x22, [x0, #144] - stp x23, x24, [x0, #160] - stp x25, x26, [x0, #176] - str x27, [x0, #192] - ldp x17, x19, [x29, #72] - ldp x20, x21, [x29, #88] - ldp x22, x23, [x29, #104] - ldp x24, x25, [x29, #120] - ldp x26, x27, [x29, #136] - ldr x28, [x29, #152] - ldp D8, D9, [x29, #160] - ldp D10, D11, [x29, #176] - ldp D12, D13, [x29, #192] - ldp D14, D15, [x29, #208] - ldp x29, x30, [sp], #0xe0 - ret - ENDP - ELSE - AREA |.text|, CODE, READONLY - ALIGN 4 - EXPORT mlkem_sha3_blocksx3_neon -mlkem_sha3_blocksx3_neon PROC - stp x29, x30, [sp, #-224]! - add x29, sp, #0 - stp x17, x19, [x29, #72] - stp x20, x21, [x29, #88] - stp x22, x23, [x29, #104] - stp x24, x25, [x29, #120] - stp x26, x27, [x29, #136] - str x28, [x29, #152] - stp D8, D9, [x29, #160] - stp D10, D11, [x29, #176] - stp D12, D13, [x29, #192] - stp D14, D15, [x29, #208] - adrp x27, L_sha3_aarch64_r - add x27, x27, L_sha3_aarch64_r - str x0, [x29, #40] - ld4 {V0.D, V1.D, V2.D, V3.D}[0], [x0], #32 - ld4 {V4.D, V5.D, V6.D, V7.D}[0], [x0], #32 - ld4 {V8.D, V9.D, V10.D, V11.D}[0], [x0], #32 - ld4 {V12.D, V13.D, V14.D, V15.D}[0], [x0], #32 - ld4 {V16.D, V17.D, V18.D, V19.D}[0], [x0], #32 - ld4 {V20.D, V21.D, V22.D, V23.D}[0], [x0], #32 - ld1 {V24.D}[0], [x0] - add x0, x0, #8 - ld4 {V0.D, V1.D, V2.D, V3.D}[1], [x0], #32 - ld4 {V4.D, V5.D, V6.D, V7.D}[1], [x0], #32 - ld4 {V8.D, V9.D, V10.D, V11.D}[1], [x0], #32 - ld4 {V12.D, V13.D, V14.D, V15.D}[1], [x0], #32 - ld4 {V16.D, V17.D, V18.D, V19.D}[1], [x0], #32 - ld4 {V20.D, V21.D, V22.D, V23.D}[1], [x0], #32 - ld1 {V24.D}[1], [x0] - add x0, x0, #8 - ldp x1, x2, [x0] - ldp x3, x4, [x0, #16] - ldp x5, x6, [x0, #32] - ldp x7, x8, [x0, #48] - ldp x9, x10, [x0, #64] - ldp x11, x12, [x0, #80] - ldp x13, x14, [x0, #96] - ldp x15, x16, [x0, #112] - ldp x17, x19, [x0, #128] - ldp x20, x21, [x0, #144] - ldp x22, x23, [x0, #160] - ldp x24, x25, [x0, #176] - ldr x26, [x0, #192] - mov x28, #24 - ; Start of 24 rounds -L_SHA3_transform_blocksx3_neon_begin - stp x27, x28, [x29, #48] - ; Col Mix NEON - eor V30.16B, V4.16B, V9.16B - eor x0, x5, x10 - eor V27.16B, V1.16B, V6.16B - eor x30, x1, x6 - eor V30.16B, V30.16B, V14.16B - eor x28, x3, x8 - eor V27.16B, V27.16B, V11.16B - eor x0, x0, x15 - eor V30.16B, V30.16B, V19.16B - eor x30, x30, x11 - eor V27.16B, V27.16B, V16.16B - eor x28, x28, x13 - eor V30.16B, V30.16B, V24.16B - eor x0, x0, x21 - eor V27.16B, V27.16B, V21.16B - eor x30, x30, x16 - ushr V25.2D, V27.2D, #63 - eor x28, x28, x19 - sli V25.2D, V27.2D, #1 - eor x0, x0, x26 - eor V25.16B, V25.16B, V30.16B - eor x30, x30, x22 - eor V31.16B, V0.16B, V5.16B - eor x28, x28, x24 - eor V28.16B, V2.16B, V7.16B - str x0, [x29, #32] - eor V31.16B, V31.16B, V10.16B - str x28, [x29, #24] - eor V28.16B, V28.16B, V12.16B - eor x27, x2, x7 - eor V31.16B, V31.16B, V15.16B - eor x28, x4, x9 - eor V28.16B, V28.16B, V17.16B - eor x27, x27, x12 - eor V31.16B, V31.16B, V20.16B - eor x28, x28, x14 - eor V28.16B, V28.16B, V22.16B - eor x27, x27, x17 - ushr V29.2D, V30.2D, #63 - eor x28, x28, x20 - ushr V26.2D, V28.2D, #63 - eor x27, x27, x23 - sli V29.2D, V30.2D, #1 - eor x28, x28, x25 - sli V26.2D, V28.2D, #1 - eor x0, x0, x27, ror 63 - eor V28.16B, V28.16B, V29.16B - eor x27, x27, x28, ror 63 - eor V29.16B, V3.16B, V8.16B - eor x1, x1, x0 - eor V26.16B, V26.16B, V31.16B - eor x6, x6, x0 - eor V29.16B, V29.16B, V13.16B - eor x11, x11, x0 - eor V29.16B, V29.16B, V18.16B - eor x16, x16, x0 - eor V29.16B, V29.16B, V23.16B - eor x22, x22, x0 - ushr V30.2D, V29.2D, #63 - eor x3, x3, x27 - sli V30.2D, V29.2D, #1 - eor x8, x8, x27 - eor V27.16B, V27.16B, V30.16B - eor x13, x13, x27 - ushr V30.2D, V31.2D, #63 - eor x19, x19, x27 - sli V30.2D, V31.2D, #1 - eor x24, x24, x27 - eor V29.16B, V29.16B, V30.16B - ldr x0, [x29, #32] - ; Swap Rotate NEON - eor V0.16B, V0.16B, V25.16B - eor V31.16B, V1.16B, V26.16B - ldr x27, [x29, #24] - eor V6.16B, V6.16B, V26.16B - eor x28, x28, x30, ror 63 - ushr V30.2D, V31.2D, #63 - eor x30, x30, x27, ror 63 - ushr V1.2D, V6.2D, #20 - eor x27, x27, x0, ror 63 - sli V30.2D, V31.2D, #1 - eor x5, x5, x28 - sli V1.2D, V6.2D, #44 - eor x10, x10, x28 - eor V31.16B, V9.16B, V29.16B - eor x15, x15, x28 - eor V22.16B, V22.16B, V27.16B - eor x21, x21, x28 - ushr V6.2D, V31.2D, #44 - eor x26, x26, x28 - ushr V9.2D, V22.2D, #3 - eor x2, x2, x30 - sli V6.2D, V31.2D, #20 - eor x7, x7, x30 - sli V9.2D, V22.2D, #61 - eor x12, x12, x30 - eor V31.16B, V14.16B, V29.16B - eor x17, x17, x30 - eor V20.16B, V20.16B, V25.16B - eor x23, x23, x30 - ushr V22.2D, V31.2D, #25 - eor x4, x4, x27 - ushr V14.2D, V20.2D, #46 - eor x9, x9, x27 - sli V22.2D, V31.2D, #39 - eor x14, x14, x27 - sli V14.2D, V20.2D, #18 - eor x20, x20, x27 - eor V31.16B, V2.16B, V27.16B - eor x25, x25, x27 - ; Swap Rotate Base - eor V12.16B, V12.16B, V27.16B - ror x0, x2, #63 - ushr V20.2D, V31.2D, #2 - ror x2, x7, #20 - ushr V2.2D, V12.2D, #21 - ror x7, x10, #44 - sli V20.2D, V31.2D, #62 - ror x10, x24, #3 - sli V2.2D, V12.2D, #43 - ror x24, x15, #25 - eor V31.16B, V13.16B, V28.16B - ror x15, x22, #46 - eor V19.16B, V19.16B, V29.16B - ror x22, x3, #2 - ushr V12.2D, V31.2D, #39 - ror x3, x13, #21 - ushr V13.2D, V19.2D, #56 - ror x13, x14, #39 - sli V12.2D, V31.2D, #25 - ror x14, x21, #56 - sli V13.2D, V19.2D, #8 - ror x21, x25, #8 - eor V31.16B, V23.16B, V28.16B - ror x25, x16, #23 - eor V15.16B, V15.16B, V25.16B - ror x16, x5, #37 - ushr V19.2D, V31.2D, #8 - ror x5, x26, #50 - ushr V23.2D, V15.2D, #23 - ror x26, x23, #62 - sli V19.2D, V31.2D, #56 - ror x23, x9, #9 - sli V23.2D, V15.2D, #41 - ror x9, x17, #19 - eor V31.16B, V4.16B, V29.16B - ror x17, x6, #28 - eor V24.16B, V24.16B, V29.16B - ror x6, x4, #36 - ushr V15.2D, V31.2D, #37 - ror x4, x20, #43 - ushr V4.2D, V24.2D, #50 - ror x20, x19, #49 - sli V15.2D, V31.2D, #27 - ror x19, x12, #54 - sli V4.2D, V24.2D, #14 - ror x12, x8, #58 - eor V31.16B, V21.16B, V26.16B - ror x8, x11, #61 - ; Row Mix Base - eor V8.16B, V8.16B, V28.16B - bic x11, x3, x2 - ushr V24.2D, V31.2D, #62 - bic x27, x4, x3 - ushr V21.2D, V8.2D, #9 - bic x28, x1, x5 - sli V24.2D, V31.2D, #2 - bic x30, x2, x1 - sli V21.2D, V8.2D, #55 - eor x1, x1, x11 - eor V31.16B, V16.16B, V26.16B - eor x2, x2, x27 - eor V5.16B, V5.16B, V25.16B - bic x11, x5, x4 - ushr V8.2D, V31.2D, #19 - eor x4, x4, x28 - ushr V16.2D, V5.2D, #28 - eor x3, x3, x11 - sli V8.2D, V31.2D, #45 - eor x5, x5, x30 - sli V16.2D, V5.2D, #36 - bic x11, x8, x7 - eor V31.16B, V3.16B, V28.16B - bic x27, x9, x8 - eor V18.16B, V18.16B, V28.16B - bic x28, x6, x10 - ushr V5.2D, V31.2D, #36 - bic x30, x7, x6 - ushr V3.2D, V18.2D, #43 - eor x6, x6, x11 - sli V5.2D, V31.2D, #28 - eor x7, x7, x27 - sli V3.2D, V18.2D, #21 - bic x11, x10, x9 - eor V31.16B, V17.16B, V27.16B - eor x9, x9, x28 - eor V11.16B, V11.16B, V26.16B - eor x8, x8, x11 - ushr V18.2D, V31.2D, #49 - eor x10, x10, x30 - ushr V17.2D, V11.2D, #54 - bic x11, x13, x12 - sli V18.2D, V31.2D, #15 - bic x27, x14, x13 - sli V17.2D, V11.2D, #10 - bic x28, x0, x15 - eor V31.16B, V7.16B, V27.16B - bic x30, x12, x0 - eor V10.16B, V10.16B, V25.16B - eor x11, x0, x11 - ushr V11.2D, V31.2D, #58 - eor x12, x12, x27 - ushr V7.2D, V10.2D, #61 - bic x0, x15, x14 - sli V11.2D, V31.2D, #6 - eor x14, x14, x28 - sli V7.2D, V10.2D, #3 - eor x13, x13, x0 - ; Row Mix NEON - bic V25.16B, V2.16B, V1.16B - eor x15, x15, x30 - bic V26.16B, V3.16B, V2.16B - bic x0, x19, x17 - bic V27.16B, V4.16B, V3.16B - bic x27, x20, x19 - bic V28.16B, V0.16B, V4.16B - bic x28, x16, x21 - bic V29.16B, V1.16B, V0.16B - bic x30, x17, x16 - eor V0.16B, V0.16B, V25.16B - eor x16, x16, x0 - eor V1.16B, V1.16B, V26.16B - eor x17, x17, x27 - eor V2.16B, V2.16B, V27.16B - bic x0, x21, x20 - eor V3.16B, V3.16B, V28.16B - eor x20, x20, x28 - eor V4.16B, V4.16B, V29.16B - eor x19, x19, x0 - bic V25.16B, V7.16B, V6.16B - eor x21, x21, x30 - bic V26.16B, V8.16B, V7.16B - bic x0, x24, x23 - bic V27.16B, V9.16B, V8.16B - bic x27, x25, x24 - bic V28.16B, V5.16B, V9.16B - bic x28, x22, x26 - bic V29.16B, V6.16B, V5.16B - bic x30, x23, x22 - eor V5.16B, V5.16B, V25.16B - eor x22, x22, x0 - eor V6.16B, V6.16B, V26.16B - eor x23, x23, x27 - eor V7.16B, V7.16B, V27.16B - bic x0, x26, x25 - eor V8.16B, V8.16B, V28.16B - eor x25, x25, x28 - eor V9.16B, V9.16B, V29.16B - eor x24, x24, x0 - bic V25.16B, V12.16B, V11.16B - eor x26, x26, x30 - bic V26.16B, V13.16B, V12.16B - bic V27.16B, V14.16B, V13.16B - bic V28.16B, V30.16B, V14.16B - bic V29.16B, V11.16B, V30.16B - eor V10.16B, V30.16B, V25.16B - eor V11.16B, V11.16B, V26.16B - eor V12.16B, V12.16B, V27.16B - eor V13.16B, V13.16B, V28.16B - eor V14.16B, V14.16B, V29.16B - bic V25.16B, V17.16B, V16.16B - bic V26.16B, V18.16B, V17.16B - bic V27.16B, V19.16B, V18.16B - bic V28.16B, V15.16B, V19.16B - bic V29.16B, V16.16B, V15.16B - eor V15.16B, V15.16B, V25.16B - eor V16.16B, V16.16B, V26.16B - eor V17.16B, V17.16B, V27.16B - eor V18.16B, V18.16B, V28.16B - eor V19.16B, V19.16B, V29.16B - bic V25.16B, V22.16B, V21.16B - bic V26.16B, V23.16B, V22.16B - bic V27.16B, V24.16B, V23.16B - bic V28.16B, V20.16B, V24.16B - bic V29.16B, V21.16B, V20.16B - eor V20.16B, V20.16B, V25.16B - eor V21.16B, V21.16B, V26.16B - eor V22.16B, V22.16B, V27.16B - eor V23.16B, V23.16B, V28.16B - eor V24.16B, V24.16B, V29.16B - ; Done transforming - ldp x27, x28, [x29, #48] - ldr x0, [x27], #8 - subs x28, x28, #1 - mov V30.D[0], x0 - mov V30.D[1], x0 - eor x1, x1, x0 - eor V0.16B, V0.16B, V30.16B - bne L_SHA3_transform_blocksx3_neon_begin - ldr x0, [x29, #40] - st4 {V0.D, V1.D, V2.D, V3.D}[0], [x0], #32 - st4 {V4.D, V5.D, V6.D, V7.D}[0], [x0], #32 - st4 {V8.D, V9.D, V10.D, V11.D}[0], [x0], #32 - st4 {V12.D, V13.D, V14.D, V15.D}[0], [x0], #32 - st4 {V16.D, V17.D, V18.D, V19.D}[0], [x0], #32 - st4 {V20.D, V21.D, V22.D, V23.D}[0], [x0], #32 - st1 {V24.D}[0], [x0] - add x0, x0, #8 - st4 {V0.D, V1.D, V2.D, V3.D}[1], [x0], #32 - st4 {V4.D, V5.D, V6.D, V7.D}[1], [x0], #32 - st4 {V8.D, V9.D, V10.D, V11.D}[1], [x0], #32 - st4 {V12.D, V13.D, V14.D, V15.D}[1], [x0], #32 - st4 {V16.D, V17.D, V18.D, V19.D}[1], [x0], #32 - st4 {V20.D, V21.D, V22.D, V23.D}[1], [x0], #32 - st1 {V24.D}[1], [x0] - add x0, x0, #8 - stp x1, x2, [x0] - stp x3, x4, [x0, #16] - stp x5, x6, [x0, #32] - stp x7, x8, [x0, #48] - stp x9, x10, [x0, #64] - stp x11, x12, [x0, #80] - stp x13, x14, [x0, #96] - stp x15, x16, [x0, #112] - stp x17, x19, [x0, #128] - stp x20, x21, [x0, #144] - stp x22, x23, [x0, #160] - stp x24, x25, [x0, #176] - str x26, [x0, #192] - ldp x17, x19, [x29, #72] - ldp x20, x21, [x29, #88] - ldp x22, x23, [x29, #104] - ldp x24, x25, [x29, #120] - ldp x26, x27, [x29, #136] - ldr x28, [x29, #152] - ldp D8, D9, [x29, #160] - ldp D10, D11, [x29, #176] - ldp D12, D13, [x29, #192] - ldp D14, D15, [x29, #208] - ldp x29, x30, [sp], #0xe0 - ret - ENDP - AREA |.text|, CODE, READONLY - ALIGN 4 - EXPORT mlkem_shake128_blocksx3_seed_neon -mlkem_shake128_blocksx3_seed_neon PROC - stp x29, x30, [sp, #-224]! - add x29, sp, #0 - stp x17, x19, [x29, #72] - stp x20, x21, [x29, #88] - stp x22, x23, [x29, #104] - stp x24, x25, [x29, #120] - stp x26, x27, [x29, #136] - str x28, [x29, #152] - stp D8, D9, [x29, #160] - stp D10, D11, [x29, #176] - stp D12, D13, [x29, #192] - stp D14, D15, [x29, #208] - adrp x28, L_sha3_aarch64_r - add x28, x28, L_sha3_aarch64_r - str x0, [x29, #40] - add x0, x0, #32 - ld1 {V4.D}[0], [x0] - ldp x2, x3, [x1], #16 - add x0, x0, #0xc8 - ld1 {V4.D}[1], [x0] - ldp x4, x5, [x1], #16 - ldr x6, [x0, #200] - eor V5.16B, V5.16B, V5.16B - eor x7, x7, x7 - eor V6.16B, V6.16B, V6.16B - eor x8, x8, x8 - eor V7.16B, V7.16B, V7.16B - eor x9, x9, x9 - eor V8.16B, V8.16B, V8.16B - eor x10, x10, x10 - eor V9.16B, V9.16B, V9.16B - eor x11, x11, x11 - eor V10.16B, V10.16B, V10.16B - eor x12, x12, x12 - eor V11.16B, V11.16B, V11.16B - eor x13, x13, x13 - eor V12.16B, V12.16B, V12.16B - eor x14, x14, x14 - eor V13.16B, V13.16B, V13.16B - eor x15, x15, x15 - eor V14.16B, V14.16B, V14.16B - eor x16, x16, x16 - eor V15.16B, V15.16B, V15.16B - eor x17, x17, x17 - eor V16.16B, V16.16B, V16.16B - eor x19, x19, x19 - eor V17.16B, V17.16B, V17.16B - eor x20, x20, x20 - eor V18.16B, V18.16B, V18.16B - eor x21, x21, x21 - eor V19.16B, V19.16B, V19.16B - eor x22, x22, x22 - movz x23, #0x8000, lsl 48 - eor V21.16B, V21.16B, V21.16B - eor x24, x24, x24 - eor V22.16B, V22.16B, V22.16B - eor x25, x25, x25 - eor V23.16B, V23.16B, V23.16B - eor x26, x26, x26 - eor V24.16B, V24.16B, V24.16B - eor x27, x27, x27 - dup V0.2D, x2 - dup V1.2D, x3 - dup V2.2D, x4 - dup V3.2D, x5 - dup V20.2D, x23 - mov x1, #24 - ; Start of 24 rounds -L_SHA3_shake128_blocksx3_seed_neon_begin - stp x28, x1, [x29, #48] - ; Col Mix NEON - eor V30.16B, V4.16B, V9.16B - eor x0, x6, x11 - eor V27.16B, V1.16B, V6.16B - eor x30, x2, x7 - eor V30.16B, V30.16B, V14.16B - eor x28, x4, x9 - eor V27.16B, V27.16B, V11.16B - eor x0, x0, x16 - eor V30.16B, V30.16B, V19.16B - eor x30, x30, x12 - eor V27.16B, V27.16B, V16.16B - eor x28, x28, x14 - eor V30.16B, V30.16B, V24.16B - eor x0, x0, x22 - eor V27.16B, V27.16B, V21.16B - eor x30, x30, x17 - ushr V25.2D, V27.2D, #63 - eor x28, x28, x20 - sli V25.2D, V27.2D, #1 - eor x0, x0, x27 - eor V25.16B, V25.16B, V30.16B - eor x30, x30, x23 - eor V31.16B, V0.16B, V5.16B - eor x28, x28, x25 - eor V28.16B, V2.16B, V7.16B - str x0, [x29, #32] - eor V31.16B, V31.16B, V10.16B - str x28, [x29, #24] - eor V28.16B, V28.16B, V12.16B - eor x1, x3, x8 - eor V31.16B, V31.16B, V15.16B - eor x28, x5, x10 - eor V28.16B, V28.16B, V17.16B - eor x1, x1, x13 - eor V31.16B, V31.16B, V20.16B - eor x28, x28, x15 - eor V28.16B, V28.16B, V22.16B - eor x1, x1, x19 - ushr V29.2D, V30.2D, #63 - eor x28, x28, x21 - ushr V26.2D, V28.2D, #63 - eor x1, x1, x24 - sli V29.2D, V30.2D, #1 - eor x28, x28, x26 - sli V26.2D, V28.2D, #1 - eor x0, x0, x1, ror 63 - eor V28.16B, V28.16B, V29.16B - eor x1, x1, x28, ror 63 - eor V29.16B, V3.16B, V8.16B - eor x2, x2, x0 - eor V26.16B, V26.16B, V31.16B - eor x7, x7, x0 - eor V29.16B, V29.16B, V13.16B - eor x12, x12, x0 - eor V29.16B, V29.16B, V18.16B - eor x17, x17, x0 - eor V29.16B, V29.16B, V23.16B - eor x23, x23, x0 - ushr V30.2D, V29.2D, #63 - eor x4, x4, x1 - sli V30.2D, V29.2D, #1 - eor x9, x9, x1 - eor V27.16B, V27.16B, V30.16B - eor x14, x14, x1 - ushr V30.2D, V31.2D, #63 - eor x20, x20, x1 - sli V30.2D, V31.2D, #1 - eor x25, x25, x1 - eor V29.16B, V29.16B, V30.16B - ldr x0, [x29, #32] - ; Swap Rotate NEON - eor V0.16B, V0.16B, V25.16B - eor V31.16B, V1.16B, V26.16B - ldr x1, [x29, #24] - eor V6.16B, V6.16B, V26.16B - eor x28, x28, x30, ror 63 - ushr V30.2D, V31.2D, #63 - eor x30, x30, x1, ror 63 - ushr V1.2D, V6.2D, #20 - eor x1, x1, x0, ror 63 - sli V30.2D, V31.2D, #1 - eor x6, x6, x28 - sli V1.2D, V6.2D, #44 - eor x11, x11, x28 - eor V31.16B, V9.16B, V29.16B - eor x16, x16, x28 - eor V22.16B, V22.16B, V27.16B - eor x22, x22, x28 - ushr V6.2D, V31.2D, #44 - eor x27, x27, x28 - ushr V9.2D, V22.2D, #3 - eor x3, x3, x30 - sli V6.2D, V31.2D, #20 - eor x8, x8, x30 - sli V9.2D, V22.2D, #61 - eor x13, x13, x30 - eor V31.16B, V14.16B, V29.16B - eor x19, x19, x30 - eor V20.16B, V20.16B, V25.16B - eor x24, x24, x30 - ushr V22.2D, V31.2D, #25 - eor x5, x5, x1 - ushr V14.2D, V20.2D, #46 - eor x10, x10, x1 - sli V22.2D, V31.2D, #39 - eor x15, x15, x1 - sli V14.2D, V20.2D, #18 - eor x21, x21, x1 - eor V31.16B, V2.16B, V27.16B - eor x26, x26, x1 - ; Swap Rotate Base - eor V12.16B, V12.16B, V27.16B - ror x0, x3, #63 - ushr V20.2D, V31.2D, #2 - ror x3, x8, #20 - ushr V2.2D, V12.2D, #21 - ror x8, x11, #44 - sli V20.2D, V31.2D, #62 - ror x11, x25, #3 - sli V2.2D, V12.2D, #43 - ror x25, x16, #25 - eor V31.16B, V13.16B, V28.16B - ror x16, x23, #46 - eor V19.16B, V19.16B, V29.16B - ror x23, x4, #2 - ushr V12.2D, V31.2D, #39 - ror x4, x14, #21 - ushr V13.2D, V19.2D, #56 - ror x14, x15, #39 - sli V12.2D, V31.2D, #25 - ror x15, x22, #56 - sli V13.2D, V19.2D, #8 - ror x22, x26, #8 - eor V31.16B, V23.16B, V28.16B - ror x26, x17, #23 - eor V15.16B, V15.16B, V25.16B - ror x17, x6, #37 - ushr V19.2D, V31.2D, #8 - ror x6, x27, #50 - ushr V23.2D, V15.2D, #23 - ror x27, x24, #62 - sli V19.2D, V31.2D, #56 - ror x24, x10, #9 - sli V23.2D, V15.2D, #41 - ror x10, x19, #19 - eor V31.16B, V4.16B, V29.16B - ror x19, x7, #28 - eor V24.16B, V24.16B, V29.16B - ror x7, x5, #36 - ushr V15.2D, V31.2D, #37 - ror x5, x21, #43 - ushr V4.2D, V24.2D, #50 - ror x21, x20, #49 - sli V15.2D, V31.2D, #27 - ror x20, x13, #54 - sli V4.2D, V24.2D, #14 - ror x13, x9, #58 - eor V31.16B, V21.16B, V26.16B - ror x9, x12, #61 - ; Row Mix Base - eor V8.16B, V8.16B, V28.16B - bic x12, x4, x3 - ushr V24.2D, V31.2D, #62 - bic x1, x5, x4 - ushr V21.2D, V8.2D, #9 - bic x28, x2, x6 - sli V24.2D, V31.2D, #2 - bic x30, x3, x2 - sli V21.2D, V8.2D, #55 - eor x2, x2, x12 - eor V31.16B, V16.16B, V26.16B - eor x3, x3, x1 - eor V5.16B, V5.16B, V25.16B - bic x12, x6, x5 - ushr V8.2D, V31.2D, #19 - eor x5, x5, x28 - ushr V16.2D, V5.2D, #28 - eor x4, x4, x12 - sli V8.2D, V31.2D, #45 - eor x6, x6, x30 - sli V16.2D, V5.2D, #36 - bic x12, x9, x8 - eor V31.16B, V3.16B, V28.16B - bic x1, x10, x9 - eor V18.16B, V18.16B, V28.16B - bic x28, x7, x11 - ushr V5.2D, V31.2D, #36 - bic x30, x8, x7 - ushr V3.2D, V18.2D, #43 - eor x7, x7, x12 - sli V5.2D, V31.2D, #28 - eor x8, x8, x1 - sli V3.2D, V18.2D, #21 - bic x12, x11, x10 - eor V31.16B, V17.16B, V27.16B - eor x10, x10, x28 - eor V11.16B, V11.16B, V26.16B - eor x9, x9, x12 - ushr V18.2D, V31.2D, #49 - eor x11, x11, x30 - ushr V17.2D, V11.2D, #54 - bic x12, x14, x13 - sli V18.2D, V31.2D, #15 - bic x1, x15, x14 - sli V17.2D, V11.2D, #10 - bic x28, x0, x16 - eor V31.16B, V7.16B, V27.16B - bic x30, x13, x0 - eor V10.16B, V10.16B, V25.16B - eor x12, x0, x12 - ushr V11.2D, V31.2D, #58 - eor x13, x13, x1 - ushr V7.2D, V10.2D, #61 - bic x0, x16, x15 - sli V11.2D, V31.2D, #6 - eor x15, x15, x28 - sli V7.2D, V10.2D, #3 - eor x14, x14, x0 - ; Row Mix NEON - bic V25.16B, V2.16B, V1.16B - eor x16, x16, x30 - bic V26.16B, V3.16B, V2.16B - bic x0, x20, x19 - bic V27.16B, V4.16B, V3.16B - bic x1, x21, x20 - bic V28.16B, V0.16B, V4.16B - bic x28, x17, x22 - bic V29.16B, V1.16B, V0.16B - bic x30, x19, x17 - eor V0.16B, V0.16B, V25.16B - eor x17, x17, x0 - eor V1.16B, V1.16B, V26.16B - eor x19, x19, x1 - eor V2.16B, V2.16B, V27.16B - bic x0, x22, x21 - eor V3.16B, V3.16B, V28.16B - eor x21, x21, x28 - eor V4.16B, V4.16B, V29.16B - eor x20, x20, x0 - bic V25.16B, V7.16B, V6.16B - eor x22, x22, x30 - bic V26.16B, V8.16B, V7.16B - bic x0, x25, x24 - bic V27.16B, V9.16B, V8.16B - bic x1, x26, x25 - bic V28.16B, V5.16B, V9.16B - bic x28, x23, x27 - bic V29.16B, V6.16B, V5.16B - bic x30, x24, x23 - eor V5.16B, V5.16B, V25.16B - eor x23, x23, x0 - eor V6.16B, V6.16B, V26.16B - eor x24, x24, x1 - eor V7.16B, V7.16B, V27.16B - bic x0, x27, x26 - eor V8.16B, V8.16B, V28.16B - eor x26, x26, x28 - eor V9.16B, V9.16B, V29.16B - eor x25, x25, x0 - bic V25.16B, V12.16B, V11.16B - eor x27, x27, x30 - bic V26.16B, V13.16B, V12.16B - bic V27.16B, V14.16B, V13.16B - bic V28.16B, V30.16B, V14.16B - bic V29.16B, V11.16B, V30.16B - eor V10.16B, V30.16B, V25.16B - eor V11.16B, V11.16B, V26.16B - eor V12.16B, V12.16B, V27.16B - eor V13.16B, V13.16B, V28.16B - eor V14.16B, V14.16B, V29.16B - bic V25.16B, V17.16B, V16.16B - bic V26.16B, V18.16B, V17.16B - bic V27.16B, V19.16B, V18.16B - bic V28.16B, V15.16B, V19.16B - bic V29.16B, V16.16B, V15.16B - eor V15.16B, V15.16B, V25.16B - eor V16.16B, V16.16B, V26.16B - eor V17.16B, V17.16B, V27.16B - eor V18.16B, V18.16B, V28.16B - eor V19.16B, V19.16B, V29.16B - bic V25.16B, V22.16B, V21.16B - bic V26.16B, V23.16B, V22.16B - bic V27.16B, V24.16B, V23.16B - bic V28.16B, V20.16B, V24.16B - bic V29.16B, V21.16B, V20.16B - eor V20.16B, V20.16B, V25.16B - eor V21.16B, V21.16B, V26.16B - eor V22.16B, V22.16B, V27.16B - eor V23.16B, V23.16B, V28.16B - eor V24.16B, V24.16B, V29.16B - ; Done transforming - ldp x28, x1, [x29, #48] - ldr x0, [x28], #8 - subs x1, x1, #1 - mov V30.D[0], x0 - mov V30.D[1], x0 - eor x2, x2, x0 - eor V0.16B, V0.16B, V30.16B - bne L_SHA3_shake128_blocksx3_seed_neon_begin - ldr x0, [x29, #40] - st4 {V0.D, V1.D, V2.D, V3.D}[0], [x0], #32 - st4 {V4.D, V5.D, V6.D, V7.D}[0], [x0], #32 - st4 {V8.D, V9.D, V10.D, V11.D}[0], [x0], #32 - st4 {V12.D, V13.D, V14.D, V15.D}[0], [x0], #32 - st4 {V16.D, V17.D, V18.D, V19.D}[0], [x0], #32 - st4 {V20.D, V21.D, V22.D, V23.D}[0], [x0], #32 - st1 {V24.D}[0], [x0] - add x0, x0, #8 - st4 {V0.D, V1.D, V2.D, V3.D}[1], [x0], #32 - st4 {V4.D, V5.D, V6.D, V7.D}[1], [x0], #32 - st4 {V8.D, V9.D, V10.D, V11.D}[1], [x0], #32 - st4 {V12.D, V13.D, V14.D, V15.D}[1], [x0], #32 - st4 {V16.D, V17.D, V18.D, V19.D}[1], [x0], #32 - st4 {V20.D, V21.D, V22.D, V23.D}[1], [x0], #32 - st1 {V24.D}[1], [x0] - add x0, x0, #8 - stp x2, x3, [x0] - stp x4, x5, [x0, #16] - stp x6, x7, [x0, #32] - stp x8, x9, [x0, #48] - stp x10, x11, [x0, #64] - stp x12, x13, [x0, #80] - stp x14, x15, [x0, #96] - stp x16, x17, [x0, #112] - stp x19, x20, [x0, #128] - stp x21, x22, [x0, #144] - stp x23, x24, [x0, #160] - stp x25, x26, [x0, #176] - str x27, [x0, #192] - ldp x17, x19, [x29, #72] - ldp x20, x21, [x29, #88] - ldp x22, x23, [x29, #104] - ldp x24, x25, [x29, #120] - ldp x26, x27, [x29, #136] - ldr x28, [x29, #152] - ldp D8, D9, [x29, #160] - ldp D10, D11, [x29, #176] - ldp D12, D13, [x29, #192] - ldp D14, D15, [x29, #208] - ldp x29, x30, [sp], #0xe0 - ret - ENDP - AREA |.text|, CODE, READONLY - ALIGN 4 - EXPORT mlkem_shake256_blocksx3_seed_neon -mlkem_shake256_blocksx3_seed_neon PROC - stp x29, x30, [sp, #-224]! - add x29, sp, #0 - stp x17, x19, [x29, #72] - stp x20, x21, [x29, #88] - stp x22, x23, [x29, #104] - stp x24, x25, [x29, #120] - stp x26, x27, [x29, #136] - str x28, [x29, #152] - stp D8, D9, [x29, #160] - stp D10, D11, [x29, #176] - stp D12, D13, [x29, #192] - stp D14, D15, [x29, #208] - adrp x28, L_sha3_aarch64_r - add x28, x28, L_sha3_aarch64_r - str x0, [x29, #40] - add x0, x0, #32 - ld1 {V4.D}[0], [x0] - ldp x2, x3, [x1], #16 - add x0, x0, #0xc8 - ld1 {V4.D}[1], [x0] - ldp x4, x5, [x1], #16 - ldr x6, [x0, #200] - eor V5.16B, V5.16B, V5.16B - eor x7, x7, x7 - eor V6.16B, V6.16B, V6.16B - eor x8, x8, x8 - eor V7.16B, V7.16B, V7.16B - eor x9, x9, x9 - eor V8.16B, V8.16B, V8.16B - eor x10, x10, x10 - eor V9.16B, V9.16B, V9.16B - eor x11, x11, x11 - eor V10.16B, V10.16B, V10.16B - eor x12, x12, x12 - eor V11.16B, V11.16B, V11.16B - eor x13, x13, x13 - eor V12.16B, V12.16B, V12.16B - eor x14, x14, x14 - eor V13.16B, V13.16B, V13.16B - eor x15, x15, x15 - eor V14.16B, V14.16B, V14.16B - eor x16, x16, x16 - eor V15.16B, V15.16B, V15.16B - eor x17, x17, x17 - movz x19, #0x8000, lsl 48 - eor V17.16B, V17.16B, V17.16B - eor x20, x20, x20 - eor V18.16B, V18.16B, V18.16B - eor x21, x21, x21 - eor V19.16B, V19.16B, V19.16B - eor x22, x22, x22 - eor V20.16B, V20.16B, V20.16B - eor x23, x23, x23 - eor V21.16B, V21.16B, V21.16B - eor x24, x24, x24 - eor V22.16B, V22.16B, V22.16B - eor x25, x25, x25 - eor V23.16B, V23.16B, V23.16B - eor x26, x26, x26 - eor V24.16B, V24.16B, V24.16B - eor x27, x27, x27 - dup V0.2D, x2 - dup V1.2D, x3 - dup V2.2D, x4 - dup V3.2D, x5 - dup V16.2D, x19 - mov x1, #24 - ; Start of 24 rounds -L_SHA3_shake256_blocksx3_seed_neon_begin - stp x28, x1, [x29, #48] - ; Col Mix NEON - eor V30.16B, V4.16B, V9.16B - eor x0, x6, x11 - eor V27.16B, V1.16B, V6.16B - eor x30, x2, x7 - eor V30.16B, V30.16B, V14.16B - eor x28, x4, x9 - eor V27.16B, V27.16B, V11.16B - eor x0, x0, x16 - eor V30.16B, V30.16B, V19.16B - eor x30, x30, x12 - eor V27.16B, V27.16B, V16.16B - eor x28, x28, x14 - eor V30.16B, V30.16B, V24.16B - eor x0, x0, x22 - eor V27.16B, V27.16B, V21.16B - eor x30, x30, x17 - ushr V25.2D, V27.2D, #63 - eor x28, x28, x20 - sli V25.2D, V27.2D, #1 - eor x0, x0, x27 - eor V25.16B, V25.16B, V30.16B - eor x30, x30, x23 - eor V31.16B, V0.16B, V5.16B - eor x28, x28, x25 - eor V28.16B, V2.16B, V7.16B - str x0, [x29, #32] - eor V31.16B, V31.16B, V10.16B - str x28, [x29, #24] - eor V28.16B, V28.16B, V12.16B - eor x1, x3, x8 - eor V31.16B, V31.16B, V15.16B - eor x28, x5, x10 - eor V28.16B, V28.16B, V17.16B - eor x1, x1, x13 - eor V31.16B, V31.16B, V20.16B - eor x28, x28, x15 - eor V28.16B, V28.16B, V22.16B - eor x1, x1, x19 - ushr V29.2D, V30.2D, #63 - eor x28, x28, x21 - ushr V26.2D, V28.2D, #63 - eor x1, x1, x24 - sli V29.2D, V30.2D, #1 - eor x28, x28, x26 - sli V26.2D, V28.2D, #1 - eor x0, x0, x1, ror 63 - eor V28.16B, V28.16B, V29.16B - eor x1, x1, x28, ror 63 - eor V29.16B, V3.16B, V8.16B - eor x2, x2, x0 - eor V26.16B, V26.16B, V31.16B - eor x7, x7, x0 - eor V29.16B, V29.16B, V13.16B - eor x12, x12, x0 - eor V29.16B, V29.16B, V18.16B - eor x17, x17, x0 - eor V29.16B, V29.16B, V23.16B - eor x23, x23, x0 - ushr V30.2D, V29.2D, #63 - eor x4, x4, x1 - sli V30.2D, V29.2D, #1 - eor x9, x9, x1 - eor V27.16B, V27.16B, V30.16B - eor x14, x14, x1 - ushr V30.2D, V31.2D, #63 - eor x20, x20, x1 - sli V30.2D, V31.2D, #1 - eor x25, x25, x1 - eor V29.16B, V29.16B, V30.16B - ldr x0, [x29, #32] - ; Swap Rotate NEON - eor V0.16B, V0.16B, V25.16B - eor V31.16B, V1.16B, V26.16B - ldr x1, [x29, #24] - eor V6.16B, V6.16B, V26.16B - eor x28, x28, x30, ror 63 - ushr V30.2D, V31.2D, #63 - eor x30, x30, x1, ror 63 - ushr V1.2D, V6.2D, #20 - eor x1, x1, x0, ror 63 - sli V30.2D, V31.2D, #1 - eor x6, x6, x28 - sli V1.2D, V6.2D, #44 - eor x11, x11, x28 - eor V31.16B, V9.16B, V29.16B - eor x16, x16, x28 - eor V22.16B, V22.16B, V27.16B - eor x22, x22, x28 - ushr V6.2D, V31.2D, #44 - eor x27, x27, x28 - ushr V9.2D, V22.2D, #3 - eor x3, x3, x30 - sli V6.2D, V31.2D, #20 - eor x8, x8, x30 - sli V9.2D, V22.2D, #61 - eor x13, x13, x30 - eor V31.16B, V14.16B, V29.16B - eor x19, x19, x30 - eor V20.16B, V20.16B, V25.16B - eor x24, x24, x30 - ushr V22.2D, V31.2D, #25 - eor x5, x5, x1 - ushr V14.2D, V20.2D, #46 - eor x10, x10, x1 - sli V22.2D, V31.2D, #39 - eor x15, x15, x1 - sli V14.2D, V20.2D, #18 - eor x21, x21, x1 - eor V31.16B, V2.16B, V27.16B - eor x26, x26, x1 - ; Swap Rotate Base - eor V12.16B, V12.16B, V27.16B - ror x0, x3, #63 - ushr V20.2D, V31.2D, #2 - ror x3, x8, #20 - ushr V2.2D, V12.2D, #21 - ror x8, x11, #44 - sli V20.2D, V31.2D, #62 - ror x11, x25, #3 - sli V2.2D, V12.2D, #43 - ror x25, x16, #25 - eor V31.16B, V13.16B, V28.16B - ror x16, x23, #46 - eor V19.16B, V19.16B, V29.16B - ror x23, x4, #2 - ushr V12.2D, V31.2D, #39 - ror x4, x14, #21 - ushr V13.2D, V19.2D, #56 - ror x14, x15, #39 - sli V12.2D, V31.2D, #25 - ror x15, x22, #56 - sli V13.2D, V19.2D, #8 - ror x22, x26, #8 - eor V31.16B, V23.16B, V28.16B - ror x26, x17, #23 - eor V15.16B, V15.16B, V25.16B - ror x17, x6, #37 - ushr V19.2D, V31.2D, #8 - ror x6, x27, #50 - ushr V23.2D, V15.2D, #23 - ror x27, x24, #62 - sli V19.2D, V31.2D, #56 - ror x24, x10, #9 - sli V23.2D, V15.2D, #41 - ror x10, x19, #19 - eor V31.16B, V4.16B, V29.16B - ror x19, x7, #28 - eor V24.16B, V24.16B, V29.16B - ror x7, x5, #36 - ushr V15.2D, V31.2D, #37 - ror x5, x21, #43 - ushr V4.2D, V24.2D, #50 - ror x21, x20, #49 - sli V15.2D, V31.2D, #27 - ror x20, x13, #54 - sli V4.2D, V24.2D, #14 - ror x13, x9, #58 - eor V31.16B, V21.16B, V26.16B - ror x9, x12, #61 - ; Row Mix Base - eor V8.16B, V8.16B, V28.16B - bic x12, x4, x3 - ushr V24.2D, V31.2D, #62 - bic x1, x5, x4 - ushr V21.2D, V8.2D, #9 - bic x28, x2, x6 - sli V24.2D, V31.2D, #2 - bic x30, x3, x2 - sli V21.2D, V8.2D, #55 - eor x2, x2, x12 - eor V31.16B, V16.16B, V26.16B - eor x3, x3, x1 - eor V5.16B, V5.16B, V25.16B - bic x12, x6, x5 - ushr V8.2D, V31.2D, #19 - eor x5, x5, x28 - ushr V16.2D, V5.2D, #28 - eor x4, x4, x12 - sli V8.2D, V31.2D, #45 - eor x6, x6, x30 - sli V16.2D, V5.2D, #36 - bic x12, x9, x8 - eor V31.16B, V3.16B, V28.16B - bic x1, x10, x9 - eor V18.16B, V18.16B, V28.16B - bic x28, x7, x11 - ushr V5.2D, V31.2D, #36 - bic x30, x8, x7 - ushr V3.2D, V18.2D, #43 - eor x7, x7, x12 - sli V5.2D, V31.2D, #28 - eor x8, x8, x1 - sli V3.2D, V18.2D, #21 - bic x12, x11, x10 - eor V31.16B, V17.16B, V27.16B - eor x10, x10, x28 - eor V11.16B, V11.16B, V26.16B - eor x9, x9, x12 - ushr V18.2D, V31.2D, #49 - eor x11, x11, x30 - ushr V17.2D, V11.2D, #54 - bic x12, x14, x13 - sli V18.2D, V31.2D, #15 - bic x1, x15, x14 - sli V17.2D, V11.2D, #10 - bic x28, x0, x16 - eor V31.16B, V7.16B, V27.16B - bic x30, x13, x0 - eor V10.16B, V10.16B, V25.16B - eor x12, x0, x12 - ushr V11.2D, V31.2D, #58 - eor x13, x13, x1 - ushr V7.2D, V10.2D, #61 - bic x0, x16, x15 - sli V11.2D, V31.2D, #6 - eor x15, x15, x28 - sli V7.2D, V10.2D, #3 - eor x14, x14, x0 - ; Row Mix NEON - bic V25.16B, V2.16B, V1.16B - eor x16, x16, x30 - bic V26.16B, V3.16B, V2.16B - bic x0, x20, x19 - bic V27.16B, V4.16B, V3.16B - bic x1, x21, x20 - bic V28.16B, V0.16B, V4.16B - bic x28, x17, x22 - bic V29.16B, V1.16B, V0.16B - bic x30, x19, x17 - eor V0.16B, V0.16B, V25.16B - eor x17, x17, x0 - eor V1.16B, V1.16B, V26.16B - eor x19, x19, x1 - eor V2.16B, V2.16B, V27.16B - bic x0, x22, x21 - eor V3.16B, V3.16B, V28.16B - eor x21, x21, x28 - eor V4.16B, V4.16B, V29.16B - eor x20, x20, x0 - bic V25.16B, V7.16B, V6.16B - eor x22, x22, x30 - bic V26.16B, V8.16B, V7.16B - bic x0, x25, x24 - bic V27.16B, V9.16B, V8.16B - bic x1, x26, x25 - bic V28.16B, V5.16B, V9.16B - bic x28, x23, x27 - bic V29.16B, V6.16B, V5.16B - bic x30, x24, x23 - eor V5.16B, V5.16B, V25.16B - eor x23, x23, x0 - eor V6.16B, V6.16B, V26.16B - eor x24, x24, x1 - eor V7.16B, V7.16B, V27.16B - bic x0, x27, x26 - eor V8.16B, V8.16B, V28.16B - eor x26, x26, x28 - eor V9.16B, V9.16B, V29.16B - eor x25, x25, x0 - bic V25.16B, V12.16B, V11.16B - eor x27, x27, x30 - bic V26.16B, V13.16B, V12.16B - bic V27.16B, V14.16B, V13.16B - bic V28.16B, V30.16B, V14.16B - bic V29.16B, V11.16B, V30.16B - eor V10.16B, V30.16B, V25.16B - eor V11.16B, V11.16B, V26.16B - eor V12.16B, V12.16B, V27.16B - eor V13.16B, V13.16B, V28.16B - eor V14.16B, V14.16B, V29.16B - bic V25.16B, V17.16B, V16.16B - bic V26.16B, V18.16B, V17.16B - bic V27.16B, V19.16B, V18.16B - bic V28.16B, V15.16B, V19.16B - bic V29.16B, V16.16B, V15.16B - eor V15.16B, V15.16B, V25.16B - eor V16.16B, V16.16B, V26.16B - eor V17.16B, V17.16B, V27.16B - eor V18.16B, V18.16B, V28.16B - eor V19.16B, V19.16B, V29.16B - bic V25.16B, V22.16B, V21.16B - bic V26.16B, V23.16B, V22.16B - bic V27.16B, V24.16B, V23.16B - bic V28.16B, V20.16B, V24.16B - bic V29.16B, V21.16B, V20.16B - eor V20.16B, V20.16B, V25.16B - eor V21.16B, V21.16B, V26.16B - eor V22.16B, V22.16B, V27.16B - eor V23.16B, V23.16B, V28.16B - eor V24.16B, V24.16B, V29.16B - ; Done transforming - ldp x28, x1, [x29, #48] - ldr x0, [x28], #8 - subs x1, x1, #1 - mov V30.D[0], x0 - mov V30.D[1], x0 - eor x2, x2, x0 - eor V0.16B, V0.16B, V30.16B - bne L_SHA3_shake256_blocksx3_seed_neon_begin + bne L_SHA3_shake256_blocksx3_seed_crypto_begin ldr x0, [x29, #40] st4 {V0.D, V1.D, V2.D, V3.D}[0], [x0], #32 st4 {V4.D, V5.D, V6.D, V7.D}[0], [x0], #32 @@ -11421,5 +10191,1234 @@ L_SHA3_shake256_blocksx3_seed_neon_begin ret ENDP ENDIF + AREA |.text|, CODE, READONLY + ALIGN 4 + EXPORT mlkem_sha3_blocksx3_neon +mlkem_sha3_blocksx3_neon PROC + stp x29, x30, [sp, #-224]! + add x29, sp, #0 + stp x17, x19, [x29, #72] + stp x20, x21, [x29, #88] + stp x22, x23, [x29, #104] + stp x24, x25, [x29, #120] + stp x26, x27, [x29, #136] + str x28, [x29, #152] + stp D8, D9, [x29, #160] + stp D10, D11, [x29, #176] + stp D12, D13, [x29, #192] + stp D14, D15, [x29, #208] + adrp x27, L_sha3_aarch64_r + add x27, x27, L_sha3_aarch64_r + str x0, [x29, #40] + ld4 {V0.D, V1.D, V2.D, V3.D}[0], [x0], #32 + ld4 {V4.D, V5.D, V6.D, V7.D}[0], [x0], #32 + ld4 {V8.D, V9.D, V10.D, V11.D}[0], [x0], #32 + ld4 {V12.D, V13.D, V14.D, V15.D}[0], [x0], #32 + ld4 {V16.D, V17.D, V18.D, V19.D}[0], [x0], #32 + ld4 {V20.D, V21.D, V22.D, V23.D}[0], [x0], #32 + ld1 {V24.D}[0], [x0] + add x0, x0, #8 + ld4 {V0.D, V1.D, V2.D, V3.D}[1], [x0], #32 + ld4 {V4.D, V5.D, V6.D, V7.D}[1], [x0], #32 + ld4 {V8.D, V9.D, V10.D, V11.D}[1], [x0], #32 + ld4 {V12.D, V13.D, V14.D, V15.D}[1], [x0], #32 + ld4 {V16.D, V17.D, V18.D, V19.D}[1], [x0], #32 + ld4 {V20.D, V21.D, V22.D, V23.D}[1], [x0], #32 + ld1 {V24.D}[1], [x0] + add x0, x0, #8 + ldp x1, x2, [x0] + ldp x3, x4, [x0, #16] + ldp x5, x6, [x0, #32] + ldp x7, x8, [x0, #48] + ldp x9, x10, [x0, #64] + ldp x11, x12, [x0, #80] + ldp x13, x14, [x0, #96] + ldp x15, x16, [x0, #112] + ldp x17, x19, [x0, #128] + ldp x20, x21, [x0, #144] + ldp x22, x23, [x0, #160] + ldp x24, x25, [x0, #176] + ldr x26, [x0, #192] + mov x28, #24 + ; Start of 24 rounds +L_SHA3_transform_blocksx3_neon_begin + stp x27, x28, [x29, #48] + ; Col Mix NEON + eor V30.16B, V4.16B, V9.16B + eor x0, x5, x10 + eor V27.16B, V1.16B, V6.16B + eor x30, x1, x6 + eor V30.16B, V30.16B, V14.16B + eor x28, x3, x8 + eor V27.16B, V27.16B, V11.16B + eor x0, x0, x15 + eor V30.16B, V30.16B, V19.16B + eor x30, x30, x11 + eor V27.16B, V27.16B, V16.16B + eor x28, x28, x13 + eor V30.16B, V30.16B, V24.16B + eor x0, x0, x21 + eor V27.16B, V27.16B, V21.16B + eor x30, x30, x16 + ushr V25.2D, V27.2D, #63 + eor x28, x28, x19 + sli V25.2D, V27.2D, #1 + eor x0, x0, x26 + eor V25.16B, V25.16B, V30.16B + eor x30, x30, x22 + eor V31.16B, V0.16B, V5.16B + eor x28, x28, x24 + eor V28.16B, V2.16B, V7.16B + str x0, [x29, #32] + eor V31.16B, V31.16B, V10.16B + str x28, [x29, #24] + eor V28.16B, V28.16B, V12.16B + eor x27, x2, x7 + eor V31.16B, V31.16B, V15.16B + eor x28, x4, x9 + eor V28.16B, V28.16B, V17.16B + eor x27, x27, x12 + eor V31.16B, V31.16B, V20.16B + eor x28, x28, x14 + eor V28.16B, V28.16B, V22.16B + eor x27, x27, x17 + ushr V29.2D, V30.2D, #63 + eor x28, x28, x20 + ushr V26.2D, V28.2D, #63 + eor x27, x27, x23 + sli V29.2D, V30.2D, #1 + eor x28, x28, x25 + sli V26.2D, V28.2D, #1 + eor x0, x0, x27, ror 63 + eor V28.16B, V28.16B, V29.16B + eor x27, x27, x28, ror 63 + eor V29.16B, V3.16B, V8.16B + eor x1, x1, x0 + eor V26.16B, V26.16B, V31.16B + eor x6, x6, x0 + eor V29.16B, V29.16B, V13.16B + eor x11, x11, x0 + eor V29.16B, V29.16B, V18.16B + eor x16, x16, x0 + eor V29.16B, V29.16B, V23.16B + eor x22, x22, x0 + ushr V30.2D, V29.2D, #63 + eor x3, x3, x27 + sli V30.2D, V29.2D, #1 + eor x8, x8, x27 + eor V27.16B, V27.16B, V30.16B + eor x13, x13, x27 + ushr V30.2D, V31.2D, #63 + eor x19, x19, x27 + sli V30.2D, V31.2D, #1 + eor x24, x24, x27 + eor V29.16B, V29.16B, V30.16B + ldr x0, [x29, #32] + ; Swap Rotate NEON + eor V0.16B, V0.16B, V25.16B + eor V31.16B, V1.16B, V26.16B + ldr x27, [x29, #24] + eor V6.16B, V6.16B, V26.16B + eor x28, x28, x30, ror 63 + ushr V30.2D, V31.2D, #63 + eor x30, x30, x27, ror 63 + ushr V1.2D, V6.2D, #20 + eor x27, x27, x0, ror 63 + sli V30.2D, V31.2D, #1 + eor x5, x5, x28 + sli V1.2D, V6.2D, #44 + eor x10, x10, x28 + eor V31.16B, V9.16B, V29.16B + eor x15, x15, x28 + eor V22.16B, V22.16B, V27.16B + eor x21, x21, x28 + ushr V6.2D, V31.2D, #44 + eor x26, x26, x28 + ushr V9.2D, V22.2D, #3 + eor x2, x2, x30 + sli V6.2D, V31.2D, #20 + eor x7, x7, x30 + sli V9.2D, V22.2D, #61 + eor x12, x12, x30 + eor V31.16B, V14.16B, V29.16B + eor x17, x17, x30 + eor V20.16B, V20.16B, V25.16B + eor x23, x23, x30 + ushr V22.2D, V31.2D, #25 + eor x4, x4, x27 + ushr V14.2D, V20.2D, #46 + eor x9, x9, x27 + sli V22.2D, V31.2D, #39 + eor x14, x14, x27 + sli V14.2D, V20.2D, #18 + eor x20, x20, x27 + eor V31.16B, V2.16B, V27.16B + eor x25, x25, x27 + ; Swap Rotate Base + eor V12.16B, V12.16B, V27.16B + ror x0, x2, #63 + ushr V20.2D, V31.2D, #2 + ror x2, x7, #20 + ushr V2.2D, V12.2D, #21 + ror x7, x10, #44 + sli V20.2D, V31.2D, #62 + ror x10, x24, #3 + sli V2.2D, V12.2D, #43 + ror x24, x15, #25 + eor V31.16B, V13.16B, V28.16B + ror x15, x22, #46 + eor V19.16B, V19.16B, V29.16B + ror x22, x3, #2 + ushr V12.2D, V31.2D, #39 + ror x3, x13, #21 + ushr V13.2D, V19.2D, #56 + ror x13, x14, #39 + sli V12.2D, V31.2D, #25 + ror x14, x21, #56 + sli V13.2D, V19.2D, #8 + ror x21, x25, #8 + eor V31.16B, V23.16B, V28.16B + ror x25, x16, #23 + eor V15.16B, V15.16B, V25.16B + ror x16, x5, #37 + ushr V19.2D, V31.2D, #8 + ror x5, x26, #50 + ushr V23.2D, V15.2D, #23 + ror x26, x23, #62 + sli V19.2D, V31.2D, #56 + ror x23, x9, #9 + sli V23.2D, V15.2D, #41 + ror x9, x17, #19 + eor V31.16B, V4.16B, V29.16B + ror x17, x6, #28 + eor V24.16B, V24.16B, V29.16B + ror x6, x4, #36 + ushr V15.2D, V31.2D, #37 + ror x4, x20, #43 + ushr V4.2D, V24.2D, #50 + ror x20, x19, #49 + sli V15.2D, V31.2D, #27 + ror x19, x12, #54 + sli V4.2D, V24.2D, #14 + ror x12, x8, #58 + eor V31.16B, V21.16B, V26.16B + ror x8, x11, #61 + ; Row Mix Base + eor V8.16B, V8.16B, V28.16B + bic x11, x3, x2 + ushr V24.2D, V31.2D, #62 + bic x27, x4, x3 + ushr V21.2D, V8.2D, #9 + bic x28, x1, x5 + sli V24.2D, V31.2D, #2 + bic x30, x2, x1 + sli V21.2D, V8.2D, #55 + eor x1, x1, x11 + eor V31.16B, V16.16B, V26.16B + eor x2, x2, x27 + eor V5.16B, V5.16B, V25.16B + bic x11, x5, x4 + ushr V8.2D, V31.2D, #19 + eor x4, x4, x28 + ushr V16.2D, V5.2D, #28 + eor x3, x3, x11 + sli V8.2D, V31.2D, #45 + eor x5, x5, x30 + sli V16.2D, V5.2D, #36 + bic x11, x8, x7 + eor V31.16B, V3.16B, V28.16B + bic x27, x9, x8 + eor V18.16B, V18.16B, V28.16B + bic x28, x6, x10 + ushr V5.2D, V31.2D, #36 + bic x30, x7, x6 + ushr V3.2D, V18.2D, #43 + eor x6, x6, x11 + sli V5.2D, V31.2D, #28 + eor x7, x7, x27 + sli V3.2D, V18.2D, #21 + bic x11, x10, x9 + eor V31.16B, V17.16B, V27.16B + eor x9, x9, x28 + eor V11.16B, V11.16B, V26.16B + eor x8, x8, x11 + ushr V18.2D, V31.2D, #49 + eor x10, x10, x30 + ushr V17.2D, V11.2D, #54 + bic x11, x13, x12 + sli V18.2D, V31.2D, #15 + bic x27, x14, x13 + sli V17.2D, V11.2D, #10 + bic x28, x0, x15 + eor V31.16B, V7.16B, V27.16B + bic x30, x12, x0 + eor V10.16B, V10.16B, V25.16B + eor x11, x0, x11 + ushr V11.2D, V31.2D, #58 + eor x12, x12, x27 + ushr V7.2D, V10.2D, #61 + bic x0, x15, x14 + sli V11.2D, V31.2D, #6 + eor x14, x14, x28 + sli V7.2D, V10.2D, #3 + eor x13, x13, x0 + ; Row Mix NEON + bic V25.16B, V2.16B, V1.16B + eor x15, x15, x30 + bic V26.16B, V3.16B, V2.16B + bic x0, x19, x17 + bic V27.16B, V4.16B, V3.16B + bic x27, x20, x19 + bic V28.16B, V0.16B, V4.16B + bic x28, x16, x21 + bic V29.16B, V1.16B, V0.16B + bic x30, x17, x16 + eor V0.16B, V0.16B, V25.16B + eor x16, x16, x0 + eor V1.16B, V1.16B, V26.16B + eor x17, x17, x27 + eor V2.16B, V2.16B, V27.16B + bic x0, x21, x20 + eor V3.16B, V3.16B, V28.16B + eor x20, x20, x28 + eor V4.16B, V4.16B, V29.16B + eor x19, x19, x0 + bic V25.16B, V7.16B, V6.16B + eor x21, x21, x30 + bic V26.16B, V8.16B, V7.16B + bic x0, x24, x23 + bic V27.16B, V9.16B, V8.16B + bic x27, x25, x24 + bic V28.16B, V5.16B, V9.16B + bic x28, x22, x26 + bic V29.16B, V6.16B, V5.16B + bic x30, x23, x22 + eor V5.16B, V5.16B, V25.16B + eor x22, x22, x0 + eor V6.16B, V6.16B, V26.16B + eor x23, x23, x27 + eor V7.16B, V7.16B, V27.16B + bic x0, x26, x25 + eor V8.16B, V8.16B, V28.16B + eor x25, x25, x28 + eor V9.16B, V9.16B, V29.16B + eor x24, x24, x0 + bic V25.16B, V12.16B, V11.16B + eor x26, x26, x30 + bic V26.16B, V13.16B, V12.16B + bic V27.16B, V14.16B, V13.16B + bic V28.16B, V30.16B, V14.16B + bic V29.16B, V11.16B, V30.16B + eor V10.16B, V30.16B, V25.16B + eor V11.16B, V11.16B, V26.16B + eor V12.16B, V12.16B, V27.16B + eor V13.16B, V13.16B, V28.16B + eor V14.16B, V14.16B, V29.16B + bic V25.16B, V17.16B, V16.16B + bic V26.16B, V18.16B, V17.16B + bic V27.16B, V19.16B, V18.16B + bic V28.16B, V15.16B, V19.16B + bic V29.16B, V16.16B, V15.16B + eor V15.16B, V15.16B, V25.16B + eor V16.16B, V16.16B, V26.16B + eor V17.16B, V17.16B, V27.16B + eor V18.16B, V18.16B, V28.16B + eor V19.16B, V19.16B, V29.16B + bic V25.16B, V22.16B, V21.16B + bic V26.16B, V23.16B, V22.16B + bic V27.16B, V24.16B, V23.16B + bic V28.16B, V20.16B, V24.16B + bic V29.16B, V21.16B, V20.16B + eor V20.16B, V20.16B, V25.16B + eor V21.16B, V21.16B, V26.16B + eor V22.16B, V22.16B, V27.16B + eor V23.16B, V23.16B, V28.16B + eor V24.16B, V24.16B, V29.16B + ; Done transforming + ldp x27, x28, [x29, #48] + ldr x0, [x27], #8 + subs x28, x28, #1 + mov V30.D[0], x0 + mov V30.D[1], x0 + eor x1, x1, x0 + eor V0.16B, V0.16B, V30.16B + bne L_SHA3_transform_blocksx3_neon_begin + ldr x0, [x29, #40] + st4 {V0.D, V1.D, V2.D, V3.D}[0], [x0], #32 + st4 {V4.D, V5.D, V6.D, V7.D}[0], [x0], #32 + st4 {V8.D, V9.D, V10.D, V11.D}[0], [x0], #32 + st4 {V12.D, V13.D, V14.D, V15.D}[0], [x0], #32 + st4 {V16.D, V17.D, V18.D, V19.D}[0], [x0], #32 + st4 {V20.D, V21.D, V22.D, V23.D}[0], [x0], #32 + st1 {V24.D}[0], [x0] + add x0, x0, #8 + st4 {V0.D, V1.D, V2.D, V3.D}[1], [x0], #32 + st4 {V4.D, V5.D, V6.D, V7.D}[1], [x0], #32 + st4 {V8.D, V9.D, V10.D, V11.D}[1], [x0], #32 + st4 {V12.D, V13.D, V14.D, V15.D}[1], [x0], #32 + st4 {V16.D, V17.D, V18.D, V19.D}[1], [x0], #32 + st4 {V20.D, V21.D, V22.D, V23.D}[1], [x0], #32 + st1 {V24.D}[1], [x0] + add x0, x0, #8 + stp x1, x2, [x0] + stp x3, x4, [x0, #16] + stp x5, x6, [x0, #32] + stp x7, x8, [x0, #48] + stp x9, x10, [x0, #64] + stp x11, x12, [x0, #80] + stp x13, x14, [x0, #96] + stp x15, x16, [x0, #112] + stp x17, x19, [x0, #128] + stp x20, x21, [x0, #144] + stp x22, x23, [x0, #160] + stp x24, x25, [x0, #176] + str x26, [x0, #192] + ldp x17, x19, [x29, #72] + ldp x20, x21, [x29, #88] + ldp x22, x23, [x29, #104] + ldp x24, x25, [x29, #120] + ldp x26, x27, [x29, #136] + ldr x28, [x29, #152] + ldp D8, D9, [x29, #160] + ldp D10, D11, [x29, #176] + ldp D12, D13, [x29, #192] + ldp D14, D15, [x29, #208] + ldp x29, x30, [sp], #0xe0 + ret + ENDP + AREA |.text|, CODE, READONLY + ALIGN 4 + EXPORT mlkem_shake128_blocksx3_seed_neon +mlkem_shake128_blocksx3_seed_neon PROC + stp x29, x30, [sp, #-224]! + add x29, sp, #0 + stp x17, x19, [x29, #72] + stp x20, x21, [x29, #88] + stp x22, x23, [x29, #104] + stp x24, x25, [x29, #120] + stp x26, x27, [x29, #136] + str x28, [x29, #152] + stp D8, D9, [x29, #160] + stp D10, D11, [x29, #176] + stp D12, D13, [x29, #192] + stp D14, D15, [x29, #208] + adrp x28, L_sha3_aarch64_r + add x28, x28, L_sha3_aarch64_r + str x0, [x29, #40] + add x0, x0, #32 + ld1 {V4.D}[0], [x0] + ldp x2, x3, [x1], #16 + add x0, x0, #0xc8 + ld1 {V4.D}[1], [x0] + ldp x4, x5, [x1], #16 + ldr x6, [x0, #200] + eor V5.16B, V5.16B, V5.16B + eor x7, x7, x7 + eor V6.16B, V6.16B, V6.16B + eor x8, x8, x8 + eor V7.16B, V7.16B, V7.16B + eor x9, x9, x9 + eor V8.16B, V8.16B, V8.16B + eor x10, x10, x10 + eor V9.16B, V9.16B, V9.16B + eor x11, x11, x11 + eor V10.16B, V10.16B, V10.16B + eor x12, x12, x12 + eor V11.16B, V11.16B, V11.16B + eor x13, x13, x13 + eor V12.16B, V12.16B, V12.16B + eor x14, x14, x14 + eor V13.16B, V13.16B, V13.16B + eor x15, x15, x15 + eor V14.16B, V14.16B, V14.16B + eor x16, x16, x16 + eor V15.16B, V15.16B, V15.16B + eor x17, x17, x17 + eor V16.16B, V16.16B, V16.16B + eor x19, x19, x19 + eor V17.16B, V17.16B, V17.16B + eor x20, x20, x20 + eor V18.16B, V18.16B, V18.16B + eor x21, x21, x21 + eor V19.16B, V19.16B, V19.16B + eor x22, x22, x22 + movz x23, #0x8000, lsl 48 + eor V21.16B, V21.16B, V21.16B + eor x24, x24, x24 + eor V22.16B, V22.16B, V22.16B + eor x25, x25, x25 + eor V23.16B, V23.16B, V23.16B + eor x26, x26, x26 + eor V24.16B, V24.16B, V24.16B + eor x27, x27, x27 + dup V0.2D, x2 + dup V1.2D, x3 + dup V2.2D, x4 + dup V3.2D, x5 + dup V20.2D, x23 + mov x1, #24 + ; Start of 24 rounds +L_SHA3_shake128_blocksx3_seed_neon_begin + stp x28, x1, [x29, #48] + ; Col Mix NEON + eor V30.16B, V4.16B, V9.16B + eor x0, x6, x11 + eor V27.16B, V1.16B, V6.16B + eor x30, x2, x7 + eor V30.16B, V30.16B, V14.16B + eor x28, x4, x9 + eor V27.16B, V27.16B, V11.16B + eor x0, x0, x16 + eor V30.16B, V30.16B, V19.16B + eor x30, x30, x12 + eor V27.16B, V27.16B, V16.16B + eor x28, x28, x14 + eor V30.16B, V30.16B, V24.16B + eor x0, x0, x22 + eor V27.16B, V27.16B, V21.16B + eor x30, x30, x17 + ushr V25.2D, V27.2D, #63 + eor x28, x28, x20 + sli V25.2D, V27.2D, #1 + eor x0, x0, x27 + eor V25.16B, V25.16B, V30.16B + eor x30, x30, x23 + eor V31.16B, V0.16B, V5.16B + eor x28, x28, x25 + eor V28.16B, V2.16B, V7.16B + str x0, [x29, #32] + eor V31.16B, V31.16B, V10.16B + str x28, [x29, #24] + eor V28.16B, V28.16B, V12.16B + eor x1, x3, x8 + eor V31.16B, V31.16B, V15.16B + eor x28, x5, x10 + eor V28.16B, V28.16B, V17.16B + eor x1, x1, x13 + eor V31.16B, V31.16B, V20.16B + eor x28, x28, x15 + eor V28.16B, V28.16B, V22.16B + eor x1, x1, x19 + ushr V29.2D, V30.2D, #63 + eor x28, x28, x21 + ushr V26.2D, V28.2D, #63 + eor x1, x1, x24 + sli V29.2D, V30.2D, #1 + eor x28, x28, x26 + sli V26.2D, V28.2D, #1 + eor x0, x0, x1, ror 63 + eor V28.16B, V28.16B, V29.16B + eor x1, x1, x28, ror 63 + eor V29.16B, V3.16B, V8.16B + eor x2, x2, x0 + eor V26.16B, V26.16B, V31.16B + eor x7, x7, x0 + eor V29.16B, V29.16B, V13.16B + eor x12, x12, x0 + eor V29.16B, V29.16B, V18.16B + eor x17, x17, x0 + eor V29.16B, V29.16B, V23.16B + eor x23, x23, x0 + ushr V30.2D, V29.2D, #63 + eor x4, x4, x1 + sli V30.2D, V29.2D, #1 + eor x9, x9, x1 + eor V27.16B, V27.16B, V30.16B + eor x14, x14, x1 + ushr V30.2D, V31.2D, #63 + eor x20, x20, x1 + sli V30.2D, V31.2D, #1 + eor x25, x25, x1 + eor V29.16B, V29.16B, V30.16B + ldr x0, [x29, #32] + ; Swap Rotate NEON + eor V0.16B, V0.16B, V25.16B + eor V31.16B, V1.16B, V26.16B + ldr x1, [x29, #24] + eor V6.16B, V6.16B, V26.16B + eor x28, x28, x30, ror 63 + ushr V30.2D, V31.2D, #63 + eor x30, x30, x1, ror 63 + ushr V1.2D, V6.2D, #20 + eor x1, x1, x0, ror 63 + sli V30.2D, V31.2D, #1 + eor x6, x6, x28 + sli V1.2D, V6.2D, #44 + eor x11, x11, x28 + eor V31.16B, V9.16B, V29.16B + eor x16, x16, x28 + eor V22.16B, V22.16B, V27.16B + eor x22, x22, x28 + ushr V6.2D, V31.2D, #44 + eor x27, x27, x28 + ushr V9.2D, V22.2D, #3 + eor x3, x3, x30 + sli V6.2D, V31.2D, #20 + eor x8, x8, x30 + sli V9.2D, V22.2D, #61 + eor x13, x13, x30 + eor V31.16B, V14.16B, V29.16B + eor x19, x19, x30 + eor V20.16B, V20.16B, V25.16B + eor x24, x24, x30 + ushr V22.2D, V31.2D, #25 + eor x5, x5, x1 + ushr V14.2D, V20.2D, #46 + eor x10, x10, x1 + sli V22.2D, V31.2D, #39 + eor x15, x15, x1 + sli V14.2D, V20.2D, #18 + eor x21, x21, x1 + eor V31.16B, V2.16B, V27.16B + eor x26, x26, x1 + ; Swap Rotate Base + eor V12.16B, V12.16B, V27.16B + ror x0, x3, #63 + ushr V20.2D, V31.2D, #2 + ror x3, x8, #20 + ushr V2.2D, V12.2D, #21 + ror x8, x11, #44 + sli V20.2D, V31.2D, #62 + ror x11, x25, #3 + sli V2.2D, V12.2D, #43 + ror x25, x16, #25 + eor V31.16B, V13.16B, V28.16B + ror x16, x23, #46 + eor V19.16B, V19.16B, V29.16B + ror x23, x4, #2 + ushr V12.2D, V31.2D, #39 + ror x4, x14, #21 + ushr V13.2D, V19.2D, #56 + ror x14, x15, #39 + sli V12.2D, V31.2D, #25 + ror x15, x22, #56 + sli V13.2D, V19.2D, #8 + ror x22, x26, #8 + eor V31.16B, V23.16B, V28.16B + ror x26, x17, #23 + eor V15.16B, V15.16B, V25.16B + ror x17, x6, #37 + ushr V19.2D, V31.2D, #8 + ror x6, x27, #50 + ushr V23.2D, V15.2D, #23 + ror x27, x24, #62 + sli V19.2D, V31.2D, #56 + ror x24, x10, #9 + sli V23.2D, V15.2D, #41 + ror x10, x19, #19 + eor V31.16B, V4.16B, V29.16B + ror x19, x7, #28 + eor V24.16B, V24.16B, V29.16B + ror x7, x5, #36 + ushr V15.2D, V31.2D, #37 + ror x5, x21, #43 + ushr V4.2D, V24.2D, #50 + ror x21, x20, #49 + sli V15.2D, V31.2D, #27 + ror x20, x13, #54 + sli V4.2D, V24.2D, #14 + ror x13, x9, #58 + eor V31.16B, V21.16B, V26.16B + ror x9, x12, #61 + ; Row Mix Base + eor V8.16B, V8.16B, V28.16B + bic x12, x4, x3 + ushr V24.2D, V31.2D, #62 + bic x1, x5, x4 + ushr V21.2D, V8.2D, #9 + bic x28, x2, x6 + sli V24.2D, V31.2D, #2 + bic x30, x3, x2 + sli V21.2D, V8.2D, #55 + eor x2, x2, x12 + eor V31.16B, V16.16B, V26.16B + eor x3, x3, x1 + eor V5.16B, V5.16B, V25.16B + bic x12, x6, x5 + ushr V8.2D, V31.2D, #19 + eor x5, x5, x28 + ushr V16.2D, V5.2D, #28 + eor x4, x4, x12 + sli V8.2D, V31.2D, #45 + eor x6, x6, x30 + sli V16.2D, V5.2D, #36 + bic x12, x9, x8 + eor V31.16B, V3.16B, V28.16B + bic x1, x10, x9 + eor V18.16B, V18.16B, V28.16B + bic x28, x7, x11 + ushr V5.2D, V31.2D, #36 + bic x30, x8, x7 + ushr V3.2D, V18.2D, #43 + eor x7, x7, x12 + sli V5.2D, V31.2D, #28 + eor x8, x8, x1 + sli V3.2D, V18.2D, #21 + bic x12, x11, x10 + eor V31.16B, V17.16B, V27.16B + eor x10, x10, x28 + eor V11.16B, V11.16B, V26.16B + eor x9, x9, x12 + ushr V18.2D, V31.2D, #49 + eor x11, x11, x30 + ushr V17.2D, V11.2D, #54 + bic x12, x14, x13 + sli V18.2D, V31.2D, #15 + bic x1, x15, x14 + sli V17.2D, V11.2D, #10 + bic x28, x0, x16 + eor V31.16B, V7.16B, V27.16B + bic x30, x13, x0 + eor V10.16B, V10.16B, V25.16B + eor x12, x0, x12 + ushr V11.2D, V31.2D, #58 + eor x13, x13, x1 + ushr V7.2D, V10.2D, #61 + bic x0, x16, x15 + sli V11.2D, V31.2D, #6 + eor x15, x15, x28 + sli V7.2D, V10.2D, #3 + eor x14, x14, x0 + ; Row Mix NEON + bic V25.16B, V2.16B, V1.16B + eor x16, x16, x30 + bic V26.16B, V3.16B, V2.16B + bic x0, x20, x19 + bic V27.16B, V4.16B, V3.16B + bic x1, x21, x20 + bic V28.16B, V0.16B, V4.16B + bic x28, x17, x22 + bic V29.16B, V1.16B, V0.16B + bic x30, x19, x17 + eor V0.16B, V0.16B, V25.16B + eor x17, x17, x0 + eor V1.16B, V1.16B, V26.16B + eor x19, x19, x1 + eor V2.16B, V2.16B, V27.16B + bic x0, x22, x21 + eor V3.16B, V3.16B, V28.16B + eor x21, x21, x28 + eor V4.16B, V4.16B, V29.16B + eor x20, x20, x0 + bic V25.16B, V7.16B, V6.16B + eor x22, x22, x30 + bic V26.16B, V8.16B, V7.16B + bic x0, x25, x24 + bic V27.16B, V9.16B, V8.16B + bic x1, x26, x25 + bic V28.16B, V5.16B, V9.16B + bic x28, x23, x27 + bic V29.16B, V6.16B, V5.16B + bic x30, x24, x23 + eor V5.16B, V5.16B, V25.16B + eor x23, x23, x0 + eor V6.16B, V6.16B, V26.16B + eor x24, x24, x1 + eor V7.16B, V7.16B, V27.16B + bic x0, x27, x26 + eor V8.16B, V8.16B, V28.16B + eor x26, x26, x28 + eor V9.16B, V9.16B, V29.16B + eor x25, x25, x0 + bic V25.16B, V12.16B, V11.16B + eor x27, x27, x30 + bic V26.16B, V13.16B, V12.16B + bic V27.16B, V14.16B, V13.16B + bic V28.16B, V30.16B, V14.16B + bic V29.16B, V11.16B, V30.16B + eor V10.16B, V30.16B, V25.16B + eor V11.16B, V11.16B, V26.16B + eor V12.16B, V12.16B, V27.16B + eor V13.16B, V13.16B, V28.16B + eor V14.16B, V14.16B, V29.16B + bic V25.16B, V17.16B, V16.16B + bic V26.16B, V18.16B, V17.16B + bic V27.16B, V19.16B, V18.16B + bic V28.16B, V15.16B, V19.16B + bic V29.16B, V16.16B, V15.16B + eor V15.16B, V15.16B, V25.16B + eor V16.16B, V16.16B, V26.16B + eor V17.16B, V17.16B, V27.16B + eor V18.16B, V18.16B, V28.16B + eor V19.16B, V19.16B, V29.16B + bic V25.16B, V22.16B, V21.16B + bic V26.16B, V23.16B, V22.16B + bic V27.16B, V24.16B, V23.16B + bic V28.16B, V20.16B, V24.16B + bic V29.16B, V21.16B, V20.16B + eor V20.16B, V20.16B, V25.16B + eor V21.16B, V21.16B, V26.16B + eor V22.16B, V22.16B, V27.16B + eor V23.16B, V23.16B, V28.16B + eor V24.16B, V24.16B, V29.16B + ; Done transforming + ldp x28, x1, [x29, #48] + ldr x0, [x28], #8 + subs x1, x1, #1 + mov V30.D[0], x0 + mov V30.D[1], x0 + eor x2, x2, x0 + eor V0.16B, V0.16B, V30.16B + bne L_SHA3_shake128_blocksx3_seed_neon_begin + ldr x0, [x29, #40] + st4 {V0.D, V1.D, V2.D, V3.D}[0], [x0], #32 + st4 {V4.D, V5.D, V6.D, V7.D}[0], [x0], #32 + st4 {V8.D, V9.D, V10.D, V11.D}[0], [x0], #32 + st4 {V12.D, V13.D, V14.D, V15.D}[0], [x0], #32 + st4 {V16.D, V17.D, V18.D, V19.D}[0], [x0], #32 + st4 {V20.D, V21.D, V22.D, V23.D}[0], [x0], #32 + st1 {V24.D}[0], [x0] + add x0, x0, #8 + st4 {V0.D, V1.D, V2.D, V3.D}[1], [x0], #32 + st4 {V4.D, V5.D, V6.D, V7.D}[1], [x0], #32 + st4 {V8.D, V9.D, V10.D, V11.D}[1], [x0], #32 + st4 {V12.D, V13.D, V14.D, V15.D}[1], [x0], #32 + st4 {V16.D, V17.D, V18.D, V19.D}[1], [x0], #32 + st4 {V20.D, V21.D, V22.D, V23.D}[1], [x0], #32 + st1 {V24.D}[1], [x0] + add x0, x0, #8 + stp x2, x3, [x0] + stp x4, x5, [x0, #16] + stp x6, x7, [x0, #32] + stp x8, x9, [x0, #48] + stp x10, x11, [x0, #64] + stp x12, x13, [x0, #80] + stp x14, x15, [x0, #96] + stp x16, x17, [x0, #112] + stp x19, x20, [x0, #128] + stp x21, x22, [x0, #144] + stp x23, x24, [x0, #160] + stp x25, x26, [x0, #176] + str x27, [x0, #192] + ldp x17, x19, [x29, #72] + ldp x20, x21, [x29, #88] + ldp x22, x23, [x29, #104] + ldp x24, x25, [x29, #120] + ldp x26, x27, [x29, #136] + ldr x28, [x29, #152] + ldp D8, D9, [x29, #160] + ldp D10, D11, [x29, #176] + ldp D12, D13, [x29, #192] + ldp D14, D15, [x29, #208] + ldp x29, x30, [sp], #0xe0 + ret + ENDP + AREA |.text|, CODE, READONLY + ALIGN 4 + EXPORT mlkem_shake256_blocksx3_seed_neon +mlkem_shake256_blocksx3_seed_neon PROC + stp x29, x30, [sp, #-224]! + add x29, sp, #0 + stp x17, x19, [x29, #72] + stp x20, x21, [x29, #88] + stp x22, x23, [x29, #104] + stp x24, x25, [x29, #120] + stp x26, x27, [x29, #136] + str x28, [x29, #152] + stp D8, D9, [x29, #160] + stp D10, D11, [x29, #176] + stp D12, D13, [x29, #192] + stp D14, D15, [x29, #208] + adrp x28, L_sha3_aarch64_r + add x28, x28, L_sha3_aarch64_r + str x0, [x29, #40] + add x0, x0, #32 + ld1 {V4.D}[0], [x0] + ldp x2, x3, [x1], #16 + add x0, x0, #0xc8 + ld1 {V4.D}[1], [x0] + ldp x4, x5, [x1], #16 + ldr x6, [x0, #200] + eor V5.16B, V5.16B, V5.16B + eor x7, x7, x7 + eor V6.16B, V6.16B, V6.16B + eor x8, x8, x8 + eor V7.16B, V7.16B, V7.16B + eor x9, x9, x9 + eor V8.16B, V8.16B, V8.16B + eor x10, x10, x10 + eor V9.16B, V9.16B, V9.16B + eor x11, x11, x11 + eor V10.16B, V10.16B, V10.16B + eor x12, x12, x12 + eor V11.16B, V11.16B, V11.16B + eor x13, x13, x13 + eor V12.16B, V12.16B, V12.16B + eor x14, x14, x14 + eor V13.16B, V13.16B, V13.16B + eor x15, x15, x15 + eor V14.16B, V14.16B, V14.16B + eor x16, x16, x16 + eor V15.16B, V15.16B, V15.16B + eor x17, x17, x17 + movz x19, #0x8000, lsl 48 + eor V17.16B, V17.16B, V17.16B + eor x20, x20, x20 + eor V18.16B, V18.16B, V18.16B + eor x21, x21, x21 + eor V19.16B, V19.16B, V19.16B + eor x22, x22, x22 + eor V20.16B, V20.16B, V20.16B + eor x23, x23, x23 + eor V21.16B, V21.16B, V21.16B + eor x24, x24, x24 + eor V22.16B, V22.16B, V22.16B + eor x25, x25, x25 + eor V23.16B, V23.16B, V23.16B + eor x26, x26, x26 + eor V24.16B, V24.16B, V24.16B + eor x27, x27, x27 + dup V0.2D, x2 + dup V1.2D, x3 + dup V2.2D, x4 + dup V3.2D, x5 + dup V16.2D, x19 + mov x1, #24 + ; Start of 24 rounds +L_SHA3_shake256_blocksx3_seed_neon_begin + stp x28, x1, [x29, #48] + ; Col Mix NEON + eor V30.16B, V4.16B, V9.16B + eor x0, x6, x11 + eor V27.16B, V1.16B, V6.16B + eor x30, x2, x7 + eor V30.16B, V30.16B, V14.16B + eor x28, x4, x9 + eor V27.16B, V27.16B, V11.16B + eor x0, x0, x16 + eor V30.16B, V30.16B, V19.16B + eor x30, x30, x12 + eor V27.16B, V27.16B, V16.16B + eor x28, x28, x14 + eor V30.16B, V30.16B, V24.16B + eor x0, x0, x22 + eor V27.16B, V27.16B, V21.16B + eor x30, x30, x17 + ushr V25.2D, V27.2D, #63 + eor x28, x28, x20 + sli V25.2D, V27.2D, #1 + eor x0, x0, x27 + eor V25.16B, V25.16B, V30.16B + eor x30, x30, x23 + eor V31.16B, V0.16B, V5.16B + eor x28, x28, x25 + eor V28.16B, V2.16B, V7.16B + str x0, [x29, #32] + eor V31.16B, V31.16B, V10.16B + str x28, [x29, #24] + eor V28.16B, V28.16B, V12.16B + eor x1, x3, x8 + eor V31.16B, V31.16B, V15.16B + eor x28, x5, x10 + eor V28.16B, V28.16B, V17.16B + eor x1, x1, x13 + eor V31.16B, V31.16B, V20.16B + eor x28, x28, x15 + eor V28.16B, V28.16B, V22.16B + eor x1, x1, x19 + ushr V29.2D, V30.2D, #63 + eor x28, x28, x21 + ushr V26.2D, V28.2D, #63 + eor x1, x1, x24 + sli V29.2D, V30.2D, #1 + eor x28, x28, x26 + sli V26.2D, V28.2D, #1 + eor x0, x0, x1, ror 63 + eor V28.16B, V28.16B, V29.16B + eor x1, x1, x28, ror 63 + eor V29.16B, V3.16B, V8.16B + eor x2, x2, x0 + eor V26.16B, V26.16B, V31.16B + eor x7, x7, x0 + eor V29.16B, V29.16B, V13.16B + eor x12, x12, x0 + eor V29.16B, V29.16B, V18.16B + eor x17, x17, x0 + eor V29.16B, V29.16B, V23.16B + eor x23, x23, x0 + ushr V30.2D, V29.2D, #63 + eor x4, x4, x1 + sli V30.2D, V29.2D, #1 + eor x9, x9, x1 + eor V27.16B, V27.16B, V30.16B + eor x14, x14, x1 + ushr V30.2D, V31.2D, #63 + eor x20, x20, x1 + sli V30.2D, V31.2D, #1 + eor x25, x25, x1 + eor V29.16B, V29.16B, V30.16B + ldr x0, [x29, #32] + ; Swap Rotate NEON + eor V0.16B, V0.16B, V25.16B + eor V31.16B, V1.16B, V26.16B + ldr x1, [x29, #24] + eor V6.16B, V6.16B, V26.16B + eor x28, x28, x30, ror 63 + ushr V30.2D, V31.2D, #63 + eor x30, x30, x1, ror 63 + ushr V1.2D, V6.2D, #20 + eor x1, x1, x0, ror 63 + sli V30.2D, V31.2D, #1 + eor x6, x6, x28 + sli V1.2D, V6.2D, #44 + eor x11, x11, x28 + eor V31.16B, V9.16B, V29.16B + eor x16, x16, x28 + eor V22.16B, V22.16B, V27.16B + eor x22, x22, x28 + ushr V6.2D, V31.2D, #44 + eor x27, x27, x28 + ushr V9.2D, V22.2D, #3 + eor x3, x3, x30 + sli V6.2D, V31.2D, #20 + eor x8, x8, x30 + sli V9.2D, V22.2D, #61 + eor x13, x13, x30 + eor V31.16B, V14.16B, V29.16B + eor x19, x19, x30 + eor V20.16B, V20.16B, V25.16B + eor x24, x24, x30 + ushr V22.2D, V31.2D, #25 + eor x5, x5, x1 + ushr V14.2D, V20.2D, #46 + eor x10, x10, x1 + sli V22.2D, V31.2D, #39 + eor x15, x15, x1 + sli V14.2D, V20.2D, #18 + eor x21, x21, x1 + eor V31.16B, V2.16B, V27.16B + eor x26, x26, x1 + ; Swap Rotate Base + eor V12.16B, V12.16B, V27.16B + ror x0, x3, #63 + ushr V20.2D, V31.2D, #2 + ror x3, x8, #20 + ushr V2.2D, V12.2D, #21 + ror x8, x11, #44 + sli V20.2D, V31.2D, #62 + ror x11, x25, #3 + sli V2.2D, V12.2D, #43 + ror x25, x16, #25 + eor V31.16B, V13.16B, V28.16B + ror x16, x23, #46 + eor V19.16B, V19.16B, V29.16B + ror x23, x4, #2 + ushr V12.2D, V31.2D, #39 + ror x4, x14, #21 + ushr V13.2D, V19.2D, #56 + ror x14, x15, #39 + sli V12.2D, V31.2D, #25 + ror x15, x22, #56 + sli V13.2D, V19.2D, #8 + ror x22, x26, #8 + eor V31.16B, V23.16B, V28.16B + ror x26, x17, #23 + eor V15.16B, V15.16B, V25.16B + ror x17, x6, #37 + ushr V19.2D, V31.2D, #8 + ror x6, x27, #50 + ushr V23.2D, V15.2D, #23 + ror x27, x24, #62 + sli V19.2D, V31.2D, #56 + ror x24, x10, #9 + sli V23.2D, V15.2D, #41 + ror x10, x19, #19 + eor V31.16B, V4.16B, V29.16B + ror x19, x7, #28 + eor V24.16B, V24.16B, V29.16B + ror x7, x5, #36 + ushr V15.2D, V31.2D, #37 + ror x5, x21, #43 + ushr V4.2D, V24.2D, #50 + ror x21, x20, #49 + sli V15.2D, V31.2D, #27 + ror x20, x13, #54 + sli V4.2D, V24.2D, #14 + ror x13, x9, #58 + eor V31.16B, V21.16B, V26.16B + ror x9, x12, #61 + ; Row Mix Base + eor V8.16B, V8.16B, V28.16B + bic x12, x4, x3 + ushr V24.2D, V31.2D, #62 + bic x1, x5, x4 + ushr V21.2D, V8.2D, #9 + bic x28, x2, x6 + sli V24.2D, V31.2D, #2 + bic x30, x3, x2 + sli V21.2D, V8.2D, #55 + eor x2, x2, x12 + eor V31.16B, V16.16B, V26.16B + eor x3, x3, x1 + eor V5.16B, V5.16B, V25.16B + bic x12, x6, x5 + ushr V8.2D, V31.2D, #19 + eor x5, x5, x28 + ushr V16.2D, V5.2D, #28 + eor x4, x4, x12 + sli V8.2D, V31.2D, #45 + eor x6, x6, x30 + sli V16.2D, V5.2D, #36 + bic x12, x9, x8 + eor V31.16B, V3.16B, V28.16B + bic x1, x10, x9 + eor V18.16B, V18.16B, V28.16B + bic x28, x7, x11 + ushr V5.2D, V31.2D, #36 + bic x30, x8, x7 + ushr V3.2D, V18.2D, #43 + eor x7, x7, x12 + sli V5.2D, V31.2D, #28 + eor x8, x8, x1 + sli V3.2D, V18.2D, #21 + bic x12, x11, x10 + eor V31.16B, V17.16B, V27.16B + eor x10, x10, x28 + eor V11.16B, V11.16B, V26.16B + eor x9, x9, x12 + ushr V18.2D, V31.2D, #49 + eor x11, x11, x30 + ushr V17.2D, V11.2D, #54 + bic x12, x14, x13 + sli V18.2D, V31.2D, #15 + bic x1, x15, x14 + sli V17.2D, V11.2D, #10 + bic x28, x0, x16 + eor V31.16B, V7.16B, V27.16B + bic x30, x13, x0 + eor V10.16B, V10.16B, V25.16B + eor x12, x0, x12 + ushr V11.2D, V31.2D, #58 + eor x13, x13, x1 + ushr V7.2D, V10.2D, #61 + bic x0, x16, x15 + sli V11.2D, V31.2D, #6 + eor x15, x15, x28 + sli V7.2D, V10.2D, #3 + eor x14, x14, x0 + ; Row Mix NEON + bic V25.16B, V2.16B, V1.16B + eor x16, x16, x30 + bic V26.16B, V3.16B, V2.16B + bic x0, x20, x19 + bic V27.16B, V4.16B, V3.16B + bic x1, x21, x20 + bic V28.16B, V0.16B, V4.16B + bic x28, x17, x22 + bic V29.16B, V1.16B, V0.16B + bic x30, x19, x17 + eor V0.16B, V0.16B, V25.16B + eor x17, x17, x0 + eor V1.16B, V1.16B, V26.16B + eor x19, x19, x1 + eor V2.16B, V2.16B, V27.16B + bic x0, x22, x21 + eor V3.16B, V3.16B, V28.16B + eor x21, x21, x28 + eor V4.16B, V4.16B, V29.16B + eor x20, x20, x0 + bic V25.16B, V7.16B, V6.16B + eor x22, x22, x30 + bic V26.16B, V8.16B, V7.16B + bic x0, x25, x24 + bic V27.16B, V9.16B, V8.16B + bic x1, x26, x25 + bic V28.16B, V5.16B, V9.16B + bic x28, x23, x27 + bic V29.16B, V6.16B, V5.16B + bic x30, x24, x23 + eor V5.16B, V5.16B, V25.16B + eor x23, x23, x0 + eor V6.16B, V6.16B, V26.16B + eor x24, x24, x1 + eor V7.16B, V7.16B, V27.16B + bic x0, x27, x26 + eor V8.16B, V8.16B, V28.16B + eor x26, x26, x28 + eor V9.16B, V9.16B, V29.16B + eor x25, x25, x0 + bic V25.16B, V12.16B, V11.16B + eor x27, x27, x30 + bic V26.16B, V13.16B, V12.16B + bic V27.16B, V14.16B, V13.16B + bic V28.16B, V30.16B, V14.16B + bic V29.16B, V11.16B, V30.16B + eor V10.16B, V30.16B, V25.16B + eor V11.16B, V11.16B, V26.16B + eor V12.16B, V12.16B, V27.16B + eor V13.16B, V13.16B, V28.16B + eor V14.16B, V14.16B, V29.16B + bic V25.16B, V17.16B, V16.16B + bic V26.16B, V18.16B, V17.16B + bic V27.16B, V19.16B, V18.16B + bic V28.16B, V15.16B, V19.16B + bic V29.16B, V16.16B, V15.16B + eor V15.16B, V15.16B, V25.16B + eor V16.16B, V16.16B, V26.16B + eor V17.16B, V17.16B, V27.16B + eor V18.16B, V18.16B, V28.16B + eor V19.16B, V19.16B, V29.16B + bic V25.16B, V22.16B, V21.16B + bic V26.16B, V23.16B, V22.16B + bic V27.16B, V24.16B, V23.16B + bic V28.16B, V20.16B, V24.16B + bic V29.16B, V21.16B, V20.16B + eor V20.16B, V20.16B, V25.16B + eor V21.16B, V21.16B, V26.16B + eor V22.16B, V22.16B, V27.16B + eor V23.16B, V23.16B, V28.16B + eor V24.16B, V24.16B, V29.16B + ; Done transforming + ldp x28, x1, [x29, #48] + ldr x0, [x28], #8 + subs x1, x1, #1 + mov V30.D[0], x0 + mov V30.D[1], x0 + eor x2, x2, x0 + eor V0.16B, V0.16B, V30.16B + bne L_SHA3_shake256_blocksx3_seed_neon_begin + ldr x0, [x29, #40] + st4 {V0.D, V1.D, V2.D, V3.D}[0], [x0], #32 + st4 {V4.D, V5.D, V6.D, V7.D}[0], [x0], #32 + st4 {V8.D, V9.D, V10.D, V11.D}[0], [x0], #32 + st4 {V12.D, V13.D, V14.D, V15.D}[0], [x0], #32 + st4 {V16.D, V17.D, V18.D, V19.D}[0], [x0], #32 + st4 {V20.D, V21.D, V22.D, V23.D}[0], [x0], #32 + st1 {V24.D}[0], [x0] + add x0, x0, #8 + st4 {V0.D, V1.D, V2.D, V3.D}[1], [x0], #32 + st4 {V4.D, V5.D, V6.D, V7.D}[1], [x0], #32 + st4 {V8.D, V9.D, V10.D, V11.D}[1], [x0], #32 + st4 {V12.D, V13.D, V14.D, V15.D}[1], [x0], #32 + st4 {V16.D, V17.D, V18.D, V19.D}[1], [x0], #32 + st4 {V20.D, V21.D, V22.D, V23.D}[1], [x0], #32 + st1 {V24.D}[1], [x0] + add x0, x0, #8 + stp x2, x3, [x0] + stp x4, x5, [x0, #16] + stp x6, x7, [x0, #32] + stp x8, x9, [x0, #48] + stp x10, x11, [x0, #64] + stp x12, x13, [x0, #80] + stp x14, x15, [x0, #96] + stp x16, x17, [x0, #112] + stp x19, x20, [x0, #128] + stp x21, x22, [x0, #144] + stp x23, x24, [x0, #160] + stp x25, x26, [x0, #176] + str x27, [x0, #192] + ldp x17, x19, [x29, #72] + ldp x20, x21, [x29, #88] + ldp x22, x23, [x29, #104] + ldp x24, x25, [x29, #120] + ldp x26, x27, [x29, #136] + ldr x28, [x29, #152] + ldp D8, D9, [x29, #160] + ldp D10, D11, [x29, #176] + ldp D12, D13, [x29, #192] + ldp D14, D15, [x29, #208] + ldp x29, x30, [sp], #0xe0 + ret + ENDP ENDIF END diff --git a/wolfcrypt/src/port/arm/armv8-mlkem-asm_c.c b/wolfcrypt/src/port/arm/armv8-mlkem-asm_c.c index 8a392c2e17..2b1f5b40b0 100644 --- a/wolfcrypt/src/port/arm/armv8-mlkem-asm_c.c +++ b/wolfcrypt/src/port/arm/armv8-mlkem-asm_c.c @@ -9120,7 +9120,7 @@ XALIGNED(16) static const word64 L_sha3_aarch64_r[] = { }; #ifdef WOLFSSL_ARMASM_CRYPTO_SHA3 -void mlkem_sha3_blocksx3_neon(word64* state) +void mlkem_sha3_blocksx3_crypto(word64* state) { const word64* r = L_sha3_aarch64_r; __asm__ __volatile__ ( @@ -9159,7 +9159,7 @@ void mlkem_sha3_blocksx3_neon(word64* state) "mov x28, #24\n\t" /* Start of 24 rounds */ "\n" - "L_SHA3_transform_blocksx3_neon_begin_%=:\n\t" + "L_SHA3_transform_blocksx3_crypto_begin_%=:\n\t" "stp %[r], x28, [x29, #48]\n\t" /* Col Mix */ "eor3 v31.16b, v0.16b, v5.16b, v10.16b\n\t" @@ -9375,7 +9375,7 @@ void mlkem_sha3_blocksx3_neon(word64* state) "mov v30.d[1], %x[state]\n\t" "eor x1, x1, %x[state]\n\t" "eor v0.16b, v0.16b, v30.16b\n\t" - "b.ne L_SHA3_transform_blocksx3_neon_begin_%=\n\t" + "b.ne L_SHA3_transform_blocksx3_crypto_begin_%=\n\t" "ldr %x[state], [x29, #40]\n\t" "st4 {v0.d, v1.d, v2.d, v3.d}[0], [%x[state]], #32\n\t" "st4 {v4.d, v5.d, v6.d, v7.d}[0], [%x[state]], #32\n\t" @@ -9419,7 +9419,7 @@ void mlkem_sha3_blocksx3_neon(word64* state) ); } -void mlkem_shake128_blocksx3_seed_neon(word64* state, byte* seed) +void mlkem_shake128_blocksx3_seed_crypto(word64* state, byte* seed) { const word64* r = L_sha3_aarch64_r; __asm__ __volatile__ ( @@ -9480,7 +9480,7 @@ void mlkem_shake128_blocksx3_seed_neon(word64* state, byte* seed) "mov %x[seed], #24\n\t" /* Start of 24 rounds */ "\n" - "L_SHA3_shake128_blocksx3_seed_neon_begin_%=:\n\t" + "L_SHA3_shake128_blocksx3_seed_crypto_begin_%=:\n\t" "stp %[r], %x[seed], [x29, #48]\n\t" /* Col Mix */ "eor3 v31.16b, v0.16b, v5.16b, v10.16b\n\t" @@ -9696,7 +9696,7 @@ void mlkem_shake128_blocksx3_seed_neon(word64* state, byte* seed) "mov v30.d[1], %x[state]\n\t" "eor x2, x2, %x[state]\n\t" "eor v0.16b, v0.16b, v30.16b\n\t" - "b.ne L_SHA3_shake128_blocksx3_seed_neon_begin_%=\n\t" + "b.ne L_SHA3_shake128_blocksx3_seed_crypto_begin_%=\n\t" "ldr %x[state], [x29, #40]\n\t" "st4 {v0.d, v1.d, v2.d, v3.d}[0], [%x[state]], #32\n\t" "st4 {v4.d, v5.d, v6.d, v7.d}[0], [%x[state]], #32\n\t" @@ -9740,7 +9740,7 @@ void mlkem_shake128_blocksx3_seed_neon(word64* state, byte* seed) ); } -void mlkem_shake256_blocksx3_seed_neon(word64* state, byte* seed) +void mlkem_shake256_blocksx3_seed_crypto(word64* state, byte* seed) { const word64* r = L_sha3_aarch64_r; __asm__ __volatile__ ( @@ -9801,7 +9801,7 @@ void mlkem_shake256_blocksx3_seed_neon(word64* state, byte* seed) "mov %x[seed], #24\n\t" /* Start of 24 rounds */ "\n" - "L_SHA3_shake256_blocksx3_seed_neon_begin_%=:\n\t" + "L_SHA3_shake256_blocksx3_seed_crypto_begin_%=:\n\t" "stp %[r], %x[seed], [x29, #48]\n\t" /* Col Mix */ "eor3 v31.16b, v0.16b, v5.16b, v10.16b\n\t" @@ -10017,1204 +10017,7 @@ void mlkem_shake256_blocksx3_seed_neon(word64* state, byte* seed) "mov v30.d[1], %x[state]\n\t" "eor x2, x2, %x[state]\n\t" "eor v0.16b, v0.16b, v30.16b\n\t" - "b.ne L_SHA3_shake256_blocksx3_seed_neon_begin_%=\n\t" - "ldr %x[state], [x29, #40]\n\t" - "st4 {v0.d, v1.d, v2.d, v3.d}[0], [%x[state]], #32\n\t" - "st4 {v4.d, v5.d, v6.d, v7.d}[0], [%x[state]], #32\n\t" - "st4 {v8.d, v9.d, v10.d, v11.d}[0], [%x[state]], #32\n\t" - "st4 {v12.d, v13.d, v14.d, v15.d}[0], [%x[state]], #32\n\t" - "st4 {v16.d, v17.d, v18.d, v19.d}[0], [%x[state]], #32\n\t" - "st4 {v20.d, v21.d, v22.d, v23.d}[0], [%x[state]], #32\n\t" - "st1 {v24.d}[0], [%x[state]]\n\t" - "add %x[state], %x[state], #8\n\t" - "st4 {v0.d, v1.d, v2.d, v3.d}[1], [%x[state]], #32\n\t" - "st4 {v4.d, v5.d, v6.d, v7.d}[1], [%x[state]], #32\n\t" - "st4 {v8.d, v9.d, v10.d, v11.d}[1], [%x[state]], #32\n\t" - "st4 {v12.d, v13.d, v14.d, v15.d}[1], [%x[state]], #32\n\t" - "st4 {v16.d, v17.d, v18.d, v19.d}[1], [%x[state]], #32\n\t" - "st4 {v20.d, v21.d, v22.d, v23.d}[1], [%x[state]], #32\n\t" - "st1 {v24.d}[1], [%x[state]]\n\t" - "add %x[state], %x[state], #8\n\t" - "stp x2, x3, [%x[state]]\n\t" - "stp x4, x5, [%x[state], #16]\n\t" - "stp x6, x7, [%x[state], #32]\n\t" - "stp x8, x9, [%x[state], #48]\n\t" - "stp x10, x11, [%x[state], #64]\n\t" - "stp x12, x13, [%x[state], #80]\n\t" - "stp x14, x15, [%x[state], #96]\n\t" - "stp x16, x17, [%x[state], #112]\n\t" - "stp x19, x20, [%x[state], #128]\n\t" - "stp x21, x22, [%x[state], #144]\n\t" - "stp x23, x24, [%x[state], #160]\n\t" - "stp x25, x26, [%x[state], #176]\n\t" - "str x27, [%x[state], #192]\n\t" - "ldp x29, x30, [sp], #0x40\n\t" - : [state] "+r" (state), [seed] "+r" (seed) - : [r] "r" (r) - : "memory", "cc", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", - "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", - "x21", "x22", "x23", "x24", "x25", "x26", "x27", "v0", "v1", "v2", - "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", - "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", - "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", - "v31" - ); -} - -#else -void mlkem_sha3_blocksx3_neon(word64* state) -{ - const word64* r = L_sha3_aarch64_r; - __asm__ __volatile__ ( - "stp x29, x30, [sp, #-64]!\n\t" - "add x29, sp, #0\n\t" - "str %x[state], [x29, #40]\n\t" - "ld4 {v0.d, v1.d, v2.d, v3.d}[0], [%x[state]], #32\n\t" - "ld4 {v4.d, v5.d, v6.d, v7.d}[0], [%x[state]], #32\n\t" - "ld4 {v8.d, v9.d, v10.d, v11.d}[0], [%x[state]], #32\n\t" - "ld4 {v12.d, v13.d, v14.d, v15.d}[0], [%x[state]], #32\n\t" - "ld4 {v16.d, v17.d, v18.d, v19.d}[0], [%x[state]], #32\n\t" - "ld4 {v20.d, v21.d, v22.d, v23.d}[0], [%x[state]], #32\n\t" - "ld1 {v24.d}[0], [%x[state]]\n\t" - "add %x[state], %x[state], #8\n\t" - "ld4 {v0.d, v1.d, v2.d, v3.d}[1], [%x[state]], #32\n\t" - "ld4 {v4.d, v5.d, v6.d, v7.d}[1], [%x[state]], #32\n\t" - "ld4 {v8.d, v9.d, v10.d, v11.d}[1], [%x[state]], #32\n\t" - "ld4 {v12.d, v13.d, v14.d, v15.d}[1], [%x[state]], #32\n\t" - "ld4 {v16.d, v17.d, v18.d, v19.d}[1], [%x[state]], #32\n\t" - "ld4 {v20.d, v21.d, v22.d, v23.d}[1], [%x[state]], #32\n\t" - "ld1 {v24.d}[1], [%x[state]]\n\t" - "add %x[state], %x[state], #8\n\t" - "ldp x1, x2, [%x[state]]\n\t" - "ldp x3, x4, [%x[state], #16]\n\t" - "ldp x5, x6, [%x[state], #32]\n\t" - "ldp x7, x8, [%x[state], #48]\n\t" - "ldp x9, x10, [%x[state], #64]\n\t" - "ldp x11, x12, [%x[state], #80]\n\t" - "ldp x13, x14, [%x[state], #96]\n\t" - "ldp x15, x16, [%x[state], #112]\n\t" - "ldp x17, x19, [%x[state], #128]\n\t" - "ldp x20, x21, [%x[state], #144]\n\t" - "ldp x22, x23, [%x[state], #160]\n\t" - "ldp x24, x25, [%x[state], #176]\n\t" - "ldr x26, [%x[state], #192]\n\t" - "mov x28, #24\n\t" - /* Start of 24 rounds */ - "\n" - "L_SHA3_transform_blocksx3_neon_begin_%=:\n\t" - "stp %[r], x28, [x29, #48]\n\t" - /* Col Mix NEON */ - "eor v30.16b, v4.16b, v9.16b\n\t" - "eor %x[state], x5, x10\n\t" - "eor v27.16b, v1.16b, v6.16b\n\t" - "eor x30, x1, x6\n\t" - "eor v30.16b, v30.16b, v14.16b\n\t" - "eor x28, x3, x8\n\t" - "eor v27.16b, v27.16b, v11.16b\n\t" - "eor %x[state], %x[state], x15\n\t" - "eor v30.16b, v30.16b, v19.16b\n\t" - "eor x30, x30, x11\n\t" - "eor v27.16b, v27.16b, v16.16b\n\t" - "eor x28, x28, x13\n\t" - "eor v30.16b, v30.16b, v24.16b\n\t" - "eor %x[state], %x[state], x21\n\t" - "eor v27.16b, v27.16b, v21.16b\n\t" - "eor x30, x30, x16\n\t" - "ushr v25.2d, v27.2d, #63\n\t" - "eor x28, x28, x19\n\t" - "sli v25.2d, v27.2d, #1\n\t" - "eor %x[state], %x[state], x26\n\t" - "eor v25.16b, v25.16b, v30.16b\n\t" - "eor x30, x30, x22\n\t" - "eor v31.16b, v0.16b, v5.16b\n\t" - "eor x28, x28, x24\n\t" - "eor v28.16b, v2.16b, v7.16b\n\t" - "str %x[state], [x29, #32]\n\t" - "eor v31.16b, v31.16b, v10.16b\n\t" - "str x28, [x29, #24]\n\t" - "eor v28.16b, v28.16b, v12.16b\n\t" - "eor %[r], x2, x7\n\t" - "eor v31.16b, v31.16b, v15.16b\n\t" - "eor x28, x4, x9\n\t" - "eor v28.16b, v28.16b, v17.16b\n\t" - "eor %[r], %[r], x12\n\t" - "eor v31.16b, v31.16b, v20.16b\n\t" - "eor x28, x28, x14\n\t" - "eor v28.16b, v28.16b, v22.16b\n\t" - "eor %[r], %[r], x17\n\t" - "ushr v29.2d, v30.2d, #63\n\t" - "eor x28, x28, x20\n\t" - "ushr v26.2d, v28.2d, #63\n\t" - "eor %[r], %[r], x23\n\t" - "sli v29.2d, v30.2d, #1\n\t" - "eor x28, x28, x25\n\t" - "sli v26.2d, v28.2d, #1\n\t" - "eor %x[state], %x[state], %[r], ror 63\n\t" - "eor v28.16b, v28.16b, v29.16b\n\t" - "eor %[r], %[r], x28, ror 63\n\t" - "eor v29.16b, v3.16b, v8.16b\n\t" - "eor x1, x1, %x[state]\n\t" - "eor v26.16b, v26.16b, v31.16b\n\t" - "eor x6, x6, %x[state]\n\t" - "eor v29.16b, v29.16b, v13.16b\n\t" - "eor x11, x11, %x[state]\n\t" - "eor v29.16b, v29.16b, v18.16b\n\t" - "eor x16, x16, %x[state]\n\t" - "eor v29.16b, v29.16b, v23.16b\n\t" - "eor x22, x22, %x[state]\n\t" - "ushr v30.2d, v29.2d, #63\n\t" - "eor x3, x3, %[r]\n\t" - "sli v30.2d, v29.2d, #1\n\t" - "eor x8, x8, %[r]\n\t" - "eor v27.16b, v27.16b, v30.16b\n\t" - "eor x13, x13, %[r]\n\t" - "ushr v30.2d, v31.2d, #63\n\t" - "eor x19, x19, %[r]\n\t" - "sli v30.2d, v31.2d, #1\n\t" - "eor x24, x24, %[r]\n\t" - "eor v29.16b, v29.16b, v30.16b\n\t" - "ldr %x[state], [x29, #32]\n\t" - /* Swap Rotate NEON */ - "eor v0.16b, v0.16b, v25.16b\n\t" - "eor v31.16b, v1.16b, v26.16b\n\t" - "ldr %[r], [x29, #24]\n\t" - "eor v6.16b, v6.16b, v26.16b\n\t" - "eor x28, x28, x30, ror 63\n\t" - "ushr v30.2d, v31.2d, #63\n\t" - "eor x30, x30, %[r], ror 63\n\t" - "ushr v1.2d, v6.2d, #20\n\t" - "eor %[r], %[r], %x[state], ror 63\n\t" - "sli v30.2d, v31.2d, #1\n\t" - "eor x5, x5, x28\n\t" - "sli v1.2d, v6.2d, #44\n\t" - "eor x10, x10, x28\n\t" - "eor v31.16b, v9.16b, v29.16b\n\t" - "eor x15, x15, x28\n\t" - "eor v22.16b, v22.16b, v27.16b\n\t" - "eor x21, x21, x28\n\t" - "ushr v6.2d, v31.2d, #44\n\t" - "eor x26, x26, x28\n\t" - "ushr v9.2d, v22.2d, #3\n\t" - "eor x2, x2, x30\n\t" - "sli v6.2d, v31.2d, #20\n\t" - "eor x7, x7, x30\n\t" - "sli v9.2d, v22.2d, #61\n\t" - "eor x12, x12, x30\n\t" - "eor v31.16b, v14.16b, v29.16b\n\t" - "eor x17, x17, x30\n\t" - "eor v20.16b, v20.16b, v25.16b\n\t" - "eor x23, x23, x30\n\t" - "ushr v22.2d, v31.2d, #25\n\t" - "eor x4, x4, %[r]\n\t" - "ushr v14.2d, v20.2d, #46\n\t" - "eor x9, x9, %[r]\n\t" - "sli v22.2d, v31.2d, #39\n\t" - "eor x14, x14, %[r]\n\t" - "sli v14.2d, v20.2d, #18\n\t" - "eor x20, x20, %[r]\n\t" - "eor v31.16b, v2.16b, v27.16b\n\t" - "eor x25, x25, %[r]\n\t" - /* Swap Rotate Base */ - "eor v12.16b, v12.16b, v27.16b\n\t" - "ror %x[state], x2, #63\n\t" - "ushr v20.2d, v31.2d, #2\n\t" - "ror x2, x7, #20\n\t" - "ushr v2.2d, v12.2d, #21\n\t" - "ror x7, x10, #44\n\t" - "sli v20.2d, v31.2d, #62\n\t" - "ror x10, x24, #3\n\t" - "sli v2.2d, v12.2d, #43\n\t" - "ror x24, x15, #25\n\t" - "eor v31.16b, v13.16b, v28.16b\n\t" - "ror x15, x22, #46\n\t" - "eor v19.16b, v19.16b, v29.16b\n\t" - "ror x22, x3, #2\n\t" - "ushr v12.2d, v31.2d, #39\n\t" - "ror x3, x13, #21\n\t" - "ushr v13.2d, v19.2d, #56\n\t" - "ror x13, x14, #39\n\t" - "sli v12.2d, v31.2d, #25\n\t" - "ror x14, x21, #56\n\t" - "sli v13.2d, v19.2d, #8\n\t" - "ror x21, x25, #8\n\t" - "eor v31.16b, v23.16b, v28.16b\n\t" - "ror x25, x16, #23\n\t" - "eor v15.16b, v15.16b, v25.16b\n\t" - "ror x16, x5, #37\n\t" - "ushr v19.2d, v31.2d, #8\n\t" - "ror x5, x26, #50\n\t" - "ushr v23.2d, v15.2d, #23\n\t" - "ror x26, x23, #62\n\t" - "sli v19.2d, v31.2d, #56\n\t" - "ror x23, x9, #9\n\t" - "sli v23.2d, v15.2d, #41\n\t" - "ror x9, x17, #19\n\t" - "eor v31.16b, v4.16b, v29.16b\n\t" - "ror x17, x6, #28\n\t" - "eor v24.16b, v24.16b, v29.16b\n\t" - "ror x6, x4, #36\n\t" - "ushr v15.2d, v31.2d, #37\n\t" - "ror x4, x20, #43\n\t" - "ushr v4.2d, v24.2d, #50\n\t" - "ror x20, x19, #49\n\t" - "sli v15.2d, v31.2d, #27\n\t" - "ror x19, x12, #54\n\t" - "sli v4.2d, v24.2d, #14\n\t" - "ror x12, x8, #58\n\t" - "eor v31.16b, v21.16b, v26.16b\n\t" - "ror x8, x11, #61\n\t" - /* Row Mix Base */ - "eor v8.16b, v8.16b, v28.16b\n\t" - "bic x11, x3, x2\n\t" - "ushr v24.2d, v31.2d, #62\n\t" - "bic %[r], x4, x3\n\t" - "ushr v21.2d, v8.2d, #9\n\t" - "bic x28, x1, x5\n\t" - "sli v24.2d, v31.2d, #2\n\t" - "bic x30, x2, x1\n\t" - "sli v21.2d, v8.2d, #55\n\t" - "eor x1, x1, x11\n\t" - "eor v31.16b, v16.16b, v26.16b\n\t" - "eor x2, x2, %[r]\n\t" - "eor v5.16b, v5.16b, v25.16b\n\t" - "bic x11, x5, x4\n\t" - "ushr v8.2d, v31.2d, #19\n\t" - "eor x4, x4, x28\n\t" - "ushr v16.2d, v5.2d, #28\n\t" - "eor x3, x3, x11\n\t" - "sli v8.2d, v31.2d, #45\n\t" - "eor x5, x5, x30\n\t" - "sli v16.2d, v5.2d, #36\n\t" - "bic x11, x8, x7\n\t" - "eor v31.16b, v3.16b, v28.16b\n\t" - "bic %[r], x9, x8\n\t" - "eor v18.16b, v18.16b, v28.16b\n\t" - "bic x28, x6, x10\n\t" - "ushr v5.2d, v31.2d, #36\n\t" - "bic x30, x7, x6\n\t" - "ushr v3.2d, v18.2d, #43\n\t" - "eor x6, x6, x11\n\t" - "sli v5.2d, v31.2d, #28\n\t" - "eor x7, x7, %[r]\n\t" - "sli v3.2d, v18.2d, #21\n\t" - "bic x11, x10, x9\n\t" - "eor v31.16b, v17.16b, v27.16b\n\t" - "eor x9, x9, x28\n\t" - "eor v11.16b, v11.16b, v26.16b\n\t" - "eor x8, x8, x11\n\t" - "ushr v18.2d, v31.2d, #49\n\t" - "eor x10, x10, x30\n\t" - "ushr v17.2d, v11.2d, #54\n\t" - "bic x11, x13, x12\n\t" - "sli v18.2d, v31.2d, #15\n\t" - "bic %[r], x14, x13\n\t" - "sli v17.2d, v11.2d, #10\n\t" - "bic x28, %x[state], x15\n\t" - "eor v31.16b, v7.16b, v27.16b\n\t" - "bic x30, x12, %x[state]\n\t" - "eor v10.16b, v10.16b, v25.16b\n\t" - "eor x11, %x[state], x11\n\t" - "ushr v11.2d, v31.2d, #58\n\t" - "eor x12, x12, %[r]\n\t" - "ushr v7.2d, v10.2d, #61\n\t" - "bic %x[state], x15, x14\n\t" - "sli v11.2d, v31.2d, #6\n\t" - "eor x14, x14, x28\n\t" - "sli v7.2d, v10.2d, #3\n\t" - "eor x13, x13, %x[state]\n\t" - /* Row Mix NEON */ - "bic v25.16b, v2.16b, v1.16b\n\t" - "eor x15, x15, x30\n\t" - "bic v26.16b, v3.16b, v2.16b\n\t" - "bic %x[state], x19, x17\n\t" - "bic v27.16b, v4.16b, v3.16b\n\t" - "bic %[r], x20, x19\n\t" - "bic v28.16b, v0.16b, v4.16b\n\t" - "bic x28, x16, x21\n\t" - "bic v29.16b, v1.16b, v0.16b\n\t" - "bic x30, x17, x16\n\t" - "eor v0.16b, v0.16b, v25.16b\n\t" - "eor x16, x16, %x[state]\n\t" - "eor v1.16b, v1.16b, v26.16b\n\t" - "eor x17, x17, %[r]\n\t" - "eor v2.16b, v2.16b, v27.16b\n\t" - "bic %x[state], x21, x20\n\t" - "eor v3.16b, v3.16b, v28.16b\n\t" - "eor x20, x20, x28\n\t" - "eor v4.16b, v4.16b, v29.16b\n\t" - "eor x19, x19, %x[state]\n\t" - "bic v25.16b, v7.16b, v6.16b\n\t" - "eor x21, x21, x30\n\t" - "bic v26.16b, v8.16b, v7.16b\n\t" - "bic %x[state], x24, x23\n\t" - "bic v27.16b, v9.16b, v8.16b\n\t" - "bic %[r], x25, x24\n\t" - "bic v28.16b, v5.16b, v9.16b\n\t" - "bic x28, x22, x26\n\t" - "bic v29.16b, v6.16b, v5.16b\n\t" - "bic x30, x23, x22\n\t" - "eor v5.16b, v5.16b, v25.16b\n\t" - "eor x22, x22, %x[state]\n\t" - "eor v6.16b, v6.16b, v26.16b\n\t" - "eor x23, x23, %[r]\n\t" - "eor v7.16b, v7.16b, v27.16b\n\t" - "bic %x[state], x26, x25\n\t" - "eor v8.16b, v8.16b, v28.16b\n\t" - "eor x25, x25, x28\n\t" - "eor v9.16b, v9.16b, v29.16b\n\t" - "eor x24, x24, %x[state]\n\t" - "bic v25.16b, v12.16b, v11.16b\n\t" - "eor x26, x26, x30\n\t" - "bic v26.16b, v13.16b, v12.16b\n\t" - "bic v27.16b, v14.16b, v13.16b\n\t" - "bic v28.16b, v30.16b, v14.16b\n\t" - "bic v29.16b, v11.16b, v30.16b\n\t" - "eor v10.16b, v30.16b, v25.16b\n\t" - "eor v11.16b, v11.16b, v26.16b\n\t" - "eor v12.16b, v12.16b, v27.16b\n\t" - "eor v13.16b, v13.16b, v28.16b\n\t" - "eor v14.16b, v14.16b, v29.16b\n\t" - "bic v25.16b, v17.16b, v16.16b\n\t" - "bic v26.16b, v18.16b, v17.16b\n\t" - "bic v27.16b, v19.16b, v18.16b\n\t" - "bic v28.16b, v15.16b, v19.16b\n\t" - "bic v29.16b, v16.16b, v15.16b\n\t" - "eor v15.16b, v15.16b, v25.16b\n\t" - "eor v16.16b, v16.16b, v26.16b\n\t" - "eor v17.16b, v17.16b, v27.16b\n\t" - "eor v18.16b, v18.16b, v28.16b\n\t" - "eor v19.16b, v19.16b, v29.16b\n\t" - "bic v25.16b, v22.16b, v21.16b\n\t" - "bic v26.16b, v23.16b, v22.16b\n\t" - "bic v27.16b, v24.16b, v23.16b\n\t" - "bic v28.16b, v20.16b, v24.16b\n\t" - "bic v29.16b, v21.16b, v20.16b\n\t" - "eor v20.16b, v20.16b, v25.16b\n\t" - "eor v21.16b, v21.16b, v26.16b\n\t" - "eor v22.16b, v22.16b, v27.16b\n\t" - "eor v23.16b, v23.16b, v28.16b\n\t" - "eor v24.16b, v24.16b, v29.16b\n\t" - /* Done transforming */ - "ldp %[r], x28, [x29, #48]\n\t" - "ldr %x[state], [%[r]], #8\n\t" - "subs x28, x28, #1\n\t" - "mov v30.d[0], %x[state]\n\t" - "mov v30.d[1], %x[state]\n\t" - "eor x1, x1, %x[state]\n\t" - "eor v0.16b, v0.16b, v30.16b\n\t" - "b.ne L_SHA3_transform_blocksx3_neon_begin_%=\n\t" - "ldr %x[state], [x29, #40]\n\t" - "st4 {v0.d, v1.d, v2.d, v3.d}[0], [%x[state]], #32\n\t" - "st4 {v4.d, v5.d, v6.d, v7.d}[0], [%x[state]], #32\n\t" - "st4 {v8.d, v9.d, v10.d, v11.d}[0], [%x[state]], #32\n\t" - "st4 {v12.d, v13.d, v14.d, v15.d}[0], [%x[state]], #32\n\t" - "st4 {v16.d, v17.d, v18.d, v19.d}[0], [%x[state]], #32\n\t" - "st4 {v20.d, v21.d, v22.d, v23.d}[0], [%x[state]], #32\n\t" - "st1 {v24.d}[0], [%x[state]]\n\t" - "add %x[state], %x[state], #8\n\t" - "st4 {v0.d, v1.d, v2.d, v3.d}[1], [%x[state]], #32\n\t" - "st4 {v4.d, v5.d, v6.d, v7.d}[1], [%x[state]], #32\n\t" - "st4 {v8.d, v9.d, v10.d, v11.d}[1], [%x[state]], #32\n\t" - "st4 {v12.d, v13.d, v14.d, v15.d}[1], [%x[state]], #32\n\t" - "st4 {v16.d, v17.d, v18.d, v19.d}[1], [%x[state]], #32\n\t" - "st4 {v20.d, v21.d, v22.d, v23.d}[1], [%x[state]], #32\n\t" - "st1 {v24.d}[1], [%x[state]]\n\t" - "add %x[state], %x[state], #8\n\t" - "stp x1, x2, [%x[state]]\n\t" - "stp x3, x4, [%x[state], #16]\n\t" - "stp x5, x6, [%x[state], #32]\n\t" - "stp x7, x8, [%x[state], #48]\n\t" - "stp x9, x10, [%x[state], #64]\n\t" - "stp x11, x12, [%x[state], #80]\n\t" - "stp x13, x14, [%x[state], #96]\n\t" - "stp x15, x16, [%x[state], #112]\n\t" - "stp x17, x19, [%x[state], #128]\n\t" - "stp x20, x21, [%x[state], #144]\n\t" - "stp x22, x23, [%x[state], #160]\n\t" - "stp x24, x25, [%x[state], #176]\n\t" - "str x26, [%x[state], #192]\n\t" - "ldp x29, x30, [sp], #0x40\n\t" - : [state] "+r" (state) - : [r] "r" (r) - : "memory", "cc", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", - "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", - "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x28", "v0", "v1", - "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", - "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", - "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", - "v31" - ); -} - -void mlkem_shake128_blocksx3_seed_neon(word64* state, byte* seed) -{ - const word64* r = L_sha3_aarch64_r; - __asm__ __volatile__ ( - "stp x29, x30, [sp, #-64]!\n\t" - "add x29, sp, #0\n\t" - "str %x[state], [x29, #40]\n\t" - "add %x[state], %x[state], #32\n\t" - "ld1 {v4.d}[0], [%x[state]]\n\t" - "ldp x2, x3, [%x[seed]], #16\n\t" - "add %x[state], %x[state], #0xc8\n\t" - "ld1 {v4.d}[1], [%x[state]]\n\t" - "ldp x4, x5, [%x[seed]], #16\n\t" - "ldr x6, [%x[state], #200]\n\t" - "eor v5.16b, v5.16b, v5.16b\n\t" - "eor x7, x7, x7\n\t" - "eor v6.16b, v6.16b, v6.16b\n\t" - "eor x8, x8, x8\n\t" - "eor v7.16b, v7.16b, v7.16b\n\t" - "eor x9, x9, x9\n\t" - "eor v8.16b, v8.16b, v8.16b\n\t" - "eor x10, x10, x10\n\t" - "eor v9.16b, v9.16b, v9.16b\n\t" - "eor x11, x11, x11\n\t" - "eor v10.16b, v10.16b, v10.16b\n\t" - "eor x12, x12, x12\n\t" - "eor v11.16b, v11.16b, v11.16b\n\t" - "eor x13, x13, x13\n\t" - "eor v12.16b, v12.16b, v12.16b\n\t" - "eor x14, x14, x14\n\t" - "eor v13.16b, v13.16b, v13.16b\n\t" - "eor x15, x15, x15\n\t" - "eor v14.16b, v14.16b, v14.16b\n\t" - "eor x16, x16, x16\n\t" - "eor v15.16b, v15.16b, v15.16b\n\t" - "eor x17, x17, x17\n\t" - "eor v16.16b, v16.16b, v16.16b\n\t" - "eor x19, x19, x19\n\t" - "eor v17.16b, v17.16b, v17.16b\n\t" - "eor x20, x20, x20\n\t" - "eor v18.16b, v18.16b, v18.16b\n\t" - "eor x21, x21, x21\n\t" - "eor v19.16b, v19.16b, v19.16b\n\t" - "eor x22, x22, x22\n\t" - "movz x23, #0x8000, lsl 48\n\t" - "eor v21.16b, v21.16b, v21.16b\n\t" - "eor x24, x24, x24\n\t" - "eor v22.16b, v22.16b, v22.16b\n\t" - "eor x25, x25, x25\n\t" - "eor v23.16b, v23.16b, v23.16b\n\t" - "eor x26, x26, x26\n\t" - "eor v24.16b, v24.16b, v24.16b\n\t" - "eor x27, x27, x27\n\t" - "dup v0.2d, x2\n\t" - "dup v1.2d, x3\n\t" - "dup v2.2d, x4\n\t" - "dup v3.2d, x5\n\t" - "dup v20.2d, x23\n\t" - "mov %x[seed], #24\n\t" - /* Start of 24 rounds */ - "\n" - "L_SHA3_shake128_blocksx3_seed_neon_begin_%=:\n\t" - "stp %[r], %x[seed], [x29, #48]\n\t" - /* Col Mix NEON */ - "eor v30.16b, v4.16b, v9.16b\n\t" - "eor %x[state], x6, x11\n\t" - "eor v27.16b, v1.16b, v6.16b\n\t" - "eor x30, x2, x7\n\t" - "eor v30.16b, v30.16b, v14.16b\n\t" - "eor %[r], x4, x9\n\t" - "eor v27.16b, v27.16b, v11.16b\n\t" - "eor %x[state], %x[state], x16\n\t" - "eor v30.16b, v30.16b, v19.16b\n\t" - "eor x30, x30, x12\n\t" - "eor v27.16b, v27.16b, v16.16b\n\t" - "eor %[r], %[r], x14\n\t" - "eor v30.16b, v30.16b, v24.16b\n\t" - "eor %x[state], %x[state], x22\n\t" - "eor v27.16b, v27.16b, v21.16b\n\t" - "eor x30, x30, x17\n\t" - "ushr v25.2d, v27.2d, #63\n\t" - "eor %[r], %[r], x20\n\t" - "sli v25.2d, v27.2d, #1\n\t" - "eor %x[state], %x[state], x27\n\t" - "eor v25.16b, v25.16b, v30.16b\n\t" - "eor x30, x30, x23\n\t" - "eor v31.16b, v0.16b, v5.16b\n\t" - "eor %[r], %[r], x25\n\t" - "eor v28.16b, v2.16b, v7.16b\n\t" - "str %x[state], [x29, #32]\n\t" - "eor v31.16b, v31.16b, v10.16b\n\t" - "str %[r], [x29, #24]\n\t" - "eor v28.16b, v28.16b, v12.16b\n\t" - "eor %x[seed], x3, x8\n\t" - "eor v31.16b, v31.16b, v15.16b\n\t" - "eor %[r], x5, x10\n\t" - "eor v28.16b, v28.16b, v17.16b\n\t" - "eor %x[seed], %x[seed], x13\n\t" - "eor v31.16b, v31.16b, v20.16b\n\t" - "eor %[r], %[r], x15\n\t" - "eor v28.16b, v28.16b, v22.16b\n\t" - "eor %x[seed], %x[seed], x19\n\t" - "ushr v29.2d, v30.2d, #63\n\t" - "eor %[r], %[r], x21\n\t" - "ushr v26.2d, v28.2d, #63\n\t" - "eor %x[seed], %x[seed], x24\n\t" - "sli v29.2d, v30.2d, #1\n\t" - "eor %[r], %[r], x26\n\t" - "sli v26.2d, v28.2d, #1\n\t" - "eor %x[state], %x[state], %x[seed], ror 63\n\t" - "eor v28.16b, v28.16b, v29.16b\n\t" - "eor %x[seed], %x[seed], %[r], ror 63\n\t" - "eor v29.16b, v3.16b, v8.16b\n\t" - "eor x2, x2, %x[state]\n\t" - "eor v26.16b, v26.16b, v31.16b\n\t" - "eor x7, x7, %x[state]\n\t" - "eor v29.16b, v29.16b, v13.16b\n\t" - "eor x12, x12, %x[state]\n\t" - "eor v29.16b, v29.16b, v18.16b\n\t" - "eor x17, x17, %x[state]\n\t" - "eor v29.16b, v29.16b, v23.16b\n\t" - "eor x23, x23, %x[state]\n\t" - "ushr v30.2d, v29.2d, #63\n\t" - "eor x4, x4, %x[seed]\n\t" - "sli v30.2d, v29.2d, #1\n\t" - "eor x9, x9, %x[seed]\n\t" - "eor v27.16b, v27.16b, v30.16b\n\t" - "eor x14, x14, %x[seed]\n\t" - "ushr v30.2d, v31.2d, #63\n\t" - "eor x20, x20, %x[seed]\n\t" - "sli v30.2d, v31.2d, #1\n\t" - "eor x25, x25, %x[seed]\n\t" - "eor v29.16b, v29.16b, v30.16b\n\t" - "ldr %x[state], [x29, #32]\n\t" - /* Swap Rotate NEON */ - "eor v0.16b, v0.16b, v25.16b\n\t" - "eor v31.16b, v1.16b, v26.16b\n\t" - "ldr %x[seed], [x29, #24]\n\t" - "eor v6.16b, v6.16b, v26.16b\n\t" - "eor %[r], %[r], x30, ror 63\n\t" - "ushr v30.2d, v31.2d, #63\n\t" - "eor x30, x30, %x[seed], ror 63\n\t" - "ushr v1.2d, v6.2d, #20\n\t" - "eor %x[seed], %x[seed], %x[state], ror 63\n\t" - "sli v30.2d, v31.2d, #1\n\t" - "eor x6, x6, %[r]\n\t" - "sli v1.2d, v6.2d, #44\n\t" - "eor x11, x11, %[r]\n\t" - "eor v31.16b, v9.16b, v29.16b\n\t" - "eor x16, x16, %[r]\n\t" - "eor v22.16b, v22.16b, v27.16b\n\t" - "eor x22, x22, %[r]\n\t" - "ushr v6.2d, v31.2d, #44\n\t" - "eor x27, x27, %[r]\n\t" - "ushr v9.2d, v22.2d, #3\n\t" - "eor x3, x3, x30\n\t" - "sli v6.2d, v31.2d, #20\n\t" - "eor x8, x8, x30\n\t" - "sli v9.2d, v22.2d, #61\n\t" - "eor x13, x13, x30\n\t" - "eor v31.16b, v14.16b, v29.16b\n\t" - "eor x19, x19, x30\n\t" - "eor v20.16b, v20.16b, v25.16b\n\t" - "eor x24, x24, x30\n\t" - "ushr v22.2d, v31.2d, #25\n\t" - "eor x5, x5, %x[seed]\n\t" - "ushr v14.2d, v20.2d, #46\n\t" - "eor x10, x10, %x[seed]\n\t" - "sli v22.2d, v31.2d, #39\n\t" - "eor x15, x15, %x[seed]\n\t" - "sli v14.2d, v20.2d, #18\n\t" - "eor x21, x21, %x[seed]\n\t" - "eor v31.16b, v2.16b, v27.16b\n\t" - "eor x26, x26, %x[seed]\n\t" - /* Swap Rotate Base */ - "eor v12.16b, v12.16b, v27.16b\n\t" - "ror %x[state], x3, #63\n\t" - "ushr v20.2d, v31.2d, #2\n\t" - "ror x3, x8, #20\n\t" - "ushr v2.2d, v12.2d, #21\n\t" - "ror x8, x11, #44\n\t" - "sli v20.2d, v31.2d, #62\n\t" - "ror x11, x25, #3\n\t" - "sli v2.2d, v12.2d, #43\n\t" - "ror x25, x16, #25\n\t" - "eor v31.16b, v13.16b, v28.16b\n\t" - "ror x16, x23, #46\n\t" - "eor v19.16b, v19.16b, v29.16b\n\t" - "ror x23, x4, #2\n\t" - "ushr v12.2d, v31.2d, #39\n\t" - "ror x4, x14, #21\n\t" - "ushr v13.2d, v19.2d, #56\n\t" - "ror x14, x15, #39\n\t" - "sli v12.2d, v31.2d, #25\n\t" - "ror x15, x22, #56\n\t" - "sli v13.2d, v19.2d, #8\n\t" - "ror x22, x26, #8\n\t" - "eor v31.16b, v23.16b, v28.16b\n\t" - "ror x26, x17, #23\n\t" - "eor v15.16b, v15.16b, v25.16b\n\t" - "ror x17, x6, #37\n\t" - "ushr v19.2d, v31.2d, #8\n\t" - "ror x6, x27, #50\n\t" - "ushr v23.2d, v15.2d, #23\n\t" - "ror x27, x24, #62\n\t" - "sli v19.2d, v31.2d, #56\n\t" - "ror x24, x10, #9\n\t" - "sli v23.2d, v15.2d, #41\n\t" - "ror x10, x19, #19\n\t" - "eor v31.16b, v4.16b, v29.16b\n\t" - "ror x19, x7, #28\n\t" - "eor v24.16b, v24.16b, v29.16b\n\t" - "ror x7, x5, #36\n\t" - "ushr v15.2d, v31.2d, #37\n\t" - "ror x5, x21, #43\n\t" - "ushr v4.2d, v24.2d, #50\n\t" - "ror x21, x20, #49\n\t" - "sli v15.2d, v31.2d, #27\n\t" - "ror x20, x13, #54\n\t" - "sli v4.2d, v24.2d, #14\n\t" - "ror x13, x9, #58\n\t" - "eor v31.16b, v21.16b, v26.16b\n\t" - "ror x9, x12, #61\n\t" - /* Row Mix Base */ - "eor v8.16b, v8.16b, v28.16b\n\t" - "bic x12, x4, x3\n\t" - "ushr v24.2d, v31.2d, #62\n\t" - "bic %x[seed], x5, x4\n\t" - "ushr v21.2d, v8.2d, #9\n\t" - "bic %[r], x2, x6\n\t" - "sli v24.2d, v31.2d, #2\n\t" - "bic x30, x3, x2\n\t" - "sli v21.2d, v8.2d, #55\n\t" - "eor x2, x2, x12\n\t" - "eor v31.16b, v16.16b, v26.16b\n\t" - "eor x3, x3, %x[seed]\n\t" - "eor v5.16b, v5.16b, v25.16b\n\t" - "bic x12, x6, x5\n\t" - "ushr v8.2d, v31.2d, #19\n\t" - "eor x5, x5, %[r]\n\t" - "ushr v16.2d, v5.2d, #28\n\t" - "eor x4, x4, x12\n\t" - "sli v8.2d, v31.2d, #45\n\t" - "eor x6, x6, x30\n\t" - "sli v16.2d, v5.2d, #36\n\t" - "bic x12, x9, x8\n\t" - "eor v31.16b, v3.16b, v28.16b\n\t" - "bic %x[seed], x10, x9\n\t" - "eor v18.16b, v18.16b, v28.16b\n\t" - "bic %[r], x7, x11\n\t" - "ushr v5.2d, v31.2d, #36\n\t" - "bic x30, x8, x7\n\t" - "ushr v3.2d, v18.2d, #43\n\t" - "eor x7, x7, x12\n\t" - "sli v5.2d, v31.2d, #28\n\t" - "eor x8, x8, %x[seed]\n\t" - "sli v3.2d, v18.2d, #21\n\t" - "bic x12, x11, x10\n\t" - "eor v31.16b, v17.16b, v27.16b\n\t" - "eor x10, x10, %[r]\n\t" - "eor v11.16b, v11.16b, v26.16b\n\t" - "eor x9, x9, x12\n\t" - "ushr v18.2d, v31.2d, #49\n\t" - "eor x11, x11, x30\n\t" - "ushr v17.2d, v11.2d, #54\n\t" - "bic x12, x14, x13\n\t" - "sli v18.2d, v31.2d, #15\n\t" - "bic %x[seed], x15, x14\n\t" - "sli v17.2d, v11.2d, #10\n\t" - "bic %[r], %x[state], x16\n\t" - "eor v31.16b, v7.16b, v27.16b\n\t" - "bic x30, x13, %x[state]\n\t" - "eor v10.16b, v10.16b, v25.16b\n\t" - "eor x12, %x[state], x12\n\t" - "ushr v11.2d, v31.2d, #58\n\t" - "eor x13, x13, %x[seed]\n\t" - "ushr v7.2d, v10.2d, #61\n\t" - "bic %x[state], x16, x15\n\t" - "sli v11.2d, v31.2d, #6\n\t" - "eor x15, x15, %[r]\n\t" - "sli v7.2d, v10.2d, #3\n\t" - "eor x14, x14, %x[state]\n\t" - /* Row Mix NEON */ - "bic v25.16b, v2.16b, v1.16b\n\t" - "eor x16, x16, x30\n\t" - "bic v26.16b, v3.16b, v2.16b\n\t" - "bic %x[state], x20, x19\n\t" - "bic v27.16b, v4.16b, v3.16b\n\t" - "bic %x[seed], x21, x20\n\t" - "bic v28.16b, v0.16b, v4.16b\n\t" - "bic %[r], x17, x22\n\t" - "bic v29.16b, v1.16b, v0.16b\n\t" - "bic x30, x19, x17\n\t" - "eor v0.16b, v0.16b, v25.16b\n\t" - "eor x17, x17, %x[state]\n\t" - "eor v1.16b, v1.16b, v26.16b\n\t" - "eor x19, x19, %x[seed]\n\t" - "eor v2.16b, v2.16b, v27.16b\n\t" - "bic %x[state], x22, x21\n\t" - "eor v3.16b, v3.16b, v28.16b\n\t" - "eor x21, x21, %[r]\n\t" - "eor v4.16b, v4.16b, v29.16b\n\t" - "eor x20, x20, %x[state]\n\t" - "bic v25.16b, v7.16b, v6.16b\n\t" - "eor x22, x22, x30\n\t" - "bic v26.16b, v8.16b, v7.16b\n\t" - "bic %x[state], x25, x24\n\t" - "bic v27.16b, v9.16b, v8.16b\n\t" - "bic %x[seed], x26, x25\n\t" - "bic v28.16b, v5.16b, v9.16b\n\t" - "bic %[r], x23, x27\n\t" - "bic v29.16b, v6.16b, v5.16b\n\t" - "bic x30, x24, x23\n\t" - "eor v5.16b, v5.16b, v25.16b\n\t" - "eor x23, x23, %x[state]\n\t" - "eor v6.16b, v6.16b, v26.16b\n\t" - "eor x24, x24, %x[seed]\n\t" - "eor v7.16b, v7.16b, v27.16b\n\t" - "bic %x[state], x27, x26\n\t" - "eor v8.16b, v8.16b, v28.16b\n\t" - "eor x26, x26, %[r]\n\t" - "eor v9.16b, v9.16b, v29.16b\n\t" - "eor x25, x25, %x[state]\n\t" - "bic v25.16b, v12.16b, v11.16b\n\t" - "eor x27, x27, x30\n\t" - "bic v26.16b, v13.16b, v12.16b\n\t" - "bic v27.16b, v14.16b, v13.16b\n\t" - "bic v28.16b, v30.16b, v14.16b\n\t" - "bic v29.16b, v11.16b, v30.16b\n\t" - "eor v10.16b, v30.16b, v25.16b\n\t" - "eor v11.16b, v11.16b, v26.16b\n\t" - "eor v12.16b, v12.16b, v27.16b\n\t" - "eor v13.16b, v13.16b, v28.16b\n\t" - "eor v14.16b, v14.16b, v29.16b\n\t" - "bic v25.16b, v17.16b, v16.16b\n\t" - "bic v26.16b, v18.16b, v17.16b\n\t" - "bic v27.16b, v19.16b, v18.16b\n\t" - "bic v28.16b, v15.16b, v19.16b\n\t" - "bic v29.16b, v16.16b, v15.16b\n\t" - "eor v15.16b, v15.16b, v25.16b\n\t" - "eor v16.16b, v16.16b, v26.16b\n\t" - "eor v17.16b, v17.16b, v27.16b\n\t" - "eor v18.16b, v18.16b, v28.16b\n\t" - "eor v19.16b, v19.16b, v29.16b\n\t" - "bic v25.16b, v22.16b, v21.16b\n\t" - "bic v26.16b, v23.16b, v22.16b\n\t" - "bic v27.16b, v24.16b, v23.16b\n\t" - "bic v28.16b, v20.16b, v24.16b\n\t" - "bic v29.16b, v21.16b, v20.16b\n\t" - "eor v20.16b, v20.16b, v25.16b\n\t" - "eor v21.16b, v21.16b, v26.16b\n\t" - "eor v22.16b, v22.16b, v27.16b\n\t" - "eor v23.16b, v23.16b, v28.16b\n\t" - "eor v24.16b, v24.16b, v29.16b\n\t" - /* Done transforming */ - "ldp %[r], %x[seed], [x29, #48]\n\t" - "ldr %x[state], [%[r]], #8\n\t" - "subs %x[seed], %x[seed], #1\n\t" - "mov v30.d[0], %x[state]\n\t" - "mov v30.d[1], %x[state]\n\t" - "eor x2, x2, %x[state]\n\t" - "eor v0.16b, v0.16b, v30.16b\n\t" - "b.ne L_SHA3_shake128_blocksx3_seed_neon_begin_%=\n\t" - "ldr %x[state], [x29, #40]\n\t" - "st4 {v0.d, v1.d, v2.d, v3.d}[0], [%x[state]], #32\n\t" - "st4 {v4.d, v5.d, v6.d, v7.d}[0], [%x[state]], #32\n\t" - "st4 {v8.d, v9.d, v10.d, v11.d}[0], [%x[state]], #32\n\t" - "st4 {v12.d, v13.d, v14.d, v15.d}[0], [%x[state]], #32\n\t" - "st4 {v16.d, v17.d, v18.d, v19.d}[0], [%x[state]], #32\n\t" - "st4 {v20.d, v21.d, v22.d, v23.d}[0], [%x[state]], #32\n\t" - "st1 {v24.d}[0], [%x[state]]\n\t" - "add %x[state], %x[state], #8\n\t" - "st4 {v0.d, v1.d, v2.d, v3.d}[1], [%x[state]], #32\n\t" - "st4 {v4.d, v5.d, v6.d, v7.d}[1], [%x[state]], #32\n\t" - "st4 {v8.d, v9.d, v10.d, v11.d}[1], [%x[state]], #32\n\t" - "st4 {v12.d, v13.d, v14.d, v15.d}[1], [%x[state]], #32\n\t" - "st4 {v16.d, v17.d, v18.d, v19.d}[1], [%x[state]], #32\n\t" - "st4 {v20.d, v21.d, v22.d, v23.d}[1], [%x[state]], #32\n\t" - "st1 {v24.d}[1], [%x[state]]\n\t" - "add %x[state], %x[state], #8\n\t" - "stp x2, x3, [%x[state]]\n\t" - "stp x4, x5, [%x[state], #16]\n\t" - "stp x6, x7, [%x[state], #32]\n\t" - "stp x8, x9, [%x[state], #48]\n\t" - "stp x10, x11, [%x[state], #64]\n\t" - "stp x12, x13, [%x[state], #80]\n\t" - "stp x14, x15, [%x[state], #96]\n\t" - "stp x16, x17, [%x[state], #112]\n\t" - "stp x19, x20, [%x[state], #128]\n\t" - "stp x21, x22, [%x[state], #144]\n\t" - "stp x23, x24, [%x[state], #160]\n\t" - "stp x25, x26, [%x[state], #176]\n\t" - "str x27, [%x[state], #192]\n\t" - "ldp x29, x30, [sp], #0x40\n\t" - : [state] "+r" (state), [seed] "+r" (seed) - : [r] "r" (r) - : "memory", "cc", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", - "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", - "x21", "x22", "x23", "x24", "x25", "x26", "x27", "v0", "v1", "v2", - "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", - "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", - "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", - "v31" - ); -} - -void mlkem_shake256_blocksx3_seed_neon(word64* state, byte* seed) -{ - const word64* r = L_sha3_aarch64_r; - __asm__ __volatile__ ( - "stp x29, x30, [sp, #-64]!\n\t" - "add x29, sp, #0\n\t" - "str %x[state], [x29, #40]\n\t" - "add %x[state], %x[state], #32\n\t" - "ld1 {v4.d}[0], [%x[state]]\n\t" - "ldp x2, x3, [%x[seed]], #16\n\t" - "add %x[state], %x[state], #0xc8\n\t" - "ld1 {v4.d}[1], [%x[state]]\n\t" - "ldp x4, x5, [%x[seed]], #16\n\t" - "ldr x6, [%x[state], #200]\n\t" - "eor v5.16b, v5.16b, v5.16b\n\t" - "eor x7, x7, x7\n\t" - "eor v6.16b, v6.16b, v6.16b\n\t" - "eor x8, x8, x8\n\t" - "eor v7.16b, v7.16b, v7.16b\n\t" - "eor x9, x9, x9\n\t" - "eor v8.16b, v8.16b, v8.16b\n\t" - "eor x10, x10, x10\n\t" - "eor v9.16b, v9.16b, v9.16b\n\t" - "eor x11, x11, x11\n\t" - "eor v10.16b, v10.16b, v10.16b\n\t" - "eor x12, x12, x12\n\t" - "eor v11.16b, v11.16b, v11.16b\n\t" - "eor x13, x13, x13\n\t" - "eor v12.16b, v12.16b, v12.16b\n\t" - "eor x14, x14, x14\n\t" - "eor v13.16b, v13.16b, v13.16b\n\t" - "eor x15, x15, x15\n\t" - "eor v14.16b, v14.16b, v14.16b\n\t" - "eor x16, x16, x16\n\t" - "eor v15.16b, v15.16b, v15.16b\n\t" - "eor x17, x17, x17\n\t" - "movz x19, #0x8000, lsl 48\n\t" - "eor v17.16b, v17.16b, v17.16b\n\t" - "eor x20, x20, x20\n\t" - "eor v18.16b, v18.16b, v18.16b\n\t" - "eor x21, x21, x21\n\t" - "eor v19.16b, v19.16b, v19.16b\n\t" - "eor x22, x22, x22\n\t" - "eor v20.16b, v20.16b, v20.16b\n\t" - "eor x23, x23, x23\n\t" - "eor v21.16b, v21.16b, v21.16b\n\t" - "eor x24, x24, x24\n\t" - "eor v22.16b, v22.16b, v22.16b\n\t" - "eor x25, x25, x25\n\t" - "eor v23.16b, v23.16b, v23.16b\n\t" - "eor x26, x26, x26\n\t" - "eor v24.16b, v24.16b, v24.16b\n\t" - "eor x27, x27, x27\n\t" - "dup v0.2d, x2\n\t" - "dup v1.2d, x3\n\t" - "dup v2.2d, x4\n\t" - "dup v3.2d, x5\n\t" - "dup v16.2d, x19\n\t" - "mov %x[seed], #24\n\t" - /* Start of 24 rounds */ - "\n" - "L_SHA3_shake256_blocksx3_seed_neon_begin_%=:\n\t" - "stp %[r], %x[seed], [x29, #48]\n\t" - /* Col Mix NEON */ - "eor v30.16b, v4.16b, v9.16b\n\t" - "eor %x[state], x6, x11\n\t" - "eor v27.16b, v1.16b, v6.16b\n\t" - "eor x30, x2, x7\n\t" - "eor v30.16b, v30.16b, v14.16b\n\t" - "eor %[r], x4, x9\n\t" - "eor v27.16b, v27.16b, v11.16b\n\t" - "eor %x[state], %x[state], x16\n\t" - "eor v30.16b, v30.16b, v19.16b\n\t" - "eor x30, x30, x12\n\t" - "eor v27.16b, v27.16b, v16.16b\n\t" - "eor %[r], %[r], x14\n\t" - "eor v30.16b, v30.16b, v24.16b\n\t" - "eor %x[state], %x[state], x22\n\t" - "eor v27.16b, v27.16b, v21.16b\n\t" - "eor x30, x30, x17\n\t" - "ushr v25.2d, v27.2d, #63\n\t" - "eor %[r], %[r], x20\n\t" - "sli v25.2d, v27.2d, #1\n\t" - "eor %x[state], %x[state], x27\n\t" - "eor v25.16b, v25.16b, v30.16b\n\t" - "eor x30, x30, x23\n\t" - "eor v31.16b, v0.16b, v5.16b\n\t" - "eor %[r], %[r], x25\n\t" - "eor v28.16b, v2.16b, v7.16b\n\t" - "str %x[state], [x29, #32]\n\t" - "eor v31.16b, v31.16b, v10.16b\n\t" - "str %[r], [x29, #24]\n\t" - "eor v28.16b, v28.16b, v12.16b\n\t" - "eor %x[seed], x3, x8\n\t" - "eor v31.16b, v31.16b, v15.16b\n\t" - "eor %[r], x5, x10\n\t" - "eor v28.16b, v28.16b, v17.16b\n\t" - "eor %x[seed], %x[seed], x13\n\t" - "eor v31.16b, v31.16b, v20.16b\n\t" - "eor %[r], %[r], x15\n\t" - "eor v28.16b, v28.16b, v22.16b\n\t" - "eor %x[seed], %x[seed], x19\n\t" - "ushr v29.2d, v30.2d, #63\n\t" - "eor %[r], %[r], x21\n\t" - "ushr v26.2d, v28.2d, #63\n\t" - "eor %x[seed], %x[seed], x24\n\t" - "sli v29.2d, v30.2d, #1\n\t" - "eor %[r], %[r], x26\n\t" - "sli v26.2d, v28.2d, #1\n\t" - "eor %x[state], %x[state], %x[seed], ror 63\n\t" - "eor v28.16b, v28.16b, v29.16b\n\t" - "eor %x[seed], %x[seed], %[r], ror 63\n\t" - "eor v29.16b, v3.16b, v8.16b\n\t" - "eor x2, x2, %x[state]\n\t" - "eor v26.16b, v26.16b, v31.16b\n\t" - "eor x7, x7, %x[state]\n\t" - "eor v29.16b, v29.16b, v13.16b\n\t" - "eor x12, x12, %x[state]\n\t" - "eor v29.16b, v29.16b, v18.16b\n\t" - "eor x17, x17, %x[state]\n\t" - "eor v29.16b, v29.16b, v23.16b\n\t" - "eor x23, x23, %x[state]\n\t" - "ushr v30.2d, v29.2d, #63\n\t" - "eor x4, x4, %x[seed]\n\t" - "sli v30.2d, v29.2d, #1\n\t" - "eor x9, x9, %x[seed]\n\t" - "eor v27.16b, v27.16b, v30.16b\n\t" - "eor x14, x14, %x[seed]\n\t" - "ushr v30.2d, v31.2d, #63\n\t" - "eor x20, x20, %x[seed]\n\t" - "sli v30.2d, v31.2d, #1\n\t" - "eor x25, x25, %x[seed]\n\t" - "eor v29.16b, v29.16b, v30.16b\n\t" - "ldr %x[state], [x29, #32]\n\t" - /* Swap Rotate NEON */ - "eor v0.16b, v0.16b, v25.16b\n\t" - "eor v31.16b, v1.16b, v26.16b\n\t" - "ldr %x[seed], [x29, #24]\n\t" - "eor v6.16b, v6.16b, v26.16b\n\t" - "eor %[r], %[r], x30, ror 63\n\t" - "ushr v30.2d, v31.2d, #63\n\t" - "eor x30, x30, %x[seed], ror 63\n\t" - "ushr v1.2d, v6.2d, #20\n\t" - "eor %x[seed], %x[seed], %x[state], ror 63\n\t" - "sli v30.2d, v31.2d, #1\n\t" - "eor x6, x6, %[r]\n\t" - "sli v1.2d, v6.2d, #44\n\t" - "eor x11, x11, %[r]\n\t" - "eor v31.16b, v9.16b, v29.16b\n\t" - "eor x16, x16, %[r]\n\t" - "eor v22.16b, v22.16b, v27.16b\n\t" - "eor x22, x22, %[r]\n\t" - "ushr v6.2d, v31.2d, #44\n\t" - "eor x27, x27, %[r]\n\t" - "ushr v9.2d, v22.2d, #3\n\t" - "eor x3, x3, x30\n\t" - "sli v6.2d, v31.2d, #20\n\t" - "eor x8, x8, x30\n\t" - "sli v9.2d, v22.2d, #61\n\t" - "eor x13, x13, x30\n\t" - "eor v31.16b, v14.16b, v29.16b\n\t" - "eor x19, x19, x30\n\t" - "eor v20.16b, v20.16b, v25.16b\n\t" - "eor x24, x24, x30\n\t" - "ushr v22.2d, v31.2d, #25\n\t" - "eor x5, x5, %x[seed]\n\t" - "ushr v14.2d, v20.2d, #46\n\t" - "eor x10, x10, %x[seed]\n\t" - "sli v22.2d, v31.2d, #39\n\t" - "eor x15, x15, %x[seed]\n\t" - "sli v14.2d, v20.2d, #18\n\t" - "eor x21, x21, %x[seed]\n\t" - "eor v31.16b, v2.16b, v27.16b\n\t" - "eor x26, x26, %x[seed]\n\t" - /* Swap Rotate Base */ - "eor v12.16b, v12.16b, v27.16b\n\t" - "ror %x[state], x3, #63\n\t" - "ushr v20.2d, v31.2d, #2\n\t" - "ror x3, x8, #20\n\t" - "ushr v2.2d, v12.2d, #21\n\t" - "ror x8, x11, #44\n\t" - "sli v20.2d, v31.2d, #62\n\t" - "ror x11, x25, #3\n\t" - "sli v2.2d, v12.2d, #43\n\t" - "ror x25, x16, #25\n\t" - "eor v31.16b, v13.16b, v28.16b\n\t" - "ror x16, x23, #46\n\t" - "eor v19.16b, v19.16b, v29.16b\n\t" - "ror x23, x4, #2\n\t" - "ushr v12.2d, v31.2d, #39\n\t" - "ror x4, x14, #21\n\t" - "ushr v13.2d, v19.2d, #56\n\t" - "ror x14, x15, #39\n\t" - "sli v12.2d, v31.2d, #25\n\t" - "ror x15, x22, #56\n\t" - "sli v13.2d, v19.2d, #8\n\t" - "ror x22, x26, #8\n\t" - "eor v31.16b, v23.16b, v28.16b\n\t" - "ror x26, x17, #23\n\t" - "eor v15.16b, v15.16b, v25.16b\n\t" - "ror x17, x6, #37\n\t" - "ushr v19.2d, v31.2d, #8\n\t" - "ror x6, x27, #50\n\t" - "ushr v23.2d, v15.2d, #23\n\t" - "ror x27, x24, #62\n\t" - "sli v19.2d, v31.2d, #56\n\t" - "ror x24, x10, #9\n\t" - "sli v23.2d, v15.2d, #41\n\t" - "ror x10, x19, #19\n\t" - "eor v31.16b, v4.16b, v29.16b\n\t" - "ror x19, x7, #28\n\t" - "eor v24.16b, v24.16b, v29.16b\n\t" - "ror x7, x5, #36\n\t" - "ushr v15.2d, v31.2d, #37\n\t" - "ror x5, x21, #43\n\t" - "ushr v4.2d, v24.2d, #50\n\t" - "ror x21, x20, #49\n\t" - "sli v15.2d, v31.2d, #27\n\t" - "ror x20, x13, #54\n\t" - "sli v4.2d, v24.2d, #14\n\t" - "ror x13, x9, #58\n\t" - "eor v31.16b, v21.16b, v26.16b\n\t" - "ror x9, x12, #61\n\t" - /* Row Mix Base */ - "eor v8.16b, v8.16b, v28.16b\n\t" - "bic x12, x4, x3\n\t" - "ushr v24.2d, v31.2d, #62\n\t" - "bic %x[seed], x5, x4\n\t" - "ushr v21.2d, v8.2d, #9\n\t" - "bic %[r], x2, x6\n\t" - "sli v24.2d, v31.2d, #2\n\t" - "bic x30, x3, x2\n\t" - "sli v21.2d, v8.2d, #55\n\t" - "eor x2, x2, x12\n\t" - "eor v31.16b, v16.16b, v26.16b\n\t" - "eor x3, x3, %x[seed]\n\t" - "eor v5.16b, v5.16b, v25.16b\n\t" - "bic x12, x6, x5\n\t" - "ushr v8.2d, v31.2d, #19\n\t" - "eor x5, x5, %[r]\n\t" - "ushr v16.2d, v5.2d, #28\n\t" - "eor x4, x4, x12\n\t" - "sli v8.2d, v31.2d, #45\n\t" - "eor x6, x6, x30\n\t" - "sli v16.2d, v5.2d, #36\n\t" - "bic x12, x9, x8\n\t" - "eor v31.16b, v3.16b, v28.16b\n\t" - "bic %x[seed], x10, x9\n\t" - "eor v18.16b, v18.16b, v28.16b\n\t" - "bic %[r], x7, x11\n\t" - "ushr v5.2d, v31.2d, #36\n\t" - "bic x30, x8, x7\n\t" - "ushr v3.2d, v18.2d, #43\n\t" - "eor x7, x7, x12\n\t" - "sli v5.2d, v31.2d, #28\n\t" - "eor x8, x8, %x[seed]\n\t" - "sli v3.2d, v18.2d, #21\n\t" - "bic x12, x11, x10\n\t" - "eor v31.16b, v17.16b, v27.16b\n\t" - "eor x10, x10, %[r]\n\t" - "eor v11.16b, v11.16b, v26.16b\n\t" - "eor x9, x9, x12\n\t" - "ushr v18.2d, v31.2d, #49\n\t" - "eor x11, x11, x30\n\t" - "ushr v17.2d, v11.2d, #54\n\t" - "bic x12, x14, x13\n\t" - "sli v18.2d, v31.2d, #15\n\t" - "bic %x[seed], x15, x14\n\t" - "sli v17.2d, v11.2d, #10\n\t" - "bic %[r], %x[state], x16\n\t" - "eor v31.16b, v7.16b, v27.16b\n\t" - "bic x30, x13, %x[state]\n\t" - "eor v10.16b, v10.16b, v25.16b\n\t" - "eor x12, %x[state], x12\n\t" - "ushr v11.2d, v31.2d, #58\n\t" - "eor x13, x13, %x[seed]\n\t" - "ushr v7.2d, v10.2d, #61\n\t" - "bic %x[state], x16, x15\n\t" - "sli v11.2d, v31.2d, #6\n\t" - "eor x15, x15, %[r]\n\t" - "sli v7.2d, v10.2d, #3\n\t" - "eor x14, x14, %x[state]\n\t" - /* Row Mix NEON */ - "bic v25.16b, v2.16b, v1.16b\n\t" - "eor x16, x16, x30\n\t" - "bic v26.16b, v3.16b, v2.16b\n\t" - "bic %x[state], x20, x19\n\t" - "bic v27.16b, v4.16b, v3.16b\n\t" - "bic %x[seed], x21, x20\n\t" - "bic v28.16b, v0.16b, v4.16b\n\t" - "bic %[r], x17, x22\n\t" - "bic v29.16b, v1.16b, v0.16b\n\t" - "bic x30, x19, x17\n\t" - "eor v0.16b, v0.16b, v25.16b\n\t" - "eor x17, x17, %x[state]\n\t" - "eor v1.16b, v1.16b, v26.16b\n\t" - "eor x19, x19, %x[seed]\n\t" - "eor v2.16b, v2.16b, v27.16b\n\t" - "bic %x[state], x22, x21\n\t" - "eor v3.16b, v3.16b, v28.16b\n\t" - "eor x21, x21, %[r]\n\t" - "eor v4.16b, v4.16b, v29.16b\n\t" - "eor x20, x20, %x[state]\n\t" - "bic v25.16b, v7.16b, v6.16b\n\t" - "eor x22, x22, x30\n\t" - "bic v26.16b, v8.16b, v7.16b\n\t" - "bic %x[state], x25, x24\n\t" - "bic v27.16b, v9.16b, v8.16b\n\t" - "bic %x[seed], x26, x25\n\t" - "bic v28.16b, v5.16b, v9.16b\n\t" - "bic %[r], x23, x27\n\t" - "bic v29.16b, v6.16b, v5.16b\n\t" - "bic x30, x24, x23\n\t" - "eor v5.16b, v5.16b, v25.16b\n\t" - "eor x23, x23, %x[state]\n\t" - "eor v6.16b, v6.16b, v26.16b\n\t" - "eor x24, x24, %x[seed]\n\t" - "eor v7.16b, v7.16b, v27.16b\n\t" - "bic %x[state], x27, x26\n\t" - "eor v8.16b, v8.16b, v28.16b\n\t" - "eor x26, x26, %[r]\n\t" - "eor v9.16b, v9.16b, v29.16b\n\t" - "eor x25, x25, %x[state]\n\t" - "bic v25.16b, v12.16b, v11.16b\n\t" - "eor x27, x27, x30\n\t" - "bic v26.16b, v13.16b, v12.16b\n\t" - "bic v27.16b, v14.16b, v13.16b\n\t" - "bic v28.16b, v30.16b, v14.16b\n\t" - "bic v29.16b, v11.16b, v30.16b\n\t" - "eor v10.16b, v30.16b, v25.16b\n\t" - "eor v11.16b, v11.16b, v26.16b\n\t" - "eor v12.16b, v12.16b, v27.16b\n\t" - "eor v13.16b, v13.16b, v28.16b\n\t" - "eor v14.16b, v14.16b, v29.16b\n\t" - "bic v25.16b, v17.16b, v16.16b\n\t" - "bic v26.16b, v18.16b, v17.16b\n\t" - "bic v27.16b, v19.16b, v18.16b\n\t" - "bic v28.16b, v15.16b, v19.16b\n\t" - "bic v29.16b, v16.16b, v15.16b\n\t" - "eor v15.16b, v15.16b, v25.16b\n\t" - "eor v16.16b, v16.16b, v26.16b\n\t" - "eor v17.16b, v17.16b, v27.16b\n\t" - "eor v18.16b, v18.16b, v28.16b\n\t" - "eor v19.16b, v19.16b, v29.16b\n\t" - "bic v25.16b, v22.16b, v21.16b\n\t" - "bic v26.16b, v23.16b, v22.16b\n\t" - "bic v27.16b, v24.16b, v23.16b\n\t" - "bic v28.16b, v20.16b, v24.16b\n\t" - "bic v29.16b, v21.16b, v20.16b\n\t" - "eor v20.16b, v20.16b, v25.16b\n\t" - "eor v21.16b, v21.16b, v26.16b\n\t" - "eor v22.16b, v22.16b, v27.16b\n\t" - "eor v23.16b, v23.16b, v28.16b\n\t" - "eor v24.16b, v24.16b, v29.16b\n\t" - /* Done transforming */ - "ldp %[r], %x[seed], [x29, #48]\n\t" - "ldr %x[state], [%[r]], #8\n\t" - "subs %x[seed], %x[seed], #1\n\t" - "mov v30.d[0], %x[state]\n\t" - "mov v30.d[1], %x[state]\n\t" - "eor x2, x2, %x[state]\n\t" - "eor v0.16b, v0.16b, v30.16b\n\t" - "b.ne L_SHA3_shake256_blocksx3_seed_neon_begin_%=\n\t" + "b.ne L_SHA3_shake256_blocksx3_seed_crypto_begin_%=\n\t" "ldr %x[state], [x29, #40]\n\t" "st4 {v0.d, v1.d, v2.d, v3.d}[0], [%x[state]], #32\n\t" "st4 {v4.d, v5.d, v6.d, v7.d}[0], [%x[state]], #32\n\t" @@ -11259,6 +10062,1202 @@ void mlkem_shake256_blocksx3_seed_neon(word64* state, byte* seed) } #endif /* WOLFSSL_ARMASM_CRYPTO_SHA3 */ +void mlkem_sha3_blocksx3_neon(word64* state) +{ + const word64* r = L_sha3_aarch64_r; + __asm__ __volatile__ ( + "stp x29, x30, [sp, #-64]!\n\t" + "add x29, sp, #0\n\t" + "str %x[state], [x29, #40]\n\t" + "ld4 {v0.d, v1.d, v2.d, v3.d}[0], [%x[state]], #32\n\t" + "ld4 {v4.d, v5.d, v6.d, v7.d}[0], [%x[state]], #32\n\t" + "ld4 {v8.d, v9.d, v10.d, v11.d}[0], [%x[state]], #32\n\t" + "ld4 {v12.d, v13.d, v14.d, v15.d}[0], [%x[state]], #32\n\t" + "ld4 {v16.d, v17.d, v18.d, v19.d}[0], [%x[state]], #32\n\t" + "ld4 {v20.d, v21.d, v22.d, v23.d}[0], [%x[state]], #32\n\t" + "ld1 {v24.d}[0], [%x[state]]\n\t" + "add %x[state], %x[state], #8\n\t" + "ld4 {v0.d, v1.d, v2.d, v3.d}[1], [%x[state]], #32\n\t" + "ld4 {v4.d, v5.d, v6.d, v7.d}[1], [%x[state]], #32\n\t" + "ld4 {v8.d, v9.d, v10.d, v11.d}[1], [%x[state]], #32\n\t" + "ld4 {v12.d, v13.d, v14.d, v15.d}[1], [%x[state]], #32\n\t" + "ld4 {v16.d, v17.d, v18.d, v19.d}[1], [%x[state]], #32\n\t" + "ld4 {v20.d, v21.d, v22.d, v23.d}[1], [%x[state]], #32\n\t" + "ld1 {v24.d}[1], [%x[state]]\n\t" + "add %x[state], %x[state], #8\n\t" + "ldp x1, x2, [%x[state]]\n\t" + "ldp x3, x4, [%x[state], #16]\n\t" + "ldp x5, x6, [%x[state], #32]\n\t" + "ldp x7, x8, [%x[state], #48]\n\t" + "ldp x9, x10, [%x[state], #64]\n\t" + "ldp x11, x12, [%x[state], #80]\n\t" + "ldp x13, x14, [%x[state], #96]\n\t" + "ldp x15, x16, [%x[state], #112]\n\t" + "ldp x17, x19, [%x[state], #128]\n\t" + "ldp x20, x21, [%x[state], #144]\n\t" + "ldp x22, x23, [%x[state], #160]\n\t" + "ldp x24, x25, [%x[state], #176]\n\t" + "ldr x26, [%x[state], #192]\n\t" + "mov x28, #24\n\t" + /* Start of 24 rounds */ + "\n" + "L_SHA3_transform_blocksx3_neon_begin_%=:\n\t" + "stp %[r], x28, [x29, #48]\n\t" + /* Col Mix NEON */ + "eor v30.16b, v4.16b, v9.16b\n\t" + "eor %x[state], x5, x10\n\t" + "eor v27.16b, v1.16b, v6.16b\n\t" + "eor x30, x1, x6\n\t" + "eor v30.16b, v30.16b, v14.16b\n\t" + "eor x28, x3, x8\n\t" + "eor v27.16b, v27.16b, v11.16b\n\t" + "eor %x[state], %x[state], x15\n\t" + "eor v30.16b, v30.16b, v19.16b\n\t" + "eor x30, x30, x11\n\t" + "eor v27.16b, v27.16b, v16.16b\n\t" + "eor x28, x28, x13\n\t" + "eor v30.16b, v30.16b, v24.16b\n\t" + "eor %x[state], %x[state], x21\n\t" + "eor v27.16b, v27.16b, v21.16b\n\t" + "eor x30, x30, x16\n\t" + "ushr v25.2d, v27.2d, #63\n\t" + "eor x28, x28, x19\n\t" + "sli v25.2d, v27.2d, #1\n\t" + "eor %x[state], %x[state], x26\n\t" + "eor v25.16b, v25.16b, v30.16b\n\t" + "eor x30, x30, x22\n\t" + "eor v31.16b, v0.16b, v5.16b\n\t" + "eor x28, x28, x24\n\t" + "eor v28.16b, v2.16b, v7.16b\n\t" + "str %x[state], [x29, #32]\n\t" + "eor v31.16b, v31.16b, v10.16b\n\t" + "str x28, [x29, #24]\n\t" + "eor v28.16b, v28.16b, v12.16b\n\t" + "eor %[r], x2, x7\n\t" + "eor v31.16b, v31.16b, v15.16b\n\t" + "eor x28, x4, x9\n\t" + "eor v28.16b, v28.16b, v17.16b\n\t" + "eor %[r], %[r], x12\n\t" + "eor v31.16b, v31.16b, v20.16b\n\t" + "eor x28, x28, x14\n\t" + "eor v28.16b, v28.16b, v22.16b\n\t" + "eor %[r], %[r], x17\n\t" + "ushr v29.2d, v30.2d, #63\n\t" + "eor x28, x28, x20\n\t" + "ushr v26.2d, v28.2d, #63\n\t" + "eor %[r], %[r], x23\n\t" + "sli v29.2d, v30.2d, #1\n\t" + "eor x28, x28, x25\n\t" + "sli v26.2d, v28.2d, #1\n\t" + "eor %x[state], %x[state], %[r], ror 63\n\t" + "eor v28.16b, v28.16b, v29.16b\n\t" + "eor %[r], %[r], x28, ror 63\n\t" + "eor v29.16b, v3.16b, v8.16b\n\t" + "eor x1, x1, %x[state]\n\t" + "eor v26.16b, v26.16b, v31.16b\n\t" + "eor x6, x6, %x[state]\n\t" + "eor v29.16b, v29.16b, v13.16b\n\t" + "eor x11, x11, %x[state]\n\t" + "eor v29.16b, v29.16b, v18.16b\n\t" + "eor x16, x16, %x[state]\n\t" + "eor v29.16b, v29.16b, v23.16b\n\t" + "eor x22, x22, %x[state]\n\t" + "ushr v30.2d, v29.2d, #63\n\t" + "eor x3, x3, %[r]\n\t" + "sli v30.2d, v29.2d, #1\n\t" + "eor x8, x8, %[r]\n\t" + "eor v27.16b, v27.16b, v30.16b\n\t" + "eor x13, x13, %[r]\n\t" + "ushr v30.2d, v31.2d, #63\n\t" + "eor x19, x19, %[r]\n\t" + "sli v30.2d, v31.2d, #1\n\t" + "eor x24, x24, %[r]\n\t" + "eor v29.16b, v29.16b, v30.16b\n\t" + "ldr %x[state], [x29, #32]\n\t" + /* Swap Rotate NEON */ + "eor v0.16b, v0.16b, v25.16b\n\t" + "eor v31.16b, v1.16b, v26.16b\n\t" + "ldr %[r], [x29, #24]\n\t" + "eor v6.16b, v6.16b, v26.16b\n\t" + "eor x28, x28, x30, ror 63\n\t" + "ushr v30.2d, v31.2d, #63\n\t" + "eor x30, x30, %[r], ror 63\n\t" + "ushr v1.2d, v6.2d, #20\n\t" + "eor %[r], %[r], %x[state], ror 63\n\t" + "sli v30.2d, v31.2d, #1\n\t" + "eor x5, x5, x28\n\t" + "sli v1.2d, v6.2d, #44\n\t" + "eor x10, x10, x28\n\t" + "eor v31.16b, v9.16b, v29.16b\n\t" + "eor x15, x15, x28\n\t" + "eor v22.16b, v22.16b, v27.16b\n\t" + "eor x21, x21, x28\n\t" + "ushr v6.2d, v31.2d, #44\n\t" + "eor x26, x26, x28\n\t" + "ushr v9.2d, v22.2d, #3\n\t" + "eor x2, x2, x30\n\t" + "sli v6.2d, v31.2d, #20\n\t" + "eor x7, x7, x30\n\t" + "sli v9.2d, v22.2d, #61\n\t" + "eor x12, x12, x30\n\t" + "eor v31.16b, v14.16b, v29.16b\n\t" + "eor x17, x17, x30\n\t" + "eor v20.16b, v20.16b, v25.16b\n\t" + "eor x23, x23, x30\n\t" + "ushr v22.2d, v31.2d, #25\n\t" + "eor x4, x4, %[r]\n\t" + "ushr v14.2d, v20.2d, #46\n\t" + "eor x9, x9, %[r]\n\t" + "sli v22.2d, v31.2d, #39\n\t" + "eor x14, x14, %[r]\n\t" + "sli v14.2d, v20.2d, #18\n\t" + "eor x20, x20, %[r]\n\t" + "eor v31.16b, v2.16b, v27.16b\n\t" + "eor x25, x25, %[r]\n\t" + /* Swap Rotate Base */ + "eor v12.16b, v12.16b, v27.16b\n\t" + "ror %x[state], x2, #63\n\t" + "ushr v20.2d, v31.2d, #2\n\t" + "ror x2, x7, #20\n\t" + "ushr v2.2d, v12.2d, #21\n\t" + "ror x7, x10, #44\n\t" + "sli v20.2d, v31.2d, #62\n\t" + "ror x10, x24, #3\n\t" + "sli v2.2d, v12.2d, #43\n\t" + "ror x24, x15, #25\n\t" + "eor v31.16b, v13.16b, v28.16b\n\t" + "ror x15, x22, #46\n\t" + "eor v19.16b, v19.16b, v29.16b\n\t" + "ror x22, x3, #2\n\t" + "ushr v12.2d, v31.2d, #39\n\t" + "ror x3, x13, #21\n\t" + "ushr v13.2d, v19.2d, #56\n\t" + "ror x13, x14, #39\n\t" + "sli v12.2d, v31.2d, #25\n\t" + "ror x14, x21, #56\n\t" + "sli v13.2d, v19.2d, #8\n\t" + "ror x21, x25, #8\n\t" + "eor v31.16b, v23.16b, v28.16b\n\t" + "ror x25, x16, #23\n\t" + "eor v15.16b, v15.16b, v25.16b\n\t" + "ror x16, x5, #37\n\t" + "ushr v19.2d, v31.2d, #8\n\t" + "ror x5, x26, #50\n\t" + "ushr v23.2d, v15.2d, #23\n\t" + "ror x26, x23, #62\n\t" + "sli v19.2d, v31.2d, #56\n\t" + "ror x23, x9, #9\n\t" + "sli v23.2d, v15.2d, #41\n\t" + "ror x9, x17, #19\n\t" + "eor v31.16b, v4.16b, v29.16b\n\t" + "ror x17, x6, #28\n\t" + "eor v24.16b, v24.16b, v29.16b\n\t" + "ror x6, x4, #36\n\t" + "ushr v15.2d, v31.2d, #37\n\t" + "ror x4, x20, #43\n\t" + "ushr v4.2d, v24.2d, #50\n\t" + "ror x20, x19, #49\n\t" + "sli v15.2d, v31.2d, #27\n\t" + "ror x19, x12, #54\n\t" + "sli v4.2d, v24.2d, #14\n\t" + "ror x12, x8, #58\n\t" + "eor v31.16b, v21.16b, v26.16b\n\t" + "ror x8, x11, #61\n\t" + /* Row Mix Base */ + "eor v8.16b, v8.16b, v28.16b\n\t" + "bic x11, x3, x2\n\t" + "ushr v24.2d, v31.2d, #62\n\t" + "bic %[r], x4, x3\n\t" + "ushr v21.2d, v8.2d, #9\n\t" + "bic x28, x1, x5\n\t" + "sli v24.2d, v31.2d, #2\n\t" + "bic x30, x2, x1\n\t" + "sli v21.2d, v8.2d, #55\n\t" + "eor x1, x1, x11\n\t" + "eor v31.16b, v16.16b, v26.16b\n\t" + "eor x2, x2, %[r]\n\t" + "eor v5.16b, v5.16b, v25.16b\n\t" + "bic x11, x5, x4\n\t" + "ushr v8.2d, v31.2d, #19\n\t" + "eor x4, x4, x28\n\t" + "ushr v16.2d, v5.2d, #28\n\t" + "eor x3, x3, x11\n\t" + "sli v8.2d, v31.2d, #45\n\t" + "eor x5, x5, x30\n\t" + "sli v16.2d, v5.2d, #36\n\t" + "bic x11, x8, x7\n\t" + "eor v31.16b, v3.16b, v28.16b\n\t" + "bic %[r], x9, x8\n\t" + "eor v18.16b, v18.16b, v28.16b\n\t" + "bic x28, x6, x10\n\t" + "ushr v5.2d, v31.2d, #36\n\t" + "bic x30, x7, x6\n\t" + "ushr v3.2d, v18.2d, #43\n\t" + "eor x6, x6, x11\n\t" + "sli v5.2d, v31.2d, #28\n\t" + "eor x7, x7, %[r]\n\t" + "sli v3.2d, v18.2d, #21\n\t" + "bic x11, x10, x9\n\t" + "eor v31.16b, v17.16b, v27.16b\n\t" + "eor x9, x9, x28\n\t" + "eor v11.16b, v11.16b, v26.16b\n\t" + "eor x8, x8, x11\n\t" + "ushr v18.2d, v31.2d, #49\n\t" + "eor x10, x10, x30\n\t" + "ushr v17.2d, v11.2d, #54\n\t" + "bic x11, x13, x12\n\t" + "sli v18.2d, v31.2d, #15\n\t" + "bic %[r], x14, x13\n\t" + "sli v17.2d, v11.2d, #10\n\t" + "bic x28, %x[state], x15\n\t" + "eor v31.16b, v7.16b, v27.16b\n\t" + "bic x30, x12, %x[state]\n\t" + "eor v10.16b, v10.16b, v25.16b\n\t" + "eor x11, %x[state], x11\n\t" + "ushr v11.2d, v31.2d, #58\n\t" + "eor x12, x12, %[r]\n\t" + "ushr v7.2d, v10.2d, #61\n\t" + "bic %x[state], x15, x14\n\t" + "sli v11.2d, v31.2d, #6\n\t" + "eor x14, x14, x28\n\t" + "sli v7.2d, v10.2d, #3\n\t" + "eor x13, x13, %x[state]\n\t" + /* Row Mix NEON */ + "bic v25.16b, v2.16b, v1.16b\n\t" + "eor x15, x15, x30\n\t" + "bic v26.16b, v3.16b, v2.16b\n\t" + "bic %x[state], x19, x17\n\t" + "bic v27.16b, v4.16b, v3.16b\n\t" + "bic %[r], x20, x19\n\t" + "bic v28.16b, v0.16b, v4.16b\n\t" + "bic x28, x16, x21\n\t" + "bic v29.16b, v1.16b, v0.16b\n\t" + "bic x30, x17, x16\n\t" + "eor v0.16b, v0.16b, v25.16b\n\t" + "eor x16, x16, %x[state]\n\t" + "eor v1.16b, v1.16b, v26.16b\n\t" + "eor x17, x17, %[r]\n\t" + "eor v2.16b, v2.16b, v27.16b\n\t" + "bic %x[state], x21, x20\n\t" + "eor v3.16b, v3.16b, v28.16b\n\t" + "eor x20, x20, x28\n\t" + "eor v4.16b, v4.16b, v29.16b\n\t" + "eor x19, x19, %x[state]\n\t" + "bic v25.16b, v7.16b, v6.16b\n\t" + "eor x21, x21, x30\n\t" + "bic v26.16b, v8.16b, v7.16b\n\t" + "bic %x[state], x24, x23\n\t" + "bic v27.16b, v9.16b, v8.16b\n\t" + "bic %[r], x25, x24\n\t" + "bic v28.16b, v5.16b, v9.16b\n\t" + "bic x28, x22, x26\n\t" + "bic v29.16b, v6.16b, v5.16b\n\t" + "bic x30, x23, x22\n\t" + "eor v5.16b, v5.16b, v25.16b\n\t" + "eor x22, x22, %x[state]\n\t" + "eor v6.16b, v6.16b, v26.16b\n\t" + "eor x23, x23, %[r]\n\t" + "eor v7.16b, v7.16b, v27.16b\n\t" + "bic %x[state], x26, x25\n\t" + "eor v8.16b, v8.16b, v28.16b\n\t" + "eor x25, x25, x28\n\t" + "eor v9.16b, v9.16b, v29.16b\n\t" + "eor x24, x24, %x[state]\n\t" + "bic v25.16b, v12.16b, v11.16b\n\t" + "eor x26, x26, x30\n\t" + "bic v26.16b, v13.16b, v12.16b\n\t" + "bic v27.16b, v14.16b, v13.16b\n\t" + "bic v28.16b, v30.16b, v14.16b\n\t" + "bic v29.16b, v11.16b, v30.16b\n\t" + "eor v10.16b, v30.16b, v25.16b\n\t" + "eor v11.16b, v11.16b, v26.16b\n\t" + "eor v12.16b, v12.16b, v27.16b\n\t" + "eor v13.16b, v13.16b, v28.16b\n\t" + "eor v14.16b, v14.16b, v29.16b\n\t" + "bic v25.16b, v17.16b, v16.16b\n\t" + "bic v26.16b, v18.16b, v17.16b\n\t" + "bic v27.16b, v19.16b, v18.16b\n\t" + "bic v28.16b, v15.16b, v19.16b\n\t" + "bic v29.16b, v16.16b, v15.16b\n\t" + "eor v15.16b, v15.16b, v25.16b\n\t" + "eor v16.16b, v16.16b, v26.16b\n\t" + "eor v17.16b, v17.16b, v27.16b\n\t" + "eor v18.16b, v18.16b, v28.16b\n\t" + "eor v19.16b, v19.16b, v29.16b\n\t" + "bic v25.16b, v22.16b, v21.16b\n\t" + "bic v26.16b, v23.16b, v22.16b\n\t" + "bic v27.16b, v24.16b, v23.16b\n\t" + "bic v28.16b, v20.16b, v24.16b\n\t" + "bic v29.16b, v21.16b, v20.16b\n\t" + "eor v20.16b, v20.16b, v25.16b\n\t" + "eor v21.16b, v21.16b, v26.16b\n\t" + "eor v22.16b, v22.16b, v27.16b\n\t" + "eor v23.16b, v23.16b, v28.16b\n\t" + "eor v24.16b, v24.16b, v29.16b\n\t" + /* Done transforming */ + "ldp %[r], x28, [x29, #48]\n\t" + "ldr %x[state], [%[r]], #8\n\t" + "subs x28, x28, #1\n\t" + "mov v30.d[0], %x[state]\n\t" + "mov v30.d[1], %x[state]\n\t" + "eor x1, x1, %x[state]\n\t" + "eor v0.16b, v0.16b, v30.16b\n\t" + "b.ne L_SHA3_transform_blocksx3_neon_begin_%=\n\t" + "ldr %x[state], [x29, #40]\n\t" + "st4 {v0.d, v1.d, v2.d, v3.d}[0], [%x[state]], #32\n\t" + "st4 {v4.d, v5.d, v6.d, v7.d}[0], [%x[state]], #32\n\t" + "st4 {v8.d, v9.d, v10.d, v11.d}[0], [%x[state]], #32\n\t" + "st4 {v12.d, v13.d, v14.d, v15.d}[0], [%x[state]], #32\n\t" + "st4 {v16.d, v17.d, v18.d, v19.d}[0], [%x[state]], #32\n\t" + "st4 {v20.d, v21.d, v22.d, v23.d}[0], [%x[state]], #32\n\t" + "st1 {v24.d}[0], [%x[state]]\n\t" + "add %x[state], %x[state], #8\n\t" + "st4 {v0.d, v1.d, v2.d, v3.d}[1], [%x[state]], #32\n\t" + "st4 {v4.d, v5.d, v6.d, v7.d}[1], [%x[state]], #32\n\t" + "st4 {v8.d, v9.d, v10.d, v11.d}[1], [%x[state]], #32\n\t" + "st4 {v12.d, v13.d, v14.d, v15.d}[1], [%x[state]], #32\n\t" + "st4 {v16.d, v17.d, v18.d, v19.d}[1], [%x[state]], #32\n\t" + "st4 {v20.d, v21.d, v22.d, v23.d}[1], [%x[state]], #32\n\t" + "st1 {v24.d}[1], [%x[state]]\n\t" + "add %x[state], %x[state], #8\n\t" + "stp x1, x2, [%x[state]]\n\t" + "stp x3, x4, [%x[state], #16]\n\t" + "stp x5, x6, [%x[state], #32]\n\t" + "stp x7, x8, [%x[state], #48]\n\t" + "stp x9, x10, [%x[state], #64]\n\t" + "stp x11, x12, [%x[state], #80]\n\t" + "stp x13, x14, [%x[state], #96]\n\t" + "stp x15, x16, [%x[state], #112]\n\t" + "stp x17, x19, [%x[state], #128]\n\t" + "stp x20, x21, [%x[state], #144]\n\t" + "stp x22, x23, [%x[state], #160]\n\t" + "stp x24, x25, [%x[state], #176]\n\t" + "str x26, [%x[state], #192]\n\t" + "ldp x29, x30, [sp], #0x40\n\t" + : [state] "+r" (state) + : [r] "r" (r) + : "memory", "cc", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", + "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", + "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x28", "v0", "v1", + "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", + "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", + "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", + "v31" + ); +} + +void mlkem_shake128_blocksx3_seed_neon(word64* state, byte* seed) +{ + const word64* r = L_sha3_aarch64_r; + __asm__ __volatile__ ( + "stp x29, x30, [sp, #-64]!\n\t" + "add x29, sp, #0\n\t" + "str %x[state], [x29, #40]\n\t" + "add %x[state], %x[state], #32\n\t" + "ld1 {v4.d}[0], [%x[state]]\n\t" + "ldp x2, x3, [%x[seed]], #16\n\t" + "add %x[state], %x[state], #0xc8\n\t" + "ld1 {v4.d}[1], [%x[state]]\n\t" + "ldp x4, x5, [%x[seed]], #16\n\t" + "ldr x6, [%x[state], #200]\n\t" + "eor v5.16b, v5.16b, v5.16b\n\t" + "eor x7, x7, x7\n\t" + "eor v6.16b, v6.16b, v6.16b\n\t" + "eor x8, x8, x8\n\t" + "eor v7.16b, v7.16b, v7.16b\n\t" + "eor x9, x9, x9\n\t" + "eor v8.16b, v8.16b, v8.16b\n\t" + "eor x10, x10, x10\n\t" + "eor v9.16b, v9.16b, v9.16b\n\t" + "eor x11, x11, x11\n\t" + "eor v10.16b, v10.16b, v10.16b\n\t" + "eor x12, x12, x12\n\t" + "eor v11.16b, v11.16b, v11.16b\n\t" + "eor x13, x13, x13\n\t" + "eor v12.16b, v12.16b, v12.16b\n\t" + "eor x14, x14, x14\n\t" + "eor v13.16b, v13.16b, v13.16b\n\t" + "eor x15, x15, x15\n\t" + "eor v14.16b, v14.16b, v14.16b\n\t" + "eor x16, x16, x16\n\t" + "eor v15.16b, v15.16b, v15.16b\n\t" + "eor x17, x17, x17\n\t" + "eor v16.16b, v16.16b, v16.16b\n\t" + "eor x19, x19, x19\n\t" + "eor v17.16b, v17.16b, v17.16b\n\t" + "eor x20, x20, x20\n\t" + "eor v18.16b, v18.16b, v18.16b\n\t" + "eor x21, x21, x21\n\t" + "eor v19.16b, v19.16b, v19.16b\n\t" + "eor x22, x22, x22\n\t" + "movz x23, #0x8000, lsl 48\n\t" + "eor v21.16b, v21.16b, v21.16b\n\t" + "eor x24, x24, x24\n\t" + "eor v22.16b, v22.16b, v22.16b\n\t" + "eor x25, x25, x25\n\t" + "eor v23.16b, v23.16b, v23.16b\n\t" + "eor x26, x26, x26\n\t" + "eor v24.16b, v24.16b, v24.16b\n\t" + "eor x27, x27, x27\n\t" + "dup v0.2d, x2\n\t" + "dup v1.2d, x3\n\t" + "dup v2.2d, x4\n\t" + "dup v3.2d, x5\n\t" + "dup v20.2d, x23\n\t" + "mov %x[seed], #24\n\t" + /* Start of 24 rounds */ + "\n" + "L_SHA3_shake128_blocksx3_seed_neon_begin_%=:\n\t" + "stp %[r], %x[seed], [x29, #48]\n\t" + /* Col Mix NEON */ + "eor v30.16b, v4.16b, v9.16b\n\t" + "eor %x[state], x6, x11\n\t" + "eor v27.16b, v1.16b, v6.16b\n\t" + "eor x30, x2, x7\n\t" + "eor v30.16b, v30.16b, v14.16b\n\t" + "eor %[r], x4, x9\n\t" + "eor v27.16b, v27.16b, v11.16b\n\t" + "eor %x[state], %x[state], x16\n\t" + "eor v30.16b, v30.16b, v19.16b\n\t" + "eor x30, x30, x12\n\t" + "eor v27.16b, v27.16b, v16.16b\n\t" + "eor %[r], %[r], x14\n\t" + "eor v30.16b, v30.16b, v24.16b\n\t" + "eor %x[state], %x[state], x22\n\t" + "eor v27.16b, v27.16b, v21.16b\n\t" + "eor x30, x30, x17\n\t" + "ushr v25.2d, v27.2d, #63\n\t" + "eor %[r], %[r], x20\n\t" + "sli v25.2d, v27.2d, #1\n\t" + "eor %x[state], %x[state], x27\n\t" + "eor v25.16b, v25.16b, v30.16b\n\t" + "eor x30, x30, x23\n\t" + "eor v31.16b, v0.16b, v5.16b\n\t" + "eor %[r], %[r], x25\n\t" + "eor v28.16b, v2.16b, v7.16b\n\t" + "str %x[state], [x29, #32]\n\t" + "eor v31.16b, v31.16b, v10.16b\n\t" + "str %[r], [x29, #24]\n\t" + "eor v28.16b, v28.16b, v12.16b\n\t" + "eor %x[seed], x3, x8\n\t" + "eor v31.16b, v31.16b, v15.16b\n\t" + "eor %[r], x5, x10\n\t" + "eor v28.16b, v28.16b, v17.16b\n\t" + "eor %x[seed], %x[seed], x13\n\t" + "eor v31.16b, v31.16b, v20.16b\n\t" + "eor %[r], %[r], x15\n\t" + "eor v28.16b, v28.16b, v22.16b\n\t" + "eor %x[seed], %x[seed], x19\n\t" + "ushr v29.2d, v30.2d, #63\n\t" + "eor %[r], %[r], x21\n\t" + "ushr v26.2d, v28.2d, #63\n\t" + "eor %x[seed], %x[seed], x24\n\t" + "sli v29.2d, v30.2d, #1\n\t" + "eor %[r], %[r], x26\n\t" + "sli v26.2d, v28.2d, #1\n\t" + "eor %x[state], %x[state], %x[seed], ror 63\n\t" + "eor v28.16b, v28.16b, v29.16b\n\t" + "eor %x[seed], %x[seed], %[r], ror 63\n\t" + "eor v29.16b, v3.16b, v8.16b\n\t" + "eor x2, x2, %x[state]\n\t" + "eor v26.16b, v26.16b, v31.16b\n\t" + "eor x7, x7, %x[state]\n\t" + "eor v29.16b, v29.16b, v13.16b\n\t" + "eor x12, x12, %x[state]\n\t" + "eor v29.16b, v29.16b, v18.16b\n\t" + "eor x17, x17, %x[state]\n\t" + "eor v29.16b, v29.16b, v23.16b\n\t" + "eor x23, x23, %x[state]\n\t" + "ushr v30.2d, v29.2d, #63\n\t" + "eor x4, x4, %x[seed]\n\t" + "sli v30.2d, v29.2d, #1\n\t" + "eor x9, x9, %x[seed]\n\t" + "eor v27.16b, v27.16b, v30.16b\n\t" + "eor x14, x14, %x[seed]\n\t" + "ushr v30.2d, v31.2d, #63\n\t" + "eor x20, x20, %x[seed]\n\t" + "sli v30.2d, v31.2d, #1\n\t" + "eor x25, x25, %x[seed]\n\t" + "eor v29.16b, v29.16b, v30.16b\n\t" + "ldr %x[state], [x29, #32]\n\t" + /* Swap Rotate NEON */ + "eor v0.16b, v0.16b, v25.16b\n\t" + "eor v31.16b, v1.16b, v26.16b\n\t" + "ldr %x[seed], [x29, #24]\n\t" + "eor v6.16b, v6.16b, v26.16b\n\t" + "eor %[r], %[r], x30, ror 63\n\t" + "ushr v30.2d, v31.2d, #63\n\t" + "eor x30, x30, %x[seed], ror 63\n\t" + "ushr v1.2d, v6.2d, #20\n\t" + "eor %x[seed], %x[seed], %x[state], ror 63\n\t" + "sli v30.2d, v31.2d, #1\n\t" + "eor x6, x6, %[r]\n\t" + "sli v1.2d, v6.2d, #44\n\t" + "eor x11, x11, %[r]\n\t" + "eor v31.16b, v9.16b, v29.16b\n\t" + "eor x16, x16, %[r]\n\t" + "eor v22.16b, v22.16b, v27.16b\n\t" + "eor x22, x22, %[r]\n\t" + "ushr v6.2d, v31.2d, #44\n\t" + "eor x27, x27, %[r]\n\t" + "ushr v9.2d, v22.2d, #3\n\t" + "eor x3, x3, x30\n\t" + "sli v6.2d, v31.2d, #20\n\t" + "eor x8, x8, x30\n\t" + "sli v9.2d, v22.2d, #61\n\t" + "eor x13, x13, x30\n\t" + "eor v31.16b, v14.16b, v29.16b\n\t" + "eor x19, x19, x30\n\t" + "eor v20.16b, v20.16b, v25.16b\n\t" + "eor x24, x24, x30\n\t" + "ushr v22.2d, v31.2d, #25\n\t" + "eor x5, x5, %x[seed]\n\t" + "ushr v14.2d, v20.2d, #46\n\t" + "eor x10, x10, %x[seed]\n\t" + "sli v22.2d, v31.2d, #39\n\t" + "eor x15, x15, %x[seed]\n\t" + "sli v14.2d, v20.2d, #18\n\t" + "eor x21, x21, %x[seed]\n\t" + "eor v31.16b, v2.16b, v27.16b\n\t" + "eor x26, x26, %x[seed]\n\t" + /* Swap Rotate Base */ + "eor v12.16b, v12.16b, v27.16b\n\t" + "ror %x[state], x3, #63\n\t" + "ushr v20.2d, v31.2d, #2\n\t" + "ror x3, x8, #20\n\t" + "ushr v2.2d, v12.2d, #21\n\t" + "ror x8, x11, #44\n\t" + "sli v20.2d, v31.2d, #62\n\t" + "ror x11, x25, #3\n\t" + "sli v2.2d, v12.2d, #43\n\t" + "ror x25, x16, #25\n\t" + "eor v31.16b, v13.16b, v28.16b\n\t" + "ror x16, x23, #46\n\t" + "eor v19.16b, v19.16b, v29.16b\n\t" + "ror x23, x4, #2\n\t" + "ushr v12.2d, v31.2d, #39\n\t" + "ror x4, x14, #21\n\t" + "ushr v13.2d, v19.2d, #56\n\t" + "ror x14, x15, #39\n\t" + "sli v12.2d, v31.2d, #25\n\t" + "ror x15, x22, #56\n\t" + "sli v13.2d, v19.2d, #8\n\t" + "ror x22, x26, #8\n\t" + "eor v31.16b, v23.16b, v28.16b\n\t" + "ror x26, x17, #23\n\t" + "eor v15.16b, v15.16b, v25.16b\n\t" + "ror x17, x6, #37\n\t" + "ushr v19.2d, v31.2d, #8\n\t" + "ror x6, x27, #50\n\t" + "ushr v23.2d, v15.2d, #23\n\t" + "ror x27, x24, #62\n\t" + "sli v19.2d, v31.2d, #56\n\t" + "ror x24, x10, #9\n\t" + "sli v23.2d, v15.2d, #41\n\t" + "ror x10, x19, #19\n\t" + "eor v31.16b, v4.16b, v29.16b\n\t" + "ror x19, x7, #28\n\t" + "eor v24.16b, v24.16b, v29.16b\n\t" + "ror x7, x5, #36\n\t" + "ushr v15.2d, v31.2d, #37\n\t" + "ror x5, x21, #43\n\t" + "ushr v4.2d, v24.2d, #50\n\t" + "ror x21, x20, #49\n\t" + "sli v15.2d, v31.2d, #27\n\t" + "ror x20, x13, #54\n\t" + "sli v4.2d, v24.2d, #14\n\t" + "ror x13, x9, #58\n\t" + "eor v31.16b, v21.16b, v26.16b\n\t" + "ror x9, x12, #61\n\t" + /* Row Mix Base */ + "eor v8.16b, v8.16b, v28.16b\n\t" + "bic x12, x4, x3\n\t" + "ushr v24.2d, v31.2d, #62\n\t" + "bic %x[seed], x5, x4\n\t" + "ushr v21.2d, v8.2d, #9\n\t" + "bic %[r], x2, x6\n\t" + "sli v24.2d, v31.2d, #2\n\t" + "bic x30, x3, x2\n\t" + "sli v21.2d, v8.2d, #55\n\t" + "eor x2, x2, x12\n\t" + "eor v31.16b, v16.16b, v26.16b\n\t" + "eor x3, x3, %x[seed]\n\t" + "eor v5.16b, v5.16b, v25.16b\n\t" + "bic x12, x6, x5\n\t" + "ushr v8.2d, v31.2d, #19\n\t" + "eor x5, x5, %[r]\n\t" + "ushr v16.2d, v5.2d, #28\n\t" + "eor x4, x4, x12\n\t" + "sli v8.2d, v31.2d, #45\n\t" + "eor x6, x6, x30\n\t" + "sli v16.2d, v5.2d, #36\n\t" + "bic x12, x9, x8\n\t" + "eor v31.16b, v3.16b, v28.16b\n\t" + "bic %x[seed], x10, x9\n\t" + "eor v18.16b, v18.16b, v28.16b\n\t" + "bic %[r], x7, x11\n\t" + "ushr v5.2d, v31.2d, #36\n\t" + "bic x30, x8, x7\n\t" + "ushr v3.2d, v18.2d, #43\n\t" + "eor x7, x7, x12\n\t" + "sli v5.2d, v31.2d, #28\n\t" + "eor x8, x8, %x[seed]\n\t" + "sli v3.2d, v18.2d, #21\n\t" + "bic x12, x11, x10\n\t" + "eor v31.16b, v17.16b, v27.16b\n\t" + "eor x10, x10, %[r]\n\t" + "eor v11.16b, v11.16b, v26.16b\n\t" + "eor x9, x9, x12\n\t" + "ushr v18.2d, v31.2d, #49\n\t" + "eor x11, x11, x30\n\t" + "ushr v17.2d, v11.2d, #54\n\t" + "bic x12, x14, x13\n\t" + "sli v18.2d, v31.2d, #15\n\t" + "bic %x[seed], x15, x14\n\t" + "sli v17.2d, v11.2d, #10\n\t" + "bic %[r], %x[state], x16\n\t" + "eor v31.16b, v7.16b, v27.16b\n\t" + "bic x30, x13, %x[state]\n\t" + "eor v10.16b, v10.16b, v25.16b\n\t" + "eor x12, %x[state], x12\n\t" + "ushr v11.2d, v31.2d, #58\n\t" + "eor x13, x13, %x[seed]\n\t" + "ushr v7.2d, v10.2d, #61\n\t" + "bic %x[state], x16, x15\n\t" + "sli v11.2d, v31.2d, #6\n\t" + "eor x15, x15, %[r]\n\t" + "sli v7.2d, v10.2d, #3\n\t" + "eor x14, x14, %x[state]\n\t" + /* Row Mix NEON */ + "bic v25.16b, v2.16b, v1.16b\n\t" + "eor x16, x16, x30\n\t" + "bic v26.16b, v3.16b, v2.16b\n\t" + "bic %x[state], x20, x19\n\t" + "bic v27.16b, v4.16b, v3.16b\n\t" + "bic %x[seed], x21, x20\n\t" + "bic v28.16b, v0.16b, v4.16b\n\t" + "bic %[r], x17, x22\n\t" + "bic v29.16b, v1.16b, v0.16b\n\t" + "bic x30, x19, x17\n\t" + "eor v0.16b, v0.16b, v25.16b\n\t" + "eor x17, x17, %x[state]\n\t" + "eor v1.16b, v1.16b, v26.16b\n\t" + "eor x19, x19, %x[seed]\n\t" + "eor v2.16b, v2.16b, v27.16b\n\t" + "bic %x[state], x22, x21\n\t" + "eor v3.16b, v3.16b, v28.16b\n\t" + "eor x21, x21, %[r]\n\t" + "eor v4.16b, v4.16b, v29.16b\n\t" + "eor x20, x20, %x[state]\n\t" + "bic v25.16b, v7.16b, v6.16b\n\t" + "eor x22, x22, x30\n\t" + "bic v26.16b, v8.16b, v7.16b\n\t" + "bic %x[state], x25, x24\n\t" + "bic v27.16b, v9.16b, v8.16b\n\t" + "bic %x[seed], x26, x25\n\t" + "bic v28.16b, v5.16b, v9.16b\n\t" + "bic %[r], x23, x27\n\t" + "bic v29.16b, v6.16b, v5.16b\n\t" + "bic x30, x24, x23\n\t" + "eor v5.16b, v5.16b, v25.16b\n\t" + "eor x23, x23, %x[state]\n\t" + "eor v6.16b, v6.16b, v26.16b\n\t" + "eor x24, x24, %x[seed]\n\t" + "eor v7.16b, v7.16b, v27.16b\n\t" + "bic %x[state], x27, x26\n\t" + "eor v8.16b, v8.16b, v28.16b\n\t" + "eor x26, x26, %[r]\n\t" + "eor v9.16b, v9.16b, v29.16b\n\t" + "eor x25, x25, %x[state]\n\t" + "bic v25.16b, v12.16b, v11.16b\n\t" + "eor x27, x27, x30\n\t" + "bic v26.16b, v13.16b, v12.16b\n\t" + "bic v27.16b, v14.16b, v13.16b\n\t" + "bic v28.16b, v30.16b, v14.16b\n\t" + "bic v29.16b, v11.16b, v30.16b\n\t" + "eor v10.16b, v30.16b, v25.16b\n\t" + "eor v11.16b, v11.16b, v26.16b\n\t" + "eor v12.16b, v12.16b, v27.16b\n\t" + "eor v13.16b, v13.16b, v28.16b\n\t" + "eor v14.16b, v14.16b, v29.16b\n\t" + "bic v25.16b, v17.16b, v16.16b\n\t" + "bic v26.16b, v18.16b, v17.16b\n\t" + "bic v27.16b, v19.16b, v18.16b\n\t" + "bic v28.16b, v15.16b, v19.16b\n\t" + "bic v29.16b, v16.16b, v15.16b\n\t" + "eor v15.16b, v15.16b, v25.16b\n\t" + "eor v16.16b, v16.16b, v26.16b\n\t" + "eor v17.16b, v17.16b, v27.16b\n\t" + "eor v18.16b, v18.16b, v28.16b\n\t" + "eor v19.16b, v19.16b, v29.16b\n\t" + "bic v25.16b, v22.16b, v21.16b\n\t" + "bic v26.16b, v23.16b, v22.16b\n\t" + "bic v27.16b, v24.16b, v23.16b\n\t" + "bic v28.16b, v20.16b, v24.16b\n\t" + "bic v29.16b, v21.16b, v20.16b\n\t" + "eor v20.16b, v20.16b, v25.16b\n\t" + "eor v21.16b, v21.16b, v26.16b\n\t" + "eor v22.16b, v22.16b, v27.16b\n\t" + "eor v23.16b, v23.16b, v28.16b\n\t" + "eor v24.16b, v24.16b, v29.16b\n\t" + /* Done transforming */ + "ldp %[r], %x[seed], [x29, #48]\n\t" + "ldr %x[state], [%[r]], #8\n\t" + "subs %x[seed], %x[seed], #1\n\t" + "mov v30.d[0], %x[state]\n\t" + "mov v30.d[1], %x[state]\n\t" + "eor x2, x2, %x[state]\n\t" + "eor v0.16b, v0.16b, v30.16b\n\t" + "b.ne L_SHA3_shake128_blocksx3_seed_neon_begin_%=\n\t" + "ldr %x[state], [x29, #40]\n\t" + "st4 {v0.d, v1.d, v2.d, v3.d}[0], [%x[state]], #32\n\t" + "st4 {v4.d, v5.d, v6.d, v7.d}[0], [%x[state]], #32\n\t" + "st4 {v8.d, v9.d, v10.d, v11.d}[0], [%x[state]], #32\n\t" + "st4 {v12.d, v13.d, v14.d, v15.d}[0], [%x[state]], #32\n\t" + "st4 {v16.d, v17.d, v18.d, v19.d}[0], [%x[state]], #32\n\t" + "st4 {v20.d, v21.d, v22.d, v23.d}[0], [%x[state]], #32\n\t" + "st1 {v24.d}[0], [%x[state]]\n\t" + "add %x[state], %x[state], #8\n\t" + "st4 {v0.d, v1.d, v2.d, v3.d}[1], [%x[state]], #32\n\t" + "st4 {v4.d, v5.d, v6.d, v7.d}[1], [%x[state]], #32\n\t" + "st4 {v8.d, v9.d, v10.d, v11.d}[1], [%x[state]], #32\n\t" + "st4 {v12.d, v13.d, v14.d, v15.d}[1], [%x[state]], #32\n\t" + "st4 {v16.d, v17.d, v18.d, v19.d}[1], [%x[state]], #32\n\t" + "st4 {v20.d, v21.d, v22.d, v23.d}[1], [%x[state]], #32\n\t" + "st1 {v24.d}[1], [%x[state]]\n\t" + "add %x[state], %x[state], #8\n\t" + "stp x2, x3, [%x[state]]\n\t" + "stp x4, x5, [%x[state], #16]\n\t" + "stp x6, x7, [%x[state], #32]\n\t" + "stp x8, x9, [%x[state], #48]\n\t" + "stp x10, x11, [%x[state], #64]\n\t" + "stp x12, x13, [%x[state], #80]\n\t" + "stp x14, x15, [%x[state], #96]\n\t" + "stp x16, x17, [%x[state], #112]\n\t" + "stp x19, x20, [%x[state], #128]\n\t" + "stp x21, x22, [%x[state], #144]\n\t" + "stp x23, x24, [%x[state], #160]\n\t" + "stp x25, x26, [%x[state], #176]\n\t" + "str x27, [%x[state], #192]\n\t" + "ldp x29, x30, [sp], #0x40\n\t" + : [state] "+r" (state), [seed] "+r" (seed) + : [r] "r" (r) + : "memory", "cc", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", + "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", + "x21", "x22", "x23", "x24", "x25", "x26", "x27", "v0", "v1", "v2", + "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", + "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", + "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", + "v31" + ); +} + +void mlkem_shake256_blocksx3_seed_neon(word64* state, byte* seed) +{ + const word64* r = L_sha3_aarch64_r; + __asm__ __volatile__ ( + "stp x29, x30, [sp, #-64]!\n\t" + "add x29, sp, #0\n\t" + "str %x[state], [x29, #40]\n\t" + "add %x[state], %x[state], #32\n\t" + "ld1 {v4.d}[0], [%x[state]]\n\t" + "ldp x2, x3, [%x[seed]], #16\n\t" + "add %x[state], %x[state], #0xc8\n\t" + "ld1 {v4.d}[1], [%x[state]]\n\t" + "ldp x4, x5, [%x[seed]], #16\n\t" + "ldr x6, [%x[state], #200]\n\t" + "eor v5.16b, v5.16b, v5.16b\n\t" + "eor x7, x7, x7\n\t" + "eor v6.16b, v6.16b, v6.16b\n\t" + "eor x8, x8, x8\n\t" + "eor v7.16b, v7.16b, v7.16b\n\t" + "eor x9, x9, x9\n\t" + "eor v8.16b, v8.16b, v8.16b\n\t" + "eor x10, x10, x10\n\t" + "eor v9.16b, v9.16b, v9.16b\n\t" + "eor x11, x11, x11\n\t" + "eor v10.16b, v10.16b, v10.16b\n\t" + "eor x12, x12, x12\n\t" + "eor v11.16b, v11.16b, v11.16b\n\t" + "eor x13, x13, x13\n\t" + "eor v12.16b, v12.16b, v12.16b\n\t" + "eor x14, x14, x14\n\t" + "eor v13.16b, v13.16b, v13.16b\n\t" + "eor x15, x15, x15\n\t" + "eor v14.16b, v14.16b, v14.16b\n\t" + "eor x16, x16, x16\n\t" + "eor v15.16b, v15.16b, v15.16b\n\t" + "eor x17, x17, x17\n\t" + "movz x19, #0x8000, lsl 48\n\t" + "eor v17.16b, v17.16b, v17.16b\n\t" + "eor x20, x20, x20\n\t" + "eor v18.16b, v18.16b, v18.16b\n\t" + "eor x21, x21, x21\n\t" + "eor v19.16b, v19.16b, v19.16b\n\t" + "eor x22, x22, x22\n\t" + "eor v20.16b, v20.16b, v20.16b\n\t" + "eor x23, x23, x23\n\t" + "eor v21.16b, v21.16b, v21.16b\n\t" + "eor x24, x24, x24\n\t" + "eor v22.16b, v22.16b, v22.16b\n\t" + "eor x25, x25, x25\n\t" + "eor v23.16b, v23.16b, v23.16b\n\t" + "eor x26, x26, x26\n\t" + "eor v24.16b, v24.16b, v24.16b\n\t" + "eor x27, x27, x27\n\t" + "dup v0.2d, x2\n\t" + "dup v1.2d, x3\n\t" + "dup v2.2d, x4\n\t" + "dup v3.2d, x5\n\t" + "dup v16.2d, x19\n\t" + "mov %x[seed], #24\n\t" + /* Start of 24 rounds */ + "\n" + "L_SHA3_shake256_blocksx3_seed_neon_begin_%=:\n\t" + "stp %[r], %x[seed], [x29, #48]\n\t" + /* Col Mix NEON */ + "eor v30.16b, v4.16b, v9.16b\n\t" + "eor %x[state], x6, x11\n\t" + "eor v27.16b, v1.16b, v6.16b\n\t" + "eor x30, x2, x7\n\t" + "eor v30.16b, v30.16b, v14.16b\n\t" + "eor %[r], x4, x9\n\t" + "eor v27.16b, v27.16b, v11.16b\n\t" + "eor %x[state], %x[state], x16\n\t" + "eor v30.16b, v30.16b, v19.16b\n\t" + "eor x30, x30, x12\n\t" + "eor v27.16b, v27.16b, v16.16b\n\t" + "eor %[r], %[r], x14\n\t" + "eor v30.16b, v30.16b, v24.16b\n\t" + "eor %x[state], %x[state], x22\n\t" + "eor v27.16b, v27.16b, v21.16b\n\t" + "eor x30, x30, x17\n\t" + "ushr v25.2d, v27.2d, #63\n\t" + "eor %[r], %[r], x20\n\t" + "sli v25.2d, v27.2d, #1\n\t" + "eor %x[state], %x[state], x27\n\t" + "eor v25.16b, v25.16b, v30.16b\n\t" + "eor x30, x30, x23\n\t" + "eor v31.16b, v0.16b, v5.16b\n\t" + "eor %[r], %[r], x25\n\t" + "eor v28.16b, v2.16b, v7.16b\n\t" + "str %x[state], [x29, #32]\n\t" + "eor v31.16b, v31.16b, v10.16b\n\t" + "str %[r], [x29, #24]\n\t" + "eor v28.16b, v28.16b, v12.16b\n\t" + "eor %x[seed], x3, x8\n\t" + "eor v31.16b, v31.16b, v15.16b\n\t" + "eor %[r], x5, x10\n\t" + "eor v28.16b, v28.16b, v17.16b\n\t" + "eor %x[seed], %x[seed], x13\n\t" + "eor v31.16b, v31.16b, v20.16b\n\t" + "eor %[r], %[r], x15\n\t" + "eor v28.16b, v28.16b, v22.16b\n\t" + "eor %x[seed], %x[seed], x19\n\t" + "ushr v29.2d, v30.2d, #63\n\t" + "eor %[r], %[r], x21\n\t" + "ushr v26.2d, v28.2d, #63\n\t" + "eor %x[seed], %x[seed], x24\n\t" + "sli v29.2d, v30.2d, #1\n\t" + "eor %[r], %[r], x26\n\t" + "sli v26.2d, v28.2d, #1\n\t" + "eor %x[state], %x[state], %x[seed], ror 63\n\t" + "eor v28.16b, v28.16b, v29.16b\n\t" + "eor %x[seed], %x[seed], %[r], ror 63\n\t" + "eor v29.16b, v3.16b, v8.16b\n\t" + "eor x2, x2, %x[state]\n\t" + "eor v26.16b, v26.16b, v31.16b\n\t" + "eor x7, x7, %x[state]\n\t" + "eor v29.16b, v29.16b, v13.16b\n\t" + "eor x12, x12, %x[state]\n\t" + "eor v29.16b, v29.16b, v18.16b\n\t" + "eor x17, x17, %x[state]\n\t" + "eor v29.16b, v29.16b, v23.16b\n\t" + "eor x23, x23, %x[state]\n\t" + "ushr v30.2d, v29.2d, #63\n\t" + "eor x4, x4, %x[seed]\n\t" + "sli v30.2d, v29.2d, #1\n\t" + "eor x9, x9, %x[seed]\n\t" + "eor v27.16b, v27.16b, v30.16b\n\t" + "eor x14, x14, %x[seed]\n\t" + "ushr v30.2d, v31.2d, #63\n\t" + "eor x20, x20, %x[seed]\n\t" + "sli v30.2d, v31.2d, #1\n\t" + "eor x25, x25, %x[seed]\n\t" + "eor v29.16b, v29.16b, v30.16b\n\t" + "ldr %x[state], [x29, #32]\n\t" + /* Swap Rotate NEON */ + "eor v0.16b, v0.16b, v25.16b\n\t" + "eor v31.16b, v1.16b, v26.16b\n\t" + "ldr %x[seed], [x29, #24]\n\t" + "eor v6.16b, v6.16b, v26.16b\n\t" + "eor %[r], %[r], x30, ror 63\n\t" + "ushr v30.2d, v31.2d, #63\n\t" + "eor x30, x30, %x[seed], ror 63\n\t" + "ushr v1.2d, v6.2d, #20\n\t" + "eor %x[seed], %x[seed], %x[state], ror 63\n\t" + "sli v30.2d, v31.2d, #1\n\t" + "eor x6, x6, %[r]\n\t" + "sli v1.2d, v6.2d, #44\n\t" + "eor x11, x11, %[r]\n\t" + "eor v31.16b, v9.16b, v29.16b\n\t" + "eor x16, x16, %[r]\n\t" + "eor v22.16b, v22.16b, v27.16b\n\t" + "eor x22, x22, %[r]\n\t" + "ushr v6.2d, v31.2d, #44\n\t" + "eor x27, x27, %[r]\n\t" + "ushr v9.2d, v22.2d, #3\n\t" + "eor x3, x3, x30\n\t" + "sli v6.2d, v31.2d, #20\n\t" + "eor x8, x8, x30\n\t" + "sli v9.2d, v22.2d, #61\n\t" + "eor x13, x13, x30\n\t" + "eor v31.16b, v14.16b, v29.16b\n\t" + "eor x19, x19, x30\n\t" + "eor v20.16b, v20.16b, v25.16b\n\t" + "eor x24, x24, x30\n\t" + "ushr v22.2d, v31.2d, #25\n\t" + "eor x5, x5, %x[seed]\n\t" + "ushr v14.2d, v20.2d, #46\n\t" + "eor x10, x10, %x[seed]\n\t" + "sli v22.2d, v31.2d, #39\n\t" + "eor x15, x15, %x[seed]\n\t" + "sli v14.2d, v20.2d, #18\n\t" + "eor x21, x21, %x[seed]\n\t" + "eor v31.16b, v2.16b, v27.16b\n\t" + "eor x26, x26, %x[seed]\n\t" + /* Swap Rotate Base */ + "eor v12.16b, v12.16b, v27.16b\n\t" + "ror %x[state], x3, #63\n\t" + "ushr v20.2d, v31.2d, #2\n\t" + "ror x3, x8, #20\n\t" + "ushr v2.2d, v12.2d, #21\n\t" + "ror x8, x11, #44\n\t" + "sli v20.2d, v31.2d, #62\n\t" + "ror x11, x25, #3\n\t" + "sli v2.2d, v12.2d, #43\n\t" + "ror x25, x16, #25\n\t" + "eor v31.16b, v13.16b, v28.16b\n\t" + "ror x16, x23, #46\n\t" + "eor v19.16b, v19.16b, v29.16b\n\t" + "ror x23, x4, #2\n\t" + "ushr v12.2d, v31.2d, #39\n\t" + "ror x4, x14, #21\n\t" + "ushr v13.2d, v19.2d, #56\n\t" + "ror x14, x15, #39\n\t" + "sli v12.2d, v31.2d, #25\n\t" + "ror x15, x22, #56\n\t" + "sli v13.2d, v19.2d, #8\n\t" + "ror x22, x26, #8\n\t" + "eor v31.16b, v23.16b, v28.16b\n\t" + "ror x26, x17, #23\n\t" + "eor v15.16b, v15.16b, v25.16b\n\t" + "ror x17, x6, #37\n\t" + "ushr v19.2d, v31.2d, #8\n\t" + "ror x6, x27, #50\n\t" + "ushr v23.2d, v15.2d, #23\n\t" + "ror x27, x24, #62\n\t" + "sli v19.2d, v31.2d, #56\n\t" + "ror x24, x10, #9\n\t" + "sli v23.2d, v15.2d, #41\n\t" + "ror x10, x19, #19\n\t" + "eor v31.16b, v4.16b, v29.16b\n\t" + "ror x19, x7, #28\n\t" + "eor v24.16b, v24.16b, v29.16b\n\t" + "ror x7, x5, #36\n\t" + "ushr v15.2d, v31.2d, #37\n\t" + "ror x5, x21, #43\n\t" + "ushr v4.2d, v24.2d, #50\n\t" + "ror x21, x20, #49\n\t" + "sli v15.2d, v31.2d, #27\n\t" + "ror x20, x13, #54\n\t" + "sli v4.2d, v24.2d, #14\n\t" + "ror x13, x9, #58\n\t" + "eor v31.16b, v21.16b, v26.16b\n\t" + "ror x9, x12, #61\n\t" + /* Row Mix Base */ + "eor v8.16b, v8.16b, v28.16b\n\t" + "bic x12, x4, x3\n\t" + "ushr v24.2d, v31.2d, #62\n\t" + "bic %x[seed], x5, x4\n\t" + "ushr v21.2d, v8.2d, #9\n\t" + "bic %[r], x2, x6\n\t" + "sli v24.2d, v31.2d, #2\n\t" + "bic x30, x3, x2\n\t" + "sli v21.2d, v8.2d, #55\n\t" + "eor x2, x2, x12\n\t" + "eor v31.16b, v16.16b, v26.16b\n\t" + "eor x3, x3, %x[seed]\n\t" + "eor v5.16b, v5.16b, v25.16b\n\t" + "bic x12, x6, x5\n\t" + "ushr v8.2d, v31.2d, #19\n\t" + "eor x5, x5, %[r]\n\t" + "ushr v16.2d, v5.2d, #28\n\t" + "eor x4, x4, x12\n\t" + "sli v8.2d, v31.2d, #45\n\t" + "eor x6, x6, x30\n\t" + "sli v16.2d, v5.2d, #36\n\t" + "bic x12, x9, x8\n\t" + "eor v31.16b, v3.16b, v28.16b\n\t" + "bic %x[seed], x10, x9\n\t" + "eor v18.16b, v18.16b, v28.16b\n\t" + "bic %[r], x7, x11\n\t" + "ushr v5.2d, v31.2d, #36\n\t" + "bic x30, x8, x7\n\t" + "ushr v3.2d, v18.2d, #43\n\t" + "eor x7, x7, x12\n\t" + "sli v5.2d, v31.2d, #28\n\t" + "eor x8, x8, %x[seed]\n\t" + "sli v3.2d, v18.2d, #21\n\t" + "bic x12, x11, x10\n\t" + "eor v31.16b, v17.16b, v27.16b\n\t" + "eor x10, x10, %[r]\n\t" + "eor v11.16b, v11.16b, v26.16b\n\t" + "eor x9, x9, x12\n\t" + "ushr v18.2d, v31.2d, #49\n\t" + "eor x11, x11, x30\n\t" + "ushr v17.2d, v11.2d, #54\n\t" + "bic x12, x14, x13\n\t" + "sli v18.2d, v31.2d, #15\n\t" + "bic %x[seed], x15, x14\n\t" + "sli v17.2d, v11.2d, #10\n\t" + "bic %[r], %x[state], x16\n\t" + "eor v31.16b, v7.16b, v27.16b\n\t" + "bic x30, x13, %x[state]\n\t" + "eor v10.16b, v10.16b, v25.16b\n\t" + "eor x12, %x[state], x12\n\t" + "ushr v11.2d, v31.2d, #58\n\t" + "eor x13, x13, %x[seed]\n\t" + "ushr v7.2d, v10.2d, #61\n\t" + "bic %x[state], x16, x15\n\t" + "sli v11.2d, v31.2d, #6\n\t" + "eor x15, x15, %[r]\n\t" + "sli v7.2d, v10.2d, #3\n\t" + "eor x14, x14, %x[state]\n\t" + /* Row Mix NEON */ + "bic v25.16b, v2.16b, v1.16b\n\t" + "eor x16, x16, x30\n\t" + "bic v26.16b, v3.16b, v2.16b\n\t" + "bic %x[state], x20, x19\n\t" + "bic v27.16b, v4.16b, v3.16b\n\t" + "bic %x[seed], x21, x20\n\t" + "bic v28.16b, v0.16b, v4.16b\n\t" + "bic %[r], x17, x22\n\t" + "bic v29.16b, v1.16b, v0.16b\n\t" + "bic x30, x19, x17\n\t" + "eor v0.16b, v0.16b, v25.16b\n\t" + "eor x17, x17, %x[state]\n\t" + "eor v1.16b, v1.16b, v26.16b\n\t" + "eor x19, x19, %x[seed]\n\t" + "eor v2.16b, v2.16b, v27.16b\n\t" + "bic %x[state], x22, x21\n\t" + "eor v3.16b, v3.16b, v28.16b\n\t" + "eor x21, x21, %[r]\n\t" + "eor v4.16b, v4.16b, v29.16b\n\t" + "eor x20, x20, %x[state]\n\t" + "bic v25.16b, v7.16b, v6.16b\n\t" + "eor x22, x22, x30\n\t" + "bic v26.16b, v8.16b, v7.16b\n\t" + "bic %x[state], x25, x24\n\t" + "bic v27.16b, v9.16b, v8.16b\n\t" + "bic %x[seed], x26, x25\n\t" + "bic v28.16b, v5.16b, v9.16b\n\t" + "bic %[r], x23, x27\n\t" + "bic v29.16b, v6.16b, v5.16b\n\t" + "bic x30, x24, x23\n\t" + "eor v5.16b, v5.16b, v25.16b\n\t" + "eor x23, x23, %x[state]\n\t" + "eor v6.16b, v6.16b, v26.16b\n\t" + "eor x24, x24, %x[seed]\n\t" + "eor v7.16b, v7.16b, v27.16b\n\t" + "bic %x[state], x27, x26\n\t" + "eor v8.16b, v8.16b, v28.16b\n\t" + "eor x26, x26, %[r]\n\t" + "eor v9.16b, v9.16b, v29.16b\n\t" + "eor x25, x25, %x[state]\n\t" + "bic v25.16b, v12.16b, v11.16b\n\t" + "eor x27, x27, x30\n\t" + "bic v26.16b, v13.16b, v12.16b\n\t" + "bic v27.16b, v14.16b, v13.16b\n\t" + "bic v28.16b, v30.16b, v14.16b\n\t" + "bic v29.16b, v11.16b, v30.16b\n\t" + "eor v10.16b, v30.16b, v25.16b\n\t" + "eor v11.16b, v11.16b, v26.16b\n\t" + "eor v12.16b, v12.16b, v27.16b\n\t" + "eor v13.16b, v13.16b, v28.16b\n\t" + "eor v14.16b, v14.16b, v29.16b\n\t" + "bic v25.16b, v17.16b, v16.16b\n\t" + "bic v26.16b, v18.16b, v17.16b\n\t" + "bic v27.16b, v19.16b, v18.16b\n\t" + "bic v28.16b, v15.16b, v19.16b\n\t" + "bic v29.16b, v16.16b, v15.16b\n\t" + "eor v15.16b, v15.16b, v25.16b\n\t" + "eor v16.16b, v16.16b, v26.16b\n\t" + "eor v17.16b, v17.16b, v27.16b\n\t" + "eor v18.16b, v18.16b, v28.16b\n\t" + "eor v19.16b, v19.16b, v29.16b\n\t" + "bic v25.16b, v22.16b, v21.16b\n\t" + "bic v26.16b, v23.16b, v22.16b\n\t" + "bic v27.16b, v24.16b, v23.16b\n\t" + "bic v28.16b, v20.16b, v24.16b\n\t" + "bic v29.16b, v21.16b, v20.16b\n\t" + "eor v20.16b, v20.16b, v25.16b\n\t" + "eor v21.16b, v21.16b, v26.16b\n\t" + "eor v22.16b, v22.16b, v27.16b\n\t" + "eor v23.16b, v23.16b, v28.16b\n\t" + "eor v24.16b, v24.16b, v29.16b\n\t" + /* Done transforming */ + "ldp %[r], %x[seed], [x29, #48]\n\t" + "ldr %x[state], [%[r]], #8\n\t" + "subs %x[seed], %x[seed], #1\n\t" + "mov v30.d[0], %x[state]\n\t" + "mov v30.d[1], %x[state]\n\t" + "eor x2, x2, %x[state]\n\t" + "eor v0.16b, v0.16b, v30.16b\n\t" + "b.ne L_SHA3_shake256_blocksx3_seed_neon_begin_%=\n\t" + "ldr %x[state], [x29, #40]\n\t" + "st4 {v0.d, v1.d, v2.d, v3.d}[0], [%x[state]], #32\n\t" + "st4 {v4.d, v5.d, v6.d, v7.d}[0], [%x[state]], #32\n\t" + "st4 {v8.d, v9.d, v10.d, v11.d}[0], [%x[state]], #32\n\t" + "st4 {v12.d, v13.d, v14.d, v15.d}[0], [%x[state]], #32\n\t" + "st4 {v16.d, v17.d, v18.d, v19.d}[0], [%x[state]], #32\n\t" + "st4 {v20.d, v21.d, v22.d, v23.d}[0], [%x[state]], #32\n\t" + "st1 {v24.d}[0], [%x[state]]\n\t" + "add %x[state], %x[state], #8\n\t" + "st4 {v0.d, v1.d, v2.d, v3.d}[1], [%x[state]], #32\n\t" + "st4 {v4.d, v5.d, v6.d, v7.d}[1], [%x[state]], #32\n\t" + "st4 {v8.d, v9.d, v10.d, v11.d}[1], [%x[state]], #32\n\t" + "st4 {v12.d, v13.d, v14.d, v15.d}[1], [%x[state]], #32\n\t" + "st4 {v16.d, v17.d, v18.d, v19.d}[1], [%x[state]], #32\n\t" + "st4 {v20.d, v21.d, v22.d, v23.d}[1], [%x[state]], #32\n\t" + "st1 {v24.d}[1], [%x[state]]\n\t" + "add %x[state], %x[state], #8\n\t" + "stp x2, x3, [%x[state]]\n\t" + "stp x4, x5, [%x[state], #16]\n\t" + "stp x6, x7, [%x[state], #32]\n\t" + "stp x8, x9, [%x[state], #48]\n\t" + "stp x10, x11, [%x[state], #64]\n\t" + "stp x12, x13, [%x[state], #80]\n\t" + "stp x14, x15, [%x[state], #96]\n\t" + "stp x16, x17, [%x[state], #112]\n\t" + "stp x19, x20, [%x[state], #128]\n\t" + "stp x21, x22, [%x[state], #144]\n\t" + "stp x23, x24, [%x[state], #160]\n\t" + "stp x25, x26, [%x[state], #176]\n\t" + "str x27, [%x[state], #192]\n\t" + "ldp x29, x30, [sp], #0x40\n\t" + : [state] "+r" (state), [seed] "+r" (seed) + : [r] "r" (r) + : "memory", "cc", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", + "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", + "x21", "x22", "x23", "x24", "x25", "x26", "x27", "v0", "v1", "v2", + "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", + "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", + "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", + "v31" + ); +} + #endif /* WOLFSSL_HAVE_MLKEM */ #endif /* __aarch64__ */ #endif /* WOLFSSL_ARMASM */ diff --git a/wolfcrypt/src/wc_mlkem_poly.c b/wolfcrypt/src/wc_mlkem_poly.c index b164972232..c587c9f660 100644 --- a/wolfcrypt/src/wc_mlkem_poly.c +++ b/wolfcrypt/src/wc_mlkem_poly.c @@ -1258,6 +1258,55 @@ void mlkem_init(void) #if defined(__aarch64__) && defined(WOLFSSL_ARMASM) +/* The three-way Keccak helpers have a NEON implementation and one using the + * SHA-3 crypto extension instructions (EOR3/RAX1/XAR/BCAX). Those instructions + * are OPTIONAL in ARMv8.2 and are absent on Cortex-A55 parts such as the NXP + * i.MX95, so the choice has to be made from the CPU ID flags at run time -- the + * same way sha3.c selects between BlockSha3_crypto and BlockSha3_base. + * Selecting at build time made ML-KEM abort with SIGILL on any aarch64 CPU + * without FEAT_SHA3 whenever wolfSSL was configured + * --enable-armasm=sha3-crypto, even though SHA-3 itself fell back correctly. + */ +#ifdef WOLFSSL_ARMASM_CRYPTO_SHA3 + +static void mlkem_sha3_blocksx3(word64* state) +{ + if (IS_AARCH64_SHA3(cpuid_flags)) { + mlkem_sha3_blocksx3_crypto(state); + } + else { + mlkem_sha3_blocksx3_neon(state); + } +} + +static void mlkem_shake128_blocksx3_seed(word64* state, byte* seed) +{ + if (IS_AARCH64_SHA3(cpuid_flags)) { + mlkem_shake128_blocksx3_seed_crypto(state, seed); + } + else { + mlkem_shake128_blocksx3_seed_neon(state, seed); + } +} + +static void mlkem_shake256_blocksx3_seed(word64* state, byte* seed) +{ + if (IS_AARCH64_SHA3(cpuid_flags)) { + mlkem_shake256_blocksx3_seed_crypto(state, seed); + } + else { + mlkem_shake256_blocksx3_seed_neon(state, seed); + } +} + +#else + +#define mlkem_sha3_blocksx3 mlkem_sha3_blocksx3_neon +#define mlkem_shake128_blocksx3_seed mlkem_shake128_blocksx3_seed_neon +#define mlkem_shake256_blocksx3_seed mlkem_shake256_blocksx3_seed_neon + +#endif /* WOLFSSL_ARMASM_CRYPTO_SHA3 */ + #ifndef WOLFSSL_MLKEM_NO_MAKE_KEY /* Generate a public-private key pair from randomly generated data. * @@ -3140,7 +3189,7 @@ static int mlkem_gen_matrix_k2_aarch64(sword16* a, byte* seed, int transposed) state[2*25 + 4] = 0x1f0000 + (0 << 8) + 1; } - mlkem_shake128_blocksx3_seed_neon(state, seed); + mlkem_shake128_blocksx3_seed(state, seed); /* Sample random bytes to create a polynomial. */ p = (byte*)st; ctr0 = mlkem_rej_uniform_neon(a + 0 * MLKEM_N, MLKEM_N, p, XOF_BLOCK_SIZE); @@ -3149,7 +3198,7 @@ static int mlkem_gen_matrix_k2_aarch64(sword16* a, byte* seed, int transposed) p += 25 * 8; ctr2 = mlkem_rej_uniform_neon(a + 2 * MLKEM_N, MLKEM_N, p, XOF_BLOCK_SIZE); while ((ctr0 < MLKEM_N) || (ctr1 < MLKEM_N) || (ctr2 < MLKEM_N)) { - mlkem_sha3_blocksx3_neon(st); + mlkem_sha3_blocksx3(st); p = (byte*)st; ctr0 += mlkem_rej_uniform_neon(a + 0 * MLKEM_N + ctr0, MLKEM_N - ctr0, @@ -3213,7 +3262,7 @@ static int mlkem_gen_matrix_k3_aarch64(sword16* a, byte* seed, int transposed) } } - mlkem_shake128_blocksx3_seed_neon(state, seed); + mlkem_shake128_blocksx3_seed(state, seed); /* Sample random bytes to create a polynomial. */ p = (byte*)st; ctr0 = mlkem_rej_uniform_neon(a + 0 * MLKEM_N, MLKEM_N, p, @@ -3226,7 +3275,7 @@ static int mlkem_gen_matrix_k3_aarch64(sword16* a, byte* seed, int transposed) XOF_BLOCK_SIZE); /* Create more blocks if too many rejected. */ while ((ctr0 < MLKEM_N) || (ctr1 < MLKEM_N) || (ctr2 < MLKEM_N)) { - mlkem_sha3_blocksx3_neon(st); + mlkem_sha3_blocksx3(st); p = (byte*)st; ctr0 += mlkem_rej_uniform_neon(a + 0 * MLKEM_N + ctr0, @@ -3279,7 +3328,7 @@ static int mlkem_gen_matrix_k4_aarch64(sword16* a, byte* seed, int transposed) } } - mlkem_shake128_blocksx3_seed_neon(state, seed); + mlkem_shake128_blocksx3_seed(state, seed); /* Sample random bytes to create a polynomial. */ p = (byte*)st; ctr0 = mlkem_rej_uniform_neon(a + 0 * MLKEM_N, MLKEM_N, p, @@ -3292,7 +3341,7 @@ static int mlkem_gen_matrix_k4_aarch64(sword16* a, byte* seed, int transposed) XOF_BLOCK_SIZE); /* Create more blocks if too many rejected. */ while ((ctr0 < MLKEM_N) || (ctr1 < MLKEM_N) || (ctr2 < MLKEM_N)) { - mlkem_sha3_blocksx3_neon(st); + mlkem_sha3_blocksx3(st); p = (byte*)st; ctr0 += mlkem_rej_uniform_neon(a + 0 * MLKEM_N + ctr0, @@ -5140,7 +5189,7 @@ static void mlkem_get_noise_x3_eta2_aarch64(word64* rand, byte* seed, byte o) rand[1*25 + 4] = 0x1f00 + 1 + o; rand[2*25 + 4] = 0x1f00 + 2 + o; - mlkem_shake256_blocksx3_seed_neon(rand, seed); + mlkem_shake256_blocksx3_seed(rand, seed); } #if defined(WOLFSSL_KYBER512) || defined(WOLFSSL_WC_ML_KEM_512) @@ -5171,11 +5220,11 @@ static void mlkem_get_noise_x3_eta3_aarch64(byte* rand, byte* seed, byte o) state[1*25 + 4] = 0x1f00 + 1 + o; state[2*25 + 4] = 0x1f00 + 2 + o; - mlkem_shake256_blocksx3_seed_neon(state, seed); + mlkem_shake256_blocksx3_seed(state, seed); XMEMCPY(rand + 0 * ETA3_RAND_SIZE, state + 0*25, SHA3_256_BYTES); XMEMCPY(rand + 1 * ETA3_RAND_SIZE, state + 1*25, SHA3_256_BYTES); XMEMCPY(rand + 2 * ETA3_RAND_SIZE, state + 2*25, SHA3_256_BYTES); - mlkem_sha3_blocksx3_neon(state); + mlkem_sha3_blocksx3(state); rand += SHA3_256_BYTES; XMEMCPY(rand + 0 * ETA3_RAND_SIZE, state + 0*25, ETA3_RAND_SIZE - SHA3_256_BYTES); diff --git a/wolfssl/wolfcrypt/wc_mlkem.h b/wolfssl/wolfcrypt/wc_mlkem.h index 038c2a62a6..739bd4a793 100644 --- a/wolfssl/wolfcrypt/wc_mlkem.h +++ b/wolfssl/wolfcrypt/wc_mlkem.h @@ -814,6 +814,13 @@ WOLFSSL_LOCAL void mlkem_to_mont_sqrdmlsh(sword16* p); WOLFSSL_LOCAL void mlkem_sha3_blocksx3_neon(word64* state); WOLFSSL_LOCAL void mlkem_shake128_blocksx3_seed_neon(word64* state, byte* seed); WOLFSSL_LOCAL void mlkem_shake256_blocksx3_seed_neon(word64* state, byte* seed); +#ifdef WOLFSSL_ARMASM_CRYPTO_SHA3 +WOLFSSL_LOCAL void mlkem_sha3_blocksx3_crypto(word64* state); +WOLFSSL_LOCAL void mlkem_shake128_blocksx3_seed_crypto(word64* state, + byte* seed); +WOLFSSL_LOCAL void mlkem_shake256_blocksx3_seed_crypto(word64* state, + byte* seed); +#endif WOLFSSL_LOCAL unsigned int mlkem_rej_uniform_neon(sword16* p, unsigned int len, const byte* r, unsigned int rLen); WOLFSSL_LOCAL int mlkem_cmp_neon(const byte* a, const byte* b, int sz);