wolfSSL/wolfssl
1
0
Fork 1
mirror of https://github.com/wolfSSL/wolfssl.git synced 2026-08-07 14:25:35 +02:00
Code Issues Packages Projects Releases Wiki Activity

PPC64 Assembly: ELF V1 headers added. Little-endian asm added.

Browse Source 
ELF V1 headers are need for cross-compiles.
PPC64 is big or little endian so assembly code modified to be able to be built for both.
Fix wiring of vector AES implementations in aes.c.
This commit is contained in:
Sean Parkinson
2026-07-09 19:18:48 +10:00
parent a4aab71ffe
commit 3f3ebcac58
12 changed files with 20253 additions and 169 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.wolfssl_known_macro_extras
+2
View File
@@ -1084,6 +1084,7 @@ __INTEL_COMPILER
__KEIL__
__KEY_DATA_H__
__LINUX__
__LITTLE_ENDIAN__
__LP64
__LP64__
__MACH__
@@ -1098,6 +1099,7 @@ __OS2__
__OpenBSD__
__PIC__
__PIE__
__POWER9_VECTOR__
__POWERPC__
__PPC__
__PPU
wolfcrypt/src/aes.c
+97 -18
View File
@@ -749,17 +749,16 @@ block cipher mechanism that uses n-bit binary string parameter key with 128-bits
#define AESNI_ALIGN 16
#endif
/* note that all write access to these static variables must be idempotent,
* as arranged by Check_CPU_support_AES(), else they will be susceptible to
* data races.
*/
static int checkedAESNI = 0;
static int haveAESNI = 0;
static cpuid_flags_t intel_flags = WC_CPUID_INITIALIZER;
/* Accessed with the wolfSSL atomic APIs so the one-time detection is free of
* data races. Writes are also idempotent (all callers compute the same
* value), so a benign concurrent double-write is harmless. */
static wolfSSL_Atomic_Uint checkedAESNI = WOLFSSL_ATOMIC_INITIALIZER(0);
static wolfSSL_Atomic_Uint haveAESNI = WOLFSSL_ATOMIC_INITIALIZER(0);
static cpuid_flags_atomic_t intel_flags = WC_CPUID_ATOMIC_INITIALIZER;
static WARN_UNUSED_RESULT int Check_CPU_support_AES(void)
{
cpuid_get_flags_ex(&intel_flags);
cpuid_get_flags_atomic(&intel_flags);
return IS_INTEL_AESNI(intel_flags) != 0;
}
@@ -1163,11 +1162,11 @@ block cipher mechanism that uses n-bit binary string parameter key with 128-bits
#elif defined(WOLFSSL_ARMASM)
#if defined(__aarch64__) && !defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
static cpuid_flags_t cpuid_flags = WC_CPUID_INITIALIZER;
static cpuid_flags_atomic_t cpuid_flags = WC_CPUID_ATOMIC_INITIALIZER;
static void Check_CPU_support_HwCrypto(Aes* aes)
{
cpuid_get_flags_ex(&cpuid_flags);
cpuid_get_flags_atomic(&cpuid_flags);
aes->use_aes_hw_crypto = IS_AARCH64_AES(cpuid_flags);
#ifdef HAVE_AESGCM
aes->use_pmull_hw_crypto = IS_AARCH64_PMULL(cpuid_flags);
@@ -1242,6 +1241,83 @@ static WARN_UNUSED_RESULT int wc_AesDecrypt(Aes* aes, const byte* inBlock,
#elif (defined(WOLFSSL_PPC64_ASM) || defined(WOLFSSL_PPC32_ASM))
#if defined(WOLFSSL_PPC64_ASM) && defined(WOLFSSL_PPC64_ASM_CRYPTO)
/* POWER8+ has vector AES (vcipher/vncipher...) instructions. When built in,
* select the "_crypto" implementations at run time if the CPU supports them.
*
* A run-time flag with direct calls is used rather than a function pointer: an
* indirect call would require an ELFv1 function descriptor, whereas direct
* calls work under both the ELFv1 and ELFv2 ABIs. The dispatch is expressed as
* self-referential macros - the base name inside each macro is not re-expanded
* (C99 6.10.3.4), so it names the real base function. In a PPC build the ARM
* branches that also call these names are #if'd out, so only the live PPC call
* sites are redirected. */
/* Resolved dispatch decision (0 = base, 1 = vector-crypto), accessed with the
* wolfSSL atomic APIs so the one-time detection is free of data races. The
* master cpuid flags read by cpuid_get_flags() are themselves atomic; the write
* here is idempotent so a benign concurrent double-write is harmless. */
static wolfSSL_Atomic_Uint aes_ppc64_use_crypto = WOLFSSL_ATOMIC_INITIALIZER(0);
/* True when the CPU supports the vector-crypto instructions. */
#define AES_PPC64_USE_CRYPTO() (WOLFSSL_ATOMIC_LOAD(aes_ppc64_use_crypto) != 0)
/* Check and set the decision together (as Check_CPU_support_AES/HwCrypto do);
* called from the key-setup path before any AES_*_crypto use. */
static void Aes_SetCrypto(void)
{
WOLFSSL_ATOMIC_STORE(aes_ppc64_use_crypto,
(unsigned int)(IS_PPC64_VEC_CRYPTO(cpuid_get_flags()) != 0));
}
#define AES_set_encrypt_key(key, len, ks) \
(AES_PPC64_USE_CRYPTO() ? \
AES_set_encrypt_key_crypto((key), (len), (ks)) : \
AES_set_encrypt_key((key), (len), (ks)))
#define AES_invert_key(ks, rounds) \
(AES_PPC64_USE_CRYPTO() ? \
AES_invert_key_crypto((ks), (rounds)) : \
AES_invert_key((ks), (rounds)))
#define AES_ECB_encrypt(in, out, len, ks, nr) \
(AES_PPC64_USE_CRYPTO() ? \
AES_ECB_encrypt_crypto((in), (out), (len), (ks), (nr)) : \
AES_ECB_encrypt((in), (out), (len), (ks), (nr)))
#define AES_ECB_decrypt(in, out, len, ks, nr) \
(AES_PPC64_USE_CRYPTO() ? \
AES_ECB_decrypt_crypto((in), (out), (len), (ks), (nr)) : \
AES_ECB_decrypt((in), (out), (len), (ks), (nr)))
#define AES_CBC_encrypt(in, out, len, ks, nr, iv) \
(AES_PPC64_USE_CRYPTO() ? \
AES_CBC_encrypt_crypto((in), (out), (len), (ks), (nr), (iv)) : \
AES_CBC_encrypt((in), (out), (len), (ks), (nr), (iv)))
#define AES_CBC_decrypt(in, out, len, ks, nr, iv) \
(AES_PPC64_USE_CRYPTO() ? \
AES_CBC_decrypt_crypto((in), (out), (len), (ks), (nr), (iv)) : \
AES_CBC_decrypt((in), (out), (len), (ks), (nr), (iv)))
#define AES_CTR_encrypt(in, out, len, ks, nr, ctr) \
(AES_PPC64_USE_CRYPTO() ? \
AES_CTR_encrypt_crypto((in), (out), (len), (ks), (nr), (ctr)) : \
AES_CTR_encrypt((in), (out), (len), (ks), (nr), (ctr)))
#define AES_GCM_encrypt(in, out, len, ks, nr, ctr) \
(AES_PPC64_USE_CRYPTO() ? \
AES_GCM_encrypt_crypto((in), (out), (len), (ks), (nr), (ctr)) : \
AES_GCM_encrypt((in), (out), (len), (ks), (nr), (ctr)))
#if defined(WOLFSSL_AES_XTS)
#define AES_XTS_encrypt(in, out, sz, i, key, key2, tmp, nr) \
(AES_PPC64_USE_CRYPTO() ? \
AES_XTS_encrypt_crypto((in), (out), (sz), (i), (key), (key2), (tmp), \
(nr)) : \
AES_XTS_encrypt((in), (out), (sz), (i), (key), (key2), (tmp), (nr)))
#define AES_XTS_decrypt(in, out, sz, i, key, key2, tmp, nr) \
(AES_PPC64_USE_CRYPTO() ? \
AES_XTS_decrypt_crypto((in), (out), (sz), (i), (key), (key2), (tmp), \
(nr)) : \
AES_XTS_decrypt((in), (out), (sz), (i), (key), (key2), (tmp), (nr)))
#endif /* WOLFSSL_AES_XTS */
#else
#define Aes_SetCrypto() WC_DO_NOTHING
#endif /* WOLFSSL_PPC64_ASM && WOLFSSL_PPC64_ASM_CRYPTO */
#if defined(WOLFSSL_AES_DIRECT) || defined(HAVE_AESCCM) || \
defined(WOLFSSL_AESGCM_STREAM) || defined(HAVE_AESGCM)
static WARN_UNUSED_RESULT int wc_AesEncrypt(Aes* aes, const byte* inBlock,
@@ -4955,6 +5031,9 @@ static WARN_UNUSED_RESULT int wc_AesDecrypt(Aes* aes, const byte* inBlock,
aes->keylen = (int)keylen;
aes->rounds = (keylen/4) + 6;
/* Determine base vs vector-crypto before the (dispatched) key setup so
* the schedule matches the mode functions that later consume it. */
Aes_SetCrypto();
AES_set_encrypt_key(userKey, keylen * 8, (byte*)aes->key);
#ifdef HAVE_AES_DECRYPT
@@ -5637,11 +5716,11 @@ static void AesSetKey_C(Aes* aes, const byte* key, word32 keySz, int dir)
* function correctly with default build settings.
*/
if (checkedAESNI == 0) {
haveAESNI = Check_CPU_support_AES();
checkedAESNI = 1;
if (WOLFSSL_ATOMIC_LOAD(checkedAESNI) == 0) {
WOLFSSL_ATOMIC_STORE(haveAESNI, Check_CPU_support_AES());
WOLFSSL_ATOMIC_STORE(checkedAESNI, 1);
}
if (haveAESNI
if (WOLFSSL_ATOMIC_LOAD(haveAESNI)
#if defined(WC_FLAG_DONT_USE_VECTOR_OPS) && !defined(WC_C_DYNAMIC_FALLBACK)
&& (aes->use_aesni != WC_FLAG_DONT_USE_VECTOR_OPS)
#endif
@@ -18887,7 +18966,7 @@ static AesGcmSivPolyvalFn AesGcmSivPolyvalAsm(void)
static AesGcmSivPolyvalFn AesGcmSivPolyvalAsm(void)
{
#ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO
cpuid_get_flags_ex(&cpuid_flags);
cpuid_get_flags_atomic(&cpuid_flags);
if (IS_AARCH64_PMULL(cpuid_flags)) {
return &AES_GCMSIV_polyval_pmull;
}
@@ -18906,7 +18985,7 @@ static AesGcmSivPolyvalFn AesGcmSivPolyvalAsm(void)
* AES-NI gates the base path (matching wolfSSL's AES-GCM). */
static AesGcmSivPolyvalFn AesGcmSivPolyvalAsm(void)
{
cpuid_get_flags_ex(&intel_flags);
cpuid_get_flags_atomic(&intel_flags);
if (!IS_INTEL_AESNI(intel_flags)) {
return NULL;
}
@@ -18951,7 +19030,7 @@ static AesGcmSivCtrFn AesGcmSivCtrAsm(void)
static AesGcmSivCtrFn AesGcmSivCtrAsm(void)
{
#ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO
cpuid_get_flags_ex(&cpuid_flags);
cpuid_get_flags_atomic(&cpuid_flags);
if (IS_AARCH64_AES(cpuid_flags)) {
return &AES_GCMSIV_ctr_aarch64;
}
@@ -18969,7 +19048,7 @@ static AesGcmSivCtrFn AesGcmSivCtrAsm(void)
* the base; AVX1/VAES/AVX512 are progressively wider pipelines. */
static AesGcmSivCtrFn AesGcmSivCtrAsm(void)
{
cpuid_get_flags_ex(&intel_flags);
cpuid_get_flags_atomic(&intel_flags);
if (!IS_INTEL_AESNI(intel_flags)) {
return NULL;
}
wolfcrypt/src/port/ppc64/ppc64-aes-asm.S
+10002 -60
View File
File diff suppressed because it is too large Load Diff
wolfcrypt/src/port/ppc64/ppc64-aes-asm_c.c
+9632 -48
View File
File diff suppressed because it is too large Load Diff
wolfcrypt/src/port/ppc64/ppc64-sha256-asm.S
+108 -4
View File
@@ -60,16 +60,29 @@ L_SHA256_transform_len_k:
.long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
.long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
#ifndef __APPLE__
#if defined(_CALL_ELF) && _CALL_ELF == 2
.text
.globl Transform_Sha256_Len
.type Transform_Sha256_Len,@function
.align 16
Transform_Sha256_Len:
#if defined(_CALL_ELF) && _CALL_ELF == 2
0: addis 2, 12, .TOC.-0b@ha
addi 2, 2, .TOC.-0b@l
.localentry Transform_Sha256_Len, .-Transform_Sha256_Len
#endif
#else
/* Power ELF V1 ABI: emit a function descriptor in .opd that
* points at the code (.L.Transform_Sha256_Len) and the TOC base. */
.section .opd,"aw",@progbits
.globl Transform_Sha256_Len
.type Transform_Sha256_Len,@function
.align 3
Transform_Sha256_Len:
.quad .L.Transform_Sha256_Len,.TOC.@tocbase,0
.size Transform_Sha256_Len,24
.text
.align 16
.L.Transform_Sha256_Len:
#endif /* _CALL_ELF == 2 */
#else
.section __TEXT,__text
.globl _Transform_Sha256_Len
@@ -114,6 +127,40 @@ _Transform_Sha256_Len:
# Start of loop processing a block
L_SHA256_transform_len_begin:
# Load W - 64 bytes
#ifdef __LITTLE_ENDIAN__
li 0, 0
lwbrx 16, 4, 0
li 0, 4
lwbrx 17, 4, 0
li 0, 8
lwbrx 18, 4, 0
li 0, 12
lwbrx 19, 4, 0
li 0, 16
lwbrx 20, 4, 0
li 0, 20
lwbrx 21, 4, 0
li 0, 24
lwbrx 22, 4, 0
li 0, 28
lwbrx 23, 4, 0
li 0, 32
lwbrx 24, 4, 0
li 0, 36
lwbrx 25, 4, 0
li 0, 40
lwbrx 26, 4, 0
li 0, 44
lwbrx 27, 4, 0
li 0, 48
lwbrx 28, 4, 0
li 0, 52
lwbrx 29, 4, 0
li 0, 56
lwbrx 30, 4, 0
li 0, 60
lwbrx 31, 4, 0
#else
lwz 16, 0(4)
lwz 17, 4(4)
lwz 18, 8(4)
@@ -130,6 +177,7 @@ L_SHA256_transform_len_begin:
lwz 29, 52(4)
lwz 30, 56(4)
lwz 31, 60(4)
#endif /* __LITTLE_ENDIAN__ */
# Start of 16 rounds
# Round 0
rotlwi 0, 11, 26
@@ -2504,6 +2552,40 @@ L_SHA256_transform_len_begin:
# Start of loop processing a block
L_SHA256_transform_len_begin:
# Load W - 64 bytes
#ifdef __LITTLE_ENDIAN__
li 0, 0
lwbrx 16, 4, 0
li 0, 4
lwbrx 17, 4, 0
li 0, 8
lwbrx 18, 4, 0
li 0, 12
lwbrx 19, 4, 0
li 0, 16
lwbrx 20, 4, 0
li 0, 20
lwbrx 21, 4, 0
li 0, 24
lwbrx 22, 4, 0
li 0, 28
lwbrx 23, 4, 0
li 0, 32
lwbrx 24, 4, 0
li 0, 36
lwbrx 25, 4, 0
li 0, 40
lwbrx 26, 4, 0
li 0, 44
lwbrx 27, 4, 0
li 0, 48
lwbrx 28, 4, 0
li 0, 52
lwbrx 29, 4, 0
li 0, 56
lwbrx 30, 4, 0
li 0, 60
lwbrx 31, 4, 0
#else
lwz 16, 0(4)
lwz 17, 4(4)
lwz 18, 8(4)
@@ -2520,6 +2602,7 @@ L_SHA256_transform_len_begin:
lwz 29, 52(4)
lwz 30, 56(4)
lwz 31, 60(4)
#endif /* __LITTLE_ENDIAN__ */
li 0, 4
mtctr 0
# Start of 16 rounds
@@ -3287,7 +3370,11 @@ L_SHA256_transform_len_after_blk_15:
addi 1, 1, 0x98
blr
#ifndef __APPLE__
#if defined(_CALL_ELF) && _CALL_ELF == 2
.size Transform_Sha256_Len,.-Transform_Sha256_Len
#else
.size .L.Transform_Sha256_Len,.-.L.Transform_Sha256_Len
#endif /* _CALL_ELF == 2 */
#endif /* __APPLE__ */
#ifdef WOLFSSL_PPC64_ASM_CRYPTO
#ifndef __APPLE__
@@ -3324,16 +3411,29 @@ L_SHA256_crypto_k:
.machine "push"
.machine "power8"
#ifndef __APPLE__
#if defined(_CALL_ELF) && _CALL_ELF == 2
.text
.globl Transform_Sha256_Len_crypto
.type Transform_Sha256_Len_crypto,@function
.align 16
Transform_Sha256_Len_crypto:
#if defined(_CALL_ELF) && _CALL_ELF == 2
0: addis 2, 12, .TOC.-0b@ha
addi 2, 2, .TOC.-0b@l
.localentry Transform_Sha256_Len_crypto, .-Transform_Sha256_Len_crypto
#endif
#else
/* Power ELF V1 ABI: emit a function descriptor in .opd that
* points at the code (.L.Transform_Sha256_Len_crypto) and the TOC base. */
.section .opd,"aw",@progbits
.globl Transform_Sha256_Len_crypto
.type Transform_Sha256_Len_crypto,@function
.align 3
Transform_Sha256_Len_crypto:
.quad .L.Transform_Sha256_Len_crypto,.TOC.@tocbase,0
.size Transform_Sha256_Len_crypto,24
.text
.align 16
.L.Transform_Sha256_Len_crypto:
#endif /* _CALL_ELF == 2 */
#else
.section __TEXT,__text
.globl _Transform_Sha256_Len_crypto
@@ -4781,7 +4881,11 @@ L_SHA256_crypto_begin:
addi 1, 1, 0xc0
blr
#ifndef __APPLE__
#if defined(_CALL_ELF) && _CALL_ELF == 2
.size Transform_Sha256_Len_crypto,.-Transform_Sha256_Len_crypto
#else
.size .L.Transform_Sha256_Len_crypto,.-.L.Transform_Sha256_Len_crypto
#endif /* _CALL_ELF == 2 */
#endif /* __APPLE__ */
.machine "pop"
#endif /* WOLFSSL_PPC64_ASM_CRYPTO */
wolfcrypt/src/port/ppc64/ppc64-sha256-asm_c.c
+70
View File
@@ -104,6 +104,40 @@ void Transform_Sha256_Len(wc_Sha256* sha256, const byte* data, word32 len)
"\n"
"L_SHA256_transform_len_begin_%=: \n\t"
/* Load W - 64 bytes */
#ifdef __LITTLE_ENDIAN__
"li 0, 0\n\t"
"lwbrx 16, %[data], 0\n\t"
"li 0, 4\n\t"
"lwbrx 17, %[data], 0\n\t"
"li 0, 8\n\t"
"lwbrx 18, %[data], 0\n\t"
"li 0, 12\n\t"
"lwbrx 19, %[data], 0\n\t"
"li 0, 16\n\t"
"lwbrx 20, %[data], 0\n\t"
"li 0, 20\n\t"
"lwbrx 21, %[data], 0\n\t"
"li 0, 24\n\t"
"lwbrx 22, %[data], 0\n\t"
"li 0, 28\n\t"
"lwbrx 23, %[data], 0\n\t"
"li 0, 32\n\t"
"lwbrx 24, %[data], 0\n\t"
"li 0, 36\n\t"
"lwbrx 25, %[data], 0\n\t"
"li 0, 40\n\t"
"lwbrx 26, %[data], 0\n\t"
"li 0, 44\n\t"
"lwbrx 27, %[data], 0\n\t"
"li 0, 48\n\t"
"lwbrx 28, %[data], 0\n\t"
"li 0, 52\n\t"
"lwbrx 29, %[data], 0\n\t"
"li 0, 56\n\t"
"lwbrx 30, %[data], 0\n\t"
"li 0, 60\n\t"
"lwbrx 31, %[data], 0\n\t"
#else
"lwz 16, 0(%[data])\n\t"
"lwz 17, 4(%[data])\n\t"
"lwz 18, 8(%[data])\n\t"
@@ -120,6 +154,7 @@ void Transform_Sha256_Len(wc_Sha256* sha256, const byte* data, word32 len)
"lwz 29, 52(%[data])\n\t"
"lwz 30, 56(%[data])\n\t"
"lwz 31, 60(%[data])\n\t"
#endif /* __LITTLE_ENDIAN__ */
/* Start of 16 rounds */
/* Round 0 */
"rotlwi 0, 11, 26\n\t"
@@ -2495,6 +2530,40 @@ void Transform_Sha256_Len(wc_Sha256* sha256, const byte* data, word32 len)
"\n"
"L_SHA256_transform_len_begin_%=: \n\t"
/* Load W - 64 bytes */
#ifdef __LITTLE_ENDIAN__
"li 0, 0\n\t"
"lwbrx 16, %[data], 0\n\t"
"li 0, 4\n\t"
"lwbrx 17, %[data], 0\n\t"
"li 0, 8\n\t"
"lwbrx 18, %[data], 0\n\t"
"li 0, 12\n\t"
"lwbrx 19, %[data], 0\n\t"
"li 0, 16\n\t"
"lwbrx 20, %[data], 0\n\t"
"li 0, 20\n\t"
"lwbrx 21, %[data], 0\n\t"
"li 0, 24\n\t"
"lwbrx 22, %[data], 0\n\t"
"li 0, 28\n\t"
"lwbrx 23, %[data], 0\n\t"
"li 0, 32\n\t"
"lwbrx 24, %[data], 0\n\t"
"li 0, 36\n\t"
"lwbrx 25, %[data], 0\n\t"
"li 0, 40\n\t"
"lwbrx 26, %[data], 0\n\t"
"li 0, 44\n\t"
"lwbrx 27, %[data], 0\n\t"
"li 0, 48\n\t"
"lwbrx 28, %[data], 0\n\t"
"li 0, 52\n\t"
"lwbrx 29, %[data], 0\n\t"
"li 0, 56\n\t"
"lwbrx 30, %[data], 0\n\t"
"li 0, 60\n\t"
"lwbrx 31, %[data], 0\n\t"
#else
"lwz 16, 0(%[data])\n\t"
"lwz 17, 4(%[data])\n\t"
"lwz 18, 8(%[data])\n\t"
@@ -2511,6 +2580,7 @@ void Transform_Sha256_Len(wc_Sha256* sha256, const byte* data, word32 len)
"lwz 29, 52(%[data])\n\t"
"lwz 30, 56(%[data])\n\t"
"lwz 31, 60(%[data])\n\t"
#endif /* __LITTLE_ENDIAN__ */
"li 0, 4\n\t"
"mtctr 0\n\t"
/* Start of 16 rounds */
wolfcrypt/src/port/ppc64/ppc64-sha3-asm.S
+76 -8
View File
@@ -56,16 +56,29 @@ L_SHA3_transform_base_r:
.quad 0x8000000080008081,0x8000000000008080
.quad 0x0000000080000001,0x8000000080008008
#ifndef __APPLE__
#if defined(_CALL_ELF) && _CALL_ELF == 2
.text
.globl BlockSha3_base
.type BlockSha3_base,@function
.align 16
BlockSha3_base:
#if defined(_CALL_ELF) && _CALL_ELF == 2
0: addis 2, 12, .TOC.-0b@ha
addi 2, 2, .TOC.-0b@l
.localentry BlockSha3_base, .-BlockSha3_base
#endif
#else
/* Power ELF V1 ABI: emit a function descriptor in .opd that
* points at the code (.L.BlockSha3_base) and the TOC base. */
.section .opd,"aw",@progbits
.globl BlockSha3_base
.type BlockSha3_base,@function
.align 3
BlockSha3_base:
.quad .L.BlockSha3_base,.TOC.@tocbase,0
.size BlockSha3_base,24
.text
.align 16
.L.BlockSha3_base:
#endif /* _CALL_ELF == 2 */
#else
.section __TEXT,__text
.globl _BlockSha3_base
@@ -330,7 +343,11 @@ L_SHA3_transform_base_begin:
addi 1, 1, 0x98
blr
#ifndef __APPLE__
#if defined(_CALL_ELF) && _CALL_ELF == 2
.size BlockSha3_base,.-BlockSha3_base
#else
.size .L.BlockSha3_base,.-.L.BlockSha3_base
#endif /* _CALL_ELF == 2 */
#endif /* __APPLE__ */
#ifdef WOLFSSL_PPC64_ASM_POWER8
#ifndef __APPLE__
@@ -362,16 +379,29 @@ L_SHA3_transform_power8_r:
.machine "push"
.machine "power8"
#ifndef __APPLE__
#if defined(_CALL_ELF) && _CALL_ELF == 2
.text
.globl BlockSha3_power8
.type BlockSha3_power8,@function
.align 16
BlockSha3_power8:
#if defined(_CALL_ELF) && _CALL_ELF == 2
0: addis 2, 12, .TOC.-0b@ha
addi 2, 2, .TOC.-0b@l
.localentry BlockSha3_power8, .-BlockSha3_power8
#endif
#else
/* Power ELF V1 ABI: emit a function descriptor in .opd that
* points at the code (.L.BlockSha3_power8) and the TOC base. */
.section .opd,"aw",@progbits
.globl BlockSha3_power8
.type BlockSha3_power8,@function
.align 3
BlockSha3_power8:
.quad .L.BlockSha3_power8,.TOC.@tocbase,0
.size BlockSha3_power8,24
.text
.align 16
.L.BlockSha3_power8:
#endif /* _CALL_ELF == 2 */
#else
.section __TEXT,__text
.globl _BlockSha3_power8
@@ -731,7 +761,11 @@ L_SHA3_transform_power8_begin:
addi 1, 1, 0xc0
blr
#ifndef __APPLE__
#if defined(_CALL_ELF) && _CALL_ELF == 2
.size BlockSha3_power8,.-BlockSha3_power8
#else
.size .L.BlockSha3_power8,.-.L.BlockSha3_power8
#endif /* _CALL_ELF == 2 */
#endif /* __APPLE__ */
.machine "pop"
#if (defined(WOLFSSL_HAVE_MLKEM) || defined(HAVE_DILITHIUM)) && defined(WOLFSSL_SHA3_PPC64_BLOCKS_N)
@@ -791,16 +825,29 @@ L_SHA3_blocksx2_power8_r:
.machine "push"
.machine "power8"
#ifndef __APPLE__
#if defined(_CALL_ELF) && _CALL_ELF == 2
.text
.globl sha3_blocksx2_power8
.type sha3_blocksx2_power8,@function
.align 16
sha3_blocksx2_power8:
#if defined(_CALL_ELF) && _CALL_ELF == 2
0: addis 2, 12, .TOC.-0b@ha
addi 2, 2, .TOC.-0b@l
.localentry sha3_blocksx2_power8, .-sha3_blocksx2_power8
#endif
#else
/* Power ELF V1 ABI: emit a function descriptor in .opd that
* points at the code (.L.sha3_blocksx2_power8) and the TOC base. */
.section .opd,"aw",@progbits
.globl sha3_blocksx2_power8
.type sha3_blocksx2_power8,@function
.align 3
sha3_blocksx2_power8:
.quad .L.sha3_blocksx2_power8,.TOC.@tocbase,0
.size sha3_blocksx2_power8,24
.text
.align 16
.L.sha3_blocksx2_power8:
#endif /* _CALL_ELF == 2 */
#else
.section __TEXT,__text
.globl _sha3_blocksx2_power8
@@ -1163,7 +1210,11 @@ L_SHA3_blocksx2_power8_begin:
addi 1, 1, 0xc0
blr
#ifndef __APPLE__
#if defined(_CALL_ELF) && _CALL_ELF == 2
.size sha3_blocksx2_power8,.-sha3_blocksx2_power8
#else
.size .L.sha3_blocksx2_power8,.-.L.sha3_blocksx2_power8
#endif /* _CALL_ELF == 2 */
#endif /* __APPLE__ */
.machine "pop"
#ifndef __APPLE__
@@ -1222,16 +1273,29 @@ L_SHA3_blocksx3_power8_r:
.machine "push"
.machine "power8"
#ifndef __APPLE__
#if defined(_CALL_ELF) && _CALL_ELF == 2
.text
.globl sha3_blocksx3_power8
.type sha3_blocksx3_power8,@function
.align 16
sha3_blocksx3_power8:
#if defined(_CALL_ELF) && _CALL_ELF == 2
0: addis 2, 12, .TOC.-0b@ha
addi 2, 2, .TOC.-0b@l
.localentry sha3_blocksx3_power8, .-sha3_blocksx3_power8
#endif
#else
/* Power ELF V1 ABI: emit a function descriptor in .opd that
* points at the code (.L.sha3_blocksx3_power8) and the TOC base. */
.section .opd,"aw",@progbits
.globl sha3_blocksx3_power8
.type sha3_blocksx3_power8,@function
.align 3
sha3_blocksx3_power8:
.quad .L.sha3_blocksx3_power8,.TOC.@tocbase,0
.size sha3_blocksx3_power8,24
.text
.align 16
.L.sha3_blocksx3_power8:
#endif /* _CALL_ELF == 2 */
#else
.section __TEXT,__text
.globl _sha3_blocksx3_power8
@@ -1836,7 +1900,11 @@ L_SHA3_blocksx3_power8_begin:
addi 1, 1, 0xc0
blr
#ifndef __APPLE__
#if defined(_CALL_ELF) && _CALL_ELF == 2
.size sha3_blocksx3_power8,.-sha3_blocksx3_power8
#else
.size .L.sha3_blocksx3_power8,.-.L.sha3_blocksx3_power8
#endif /* _CALL_ELF == 2 */
#endif /* __APPLE__ */
.machine "pop"
#endif /* (WOLFSSL_HAVE_MLKEM || HAVE_DILITHIUM) && WOLFSSL_SHA3_PPC64_BLOCKS_N */
wolfcrypt/src/port/ppc64/ppc64-sha512-asm.S
+108 -4
View File
@@ -84,16 +84,29 @@ L_SHA512_transform_len_k:
.quad 0x4cc5d4becb3e42b6,0x597f299cfc657e2a
.quad 0x5fcb6fab3ad6faec,0x6c44198c4a475817
#ifndef __APPLE__
#if defined(_CALL_ELF) && _CALL_ELF == 2
.text
.globl Transform_Sha512_Len
.type Transform_Sha512_Len,@function
.align 16
Transform_Sha512_Len:
#if defined(_CALL_ELF) && _CALL_ELF == 2
0: addis 2, 12, .TOC.-0b@ha
addi 2, 2, .TOC.-0b@l
.localentry Transform_Sha512_Len, .-Transform_Sha512_Len
#endif
#else
/* Power ELF V1 ABI: emit a function descriptor in .opd that
* points at the code (.L.Transform_Sha512_Len) and the TOC base. */
.section .opd,"aw",@progbits
.globl Transform_Sha512_Len
.type Transform_Sha512_Len,@function
.align 3
Transform_Sha512_Len:
.quad .L.Transform_Sha512_Len,.TOC.@tocbase,0
.size Transform_Sha512_Len,24
.text
.align 16
.L.Transform_Sha512_Len:
#endif /* _CALL_ELF == 2 */
#else
.section __TEXT,__text
.globl _Transform_Sha512_Len
@@ -138,6 +151,40 @@ _Transform_Sha512_Len:
# Start of loop processing a block
L_SHA512_transform_len_begin:
# Load W - 128 bytes
#ifdef __LITTLE_ENDIAN__
li 0, 0
ldbrx 16, 4, 0
li 0, 8
ldbrx 17, 4, 0
li 0, 16
ldbrx 18, 4, 0
li 0, 24
ldbrx 19, 4, 0
li 0, 32
ldbrx 20, 4, 0
li 0, 40
ldbrx 21, 4, 0
li 0, 48
ldbrx 22, 4, 0
li 0, 56
ldbrx 23, 4, 0
li 0, 0x40
ldbrx 24, 4, 0
li 0, 0x48
ldbrx 25, 4, 0
li 0, 0x50
ldbrx 26, 4, 0
li 0, 0x58
ldbrx 27, 4, 0
li 0, 0x60
ldbrx 28, 4, 0
li 0, 0x68
ldbrx 29, 4, 0
li 0, 0x70
ldbrx 30, 4, 0
li 0, 0x78
ldbrx 31, 4, 0
#else
ld 16, 0(4)
ld 17, 8(4)
ld 18, 16(4)
@@ -154,6 +201,7 @@ L_SHA512_transform_len_begin:
ld 29, 104(4)
ld 30, 112(4)
ld 31, 120(4)
#endif /* __LITTLE_ENDIAN__ */
# Start of 80 rounds
# Round 0
rotrdi 0, 11, 14
@@ -3169,6 +3217,40 @@ L_SHA512_transform_len_begin:
# Start of loop processing a block
L_SHA512_transform_len_begin:
# Load W - 128 bytes
#ifdef __LITTLE_ENDIAN__
li 0, 0
ldbrx 16, 4, 0
li 0, 8
ldbrx 17, 4, 0
li 0, 16
ldbrx 18, 4, 0
li 0, 24
ldbrx 19, 4, 0
li 0, 32
ldbrx 20, 4, 0
li 0, 40
ldbrx 21, 4, 0
li 0, 48
ldbrx 22, 4, 0
li 0, 56
ldbrx 23, 4, 0
li 0, 0x40
ldbrx 24, 4, 0
li 0, 0x48
ldbrx 25, 4, 0
li 0, 0x50
ldbrx 26, 4, 0
li 0, 0x58
ldbrx 27, 4, 0
li 0, 0x60
ldbrx 28, 4, 0
li 0, 0x68
ldbrx 29, 4, 0
li 0, 0x70
ldbrx 30, 4, 0
li 0, 0x78
ldbrx 31, 4, 0
#else
ld 16, 0(4)
ld 17, 8(4)
ld 18, 16(4)
@@ -3185,6 +3267,7 @@ L_SHA512_transform_len_begin:
ld 29, 104(4)
ld 30, 112(4)
ld 31, 120(4)
#endif /* __LITTLE_ENDIAN__ */
li 0, 5
mtctr 0
# Start of 80 rounds
@@ -3952,7 +4035,11 @@ L_SHA512_transform_len_after_blk_15:
addi 1, 1, 0x98
blr
#ifndef __APPLE__
#if defined(_CALL_ELF) && _CALL_ELF == 2
.size Transform_Sha512_Len,.-Transform_Sha512_Len
#else
.size .L.Transform_Sha512_Len,.-.L.Transform_Sha512_Len
#endif /* _CALL_ELF == 2 */
#endif /* __APPLE__ */
#ifdef WOLFSSL_PPC64_ASM_CRYPTO
#ifndef __APPLE__
@@ -4013,16 +4100,29 @@ L_SHA512_crypto_k:
.machine "push"
.machine "power8"
#ifndef __APPLE__
#if defined(_CALL_ELF) && _CALL_ELF == 2
.text
.globl Transform_Sha512_Len_crypto
.type Transform_Sha512_Len_crypto,@function
.align 16
Transform_Sha512_Len_crypto:
#if defined(_CALL_ELF) && _CALL_ELF == 2
0: addis 2, 12, .TOC.-0b@ha
addi 2, 2, .TOC.-0b@l
.localentry Transform_Sha512_Len_crypto, .-Transform_Sha512_Len_crypto
#endif
#else
/* Power ELF V1 ABI: emit a function descriptor in .opd that
* points at the code (.L.Transform_Sha512_Len_crypto) and the TOC base. */
.section .opd,"aw",@progbits
.globl Transform_Sha512_Len_crypto
.type Transform_Sha512_Len_crypto,@function
.align 3
Transform_Sha512_Len_crypto:
.quad .L.Transform_Sha512_Len_crypto,.TOC.@tocbase,0
.size Transform_Sha512_Len_crypto,24
.text
.align 16
.L.Transform_Sha512_Len_crypto:
#endif /* _CALL_ELF == 2 */
#else
.section __TEXT,__text
.globl _Transform_Sha512_Len_crypto
@@ -5832,7 +5932,11 @@ L_SHA512_crypto_begin:
addi 1, 1, 0xc0
blr
#ifndef __APPLE__
#if defined(_CALL_ELF) && _CALL_ELF == 2
.size Transform_Sha512_Len_crypto,.-Transform_Sha512_Len_crypto
#else
.size .L.Transform_Sha512_Len_crypto,.-.L.Transform_Sha512_Len_crypto
#endif /* _CALL_ELF == 2 */
#endif /* __APPLE__ */
.machine "pop"
#endif /* WOLFSSL_PPC64_ASM_CRYPTO */
wolfcrypt/src/port/ppc64/ppc64-sha512-asm_c.c
+70
View File
@@ -128,6 +128,40 @@ void Transform_Sha512_Len(wc_Sha512* sha512, const byte* data, word32 len)
"\n"
"L_SHA512_transform_len_begin_%=: \n\t"
/* Load W - 128 bytes */
#ifdef __LITTLE_ENDIAN__
"li 0, 0\n\t"
"ldbrx 16, %[data], 0\n\t"
"li 0, 8\n\t"
"ldbrx 17, %[data], 0\n\t"
"li 0, 16\n\t"
"ldbrx 18, %[data], 0\n\t"
"li 0, 24\n\t"
"ldbrx 19, %[data], 0\n\t"
"li 0, 32\n\t"
"ldbrx 20, %[data], 0\n\t"
"li 0, 40\n\t"
"ldbrx 21, %[data], 0\n\t"
"li 0, 48\n\t"
"ldbrx 22, %[data], 0\n\t"
"li 0, 56\n\t"
"ldbrx 23, %[data], 0\n\t"
"li 0, 0x40\n\t"
"ldbrx 24, %[data], 0\n\t"
"li 0, 0x48\n\t"
"ldbrx 25, %[data], 0\n\t"
"li 0, 0x50\n\t"
"ldbrx 26, %[data], 0\n\t"
"li 0, 0x58\n\t"
"ldbrx 27, %[data], 0\n\t"
"li 0, 0x60\n\t"
"ldbrx 28, %[data], 0\n\t"
"li 0, 0x68\n\t"
"ldbrx 29, %[data], 0\n\t"
"li 0, 0x70\n\t"
"ldbrx 30, %[data], 0\n\t"
"li 0, 0x78\n\t"
"ldbrx 31, %[data], 0\n\t"
#else
"ld 16, 0(%[data])\n\t"
"ld 17, 8(%[data])\n\t"
"ld 18, 16(%[data])\n\t"
@@ -144,6 +178,7 @@ void Transform_Sha512_Len(wc_Sha512* sha512, const byte* data, word32 len)
"ld 29, 104(%[data])\n\t"
"ld 30, 112(%[data])\n\t"
"ld 31, 120(%[data])\n\t"
#endif /* __LITTLE_ENDIAN__ */
/* Start of 80 rounds */
/* Round 0 */
"rotrdi 0, 11, 14\n\t"
@@ -3160,6 +3195,40 @@ void Transform_Sha512_Len(wc_Sha512* sha512, const byte* data, word32 len)
"\n"
"L_SHA512_transform_len_begin_%=: \n\t"
/* Load W - 128 bytes */
#ifdef __LITTLE_ENDIAN__
"li 0, 0\n\t"
"ldbrx 16, %[data], 0\n\t"
"li 0, 8\n\t"
"ldbrx 17, %[data], 0\n\t"
"li 0, 16\n\t"
"ldbrx 18, %[data], 0\n\t"
"li 0, 24\n\t"
"ldbrx 19, %[data], 0\n\t"
"li 0, 32\n\t"
"ldbrx 20, %[data], 0\n\t"
"li 0, 40\n\t"
"ldbrx 21, %[data], 0\n\t"
"li 0, 48\n\t"
"ldbrx 22, %[data], 0\n\t"
"li 0, 56\n\t"
"ldbrx 23, %[data], 0\n\t"
"li 0, 0x40\n\t"
"ldbrx 24, %[data], 0\n\t"
"li 0, 0x48\n\t"
"ldbrx 25, %[data], 0\n\t"
"li 0, 0x50\n\t"
"ldbrx 26, %[data], 0\n\t"
"li 0, 0x58\n\t"
"ldbrx 27, %[data], 0\n\t"
"li 0, 0x60\n\t"
"ldbrx 28, %[data], 0\n\t"
"li 0, 0x68\n\t"
"ldbrx 29, %[data], 0\n\t"
"li 0, 0x70\n\t"
"ldbrx 30, %[data], 0\n\t"
"li 0, 0x78\n\t"
"ldbrx 31, %[data], 0\n\t"
#else
"ld 16, 0(%[data])\n\t"
"ld 17, 8(%[data])\n\t"
"ld 18, 16(%[data])\n\t"
@@ -3176,6 +3245,7 @@ void Transform_Sha512_Len(wc_Sha512* sha512, const byte* data, word32 len)
"ld 29, 104(%[data])\n\t"
"ld 30, 112(%[data])\n\t"
"ld 31, 120(%[data])\n\t"
#endif /* __LITTLE_ENDIAN__ */
"li 0, 5\n\t"
"mtctr 0\n\t"
/* Start of 80 rounds */
wolfcrypt/src/sha256.c
+24 -12
View File
@@ -438,7 +438,7 @@ static int InitSha256(wc_Sha256* sha256)
} /* extern "C" */
#endif
static cpuid_flags_t intel_flags = WC_CPUID_INITIALIZER;
static cpuid_flags_atomic_t intel_flags = WC_CPUID_ATOMIC_INITIALIZER;
#if defined(WC_C_DYNAMIC_FALLBACK) && !defined(WC_NO_INTERNAL_FUNCTION_POINTERS)
#define WC_NO_INTERNAL_FUNCTION_POINTERS
@@ -475,7 +475,7 @@ static int InitSha256(wc_Sha256* sha256)
}
#endif
cpuid_get_flags_ex(&intel_flags);
cpuid_get_flags_atomic(&intel_flags);
if (IS_INTEL_SHA(intel_flags)) {
#ifdef HAVE_INTEL_AVX1
@@ -652,7 +652,7 @@ static int InitSha256(wc_Sha256* sha256)
if (transform_check)
return;
cpuid_get_flags_ex(&intel_flags);
cpuid_get_flags_atomic(&intel_flags);
if (IS_INTEL_SHA(intel_flags)) {
#ifdef HAVE_INTEL_AVX1
@@ -1149,21 +1149,23 @@ extern void Transform_Sha256_Len_crypto(wc_Sha256* sha256, const byte* data,
word32 len);
/* -1 = not yet determined, 0 = base, 1 = vector-crypto */
static int sha256_use_crypto = -1;
/* Resolved dispatch decision (0 = base, 1 = vector-crypto), accessed with the
* wolfSSL atomic APIs so the one-time detection is free of data races. The
* write is idempotent (all callers compute the same value from the atomic
* master flags), so a benign concurrent double-write is harmless. */
static wolfSSL_Atomic_Uint sha256_use_crypto = WOLFSSL_ATOMIC_INITIALIZER(0);
/* Detect CPU support via the central cpuid module. Idempotent - safe to call
* from multiple threads as all callers compute the same value. */
/* Detect CPU support via the central cpuid module. */
static void Sha256_SetTransform(void)
{
if (sha256_use_crypto < 0) {
sha256_use_crypto = IS_PPC64_VEC_CRYPTO(cpuid_get_flags()) != 0;
}
WOLFSSL_ATOMIC_STORE(sha256_use_crypto,
(unsigned int)(IS_PPC64_VEC_CRYPTO(cpuid_get_flags()) != 0));
}
static WC_INLINE void SHA256_TRANSFORM_LEN(wc_Sha256* sha256, const byte* data,
word32 len)
{
if (sha256_use_crypto)
if (WOLFSSL_ATOMIC_LOAD(sha256_use_crypto))
Transform_Sha256_Len_crypto(sha256, data, len);
else
Transform_Sha256_Len(sha256, data, len);
@@ -1213,7 +1215,7 @@ static int Transform_Sha256(wc_Sha256* sha256, const byte* data)
!defined(WOLF_CRYPTO_CB_ONLY_SHA256)
static int transform_check = 0;
static cpuid_flags_t cpuid_flags = WC_CPUID_INITIALIZER;
static cpuid_flags_atomic_t cpuid_flags = WC_CPUID_ATOMIC_INITIALIZER;
static int Transform_Sha256(wc_Sha256* sha256, const byte* data);
static int Transform_Sha256_Len(wc_Sha256* sha256, const byte* data,
@@ -1281,7 +1283,7 @@ static void Sha256_SetTransform(void)
if (transform_check)
return;
cpuid_get_flags_ex(&cpuid_flags);
cpuid_get_flags_atomic(&cpuid_flags);
#if !defined(WOLFSSL_ARMASM_NO_NEON)
#if !defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
@@ -1990,6 +1992,16 @@ static WC_INLINE int Transform_Sha256_Len(wc_Sha256* sha256, const byte* data,
&sha256->buffer[WC_SHA256_PAD_SIZE / sizeof(word32)],
2 * sizeof(word32));
#endif
#if defined(WOLFSSL_PPC64_ASM) && defined(LITTLE_ENDIAN_ORDER)
/* The PPC64 assembly loads the message with byte-reversed loads on
* little-endian, treating the whole block as a big-endian byte stream.
* The 64-bit length above is stored in native (little-endian) word
* order, so reverse it here to keep the block a consistent big-endian
* stream. */
ByteReverseWords( &sha256->buffer[WC_SHA256_PAD_SIZE / sizeof(word32)],
&sha256->buffer[WC_SHA256_PAD_SIZE / sizeof(word32)],
2 * sizeof(word32));
#endif
#if defined(WOLFSSL_USE_ESP32_CRYPT_HASH_HW) && \
!defined(NO_WOLFSSL_ESP32_CRYPT_HASH_SHA256)
wolfcrypt/src/sha512.c
+34 -15
View File
@@ -1182,7 +1182,7 @@ static int InitSha512_256(wc_Sha512* sha512)
} /* extern "C" */
#endif
static cpuid_flags_t intel_flags = WC_CPUID_INITIALIZER;
static cpuid_flags_atomic_t intel_flags = WC_CPUID_ATOMIC_INITIALIZER;
#if defined(WC_C_DYNAMIC_FALLBACK) && !defined(WC_NO_INTERNAL_FUNCTION_POINTERS)
#define WC_NO_INTERNAL_FUNCTION_POINTERS
@@ -1220,7 +1220,7 @@ static int InitSha512_256(wc_Sha512* sha512)
}
#endif
cpuid_get_flags_ex(&intel_flags);
cpuid_get_flags_atomic(&intel_flags);
#if defined(HAVE_INTEL_AVX2)
if (IS_INTEL_AVX2(intel_flags)) {
@@ -1363,7 +1363,7 @@ static int InitSha512_256(wc_Sha512* sha512)
if (transform_check)
return;
cpuid_get_flags_ex(&intel_flags);
cpuid_get_flags_atomic(&intel_flags);
#if defined(HAVE_INTEL_AVX2)
if (IS_INTEL_AVX2(intel_flags)) {
@@ -1432,7 +1432,7 @@ static int InitSha512_256(wc_Sha512* sha512)
#define NEED_SOFT_SHA512
static int transform_check = 0;
static cpuid_flags_t cpuid_flags = WC_CPUID_INITIALIZER;
static cpuid_flags_atomic_t cpuid_flags = WC_CPUID_ATOMIC_INITIALIZER;
static int _Transform_Sha512(wc_Sha512* sha512);
static int Transform_Sha512_C(wc_Sha512* sha512, const byte* data);
@@ -1517,7 +1517,7 @@ static void Sha512_SetTransform(void)
if (transform_check)
return;
cpuid_get_flags_ex(&cpuid_flags);
cpuid_get_flags_atomic(&cpuid_flags);
#ifdef WOLFSSL_ARMASM_CRYPTO_SHA512
if (IS_AARCH64_SHA512(cpuid_flags)) {
@@ -1614,18 +1614,26 @@ extern void Transform_Sha512_Len(wc_Sha512* sha512, const byte* data,
extern void Transform_Sha512_Len_crypto(wc_Sha512* sha512, const byte* data,
word32 len);
/* -1 = not yet determined, 0 = base, 1 = vector-crypto */
static int sha512_use_crypto = -1;
/* Resolved dispatch decision, accessed with the wolfSSL atomic APIs so the
* lazy one-time detection is free of data races. WC_CPUID_INITIALIZER means
* "not yet determined"; the write is idempotent (all callers compute the same
* value from the atomic master flags), so a benign concurrent double-write is
* harmless. */
static wolfSSL_Atomic_Uint sha512_use_crypto =
WOLFSSL_ATOMIC_INITIALIZER(WC_CPUID_INITIALIZER);
/* Detect CPU support via the central cpuid module on first use. Idempotent -
* safe to call from multiple threads as all callers compute the same value. */
/* Detect CPU support via the central cpuid module on first use. */
static WC_INLINE void SHA512_TRANSFORM_LEN(wc_Sha512* sha512, const byte* data,
word32 len)
{
if (sha512_use_crypto < 0)
sha512_use_crypto = IS_PPC64_VEC_CRYPTO(cpuid_get_flags()) != 0;
unsigned int use_crypto = WOLFSSL_ATOMIC_LOAD(sha512_use_crypto);
if (sha512_use_crypto)
if (use_crypto == WC_CPUID_INITIALIZER) {
use_crypto = (unsigned int)(IS_PPC64_VEC_CRYPTO(cpuid_get_flags()) != 0);
WOLFSSL_ATOMIC_STORE(sha512_use_crypto, use_crypto);
}
if (use_crypto)
Transform_Sha512_Len_crypto(sha512, data, len);
else
Transform_Sha512_Len(sha512, data, len);
@@ -1953,7 +1961,7 @@ static WC_INLINE int Sha512Update(wc_Sha512* sha512, const byte* data, word32 le
#if (!defined(WOLFSSL_ESP32_CRYPT) || \
defined(NO_WOLFSSL_ESP32_CRYPT_HASH) || \
defined(NO_WOLFSSL_ESP32_CRYPT_HASH_SHA512)) && \
!defined(WOLFSSL_ARMASM)
!defined(WOLFSSL_ARMASM) && !defined(WOLFSSL_PPC64_ASM)
ByteReverseWords64(sha512->buffer, sha512->buffer,
WC_SHA512_BLOCK_SIZE);
#endif
@@ -2195,7 +2203,7 @@ static WC_INLINE int Sha512Final(wc_Sha512* sha512)
#if (!defined(WOLFSSL_ESP32_CRYPT) || \
defined(NO_WOLFSSL_ESP32_CRYPT_HASH) || \
defined(NO_WOLFSSL_ESP32_CRYPT_HASH_SHA512)) && \
!defined(WOLFSSL_ARMASM)
!defined(WOLFSSL_ARMASM) && !defined(WOLFSSL_PPC64_ASM)
ByteReverseWords64(sha512->buffer,sha512->buffer,
WC_SHA512_BLOCK_SIZE);
#endif
@@ -2252,7 +2260,7 @@ static WC_INLINE int Sha512Final(wc_Sha512* sha512)
#if (!defined(WOLFSSL_ESP32_CRYPT) || \
defined(NO_WOLFSSL_ESP32_CRYPT_HASH) || \
defined(NO_WOLFSSL_ESP32_CRYPT_HASH_SHA512)) && \
!defined(WOLFSSL_ARMASM)
!defined(WOLFSSL_ARMASM) && !defined(WOLFSSL_PPC64_ASM)
ByteReverseWords64(sha512->buffer, sha512->buffer, WC_SHA512_PAD_SIZE);
#endif
#endif
@@ -2284,6 +2292,17 @@ static WC_INLINE int Sha512Final(wc_Sha512* sha512)
&(sha512->buffer[SHA512_PAD_LEN_64]),
WC_SHA512_BLOCK_SIZE - WC_SHA512_PAD_SIZE);
}
#elif defined(WOLFSSL_PPC64_ASM) && defined(LITTLE_ENDIAN_ORDER)
/* The PPC64 assembly loads the message with byte-reversed loads on
* little-endian, treating the whole block as a big-endian byte stream. The
* 128-bit length just stored is in native (little-endian) word order, so
* reverse it here to keep the block a consistent big-endian stream. */
{
ByteReverseWords64(
&(sha512->buffer[WC_SHA512_PAD_SIZE / sizeof(word64)]),
&(sha512->buffer[WC_SHA512_PAD_SIZE / sizeof(word64)]),
WC_SHA512_BLOCK_SIZE - WC_SHA512_PAD_SIZE);
}
#endif
#if defined(WOLFSSL_ARMASM) || defined(WOLFSSL_PPC64_ASM) || \
wolfssl/wolfcrypt/aes.h
+30
View File
@@ -1183,6 +1183,36 @@ WOLFSSL_LOCAL void AES_XTS_encrypt(const byte* in, byte* out, word32 sz,
WOLFSSL_LOCAL void AES_XTS_decrypt(const byte* in, byte* out, word32 sz,
const byte* i, byte* key, byte* key2, byte* tmp, int nr);
#endif
#if defined(WOLFSSL_PPC64_ASM_CRYPTO)
/* POWER8+ vector-crypto (vcipher family) variants, selected at run time when
* the CPU reports VEC_CRYPTO support (see aes.c). */
WOLFSSL_LOCAL void AES_set_encrypt_key_crypto(const unsigned char* key,
word32 len, unsigned char* ks);
WOLFSSL_LOCAL void AES_invert_key_crypto(unsigned char* ks, word32 rounds);
WOLFSSL_LOCAL void AES_ECB_encrypt_crypto(const unsigned char* in,
unsigned char* out, unsigned long len, const unsigned char* ks, int nr);
WOLFSSL_LOCAL void AES_ECB_decrypt_crypto(const unsigned char* in,
unsigned char* out, unsigned long len, const unsigned char* ks, int nr);
WOLFSSL_LOCAL void AES_CBC_encrypt_crypto(const unsigned char* in,
unsigned char* out, unsigned long len, const unsigned char* ks, int nr,
unsigned char* iv);
WOLFSSL_LOCAL void AES_CBC_decrypt_crypto(const unsigned char* in,
unsigned char* out, unsigned long len, const unsigned char* ks, int nr,
unsigned char* iv);
WOLFSSL_LOCAL void AES_CTR_encrypt_crypto(const unsigned char* in,
unsigned char* out, unsigned long len, const unsigned char* ks, int nr,
unsigned char* ctr);
WOLFSSL_LOCAL void AES_GCM_encrypt_crypto(const unsigned char* in,
unsigned char* out, unsigned long len, const unsigned char* ks, int nr,
unsigned char* ctr);
#if defined(WOLFSSL_AES_XTS)
WOLFSSL_LOCAL void AES_XTS_encrypt_crypto(const byte* in, byte* out, word32 sz,
const byte* i, byte* key, byte* key2, byte* tmp, int nr);
WOLFSSL_LOCAL void AES_XTS_decrypt_crypto(const byte* in, byte* out, word32 sz,
const byte* i, byte* key, byte* key2, byte* tmp, int nr);
#endif
#endif /* WOLFSSL_PPC64_ASM_CRYPTO */
#endif /* WOLFSSL_PPC64_ASM */
#if defined(WOLFSSL_PPC32_ASM)