feedc0de/wolfssl
1
0
Fork 0
forked from wolfSSL/wolfssl
Code Pull Requests Activity

Revert "Aarch64: make code compile when no hardware crypto avail"

Browse Source
This commit is contained in:
David Garske
2024-12-13 13:52:53 -08:00
committed by GitHub
parent d7e40e7413
commit 71325a2a32
9 changed files with 725 additions and 1564 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.wolfssl_known_macro_extras
View File

@ -546,7 +546,6 @@ WOLFCRYPT_FIPS_CORE_DYNAMIC_HASH_VALUE
WOLFSENTRY_H
WOLFSENTRY_NO_JSON
WOLFSSL_32BIT_MILLI_TIME
WOLFSSL_AARCH64_PRIVILEGE_MODE
WOLFSSL_AESNI_BY4
WOLFSSL_AESNI_BY6
WOLFSSL_AFTER_DATE_CLOCK_SKEW
@ -905,7 +904,6 @@ __MINGW32__
__MINGW64_VERSION_MAJOR
__MINGW64__
__MWERKS__
__OpenBSD__
__PIE__
__POWERPC__
__PPC__

11
configure.ac
View File

@ -2974,7 +2974,6 @@ then
fi
ENABLED_ARMASM_CRYPTO="unknown"
ENABLED_ARMASM_INLINE="no"
ENABLED_ARMASM_SHA3="no"
ENABLED_ARMASM_CRYPTO_SM4="no"
@ -2996,9 +2995,6 @@ then
inline)
ENABLED_ARMASM_INLINE=yes
;;
no-crypto)
ENABLED_ARMASM_CRYPTO=no
;;
sha512-crypto | sha3-crypto)
case $host_cpu in
*aarch64*)
@ -3074,9 +3070,7 @@ then
esac
# Include options.h
AM_CCASFLAGS="$AM_CCASFLAGS -DEXTERNAL_OPTS_OPENVPN"
if test "$ENABLED_ARMASM_CRYPTO" = "unknown"; then
ENABLED_ARMASM_CRYPTO=yes
fi
ENABLED_ARMASM_CRYPTO=yes
ENABLED_ARMASM_NEON=yes
ENABLED_ARM_64=yes
@ -3177,9 +3171,6 @@ fi
if test "$ENABLED_ARMASM_SM4" = "yes"; then
AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_ARMASM_CRYPTO_SM4"
fi
if test "$ENABLED_ARMASM_CRYPTO" = "unknown"; then
ENABLED_ARMASM_CRYPTO=no
fi
if test "$ENABLED_ARMASM_CRYPTO" = "no"; then
AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_ARMASM_NO_HW_CRYPTO"
fi

46
wolfcrypt/benchmark/benchmark.c
View File

@ -229,8 +229,6 @@
#include <wolfssl/wolfcrypt/async.h>
#endif
#include <wolfssl/wolfcrypt/cpuid.h>
#ifdef USE_FLAT_BENCHMARK_H
#include "benchmark.h"
#else
@ -3941,46 +3939,6 @@ exit:
return NULL;
}
#if defined(HAVE_CPUID) && defined(WOLFSSL_TEST_STATIC_BUILD)
static void print_cpu_features(void)
{
word32 cpuid_flags = cpuid_get_flags();
printf("CPU: ");
#ifdef HAVE_CPUID_INTEL
printf("Intel");
#ifdef WOLFSSL_X86_64_BUILD
printf(" x86_64");
#else
printf(" x86");
#endif
printf(" -");
if (IS_INTEL_AVX1(cpuid_flags)) printf(" avx1");
if (IS_INTEL_AVX2(cpuid_flags)) printf(" avx2");
if (IS_INTEL_RDRAND(cpuid_flags)) printf(" rdrand");
if (IS_INTEL_RDSEED(cpuid_flags)) printf(" rdseed");
if (IS_INTEL_BMI2(cpuid_flags)) printf(" bmi2");
if (IS_INTEL_AESNI(cpuid_flags)) printf(" aesni");
if (IS_INTEL_ADX(cpuid_flags)) printf(" adx");
if (IS_INTEL_MOVBE(cpuid_flags)) printf(" movbe");
if (IS_INTEL_BMI1(cpuid_flags)) printf(" bmi1");
if (IS_INTEL_SHA(cpuid_flags)) printf(" sha");
#endif
#ifdef __aarch64__
printf("Aarch64 -");
if (IS_AARCH64_AES(cpuid_flags)) printf(" aes");
if (IS_AARCH64_PMULL(cpuid_flags)) printf(" pmull");
if (IS_AARCH64_SHA256(cpuid_flags)) printf(" sha256");
if (IS_AARCH64_SHA512(cpuid_flags)) printf(" sha512");
if (IS_AARCH64_RDM(cpuid_flags)) printf(" rdm");
if (IS_AARCH64_SHA3(cpuid_flags)) printf(" sha3");
if (IS_AARCH64_SM3(cpuid_flags)) printf(" sm3");
if (IS_AARCH64_SM4(cpuid_flags)) printf(" sm4");
#endif
printf("\n");
}
#endif
int benchmark_init(void)
{
int ret = 0;
@ -4001,10 +3959,6 @@ int benchmark_init(void)
return EXIT_FAILURE;
}
#if defined(HAVE_CPUID) && defined(WOLFSSL_TEST_STATIC_BUILD)
print_cpu_features();
#endif
#ifdef HAVE_WC_INTROSPECTION
printf("Math: %s\n", wc_GetMathInfo());
#endif

193
wolfcrypt/src/aes.c
View File

@ -106,7 +106,7 @@ block cipher mechanism that uses n-bit binary string parameter key with 128-bits
#include <wolfcrypt/src/misc.c>
#endif
#if !defined(WOLFSSL_RISCV_ASM)
#if !defined(WOLFSSL_ARMASM) && !defined(WOLFSSL_RISCV_ASM)
#ifdef WOLFSSL_IMX6_CAAM_BLOB
/* case of possibly not using hardware acceleration for AES but using key
@ -787,26 +787,6 @@ block cipher mechanism that uses n-bit binary string parameter key with 128-bits
}
#endif /* HAVE_AES_DECRYPT */
#elif defined(__aarch64__) && defined(WOLFSSL_ARMASM) && \
!defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
#define NEED_AES_TABLES
static int checkedCpuIdFlags = 0;
static word32 cpuid_flags = 0;
static void Check_CPU_support_HwCrypto(Aes* aes)
{
if (checkedCpuIdFlags == 0) {
cpuid_flags = cpuid_get_flags();
checkedCpuIdFlags = 1;
}
aes->use_aes_hw_crypto = IS_AARCH64_AES(cpuid_flags);
#ifdef HAVE_AESGCM
aes->use_pmull_hw_crypto = IS_AARCH64_PMULL(cpuid_flags);
#endif
}
#elif (defined(WOLFSSL_IMX6_CAAM) && !defined(NO_IMX6_CAAM_AES) \
&& !defined(WOLFSSL_QNX_CAAM)) || \
((defined(WOLFSSL_AFALG) || defined(WOLFSSL_DEVCRYPTO_AES)) && \
@ -2895,13 +2875,6 @@ static WARN_UNUSED_RESULT int wc_AesEncrypt(
printf("Skipping AES-NI\n");
#endif
}
#elif defined(__aarch64__) && defined(WOLFSSL_ARMASM) && \
!defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
if (aes->use_aes_hw_crypto) {
AES_encrypt_AARCH64(inBlock, outBlock, (byte*)aes->key,
(int)aes->rounds);
return 0;
}
#endif /* WOLFSSL_AESNI */
#if defined(WOLFSSL_SCE) && !defined(WOLFSSL_SCE_NO_AES)
AES_ECB_encrypt(aes, inBlock, outBlock, WC_AES_BLOCK_SIZE);
@ -3657,13 +3630,6 @@ static WARN_UNUSED_RESULT int wc_AesDecrypt(
printf("Skipping AES-NI\n");
#endif
}
#elif defined(__aarch64__) && defined(WOLFSSL_ARMASM) && \
!defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
if (aes->use_aes_hw_crypto) {
AES_decrypt_AARCH64(inBlock, outBlock, (byte*)aes->key,
(int)aes->rounds);
return 0;
}
#endif /* WOLFSSL_AESNI */
#if defined(WOLFSSL_SCE) && !defined(WOLFSSL_SCE_NO_AES)
return AES_ECB_decrypt(aes, inBlock, outBlock, WC_AES_BLOCK_SIZE);
@ -4614,14 +4580,6 @@ static void AesSetKey_C(Aes* aes, const byte* key, word32 keySz, int dir)
}
#endif /* WOLFSSL_AESNI */
#if defined(__aarch64__) && defined(WOLFSSL_ARMASM) && \
!defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
Check_CPU_support_HwCrypto(aes);
if (aes->use_aes_hw_crypto) {
return AES_set_key_AARCH64(userKey, keylen, aes, dir);
}
#endif
#ifdef WOLFSSL_KCAPI_AES
XMEMCPY(aes->devKey, userKey, keylen);
if (aes->init != 0) {
@ -5817,14 +5775,6 @@ int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
}
}
else
#elif defined(__aarch64__) && defined(WOLFSSL_ARMASM) && \
!defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
if (aes->use_aes_hw_crypto) {
AES_CBC_encrypt_AARCH64(in, out, sz, (byte*)aes->reg,
(byte*)aes->key, (int)aes->rounds);
ret = 0;
}
else
#endif
{
ret = 0;
@ -5965,14 +5915,6 @@ int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
ret = 0;
}
else
#elif defined(__aarch64__) && defined(WOLFSSL_ARMASM) && \
!defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
if (aes->use_aes_hw_crypto) {
AES_CBC_decrypt_AARCH64(in, out, sz, (byte*)aes->reg,
(byte*)aes->key, (int)aes->rounds);
ret = 0;
}
else
#endif
{
ret = 0;
@ -6311,14 +6253,6 @@ int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
aes->left -= processed;
sz -= processed;
#if defined(__aarch64__) && defined(WOLFSSL_ARMASM) && \
!defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
if (aes->use_aes_hw_crypto) {
AES_CTR_encrypt_AARCH64(aes, out, in, sz);
return 0;
}
#endif
VECTOR_REGISTERS_PUSH;
#if defined(HAVE_AES_ECB) && !defined(WOLFSSL_PIC32MZ_CRYPT) && \
@ -6407,7 +6341,7 @@ int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
#endif /* NEED_AES_CTR_SOFT */
#endif /* WOLFSSL_AES_COUNTER */
#endif /* !WOLFSSL_RISCV_ASM */
#endif /* !WOLFSSL_ARMASM && ! WOLFSSL_RISCV_ASM */
/*
@ -6454,7 +6388,10 @@ static WC_INLINE void IncCtr(byte* ctr, word32 ctrSz)
#endif
#if defined(WOLFSSL_RISCV_ASM)
#ifdef WOLFSSL_ARMASM
/* implementation is located in wolfcrypt/src/port/arm/armv8-aes.c */
#elif defined(WOLFSSL_RISCV_ASM)
/* implemented in wolfcrypt/src/port/risc-v/riscv-64-aes.c */
#elif defined(WOLFSSL_AFALG)
@ -6664,13 +6601,6 @@ int wc_AesGcmSetKey(Aes* aes, const byte* key, word32 len)
return ret;
#endif /* WOLFSSL_RENESAS_RSIP && WOLFSSL_RENESAS_FSPSM_CRYPTONLY*/
#if defined(__aarch64__) && defined(WOLFSSL_ARMASM) && \
!defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
if (ret == 0 && aes->use_aes_hw_crypto && aes->use_pmull_hw_crypto) {
AES_GCM_set_key_AARCH64(aes, iv);
}
else
#endif
#if !defined(FREESCALE_LTC_AES_GCM)
if (ret == 0) {
VECTOR_REGISTERS_PUSH;
@ -7388,8 +7318,6 @@ void GHASH(Gcm* gcm, const byte* a, word32 aSz, const byte* c,
*/
#define GHASH_INIT_EXTRA(aes) WC_DO_NOTHING
#if !defined(__aarch64__) || !defined(WOLFSSL_ARMASM) || \
defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
/* GHASH one block of data..
*
* XOR block into tag and GMULT with H using pre-computed table.
@ -7403,7 +7331,6 @@ void GHASH(Gcm* gcm, const byte* a, word32 aSz, const byte* c,
GMULT(AES_TAG(aes), (aes)->gcm.M0); \
} \
while (0)
#endif
#endif /* WOLFSSL_AESGCM_STREAM */
#elif defined(WORD64_AVAILABLE) && !defined(GCM_WORD32)
@ -7999,17 +7926,8 @@ static void GHASH_INIT(Aes* aes) {
/* Reset counts of AAD and cipher text. */
aes->aOver = 0;
aes->cOver = 0;
#if defined(__aarch64__) && defined(WOLFSSL_ARMASM) && \
!defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
if (aes->use_aes_hw_crypto && aes->use_pmull_hw_crypto) {
; /* Don't do extra initialization. */
}
else
#endif
{
/* Extra initialization based on implementation. */
GHASH_INIT_EXTRA(aes);
}
/* Extra initialization based on implementation. */
GHASH_INIT_EXTRA(aes);
}
/* Update the GHASH with AAD and/or cipher text.
@ -8670,14 +8588,6 @@ int wc_AesGcmEncrypt(Aes* aes, byte* out, const byte* in, word32 sz,
}
}
else
#elif defined(__aarch64__) && defined(WOLFSSL_ARMASM) && \
!defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
if (aes->use_aes_hw_crypto && aes->use_pmull_hw_crypto) {
AES_GCM_encrypt_AARCH64(aes, out, in, sz, iv, ivSz, authTag, authTagSz,
authIn, authInSz);
ret = 0;
}
else
#endif /* WOLFSSL_AESNI */
{
ret = AES_GCM_encrypt_C(aes, out, in, sz, iv, ivSz, authTag, authTagSz,
@ -9262,13 +9172,6 @@ int wc_AesGcmDecrypt(Aes* aes, byte* out, const byte* in, word32 sz,
}
}
else
#elif defined(__aarch64__) && defined(WOLFSSL_ARMASM) && \
!defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
if (aes->use_aes_hw_crypto && aes->use_pmull_hw_crypto) {
ret = AES_GCM_decrypt_AARCH64(aes, out, in, sz, iv, ivSz, authTag,
authTagSz, authIn, authInSz);
}
else
#endif /* WOLFSSL_AESNI */
{
ret = AES_GCM_decrypt_C(aes, out, in, sz, iv, ivSz, authTag, authTagSz,
@ -10183,20 +10086,7 @@ int wc_AesGcmInit(Aes* aes, const byte* key, word32 len, const byte* iv,
RESTORE_VECTOR_REGISTERS();
}
else
#elif defined(__aarch64__) && defined(WOLFSSL_ARMASM) && \
!defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
if (aes->use_aes_hw_crypto) {
AES_GCM_init_AARCH64(aes, iv, ivSz);
/* Reset state fields. */
aes->over = 0;
aes->aSz = 0;
aes->cSz = 0;
/* Initialization for GHASH. */
GHASH_INIT(aes);
}
else
#endif /* WOLFSSL_AESNI */
#endif
{
ret = AesGcmInit_C(aes, iv, ivSz);
}
@ -10322,13 +10212,6 @@ int wc_AesGcmEncryptUpdate(Aes* aes, byte* out, const byte* in, word32 sz,
RESTORE_VECTOR_REGISTERS();
}
else
#elif defined(__aarch64__) && defined(WOLFSSL_ARMASM) && \
!defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
if (aes->use_aes_hw_crypto) {
AES_GCM_crypt_update_AARCH64(aes, out, in, sz);
GHASH_UPDATE_AARCH64(aes, authIn, authInSz, out, sz);
}
else
#endif
{
/* Encrypt the plaintext. */
@ -10382,12 +10265,6 @@ int wc_AesGcmEncryptFinal(Aes* aes, byte* authTag, word32 authTagSz)
RESTORE_VECTOR_REGISTERS();
}
else
#elif defined(__aarch64__) && defined(WOLFSSL_ARMASM) && \
!defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
if (aes->use_aes_hw_crypto) {
AES_GCM_final_AARCH64(aes, authTag, authTagSz);
}
else
#endif
{
ret = AesGcmFinal_C(aes, authTag, authTagSz);
@ -10471,13 +10348,6 @@ int wc_AesGcmDecryptUpdate(Aes* aes, byte* out, const byte* in, word32 sz,
RESTORE_VECTOR_REGISTERS();
}
else
#elif defined(__aarch64__) && defined(WOLFSSL_ARMASM) && \
!defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
if (aes->use_aes_hw_crypto) {
GHASH_UPDATE_AARCH64(aes, authIn, authInSz, in, sz);
AES_GCM_crypt_update_AARCH64(aes, out, in, sz);
}
else
#endif
{
/* Update the authentication tag with any authentication data and
@ -10529,17 +10399,6 @@ int wc_AesGcmDecryptFinal(Aes* aes, const byte* authTag, word32 authTagSz)
RESTORE_VECTOR_REGISTERS();
}
else
#elif defined(__aarch64__) && defined(WOLFSSL_ARMASM) && \
!defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
if (aes->use_aes_hw_crypto) {
ALIGN32 byte calcTag[WC_AES_BLOCK_SIZE];
AES_GCM_final_AARCH64(aes, calcTag, authTagSz);
/* Check calculated tag matches the one passed in. */
if (ConstantCompare(authTag, calcTag, (int)authTagSz) != 0) {
ret = AES_GCM_AUTH_E;
}
}
else
#endif
{
ALIGN32 byte calcTag[WC_AES_BLOCK_SIZE];
@ -10816,7 +10675,10 @@ int wc_AesCcmCheckTagSize(int sz)
return 0;
}
#if defined(WOLFSSL_RISCV_ASM)
#ifdef WOLFSSL_ARMASM
/* implementation located in wolfcrypt/src/port/arm/armv8-aes.c */
#elif defined(WOLFSSL_RISCV_ASM)
/* implementation located in wolfcrypt/src/port/risc-v/riscv-64-aes.c */
#elif defined(HAVE_COLDFIRE_SEC)
@ -11822,12 +11684,6 @@ static WARN_UNUSED_RESULT int _AesEcbEncrypt(
AES_ECB_encrypt_AESNI(in, out, sz, (byte*)aes->key, (int)aes->rounds);
}
else
#elif defined(__aarch64__) && defined(WOLFSSL_ARMASM) && \
!defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
if (aes->use_aes_hw_crypto) {
AES_encrypt_AARCH64(in, out, (byte*)aes->key, (int)aes->rounds);
}
else
#endif
{
#ifdef NEED_AES_TABLES
@ -11880,12 +11736,6 @@ static WARN_UNUSED_RESULT int _AesEcbDecrypt(
AES_ECB_decrypt_AESNI(in, out, sz, (byte*)aes->key, (int)aes->rounds);
}
else
#elif defined(__aarch64__) && defined(WOLFSSL_ARMASM) && \
!defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
if (aes->use_aes_hw_crypto) {
AES_decrypt_AARCH64(in, out, (byte*)aes->key, (int)aes->rounds);
}
else
#endif
{
#ifdef NEED_AES_TABLES
@ -12988,6 +12838,7 @@ void AES_XTS_decrypt_update_avx1(const unsigned char *in, unsigned char *out, wo
#endif /* WOLFSSL_AESNI */
#if !defined(WOLFSSL_ARMASM) || defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
#ifdef HAVE_AES_ECB
/* helper function for encrypting / decrypting full buffer at once */
static WARN_UNUSED_RESULT int _AesXtsHelper(
@ -13249,13 +13100,6 @@ int wc_AesXtsEncrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz,
RESTORE_VECTOR_REGISTERS();
}
else
#elif defined(__aarch64__) && defined(WOLFSSL_ARMASM) && \
!defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
if (aes->use_aes_hw_crypto) {
AES_XTS_encrypt_AARCH64(xaes, out, in, sz, i);
ret = 0;
}
else
#endif
{
ret = AesXtsEncrypt_sw(xaes, out, in, sz, i);
@ -13689,13 +13533,6 @@ int wc_AesXtsDecrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz,
RESTORE_VECTOR_REGISTERS();
}
else
#elif defined(__aarch64__) && defined(WOLFSSL_ARMASM) && \
!defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
if (aes->use_aes_hw_crypto) {
AES_XTS_decrypt_AARCH64(xaes, out, in, sz, i);
ret = 0;
}
else
#endif
{
ret = AesXtsDecrypt_sw(xaes, out, in, sz, i);
@ -13893,6 +13730,8 @@ int wc_AesXtsDecryptFinal(XtsAes* xaes, byte* out, const byte* in, word32 sz,
#endif /* WOLFSSL_AESXTS_STREAM */
#endif /* !WOLFSSL_ARMASM || WOLFSSL_ARMASM_NO_HW_CRYPTO */
/* Same as wc_AesXtsEncryptSector but the sector gets incremented by one every
* sectorSz bytes
*

205
wolfcrypt/src/cpuid.c
View File

@ -28,8 +28,7 @@
#include <wolfssl/wolfcrypt/cpuid.h>
#if defined(HAVE_CPUID) || defined(HAVE_CPUID_INTEL) || \
defined(HAVE_CPUID_AARCH64)
#if defined(HAVE_CPUID) || defined(HAVE_CPUID_INTEL)
static word32 cpuid_check = 0;
static word32 cpuid_flags = 0;
#endif
@ -102,208 +101,6 @@
cpuid_check = 1;
}
}
#elif defined(HAVE_CPUID_AARCH64)
#define CPUID_AARCH64_FEAT_AES ((word64)1 << 4)
#define CPUID_AARCH64_FEAT_PMULL ((word64)1 << 5)
#define CPUID_AARCH64_FEAT_SHA256 ((word64)1 << 12)
#define CPUID_AARCH64_FEAT_SHA256_512 ((word64)1 << 13)
#define CPUID_AARCH64_FEAT_RDM ((word64)1 << 28)
#define CPUID_AARCH64_FEAT_SHA3 ((word64)1 << 32)
#define CPUID_AARCH64_FEAT_SM3 ((word64)1 << 36)
#define CPUID_AARCH64_FEAT_SM4 ((word64)1 << 40)
#ifdef WOLFSSL_AARCH64_PRIVILEGE_MODE
/* https://developer.arm.com/documentation/ddi0601/2024-09/AArch64-Registers
* /ID-AA64ISAR0-EL1--AArch64-Instruction-Set-Attribute-Register-0 */
void cpuid_set_flags(void)
{
if (!cpuid_check) {
word64 features;
__asm__ __volatile (
"mrs %[feat], ID_AA64ISAR0_EL1\n"
: [feat] "=r" (features)
:
:
);
if (features & CPUID_AARCH64_FEAT_AES)
cpuid_flags |= CPUID_AES;
if (features & CPUID_AARCH64_FEAT_PMULL)
cpuid_flags |= CPUID_PMULL;
if (features & CPUID_AARCH64_FEAT_SHA256)
cpuid_flags |= CPUID_SHA256;
if (features & CPUID_AARCH64_FEAT_SHA256_512)
cpuid_flags |= CPUID_SHA256 | CPUID_SHA512;
if (features & CPUID_AARCH64_FEAT_RDM)
cpuid_flags |= CPUID_RDM;
if (features & CPUID_AARCH64_FEAT_SHA3)
cpuid_flags |= CPUID_SHA3;
if (features & CPUID_AARCH64_FEAT_SM3)
cpuid_flags |= CPUID_SM3;
if (features & CPUID_AARCH64_FEAT_SM4)
cpuid_flags |= CPUID_SM4;
cpuid_check = 1;
}
}
#elif defined(__linux__)
/* https://community.arm.com/arm-community-blogs/b/operating-systems-blog/
* posts/runtime-detection-of-cpu-features-on-an-armv8-a-cpu */
#include <sys/auxv.h>
#include <asm/hwcap.h>
void cpuid_set_flags(void)
{
if (!cpuid_check) {
word64 hwcaps = getauxval(AT_HWCAP);
if (hwcaps & HWCAP_AES)
cpuid_flags |= CPUID_AES;
if (hwcaps & HWCAP_PMULL)
cpuid_flags |= CPUID_PMULL;
if (hwcaps & HWCAP_SHA2)
cpuid_flags |= CPUID_SHA256;
if (hwcaps & HWCAP_SHA512)
cpuid_flags |= CPUID_SHA512;
if (hwcaps & HWCAP_ASIMDRDM)
cpuid_flags |= CPUID_RDM;
if (hwcaps & HWCAP_SHA3)
cpuid_flags |= CPUID_SHA3;
if (hwcaps & HWCAP_SM3)
cpuid_flags |= CPUID_SM3;
if (hwcaps & HWCAP_SM4)
cpuid_flags |= CPUID_SM4;
cpuid_check = 1;
}
}
#elif defined(__ANDROID__) || defined(ANDROID)
/* https://community.arm.com/arm-community-blogs/b/operating-systems-blog/
* posts/runtime-detection-of-cpu-features-on-an-armv8-a-cpu */
#include "cpu-features.h"
void cpuid_set_flags(void)
{
if (!cpuid_check) {
word64 features = android_getCpuFeatures();
if (features & ANDROID_CPU_ARM_FEATURE_AES)
cpuid_flags |= CPUID_AES;
if (features & ANDROID_CPU_ARM_FEATURE_PMULL)
cpuid_flags |= CPUID_PMULL;
if (features & ANDROID_CPU_ARM_FEATURE_SHA2)
cpuid_flags |= CPUID_SHA256;
cpuid_check = 1;
}
}
#elif defined(__APPLE__)
/* https://developer.apple.com/documentation/kernel/1387446-sysctlbyname/
* determining_instruction_set_characteristics */
#include <sys/sysctl.h>
static word64 cpuid_get_sysctlbyname(const char* name)
{
word64 ret = 0;
size_t size = sizeof(ret);
sysctlbyname(name, &ret, &size, NULL, 0);
return ret;
}
void cpuid_set_flags(void)
{
if (!cpuid_check) {
if (cpuid_get_sysctlbyname("hw.optional.arm.FEAT_AES") != 0)
cpuid_flags |= CPUID_AES;
if (cpuid_get_sysctlbyname("hw.optional.arm.FEAT_PMULL") != 0)
cpuid_flags |= CPUID_PMULL;
if (cpuid_get_sysctlbyname("hw.optional.arm.FEAT_SHA256") != 0)
cpuid_flags |= CPUID_SHA256;
if (cpuid_get_sysctlbyname("hw.optional.arm.FEAT_SHA512") != 0)
cpuid_flags |= CPUID_SHA512;
if (cpuid_get_sysctlbyname("hw.optional.arm.FEAT_RDM") != 0)
cpuid_flags |= CPUID_RDM;
if (cpuid_get_sysctlbyname("hw.optional.arm.FEAT_SHA3") != 0)
cpuid_flags |= CPUID_SHA3;
#ifdef WOLFSSL_ARMASM_CRYPTO_SM3
cpuid_flags |= CPUID_SM3;
#endif
#ifdef WOLFSSL_ARMASM_CRYPTO_SM4
cpuid_flags |= CPUID_SM4;
#endif
cpuid_check = 1;
}
}
#elif defined(__FreeBSD__) || defined(__OpenBSD__)
/* https://man.freebsd.org/cgi/man.cgi?elf_aux_info(3) */
#include <sys/auxv.h>
void cpuid_set_flags(void)
{
if (!cpuid_check) {
word64 features = 0;
elf_aux_info(AT_HWCAP, &features, sizeof(features));
if (features & CPUID_AARCH64_FEAT_AES)
cpuid_flags |= CPUID_AES;
if (features & CPUID_AARCH64_FEAT_PMULL)
cpuid_flags |= CPUID_PMULL;
if (features & CPUID_AARCH64_FEAT_SHA256)
cpuid_flags |= CPUID_SHA256;
if (features & CPUID_AARCH64_FEAT_SHA256_512)
cpuid_flags |= CPUID_SHA256 | CPUID_SHA512;
if (features & CPUID_AARCH64_FEAT_RDM)
cpuid_flags |= CPUID_RDM;
if (features & CPUID_AARCH64_FEAT_SHA3)
cpuid_flags |= CPUID_SHA3;
if (features & CPUID_AARCH64_FEAT_SM3)
cpuid_flags |= CPUID_SM3;
if (features & CPUID_AARCH64_FEAT_SM4)
cpuid_flags |= CPUID_SM4;
cpuid_check = 1;
}
}
#else
void cpuid_set_flags(void)
{
if (!cpuid_check) {
#ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO
cpuid_flags |= CPUID_AES;
cpuid_flags |= CPUID_PMULL;
cpuid_flags |= CPUID_SHA256;
#endif
#ifdef WOLFSSL_ARMASM_CRYPTO_SHA512
cpuid_flags |= CPUID_SHA512;
#endif
#ifndef WOLFSSL_AARCH64_NO_SQRMLSH
cpuid_flags |= CPUID_RDM;
#endif
#ifdef WOLFSSL_ARMASM_CRYPTO_SHA3
cpuid_flags |= CPUID_SHA3;
#endif
#ifdef WOLFSSL_ARMASM_CRYPTO_SM3
cpuid_flags |= CPUID_SM3;
#endif
#ifdef WOLFSSL_ARMASM_CRYPTO_SM4
cpuid_flags |= CPUID_SM4;
#endif
cpuid_check = 1;
}
}
#endif
#elif defined(HAVE_CPUID)
void cpuid_set_flags(void)
{

1536
wolfcrypt/src/port/arm/armv8-aes.c
View File

File diff suppressed because it is too large Load Diff

209
wolfcrypt/src/port/arm/armv8-sha256.c
View File

@ -1407,214 +1407,7 @@ static WC_INLINE int Sha256Final(wc_Sha256* sha256, byte* hash)
return ret;
}
#elif defined(__aarch64__)
static const FLASH_QUALIFIER ALIGN32 word32 K[64] = {
0x428A2F98L, 0x71374491L, 0xB5C0FBCFL, 0xE9B5DBA5L, 0x3956C25BL,
0x59F111F1L, 0x923F82A4L, 0xAB1C5ED5L, 0xD807AA98L, 0x12835B01L,
0x243185BEL, 0x550C7DC3L, 0x72BE5D74L, 0x80DEB1FEL, 0x9BDC06A7L,
0xC19BF174L, 0xE49B69C1L, 0xEFBE4786L, 0x0FC19DC6L, 0x240CA1CCL,
0x2DE92C6FL, 0x4A7484AAL, 0x5CB0A9DCL, 0x76F988DAL, 0x983E5152L,
0xA831C66DL, 0xB00327C8L, 0xBF597FC7L, 0xC6E00BF3L, 0xD5A79147L,
0x06CA6351L, 0x14292967L, 0x27B70A85L, 0x2E1B2138L, 0x4D2C6DFCL,
0x53380D13L, 0x650A7354L, 0x766A0ABBL, 0x81C2C92EL, 0x92722C85L,
0xA2BFE8A1L, 0xA81A664BL, 0xC24B8B70L, 0xC76C51A3L, 0xD192E819L,
0xD6990624L, 0xF40E3585L, 0x106AA070L, 0x19A4C116L, 0x1E376C08L,
0x2748774CL, 0x34B0BCB5L, 0x391C0CB3L, 0x4ED8AA4AL, 0x5B9CCA4FL,
0x682E6FF3L, 0x748F82EEL, 0x78A5636FL, 0x84C87814L, 0x8CC70208L,
0x90BEFFFAL, 0xA4506CEBL, 0xBEF9A3F7L, 0xC67178F2L
};
/* Both versions of Ch and Maj are logically the same, but with the second set
the compilers can recognize them better for optimization */
#ifdef WOLFSSL_SHA256_BY_SPEC
/* SHA256 math based on specification */
#define Ch(x,y,z) ((z) ^ ((x) & ((y) ^ (z))))
#define Maj(x,y,z) ((((x) | (y)) & (z)) | ((x) & (y)))
#else
/* SHA256 math reworked for easier compiler optimization */
#define Ch(x,y,z) ((((y) ^ (z)) & (x)) ^ (z))
#define Maj(x,y,z) ((((x) ^ (y)) & ((y) ^ (z))) ^ (y))
#endif
#define R(x, n) (((x) & 0xFFFFFFFFU) >> (n))
#define S(x, n) rotrFixed(x, n)
#define Sigma0(x) (S(x, 2) ^ S(x, 13) ^ S(x, 22))
#define Sigma1(x) (S(x, 6) ^ S(x, 11) ^ S(x, 25))
#define Gamma0(x) (S(x, 7) ^ S(x, 18) ^ R(x, 3))
#define Gamma1(x) (S(x, 17) ^ S(x, 19) ^ R(x, 10))
#define a(i) S[(0-(i)) & 7]
#define b(i) S[(1-(i)) & 7]
#define c(i) S[(2-(i)) & 7]
#define d(i) S[(3-(i)) & 7]
#define e(i) S[(4-(i)) & 7]
#define f(i) S[(5-(i)) & 7]
#define g(i) S[(6-(i)) & 7]
#define h(i) S[(7-(i)) & 7]
#ifndef XTRANSFORM
#define XTRANSFORM(S, D) Transform_Sha256((S),(D))
#endif
#ifndef SHA256_MANY_REGISTERS
#define RND(j) \
t0 = h(j) + Sigma1(e(j)) + Ch(e(j), f(j), g(j)) + K[i+(j)] + \
W[i+(j)]; \
t1 = Sigma0(a(j)) + Maj(a(j), b(j), c(j)); \
d(j) += t0; \
h(j) = t0 + t1
static void Transform_Sha256(wc_Sha256* sha256, const byte* data)
{
word32 S[8], t0, t1;
int i;
#ifdef WOLFSSL_SMALL_STACK_CACHE
word32* W = sha256->W;
if (W == NULL) {
W = (word32*)XMALLOC(sizeof(word32) * WC_SHA256_BLOCK_SIZE, NULL,
DYNAMIC_TYPE_DIGEST);
if (W == NULL)
return MEMORY_E;
sha256->W = W;
}
#elif defined(WOLFSSL_SMALL_STACK)
word32* W;
W = (word32*)XMALLOC(sizeof(word32) * WC_SHA256_BLOCK_SIZE, NULL,
DYNAMIC_TYPE_TMP_BUFFER);
if (W == NULL)
return MEMORY_E;
#else
word32 W[WC_SHA256_BLOCK_SIZE];
#endif
/* Copy context->state[] to working vars */
for (i = 0; i < 8; i++)
S[i] = sha256->digest[i];
for (i = 0; i < 16; i++)
W[i] = *((const word32*)&data[i*(int)sizeof(word32)]);
for (i = 16; i < WC_SHA256_BLOCK_SIZE; i++)
W[i] = Gamma1(W[i-2]) + W[i-7] + Gamma0(W[i-15]) + W[i-16];
#ifdef USE_SLOW_SHA256
/* not unrolled - ~2k smaller and ~25% slower */
for (i = 0; i < WC_SHA256_BLOCK_SIZE; i += 8) {
int j;
for (j = 0; j < 8; j++) { /* braces needed here for macros {} */
RND(j);
}
}
#else
/* partially loop unrolled */
for (i = 0; i < WC_SHA256_BLOCK_SIZE; i += 8) {
RND(0); RND(1); RND(2); RND(3);
RND(4); RND(5); RND(6); RND(7);
}
#endif /* USE_SLOW_SHA256 */
/* Add the working vars back into digest state[] */
for (i = 0; i < 8; i++) {
sha256->digest[i] += S[i];
}
#if defined(WOLFSSL_SMALL_STACK) && !defined(WOLFSSL_SMALL_STACK_CACHE)
ForceZero(W, sizeof(word32) * WC_SHA256_BLOCK_SIZE);
XFREE(W, NULL, DYNAMIC_TYPE_TMP_BUFFER);
#endif
}
#else
/* SHA256 version that keeps all data in registers */
#define SCHED1(j) (W[j] = *((word32*)&data[j*sizeof(word32)]))
#define SCHED(j) ( \
W[ j & 15] += \
Gamma1(W[(j-2) & 15])+ \
W[(j-7) & 15] + \
Gamma0(W[(j-15) & 15]) \
)
#define RND1(j) \
t0 = h(j) + Sigma1(e(j)) + Ch(e(j), f(j), g(j)) + K[i+j] + SCHED1(j); \
t1 = Sigma0(a(j)) + Maj(a(j), b(j), c(j)); \
d(j) += t0; \
h(j) = t0 + t1
#define RNDN(j) \
t0 = h(j) + Sigma1(e(j)) + Ch(e(j), f(j), g(j)) + K[i+j] + SCHED(j); \
t1 = Sigma0(a(j)) + Maj(a(j), b(j), c(j)); \
d(j) += t0; \
h(j) = t0 + t1
static void Transform_Sha256(wc_Sha256* sha256, const byte* data)
{
word32 S[8], t0, t1;
int i;
#ifdef USE_SLOW_SHA256
int j;
#endif
word32 W[WC_SHA256_BLOCK_SIZE/sizeof(word32)];
/* Copy digest to working vars */
S[0] = sha256->digest[0];
S[1] = sha256->digest[1];
S[2] = sha256->digest[2];
S[3] = sha256->digest[3];
S[4] = sha256->digest[4];
S[5] = sha256->digest[5];
S[6] = sha256->digest[6];
S[7] = sha256->digest[7];
i = 0;
#ifdef USE_SLOW_SHA256
for (j = 0; j < 16; j++) {
RND1(j);
}
for (i = 16; i < 64; i += 16) {
for (j = 0; j < 16; j++) {
RNDN(j);
}
}
#else
RND1( 0); RND1( 1); RND1( 2); RND1( 3);
RND1( 4); RND1( 5); RND1( 6); RND1( 7);
RND1( 8); RND1( 9); RND1(10); RND1(11);
RND1(12); RND1(13); RND1(14); RND1(15);
/* 64 operations, partially loop unrolled */
for (i = 16; i < 64; i += 16) {
RNDN( 0); RNDN( 1); RNDN( 2); RNDN( 3);
RNDN( 4); RNDN( 5); RNDN( 6); RNDN( 7);
RNDN( 8); RNDN( 9); RNDN(10); RNDN(11);
RNDN(12); RNDN(13); RNDN(14); RNDN(15);
}
#endif
/* Add the working vars back into digest */
sha256->digest[0] += S[0];
sha256->digest[1] += S[1];
sha256->digest[2] += S[2];
sha256->digest[3] += S[3];
sha256->digest[4] += S[4];
sha256->digest[5] += S[5];
sha256->digest[6] += S[6];
sha256->digest[7] += S[7];
}
#endif /* SHA256_MANY_REGISTERS */
static void Transform_Sha256_Len(wc_Sha256* sha256, const byte* data,
word32 len)
{
while (len > 0) {
byte tmp[WC_SHA256_BLOCK_SIZE];
ByteReverseWords((word32*)tmp, (const word32*)data,
WC_SHA256_BLOCK_SIZE);
Transform_Sha256(sha256, tmp);
data += WC_SHA256_BLOCK_SIZE;
len -= WC_SHA256_BLOCK_SIZE;
}
}
#else
#else /* */
extern void Transform_Sha256_Len(wc_Sha256* sha256, const byte* data,
word32 len);

62
wolfssl/wolfcrypt/aes.h
View File

@ -61,7 +61,7 @@ typedef struct Gcm {
#endif
WOLFSSL_LOCAL void GenerateM0(Gcm* gcm);
#if !defined(__aarch64__) && defined(WOLFSSL_ARMASM)
#ifdef WOLFSSL_ARMASM
WOLFSSL_LOCAL void GMULT(byte* X, byte* Y);
#endif
WOLFSSL_LOCAL void GHASH(Gcm* gcm, const byte* a, word32 aSz, const byte* c,
@ -304,13 +304,6 @@ struct Aes {
#ifdef WOLFSSL_AESNI
byte use_aesni;
#endif /* WOLFSSL_AESNI */
#if defined(__aarch64__) && defined(WOLFSSL_ARMASM) && \
!defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
byte use_aes_hw_crypto;
#ifdef HAVE_AESGCM
byte use_pmull_hw_crypto;
#endif
#endif /* __aarch64__ && WOLFSSL_ARMASM && !WOLFSSL_ARMASM_NO_HW_CRYPTO */
#ifdef WOLF_CRYPTO_CB
int devId;
void* devCtx;
@ -839,59 +832,6 @@ WOLFSSL_API int wc_AesEaxFree(AesEax* eax);
#endif /* WOLFSSL_AES_EAX */
#if defined(__aarch64__) && defined(WOLFSSL_ARMASM) && \
!defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
/* GHASH one block of data.
*
* XOR block into tag and GMULT with H.
*
* @param [in, out] aes AES GCM object.
* @param [in] block Block of AAD or cipher text.
*/
#define GHASH_ONE_BLOCK(aes, block) \
do { \
xorbuf(AES_TAG(aes), block, AES_BLOCK_SIZE); \
GMULT_AARCH64(AES_TAG(aes), aes->gcm.H); \
} \
while (0)
WOLFSSL_LOCAL int AES_set_key_AARCH64(const unsigned char *userKey,
const int keylen, Aes* aes, int dir);
WOLFSSL_LOCAL void AES_encrypt_AARCH64(const byte* inBlock, byte* outBlock,
byte* key, int nr);
WOLFSSL_LOCAL void AES_decrypt_AARCH64(const byte* inBlock, byte* outBlock,
byte* key, int nr);
WOLFSSL_LOCAL void AES_CBC_encrypt_AARCH64(const byte* in, byte* out, word32 sz,
byte* reg, byte* key, int rounds);
WOLFSSL_LOCAL void AES_CBC_decrypt_AARCH64(const byte* in, byte* out, word32 sz,
byte* reg, byte* key, int rounds);
WOLFSSL_LOCAL void AES_CTR_encrypt_AARCH64(Aes* aes, byte* out, const byte* in,
word32 sz);
WOLFSSL_LOCAL void GMULT_AARCH64(byte* X, byte* Y);
#ifdef WOLFSSL_AESGCM_STREAM
WOLFSSL_LOCAL void GHASH_UPDATE_AARCH64(Aes* aes, const byte* a, word32 aSz,
const byte* c, word32 cSz);
WOLFSSL_LOCAL void AES_GCM_init_AARCH64(Aes* aes, const byte* iv, word32 ivSz);
WOLFSSL_LOCAL void AES_GCM_crypt_update_AARCH64(Aes* aes, byte* out,
const byte* in, word32 sz);
WOLFSSL_LOCAL void AES_GCM_final_AARCH64(Aes* aes, byte* authTag,
word32 authTagSz);
#endif
WOLFSSL_LOCAL void AES_GCM_set_key_AARCH64(Aes* aes, byte* iv);
WOLFSSL_LOCAL void AES_GCM_encrypt_AARCH64(Aes* aes, byte* out, const byte* in,
word32 sz, const byte* iv, word32 ivSz, byte* authTag, word32 authTagSz,
const byte* authIn, word32 authInSz);
WOLFSSL_LOCAL int AES_GCM_decrypt_AARCH64(Aes* aes, byte* out, const byte* in,
word32 sz, const byte* iv, word32 ivSz, const byte* authTag,
word32 authTagSz, const byte* authIn, word32 authInSz);
#ifdef WOLFSSL_AES_XTS
WOLFSSL_LOCAL void AES_XTS_encrypt_AARCH64(XtsAes* xaes, byte* out,
const byte* in, word32 sz, const byte* i);
WOLFSSL_LOCAL void AES_XTS_decrypt_AARCH64(XtsAes* xaes, byte* out,
const byte* in, word32 sz, const byte* i);
#endif /* WOLFSSL_AES_XTS */
#endif /* __aarch64__ && WOLFSSL_ARMASM && !WOLFSSL_ARMASM_NO_HW_CRYPTO */
#ifdef __cplusplus
} /* extern "C" */

25
wolfssl/wolfcrypt/cpuid.h
View File

@ -38,11 +38,6 @@
#define HAVE_CPUID
#define HAVE_CPUID_INTEL
#endif
#if (defined(WOLFSSL_AARCH64_BUILD) || (defined(__aarch64__) && \
defined(WOLFSSL_ARMASM))) && !defined(WOLFSSL_NO_ASM)
#define HAVE_CPUID
#define HAVE_CPUID_AARCH64
#endif
#ifdef HAVE_CPUID_INTEL
@ -68,26 +63,6 @@
#define IS_INTEL_BMI1(f) ((f) & CPUID_BMI1)
#define IS_INTEL_SHA(f) ((f) & CPUID_SHA)
#elif defined(HAVE_CPUID_AARCH64)
#define CPUID_AES 0x0001
#define CPUID_PMULL 0x0002
#define CPUID_SHA256 0x0004
#define CPUID_SHA512 0x0008
#define CPUID_RDM 0x0010
#define CPUID_SHA3 0x0020
#define CPUID_SM3 0x0040
#define CPUID_SM4 0x0080
#define IS_AARCH64_AES(f) ((f) & CPUID_AES)
#define IS_AARCH64_PMULL(f) ((f) & CPUID_PMULL)
#define IS_AARCH64_SHA256(f) ((f) & CPUID_SHA256)
#define IS_AARCH64_SHA512(f) ((f) & CPUID_SHA512)
#define IS_AARCH64_RDM(f) ((f) & CPUID_RDM)
#define IS_AARCH64_SHA3(f) ((f) & CPUID_SHA3)
#define IS_AARCH64_SM3(f) ((f) & CPUID_SM3)
#define IS_AARCH64_SM4(f) ((f) & CPUID_SM4)
#endif
#ifdef HAVE_CPUID