wolfSSL/wolfssl
1
0
Fork 1
mirror of https://github.com/wolfSSL/wolfssl.git synced 2026-08-13 17:41:22 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #10940 from SparkiDev/chacha20_poly1305_avx512

Browse Source 
ChaCha20/Poly1305 ASM: AVX512 and improvements to other Intel x64
This commit is contained in:
David Garske
2026-08-11 10:18:18 -07:00
committed by GitHub
parent 8a31663c54 3717560a01
commit e0feaa36e9
18 changed files with 26449 additions and 549 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.wolfssl_known_macro_extras
+20
View File
@@ -55,6 +55,7 @@ BSP_SDCARD_SDHC_CHANNEL
BSP_SDCARD_SPI_CHANNEL
CAAM_OUT_INVALIDATE
CERT_REL_PREFIX
CHACHA20_POLY1305_SHORT_MAX
CIOCASYMFEAT
CIOCGSESSINFO
CMSIS_OS2_H_
@@ -845,6 +846,15 @@ WOLFSSL_CAAM_BLACK_KEY_AESCCM
WOLFSSL_CAAM_BLACK_KEY_SM
WOLFSSL_CAAM_NO_BLACK_KEY
WOLFSSL_CALLBACKS
WOLFSSL_CHACHA20_AVX512_ALWAYS
WOLFSSL_CHACHA20_AVX512_NEVER
WOLFSSL_CHACHA20_POLY1305_FUSED
WOLFSSL_CHACHA20_POLY1305_FUSED_ALWAYS
WOLFSSL_CHACHA20_POLY1305_FUSED_IFMA
WOLFSSL_CHACHA20_POLY1305_FUSED_IFMA_ALWAYS
WOLFSSL_CHACHA20_POLY1305_FUSED_IFMA_NEVER
WOLFSSL_CHACHA20_POLY1305_FUSED_NEVER
WOLFSSL_CHACHA20_POLY1305_SHORT
WOLFSSL_CHECK_DESKEY
WOLFSSL_CHIBIOS
WOLFSSL_CLANG_TIDY
@@ -941,6 +951,10 @@ WOLFSSL_NONBLOCK_OCSP
WOLFSSL_NOSHA3_384
WOLFSSL_NOT_WINDOWS_API
WOLFSSL_NO_BIO_ADDR_IN
WOLFSSL_NO_CHACHA20_POLY1305_FUSED
WOLFSSL_NO_CHACHA20_POLY1305_FUSED_IFMA
WOLFSSL_NO_CHACHA20_POLY1305_SHORT
WOLFSSL_NO_CHACHA20_POLY1305_SMALL_ASM
WOLFSSL_NO_CLIENT_CERT_ERROR
WOLFSSL_NO_COPY_CERT
WOLFSSL_NO_COPY_KEY
@@ -994,6 +1008,12 @@ WOLFSSL_PASSTHRU_ERR
WOLFSSL_PB
WOLFSSL_PEER_ADDRESS_CHANGES
WOLFSSL_PKCS11_RW_TOKENS
WOLFSSL_POLY1305_AVX512
WOLFSSL_POLY1305_AVX512_ALWAYS
WOLFSSL_POLY1305_AVX512_NEVER
WOLFSSL_POLY1305_IFMA_ALWAYS
WOLFSSL_POLY1305_NO_AVX512
WOLFSSL_POLY1305_NO_IFMA
WOLFSSL_PPC64_ASM_AES_NO_HARDEN
WOLFSSL_PRCONNECT_PRO
WOLFSSL_PREFIX
doc/dox_comments/header_files/chacha20_poly1305.h
+120
View File
@@ -123,6 +123,126 @@ int wc_ChaCha20Poly1305_Decrypt(
const byte inAuthTag[CHACHA20_POLY1305_AEAD_AUTHTAG_SIZE],
byte* outPlaintext);
/*!
\ingroup ChaCha20Poly1305
\brief This function performs the same AEAD encryption as
wc_ChaCha20Poly1305_Encrypt, but takes a caller-owned ChaCha20 context whose
key has already been set (with wc_Chacha_SetKey) plus a Poly1305 context,
instead of a raw key. This lets the ChaCha20 key be set once and reused
across many records, varying only the per-record nonce - it is intended for
protocol record layers such as TLS. A fresh per-record Poly1305 key is
derived from the ChaCha20 keystream, so this call re-keys the supplied
Poly1305 context; the caller does not key it. The 16 byte authentication tag
over the AAD and ciphertext is written to tag.
\return 0 Returned upon successfully encrypting the message
\return BAD_FUNC_ARG Returned if a required pointer argument is NULL (with
its matching length nonzero) or otherwise invalid
\param chacha pointer to a ChaCha20 context already keyed with
wc_Chacha_SetKey
\param poly pointer to a Poly1305 context used for the per-record MAC; it is
re-keyed internally on each call
\param out pointer to the buffer in which to store the ciphertext (sz bytes)
\param in pointer to the buffer containing the plaintext to encrypt
\param sz the length in bytes of the plaintext to encrypt
\param nonce pointer to the 12 byte per-record nonce
\param tag pointer to a 16 byte buffer in which to store the authentication
tag
\param aad pointer to the buffer containing arbitrary length additional
authenticated data (AAD)
\param aadSz length of the input AAD
_Example_
\code
ChaCha chacha;
Poly1305 poly;
byte key[] = { // initialize 32 byte key };
byte nonce[] = { // initialize 12 byte per-record nonce };
byte aad[] = { // initialize AAD };
byte plain[] = { // initialize message to encrypt };
byte cipher[sizeof(plain)];
byte authTag[16];
wc_Chacha_SetKey(&chacha, key, sizeof(key)); // once, then reuse
int ret = wc_ChaCha20Poly1305_Encrypt_ex(&chacha, &poly, cipher, plain,
sizeof(plain), nonce, authTag, aad, sizeof(aad));
if (ret != 0) {
// error running encrypt
}
\endcode
\sa wc_ChaCha20Poly1305_Decrypt_ex
\sa wc_ChaCha20Poly1305_Encrypt
\sa wc_Chacha_SetKey
*/
int wc_ChaCha20Poly1305_Encrypt_ex(ChaCha* chacha, Poly1305* poly,
byte* out, const byte* in, word32 sz, const byte* nonce, byte* tag,
const byte* aad, word32 aadSz);
/*!
\ingroup ChaCha20Poly1305
\brief This function is the decryption counterpart of
wc_ChaCha20Poly1305_Encrypt_ex. It takes a caller-owned ChaCha20 context
whose key has already been set (with wc_Chacha_SetKey) plus a Poly1305
context, decrypts in to out, and verifies the Poly1305 tag over the AAD and
ciphertext. On tag mismatch it returns MAC_CMP_FAILED_E and zeroizes the
output buffer, so no unauthenticated plaintext is released. The ChaCha20 key
is reused across records, varying only the per-record nonce; the Poly1305
context is re-keyed internally on each call. out may alias in (in-place
decryption is supported).
\return 0 Returned upon successfully decrypting and authenticating the
message
\return MAC_CMP_FAILED_E Returned if the computed authentication tag does not
match the supplied tag; out is zeroized in this case
\return BAD_FUNC_ARG Returned if a required pointer argument is NULL (with
its matching length nonzero) or otherwise invalid
\param chacha pointer to a ChaCha20 context already keyed with
wc_Chacha_SetKey
\param poly pointer to a Poly1305 context used for the per-record MAC; it is
re-keyed internally on each call
\param out pointer to the buffer in which to store the plaintext (sz bytes)
\param in pointer to the buffer containing the ciphertext to decrypt
\param sz the length in bytes of the ciphertext to decrypt
\param nonce pointer to the 12 byte per-record nonce
\param tag pointer to the 16 byte authentication tag to verify
\param aad pointer to the buffer containing arbitrary length additional
authenticated data (AAD)
\param aadSz length of the input AAD
_Example_
\code
ChaCha chacha;
Poly1305 poly;
byte key[] = { // initialize 32 byte key };
byte nonce[] = { // initialize 12 byte per-record nonce };
byte aad[] = { // initialize AAD };
byte cipher[] = { // received ciphertext };
byte authTag[16] = { // received authentication tag };
byte plain[sizeof(cipher)];
wc_Chacha_SetKey(&chacha, key, sizeof(key)); // once, then reuse
int ret = wc_ChaCha20Poly1305_Decrypt_ex(&chacha, &poly, plain, cipher,
sizeof(cipher), nonce, authTag, aad, sizeof(aad));
if (ret == MAC_CMP_FAILED_E) {
// authentication failed; plain has been zeroized
} else if (ret != 0) {
// error with function arguments
}
\endcode
\sa wc_ChaCha20Poly1305_Encrypt_ex
\sa wc_ChaCha20Poly1305_Decrypt
\sa wc_Chacha_SetKey
*/
int wc_ChaCha20Poly1305_Decrypt_ex(
ChaCha* chacha, Poly1305* poly, byte* out, const byte* in, word32 sz,
const byte* nonce, const byte* tag, const byte* aad, word32 aadSz);
/*!
\ingroup ChaCha20Poly1305
\brief Compares two authentication tags in constant time to prevent
src/internal.c
+142 -137
View File
@@ -21465,55 +21465,61 @@ int ChachaAEADEncrypt(WOLFSSL* ssl, byte* out, const byte* input,
wc_MemZero_Add("ChachaAEADEncrypt nonce", nonce, CHACHA20_NONCE_SZ);
#endif
/* set the nonce for chacha and get poly1305 key */
if ((ret = wc_Chacha_SetIV(ssl->encrypt.chacha, nonce, 0)) != 0) {
ForceZero(nonce, CHACHA20_NONCE_SZ);
#ifdef WOLFSSL_CHECK_MEM_ZERO
wc_MemZero_Check(nonce, CHACHA20_NONCE_SZ);
#endif
return ret;
}
/* create Poly1305 key using chacha20 keystream */
if ((ret = wc_Chacha_Process(ssl->encrypt.chacha, poly,
poly, sizeof(poly))) != 0) {
ForceZero(nonce, CHACHA20_NONCE_SZ);
#ifdef WOLFSSL_CHECK_MEM_ZERO
wc_MemZero_Check(nonce, CHACHA20_NONCE_SZ);
#endif
return ret;
}
#ifdef WOLFSSL_CHECK_MEM_ZERO
wc_MemZero_Add("ChachaAEADEncrypt poly", poly, CHACHA20_256_KEY_SIZE);
#endif
/* set the counter after getting poly1305 key */
if ((ret = wc_Chacha_SetIV(ssl->encrypt.chacha, nonce, 1)) != 0) {
ForceZero(nonce, CHACHA20_NONCE_SZ);
ForceZero(poly, sizeof(poly));
#ifdef WOLFSSL_CHECK_MEM_ZERO
wc_MemZero_Check(nonce, CHACHA20_NONCE_SZ);
wc_MemZero_Check(poly, CHACHA20_256_KEY_SIZE);
#endif
return ret;
}
ForceZero(nonce, CHACHA20_NONCE_SZ); /* done with nonce, clear it */
#ifdef WOLFSSL_CHECK_MEM_ZERO
wc_MemZero_Check(nonce, CHACHA20_NONCE_SZ);
#endif
/* encrypt the plain text */
if ((ret = wc_Chacha_Process(ssl->encrypt.chacha, out,
input, msgLen)) != 0) {
ForceZero(poly, sizeof(poly));
#ifdef WOLFSSL_CHECK_MEM_ZERO
wc_MemZero_Check(poly, CHACHA20_256_KEY_SIZE);
#endif
return ret;
}
/* get the poly1305 tag using either old padding scheme or more recent */
/* Derive the Poly1305 key, encrypt and authenticate. The legacy oldPoly
* draft keeps the manual derivation and old tag layout. RFC 7905 uses the
* persistent-key stitched helper - it derives the per-record poly key,
* encrypts and MACs in one pass (the IFMA stitch for large records, else
* two-pass), matching the split ssl->encrypt.chacha / ssl->auth.poly1305
* contexts kept keyed across the connection. */
if (ssl->options.oldPoly != 0) {
/* set the nonce for chacha and get poly1305 key */
if ((ret = wc_Chacha_SetIV(ssl->encrypt.chacha, nonce, 0)) != 0) {
ForceZero(nonce, CHACHA20_NONCE_SZ);
#ifdef WOLFSSL_CHECK_MEM_ZERO
wc_MemZero_Check(nonce, CHACHA20_NONCE_SZ);
#endif
return ret;
}
/* create Poly1305 key using chacha20 keystream */
if ((ret = wc_Chacha_Process(ssl->encrypt.chacha, poly,
poly, sizeof(poly))) != 0) {
ForceZero(nonce, CHACHA20_NONCE_SZ);
#ifdef WOLFSSL_CHECK_MEM_ZERO
wc_MemZero_Check(nonce, CHACHA20_NONCE_SZ);
#endif
return ret;
}
#ifdef WOLFSSL_CHECK_MEM_ZERO
wc_MemZero_Add("ChachaAEADEncrypt poly", poly, CHACHA20_256_KEY_SIZE);
#endif
/* set the counter after getting poly1305 key */
if ((ret = wc_Chacha_SetIV(ssl->encrypt.chacha, nonce, 1)) != 0) {
ForceZero(nonce, CHACHA20_NONCE_SZ);
ForceZero(poly, sizeof(poly));
#ifdef WOLFSSL_CHECK_MEM_ZERO
wc_MemZero_Check(nonce, CHACHA20_NONCE_SZ);
wc_MemZero_Check(poly, CHACHA20_256_KEY_SIZE);
#endif
return ret;
}
ForceZero(nonce, CHACHA20_NONCE_SZ); /* done with nonce, clear it */
#ifdef WOLFSSL_CHECK_MEM_ZERO
wc_MemZero_Check(nonce, CHACHA20_NONCE_SZ);
#endif
/* encrypt the plain text */
if ((ret = wc_Chacha_Process(ssl->encrypt.chacha, out,
input, msgLen)) != 0) {
ForceZero(poly, sizeof(poly));
#ifdef WOLFSSL_CHECK_MEM_ZERO
wc_MemZero_Check(poly, CHACHA20_256_KEY_SIZE);
#endif
return ret;
}
/* get the poly1305 tag using the old padding scheme */
if ((ret = Poly1305TagOld(ssl, add, addSz, (const byte* )out,
poly, sz, tag)) != 0) {
ForceZero(poly, sizeof(poly));
@@ -21522,29 +21528,22 @@ int ChachaAEADEncrypt(WOLFSSL* ssl, byte* out, const byte* input,
#endif
return ret;
}
ForceZero(poly, sizeof(poly)); /* done with poly1305 key, clear it */
#ifdef WOLFSSL_CHECK_MEM_ZERO
wc_MemZero_Check(poly, CHACHA20_256_KEY_SIZE);
#endif
}
else {
if ((ret = wc_Poly1305SetKey(ssl->auth.poly1305, poly,
sizeof(poly))) != 0) {
ForceZero(poly, sizeof(poly));
#ifdef WOLFSSL_CHECK_MEM_ZERO
wc_MemZero_Check(poly, CHACHA20_256_KEY_SIZE);
#endif
ret = wc_ChaCha20Poly1305_Encrypt_ex(ssl->encrypt.chacha,
ssl->auth.poly1305, out, input, msgLen, nonce, tag, add,
(word32)addSz);
ForceZero(nonce, CHACHA20_NONCE_SZ); /* done with nonce, clear it */
#ifdef WOLFSSL_CHECK_MEM_ZERO
wc_MemZero_Check(nonce, CHACHA20_NONCE_SZ);
#endif
if (ret != 0)
return ret;
}
if ((ret = wc_Poly1305_MAC(ssl->auth.poly1305, add, addSz, out, msgLen,
tag, sizeof(tag))) != 0) {
ForceZero(poly, sizeof(poly));
#ifdef WOLFSSL_CHECK_MEM_ZERO
wc_MemZero_Check(poly, CHACHA20_256_KEY_SIZE);
#endif
return ret;
}
}
ForceZero(poly, sizeof(poly)); /* done with poly1305 key, clear it */
#ifdef WOLFSSL_CHECK_MEM_ZERO
wc_MemZero_Check(poly, CHACHA20_256_KEY_SIZE);
#endif
/* append tag to ciphertext */
XMEMCPY(out + msgLen, tag, sizeof(tag));
@@ -21659,45 +21658,49 @@ int ChachaAEADDecrypt(WOLFSSL* ssl, byte* plain, const byte* input,
wc_MemZero_Add("ChachaAEADEncrypt nonce", nonce, CHACHA20_NONCE_SZ);
#endif
/* set nonce and get poly1305 key */
if ((ret = wc_Chacha_SetIV(ssl->decrypt.chacha, nonce, 0)) != 0) {
ForceZero(nonce, CHACHA20_NONCE_SZ);
#ifdef WOLFSSL_CHECK_MEM_ZERO
wc_MemZero_Check(nonce, CHACHA20_NONCE_SZ);
#endif
return ret;
}
/* use chacha20 keystream to get poly1305 key for tag */
if ((ret = wc_Chacha_Process(ssl->decrypt.chacha, poly,
poly, sizeof(poly))) != 0) {
ForceZero(nonce, CHACHA20_NONCE_SZ);
#ifdef WOLFSSL_CHECK_MEM_ZERO
wc_MemZero_Check(nonce, CHACHA20_NONCE_SZ);
#endif
return ret;
}
#ifdef WOLFSSL_CHECK_MEM_ZERO
wc_MemZero_Add("ChachaAEADEncrypt poly", poly, CHACHA20_256_KEY_SIZE);
#endif
/* set counter after getting poly1305 key */
if ((ret = wc_Chacha_SetIV(ssl->decrypt.chacha, nonce, 1)) != 0) {
ForceZero(nonce, CHACHA20_NONCE_SZ);
ForceZero(poly, sizeof(poly));
#ifdef WOLFSSL_CHECK_MEM_ZERO
wc_MemZero_Check(nonce, CHACHA20_NONCE_SZ);
wc_MemZero_Check(poly, CHACHA20_256_KEY_SIZE);
#endif
return ret;
}
ForceZero(nonce, CHACHA20_NONCE_SZ); /* done with nonce, clear it */
#ifdef WOLFSSL_CHECK_MEM_ZERO
wc_MemZero_Check(nonce, CHACHA20_NONCE_SZ);
#endif
/* get the tag using Poly1305 */
/* Verify the tag and decrypt. oldPoly keeps the manual derivation, old tag
* layout and verify-then-decrypt. RFC 7905 uses the persistent-key
* stitched helper (verify + decrypt in one pass - the IFMA decrypt stitch
* for large records, else two-pass; it zeroes plain on tag mismatch). */
if (ssl->options.oldPoly != 0) {
/* set nonce and get poly1305 key */
if ((ret = wc_Chacha_SetIV(ssl->decrypt.chacha, nonce, 0)) != 0) {
ForceZero(nonce, CHACHA20_NONCE_SZ);
#ifdef WOLFSSL_CHECK_MEM_ZERO
wc_MemZero_Check(nonce, CHACHA20_NONCE_SZ);
#endif
return ret;
}
/* use chacha20 keystream to get poly1305 key for tag */
if ((ret = wc_Chacha_Process(ssl->decrypt.chacha, poly,
poly, sizeof(poly))) != 0) {
ForceZero(nonce, CHACHA20_NONCE_SZ);
#ifdef WOLFSSL_CHECK_MEM_ZERO
wc_MemZero_Check(nonce, CHACHA20_NONCE_SZ);
#endif
return ret;
}
#ifdef WOLFSSL_CHECK_MEM_ZERO
wc_MemZero_Add("ChachaAEADEncrypt poly", poly, CHACHA20_256_KEY_SIZE);
#endif
/* set counter after getting poly1305 key */
if ((ret = wc_Chacha_SetIV(ssl->decrypt.chacha, nonce, 1)) != 0) {
ForceZero(nonce, CHACHA20_NONCE_SZ);
ForceZero(poly, sizeof(poly));
#ifdef WOLFSSL_CHECK_MEM_ZERO
wc_MemZero_Check(nonce, CHACHA20_NONCE_SZ);
wc_MemZero_Check(poly, CHACHA20_256_KEY_SIZE);
#endif
return ret;
}
ForceZero(nonce, CHACHA20_NONCE_SZ); /* done with nonce, clear it */
#ifdef WOLFSSL_CHECK_MEM_ZERO
wc_MemZero_Check(nonce, CHACHA20_NONCE_SZ);
#endif
/* get the tag using the old padding scheme */
if ((ret = Poly1305TagOld(ssl, add, addSz, input, poly, sz, tag))
!= 0) {
ForceZero(poly, sizeof(poly));
@@ -21706,43 +21709,45 @@ int ChachaAEADDecrypt(WOLFSSL* ssl, byte* plain, const byte* input,
#endif
return ret;
}
ForceZero(poly, sizeof(poly)); /* done with poly1305 key, clear it */
#ifdef WOLFSSL_CHECK_MEM_ZERO
wc_MemZero_Check(poly, CHACHA20_256_KEY_SIZE);
#endif
/* check tag sent along with packet */
if (ConstantCompare(input + msgLen, tag,
ssl->specs.aead_mac_size) != 0) {
WOLFSSL_MSG("MAC did not match");
if (!ssl->options.dtls)
SendAlert(ssl, alert_fatal, bad_record_mac);
WOLFSSL_ERROR_VERBOSE(VERIFY_MAC_ERROR);
return VERIFY_MAC_ERROR;
}
/* if the tag was good decrypt message */
if ((ret = wc_Chacha_Process(ssl->decrypt.chacha, plain,
input, (word32)msgLen)) != 0)
return ret;
}
else {
if ((ret = wc_Poly1305SetKey(ssl->auth.poly1305, poly,
sizeof(poly))) != 0) {
ForceZero(poly, sizeof(poly));
#ifdef WOLFSSL_CHECK_MEM_ZERO
wc_MemZero_Check(poly, CHACHA20_256_KEY_SIZE);
#endif
return ret;
}
if ((ret = wc_Poly1305_MAC(ssl->auth.poly1305, add, addSz, input,
(word32)msgLen, tag, sizeof(tag))) != 0) {
ForceZero(poly, sizeof(poly));
#ifdef WOLFSSL_CHECK_MEM_ZERO
wc_MemZero_Check(poly, CHACHA20_256_KEY_SIZE);
#endif
ret = wc_ChaCha20Poly1305_Decrypt_ex(ssl->decrypt.chacha,
ssl->auth.poly1305, plain, input, (word32)msgLen, nonce,
input + msgLen, add, (word32)addSz);
ForceZero(nonce, CHACHA20_NONCE_SZ); /* done with nonce, clear it */
#ifdef WOLFSSL_CHECK_MEM_ZERO
wc_MemZero_Check(nonce, CHACHA20_NONCE_SZ);
#endif
if (ret != 0) {
if (ret == WC_NO_ERR_TRACE(MAC_CMP_FAILED_E)) {
WOLFSSL_MSG("MAC did not match");
if (!ssl->options.dtls)
SendAlert(ssl, alert_fatal, bad_record_mac);
WOLFSSL_ERROR_VERBOSE(VERIFY_MAC_ERROR);
return VERIFY_MAC_ERROR;
}
return ret;
}
}
ForceZero(poly, sizeof(poly)); /* done with poly1305 key, clear it */
#ifdef WOLFSSL_CHECK_MEM_ZERO
wc_MemZero_Check(poly, CHACHA20_256_KEY_SIZE);
#endif
/* check tag sent along with packet */
if (ConstantCompare(input + msgLen, tag, ssl->specs.aead_mac_size) != 0) {
WOLFSSL_MSG("MAC did not match");
if (!ssl->options.dtls)
SendAlert(ssl, alert_fatal, bad_record_mac);
WOLFSSL_ERROR_VERBOSE(VERIFY_MAC_ERROR);
return VERIFY_MAC_ERROR;
}
/* if the tag was good decrypt message */
if ((ret = wc_Chacha_Process(ssl->decrypt.chacha, plain,
input, (word32)msgLen)) != 0)
return ret;
#ifdef CHACHA_AEAD_TEST
printf("plain after decrypt :\n");
src/tls13.c
+12 -88
View File
@@ -2543,49 +2543,11 @@ static int ChaCha20Poly1305_Encrypt(WOLFSSL* ssl, byte* output,
const byte* input, word16 sz, byte* nonce,
const byte* aad, word16 aadSz, byte* tag)
{
int ret = 0;
byte poly[CHACHA20_256_KEY_SIZE];
/* Poly1305 key is 256 bits of zero encrypted with ChaCha20. */
XMEMSET(poly, 0, sizeof(poly));
/* Set the nonce for ChaCha and get Poly1305 key. */
ret = wc_Chacha_SetIV(ssl->encrypt.chacha, nonce, 0);
if (ret != 0)
return ret;
/* Create Poly1305 key using ChaCha20 keystream. */
ret = wc_Chacha_Process(ssl->encrypt.chacha, poly, poly, sizeof(poly));
if (ret != 0)
return ret;
#ifdef WOLFSSL_CHECK_MEM_ZERO
wc_MemZero_Add("ChaCha20Poly1305_Encrypt poly", poly, sizeof(poly));
#endif
ret = wc_Chacha_SetIV(ssl->encrypt.chacha, nonce, 1);
if (ret != 0)
return ret;
/* Encrypt the plain text. */
ret = wc_Chacha_Process(ssl->encrypt.chacha, output, input, sz);
if (ret != 0) {
ForceZero(poly, sizeof(poly));
#ifdef WOLFSSL_CHECK_MEM_ZERO
wc_MemZero_Check(poly, sizeof(poly));
#endif
return ret;
}
/* Set key for Poly1305. */
ret = wc_Poly1305SetKey(ssl->auth.poly1305, poly, sizeof(poly));
ForceZero(poly, sizeof(poly)); /* done with poly1305 key, clear it */
#ifdef WOLFSSL_CHECK_MEM_ZERO
wc_MemZero_Check(poly, sizeof(poly));
#endif
if (ret != 0)
return ret;
/* Add authentication code of encrypted data to end. */
ret = wc_Poly1305_MAC(ssl->auth.poly1305, aad, aadSz, output, sz, tag,
POLY1305_AUTH_SZ);
return ret;
/* Persistent-key stitched helper: derives the per-record Poly1305 key from
* the keyed ChaCha, then encrypts and authenticates in one pass (the IFMA
* stitch for large records, else two-pass). TLS 1.3 is always RFC 8439. */
return wc_ChaCha20Poly1305_Encrypt_ex(ssl->encrypt.chacha,
ssl->auth.poly1305, output, input, sz, nonce, tag, aad, aadSz);
}
#endif
@@ -2931,55 +2893,17 @@ static int ChaCha20Poly1305_Decrypt(WOLFSSL* ssl, byte* output,
const byte* tagIn)
{
int ret;
byte tag[POLY1305_AUTH_SZ];
byte poly[CHACHA20_256_KEY_SIZE]; /* generated key for mac */
/* Poly1305 key is 256 bits of zero encrypted with ChaCha20. */
XMEMSET(poly, 0, sizeof(poly));
/* Set nonce and get Poly1305 key. */
ret = wc_Chacha_SetIV(ssl->decrypt.chacha, nonce, 0);
if (ret != 0)
return ret;
/* Use ChaCha20 keystream to get Poly1305 key for tag. */
ret = wc_Chacha_Process(ssl->decrypt.chacha, poly, poly, sizeof(poly));
if (ret != 0)
return ret;
#ifdef WOLFSSL_CHECK_MEM_ZERO
wc_MemZero_Add("ChaCha20Poly1305_Decrypt poly", poly, sizeof(poly));
#endif
ret = wc_Chacha_SetIV(ssl->decrypt.chacha, nonce, 1);
if (ret != 0) {
ForceZero(poly, sizeof(poly)); /* done with poly1305 key, clear it */
#ifdef WOLFSSL_CHECK_MEM_ZERO
wc_MemZero_Check(poly, sizeof(poly));
#endif
return ret;
}
/* Set key for Poly1305. */
ret = wc_Poly1305SetKey(ssl->auth.poly1305, poly, sizeof(poly));
ForceZero(poly, sizeof(poly)); /* done with poly1305 key, clear it */
#ifdef WOLFSSL_CHECK_MEM_ZERO
wc_MemZero_Check(poly, sizeof(poly));
#endif
if (ret != 0)
return ret;
/* Generate authentication tag for encrypted data. */
if ((ret = wc_Poly1305_MAC(ssl->auth.poly1305, aad, aadSz, input, sz, tag,
sizeof(tag))) != 0) {
return ret;
}
/* Check tag sent along with packet. */
if (ConstantCompare(tagIn, tag, POLY1305_AUTH_SZ) != 0) {
/* Persistent-key stitched helper: verifies the Poly1305 tag over
* aad+ciphertext and decrypts in one pass (the IFMA decrypt stitch for
* large records, else two-pass); it zeroes output on tag mismatch. */
ret = wc_ChaCha20Poly1305_Decrypt_ex(ssl->decrypt.chacha,
ssl->auth.poly1305, output, input, sz, nonce, tagIn, aad, aadSz);
if (ret == WC_NO_ERR_TRACE(MAC_CMP_FAILED_E)) {
WOLFSSL_MSG("MAC did not match");
return VERIFY_MAC_ERROR;
ret = VERIFY_MAC_ERROR;
}
/* If the tag was good decrypt message. */
ret = wc_Chacha_Process(ssl->decrypt.chacha, output, input, sz);
return ret;
}
#endif
tests/api/test_chacha20_poly1305.c
+563 -2
View File
@@ -359,6 +359,8 @@ int test_wc_ChaCha20Poly1305_MonteCarlo(void)
byte key[CHACHA20_POLY1305_AEAD_KEYSIZE];
byte nonce[CHACHA20_POLY1305_AEAD_IV_SIZE];
byte tag[CHACHA20_POLY1305_AEAD_AUTHTAG_SIZE];
byte aad[32];
word32 aadLen = 0;
word32 plainLen = 0;
int i;
WC_DECLARE_VAR(plain, byte, MC_CHACHA20P1305_MAX_SZ, NULL);
@@ -386,9 +388,18 @@ int test_wc_ChaCha20Poly1305_MonteCarlo(void)
plainLen = (plainLen % MC_CHACHA20P1305_MAX_SZ) + 1;
ExpectIntEQ(wc_RNG_GenerateBlock(&rng, plain, plainLen), 0);
ExpectIntEQ(wc_ChaCha20Poly1305_Encrypt(key, nonce, NULL, 0,
/* Random AAD (0..sizeof(aad)) so the AAD fold is exercised alongside
* every randomly-chosen message size, including the sz <= 64 small
* kernel band. */
ExpectIntEQ(wc_RNG_GenerateBlock(&rng, (byte*)&aadLen,
sizeof(aadLen)), 0);
aadLen = aadLen % (word32)(sizeof(aad) + 1);
if (aadLen > 0)
ExpectIntEQ(wc_RNG_GenerateBlock(&rng, aad, aadLen), 0);
ExpectIntEQ(wc_ChaCha20Poly1305_Encrypt(key, nonce, aad, aadLen,
plain, plainLen, cipher, tag), 0);
ExpectIntEQ(wc_ChaCha20Poly1305_Decrypt(key, nonce, NULL, 0,
ExpectIntEQ(wc_ChaCha20Poly1305_Decrypt(key, nonce, aad, aadLen,
cipher, plainLen, tag, decrypted), 0);
ExpectBufEQ(decrypted, plain, plainLen);
}
@@ -1308,3 +1319,553 @@ int test_wc_XChaCha20Poly1305_LargeBuffer(void)
#endif
return EXPECT_RESULT();
} /* END test_wc_XChaCha20Poly1305_LargeBuffer */
/*
* Large-message coverage for the AVX-512 + IFMA single-pass stitch, which the
* one-shot Encrypt, Encrypt_ex, Decrypt_ex and streaming paths all dispatch to
* at sz >= CHACHA20_POLY1305_STITCH_MIN (default 4096) on capable CPUs. No
* other test reaches 4096 bytes, so on AVX-512/IFMA hardware this is the only
* exercise of the stitch kernel, its scalar sub-1024 tail, and the AAD fold.
* On CPUs without AVX-512/IFMA every call transparently uses the two-pass path,
* so the test still validates (round-trips) but does not reach the stitch.
*
* Sizes span the sub-bands the stitch splits on: exact 1024-multiples (no
* tail), non-multiples (stitch + scalar tail), and both AAD present / absent
* (the poly1305_fold_avx512ifma AAD fold). Correctness is cross-checked three
* independent ways: the one-shot two-pass Decrypt round-trips the one-shot
* (stitch) ciphertext; Encrypt_ex must reproduce the one-shot ciphertext+tag
* byte-for-byte; and Decrypt_ex (the decrypt stitch, decrypt-then-verify)
* recovers the plaintext and, on a corrupted tag, returns MAC_CMP_FAILED_E with
* a zeroized output (no plaintext released though the stitch decrypts first).
*/
int test_wc_ChaCha20Poly1305_LargeMessage(void)
{
EXPECT_DECLS;
#if defined(HAVE_CHACHA) && defined(HAVE_POLY1305)
static const byte key[CHACHA20_POLY1305_AEAD_KEYSIZE] = {
0x80,0x81,0x82,0x83, 0x84,0x85,0x86,0x87,
0x88,0x89,0x8a,0x8b, 0x8c,0x8d,0x8e,0x8f,
0x90,0x91,0x92,0x93, 0x94,0x95,0x96,0x97,
0x98,0x99,0x9a,0x9b, 0x9c,0x9d,0x9e,0x9f
};
static const byte iv[CHACHA20_POLY1305_AEAD_IV_SIZE] = {
0x07,0x00,0x00,0x00, 0x40,0x41,0x42,0x43, 0x44,0x45,0x46,0x47
};
static const byte aad[12] = {
0x50,0x51,0x52,0x53, 0xc0,0xc1,0xc2,0xc3, 0xc4,0xc5,0xc6,0xc7
};
/* >= STITCH_MIN: exact 1024-multiples (4096/8192/16384) and tails
* (4097 -> 1-byte tail, 5000 -> 904-byte tail). */
static const word32 sizes[] = { 4096, 4097, 5000, 8192, 16384 };
static const word32 aadLens[] = { 0, 12 };
#define BIG_MSG_LEN 16384
byte* pt = NULL;
byte* ct = NULL;
byte* out = NULL;
byte tag[CHACHA20_POLY1305_AEAD_AUTHTAG_SIZE];
byte tag2[CHACHA20_POLY1305_AEAD_AUTHTAG_SIZE];
ChaCha chacha;
Poly1305 poly;
word32 a;
word32 s;
pt = (byte*)XMALLOC(BIG_MSG_LEN, NULL, DYNAMIC_TYPE_TMP_BUFFER);
ct = (byte*)XMALLOC(BIG_MSG_LEN, NULL, DYNAMIC_TYPE_TMP_BUFFER);
out = (byte*)XMALLOC(BIG_MSG_LEN, NULL, DYNAMIC_TYPE_TMP_BUFFER);
ExpectNotNull(pt);
ExpectNotNull(ct);
ExpectNotNull(out);
for (a = 0; a < (word32)(sizeof(aadLens) / sizeof(aadLens[0])); a++) {
word32 aadLen = aadLens[a];
for (s = 0; s < (word32)(sizeof(sizes) / sizeof(sizes[0])); s++) {
word32 sz = sizes[s];
word32 i;
if (pt == NULL || ct == NULL || out == NULL)
break;
for (i = 0; i < sz; i++)
pt[i] = (byte)(i * 3 + 1);
/* One-shot Encrypt: IFMA stitch when sz >= STITCH_MIN. */
XMEMSET(ct, 0, sz);
XMEMSET(tag, 0, sizeof(tag));
ExpectIntEQ(wc_ChaCha20Poly1305_Encrypt(key, iv, aad, aadLen,
pt, sz, ct, tag), 0);
/* Round-trip via the independent two-pass one-shot Decrypt: proves
* the encrypt stitch produced a correct ciphertext AND tag. */
XMEMSET(out, 0, sz);
ExpectIntEQ(wc_ChaCha20Poly1305_Decrypt(key, iv, aad, aadLen,
ct, sz, tag, out), 0);
ExpectBufEQ(out, pt, sz);
/* Encrypt_ex must reproduce the one-shot ciphertext + tag. */
ExpectIntEQ(wc_Chacha_SetKey(&chacha, key, sizeof(key)), 0);
XMEMSET(out, 0, sz);
XMEMSET(tag2, 0, sizeof(tag2));
ExpectIntEQ(wc_ChaCha20Poly1305_Encrypt_ex(&chacha, &poly, out, pt,
sz, iv, tag2, aad, aadLen), 0);
ExpectBufEQ(out, ct, sz);
ExpectBufEQ(tag2, tag, sizeof(tag2));
/* Decrypt_ex: the decrypt stitch (decrypt-then-verify) must recover
* the plaintext with a valid tag. */
ExpectIntEQ(wc_Chacha_SetKey(&chacha, key, sizeof(key)), 0);
XMEMSET(out, 0, sz);
ExpectIntEQ(wc_ChaCha20Poly1305_Decrypt_ex(&chacha, &poly, out, ct,
sz, iv, tag, aad, aadLen), 0);
ExpectBufEQ(out, pt, sz);
/* Bad tag: the stitch decrypts before checking the tag, so verify
* Decrypt_ex both reports the failure and zeroizes the output. */
tag[0] ^= 0xff;
ExpectIntEQ(wc_Chacha_SetKey(&chacha, key, sizeof(key)), 0);
XMEMSET(out, 0xa5, sz);
ExpectIntEQ(wc_ChaCha20Poly1305_Decrypt_ex(&chacha, &poly, out, ct,
sz, iv, tag, aad, aadLen),
WC_NO_ERR_TRACE(MAC_CMP_FAILED_E));
ExpectIntEQ(out[0], 0);
ExpectIntEQ(out[sz / 2], 0);
ExpectIntEQ(out[sz - 1], 0);
tag[0] ^= 0xff;
}
}
XFREE(pt, NULL, DYNAMIC_TYPE_TMP_BUFFER);
XFREE(ct, NULL, DYNAMIC_TYPE_TMP_BUFFER);
XFREE(out, NULL, DYNAMIC_TYPE_TMP_BUFFER);
#undef BIG_MSG_LEN
#endif
return EXPECT_RESULT();
} /* END test_wc_ChaCha20Poly1305_LargeMessage */
/*
* Small-message coverage WITH a non-empty AAD. The fused single-call asm
* kernels (chacha20_poly1305_small_enc / _dec) handle sz <= 64 on AVX2 CPUs and
* fold the AAD into Poly1305 inside assembly. No existing test drives that
* path with AAD: the KAT vectors are > 64 bytes, and MonteCarlo used NULL/0
* AAD - so the in-kernel AAD fold (the exact code the Windows stack-offset
* defects corrupt) was never executed. This is deterministic across the whole
* small band including the sz == 64 boundary.
*
* The streaming API does NOT use the short/small path, so it is an independent
* reference: small_enc must reproduce its ciphertext + tag byte-for-byte. The
* round-trip then recovers the plaintext via small_dec, and a corrupted tag
* must return MAC_CMP_FAILED_E with a fully zeroized output (small_dec decrypts
* before it verifies). On CPUs without AVX2 the same calls use the C fallback,
* so the test still validates but does not reach the asm kernel.
*/
int test_wc_ChaCha20Poly1305_SmallWithAad(void)
{
EXPECT_DECLS;
#if defined(HAVE_CHACHA) && defined(HAVE_POLY1305)
static const byte key[CHACHA20_POLY1305_AEAD_KEYSIZE] = {
0x80,0x81,0x82,0x83, 0x84,0x85,0x86,0x87,
0x88,0x89,0x8a,0x8b, 0x8c,0x8d,0x8e,0x8f,
0x90,0x91,0x92,0x93, 0x94,0x95,0x96,0x97,
0x98,0x99,0x9a,0x9b, 0x9c,0x9d,0x9e,0x9f
};
static const byte iv[CHACHA20_POLY1305_AEAD_IV_SIZE] = {
0x07,0x00,0x00,0x00, 0x40,0x41,0x42,0x43, 0x44,0x45,0x46,0x47
};
static const byte aad[20] = {
0x50,0x51,0x52,0x53, 0xc0,0xc1,0xc2,0xc3, 0xc4,0xc5,0xc6,0xc7,
0xf0,0xf1,0xf2,0xf3, 0xf4,0xf5,0xf6,0xf7
};
/* small band: the AVX2 kernel is used for sz <= 64 - cover 1, mid, the
* boundary at 64, and one just past it as a control. */
static const word32 sizes[] = { 1, 16, 32, 63, 64, 65 };
static const word32 aadLens[] = { 0, 1, 12, 20 };
byte pt[65];
byte ct[65];
byte ref[65];
byte back[65];
byte tag[CHACHA20_POLY1305_AEAD_AUTHTAG_SIZE];
byte tagRef[CHACHA20_POLY1305_AEAD_AUTHTAG_SIZE];
ChaChaPoly_Aead aead;
word32 a;
word32 s;
word32 i;
XMEMSET(&aead, 0, sizeof(aead));
for (a = 0; a < (word32)(sizeof(aadLens) / sizeof(aadLens[0])); a++) {
word32 aadLen = aadLens[a];
for (s = 0; s < (word32)(sizeof(sizes) / sizeof(sizes[0])); s++) {
word32 sz = sizes[s];
for (i = 0; i < sz; i++)
pt[i] = (byte)(i * 7 + 2);
/* One-shot Encrypt: sz <= 64 dispatches to small_enc, which folds
* the AAD in asm. */
XMEMSET(ct, 0, sizeof(ct));
XMEMSET(tag, 0, sizeof(tag));
ExpectIntEQ(wc_ChaCha20Poly1305_Encrypt(key, iv, aad, aadLen,
pt, sz, ct, tag), 0);
/* Independent reference via the streaming API (never the small
* path): validates small_enc + AAD fold against the two-pass. */
ExpectIntEQ(wc_ChaCha20Poly1305_Init(&aead, key, iv,
CHACHA20_POLY1305_AEAD_ENCRYPT), 0);
if (aadLen > 0)
ExpectIntEQ(wc_ChaCha20Poly1305_UpdateAad(&aead, aad, aadLen),
0);
XMEMSET(ref, 0, sizeof(ref));
ExpectIntEQ(wc_ChaCha20Poly1305_UpdateData(&aead, pt, ref, sz), 0);
ExpectIntEQ(wc_ChaCha20Poly1305_Final(&aead, tagRef), 0);
ExpectBufEQ(ct, ref, sz);
ExpectBufEQ(tag, tagRef, sizeof(tag));
/* Round-trip: small_dec recovers the plaintext from the reference
* ciphertext. */
XMEMSET(back, 0, sizeof(back));
ExpectIntEQ(wc_ChaCha20Poly1305_Decrypt(key, iv, aad, aadLen,
ref, sz, tagRef, back), 0);
ExpectBufEQ(back, pt, sz);
/* Bad tag: small_dec decrypts before verifying, so Decrypt must
* both report the failure and zeroize the whole output. */
tag[0] ^= 0xff;
XMEMSET(back, 0xa5, sizeof(back));
ExpectIntEQ(wc_ChaCha20Poly1305_Decrypt(key, iv, aad, aadLen,
ct, sz, tag, back), WC_NO_ERR_TRACE(MAC_CMP_FAILED_E));
for (i = 0; i < sz; i++)
ExpectIntEQ(back[i], 0);
tag[0] ^= 0xff;
}
}
#endif
return EXPECT_RESULT();
} /* END test_wc_ChaCha20Poly1305_SmallWithAad */
#if defined(HAVE_CHACHA) && defined(HAVE_POLY1305)
/* Streaming AEAD over one message, splitting the data into a first chunk of
* 'first' bytes then 'rest'-byte chunks (both clamped to what remains). enc
* selects encrypt vs decrypt; the computed/authentication tag is returned in
* 'tag' (for decrypt the caller compares it against the received tag). */
static int cp_stream(int enc, const byte* key, const byte* iv, const byte* aad,
word32 aadLen, const byte* in, word32 sz, word32 first, word32 rest,
byte* out, byte* tag)
{
ChaChaPoly_Aead aead;
word32 off;
word32 n;
int ret;
XMEMSET(&aead, 0, sizeof(aead));
ret = wc_ChaCha20Poly1305_Init(&aead, key, iv, enc ?
CHACHA20_POLY1305_AEAD_ENCRYPT : CHACHA20_POLY1305_AEAD_DECRYPT);
if (ret == 0 && aadLen > 0)
ret = wc_ChaCha20Poly1305_UpdateAad(&aead, aad, aadLen);
off = 0;
while (ret == 0 && off < sz) {
n = (off == 0) ? first : rest;
if (n > sz - off)
n = sz - off;
ret = wc_ChaCha20Poly1305_UpdateData(&aead, in + off, out + off, n);
off += n;
}
if (ret == 0)
ret = wc_ChaCha20Poly1305_Final(&aead, tag);
return ret;
}
#endif
/*
* Large-message coverage of the streaming UpdateData IFMA stitch. UpdateData
* grows two AVX-512/IFMA paths that both need dataLen >= STITCH_MIN (4096): the
* first-chunk stitch entry - which, when AAD was buffered by the vector path
* (< 128 bytes, leftover > 0), COPIES it to a 128-byte stack buffer and
* re-hashes it through the scalar path - and the per-chunk stitch gated on
* (dataLen & 63) == 0. Every existing streaming test uses <= 64-byte chunks,
* so none of this ran. The buffered-AAD re-hash is the highest-value case: a
* leftover/pad miscount there yields a wrong tag.
*
* The streaming API is chunk-invariant by contract, so 256-byte-chunk streaming
* (which never reaches STITCH_MIN, hence pure two-pass) is the reference every
* stitched chunking must match. Splits exercise: a single large call (entry +
* AAD re-hash + full bulk); a 4096 first chunk (2nd chunk 64-aligned -> the
* per-chunk stitch fires); and a 4128 first chunk (2nd chunk NOT 64-aligned ->
* the guard's false branch, two-pass). AAD lengths cover none, buffered
* (12/120 -> re-hash) and >= 128 (200 -> vector-processed, stitch entry
* blocked). A two-pass one-shot Decrypt independently round-trips each result.
*/
int test_wc_ChaCha20Poly1305_StreamLarge(void)
{
EXPECT_DECLS;
#if defined(HAVE_CHACHA) && defined(HAVE_POLY1305)
static const byte key[CHACHA20_POLY1305_AEAD_KEYSIZE] = {
0x80,0x81,0x82,0x83, 0x84,0x85,0x86,0x87,
0x88,0x89,0x8a,0x8b, 0x8c,0x8d,0x8e,0x8f,
0x90,0x91,0x92,0x93, 0x94,0x95,0x96,0x97,
0x98,0x99,0x9a,0x9b, 0x9c,0x9d,0x9e,0x9f
};
static const byte iv[CHACHA20_POLY1305_AEAD_IV_SIZE] = {
0x07,0x00,0x00,0x00, 0x40,0x41,0x42,0x43, 0x44,0x45,0x46,0x47
};
byte aad[200];
static const word32 aadLens[] = { 0, 12, 120, 200 };
/* { first-chunk, rest-chunk } data splits. */
static const word32 splits[][2] = {
{ 12288, 12288 }, /* one UpdateData call */
{ 4096, 12288 }, /* 64-aligned first chunk -> 2nd chunk stitches */
{ 4128, 12288 } /* unaligned first chunk -> 2nd chunk two-pass */
};
#define SL_LEN 12288
byte* pt = NULL;
byte* ct = NULL;
byte* ref = NULL;
byte* back = NULL;
byte tag[CHACHA20_POLY1305_AEAD_AUTHTAG_SIZE];
byte tagRef[CHACHA20_POLY1305_AEAD_AUTHTAG_SIZE];
byte calc[CHACHA20_POLY1305_AEAD_AUTHTAG_SIZE];
word32 a;
word32 sp;
word32 i;
for (i = 0; i < sizeof(aad); i++)
aad[i] = (byte)(i + 0x30);
pt = (byte*)XMALLOC(SL_LEN, NULL, DYNAMIC_TYPE_TMP_BUFFER);
ct = (byte*)XMALLOC(SL_LEN, NULL, DYNAMIC_TYPE_TMP_BUFFER);
ref = (byte*)XMALLOC(SL_LEN, NULL, DYNAMIC_TYPE_TMP_BUFFER);
back = (byte*)XMALLOC(SL_LEN, NULL, DYNAMIC_TYPE_TMP_BUFFER);
ExpectNotNull(pt);
ExpectNotNull(ct);
ExpectNotNull(ref);
ExpectNotNull(back);
if (pt != NULL && ct != NULL && ref != NULL && back != NULL) {
for (i = 0; i < SL_LEN; i++)
pt[i] = (byte)(i * 5 + 3);
for (a = 0; a < (word32)(sizeof(aadLens) / sizeof(aadLens[0]))
&& EXPECT_SUCCESS(); a++) {
word32 aadLen = aadLens[a];
/* Reference: 256-byte chunks stay below STITCH_MIN -> two-pass. */
XMEMSET(ref, 0, SL_LEN);
ExpectIntEQ(cp_stream(1, key, iv, aad, aadLen, pt, SL_LEN, 256, 256,
ref, tagRef), 0);
for (sp = 0; sp < (word32)(sizeof(splits) / sizeof(splits[0]))
&& EXPECT_SUCCESS(); sp++) {
/* Stitched streaming encrypt must match the two-pass ref. */
XMEMSET(ct, 0, SL_LEN);
ExpectIntEQ(cp_stream(1, key, iv, aad, aadLen, pt, SL_LEN,
splits[sp][0], splits[sp][1], ct, tag), 0);
ExpectBufEQ(ct, ref, SL_LEN);
ExpectBufEQ(tag, tagRef, sizeof(tag));
/* Streaming decrypt (decrypt stitch) recovers the plaintext and
* computes the matching tag. */
XMEMSET(back, 0, SL_LEN);
ExpectIntEQ(cp_stream(0, key, iv, aad, aadLen, ct, SL_LEN,
splits[sp][0], splits[sp][1], back, calc), 0);
ExpectIntEQ(wc_ChaCha20Poly1305_CheckTag(tag, calc), 0);
ExpectBufEQ(back, pt, SL_LEN);
}
/* Independent: the two-pass one-shot Decrypt round-trips it. */
XMEMSET(back, 0, SL_LEN);
ExpectIntEQ(wc_ChaCha20Poly1305_Decrypt(key, iv, aad, aadLen,
ref, SL_LEN, tagRef, back), 0);
ExpectBufEQ(back, pt, SL_LEN);
}
}
XFREE(pt, NULL, DYNAMIC_TYPE_TMP_BUFFER);
XFREE(ct, NULL, DYNAMIC_TYPE_TMP_BUFFER);
XFREE(ref, NULL, DYNAMIC_TYPE_TMP_BUFFER);
XFREE(back, NULL, DYNAMIC_TYPE_TMP_BUFFER);
#undef SL_LEN
#endif
return EXPECT_RESULT();
} /* END test_wc_ChaCha20Poly1305_StreamLarge */
/*
* Direct, full-coverage test of the new pre-keyed one-shot APIs
* wc_ChaCha20Poly1305_Encrypt_ex / _Decrypt_ex (the TLS-record analogue of
* wc_AesGcmEncrypt/Decrypt on a keyed context). Exercises every dispatch band
* the _ex path selects on - the sz <= 64 small asm kernel, the 64 < sz <= 192
* short C path, the 192 < sz < 4096 two-pass, and the sz >= 4096 IFMA stitch
* (+ sub-1024 tail) - each with AAD absent, buffered, and larger, and both
* separate and in-place (out == in) buffers. Correctness is anchored to the
* trusted one-shot wc_ChaCha20Poly1305_Encrypt, which _ex must reproduce
* byte-for-byte regardless of which internal path either takes. Also covers
* Decrypt_ex tag verification + output zeroization on a bad tag, the keyed-
* context reuse pattern (SetKey once, vary the nonce per record), and the full
* argument-validation matrix.
*/
int test_wc_ChaCha20Poly1305_Ex(void)
{
EXPECT_DECLS;
#if defined(HAVE_CHACHA) && defined(HAVE_POLY1305)
static const byte key[CHACHA20_POLY1305_AEAD_KEYSIZE] = {
0x80,0x81,0x82,0x83, 0x84,0x85,0x86,0x87,
0x88,0x89,0x8a,0x8b, 0x8c,0x8d,0x8e,0x8f,
0x90,0x91,0x92,0x93, 0x94,0x95,0x96,0x97,
0x98,0x99,0x9a,0x9b, 0x9c,0x9d,0x9e,0x9f
};
static const byte iv[CHACHA20_POLY1305_AEAD_IV_SIZE] = {
0x07,0x00,0x00,0x00, 0x40,0x41,0x42,0x43, 0x44,0x45,0x46,0x47
};
byte aad[64];
/* one size in each _ex dispatch band, plus the band boundaries. */
static const word32 sizes[] =
{ 0, 1, 64, 100, 192, 193, 1024, 4096, 5000 };
static const word32 aadLens[] = { 0, 12, 64 };
#define EX_MAX 5000
ChaCha chacha;
Poly1305 poly;
byte* pt = NULL;
byte* ct = NULL;
byte* ref = NULL;
byte* out = NULL;
byte* tmp = NULL;
byte tag[CHACHA20_POLY1305_AEAD_AUTHTAG_SIZE];
byte tagRef[CHACHA20_POLY1305_AEAD_AUTHTAG_SIZE];
word32 a;
word32 s;
word32 i;
for (i = 0; i < sizeof(aad); i++)
aad[i] = (byte)(i + 0xa0);
pt = (byte*)XMALLOC(EX_MAX, NULL, DYNAMIC_TYPE_TMP_BUFFER);
ct = (byte*)XMALLOC(EX_MAX, NULL, DYNAMIC_TYPE_TMP_BUFFER);
ref = (byte*)XMALLOC(EX_MAX, NULL, DYNAMIC_TYPE_TMP_BUFFER);
out = (byte*)XMALLOC(EX_MAX, NULL, DYNAMIC_TYPE_TMP_BUFFER);
tmp = (byte*)XMALLOC(EX_MAX, NULL, DYNAMIC_TYPE_TMP_BUFFER);
ExpectNotNull(pt);
ExpectNotNull(ct);
ExpectNotNull(ref);
ExpectNotNull(out);
ExpectNotNull(tmp);
if (pt != NULL && ct != NULL && ref != NULL && out != NULL &&
tmp != NULL) {
for (i = 0; i < EX_MAX; i++)
pt[i] = (byte)(i * 11 + 5);
for (a = 0; a < (word32)(sizeof(aadLens) / sizeof(aadLens[0]))
&& EXPECT_SUCCESS(); a++) {
word32 aadLen = aadLens[a];
for (s = 0; s < (word32)(sizeof(sizes) / sizeof(sizes[0]))
&& EXPECT_SUCCESS(); s++) {
word32 sz = sizes[s];
/* Encrypt_ex on a pre-keyed context. */
ExpectIntEQ(wc_Chacha_SetKey(&chacha, key, sizeof(key)), 0);
XMEMSET(ct, 0, EX_MAX);
XMEMSET(tag, 0, sizeof(tag));
ExpectIntEQ(wc_ChaCha20Poly1305_Encrypt_ex(&chacha, &poly, ct,
pt, sz, iv, tag, aad, aadLen), 0);
/* Must match the trusted one-shot Encrypt byte-for-byte. */
XMEMSET(ref, 0, EX_MAX);
ExpectIntEQ(wc_ChaCha20Poly1305_Encrypt(key, iv, aad, aadLen,
pt, sz, ref, tagRef), 0);
ExpectBufEQ(ct, ref, sz);
ExpectBufEQ(tag, tagRef, sizeof(tag));
/* Encrypt_ex in place (out == in). */
XMEMCPY(tmp, pt, sz);
ExpectIntEQ(wc_Chacha_SetKey(&chacha, key, sizeof(key)), 0);
ExpectIntEQ(wc_ChaCha20Poly1305_Encrypt_ex(&chacha, &poly, tmp,
tmp, sz, iv, tag, aad, aadLen), 0);
ExpectBufEQ(tmp, ct, sz);
/* Decrypt_ex round-trip, separate buffers. */
ExpectIntEQ(wc_Chacha_SetKey(&chacha, key, sizeof(key)), 0);
XMEMSET(out, 0, EX_MAX);
ExpectIntEQ(wc_ChaCha20Poly1305_Decrypt_ex(&chacha, &poly, out,
ct, sz, iv, tag, aad, aadLen), 0);
ExpectBufEQ(out, pt, sz);
/* Decrypt_ex in place (out == in). */
XMEMCPY(tmp, ct, sz);
ExpectIntEQ(wc_Chacha_SetKey(&chacha, key, sizeof(key)), 0);
ExpectIntEQ(wc_ChaCha20Poly1305_Decrypt_ex(&chacha, &poly, tmp,
tmp, sz, iv, tag, aad, aadLen), 0);
ExpectBufEQ(tmp, pt, sz);
/* Bad tag: MAC_CMP_FAILED_E and the whole output zeroized. */
tag[0] ^= 0xff;
ExpectIntEQ(wc_Chacha_SetKey(&chacha, key, sizeof(key)), 0);
XMEMSET(out, 0xa5, EX_MAX);
ExpectIntEQ(wc_ChaCha20Poly1305_Decrypt_ex(&chacha, &poly, out,
ct, sz, iv, tag, aad, aadLen),
WC_NO_ERR_TRACE(MAC_CMP_FAILED_E));
for (i = 0; i < sz; i++)
ExpectIntEQ(out[i], 0);
tag[0] ^= 0xff;
}
}
/* Keyed-context reuse: SetKey once, encrypt several records that differ
* only by nonce (the intended TLS usage), each decrypting back. */
ExpectIntEQ(wc_Chacha_SetKey(&chacha, key, sizeof(key)), 0);
for (i = 0; i < 4 && EXPECT_SUCCESS(); i++) {
byte nonce[CHACHA20_POLY1305_AEAD_IV_SIZE];
ChaCha chachaDec;
word32 sz = 200 + i * 37;
XMEMCPY(nonce, iv, sizeof(nonce));
nonce[0] = (byte)i; /* vary the nonce per record */
XMEMSET(ct, 0, EX_MAX);
ExpectIntEQ(wc_ChaCha20Poly1305_Encrypt_ex(&chacha, &poly, ct, pt,
sz, nonce, tag, aad, 12), 0);
ExpectIntEQ(wc_Chacha_SetKey(&chachaDec, key, sizeof(key)), 0);
XMEMSET(out, 0, EX_MAX);
ExpectIntEQ(wc_ChaCha20Poly1305_Decrypt_ex(&chachaDec, &poly, out,
ct, sz, nonce, tag, aad, 12), 0);
ExpectBufEQ(out, pt, sz);
}
/* Argument validation - Encrypt_ex. */
ExpectIntEQ(wc_Chacha_SetKey(&chacha, key, sizeof(key)), 0);
ExpectIntEQ(wc_ChaCha20Poly1305_Encrypt_ex(NULL, &poly, ct, pt, 64,
iv, tag, aad, 0), WC_NO_ERR_TRACE(BAD_FUNC_ARG));
ExpectIntEQ(wc_ChaCha20Poly1305_Encrypt_ex(&chacha, NULL, ct, pt, 64,
iv, tag, aad, 0), WC_NO_ERR_TRACE(BAD_FUNC_ARG));
ExpectIntEQ(wc_ChaCha20Poly1305_Encrypt_ex(&chacha, &poly, ct, pt, 64,
NULL, tag, aad, 0), WC_NO_ERR_TRACE(BAD_FUNC_ARG));
ExpectIntEQ(wc_ChaCha20Poly1305_Encrypt_ex(&chacha, &poly, ct, pt, 64,
iv, NULL, aad, 0), WC_NO_ERR_TRACE(BAD_FUNC_ARG));
ExpectIntEQ(wc_ChaCha20Poly1305_Encrypt_ex(&chacha, &poly, ct, NULL, 64,
iv, tag, aad, 0), WC_NO_ERR_TRACE(BAD_FUNC_ARG));
ExpectIntEQ(wc_ChaCha20Poly1305_Encrypt_ex(&chacha, &poly, NULL, pt, 64,
iv, tag, aad, 0), WC_NO_ERR_TRACE(BAD_FUNC_ARG));
ExpectIntEQ(wc_ChaCha20Poly1305_Encrypt_ex(&chacha, &poly, ct, pt, 64,
iv, tag, NULL, 12), WC_NO_ERR_TRACE(BAD_FUNC_ARG));
/* Argument validation - Decrypt_ex. */
ExpectIntEQ(wc_ChaCha20Poly1305_Decrypt_ex(NULL, &poly, out, ct, 64,
iv, tag, aad, 0), WC_NO_ERR_TRACE(BAD_FUNC_ARG));
ExpectIntEQ(wc_ChaCha20Poly1305_Decrypt_ex(&chacha, NULL, out, ct, 64,
iv, tag, aad, 0), WC_NO_ERR_TRACE(BAD_FUNC_ARG));
ExpectIntEQ(wc_ChaCha20Poly1305_Decrypt_ex(&chacha, &poly, out, ct, 64,
NULL, tag, aad, 0), WC_NO_ERR_TRACE(BAD_FUNC_ARG));
ExpectIntEQ(wc_ChaCha20Poly1305_Decrypt_ex(&chacha, &poly, out, ct, 64,
iv, NULL, aad, 0), WC_NO_ERR_TRACE(BAD_FUNC_ARG));
ExpectIntEQ(wc_ChaCha20Poly1305_Decrypt_ex(&chacha, &poly, out, NULL,
64, iv, tag, aad, 0), WC_NO_ERR_TRACE(BAD_FUNC_ARG));
ExpectIntEQ(wc_ChaCha20Poly1305_Decrypt_ex(&chacha, &poly, NULL, ct, 64,
iv, tag, aad, 0), WC_NO_ERR_TRACE(BAD_FUNC_ARG));
ExpectIntEQ(wc_ChaCha20Poly1305_Decrypt_ex(&chacha, &poly, out, ct, 64,
iv, tag, NULL, 12), WC_NO_ERR_TRACE(BAD_FUNC_ARG));
}
XFREE(pt, NULL, DYNAMIC_TYPE_TMP_BUFFER);
XFREE(ct, NULL, DYNAMIC_TYPE_TMP_BUFFER);
XFREE(ref, NULL, DYNAMIC_TYPE_TMP_BUFFER);
XFREE(out, NULL, DYNAMIC_TYPE_TMP_BUFFER);
XFREE(tmp, NULL, DYNAMIC_TYPE_TMP_BUFFER);
#undef EX_MAX
#endif
return EXPECT_RESULT();
} /* END test_wc_ChaCha20Poly1305_Ex */
tests/api/test_chacha20_poly1305.h
+8
View File
@@ -36,6 +36,10 @@ int test_wc_ChaCha20Poly1305_InPlace(void);
int test_wc_ChaCha20Poly1305_UnalignedBuffers(void);
int test_wc_ChaCha20Poly1305_CrossCipher(void);
int test_wc_ChaCha20Poly1305_DecisionCoverage(void);
int test_wc_ChaCha20Poly1305_LargeMessage(void);
int test_wc_ChaCha20Poly1305_SmallWithAad(void);
int test_wc_ChaCha20Poly1305_StreamLarge(void);
int test_wc_ChaCha20Poly1305_Ex(void);
int test_wc_XChaCha20Poly1305_DecisionCoverage(void);
int test_wc_XChaCha20Poly1305_LargeBuffer(void);
@@ -52,6 +56,10 @@ int test_wc_XChaCha20Poly1305_LargeBuffer(void);
TEST_DECL_GROUP("chacha20-poly1305", test_wc_ChaCha20Poly1305_UnalignedBuffers), \
TEST_DECL_GROUP("chacha20-poly1305", test_wc_ChaCha20Poly1305_CrossCipher), \
TEST_DECL_GROUP("chacha20-poly1305", test_wc_ChaCha20Poly1305_DecisionCoverage), \
TEST_DECL_GROUP("chacha20-poly1305", test_wc_ChaCha20Poly1305_LargeMessage), \
TEST_DECL_GROUP("chacha20-poly1305", test_wc_ChaCha20Poly1305_SmallWithAad), \
TEST_DECL_GROUP("chacha20-poly1305", test_wc_ChaCha20Poly1305_StreamLarge), \
TEST_DECL_GROUP("chacha20-poly1305", test_wc_ChaCha20Poly1305_Ex), \
TEST_DECL_GROUP("xchacha20-poly1305", test_wc_XChaCha20Poly1305_DecisionCoverage), \
TEST_DECL_GROUP("xchacha20-poly1305", test_wc_XChaCha20Poly1305_LargeBuffer)
wolfcrypt/benchmark/benchmark.c
+161 -1
View File
@@ -7732,6 +7732,9 @@ void bench_chacha20_poly1305_aead(void)
{
double start;
int ret = 0, i, count;
ChaCha chacha; /* keyed once, reused per record: the TLS-record path */
Poly1305 poly;
byte nonce[CHACHA20_POLY1305_AEAD_IV_SIZE];
DECLARE_MULTI_VALUE_STATS_VARS()
WC_DECLARE_VAR(bench_additional, byte, AES_AUTH_ADD_SZ, HEAP_HINT);
@@ -7743,7 +7746,9 @@ void bench_chacha20_poly1305_aead(void)
WC_ALLOC_VAR(authTag, byte, CHACHA20_POLY1305_AEAD_AUTHTAG_SIZE, HEAP_HINT);
XMEMSET(bench_additional, 0, AES_AUTH_ADD_SZ);
XMEMSET(authTag, 0, CHACHA20_POLY1305_AEAD_AUTHTAG_SIZE);
XMEMSET(nonce, 0, sizeof(nonce));
/* One-shot encrypt (re-keys per call). */
bench_stats_start(&count, &start);
do {
for (i = 0; i < numBlocks; i++) {
@@ -7763,7 +7768,162 @@ void bench_chacha20_poly1305_aead(void)
#endif
);
bench_stats_sym_finish("CHA-POLY", 0, count, bench_size, start, ret);
bench_stats_sym_finish("CHA-POLY-enc", 0, count, bench_size, start, ret);
#ifdef MULTI_VALUE_STATISTICS
bench_multi_value_stats(max, min, sum, squareSum, runs);
#endif
RESET_MULTI_VALUE_STATS_VARS();
/* Produce a valid ciphertext+tag once for the decrypt benchmarks. */
ret = wc_ChaCha20Poly1305_Encrypt(bench_key, bench_iv, bench_additional,
aesAuthAddSz, bench_plain, bench_size, bench_cipher, authTag);
if (ret < 0) {
printf("wc_ChaCha20Poly1305_Encrypt error: %d\n", ret);
goto exit;
}
/* One-shot verify+decrypt. */
bench_stats_start(&count, &start);
do {
for (i = 0; i < numBlocks; i++) {
ret = wc_ChaCha20Poly1305_Decrypt(bench_key, bench_iv,
bench_additional, aesAuthAddSz, bench_cipher, bench_size,
authTag, bench_plain);
if (ret < 0) {
printf("wc_ChaCha20Poly1305_Decrypt error: %d\n", ret);
goto exit;
}
RECORD_MULTI_VALUE_STATS();
}
count += i;
} while (bench_stats_check(start)
#ifdef MULTI_VALUE_STATISTICS
|| runs < minimum_runs
#endif
);
bench_stats_sym_finish("CHA-POLY-dec", 0, count, bench_size, start, ret);
#ifdef MULTI_VALUE_STATISTICS
bench_multi_value_stats(max, min, sum, squareSum, runs);
#endif
RESET_MULTI_VALUE_STATS_VARS();
/* TLS-record path: ChaCha keyed once, only the nonce varies per record;
* Encrypt_ex/Decrypt_ex use the single-pass stitch (no per-record
* re-key). */
ret = wc_Chacha_SetKey(&chacha, bench_key,
CHACHA20_POLY1305_AEAD_KEYSIZE);
if (ret != 0) {
printf("wc_Chacha_SetKey error: %d\n", ret);
goto exit;
}
bench_stats_start(&count, &start);
do {
for (i = 0; i < numBlocks; i++) {
ret = wc_ChaCha20Poly1305_Encrypt_ex(&chacha, &poly, bench_cipher,
bench_plain, bench_size, nonce, authTag, bench_additional,
aesAuthAddSz);
if (ret < 0) {
printf("wc_ChaCha20Poly1305_Encrypt_ex error: %d\n", ret);
goto exit;
}
RECORD_MULTI_VALUE_STATS();
}
count += i;
} while (bench_stats_check(start)
#ifdef MULTI_VALUE_STATISTICS
|| runs < minimum_runs
#endif
);
bench_stats_sym_finish("CHA-POLY-ex-enc", 0, count, bench_size, start, ret);
#ifdef MULTI_VALUE_STATISTICS
bench_multi_value_stats(max, min, sum, squareSum, runs);
#endif
RESET_MULTI_VALUE_STATS_VARS();
/* Valid ciphertext+tag for the Decrypt_ex benchmark. */
ret = wc_ChaCha20Poly1305_Encrypt_ex(&chacha, &poly, bench_cipher,
bench_plain, bench_size, nonce, authTag, bench_additional,
aesAuthAddSz);
if (ret < 0) {
printf("wc_ChaCha20Poly1305_Encrypt_ex error: %d\n", ret);
goto exit;
}
bench_stats_start(&count, &start);
do {
for (i = 0; i < numBlocks; i++) {
ret = wc_ChaCha20Poly1305_Decrypt_ex(&chacha, &poly, bench_plain,
bench_cipher, bench_size, nonce, authTag, bench_additional,
aesAuthAddSz);
if (ret < 0) {
printf("wc_ChaCha20Poly1305_Decrypt_ex error: %d\n", ret);
goto exit;
}
RECORD_MULTI_VALUE_STATS();
}
count += i;
} while (bench_stats_check(start)
#ifdef MULTI_VALUE_STATISTICS
|| runs < minimum_runs
#endif
);
bench_stats_sym_finish("CHA-POLY-ex-dec", 0, count, bench_size, start, ret);
#ifdef MULTI_VALUE_STATISTICS
bench_multi_value_stats(max, min, sum, squareSum, runs);
#endif
RESET_MULTI_VALUE_STATS_VARS();
/* Streaming AEAD interface over the FUSED stitch, "openssl speed -aead"
* methodology: Init once (framing amortized), then loop UpdateData with the
* whole buffer and NO per-chunk tag/Final. wc_ChaCha20Poly1305_UpdateData
* dispatches to the single-pass IFMA stitch for each 1024-aligned chunk
* (>= CHACHA20_POLY1305_STITCH_MIN), so this is wolfSSL's fused cipher
* throughput measured the same bare-loop way OpenSSL's cipher is - unlike a
* two-pass Process+Update, and reachable through the public API. Below the
* stitch threshold (or a non-64-aligned bench_size) UpdateData stays
* two-pass, which is the interface's real behaviour. Re-Init before the
* running dataLen would overflow CHACHA20_POLY1305_MAX. */
{
ChaChaPoly_Aead sAead;
XMEMSET(&sAead, 0, sizeof(sAead));
ret = wc_ChaCha20Poly1305_Init(&sAead, bench_key, nonce, 1);
if (ret != 0) {
printf("chacha20-poly1305 stream Init error: %d\n", ret);
goto exit;
}
bench_stats_start(&count, &start);
do {
for (i = 0; i < numBlocks; i++) {
if (sAead.dataLen > CHACHA20_POLY1305_MAX - bench_size) {
XMEMSET(&sAead, 0, sizeof(sAead));
ret = wc_ChaCha20Poly1305_Init(&sAead, bench_key, nonce, 1);
if (ret != 0) {
printf("chacha20-poly1305 stream Init error: %d\n", ret);
goto exit;
}
}
ret = wc_ChaCha20Poly1305_UpdateData(&sAead, bench_plain,
bench_cipher, bench_size);
if (ret < 0) {
printf("chacha20-poly1305 stream error: %d\n", ret);
goto exit;
}
RECORD_MULTI_VALUE_STATS();
}
count += i;
} while (bench_stats_check(start)
#ifdef MULTI_VALUE_STATISTICS
|| runs < minimum_runs
#endif
);
bench_stats_sym_finish("CHA-POLY-stream", 0, count, bench_size, start, ret);
}
#ifdef MULTI_VALUE_STATISTICS
bench_multi_value_stats(max, min, sum, squareSum, runs);
#endif
wolfcrypt/src/chacha.c
+122
View File
@@ -100,6 +100,13 @@ Public domain.
#ifndef NO_AVX2_SUPPORT
#define HAVE_INTEL_AVX2
#endif
#if !defined(NO_AVX512_SUPPORT) && !defined(HAVE_INTEL_AVX512)
#define HAVE_INTEL_AVX512
#endif
/* SSSE3 is the baseline SIMD path, used on CPUs that lack AVX. */
#ifndef HAVE_INTEL_SSSE3
#define HAVE_INTEL_SSSE3
#endif
static cpuid_flags_t cpuidFlags = WC_CPUID_INITIALIZER;
#endif
@@ -318,11 +325,51 @@ extern void chacha_encrypt_avx1(ChaCha* ctx, const byte* m, byte* c,
word32 bytes);
extern void chacha_encrypt_avx2(ChaCha* ctx, const byte* m, byte* c,
word32 bytes);
extern void chacha_encrypt_avx512(ChaCha* ctx, const byte* m, byte* c,
word32 bytes);
extern void chacha_encrypt_avx512vl(ChaCha* ctx, const byte* m, byte* c,
word32 bytes);
extern void chacha_encrypt_sse3(ChaCha* ctx, const byte* m, byte* c,
word32 bytes);
#ifdef __cplusplus
} /* extern "C" */
#endif
#if defined(USE_INTEL_CHACHA_SPEEDUP) && defined(HAVE_INTEL_AVX512)
/* Decide whether to use the 512-bit (zmm) ChaCha path for this CPU.
*
* The zmm path processes 16 blocks at a time and is the fastest option on
* microarchitectures that run 512-bit code at full clock: AMD Zen 4/5 (no
* AVX-512 license) and Intel Ice Lake and later. On Intel Skylake-SP /
* Cascade Lake-class parts, sustained 512-bit instructions trip the AVX-512
* frequency license and downclock the core - enough that the 256-bit AVX2 path
* is faster in practice (this matches OpenSSL, which suppresses its 16x zmm
* ChaCha there, and the Linux kernel, which uses only 256-bit AVX-512VL).
*
* There is no direct "does this core downclock" CPUID bit, so VAES presence is
* used as a generational proxy: the throttling parts (Skylake-SP / Skylake-X /
* Cascade Lake) predate VAES, whereas every microarchitecture that runs 512-bit
* without penalty (AMD Zen 4/5, Intel Ice Lake+) implements it. A missing VAES
* only costs a little throughput (fall back to AVX2), never correctness.
*
* Override the heuristic with:
* WOLFSSL_CHACHA20_AVX512_ALWAYS - use zmm whenever AVX-512 is present
* WOLFSSL_CHACHA20_AVX512_NEVER - never use zmm (always AVX2 or below)
*/
static WC_INLINE int chacha_avx512_beneficial(cpuid_flags_t flags)
{
#if defined(WOLFSSL_CHACHA20_AVX512_NEVER)
(void)flags;
return 0;
#elif defined(WOLFSSL_CHACHA20_AVX512_ALWAYS)
return IS_INTEL_AVX512(flags) != 0;
#else
return (IS_INTEL_AVX512(flags) != 0) && (IS_INTEL_VAES(flags) != 0);
#endif
}
#endif /* USE_INTEL_CHACHA_SPEEDUP && HAVE_INTEL_AVX512 */
#if (!defined(USE_INTEL_CHACHA_SPEEDUP) && !defined(USE_ARM_CHACHA_SPEEDUP) && \
!defined(USE_RISCV_CHACHA_SPEEDUP)) || defined(WOLFSSL_ARM_CHACHA_NEED_C)
@@ -425,6 +472,73 @@ int wc_Chacha_Process(ChaCha* ctx, byte* output, const byte* input,
cpuid_get_flags_ex(&cpuidFlags);
/* One block or less. */
#if defined(HAVE_INTEL_AVX1) && !defined(WOLFSSL_LINUXKM)
/* In userspace SAVE_VECTOR_REGISTERS is free, so a single AVX block (~285
* cyc) beats the scalar block (~435) - e.g. the per-record Poly1305 key
* derivation (a 32-byte ChaCha) in the ChaCha20-Poly1305 two-pass path.
* The AVX-512VL path already uses SIMD for one block; match that here. */
if (msglen <= CHACHA_CHUNK_BYTES && IS_INTEL_AVX512_VL(cpuidFlags) == 0 &&
IS_INTEL_AVX1(cpuidFlags)) {
SAVE_VECTOR_REGISTERS(return _svr_ret;);
chacha_encrypt_avx1(ctx, input, output, msglen);
RESTORE_VECTOR_REGISTERS();
return 0;
}
#endif
/* At most one block: the scalar path avoids the SIMD broadcast/transpose
* setup and (in the Linux kernel module) the costly vector-register
* save/restore. */
if (msglen <= CHACHA_CHUNK_BYTES) {
chacha_encrypt_x64(ctx, input, output, msglen);
return 0;
}
/* 65..255 bytes without AVX-512VL: use the SSSE3 128-bit exact-block path.
* It is ~1.8x the scalar path and beats the 8-block AVX2 kernel (which
* always emits a full 512-byte key stream) below 256 bytes - e.g. a
* 192-byte key stream is 735 vs 1335 (scalar) vs 836 (AVX2) cycles on
* Coffee Lake. This is the ChaCha20-Poly1305 short-record hot path (poly
* key + <=2 data blocks). At >=256 bytes the four-block AVX2/AVX1 kernels
* take over below. */
#ifdef HAVE_INTEL_SSSE3
if (IS_INTEL_AVX512_VL(cpuidFlags) == 0 &&
msglen < 4 * CHACHA_CHUNK_BYTES &&
IS_INTEL_SSSE3(cpuidFlags)) {
SAVE_VECTOR_REGISTERS(return _svr_ret;);
chacha_encrypt_sse3(ctx, input, output, msglen);
RESTORE_VECTOR_REGISTERS();
return 0;
}
#endif
if (IS_INTEL_AVX512_VL(cpuidFlags) == 0 &&
msglen < 4 * CHACHA_CHUNK_BYTES) {
chacha_encrypt_x64(ctx, input, output, msglen);
return 0;
}
#ifdef HAVE_INTEL_AVX512
/* Below one 16-block chunk (1024 bytes) the zmm path does no work and
* just tail-calls AVX2, so dispatch straight to AVX2 for smaller input. */
if (chacha_avx512_beneficial(cpuidFlags) &&
msglen >= 16 * CHACHA_CHUNK_BYTES) {
SAVE_VECTOR_REGISTERS(return _svr_ret;);
chacha_encrypt_avx512(ctx, input, output, msglen);
RESTORE_VECTOR_REGISTERS();
return 0;
}
/* Everything below the AVX2 512-byte minimum (1..511 bytes) is handled by
* the AVX-512VL path itself - whole 256-byte four-block chunks plus a
* partial four-block tail - using single-instruction vprold rotations on
* 128-bit registers (no AVX-512 frequency penalty). It does not fall back
* to any other implementation. */
if (IS_INTEL_AVX512_VL(cpuidFlags) && msglen < 8 * CHACHA_CHUNK_BYTES) {
SAVE_VECTOR_REGISTERS(return _svr_ret;);
chacha_encrypt_avx512vl(ctx, input, output, msglen);
RESTORE_VECTOR_REGISTERS();
return 0;
}
#endif
#ifdef HAVE_INTEL_AVX2
if (IS_INTEL_AVX2(cpuidFlags)) {
SAVE_VECTOR_REGISTERS(return _svr_ret;);
@@ -439,6 +553,14 @@ int wc_Chacha_Process(ChaCha* ctx, byte* output, const byte* input,
RESTORE_VECTOR_REGISTERS();
return 0;
}
#ifdef HAVE_INTEL_SSSE3
else if (IS_INTEL_SSSE3(cpuidFlags)) {
SAVE_VECTOR_REGISTERS(return _svr_ret;);
chacha_encrypt_sse3(ctx, input, output, msglen);
RESTORE_VECTOR_REGISTERS();
return 0;
}
#endif
else {
chacha_encrypt_x64(ctx, input, output, msglen);
return 0;
wolfcrypt/src/chacha20_poly1305.c
+748 -27
View File
@@ -32,6 +32,7 @@ or Authenticated Encryption with Additional Data (AEAD) algorithm.
#if defined(HAVE_CHACHA) && defined(HAVE_POLY1305)
#include <wolfssl/wolfcrypt/chacha20_poly1305.h>
#include <wolfssl/wolfcrypt/cpuid.h>
#ifdef NO_INLINE
#include <wolfssl/wolfcrypt/misc.h>
@@ -41,6 +42,583 @@ or Authenticated Encryption with Additional Data (AEAD) algorithm.
#endif
#define CHACHA20_POLY1305_AEAD_INITIAL_COUNTER 0
#ifdef WOLFSSL_CHACHA20_POLY1305_FUSED
/* Fused single-pass encrypt kernel (in chacha_asm.S) and the 4-way power
* precompute it depends on. */
#ifdef __cplusplus
extern "C" {
#endif
WOLFSSL_LOCAL void chacha20_poly1305_avx512(ChaCha* chacha, Poly1305* poly,
const byte* m, byte* c, word32 bytes);
WOLFSSL_LOCAL void poly1305_calc_powers_avx2(Poly1305* ctx);
#ifdef __cplusplus
}
#endif
/* The fused kernel uses 4-block ChaCha (256-bit) + 4-way Poly1305, which beats
* the wide two-pass only where 512-bit code is throttled - Intel Ice Lake and
* later, under the AVX-512 frequency license. On AMD (no throttle, very fast
* wide primitives) the two-pass wins, so gate on an Intel vendor. Override:
* WOLFSSL_CHACHA20_POLY1305_FUSED_ALWAYS / _NEVER. */
static WC_INLINE int chacha20_poly1305_use_fused(void)
{
#if defined(WOLFSSL_CHACHA20_POLY1305_FUSED_NEVER)
return 0;
#elif defined(WOLFSSL_CHACHA20_POLY1305_FUSED_ALWAYS)
return 1;
#else
cpuid_flags_t f = cpuid_get_flags();
return (IS_CPU_INTEL(f) != 0) && (IS_INTEL_AVX512(f) != 0);
#endif
}
/* Encrypt with the fused kernel: no AAD, so Poly1305 starts clean and 256-byte
* aligned. Drive Poly1305 4-way (forceAvx2) so the kernel and the tail/final
* share the layout; the kernel does the aligned bulk, the tail and length
* framing go through the normal 4-way path. */
static int chacha20_poly1305_encrypt_fused(ChaChaPoly_Aead* aead,
const byte* pt, word32 ptLen, byte* ct, byte* tag)
{
word32 bulk = ptLen & ~(word32)0xff;
int ret;
aead->poly.forceAvx2 = 1;
/* The cpuid setkey may have zeroed a different accumulator; ready the 4-way
* hash and let the kernel initialise the lanes. */
XMEMSET(aead->poly.hh, 0, sizeof(aead->poly.hh));
aead->poly.started = 0;
aead->poly.leftover = 0;
aead->state = CHACHA20_POLY1305_STATE_DATA;
SAVE_VECTOR_REGISTERS(return _svr_ret;);
poly1305_calc_powers_avx2(&aead->poly);
aead->poly.started = 1;
/* bulk is a non-zero multiple of 256 here (caller gates on >= 256), but the
* kernel has no short-length entry guard - it would run a full 256-byte unit
* off the end on a zero length, so never call it with nothing to do. */
if (bulk > 0)
chacha20_poly1305_avx512(&aead->chacha, &aead->poly, pt, ct, bulk);
RESTORE_VECTOR_REGISTERS();
aead->dataLen = bulk;
ret = 0;
if (ptLen > bulk)
ret = wc_ChaCha20Poly1305_UpdateData(aead, pt + bulk, ct + bulk,
ptLen - bulk);
if (ret == 0)
ret = wc_ChaCha20Poly1305_Final(aead, tag);
return ret;
}
#endif /* WOLFSSL_CHACHA20_POLY1305_FUSED */
#ifdef WOLFSSL_CHACHA20_POLY1305_FUSED_IFMA
/* IFMA stitched single-pass encrypt kernel (in chacha_asm.S): full 512-bit
* 16-block ChaCha interleaved with an 8-way IFMA (vpmadd52) Poly1305 that
* collapses to the scalar hash. Processes 1024-byte units. Depends on the
* radix-2^44 powers. */
#ifdef __cplusplus
extern "C" {
#endif
WOLFSSL_LOCAL void chacha20_poly1305_ifma(ChaCha* chacha, Poly1305* poly,
const byte* m, byte* c, word32 bytes);
/* Decrypt counterpart: hashes the ciphertext INPUT (m) as it decrypts to c
* (in-place safe - m is hashed before it is overwritten). Same 1024-byte
* units and radix-2^44 powers. */
WOLFSSL_LOCAL void chacha20_poly1305_ifma_decrypt(ChaCha* chacha,
Poly1305* poly, const byte* m, byte* c, word32 bytes);
WOLFSSL_LOCAL void poly1305_calc_powers_avx512ifma(Poly1305* ctx);
/* ctx->h = ctx->hh * r^nBlocks + ctx->h - advances the running hash (saved by
* the kernel to ctx->hh) past a chunk the kernel hashed from zero into ctx->h.
* Radix-2^64 scalar, so no 26<->64 conversions. */
WOLFSSL_LOCAL void poly1305_fold_avx512ifma(Poly1305* ctx, word32 nBlocks);
#ifdef __cplusplus
}
#endif
/* Minimum length to stitch. The kernel processes 1024-byte units and a
* once-per-op power precompute, and any sub-1024 remainder is authenticated by
* the slower scalar Poly1305; below this the two-pass wins (measured crossover
* on Zen5). Above it the stitch wins 1.1-1.4x, growing with size.
*
* HARD LOWER BOUND 1024: callers gate on this then pass bulk = sz & ~0x3ff to
* the kernel, and the kernel has NO short-length entry guard - its first length
* test runs only AFTER a full 1024-byte chunk. A value below 1024 lets a
* sub-1024 message compute bulk == 0, and the kernel then reads/writes a whole
* 1024-byte unit off the end of the buffer and underflows its counter into a
* multi-million-iteration loop. Enforced at compile time below; stitch_chunk()
* also guards bulk == 0 at runtime as defence in depth. */
#ifndef CHACHA20_POLY1305_STITCH_MIN
#define CHACHA20_POLY1305_STITCH_MIN 4096
#endif
#if CHACHA20_POLY1305_STITCH_MIN < 1024
#error "CHACHA20_POLY1305_STITCH_MIN must be >= 1024 (the kernel unit size)"
#endif
/* Short-message fused path: for messages this small the Poly1305 key block
* (ChaCha counter 0) and the whole ciphertext (counter 1+) fit in a single
* ChaCha keystream generation - one pass instead of two - and a scalar
* Poly1305 avoids the vector power-precompute cost. Needs the forceScalar
* flag (same builds as the fused kernels). Measured 1.1-1.5x on Zen5 for
* 64-192 byte records; above SHORT_MAX the poly key spills to a second ChaCha
* chunk and the saving is gone. */
#if defined(WOLFSSL_CHACHA20_POLY1305_FUSED_IFMA) && \
!defined(WOLFSSL_NO_CHACHA20_POLY1305_SHORT)
#define WOLFSSL_CHACHA20_POLY1305_SHORT
#ifndef CHACHA20_POLY1305_SHORT_MAX
/* 64 (poly-key block) + 192 = 256 = one AVX-512VL 4-block chunk */
#define CHACHA20_POLY1305_SHORT_MAX 192
#endif
#endif
/* The IFMA stitch runs a full-width 512-bit 16-block ChaCha interleaved with an
* 8-way IFMA Poly1305: ChaCha is the bottleneck and Poly hides under it, so it
* beats the two-pass (which runs the two passes back to back) by ~1.3-1.4x at
* >=16KB - measured on AMD Zen5, and expected wherever AVX-512 + IFMA exist
* (both use 512-bit ChaCha, so any frequency throttle hits both equally). Gate
* on AVX-512 + IFMA, any vendor. Override: ..._FUSED_IFMA_ALWAYS / _NEVER. */
static WC_INLINE int chacha20_poly1305_use_fused_ifma(void)
{
#if defined(WOLFSSL_CHACHA20_POLY1305_FUSED_IFMA_NEVER)
return 0;
#elif defined(WOLFSSL_CHACHA20_POLY1305_FUSED_IFMA_ALWAYS)
return 1;
#else
cpuid_flags_t f = cpuid_get_flags();
return (IS_INTEL_AVX512(f) != 0) && (IS_INTEL_AVX512_IFMA(f) != 0);
#endif
}
/* Stitch one 1024-byte-aligned bulk. (IFMA path) The kernel hashes this
* chunk's ciphertext from zero (leaving ctx->h = H_chunk) and advances the
* ChaCha counter, saving the running hash (the AAD, or previous chunks) to
* ctx->hh; poly1305_fold_avx512ifma then advances that hash past this chunk
* (ctx->h = ctx->hh * r^nBlocks + H_chunk). Powers are computed once (started
* flag). Caller must have ctx->h = running hash, leftover == 0, forceScalar
* and finished set, and the ChaCha counter placed for this chunk. decrypt: in
* is ciphertext, out is plaintext (in-place safe - the kernel hashes in before
* overwriting it); the hash math is identical. */
static int chacha20_poly1305_stitch_chunk(ChaCha* chacha, Poly1305* poly,
const byte* in, byte* out, word32 bulk, int decrypt)
{
int fold;
/* The kernel has no short-length entry guard and would run a full 1024-byte
* unit off the end of the buffer on a zero length. bulk is always a
* non-zero multiple of 1024 here (STITCH_MIN >= 1024, enforced at compile
* time), but never invoke the kernel with nothing to do. */
if (bulk == 0)
return 0;
/* A running hash (AAD or previous chunks) must be folded past this chunk;
* detect it before the kernel overwrites poly->h with this chunk's hash. */
fold = (poly->h[0] | poly->h[1] | poly->h[2]) != 0;
SAVE_VECTOR_REGISTERS(return _svr_ret;);
if (!poly->started) {
poly1305_calc_powers_avx512ifma(poly);
poly->started = 1;
}
if (decrypt)
chacha20_poly1305_ifma_decrypt(chacha, poly, in, out, bulk);
else
chacha20_poly1305_ifma(chacha, poly, in, out, bulk);
RESTORE_VECTOR_REGISTERS();
/* poly->h = H_chunk, poly->hh = running hash (scalar fold, no vectors). */
if (fold)
poly1305_fold_avx512ifma(poly, bulk / 16);
return 0;
}
/* Encrypt the whole message with the IFMA stitch (one-shot path). AAD is
* hashed scalar into ctx->h, the 1024-aligned bulk is stitched (folding AAD
* through it), the tail + length framing go through the scalar path.
* forceScalar/finished: see the AVX2 fused note (setkey_avx2 leaves finished
* clear). Both this and the streaming UpdateData path share stitch_chunk(). */
static int chacha20_poly1305_encrypt_fused_ifma(ChaChaPoly_Aead* aead,
const byte* aad, word32 aadLen, const byte* pt, word32 ptLen, byte* ct,
byte* tag)
{
word32 bulk = ptLen & ~(word32)0x3ff;
int ret = 0;
aead->poly.forceScalar = 1;
XMEMSET(aead->poly.h, 0, sizeof(aead->poly.h));
aead->poly.finished = 1;
aead->poly.leftover = 0;
aead->poly.started = 0;
/* Hash AAD + pad1 (scalar) -> H_aad in ctx->h. */
if (aadLen > 0) {
ret = wc_Poly1305Update(&aead->poly, aad, aadLen);
if (ret == 0)
ret = wc_Poly1305_Pad(&aead->poly, aadLen);
}
aead->aadLen = aadLen;
aead->state = CHACHA20_POLY1305_STATE_DATA;
if (ret == 0)
ret = chacha20_poly1305_stitch_chunk(&aead->chacha, &aead->poly, pt,
ct, bulk, 0);
if (ret == 0) {
aead->dataLen = bulk;
if (ptLen > bulk)
ret = wc_ChaCha20Poly1305_UpdateData(aead, pt + bulk, ct + bulk,
ptLen - bulk);
if (ret == 0)
ret = wc_ChaCha20Poly1305_Final(aead, tag);
}
return ret;
}
#endif /* WOLFSSL_CHACHA20_POLY1305_FUSED_IFMA */
#ifdef WOLFSSL_CHACHA20_POLY1305_SHORT
/* small_enc/small_dec are HAVE_INTEL_AVX2 kernels in chacha_asm.S; gate on
* NO_AVX2_SUPPORT so a -DNO_AVX2_SUPPORT build does not reference them. */
#if defined(USE_INTEL_SPEEDUP) && defined(WOLFSSL_X86_64_BUILD) && \
!defined(WOLFSSL_NO_CHACHA20_POLY1305_SMALL_ASM) && \
!defined(NO_AVX2_SUPPORT)
#define WOLFSSL_CP_SMALL_ASM
/* Fused single-call ChaCha20-Poly1305 encrypt for a one-block (<=64 byte)
* record (in chacha_asm.S): SSSE3 crypt2 produces the Poly1305 key block and
* the single data block together; the scalar poly1305_*_avx do the MAC. */
#ifdef __cplusplus
extern "C" {
#endif
WOLFSSL_LOCAL void chacha20_poly1305_small_enc(ChaCha* chacha, Poly1305* poly,
const byte* m, byte* c, word32 mLen, const byte* aad, word32 aadLen,
byte* tag);
/* Decrypt twin: decrypts in->out AND verifies the tag in one pass (decrypt-
* then-verify). Constant-time-compares the computed tag against the received
* tag internally and returns 0 on match, 1 on mismatch; the caller ForceZeros
* the output on mismatch, so no plaintext is released on a bad tag. */
WOLFSSL_LOCAL int chacha20_poly1305_small_dec(ChaCha* chacha, Poly1305* poly,
const byte* in, byte* out, word32 ctLen, const byte* aad, word32 aadLen,
const byte* tag);
#ifdef __cplusplus
}
#endif
static WC_INLINE int chacha20_poly1305_use_small(void)
{
return IS_INTEL_AVX2(cpuid_get_flags()) != 0;
}
#endif
/* Fused short-message (sz <= CHACHA20_POLY1305_SHORT_MAX) encrypt for the
* pre-keyed contexts: derive the Poly1305 key (counter 0) and the encryption
* keystream (counter 1+) in a SINGLE ChaCha pass, then scalar-hash. Saves the
* second ChaCha invocation the two-pass path would make. */
static int chacha20_poly1305_encrypt_short(ChaCha* chacha, Poly1305* poly,
byte* out, const byte* in, word32 sz, const byte* nonce, byte* tag,
const byte* aad, word32 aadSz)
{
byte ks[64 + CHACHA20_POLY1305_SHORT_MAX];
int ret;
#ifdef WOLFSSL_CP_SMALL_ASM
/* One block or less of data: the fused single-call kernel. */
if (sz <= 64 && chacha20_poly1305_use_small()) {
ret = wc_Chacha_SetIV(chacha, nonce,
CHACHA20_POLY1305_AEAD_INITIAL_COUNTER);
if (ret == 0) {
SAVE_VECTOR_REGISTERS(return _svr_ret;);
chacha20_poly1305_small_enc(chacha, poly, in, out, sz, aad, aadSz,
tag);
RESTORE_VECTOR_REGISTERS();
}
return ret;
}
#endif
XMEMSET(ks, 0, 64 + sz);
ret = wc_Chacha_SetIV(chacha, nonce,
CHACHA20_POLY1305_AEAD_INITIAL_COUNTER);
if (ret == 0) /* ctr0 (poly key) .. ctrN, one pass */
ret = wc_Chacha_Process(chacha, ks, ks, 64 + sz);
if (ret == 0)
ret = wc_Poly1305SetKey(poly, ks, CHACHA20_POLY1305_AEAD_KEYSIZE);
if (ret == 0) {
xorbufout(out, in, ks + 64, sz); /* ct = pt ^ keystream (ctr1+) */
poly->forceScalar = 1;
poly->finished = 1;
ret = wc_Poly1305_MAC(poly, aad, aadSz, out, sz, tag,
CHACHA20_POLY1305_AEAD_AUTHTAG_SIZE);
}
ForceZero(ks, 64 + sz); /* ks[0:32] was the poly key */
return ret;
}
/* Fused short-message decrypt twin: derive key + keystream in one ChaCha pass,
* MAC the ciphertext INPUT and verify the tag BEFORE decrypting, so no
* plaintext is produced on a bad tag (stronger than the stitch, cheap here
* because the message is small). In-place safe. */
static int chacha20_poly1305_decrypt_short(ChaCha* chacha, Poly1305* poly,
byte* out, const byte* in, word32 sz, const byte* nonce, const byte* tag,
const byte* aad, word32 aadSz)
{
byte ks[64 + CHACHA20_POLY1305_SHORT_MAX];
byte calcTag[CHACHA20_POLY1305_AEAD_AUTHTAG_SIZE];
int ret;
#ifdef WOLFSSL_CP_SMALL_ASM
/* One block or less of data: the fused single-call kernel decrypts in->out
* and computes calcTag in one pass (decrypt-then-verify). Zero the output
* if the tag is bad, so no plaintext is released. */
if (sz <= 64 && chacha20_poly1305_use_small()) {
ret = wc_Chacha_SetIV(chacha, nonce,
CHACHA20_POLY1305_AEAD_INITIAL_COUNTER);
if (ret == 0) {
int bad;
SAVE_VECTOR_REGISTERS(return _svr_ret;);
bad = chacha20_poly1305_small_dec(chacha, poly, in, out, sz, aad,
aadSz, tag);
RESTORE_VECTOR_REGISTERS();
if (bad) { /* bad tag: no plaintext */
if (sz > 0)
ForceZero(out, sz);
ret = MAC_CMP_FAILED_E;
}
}
(void)calcTag;
return ret;
}
#endif
XMEMSET(ks, 0, 64 + sz);
ret = wc_Chacha_SetIV(chacha, nonce,
CHACHA20_POLY1305_AEAD_INITIAL_COUNTER);
if (ret == 0)
ret = wc_Chacha_Process(chacha, ks, ks, 64 + sz);
if (ret == 0)
ret = wc_Poly1305SetKey(poly, ks, CHACHA20_POLY1305_AEAD_KEYSIZE);
if (ret == 0) {
poly->forceScalar = 1;
poly->finished = 1;
ret = wc_Poly1305_MAC(poly, aad, aadSz, in, sz, calcTag,
CHACHA20_POLY1305_AEAD_AUTHTAG_SIZE);
}
if (ret == 0)
ret = wc_ChaCha20Poly1305_CheckTag(tag, calcTag);
if (ret == 0) /* tag good: decrypt pt = ct ^ ks */
xorbufout(out, in, ks + 64, sz);
else if (sz > 0) /* bad tag/error: no stale output */
ForceZero(out, sz);
ForceZero(ks, 64 + sz);
return ret;
}
#endif /* WOLFSSL_CHACHA20_POLY1305_SHORT */
/* Encrypt + authenticate one message with PRE-KEYED ChaCha20 and Poly1305
* contexts - the ChaCha20-Poly1305 analogue of wc_AesGcmEncrypt on a keyed Aes.
* Intended for the TLS record layer, which keeps the ChaCha context keyed once
* (per traffic key) and only varies the nonce per record. The per-record
* Poly1305 key is derived here from the ChaCha keystream. Uses the single-pass
* IFMA stitch when beneficial, else the two-pass; identical output either way.
*
* chacha ChaCha20 context with the key already set (wc_Chacha_SetKey)
* poly Poly1305 scratch context (re-keyed here every call)
* out ciphertext out (may alias in)
* in/sz plaintext / length
* nonce CHACHA20_POLY1305_AEAD_IV_SIZE (12) byte record nonce
* tag CHACHA20_POLY1305_AEAD_AUTHTAG_SIZE (16) byte tag out
* aad/aadSz additional authenticated data
* returns 0 on success, negative on error.
*/
WOLFSSL_API int wc_ChaCha20Poly1305_Encrypt_ex(ChaCha* chacha, Poly1305* poly,
byte* out, const byte* in, word32 sz, const byte* nonce, byte* tag,
const byte* aad, word32 aadSz)
{
byte polyKey[CHACHA20_POLY1305_AEAD_KEYSIZE];
int ret;
if (chacha == NULL || poly == NULL || nonce == NULL || tag == NULL ||
(sz > 0 && (in == NULL || out == NULL)) ||
(aadSz > 0 && aad == NULL)) {
return BAD_FUNC_ARG;
}
#ifdef WOLFSSL_CHACHA20_POLY1305_SHORT
if (sz <= CHACHA20_POLY1305_SHORT_MAX)
return chacha20_poly1305_encrypt_short(chacha, poly, out, in, sz,
nonce, tag, aad, aadSz);
#endif
/* Per-record Poly1305 key = first 32 bytes of ChaCha20(nonce, ctr 0). */
XMEMSET(polyKey, 0, sizeof(polyKey));
ret = wc_Chacha_SetIV(chacha, nonce,
CHACHA20_POLY1305_AEAD_INITIAL_COUNTER);
if (ret == 0)
ret = wc_Chacha_Process(chacha, polyKey, polyKey, sizeof(polyKey));
if (ret == 0) /* message data starts at counter 1 */
ret = wc_Chacha_SetIV(chacha, nonce,
CHACHA20_POLY1305_AEAD_INITIAL_COUNTER + 1);
if (ret == 0)
ret = wc_Poly1305SetKey(poly, polyKey, sizeof(polyKey));
ForceZero(polyKey, sizeof(polyKey));
if (ret != 0)
return ret;
#ifdef WOLFSSL_CHACHA20_POLY1305_FUSED_IFMA
if (sz >= CHACHA20_POLY1305_STITCH_MIN &&
chacha20_poly1305_use_fused_ifma()) {
word32 bulk = sz & ~(word32)0x3ff;
/* Scalar running hash (in poly->h) so stitch and tail chain; poly->h,
* leftover and started are all zeroed by wc_Poly1305SetKey. */
poly->forceScalar = 1;
poly->finished = 1;
if (aadSz > 0) { /* H_aad + pad1 (scalar) */
ret = wc_Poly1305Update(poly, aad, aadSz);
if (ret == 0)
ret = wc_Poly1305_Pad(poly, aadSz);
}
if (ret == 0) /* stitch bulk + fold AAD */
ret = chacha20_poly1305_stitch_chunk(chacha, poly, in, out, bulk,
0);
if (ret == 0 && sz > bulk) { /* scalar tail */
ret = wc_Chacha_Process(chacha, out + bulk, in + bulk, sz - bulk);
if (ret == 0)
ret = wc_Poly1305Update(poly, out + bulk, sz - bulk);
}
if (ret == 0) /* pad2 + lengths + tag */
ret = wc_Poly1305_Pad(poly, sz);
if (ret == 0)
ret = wc_Poly1305_EncodeSizes(poly, aadSz, sz);
if (ret == 0)
ret = wc_Poly1305Final(poly, tag);
return ret;
}
#endif
/* Two-pass: fast vector Poly1305 (small msgs, or stitch not beneficial). */
ret = wc_Chacha_Process(chacha, out, in, sz);
if (ret == 0)
ret = wc_Poly1305_MAC(poly, aad, aadSz, out, sz, tag,
CHACHA20_POLY1305_AEAD_AUTHTAG_SIZE);
return ret;
}
/* Verify+decrypt one message with pre-keyed ChaCha20 and Poly1305 contexts -
* the decrypt counterpart of wc_ChaCha20Poly1305_Encrypt_ex, for the TLS record
* layer. Verifies the Poly1305 tag over AAD+ciphertext and decrypts to out
* (in-place safe). Uses the single-pass IFMA decrypt stitch when beneficial.
* The plaintext is produced while the tag is computed, so on tag mismatch out
* is zeroed and MAC_CMP_FAILED_E returned - callers must check the result.
*
* chacha ChaCha20 context with the key already set (wc_Chacha_SetKey)
* poly Poly1305 scratch context (re-keyed here every call)
* out plaintext out (may alias in)
* in/sz ciphertext / length
* nonce CHACHA20_POLY1305_AEAD_IV_SIZE (12) byte record nonce
* tag CHACHA20_POLY1305_AEAD_AUTHTAG_SIZE (16) byte tag to verify
* aad/aadSz additional authenticated data
* returns 0 on success, MAC_CMP_FAILED_E on tag mismatch, else negative.
*/
WOLFSSL_API int wc_ChaCha20Poly1305_Decrypt_ex(ChaCha* chacha, Poly1305* poly,
byte* out, const byte* in, word32 sz, const byte* nonce, const byte* tag,
const byte* aad, word32 aadSz)
{
byte polyKey[CHACHA20_POLY1305_AEAD_KEYSIZE];
byte calcTag[CHACHA20_POLY1305_AEAD_AUTHTAG_SIZE];
int ret;
if (chacha == NULL || poly == NULL || nonce == NULL || tag == NULL ||
(sz > 0 && (in == NULL || out == NULL)) ||
(aadSz > 0 && aad == NULL)) {
return BAD_FUNC_ARG;
}
#ifdef WOLFSSL_CHACHA20_POLY1305_SHORT
if (sz <= CHACHA20_POLY1305_SHORT_MAX)
return chacha20_poly1305_decrypt_short(chacha, poly, out, in, sz,
nonce, tag, aad, aadSz);
#endif
/* Per-record Poly1305 key = first 32 bytes of ChaCha20(nonce, ctr 0). */
XMEMSET(polyKey, 0, sizeof(polyKey));
ret = wc_Chacha_SetIV(chacha, nonce,
CHACHA20_POLY1305_AEAD_INITIAL_COUNTER);
if (ret == 0)
ret = wc_Chacha_Process(chacha, polyKey, polyKey, sizeof(polyKey));
if (ret == 0) /* message data starts at counter 1 */
ret = wc_Chacha_SetIV(chacha, nonce,
CHACHA20_POLY1305_AEAD_INITIAL_COUNTER + 1);
if (ret == 0)
ret = wc_Poly1305SetKey(poly, polyKey, sizeof(polyKey));
ForceZero(polyKey, sizeof(polyKey));
if (ret != 0)
return ret;
#ifdef WOLFSSL_CHACHA20_POLY1305_FUSED_IFMA
if (sz >= CHACHA20_POLY1305_STITCH_MIN &&
chacha20_poly1305_use_fused_ifma()) {
word32 bulk = sz & ~(word32)0x3ff;
/* Scalar running hash (in poly->h) so stitch and tail chain; poly->h,
* leftover and started are all zeroed by wc_Poly1305SetKey. */
poly->forceScalar = 1;
poly->finished = 1;
if (aadSz > 0) { /* H_aad + pad1 (scalar) */
ret = wc_Poly1305Update(poly, aad, aadSz);
if (ret == 0)
ret = wc_Poly1305_Pad(poly, aadSz);
}
if (ret == 0) /* stitch: hash CT + decrypt */
ret = chacha20_poly1305_stitch_chunk(chacha, poly, in, out, bulk,
1);
if (ret == 0 && sz > bulk) { /* scalar tail */
/* hash the ciphertext tail before decrypt overwrites it */
ret = wc_Poly1305Update(poly, in + bulk, sz - bulk);
if (ret == 0)
ret = wc_Chacha_Process(chacha, out + bulk, in + bulk,
sz - bulk);
}
if (ret == 0) /* pad2 + lengths + tag */
ret = wc_Poly1305_Pad(poly, sz);
if (ret == 0)
ret = wc_Poly1305_EncodeSizes(poly, aadSz, sz);
if (ret == 0)
ret = wc_Poly1305Final(poly, calcTag);
}
else
#endif
{
/* Two-pass: MAC the ciphertext (in), then decrypt in -> out. */
ret = wc_Poly1305_MAC(poly, aad, aadSz, in, sz, calcTag,
CHACHA20_POLY1305_AEAD_AUTHTAG_SIZE);
if (ret == 0)
ret = wc_Chacha_Process(chacha, out, in, sz);
}
if (ret == 0)
ret = wc_ChaCha20Poly1305_CheckTag(tag, calcTag);
if (ret != 0 && sz > 0)
ForceZero(out, sz);
return ret;
}
/* Clear a temporary ChaChaPoly_Aead. On an AVX-512/IFMA build the Poly1305
* state carries ~320 extra bytes (r5..r8 for the 16-way poly, ifma_h for the
* IFMA stitch) that are only ever written on AVX-512-capable CPUs; on a CPU
* without AVX-512 they are never touched, so zeroing them on every call is pure
* overhead - a large fraction of a small AEAD op. Skip them there (they hold
* no key material from this call), and clear the full struct otherwise. */
static WC_INLINE void chacha20_poly1305_aead_zero(ChaChaPoly_Aead* aead)
{
#ifdef WOLFSSL_POLY1305_AVX512
if (IS_INTEL_AVX512(cpuid_get_flags()) != 0)
ForceZero(aead, sizeof(ChaChaPoly_Aead));
else
ForceZero(aead, (word32)((const byte*)&aead->poly.r5
- (const byte*)aead));
#else
ForceZero(aead, sizeof(ChaChaPoly_Aead));
#endif
}
WOLFSSL_ABI
int wc_ChaCha20Poly1305_Encrypt(
const byte inKey[CHACHA20_POLY1305_AEAD_KEYSIZE],
@@ -53,9 +631,14 @@ int wc_ChaCha20Poly1305_Encrypt(
int ret;
WC_DECLARE_VAR(aead, ChaChaPoly_Aead, 1, 0);
/* Validate function arguments */
/* Validate function arguments. A NULL data pointer is rejected even at
* zero length: the direct/short paths below bypass UpdateData, whose own
* check rejects a NULL data pointer unconditionally - this preserves that
* contract. A valid pointer with zero length still succeeds (empty AEAD
* message). */
if (!inKey || !inIV ||
(inPlaintextLen > 0 && inPlaintext == NULL) ||
inPlaintext == NULL ||
(inAADLen > 0 && inAAD == NULL) ||
!outCiphertext ||
!outAuthTag)
{
@@ -65,19 +648,61 @@ int wc_ChaCha20Poly1305_Encrypt(
WC_ALLOC_VAR_EX(aead, ChaChaPoly_Aead, 1, NULL, DYNAMIC_TYPE_TMP_BUFFER,
return MEMORY_E);
#ifdef WOLFSSL_CHACHA20_POLY1305_SHORT
/* Small record: derive the Poly1305 key and the keystream in a SINGLE
* ChaCha pass (SSSE3), then scalar-hash - the same short path Encrypt_ex
* uses. Avoids wc_ChaCha20Poly1305_Init's separate scalar poly-key block
* and the second scalar data block the two-pass fallback would run. */
if (inPlaintextLen <= CHACHA20_POLY1305_SHORT_MAX) {
ret = wc_Chacha_SetKey(&aead->chacha, inKey,
CHACHA20_POLY1305_AEAD_KEYSIZE);
if (ret == 0)
ret = chacha20_poly1305_encrypt_short(&aead->chacha, &aead->poly,
outCiphertext, inPlaintext, inPlaintextLen, inIV, outAuthTag,
inAAD, inAADLen);
}
else
#endif
{
ret = wc_ChaCha20Poly1305_Init(aead, inKey, inIV,
CHACHA20_POLY1305_AEAD_ENCRYPT);
if (ret == 0)
ret = wc_ChaCha20Poly1305_UpdateAad(aead, inAAD, inAADLen);
if (ret == 0)
ret = wc_ChaCha20Poly1305_UpdateData(aead, inPlaintext, outCiphertext,
inPlaintextLen);
if (ret == 0)
ret = wc_ChaCha20Poly1305_Final(aead, outAuthTag);
/* Prefer the IFMA stitch - full 512-bit ChaCha, beats the others where
* AVX-512 + IFMA exist. */
#ifdef WOLFSSL_CHACHA20_POLY1305_FUSED_IFMA
if (ret == 0 && inPlaintextLen >= CHACHA20_POLY1305_STITCH_MIN &&
chacha20_poly1305_use_fused_ifma()) {
ret = chacha20_poly1305_encrypt_fused_ifma(aead, inAAD, inAADLen,
inPlaintext, inPlaintextLen, outCiphertext, outAuthTag);
}
else
#endif
#ifdef WOLFSSL_CHACHA20_POLY1305_FUSED
if (ret == 0 && inAADLen == 0 && inPlaintextLen >= 256 &&
chacha20_poly1305_use_fused()) {
ret = chacha20_poly1305_encrypt_fused(aead, inPlaintext,
inPlaintextLen, outCiphertext, outAuthTag);
}
else
#endif
{
/* Direct two-pass on the contexts Init already keyed (ChaCha counter is
* at 1, Poly1305 keyed). Faster than the UpdateAad/UpdateData/Final
* state machine for the common non-stitched case - in particular
* wc_Poly1305_MAC hashes the AAD inline instead of buffering it through
* UpdateAad, which is where the small-message-with-AAD cost was. */
if (ret == 0)
ret = wc_Chacha_Process(&aead->chacha, outCiphertext, inPlaintext,
inPlaintextLen);
if (ret == 0)
ret = wc_Poly1305_MAC(&aead->poly, inAAD, inAADLen, outCiphertext,
inPlaintextLen, outAuthTag,
CHACHA20_POLY1305_AEAD_AUTHTAG_SIZE);
}
}
#ifdef WOLFSSL_SMALL_STACK
if (aead != NULL)
#endif
ForceZero(aead, sizeof(ChaChaPoly_Aead));
chacha20_poly1305_aead_zero(aead);
WC_FREE_VAR_EX(aead, NULL, DYNAMIC_TYPE_TMP_BUFFER);
return ret;
@@ -96,9 +721,14 @@ int wc_ChaCha20Poly1305_Decrypt(
WC_DECLARE_VAR(aead, ChaChaPoly_Aead, 1, 0);
byte calculatedAuthTag[CHACHA20_POLY1305_AEAD_AUTHTAG_SIZE];
/* Validate function arguments */
/* Validate function arguments. A NULL data pointer is rejected even at
* zero length: the direct/short paths below bypass UpdateData, whose own
* check rejects a NULL data pointer unconditionally - this preserves that
* contract. A valid pointer with zero length still succeeds (empty AEAD
* message). */
if (!inKey || !inIV ||
(inCiphertextLen > 0 && inCiphertext == NULL) ||
inCiphertext == NULL ||
(inAADLen > 0 && inAAD == NULL) ||
!inAuthTag ||
!outPlaintext)
{
@@ -110,17 +740,38 @@ int wc_ChaCha20Poly1305_Decrypt(
XMEMSET(calculatedAuthTag, 0, sizeof(calculatedAuthTag));
#ifdef WOLFSSL_CHACHA20_POLY1305_SHORT
/* Small record: single ChaCha pass for poly key + keystream, MAC the
* ciphertext and verify BEFORE decrypting (no plaintext on bad tag). Same
* short path Decrypt_ex uses; avoids Init's extra scalar poly-key block. */
if (inCiphertextLen <= CHACHA20_POLY1305_SHORT_MAX) {
ret = wc_Chacha_SetKey(&aead->chacha, inKey,
CHACHA20_POLY1305_AEAD_KEYSIZE);
if (ret == 0)
ret = chacha20_poly1305_decrypt_short(&aead->chacha, &aead->poly,
outPlaintext, inCiphertext, inCiphertextLen, inIV, inAuthTag,
inAAD, inAADLen);
}
else
#endif
{
ret = wc_ChaCha20Poly1305_Init(aead, inKey, inIV,
CHACHA20_POLY1305_AEAD_DECRYPT);
/* Direct two-pass on the contexts Init already keyed: MAC the ciphertext,
* verify the tag, then decrypt - verify-then-decrypt, so no plaintext is
* produced on a bad tag. Faster than the UpdateAad/UpdateData/Final state
* machine (wc_Poly1305_MAC hashes the AAD inline). In-place safe: the MAC
* reads inCiphertext before the decrypt overwrites it. */
if (ret == 0)
ret = wc_ChaCha20Poly1305_UpdateAad(aead, inAAD, inAADLen);
if (ret == 0)
ret = wc_ChaCha20Poly1305_UpdateData(aead, inCiphertext, outPlaintext,
inCiphertextLen);
if (ret == 0)
ret = wc_ChaCha20Poly1305_Final(aead, calculatedAuthTag);
ret = wc_Poly1305_MAC(&aead->poly, inAAD, inAADLen, inCiphertext,
inCiphertextLen, calculatedAuthTag,
CHACHA20_POLY1305_AEAD_AUTHTAG_SIZE);
if (ret == 0)
ret = wc_ChaCha20Poly1305_CheckTag(inAuthTag, calculatedAuthTag);
if (ret == 0)
ret = wc_Chacha_Process(&aead->chacha, outPlaintext, inCiphertext,
inCiphertextLen);
}
if (ret != 0) {
/* zero plaintext on error */
@@ -129,7 +780,7 @@ int wc_ChaCha20Poly1305_Decrypt(
#ifdef WOLFSSL_SMALL_STACK
if (aead != NULL)
#endif
ForceZero(aead, sizeof(ChaChaPoly_Aead));
chacha20_poly1305_aead_zero(aead);
WC_FREE_VAR_EX(aead, NULL, DYNAMIC_TYPE_TMP_BUFFER);
return ret;
@@ -175,7 +826,8 @@ int wc_ChaCha20Poly1305_Init(ChaChaPoly_Aead* aead,
return BAD_FUNC_ARG;
}
/* setup aead context */
/* setup aead context (full clear: the aadLen/dataLen/state wrapper fields
* live after the Poly1305 member and must be initialized) */
XMEMSET(aead, 0, sizeof(ChaChaPoly_Aead));
XMEMSET(authKey, 0, sizeof(authKey));
aead->isEncrypt = isEncrypt ? 1 : 0;
@@ -262,8 +914,33 @@ int wc_ChaCha20Poly1305_UpdateData(ChaChaPoly_Aead* aead,
if (dataLen > CHACHA20_POLY1305_MAX - aead->dataLen)
return CHACHA_POLY_OVERFLOW;
#ifdef WOLFSSL_CHACHA20_POLY1305_FUSED_IFMA
/* Enter scalar-stitch mode at the first data chunk when it is large enough
* to benefit and no vector Poly1305 state exists yet (started==0): any AAD
* so far is then fully buffered by the vector path. The vector buffer can
* hold >16 bytes, which the scalar path cannot resume, so re-hash the
* buffered AAD cleanly through the scalar path; the pad below finishes it.
* If the AAD was large enough to be processed by the vector path
* (started==1) we cannot switch, so it stays two-pass - no regression.
* The IFMA stitch handles both directions (decrypt hashes the ciphertext
* input). */
if (!aead->poly.forceScalar && aead->poly.started == 0 &&
aead->dataLen == 0 && dataLen >= CHACHA20_POLY1305_STITCH_MIN &&
chacha20_poly1305_use_fused_ifma()) {
word32 aadN = (word32)aead->poly.leftover;
byte aadBuf[8 * POLY1305_BLOCK_SIZE];
if (aadN > 0)
XMEMCPY(aadBuf, aead->poly.buffer, aadN);
aead->poly.forceScalar = 1;
aead->poly.finished = 1;
aead->poly.leftover = 0;
if (aadN > 0)
ret = wc_Poly1305Update(&aead->poly, aadBuf, aadN);
}
#endif
/* Pad the AAD */
if (aead->state == CHACHA20_POLY1305_STATE_AAD) {
if (ret == 0 && aead->state == CHACHA20_POLY1305_STATE_AAD) {
ret = wc_Poly1305_Pad(&aead->poly, aead->aadLen);
}
@@ -273,14 +950,58 @@ int wc_ChaCha20Poly1305_UpdateData(ChaChaPoly_Aead* aead,
/* Perform ChaCha20 encrypt/decrypt and Poly1305 auth calc */
if (ret == 0) {
if (aead->isEncrypt) {
ret = wc_Chacha_Process(&aead->chacha, outData, inData, dataLen);
if (ret == 0)
ret = wc_Poly1305Update(&aead->poly, outData, dataLen);
const byte* in = inData;
byte* out = outData;
word32 len = dataLen;
#ifdef WOLFSSL_CHACHA20_POLY1305_FUSED_IFMA
/* Stitch the 1024-aligned bulk (encrypt + auth in one pass) when
* the data so far is 64-byte aligned - so BOTH ChaCha (no buffered
* partial keystream) and Poly1305 (leftover==0, running hash in
* ctx->h) are at a block boundary - and we are in scalar-hash mode
* (set at Init for AVX-512+IFMA encrypt). The kernel processes
* whole blocks from the ChaCha counter, so a mid-block position
* would make it skip the buffered keystream; the 64-alignment check
* prevents that. The remainder falls through to the path below. */
if (aead->poly.forceScalar && (aead->dataLen & 63) == 0 &&
len >= CHACHA20_POLY1305_STITCH_MIN) {
word32 bulk = len & ~(word32)0x3ff;
ret = chacha20_poly1305_stitch_chunk(&aead->chacha, &aead->poly,
in, out, bulk, 0);
in += bulk;
out += bulk;
len -= bulk;
}
#endif
if (ret == 0 && len > 0) {
ret = wc_Chacha_Process(&aead->chacha, out, in, len);
if (ret == 0)
ret = wc_Poly1305Update(&aead->poly, out, len);
}
}
else {
ret = wc_Poly1305Update(&aead->poly, inData, dataLen);
if (ret == 0)
ret = wc_Chacha_Process(&aead->chacha, outData, inData, dataLen);
const byte* in = inData;
byte* out = outData;
word32 len = dataLen;
#ifdef WOLFSSL_CHACHA20_POLY1305_FUSED_IFMA
/* Stitch the 1024-aligned bulk (auth + decrypt in one pass) under
* the same conditions as encrypt. The kernel hashes the ciphertext
* (in) before overwriting it, so in-place decrypt is safe. */
if (aead->poly.forceScalar && (aead->dataLen & 63) == 0 &&
len >= CHACHA20_POLY1305_STITCH_MIN) {
word32 bulk = len & ~(word32)0x3ff;
ret = chacha20_poly1305_stitch_chunk(&aead->chacha, &aead->poly,
in, out, bulk, 1);
in += bulk;
out += bulk;
len -= bulk;
}
#endif
if (ret == 0 && len > 0) {
/* hash the ciphertext before decrypt overwrites it */
ret = wc_Poly1305Update(&aead->poly, in, len);
if (ret == 0)
ret = wc_Chacha_Process(&aead->chacha, out, in, len);
}
}
}
if (ret == 0) {
wolfcrypt/src/chacha_asm.S
+9719 -144
View File
File diff suppressed because it is too large Load Diff
wolfcrypt/src/chacha_asm.asm
+9712 -144
View File
File diff suppressed because it is too large Load Diff
wolfcrypt/src/cpuid.c
+6
View File
@@ -54,9 +54,14 @@
new_cpuid_flags |= CPUID_ADX;
new_cpuid_flags |= CPUID_MOVBE;
new_cpuid_flags |= CPUID_BMI1;
new_cpuid_flags |= CPUID_SSSE3;
#ifdef WOLFSSL_SGX_CPUID_AVX512_VAES
new_cpuid_flags |= CPUID_VAES;
new_cpuid_flags |= CPUID_AVX512 | CPUID_AVX512_BW;
new_cpuid_flags |= CPUID_AVX512_VL;
new_cpuid_flags |= CPUID_AVX512_IFMA;
/* SGX is an Intel-only technology. */
new_cpuid_flags |= CPUID_INTEL;
#endif
(void)wolfSSL_Atomic_Uint_CompareExchange
@@ -166,6 +171,7 @@
}
if (cpuid_is_intel()) { new_cpuid_flags |= CPUID_INTEL ; }
if (cpuid_is_amd()) { new_cpuid_flags |= CPUID_AMD ; }
if (cpuid_flag(1, 0, ECX, 9)) { new_cpuid_flags |= CPUID_SSSE3 ; }
(void)wolfSSL_Atomic_Uint_CompareExchange
(&cpuid_flags, &old_cpuid_flags, new_cpuid_flags);
}
wolfcrypt/src/poly1305.c
+213 -2
View File
@@ -88,10 +88,80 @@ and Daniel J. Bernstein
#ifndef NO_AVX2_SUPPORT
#define HAVE_INTEL_AVX2
#endif
/* 8-way path; the struct carries r^5..r^8 only when this is on. */
#if defined(WOLFSSL_POLY1305_AVX512) && !defined(NO_AVX512_SUPPORT)
#define HAVE_INTEL_AVX512
#endif
#endif
#ifdef USE_INTEL_POLY1305_SPEEDUP
static cpuid_flags_t intel_flags = WC_CPUID_INITIALIZER;
/* The fused ChaCha20-Poly1305 kernel drives the 4-way path; when it is active
* this flag pins Update/Final to the 4-way path so the state layout matches. */
#ifdef WOLFSSL_CHACHA20_POLY1305_FUSED
#define POLY1305_FORCE_AVX2(ctx) ((ctx)->forceAvx2 != 0)
#else
#define POLY1305_FORCE_AVX2(ctx) 0
#endif
/* The fused IFMA stitch drives the scalar path; when it is active this flag
* pins Update/Final to the scalar (poly1305_blocks_avx / _final_avx) path so
* the state layout matches. */
#ifdef WOLFSSL_CHACHA20_POLY1305_FUSED_IFMA
#define POLY1305_FORCE_SCALAR(ctx) ((ctx)->forceScalar != 0)
#else
#define POLY1305_FORCE_SCALAR(ctx) 0
#endif
#ifdef HAVE_INTEL_AVX512
/* Whether to use the 8-way (512-bit) Poly1305 over the 4-way AVX2 path.
*
* The zmm path does eight 16-byte blocks per iteration and is the fastest
* option where 512-bit code runs at full clock: AMD Zen 4/5 (no AVX-512
* license) and Intel Ice Lake and later. On Intel Skylake-SP / Cascade
* Lake-class parts, sustained 512-bit instructions trip the AVX-512 frequency
* license and downclock the core; the penalty is milder here than for the
* FP-heavy ChaCha, but the 256-bit AVX2 path can still be the safer default on
* those parts. As with the ChaCha gate, VAES presence is used as a
* generational proxy: the throttling parts predate VAES, whereas every
* microarchitecture that runs 512-bit without penalty implements it. A missing
* VAES only costs a little throughput (fall back to AVX2), never correctness.
*
* Override the heuristic with:
* WOLFSSL_POLY1305_AVX512_ALWAYS - use zmm whenever AVX-512 is present
* WOLFSSL_POLY1305_AVX512_NEVER - never use zmm (always AVX2 or below)
*/
static WC_INLINE int poly1305_use_avx512(cpuid_flags_t flags)
{
#if defined(WOLFSSL_POLY1305_AVX512_NEVER)
(void)flags;
return 0;
#elif defined(WOLFSSL_POLY1305_AVX512_ALWAYS)
return IS_INTEL_AVX512(flags) != 0;
#else
return (IS_INTEL_AVX512(flags) != 0) && (IS_INTEL_VAES(flags) != 0);
#endif
}
/* The radix-2^44 IFMA path is preferred on Intel cores with AVX-512 IFMA
* (Ice Lake and later), where it beats the 26-bit vpmuludq 8-way. On AMD
* Zen 4/5 - which also implement IFMA - the vpmuludq 8-way is measurably faster
* (stronger vector-integer throughput), so IFMA is gated to an Intel vendor.
* Define WOLFSSL_POLY1305_IFMA_ALWAYS to use it on any IFMA CPU, or
* WOLFSSL_POLY1305_NO_IFMA to disable it. */
static WC_INLINE int poly1305_use_ifma(cpuid_flags_t flags)
{
#if defined(WOLFSSL_POLY1305_NO_IFMA)
(void)flags;
return 0;
#elif defined(WOLFSSL_POLY1305_IFMA_ALWAYS)
return IS_INTEL_AVX512_IFMA(flags) != 0;
#else
return (IS_INTEL_AVX512_IFMA(flags) != 0) && (IS_CPU_INTEL(flags) != 0);
#endif
}
#endif /* HAVE_INTEL_AVX512 */
#endif
#if defined(USE_INTEL_POLY1305_SPEEDUP) || defined(POLY130564)
@@ -204,6 +274,24 @@ WOLFSSL_LOCAL void poly1305_setkey_avx2(Poly1305* ctx, const byte* key);
WOLFSSL_LOCAL void poly1305_final_avx2(Poly1305* ctx, byte* mac);
#endif
#ifdef HAVE_INTEL_AVX512
/* Process multiple 128-byte (8 block) groups of data eight lanes wide. */
WOLFSSL_LOCAL void poly1305_blocks_avx512(Poly1305* ctx,
const unsigned char* m, size_t bytes);
/* Calculate R^1 .. R^8 and store them in the context. */
WOLFSSL_LOCAL void poly1305_calc_powers_avx512(Poly1305* ctx);
/* Calculate the final result - authentication data. Collapses the eight
* lanes and finishes any leftover with the AVX final function. */
WOLFSSL_LOCAL void poly1305_final_avx512(Poly1305* ctx, byte* mac);
/* AVX-512 IFMA (radix 2^44) 8-way variants - used when the CPU has IFMA. */
WOLFSSL_LOCAL void poly1305_blocks_avx512ifma(Poly1305* ctx,
const unsigned char* m, size_t bytes);
WOLFSSL_LOCAL void poly1305_calc_powers_avx512ifma(Poly1305* ctx);
WOLFSSL_LOCAL void poly1305_setkey_avx512ifma(Poly1305* ctx, const byte* key);
WOLFSSL_LOCAL void poly1305_final_avx512ifma(Poly1305* ctx, byte* mac);
#endif
#ifdef __cplusplus
} /* extern "C" */
#endif
@@ -854,6 +942,11 @@ int wc_Poly1305SetKey(Poly1305* ctx, const byte* key, word32 keySz)
#ifdef USE_INTEL_POLY1305_SPEEDUP
cpuid_get_flags_ex(&intel_flags);
SAVE_VECTOR_REGISTERS(return _svr_ret;);
#ifdef HAVE_INTEL_AVX512
if (poly1305_use_ifma(intel_flags))
poly1305_setkey_avx512ifma(ctx, key);
else
#endif
#ifdef HAVE_INTEL_AVX2
if (IS_INTEL_AVX2(intel_flags))
poly1305_setkey_avx2(ctx, key);
@@ -862,6 +955,12 @@ int wc_Poly1305SetKey(Poly1305* ctx, const byte* key, word32 keySz)
poly1305_setkey_avx(ctx, key);
RESTORE_VECTOR_REGISTERS();
ctx->started = 0;
#ifdef WOLFSSL_CHACHA20_POLY1305_FUSED
ctx->forceAvx2 = 0;
#endif
#ifdef WOLFSSL_CHACHA20_POLY1305_FUSED_IFMA
ctx->forceScalar = 0;
#endif
#elif defined(WOLFSSL_ARMASM)
#ifdef __aarch64__
poly1305_setkey_aarch64(ctx, key);
@@ -885,8 +984,18 @@ int wc_Poly1305Final(Poly1305* ctx, byte* mac)
#ifdef USE_INTEL_POLY1305_SPEEDUP
SAVE_VECTOR_REGISTERS(return _svr_ret;);
#ifdef HAVE_INTEL_AVX512
if (!POLY1305_FORCE_AVX2(ctx) && !POLY1305_FORCE_SCALAR(ctx) &&
poly1305_use_ifma(intel_flags))
poly1305_final_avx512ifma(ctx, mac);
else
if (!POLY1305_FORCE_AVX2(ctx) && !POLY1305_FORCE_SCALAR(ctx) &&
poly1305_use_avx512(intel_flags))
poly1305_final_avx512(ctx, mac);
else
#endif
#ifdef HAVE_INTEL_AVX2
if (IS_INTEL_AVX2(intel_flags))
if (!POLY1305_FORCE_SCALAR(ctx) && IS_INTEL_AVX2(intel_flags))
poly1305_final_avx2(ctx, mac);
else
#endif
@@ -1008,8 +1117,110 @@ int wc_Poly1305Update(Poly1305* ctx, const byte* m, word32 bytes)
}
#else
#ifdef USE_INTEL_POLY1305_SPEEDUP
#ifdef HAVE_INTEL_AVX512
if (!POLY1305_FORCE_AVX2(ctx) && !POLY1305_FORCE_SCALAR(ctx) &&
poly1305_use_ifma(intel_flags)) {
SAVE_VECTOR_REGISTERS(return _svr_ret;);
/* handle leftover */
if (ctx->leftover) {
size_t want = sizeof(ctx->buffer) - ctx->leftover;
if (want > bytes)
want = bytes;
for (i = 0; i < want; i++)
ctx->buffer[ctx->leftover + i] = m[i];
bytes -= (word32)want;
m += want;
ctx->leftover += want;
if (ctx->leftover < sizeof(ctx->buffer)) {
RESTORE_VECTOR_REGISTERS();
return 0;
}
if (!ctx->started) {
poly1305_calc_powers_avx512ifma(ctx);
ctx->started = 1;
}
poly1305_blocks_avx512ifma(ctx, ctx->buffer, sizeof(ctx->buffer));
ctx->leftover = 0;
}
/* process full blocks */
if (bytes >= sizeof(ctx->buffer)) {
size_t want = bytes & ~(sizeof(ctx->buffer) - 1);
if (!ctx->started) {
poly1305_calc_powers_avx512ifma(ctx);
ctx->started = 1;
}
poly1305_blocks_avx512ifma(ctx, m, want);
m += want;
bytes -= (word32)want;
}
/* store leftover */
if (bytes) {
for (i = 0; i < bytes; i++)
ctx->buffer[ctx->leftover + i] = m[i];
ctx->leftover += bytes;
}
RESTORE_VECTOR_REGISTERS();
}
else
if (!POLY1305_FORCE_AVX2(ctx) && !POLY1305_FORCE_SCALAR(ctx) &&
poly1305_use_avx512(intel_flags)) {
SAVE_VECTOR_REGISTERS(return _svr_ret;);
/* handle leftover */
if (ctx->leftover) {
size_t want = sizeof(ctx->buffer) - ctx->leftover;
if (want > bytes)
want = bytes;
for (i = 0; i < want; i++)
ctx->buffer[ctx->leftover + i] = m[i];
bytes -= (word32)want;
m += want;
ctx->leftover += want;
if (ctx->leftover < sizeof(ctx->buffer)) {
RESTORE_VECTOR_REGISTERS();
return 0;
}
if (!ctx->started) {
poly1305_calc_powers_avx512(ctx);
ctx->started = 1;
}
poly1305_blocks_avx512(ctx, ctx->buffer, sizeof(ctx->buffer));
ctx->leftover = 0;
}
/* process full blocks */
if (bytes >= sizeof(ctx->buffer)) {
size_t want = bytes & ~(sizeof(ctx->buffer) - 1);
if (!ctx->started) {
poly1305_calc_powers_avx512(ctx);
ctx->started = 1;
}
poly1305_blocks_avx512(ctx, m, want);
m += want;
bytes -= (word32)want;
}
/* store leftover */
if (bytes) {
for (i = 0; i < bytes; i++)
ctx->buffer[ctx->leftover + i] = m[i];
ctx->leftover += bytes;
}
RESTORE_VECTOR_REGISTERS();
}
else
#endif
#ifdef HAVE_INTEL_AVX2
if (IS_INTEL_AVX2(intel_flags)) {
if (!POLY1305_FORCE_SCALAR(ctx) && IS_INTEL_AVX2(intel_flags)) {
SAVE_VECTOR_REGISTERS(return _svr_ret;);
/* handle leftover */
wolfcrypt/src/poly1305_asm.S
+2464
View File
File diff suppressed because it is too large Load Diff
wolfcrypt/src/poly1305_asm.asm
+2370 -4
View File
File diff suppressed because it is too large Load Diff
wolfssl/wolfcrypt/chacha20_poly1305.h
+14
View File
@@ -104,6 +104,20 @@ int wc_ChaCha20Poly1305_Decrypt(
const byte inAuthTag[CHACHA20_POLY1305_AEAD_AUTHTAG_SIZE],
byte* outPlaintext);
/* Encrypt+authenticate one message with pre-keyed ChaCha20 and Poly1305
* contexts (analogue of wc_AesGcmEncrypt on a keyed Aes) - keeps the ChaCha key
* across records and only varies the nonce, and uses the single-pass stitch.
* Intended for the TLS record layer. */
WOLFSSL_API int wc_ChaCha20Poly1305_Encrypt_ex(ChaCha* chacha, Poly1305* poly,
byte* out, const byte* in, word32 sz, const byte* nonce, byte* tag,
const byte* aad, word32 aadSz);
/* Decrypt+verify counterpart of wc_ChaCha20Poly1305_Encrypt_ex. On tag
* mismatch returns MAC_CMP_FAILED_E and zeroizes out (no plaintext released);
* out may alias in (in-place). */
WOLFSSL_API WARN_UNUSED_RESULT int wc_ChaCha20Poly1305_Decrypt_ex(
ChaCha* chacha, Poly1305* poly, byte* out, const byte* in, word32 sz,
const byte* nonce, const byte* tag, const byte* aad, word32 aadSz);
WOLFSSL_API WARN_UNUSED_RESULT
int wc_ChaCha20Poly1305_CheckTag(
const byte authTag[CHACHA20_POLY1305_AEAD_AUTHTAG_SIZE],
wolfssl/wolfcrypt/cpuid.h
+2
View File
@@ -105,6 +105,7 @@ typedef word32 cpuid_flags_t;
/* AVX-512 Byte and Word: byte/word instructions at 512-bit width
* (vpshufb, vpaddw, vpmulhw, vpackusdw, ... on zmm). */
#define CPUID_AVX512_BW 0x80000
#define CPUID_SSSE3 0x100000 /* SSSE3 (pshufb etc.) */
#define IS_INTEL_AVX1(f) (WOLFSSL_ATOMIC_COERCE_UINT(f) & CPUID_AVX1)
#define IS_INTEL_AVX2(f) (WOLFSSL_ATOMIC_COERCE_UINT(f) & CPUID_AVX2)
@@ -139,6 +140,7 @@ typedef word32 cpuid_flags_t;
(IS_INTEL_AVX512(f) && IS_INTEL_AVX512_BW(f))
#define IS_CPU_INTEL(f) (WOLFSSL_ATOMIC_COERCE_UINT(f) & CPUID_INTEL)
#define IS_CPU_AMD(f) (WOLFSSL_ATOMIC_COERCE_UINT(f) & CPUID_AMD)
#define IS_INTEL_SSSE3(f) (WOLFSSL_ATOMIC_COERCE_UINT(f) & CPUID_SSSE3)
#elif defined(HAVE_CPUID_AARCH64)
wolfssl/wolfcrypt/poly1305.h
+53
View File
@@ -52,6 +52,34 @@
#if defined(USE_INTEL_SPEEDUP) && !defined(NO_POLY1305_ASM)
#define USE_INTEL_POLY1305_SPEEDUP
#define HAVE_INTEL_AVX1
/* 8-way AVX-512 path. Enabling it appends r^5..r^8 to the state (see the
* struct below); define WOLFSSL_POLY1305_NO_AVX512 to keep the smaller
* state and drop the path. Gated on NO_AVX512_SUPPORT to match the
* HAVE_INTEL_AVX512 kernels the .S files emit (poly1305_asm.S). */
#if !defined(WOLFSSL_POLY1305_NO_AVX512) && !defined(NO_AVX512_SUPPORT)
#define WOLFSSL_POLY1305_AVX512
#endif
/* Fused single-pass ChaCha20-Poly1305 (encrypt). It drives Poly1305 in the
* 4-way layout, so the ctx carries a flag forcing that path. The kernel is
* an AVX-512 ChaCha (chacha20_poly1305_avx512) feeding an AVX2 4-way power
* precompute (poly1305_calc_powers_avx2), so it needs BOTH ISA levels the
* .S files emit - gate off if either is disabled. */
#if !defined(WOLFSSL_NO_CHACHA20_POLY1305_FUSED) && \
!defined(NO_AVX512_SUPPORT) && !defined(NO_AVX2_SUPPORT)
#define WOLFSSL_CHACHA20_POLY1305_FUSED
#endif
/* IFMA stitched single-pass ChaCha20-Poly1305 (AVX-512 + IFMA): a full
* 512-bit 16-block ChaCha interleaved with an 8-way IFMA (vpmadd52)
* Poly1305 that collapses to the scalar hash. It BEATS the two-pass by
* ~1.3-1.4x (>=16KB) - ChaCha is the bottleneck and Poly hides under it -
* so it is ON by default and runtime-gated on the AVX-512 + IFMA flags.
* Needs the AVX-512 IFMA state (r^1..r^8, ifma_h), so it follows
* WOLFSSL_POLY1305_AVX512. Define WOLFSSL_NO_CHACHA20_POLY1305_FUSED_IFMA
* to drop it; it drives Poly1305 scalar via the forceScalar flag. */
#if defined(WOLFSSL_POLY1305_AVX512) && \
!defined(WOLFSSL_NO_CHACHA20_POLY1305_FUSED_IFMA)
#define WOLFSSL_CHACHA20_POLY1305_FUSED_IFMA
#endif
#endif
#endif
@@ -100,6 +128,31 @@ typedef struct Poly1305 {
size_t leftover;
unsigned char finished;
unsigned char started;
#ifdef WOLFSSL_POLY1305_AVX512
/* r^5..r^8 for the 8-way path, appended so the AVX1/AVX2 field offsets are
* unchanged. ALIGN8 keeps each power 8-byte aligned (26-bit limb packing
* matches r1..r4). */
ALIGN8 word32 r5[8];
word32 r6[8];
word32 r7[8];
word32 r8[8];
/* IFMA path (radix 2^44) keeps r^1..r^8 in the r1..r8 fields above (three
* 44-bit limbs each) and its eight-lane running hash here: three limbs x
* eight lanes x 64-bit. hh (32-bit packed) is too small for 44-bit lanes.
*/
ALIGN8 word64 ifma_h[24];
#endif
#ifdef WOLFSSL_CHACHA20_POLY1305_FUSED
/* When set, wc_Poly1305Update/Final use the 4-way path so the fused
* ChaCha20-Poly1305 kernel (also 4-way) stays layout-consistent. */
unsigned char forceAvx2;
#endif
#ifdef WOLFSSL_CHACHA20_POLY1305_FUSED_IFMA
/* When set, wc_Poly1305Update/Final use the scalar path so the fused IFMA
* ChaCha20-Poly1305 stitch (which finishes in the scalar hash) stays
* layout-consistent. */
unsigned char forceScalar;
#endif
#elif defined(WOLFSSL_ARMASM) && defined(__aarch64__)
ALIGN8 word64 r64[2];
ALIGN8 word32 r4[4];