wolfSSL/wolfssl
1
0
Fork 1
mirror of https://github.com/wolfSSL/wolfssl.git synced 2025-07-31 19:24:42 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #4407 from douzzer/linuxkm-SIMD-IRQ

Browse Source 
linuxkm-SIMD-IRQ
This commit is contained in:
David Garske
2021-09-20 14:57:38 -07:00
committed by GitHub
parent 753a931196 6d715130a2
commit 34c6e8f975
29 changed files with 1032 additions and 261 deletions
Download Patch File Download Diff File Expand all files Collapse all files

125
configure.ac
View File

@@ -180,6 +180,42 @@ AS_IF([test "$ax_enable_debug" = "yes"],
ENABLED_CERTS="no"
# Support for forcing 32-bit mode
# To force 32-bit instructions use:
# ./configure CFLAGS="-m32" LDFLAGS="-m32" && make
# The checks for sizeof long and long/long are run at the top of configure and require "-m32" to be set directly in the ./configure statement.
AC_ARG_ENABLE([32bit],
[AS_HELP_STRING([--enable-32bit],[Enables 32-bit support (default: disabled)])],
[ ENABLED_32BIT=$enableval ],
[ ENABLED_32BIT=no ]
)
# 16-bit compiler support
AC_ARG_ENABLE([16bit],
[AS_HELP_STRING([--enable-16bit],[Enables 16-bit support (default: disabled)])],
[ ENABLED_16BIT=$enableval ],
[ ENABLED_16BIT=no ]
)
if test "$ENABLED_16BIT" = "yes"
then
AM_CFLAGS="$AM_CFLAGS -DWC_16BIT_CPU"
fi
# Support for disabling all ASM
AC_ARG_ENABLE([asm],
[AS_HELP_STRING([--enable-asm],[Enables option for assembly (default: enabled)])],
[ ENABLED_ASM=$enableval ],
[ ENABLED_ASM=yes ]
)
if test "$ENABLED_ASM" = "no"
then
AM_CFLAGS="$AM_CFLAGS -DTFM_NO_ASM -DWOLFSSL_NO_ASM"
fi
AC_SUBST([ENABLED_ASM])
# FIPS
AC_ARG_ENABLE([fips],
@@ -373,9 +409,7 @@ then
test "$enable_aescbc_length_checks" = "" && enable_aescbc_length_checks=yes
test "$enable_camellia" = "" && enable_camellia=yes
test "$enable_ripemd" = "" && enable_ripemd=yes
test "$enable_sha512" = "" && enable_sha512=yes
test "$enable_sha224" = "" && enable_sha224=yes
test "$enable_sha3" = "" && enable_sha3=yes
test "$enable_sessioncerts" = "" && enable_sessioncerts=yes
test "$enable_keygen" = "" && enable_keygen=yes
test "$enable_certgen" = "" && enable_certgen=yes
@@ -431,6 +465,12 @@ then
test "$enable_anon" = "" && enable_anon=yes
test "$enable_mcast" = "" && enable_mcast=yes
if test "$ENABLED_32BIT" != "yes"
then
test "$enable_sha512" = "" && enable_sha512=yes
test "$enable_sha3" = "" && enable_sha3=yes
fi
if test "$ENABLED_LINUXKM_DEFAULTS" != "yes"
then
test "$enable_compkey" = "" && enable_compkey=yes
@@ -448,7 +488,10 @@ then
then
if test "$ENABLED_FIPS" = "no"
then
test "$enable_openssh" = "" && enable_openssh=yes
if test "$ENABLED_32BIT" != "yes"
then
test "$enable_openssh" = "" && enable_openssh=yes
fi
# S/MIME support requires PKCS7, which requires no FIPS.
test "$enable_smime" = "" && enable_smime=yes
fi
@@ -474,10 +517,13 @@ then
test "$enable_xchacha" = "" && enable_xchacha=yes
test "$enable_scep" = "" && enable_scep=yes
test "$enable_pkcs7" = "" && enable_pkcs7=yes
test "$enable_ed25519" = "" && enable_ed25519=yes
test "$enable_ed25519_stream" = "" && enable_ed25519_stream=yes
test "$enable_ed448" = "" && enable_ed448=yes
test "$enable_ed448_stream" = "" && enable_ed448_stream=yes
if test "$ENABLED_32BIT" != "yes"
then
test "$enable_ed25519" = "" && enable_ed25519=yes
test "$enable_ed25519_stream" = "" && enable_ed25519_stream=yes
test "$enable_ed448" = "" && enable_ed448=yes
test "$enable_ed448_stream" = "" && enable_ed448_stream=yes
fi
if test "$ENABLED_LINUXKM_DEFAULTS" != "yes"
then
@@ -525,9 +571,7 @@ then
test "$enable_aescfb" = "" && enable_aescfb=yes
test "$enable_camellia" = "" && enable_camellia=yes
test "$enable_ripemd" = "" && enable_ripemd=yes
test "$enable_sha512" = "" && enable_sha512=yes
test "$enable_sha224" = "" && enable_sha224=yes
test "$enable_sha3" = "" && enable_sha3=yes
test "$enable_sessioncerts" = "" && enable_sessioncerts=yes
test "$enable_keygen" = "" && enable_keygen=yes
test "$enable_certgen" = "" && enable_certgen=yes
@@ -569,6 +613,12 @@ then
test "$enable_cryptocb" = "" && enable_cryptocb=yes
test "$enable_anon" = "" && enable_anon=yes
if test "$ENABLED_32BIT" != "yes"
then
test "$enable_sha512" = "" && enable_sha512=yes
test "$enable_sha3" = "" && enable_sha3=yes
fi
if test "$ENABLED_LINUXKM_DEFAULTS" != "yes"
then
test "$enable_compkey" = "" && enable_compkey=yes
@@ -586,11 +636,14 @@ then
if test "$ENABLED_FIPS" = "no"
then
test "$enable_xchacha" = "" && enable_xchacha=yes
test "$enable_ed25519" = "" && enable_ed25519=yes
test "$enable_ed25519_stream" = "" && enable_ed25519_stream=yes
test "$enable_ed448" = "" && enable_ed448=yes
test "$enable_ed448_stream" = "" && enable_ed448_stream=yes
test "$enable_pkcs7" = "" && enable_pkcs7=yes
if test "$ENABLED_32BIT" != "yes"
then
test "$enable_ed25519" = "" && enable_ed25519=yes
test "$enable_ed25519_stream" = "" && enable_ed25519_stream=yes
test "$enable_ed448" = "" && enable_ed448=yes
test "$enable_ed448_stream" = "" && enable_ed448_stream=yes
fi
if test "$ENABLED_LINUXKM_DEFAULTS" != "yes"
then
@@ -615,43 +668,6 @@ then
fi
# Support for forcing 32-bit mode
# To force 32-bit instructions use:
# ./configure CFLAGS="-m32" LDFLAGS="-m32" && make
# The checks for sizeof long and long/long are run at the top of configure and require "-m32" to be set directly in the ./configure statement.
AC_ARG_ENABLE([32bit],
[AS_HELP_STRING([--enable-32bit],[Enables 32-bit support (default: disabled)])],
[ ENABLED_32BIT=$enableval ],
[ ENABLED_32BIT=no ]
)
# 16-bit compiler support
AC_ARG_ENABLE([16bit],
[AS_HELP_STRING([--enable-16bit],[Enables 16-bit support (default: disabled)])],
[ ENABLED_16BIT=$enableval ],
[ ENABLED_16BIT=no ]
)
if test "$ENABLED_16BIT" = "yes"
then
AM_CFLAGS="$AM_CFLAGS -DWC_16BIT_CPU"
fi
# Support for disabling all ASM
AC_ARG_ENABLE([asm],
[AS_HELP_STRING([--enable-asm],[Enables option for assembly (default: enabled)])],
[ ENABLED_ASM=$enableval ],
[ ENABLED_ASM=yes ]
)
if test "$ENABLED_ASM" = "no"
then
AM_CFLAGS="$AM_CFLAGS -DTFM_NO_ASM -DWOLFSSL_NO_ASM"
fi
AC_SUBST([ENABLED_ASM])
# SINGLE THREADED
AC_ARG_ENABLE([singlethreaded],
[AS_HELP_STRING([--enable-singlethreaded],[Enable wolfSSL single threaded (default: disabled)])],
@@ -1837,7 +1853,7 @@ fi
# set sha3 default
SHA3_DEFAULT=no
if test "$host_cpu" = "x86_64" || test "$host_cpu" = "aarch64"
if (test "$host_cpu" = "x86_64" || test "$host_cpu" = "aarch64") && test "$ENABLED_32BIT" = "no"
then
if test "x$ENABLED_FIPS" = "xno" || test "x$FIPS_VERSION" = "xv2"
then
@@ -6451,7 +6467,8 @@ AS_IF([test "x$ENABLED_CERT_REQ" = "xyes" && \
# ed25519 requires sha512
AS_IF([test "x$ENABLED_ED25519" = "xyes" && \
test "x$ENABLED_SHA512" = "xno"],
test "x$ENABLED_SHA512" = "xno" && \
test "x$ENABLED_32BIT" = "xno"],
[AC_MSG_ERROR([cannot enable ed25519 without enabling sha512.])])
# ed25519 stream requires ed25519
@@ -6942,7 +6959,9 @@ AM_CONDITIONAL([BUILD_OCTEON_SYNC],[test "x$ENABLED_OCTEON_SYNC" = "xyes"])
AM_CONDITIONAL([BUILD_INTEL_QA],[test "x$ENABLED_INTEL_QA" = "xyes"])
AM_CONDITIONAL([BUILD_INTEL_QA_SYNC],[test "x$ENABLED_INTEL_QA_SYNC" = "xyes"])
AM_CONDITIONAL([BUILD_SP],[test "x$ENABLED_SP" = "xyes" || test "x$ENABLED_USERSETTINGS" = "xyes"])
AM_CONDITIONAL([BUILD_SP_C],[(test "x$ENABLED_SP" = "xyes" && test "x$ENABLED_SP_ASM" = "xno") || test "x$ENABLED_USERSETTINGS" = "xyes"])
AM_CONDITIONAL([BUILD_SP_C],[((test "$ENABLED_SP_RSA" = "yes" || test "$ENABLED_SP_DH" = "yes" \
|| test "$ENABLED_SP_ECC" = "yes") && test "x$ENABLED_SP_ASM" = "xno") \
|| test "x$ENABLED_USERSETTINGS" = "xyes"])
AM_CONDITIONAL([BUILD_SP_ARM64],[test "x$ENABLED_SP_ARM64_ASM" = "xyes" || test "x$ENABLED_USERSETTINGS" = "xyes"])
AM_CONDITIONAL([BUILD_SP_ARM32],[test "x$ENABLED_SP_ARM32_ASM" = "xyes" || test "x$ENABLED_USERSETTINGS" = "xyes"])
AM_CONDITIONAL([BUILD_SP_ARM_THUMB],[test "x$ENABLED_SP_ARM_THUMB_ASM" = "xyes" || test "x$ENABLED_USERSETTINGS" = "xyes"])

17
linuxkm/Kbuild
View File

@@ -17,7 +17,7 @@
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA
#/
#
SHELL=/bin/bash
@@ -48,6 +48,7 @@ $(obj)/linuxkm/module_exports.o: $(WOLFSSL_OBJ_TARGETS)
# this mechanism only works in kernel 5.x+ (fallback to hardcoded value)
hostprogs := linuxkm/get_thread_size
always-y := $(hostprogs)
always := $(hostprogs)
HOST_EXTRACFLAGS += $(NOSTDINC_FLAGS) $(LINUXINCLUDE) $(KBUILD_CFLAGS) -static
# this rule is needed to get build to succeed in 4.x (get_thread_size still doesn't get built)
@@ -91,14 +92,20 @@ endif
asflags-y := $(WOLFSSL_ASFLAGS) $(ASFLAGS_FPUSIMD_DISABLE)
# vectorized algorithms equipped with {SAVE,RESTORE}_VECTOR_REGISTERS()
# can be safely included here:
# vectorized algorithms protected by {SAVE,RESTORE}_VECTOR_REGISTERS() can be
# safely included here, though many of these are not yet kernel-compatible:
$(obj)/wolfcrypt/src/aes_asm.o: asflags-y = $(WOLFSSL_ASFLAGS) $(ASFLAGS_FPU_DISABLE_SIMD_ENABLE)
$(obj)/wolfcrypt/src/aes_gcm_asm.o: asflags-y = $(WOLFSSL_ASFLAGS) $(ASFLAGS_FPU_DISABLE_SIMD_ENABLE)
$(obj)/wolfcrypt/src/sha256_asm.o: asflags-y = $(WOLFSSL_ASFLAGS) $(ASFLAGS_FPU_DISABLE_SIMD_ENABLE)
$(obj)/wolfcrypt/src/sp_x86_64_asm.o: asflags-y = $(WOLFSSL_ASFLAGS) $(ASFLAGS_FPU_DISABLE_SIMD_ENABLE)
$(obj)/wolfcrypt/src/sha512_asm.o: asflags-y = $(WOLFSSL_ASFLAGS) $(ASFLAGS_FPU_DISABLE_SIMD_ENABLE)
$(obj)/wolfcrypt/src/poly1305_asm.o: asflags-y = $(WOLFSSL_ASFLAGS) $(ASFLAGS_FPU_DISABLE_SIMD_ENABLE)
$(obj)/wolfcrypt/src/chacha_asm.o: asflags-y = $(WOLFSSL_ASFLAGS) $(ASFLAGS_FPU_DISABLE_SIMD_ENABLE)
# these _asms are kernel-compatible, but they still irritate objtool:
# these _asms are known kernel-compatible, but they still irritate objtool:
$(obj)/wolfcrypt/src/aes_asm.o: OBJECT_FILES_NON_STANDARD := y
$(obj)/wolfcrypt/src/aes_gcm_asm.o: OBJECT_FILES_NON_STANDARD := y
$(obj)/wolfcrypt/src/sp_x86_64_asm.o: OBJECT_FILES_NON_STANDARD := y
ifeq "$(ENABLED_LINUXKM_PIE)" "yes"
@@ -158,7 +165,7 @@ $(src)/linuxkm/module_exports.c: $(src)/linuxkm/module_exports.c.template $(WOLF
@cp $< $@
@readelf --symbols --wide $(WOLFSSL_OBJ_TARGETS) | \
awk '/^ *[0-9]+: / { \
if ($$8 !~ /^(wc_|wolf)/){next;} \
if ($$8 !~ /^(wc_|wolf|WOLF|TLSX_)/){next;} \
if (($$4 == "FUNC") && ($$5 == "GLOBAL") && ($$6 == "DEFAULT")) { \
print "EXPORT_SYMBOL_NS(" $$8 ", WOLFSSL);"; \
} \

37
linuxkm/Makefile
View File

@@ -21,7 +21,7 @@
SHELL=/bin/bash
all: libwolfssl.ko
all: libwolfssl.ko libwolfssl.ko.signed
.PHONY: libwolfssl.ko
@@ -61,7 +61,42 @@ libwolfssl.ko:
@if test -z "$(src_libwolfssl_la_OBJECTS)"; then echo '$$src_libwolfssl_la_OBJECTS is unset.' >&2; exit 1; fi
@mkdir -p linuxkm src wolfcrypt/src wolfcrypt/test
@if test ! -h $(SRC_TOP)/Kbuild; then ln -s $(MODULE_TOP)/Kbuild $(SRC_TOP)/Kbuild; fi
ifeq "$(ENABLED_LINUXKM_PIE)" "yes"
+$(MAKE) -C $(KERNEL_ROOT) M=$(MODULE_TOP) src=$(SRC_TOP) CC_FLAGS_FTRACE=
else
+$(MAKE) -C $(KERNEL_ROOT) M=$(MODULE_TOP) src=$(SRC_TOP)
endif
libwolfssl.ko.signed: libwolfssl.ko
@cd '$(KERNEL_ROOT)' || exit $$?; \
while read configline; do \
case "$$configline" in \
CONFIG_MODULE_SIG_KEY=*) \
CONFIG_MODULE_SIG_KEY="$${configline#CONFIG_MODULE_SIG_KEY=}" \
;; \
CONFIG_MODULE_SIG_HASH=*) \
CONFIG_MODULE_SIG_HASH="$${configline#CONFIG_MODULE_SIG_HASH=}" \
;; \
esac; \
done < .config || exit $$?; \
if [[ -n "$${CONFIG_MODULE_SIG_KEY}" && -n "$${CONFIG_MODULE_SIG_HASH}" && \
( ! -f '$(MODULE_TOP)/$@' || \
'$(MODULE_TOP)/$<' -nt '$(MODULE_TOP)/$@' ) ]]; then \
CONFIG_MODULE_SIG_KEY="$${CONFIG_MODULE_SIG_KEY#\"}"; \
CONFIG_MODULE_SIG_KEY="$${CONFIG_MODULE_SIG_KEY%\"}"; \
CONFIG_MODULE_SIG_HASH="$${CONFIG_MODULE_SIG_HASH#\"}"; \
CONFIG_MODULE_SIG_HASH="$${CONFIG_MODULE_SIG_HASH%\"}"; \
cp -p '$(MODULE_TOP)/$<' '$(MODULE_TOP)/$@' || exit $$?; \
./scripts/sign-file "$${CONFIG_MODULE_SIG_HASH}" \
"$${CONFIG_MODULE_SIG_KEY}" \
"$${CONFIG_MODULE_SIG_KEY/%.pem/.x509}" \
'$(MODULE_TOP)/$@' || \
$(RM) -f '$(MODULE_TOP)/$@' || exit $$?; \
if [[ "$(quiet)" != "silent_" ]]; then \
echo " Module $@ signed by $${CONFIG_MODULE_SIG_KEY}."; \
fi \
fi
.PHONY: install modules_install
install modules_install:

139
linuxkm/module_hooks.c
View File

@@ -19,6 +19,10 @@
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA
*/
#ifndef WOLFSSL_LICENSE
#define WOLFSSL_LICENSE "GPL v2"
#endif
#define FIPS_NO_WRAPPERS
#ifdef HAVE_CONFIG_H
@@ -41,13 +45,13 @@ static int libwolfssl_cleanup(void) {
#ifdef WOLFCRYPT_ONLY
ret = wolfCrypt_Cleanup();
if (ret != 0)
pr_err("wolfCrypt_Cleanup() failed: %s", wc_GetErrorString(ret));
pr_err("wolfCrypt_Cleanup() failed: %s\n", wc_GetErrorString(ret));
else
pr_info("wolfCrypt " LIBWOLFSSL_VERSION_STRING " cleanup complete.\n");
#else
ret = wolfSSL_Cleanup();
if (ret != WOLFSSL_SUCCESS)
pr_err("wolfSSL_Cleanup() failed: %s", wc_GetErrorString(ret));
pr_err("wolfSSL_Cleanup() failed: %s\n", wc_GetErrorString(ret));
else
pr_info("wolfSSL " LIBWOLFSSL_VERSION_STRING " cleanup complete.\n");
#endif
@@ -89,8 +93,8 @@ static void lkmFipsCb(int ok, int err, const char* hash)
if ((! ok) || (err != 0))
pr_err("libwolfssl FIPS error: %s\n", wc_GetErrorString(err));
if (err == IN_CORE_FIPS_E) {
pr_err("In-core integrity hash check failure.\n");
pr_err("Update verifyCore[] in fips_test.c with new hash \"%s\" and rebuild.\n",
pr_err("In-core integrity hash check failure.\n"
"Update verifyCore[] in fips_test.c with new hash \"%s\" and rebuild.\n",
hash ? hash : "<null>");
}
}
@@ -104,6 +108,13 @@ static int wolfssl_init(void)
{
int ret;
#ifdef CONFIG_MODULE_SIG
if (THIS_MODULE->sig_ok == false) {
pr_err("wolfSSL module load aborted -- bad or missing module signature with CONFIG_MODULE_SIG kernel.\n");
return -ECANCELED;
}
#endif
#ifdef USE_WOLFSSL_LINUXKM_PIE_REDIRECT_TABLE
ret = set_up_wolfssl_linuxkm_pie_redirect_table();
if (ret < 0)
@@ -112,6 +123,17 @@ static int wolfssl_init(void)
#endif
#ifdef HAVE_LINUXKM_PIE_SUPPORT
#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 5, 0)
#define THIS_MODULE_BASE (THIS_MODULE->core_layout.base)
#define THIS_MODULE_TEXT_SIZE (THIS_MODULE->core_layout.text_size)
#define THIS_MODULE_RO_SIZE (THIS_MODULE->core_layout.ro_size)
#else
#define THIS_MODULE_BASE (THIS_MODULE->module_core)
#define THIS_MODULE_TEXT_SIZE (THIS_MODULE->core_text_size)
#define THIS_MODULE_RO_SIZE (THIS_MODULE->core_ro_size)
#endif
{
char *pie_text_start = (char *)wolfCrypt_PIE_first_function;
char *pie_text_end = (char *)wolfCrypt_PIE_last_function;
@@ -120,8 +142,8 @@ static int wolfssl_init(void)
unsigned int text_hash, rodata_hash;
if ((pie_text_start < pie_text_end) &&
(pie_text_start >= (char *)(THIS_MODULE->core_layout.base)) &&
(pie_text_end - (char *)(THIS_MODULE->core_layout.base) <= THIS_MODULE->core_layout.text_size))
(pie_text_start >= (char *)THIS_MODULE_BASE) &&
(pie_text_end - (char *)THIS_MODULE_BASE <= THIS_MODULE_TEXT_SIZE))
{
text_hash = hash_span(pie_text_start, pie_text_end);
} else {
@@ -130,14 +152,14 @@ static int wolfssl_init(void)
pie_text_start,
pie_text_end,
pie_text_end-pie_text_start,
THIS_MODULE->core_layout.base,
(char *)(THIS_MODULE->core_layout.base) + THIS_MODULE->core_layout.text_size);
THIS_MODULE_BASE,
(char *)THIS_MODULE_BASE + THIS_MODULE_TEXT_SIZE);
text_hash = 0;
}
if ((pie_rodata_start < pie_rodata_end) &&
(pie_rodata_start >= (char *)(THIS_MODULE->core_layout.base) + THIS_MODULE->core_layout.text_size) &&
(pie_rodata_end - (char *)(THIS_MODULE->core_layout.base) <= THIS_MODULE->core_layout.ro_size))
(pie_rodata_start >= (char *)THIS_MODULE_BASE + THIS_MODULE_TEXT_SIZE) &&
(pie_rodata_end - (char *)THIS_MODULE_BASE <= THIS_MODULE_RO_SIZE))
{
rodata_hash = hash_span(pie_rodata_start, pie_rodata_end);
} else {
@@ -146,8 +168,8 @@ static int wolfssl_init(void)
pie_rodata_start,
pie_rodata_end,
pie_rodata_end-pie_rodata_start,
(char *)(THIS_MODULE->core_layout.base) + THIS_MODULE->core_layout.text_size,
(char *)(THIS_MODULE->core_layout.base) + THIS_MODULE->core_layout.ro_size);
(char *)THIS_MODULE_BASE + THIS_MODULE_TEXT_SIZE,
(char *)THIS_MODULE_BASE + THIS_MODULE_RO_SIZE);
rodata_hash = 0;
}
@@ -158,20 +180,20 @@ static int wolfssl_init(void)
pr_info("wolfCrypt container hashes (spans): %x (%lu) %x (%lu), module base %pK\n",
text_hash, pie_text_end-pie_text_start,
rodata_hash, pie_rodata_end-pie_rodata_start,
THIS_MODULE->core_layout.base);
THIS_MODULE_BASE);
}
#endif /* HAVE_LINUXKM_PIE_SUPPORT */
#ifdef HAVE_FIPS
ret = wolfCrypt_SetCb_fips(lkmFipsCb);
if (ret != 0) {
pr_err("wolfCrypt_SetCb_fips() failed: %s", wc_GetErrorString(ret));
pr_err("wolfCrypt_SetCb_fips() failed: %s\n", wc_GetErrorString(ret));
return -ECANCELED;
}
fipsEntry();
ret = wolfCrypt_GetStatus_fips();
if (ret != 0) {
pr_err("wolfCrypt_GetStatus_fips() failed: %s", wc_GetErrorString(ret));
pr_err("wolfCrypt_GetStatus_fips() failed: %s\n", wc_GetErrorString(ret));
if (ret == IN_CORE_FIPS_E) {
const char *newhash = wolfCrypt_GetCoreHash_fips();
pr_err("Update verifyCore[] in fips_test.c with new hash \"%s\" and rebuild.\n",
@@ -198,13 +220,13 @@ static int wolfssl_init(void)
#ifdef WOLFCRYPT_ONLY
ret = wolfCrypt_Init();
if (ret != 0) {
pr_err("wolfCrypt_Init() failed: %s", wc_GetErrorString(ret));
pr_err("wolfCrypt_Init() failed: %s\n", wc_GetErrorString(ret));
return -ECANCELED;
}
#else
ret = wolfSSL_Init();
if (ret != WOLFSSL_SUCCESS) {
pr_err("wolfSSL_Init() failed: %s", wc_GetErrorString(ret));
pr_err("wolfSSL_Init() failed: %s\n", wc_GetErrorString(ret));
return -ECANCELED;
}
#endif
@@ -212,7 +234,7 @@ static int wolfssl_init(void)
#ifndef NO_CRYPT_TEST
ret = wolfcrypt_test(NULL);
if (ret < 0) {
pr_err("wolfcrypt self-test failed with return code %d.", ret);
pr_err("wolfcrypt self-test failed with return code %d.\n", ret);
(void)libwolfssl_cleanup();
msleep(10);
return -ECANCELED;
@@ -221,12 +243,20 @@ static int wolfssl_init(void)
#endif
#ifdef WOLFCRYPT_ONLY
pr_info("wolfCrypt " LIBWOLFSSL_VERSION_STRING " loaded. See https://www.wolfssl.com/ for information.\n");
#else
pr_info("wolfSSL " LIBWOLFSSL_VERSION_STRING " loaded. See https://www.wolfssl.com/ for information.\n");
pr_info("wolfCrypt " LIBWOLFSSL_VERSION_STRING " loaded"
#ifdef CONFIG_MODULE_SIG
" with valid module signature"
#endif
".\nSee https://www.wolfssl.com/ for more information.\n"
"wolfCrypt Copyright (C) 2006-present wolfSSL Inc. Licensed under " WOLFSSL_LICENSE ".\n");
#else
pr_info("wolfSSL " LIBWOLFSSL_VERSION_STRING " loaded"
#ifdef CONFIG_MODULE_SIG
" with valid module signature"
#endif
".\nSee https://www.wolfssl.com/ for more information.\n"
"wolfSSL Copyright (C) 2006-present wolfSSL Inc. Licensed under " WOLFSSL_LICENSE ".\n");
#endif
pr_info("Copyright (C) 2006-2020 wolfSSL Inc. All Rights Reserved.\n");
return 0;
}
@@ -240,18 +270,38 @@ static void wolfssl_exit(void)
#endif
{
(void)libwolfssl_cleanup();
return;
}
module_exit(wolfssl_exit);
MODULE_LICENSE("GPL v2");
MODULE_LICENSE(WOLFSSL_LICENSE);
MODULE_AUTHOR("https://www.wolfssl.com/");
MODULE_DESCRIPTION("libwolfssl cryptographic and protocol facilities");
MODULE_VERSION(LIBWOLFSSL_VERSION_STRING);
#ifdef USE_WOLFSSL_LINUXKM_PIE_REDIRECT_TABLE
/* get_current() is an inline or macro, depending on the target -- sidestep the whole issue with a wrapper func. */
static struct task_struct *my_get_current_thread(void) {
return get_current();
}
/* ditto for preempt_count(). */
static int my_preempt_count(void) {
return preempt_count();
}
#if defined(WOLFSSL_LINUXKM_SIMD_X86_IRQ_ALLOWED) && (LINUX_VERSION_CODE < KERNEL_VERSION(5, 14, 0))
static int my_copy_fpregs_to_fpstate(struct fpu *fpu) {
return copy_fpregs_to_fpstate(fpu);
}
static void my_copy_kernel_to_fpregs(union fpregs_state *fpstate) {
copy_kernel_to_fpregs(fpstate);
}
#endif
static int set_up_wolfssl_linuxkm_pie_redirect_table(void) {
memset(
&wolfssl_linuxkm_pie_redirect_table,
@@ -310,12 +360,22 @@ static int set_up_wolfssl_linuxkm_pie_redirect_table(void) {
kmalloc_order_trace;
wolfssl_linuxkm_pie_redirect_table.get_random_bytes = get_random_bytes;
wolfssl_linuxkm_pie_redirect_table.ktime_get_real_seconds =
ktime_get_real_seconds;
wolfssl_linuxkm_pie_redirect_table.ktime_get_with_offset =
ktime_get_with_offset;
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
wolfssl_linuxkm_pie_redirect_table.getnstimeofday =
getnstimeofday;
#elif LINUX_VERSION_CODE < KERNEL_VERSION(5, 0, 0)
wolfssl_linuxkm_pie_redirect_table.current_kernel_time64 =
current_kernel_time64;
#else
wolfssl_linuxkm_pie_redirect_table.ktime_get_coarse_real_ts64 =
ktime_get_coarse_real_ts64;
#endif
#if defined(WOLFSSL_AESNI) || defined(USE_INTEL_SPEEDUP)
wolfssl_linuxkm_pie_redirect_table.get_current = my_get_current_thread;
wolfssl_linuxkm_pie_redirect_table.preempt_count = my_preempt_count;
#ifdef WOLFSSL_LINUXKM_SIMD_X86
wolfssl_linuxkm_pie_redirect_table.irq_fpu_usable = irq_fpu_usable;
#ifdef kernel_fpu_begin
wolfssl_linuxkm_pie_redirect_table.kernel_fpu_begin_mask =
kernel_fpu_begin_mask;
@@ -324,11 +384,30 @@ static int set_up_wolfssl_linuxkm_pie_redirect_table(void) {
kernel_fpu_begin;
#endif
wolfssl_linuxkm_pie_redirect_table.kernel_fpu_end = kernel_fpu_end;
#ifdef WOLFSSL_LINUXKM_SIMD_X86_IRQ_ALLOWED
#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 14, 0)
wolfssl_linuxkm_pie_redirect_table.copy_fpregs_to_fpstate = my_copy_fpregs_to_fpstate;
wolfssl_linuxkm_pie_redirect_table.copy_kernel_to_fpregs = my_copy_kernel_to_fpregs;
#else
wolfssl_linuxkm_pie_redirect_table.save_fpregs_to_fpstate = save_fpregs_to_fpstate;
wolfssl_linuxkm_pie_redirect_table.__restore_fpregs_from_fpstate = __restore_fpregs_from_fpstate;
wolfssl_linuxkm_pie_redirect_table.xfeatures_mask_all = &xfeatures_mask_all;
#endif
wolfssl_linuxkm_pie_redirect_table.cpu_number = &cpu_number;
wolfssl_linuxkm_pie_redirect_table.nr_cpu_ids = &nr_cpu_ids;
#endif /* WOLFSSL_LINUXKM_SIMD_X86_IRQ_ALLOWED */
#endif
wolfssl_linuxkm_pie_redirect_table.__mutex_init = __mutex_init;
wolfssl_linuxkm_pie_redirect_table.mutex_lock = mutex_lock;
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
wolfssl_linuxkm_pie_redirect_table.mutex_lock_nested = mutex_lock_nested;
#else
wolfssl_linuxkm_pie_redirect_table.mutex_lock = mutex_lock;
#endif
wolfssl_linuxkm_pie_redirect_table.mutex_unlock = mutex_unlock;
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
wolfssl_linuxkm_pie_redirect_table.mutex_destroy = mutex_destroy;
#endif
#ifdef HAVE_FIPS
wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_first =

4
src/internal.c
View File

@@ -8269,11 +8269,9 @@ ProtocolVersion MakeDTLSv1_2(void)
}
#elif defined(WOLFSSL_LINUXKM)
#include <linux/time.h>
#include <linux/ktime.h>
word32 LowResTimer(void)
{
return (word32)ktime_get_real_ns();
return (word32)time(NULL);
}
#else

2
src/ssl.c
View File

@@ -52105,7 +52105,7 @@ WOLFSSL_BN_ULONG wolfSSL_BN_mod_word(const WOLFSSL_BIGNUM *bn,
if (w <= MP_MASK) {
mp_digit bn_ret;
if (mp_mod_d((mp_int*)bn->internal, (WOLFSSL_BN_ULONG)w, &bn_ret) != MP_OKAY) {
if (mp_mod_d((mp_int*)bn->internal, (mp_digit)w, &bn_ret) != MP_OKAY) {
WOLFSSL_MSG("mp_add_d error");
return (WOLFSSL_BN_ULONG)WOLFSSL_FATAL_ERROR;
}

28
src/tls13.c
View File

@@ -1558,19 +1558,25 @@ end:
return (word32)(uTaskerSystemTick / (TICK_RESOLUTION / 1000));
}
#elif defined(WOLFSSL_LINUXKM)
/* The time in milliseconds.
* Used for tickets to represent difference between when first seen and when
* sending.
*
* returns the time in milliseconds as a 32-bit value.
*/
word32 TimeNowInMilliseconds(void)
{
#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 0, 0)
return (word32)(ktime_get_real_ns() / (s64)1000000);
#else
return (word32)(ktime_get_real_ns() / (ktime_t)1000000);
#endif
s64 t;
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
struct timespec ts;
getnstimeofday(&ts);
t = ts.tv_sec * (s64)1000;
t += ts.tv_nsec / (s64)1000000;
#else
struct timespec64 ts;
#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 0, 0)
ts = current_kernel_time64();
#else
ktime_get_coarse_real_ts64(&ts);
#endif
t = ts.tv_sec * 1000L;
t += ts.tv_nsec / 1000000L;
#endif
return (word32)t;
}
#elif defined(WOLFSSL_QNX_CAAM)
word32 TimeNowInMilliseconds(void)

18
tests/api.c
View File

@@ -3804,7 +3804,11 @@ static int nonblocking_accept_read(void* args, WOLFSSL* ssl, SOCKET_T* sockfd)
#endif
#if defined(OPENSSL_ALL) || defined(WOLFSSL_HAPROXY) || defined(WOLFSSL_WPAS)
#define MD_MAX_SIZE WC_SHA512_DIGEST_SIZE
#ifdef WC_SHA512_DIGEST_SIZE
#define MD_MAX_SIZE WC_SHA512_DIGEST_SIZE
#else
#define MD_MAX_SIZE WC_SHA256_DIGEST_SIZE
#endif
byte server_side_msg1[MD_MAX_SIZE] = {0};/* msg sent by server */
byte server_side_msg2[MD_MAX_SIZE] = {0};/* msg received from client */
byte client_side_msg1[MD_MAX_SIZE] = {0};/* msg sent by client */
@@ -6525,7 +6529,7 @@ static void verify_alpn_matching_http1(WOLFSSL* ssl)
AssertIntEQ(0, XMEMCMP(nego_proto, proto, protoSz));
}
static void test_wolfSSL_set_alpn_protos()
static void test_wolfSSL_set_alpn_protos(void)
{
unsigned long i;
callback_functions callbacks[] = {
@@ -23188,8 +23192,8 @@ static int test_wc_ecc_shared_secret (void)
#if defined(HAVE_ECC) && defined(HAVE_ECC_DHE) && !defined(WC_NO_RNG)
ecc_key key, pubKey;
WC_RNG rng;
int keySz = KEY16;
byte out[KEY16];
byte out[KEY32];
int keySz = sizeof(out);
word32 outlen = (word32)sizeof(out);
#if defined(HAVE_ECC) && !defined(NO_ECC256)
@@ -44450,7 +44454,11 @@ static void test_wolfSSL_SMIME_read_PKCS7(void)
#ifdef WOLFSSL_TLS13
#if defined(WOLFSSL_SEND_HRR_COOKIE) && !defined(NO_WOLFSSL_SERVER)
static byte fixedKey[WC_SHA384_DIGEST_SIZE] = { 0, };
#ifdef WC_SHA384_DIGEST_SIZE
static byte fixedKey[WC_SHA384_DIGEST_SIZE] = { 0, };
#else
static byte fixedKey[WC_SHA256_DIGEST_SIZE] = { 0, };
#endif
#endif
#ifdef WOLFSSL_EARLY_DATA
static const char earlyData[] = "Early Data";

2
wolfcrypt/benchmark/benchmark.c
View File

@@ -5770,7 +5770,7 @@ void bench_eccEncrypt(int curveId)
ecc_key userA, userB;
int keySize;
byte msg[48];
byte out[80];
byte out[sizeof(msg) + WC_SHA256_DIGEST_SIZE + (MAX_ECC_BITS+3)/4 + 2];
word32 outSz = sizeof(out);
word32 bench_plainSz = BENCH_SIZE;
int ret, i, count;

243
wolfcrypt/src/aes.c
View File

@@ -775,6 +775,7 @@ block cipher mechanism that uses n-bit binary string parameter key with 128-bits
const int bits, Aes* aes)
{
int nr;
int ret;
#ifdef WOLFSSL_SMALL_STACK
Aes *temp_key;
#else
@@ -805,7 +806,12 @@ block cipher mechanism that uses n-bit binary string parameter key with 128-bits
nr = temp_key->rounds;
aes->rounds = nr;
SAVE_VECTOR_REGISTERS();
if ((ret = SAVE_VECTOR_REGISTERS()) != 0) {
#ifdef WOLFSSL_SMALL_STACK
XFREE(temp_key, aes->heap, DYNAMIC_TYPE_AES);
#endif
return ret;
}
Key_Schedule[nr] = Temp_Key_Schedule[0];
Key_Schedule[nr-1] = _mm_aesimc_si128(Temp_Key_Schedule[1]);
@@ -1738,10 +1744,8 @@ static void wc_AesEncrypt(Aes* aes, const byte* inBlock, byte* outBlock)
tmp_align = tmp + (AESNI_ALIGN - ((wc_ptr_t)tmp % AESNI_ALIGN));
XMEMCPY(tmp_align, inBlock, AES_BLOCK_SIZE);
SAVE_VECTOR_REGISTERS();
AES_ECB_encrypt(tmp_align, tmp_align, AES_BLOCK_SIZE,
(byte*)aes->key, aes->rounds);
RESTORE_VECTOR_REGISTERS();
XMEMCPY(outBlock, tmp_align, AES_BLOCK_SIZE);
XFREE(tmp, aes->heap, DYNAMIC_TYPE_TMP_BUFFER);
return;
@@ -1751,10 +1755,8 @@ static void wc_AesEncrypt(Aes* aes, const byte* inBlock, byte* outBlock)
#endif
}
SAVE_VECTOR_REGISTERS();
AES_ECB_encrypt(inBlock, outBlock, AES_BLOCK_SIZE, (byte*)aes->key,
aes->rounds);
RESTORE_VECTOR_REGISTERS();
return;
}
@@ -2089,10 +2091,8 @@ static void wc_AesDecrypt(Aes* aes, const byte* inBlock, byte* outBlock)
/* if input and output same will overwrite input iv */
if ((const byte*)aes->tmp != inBlock)
XMEMCPY(aes->tmp, inBlock, AES_BLOCK_SIZE);
SAVE_VECTOR_REGISTERS();
AES_ECB_decrypt(inBlock, outBlock, AES_BLOCK_SIZE, (byte*)aes->key,
aes->rounds);
RESTORE_VECTOR_REGISTERS();
return;
}
else {
@@ -3060,6 +3060,64 @@ int wc_AesSetIV(Aes* aes, const byte* iv)
#elif defined(WOLFSSL_DEVCRYPTO_AES)
/* implemented in wolfcrypt/src/port/devcrypt/devcrypto_aes.c */
#elif defined(WOLFSSL_LINUXKM)
#ifdef WOLFSSL_AESNI
__must_check int wc_AesEncryptDirect(Aes* aes, byte* out, const byte* in)
{
if (haveAESNI && aes->use_aesni) {
int ret = SAVE_VECTOR_REGISTERS();
if (ret != 0)
return ret;
}
wc_AesEncrypt(aes, in, out);
if (haveAESNI && aes->use_aesni)
RESTORE_VECTOR_REGISTERS();
return 0;
}
/* vector reg save/restore is explicit in all below calls to
* wc_Aes{En,De}cryptDirect(), so bypass the public version with a
* macro.
*/
#define wc_AesEncryptDirect(aes, out, in) wc_AesEncrypt(aes, in, out)
#ifdef HAVE_AES_DECRYPT
/* Allow direct access to one block decrypt */
__must_check int wc_AesDecryptDirect(Aes* aes, byte* out, const byte* in)
{
if (haveAESNI && aes->use_aesni) {
int ret = SAVE_VECTOR_REGISTERS();
if (ret != 0)
return ret;
}
wc_AesDecrypt(aes, in, out);
if (haveAESNI && aes->use_aesni)
RESTORE_VECTOR_REGISTERS();
return 0;
}
#define wc_AesDecryptDirect(aes, out, in) wc_AesDecrypt(aes, in, out)
#endif /* HAVE_AES_DECRYPT */
#else /* !WOLFSSL_AESNI */
__must_check int wc_AesEncryptDirect(Aes* aes, byte* out, const byte* in)
{
wc_AesEncrypt(aes, in, out);
return 0;
}
#define wc_AesEncryptDirect(aes, out, in) wc_AesEncrypt(aes, in, out)
#ifdef HAVE_AES_DECRYPT
/* Allow direct access to one block decrypt */
__must_check int wc_AesDecryptDirect(Aes* aes, byte* out, const byte* in)
{
wc_AesDecrypt(aes, in, out);
return 0;
}
#define wc_AesDecryptDirect(aes, out, in) wc_AesDecrypt(aes, in, out)
#endif /* HAVE_AES_DECRYPT */
#endif /* WOLFSSL_AESNI */
#else
/* Allow direct access to one block encrypt */
void wc_AesEncryptDirect(Aes* aes, byte* out, const byte* in)
@@ -3814,6 +3872,7 @@ int wc_AesSetIV(Aes* aes, const byte* iv)
#ifdef WOLFSSL_AESNI
if (haveAESNI) {
int ret;
#ifdef DEBUG_AESNI
printf("about to aes cbc encrypt\n");
printf("in = %p\n", in);
@@ -3834,7 +3893,10 @@ int wc_AesSetIV(Aes* aes, const byte* iv)
tmp_align = tmp + (AESNI_ALIGN - ((wc_ptr_t)tmp % AESNI_ALIGN));
XMEMCPY(tmp_align, in, sz);
SAVE_VECTOR_REGISTERS();
if ((ret = SAVE_VECTOR_REGISTERS()) != 0) {
XFREE(tmp, aes->heap, DYNAMIC_TYPE_TMP_BUFFER);
return ret;
}
AES_CBC_encrypt(tmp_align, tmp_align, (byte*)aes->reg, sz,
(byte*)aes->key, aes->rounds);
RESTORE_VECTOR_REGISTERS();
@@ -3850,7 +3912,8 @@ int wc_AesSetIV(Aes* aes, const byte* iv)
#endif
}
SAVE_VECTOR_REGISTERS();
if ((ret = SAVE_VECTOR_REGISTERS()) != 0)
return ret;
AES_CBC_encrypt(in, out, (byte*)aes->reg, sz, (byte*)aes->key,
aes->rounds);
RESTORE_VECTOR_REGISTERS();
@@ -3935,6 +3998,7 @@ int wc_AesSetIV(Aes* aes, const byte* iv)
#ifdef WOLFSSL_AESNI
if (haveAESNI) {
int ret;
#ifdef DEBUG_AESNI
printf("about to aes cbc decrypt\n");
printf("in = %p\n", in);
@@ -3947,7 +4011,8 @@ int wc_AesSetIV(Aes* aes, const byte* iv)
/* if input and output same will overwrite input iv */
XMEMCPY(aes->tmp, in + sz - AES_BLOCK_SIZE, AES_BLOCK_SIZE);
SAVE_VECTOR_REGISTERS();
if ((ret = SAVE_VECTOR_REGISTERS()) != 0)
return ret;
#if defined(WOLFSSL_AESNI_BY4)
AES_CBC_decrypt_by4(in, out, (byte*)aes->reg, sz, (byte*)aes->key,
aes->rounds);
@@ -7519,7 +7584,9 @@ int wc_AesGcmEncrypt(Aes* aes, byte* out, const byte* in, word32 sz,
#ifdef WOLFSSL_AESNI
#ifdef HAVE_INTEL_AVX2
if (IS_INTEL_AVX2(intel_flags)) {
SAVE_VECTOR_REGISTERS();
int ret = SAVE_VECTOR_REGISTERS();
if (ret != 0)
return ret;
AES_GCM_encrypt_avx2(in, out, authIn, iv, authTag, sz, authInSz, ivSz,
authTagSz, (const byte*)aes->key, aes->rounds);
RESTORE_VECTOR_REGISTERS();
@@ -7529,7 +7596,9 @@ int wc_AesGcmEncrypt(Aes* aes, byte* out, const byte* in, word32 sz,
#endif
#ifdef HAVE_INTEL_AVX1
if (IS_INTEL_AVX1(intel_flags)) {
SAVE_VECTOR_REGISTERS();
int ret = SAVE_VECTOR_REGISTERS();
if (ret != 0)
return ret;
AES_GCM_encrypt_avx1(in, out, authIn, iv, authTag, sz, authInSz, ivSz,
authTagSz, (const byte*)aes->key, aes->rounds);
RESTORE_VECTOR_REGISTERS();
@@ -8041,7 +8110,9 @@ int wc_AesGcmDecrypt(Aes* aes, byte* out, const byte* in, word32 sz,
#ifdef WOLFSSL_AESNI
#ifdef HAVE_INTEL_AVX2
if (IS_INTEL_AVX2(intel_flags)) {
SAVE_VECTOR_REGISTERS();
int ret = SAVE_VECTOR_REGISTERS();
if (ret != 0)
return ret;
AES_GCM_decrypt_avx2(in, out, authIn, iv, authTag, sz, authInSz, ivSz,
authTagSz, (byte*)aes->key, aes->rounds, &res);
RESTORE_VECTOR_REGISTERS();
@@ -8053,7 +8124,9 @@ int wc_AesGcmDecrypt(Aes* aes, byte* out, const byte* in, word32 sz,
#endif
#ifdef HAVE_INTEL_AVX1
if (IS_INTEL_AVX1(intel_flags)) {
SAVE_VECTOR_REGISTERS();
int ret = SAVE_VECTOR_REGISTERS();
if (ret != 0)
return ret;
AES_GCM_decrypt_avx1(in, out, authIn, iv, authTag, sz, authInSz, ivSz,
authTagSz, (byte*)aes->key, aes->rounds, &res);
RESTORE_VECTOR_REGISTERS();
@@ -8296,7 +8369,7 @@ extern void AES_GCM_encrypt_final_aesni(unsigned char* tag,
* @param [in] iv IV/nonce buffer.
* @param [in] ivSz Length of IV/nonce data.
*/
static void AesGcmInit_aesni(Aes* aes, const byte* iv, word32 ivSz)
static int AesGcmInit_aesni(Aes* aes, const byte* iv, word32 ivSz)
{
/* Reset state fields. */
aes->aSz = 0;
@@ -8309,7 +8382,9 @@ static void AesGcmInit_aesni(Aes* aes, const byte* iv, word32 ivSz)
#ifdef HAVE_INTEL_AVX2
if (IS_INTEL_AVX2(intel_flags)) {
SAVE_VECTOR_REGISTERS();
int ret = SAVE_VECTOR_REGISTERS();
if (ret != 0)
return ret;
AES_GCM_init_avx2((byte*)aes->key, aes->rounds, iv, ivSz, aes->H,
AES_COUNTER(aes), AES_INITCTR(aes));
RESTORE_VECTOR_REGISTERS();
@@ -8318,7 +8393,9 @@ static void AesGcmInit_aesni(Aes* aes, const byte* iv, word32 ivSz)
#endif
#ifdef HAVE_INTEL_AVX1
if (IS_INTEL_AVX1(intel_flags)) {
SAVE_VECTOR_REGISTERS();
int ret = SAVE_VECTOR_REGISTERS();
if (ret != 0)
return ret;
AES_GCM_init_avx1((byte*)aes->key, aes->rounds, iv, ivSz, aes->H,
AES_COUNTER(aes), AES_INITCTR(aes));
RESTORE_VECTOR_REGISTERS();
@@ -8326,11 +8403,14 @@ static void AesGcmInit_aesni(Aes* aes, const byte* iv, word32 ivSz)
else
#endif
{
SAVE_VECTOR_REGISTERS();
int ret = SAVE_VECTOR_REGISTERS();
if (ret != 0)
return BAD_STATE_E;
AES_GCM_init_aesni((byte*)aes->key, aes->rounds, iv, ivSz, aes->H,
AES_COUNTER(aes), AES_INITCTR(aes));
RESTORE_VECTOR_REGISTERS();
}
return 0;
}
/* Update the AES GCM for encryption with authentication data.
@@ -8458,13 +8538,15 @@ static void AesGcmAadUpdate_aesni(Aes* aes, const byte* a, word32 aSz, int endA)
* @param [in] a Buffer holding authentication data.
* @param [in] aSz Length of authentication data in bytes.
*/
static void AesGcmEncryptUpdate_aesni(Aes* aes, byte* c, const byte* p,
static int AesGcmEncryptUpdate_aesni(Aes* aes, byte* c, const byte* p,
word32 cSz, const byte* a, word32 aSz)
{
word32 blocks;
int partial;
SAVE_VECTOR_REGISTERS();
int ret = SAVE_VECTOR_REGISTERS();
if (ret != 0)
return ret;
/* Hash in A, the Authentication Data */
AesGcmAadUpdate_aesni(aes, a, aSz, (cSz > 0) && (c != NULL));
@@ -8573,6 +8655,7 @@ static void AesGcmEncryptUpdate_aesni(Aes* aes, byte* c, const byte* p,
}
}
RESTORE_VECTOR_REGISTERS();
return 0;
}
/* Finalize the AES GCM for encryption and calculate the authentication tag.
@@ -8584,12 +8667,14 @@ static void AesGcmEncryptUpdate_aesni(Aes* aes, byte* c, const byte* p,
* @param [in] authTagSz Length of authentication tag in bytes.
* @return 0 on success.
*/
static void AesGcmEncryptFinal_aesni(Aes* aes, byte* authTag, word32 authTagSz)
static int AesGcmEncryptFinal_aesni(Aes* aes, byte* authTag, word32 authTagSz)
{
/* AAD block incomplete when > 0 */
byte over = aes->aOver;
SAVE_VECTOR_REGISTERS();
int ret = SAVE_VECTOR_REGISTERS();
if (ret != 0)
return ret;
if (aes->cOver > 0) {
/* Cipher text block incomplete. */
over = aes->cOver;
@@ -8635,6 +8720,7 @@ static void AesGcmEncryptFinal_aesni(Aes* aes, byte* authTag, word32 authTagSz)
aes->aSz, aes->H, AES_INITCTR(aes));
}
RESTORE_VECTOR_REGISTERS();
return 0;
}
#if defined(HAVE_AES_DECRYPT) || defined(HAVE_AESGCM_DECRYPT)
@@ -8680,13 +8766,15 @@ extern void AES_GCM_decrypt_final_aesni(unsigned char* tag,
* @param [in] a Buffer holding authentication data.
* @param [in] aSz Length of authentication data in bytes.
*/
static void AesGcmDecryptUpdate_aesni(Aes* aes, byte* p, const byte* c,
static int AesGcmDecryptUpdate_aesni(Aes* aes, byte* p, const byte* c,
word32 cSz, const byte* a, word32 aSz)
{
word32 blocks;
int partial;
SAVE_VECTOR_REGISTERS();
int ret = SAVE_VECTOR_REGISTERS();
if (ret != 0)
return ret;
/* Hash in A, the Authentication Data */
AesGcmAadUpdate_aesni(aes, a, aSz, (cSz > 0) && (c != NULL));
@@ -8797,6 +8885,7 @@ static void AesGcmDecryptUpdate_aesni(Aes* aes, byte* p, const byte* c,
}
}
RESTORE_VECTOR_REGISTERS();
return 0;
}
/* Finalize the AES GCM for decryption and check the authentication tag.
@@ -8819,7 +8908,8 @@ static int AesGcmDecryptFinal_aesni(Aes* aes, const byte* authTag,
byte over = aes->aOver;
byte *lastBlock = AES_LASTGBLOCK(aes);
SAVE_VECTOR_REGISTERS();
if ((ret = SAVE_VECTOR_REGISTERS()) != 0)
return ret;
if (aes->cOver > 0) {
/* Cipher text block incomplete. */
over = aes->cOver;
@@ -8940,7 +9030,7 @@ int wc_AesGcmInit(Aes* aes, const byte* key, word32 len, const byte* iv,
|| IS_INTEL_AVX1(intel_flags)
#endif
) {
AesGcmInit_aesni(aes, iv, ivSz);
ret = AesGcmInit_aesni(aes, iv, ivSz);
}
else
#endif
@@ -9052,7 +9142,7 @@ int wc_AesGcmEncryptUpdate(Aes* aes, byte* out, const byte* in, word32 sz,
|| IS_INTEL_AVX1(intel_flags)
#endif
) {
AesGcmEncryptUpdate_aesni(aes, out, in, sz, authIn, authInSz);
ret = AesGcmEncryptUpdate_aesni(aes, out, in, sz, authIn, authInSz);
}
else
#endif
@@ -9818,13 +9908,14 @@ int wc_AesCcmEncrypt(Aes* aes, byte* out, const byte* in, word32 inSz,
B[15] = 1;
#ifdef WOLFSSL_AESNI
if (haveAESNI && aes->use_aesni) {
int ret = SAVE_VECTOR_REGISTERS();
if (ret != 0)
return ret;
while (inSz >= AES_BLOCK_SIZE * 4) {
AesCcmCtrIncSet4(B, lenSz);
SAVE_VECTOR_REGISTERS();
AES_ECB_encrypt(B, A, AES_BLOCK_SIZE * 4, (byte*)aes->key,
aes->rounds);
RESTORE_VECTOR_REGISTERS();
xorbuf(A, in, AES_BLOCK_SIZE * 4);
XMEMCPY(out, A, AES_BLOCK_SIZE * 4);
@@ -9835,6 +9926,7 @@ int wc_AesCcmEncrypt(Aes* aes, byte* out, const byte* in, word32 inSz,
AesCcmCtrInc4(B, lenSz);
}
RESTORE_VECTOR_REGISTERS();
}
#endif
while (inSz >= AES_BLOCK_SIZE) {
@@ -9903,13 +9995,14 @@ int wc_AesCcmDecrypt(Aes* aes, byte* out, const byte* in, word32 inSz,
#ifdef WOLFSSL_AESNI
if (haveAESNI && aes->use_aesni) {
int ret = SAVE_VECTOR_REGISTERS();
if (ret != 0)
return ret;
while (oSz >= AES_BLOCK_SIZE * 4) {
AesCcmCtrIncSet4(B, lenSz);
SAVE_VECTOR_REGISTERS();
AES_ECB_encrypt(B, A, AES_BLOCK_SIZE * 4, (byte*)aes->key,
aes->rounds);
RESTORE_VECTOR_REGISTERS();
xorbuf(A, in, AES_BLOCK_SIZE * 4);
XMEMCPY(o, A, AES_BLOCK_SIZE * 4);
@@ -9920,6 +10013,7 @@ int wc_AesCcmDecrypt(Aes* aes, byte* out, const byte* in, word32 inSz,
AesCcmCtrInc4(B, lenSz);
}
RESTORE_VECTOR_REGISTERS();
}
#endif
while (oSz >= AES_BLOCK_SIZE) {
@@ -10274,12 +10368,10 @@ int wc_AesEcbDecrypt(Aes* aes, byte* out, const byte* in, word32 sz)
#else
/* Software AES - ECB */
int wc_AesEcbEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
static int _AesEcbEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
{
word32 blocks = sz / AES_BLOCK_SIZE;
if ((in == NULL) || (out == NULL) || (aes == NULL))
return BAD_FUNC_ARG;
#ifdef WOLFSSL_IMXRT_DCP
if (aes->keylen == 16)
return DCPAesEcbEncrypt(aes, out, in, sz);
@@ -10293,13 +10385,10 @@ int wc_AesEcbEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
return 0;
}
int wc_AesEcbDecrypt(Aes* aes, byte* out, const byte* in, word32 sz)
static int _AesEcbDecrypt(Aes* aes, byte* out, const byte* in, word32 sz)
{
word32 blocks = sz / AES_BLOCK_SIZE;
if ((in == NULL) || (out == NULL) || (aes == NULL))
return BAD_FUNC_ARG;
#ifdef WOLFSSL_IMXRT_DCP
if (aes->keylen == 16)
return DCPAesEcbDecrypt(aes, out, in, sz);
@@ -10312,6 +10401,36 @@ int wc_AesEcbDecrypt(Aes* aes, byte* out, const byte* in, word32 sz)
}
return 0;
}
int wc_AesEcbEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
{
int ret;
if ((in == NULL) || (out == NULL) || (aes == NULL))
return BAD_FUNC_ARG;
if ((ret = SAVE_VECTOR_REGISTERS()) != 0)
return ret;
ret = _AesEcbEncrypt(aes, out, in, sz);
RESTORE_VECTOR_REGISTERS();
return ret;
}
int wc_AesEcbDecrypt(Aes* aes, byte* out, const byte* in, word32 sz)
{
int ret;
if ((in == NULL) || (out == NULL) || (aes == NULL))
return BAD_FUNC_ARG;
if ((ret = SAVE_VECTOR_REGISTERS()) != 0)
return ret;
ret = _AesEcbDecrypt(aes, out, in, sz);
RESTORE_VECTOR_REGISTERS();
return ret;
}
#endif
#endif /* HAVE_AES_ECB */
@@ -10335,6 +10454,7 @@ static int wc_AesFeedbackEncrypt(Aes* aes, byte* out, const byte* in,
#ifdef WOLFSSL_AES_CFB
byte* reg = NULL;
#endif
int ret;
if (aes == NULL || out == NULL || in == NULL) {
return BAD_FUNC_ARG;
@@ -10360,6 +10480,9 @@ static int wc_AesFeedbackEncrypt(Aes* aes, byte* out, const byte* in,
sz--;
}
if ((ret = SAVE_VECTOR_REGISTERS()) != 0)
return ret;
while (sz >= AES_BLOCK_SIZE) {
/* Using aes->tmp here for inline case i.e. in=out */
wc_AesEncryptDirect(aes, (byte*)aes->tmp, (byte*)aes->reg);
@@ -10406,6 +10529,7 @@ static int wc_AesFeedbackEncrypt(Aes* aes, byte* out, const byte* in,
aes->left--;
}
}
RESTORE_VECTOR_REGISTERS();
return 0;
}
@@ -10427,6 +10551,7 @@ static int wc_AesFeedbackDecrypt(Aes* aes, byte* out, const byte* in, word32 sz,
byte mode)
{
byte* tmp;
int ret;
if (aes == NULL || out == NULL || in == NULL) {
return BAD_FUNC_ARG;
@@ -10448,6 +10573,9 @@ static int wc_AesFeedbackDecrypt(Aes* aes, byte* out, const byte* in, word32 sz,
sz--;
}
if ((ret = SAVE_VECTOR_REGISTERS()) != 0)
return ret;
while (sz > AES_BLOCK_SIZE) {
/* Using aes->tmp here for inline case i.e. in=out */
wc_AesEncryptDirect(aes, (byte*)aes->tmp, (byte*)aes->reg);
@@ -10491,6 +10619,7 @@ static int wc_AesFeedbackDecrypt(Aes* aes, byte* out, const byte* in, word32 sz,
aes->left--;
}
}
RESTORE_VECTOR_REGISTERS();
return 0;
}
@@ -10563,6 +10692,7 @@ static int wc_AesFeedbackCFB8(Aes* aes, byte* out, const byte* in,
word32 sz, byte dir)
{
byte *pt;
int ret;
if (aes == NULL || out == NULL || in == NULL) {
return BAD_FUNC_ARG;
@@ -10572,6 +10702,9 @@ static int wc_AesFeedbackCFB8(Aes* aes, byte* out, const byte* in,
return 0;
}
if ((ret = SAVE_VECTOR_REGISTERS()) != 0)
return ret;
while (sz > 0) {
wc_AesEncryptDirect(aes, (byte*)aes->tmp, (byte*)aes->reg);
if (dir == AES_DECRYPTION) {
@@ -10600,6 +10733,8 @@ static int wc_AesFeedbackCFB8(Aes* aes, byte* out, const byte* in,
sz -= 1;
}
RESTORE_VECTOR_REGISTERS();
return 0;
}
@@ -10612,6 +10747,7 @@ static int wc_AesFeedbackCFB1(Aes* aes, byte* out, const byte* in,
byte cur = 0; /* hold current work in order to handle inline in=out */
byte* pt;
int bit = 7;
int ret;
if (aes == NULL || out == NULL || in == NULL) {
return BAD_FUNC_ARG;
@@ -10621,6 +10757,9 @@ static int wc_AesFeedbackCFB1(Aes* aes, byte* out, const byte* in,
return 0;
}
if ((ret = SAVE_VECTOR_REGISTERS()) != 0)
return ret;
while (sz > 0) {
wc_AesEncryptDirect(aes, (byte*)aes->tmp, (byte*)aes->reg);
if (dir == AES_DECRYPTION) {
@@ -10667,6 +10806,7 @@ static int wc_AesFeedbackCFB1(Aes* aes, byte* out, const byte* in,
if (bit > 0 && bit < 7) {
out[0] = cur;
}
RESTORE_VECTOR_REGISTERS();
return 0;
}
@@ -10823,6 +10963,8 @@ int wc_AesKeyWrap_ex(Aes *aes, const byte* in, word32 inSz, byte* out,
byte t[KEYWRAP_BLOCK_SIZE];
byte tmp[AES_BLOCK_SIZE];
int ret;
/* n must be at least 2 64-bit blocks, output size is (n + 1) 8 bytes (64-bit) */
if (aes == NULL || in == NULL || inSz < 2*KEYWRAP_BLOCK_SIZE ||
out == NULL || outSz < (inSz + KEYWRAP_BLOCK_SIZE))
@@ -10843,6 +10985,9 @@ int wc_AesKeyWrap_ex(Aes *aes, const byte* in, word32 inSz, byte* out,
XMEMCPY(tmp, iv, KEYWRAP_BLOCK_SIZE);
}
if ((ret = SAVE_VECTOR_REGISTERS()) != 0)
return ret;
for (j = 0; j <= 5; j++) {
for (i = 1; i <= inSz / KEYWRAP_BLOCK_SIZE; i++) {
/* load R[i] */
@@ -10860,6 +11005,7 @@ int wc_AesKeyWrap_ex(Aes *aes, const byte* in, word32 inSz, byte* out,
}
r = out + KEYWRAP_BLOCK_SIZE;
}
RESTORE_VECTOR_REGISTERS();
/* C[0] = A */
XMEMCPY(out, tmp, KEYWRAP_BLOCK_SIZE);
@@ -10925,6 +11071,8 @@ int wc_AesKeyUnWrap_ex(Aes *aes, const byte* in, word32 inSz, byte* out,
0xA6, 0xA6, 0xA6, 0xA6, 0xA6, 0xA6, 0xA6, 0xA6
};
int ret;
if (aes == NULL || in == NULL || inSz < 3 * KEYWRAP_BLOCK_SIZE ||
out == NULL || outSz < (inSz - KEYWRAP_BLOCK_SIZE))
return BAD_FUNC_ARG;
@@ -10944,6 +11092,9 @@ int wc_AesKeyUnWrap_ex(Aes *aes, const byte* in, word32 inSz, byte* out,
XMEMCPY(out, in + KEYWRAP_BLOCK_SIZE, inSz - KEYWRAP_BLOCK_SIZE);
XMEMSET(t, 0, sizeof(t));
if ((ret = SAVE_VECTOR_REGISTERS()) != 0)
return ret;
/* initialize counter to 6n */
n = (inSz - 1) / KEYWRAP_BLOCK_SIZE;
InitKeyWrapCounter(t, 6 * n);
@@ -10964,6 +11115,7 @@ int wc_AesKeyUnWrap_ex(Aes *aes, const byte* in, word32 inSz, byte* out,
XMEMCPY(r, tmp + KEYWRAP_BLOCK_SIZE, KEYWRAP_BLOCK_SIZE);
}
}
RESTORE_VECTOR_REGISTERS();
/* verify IV */
if (XMEMCMP(tmp, expIv, KEYWRAP_BLOCK_SIZE) != 0)
@@ -11178,10 +11330,10 @@ static int _AesXtsHelper(Aes* aes, byte* out, const byte* in, word32 sz, int dir
xorbuf(out, in, totalSz);
if (dir == AES_ENCRYPTION) {
return wc_AesEcbEncrypt(aes, out, out, totalSz);
return _AesEcbEncrypt(aes, out, out, totalSz);
}
else {
return wc_AesEcbDecrypt(aes, out, out, totalSz);
return _AesEcbDecrypt(aes, out, out, totalSz);
}
}
#endif /* HAVE_AES_ECB */
@@ -11224,6 +11376,9 @@ int wc_AesXtsEncrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz,
XMEMSET(tmp, 0, AES_BLOCK_SIZE); /* set to 0's in case of improper AES
* key setup passed to encrypt direct*/
if ((ret = SAVE_VECTOR_REGISTERS()) != 0)
return ret;
wc_AesEncryptDirect(tweak, tmp, i);
#ifdef HAVE_AES_ECB
@@ -11231,6 +11386,7 @@ int wc_AesXtsEncrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz,
if (in != out) { /* can not handle inline */
XMEMCPY(out, tmp, AES_BLOCK_SIZE);
if ((ret = _AesXtsHelper(aes, out, in, sz, AES_ENCRYPTION)) != 0) {
RESTORE_VECTOR_REGISTERS();
return ret;
}
}
@@ -11285,6 +11441,7 @@ int wc_AesXtsEncrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz,
wc_AesEncryptDirect(aes, out - AES_BLOCK_SIZE, buf);
xorbuf(out - AES_BLOCK_SIZE, tmp, AES_BLOCK_SIZE);
}
RESTORE_VECTOR_REGISTERS();
}
else {
WOLFSSL_MSG("Plain text input too small for encryption");
@@ -11335,6 +11492,9 @@ int wc_AesXtsDecrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz,
XMEMSET(tmp, 0, AES_BLOCK_SIZE); /* set to 0's in case of improper AES
* key setup passed to decrypt direct*/
if ((ret = SAVE_VECTOR_REGISTERS()) != 0)
return ret;
wc_AesEncryptDirect(tweak, tmp, i);
/* if Stealing then break out of loop one block early to handle special
@@ -11348,6 +11508,7 @@ int wc_AesXtsDecrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz,
if (in != out) { /* can not handle inline */
XMEMCPY(out, tmp, AES_BLOCK_SIZE);
if ((ret = _AesXtsHelper(aes, out, in, sz, AES_DECRYPTION)) != 0) {
RESTORE_VECTOR_REGISTERS();
return ret;
}
}
@@ -11416,6 +11577,7 @@ int wc_AesXtsDecrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz,
/* Make buffer with end of cipher text | last */
XMEMCPY(buf, tmp2, AES_BLOCK_SIZE);
if (sz >= AES_BLOCK_SIZE) { /* extra sanity check before copy */
RESTORE_VECTOR_REGISTERS();
return BUFFER_E;
}
XMEMCPY(buf, in, sz);
@@ -11426,6 +11588,7 @@ int wc_AesXtsDecrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz,
xorbuf(tmp2, tmp, AES_BLOCK_SIZE);
XMEMCPY(out - AES_BLOCK_SIZE, tmp2, AES_BLOCK_SIZE);
}
RESTORE_VECTOR_REGISTERS();
}
else {
WOLFSSL_MSG("Plain text input too small for encryption");

8
wolfcrypt/src/chacha.c
View File

@@ -418,14 +418,18 @@ int wc_Chacha_Process(ChaCha* ctx, byte* output, const byte* input,
#ifdef HAVE_INTEL_AVX2
if (IS_INTEL_AVX2(cpuidFlags)) {
SAVE_VECTOR_REGISTERS();
int ret = SAVE_VECTOR_REGISTERS();
if (ret != 0)
return ret;
chacha_encrypt_avx2(ctx, input, output, msglen);
RESTORE_VECTOR_REGISTERS();
return 0;
}
#endif
if (IS_INTEL_AVX1(cpuidFlags)) {
SAVE_VECTOR_REGISTERS();
int ret = SAVE_VECTOR_REGISTERS();
if (ret != 0)
return ret;
chacha_encrypt_avx1(ctx, input, output, msglen);
RESTORE_VECTOR_REGISTERS();
return 0;

46
wolfcrypt/src/cmac.c
View File

@@ -119,10 +119,19 @@ int wc_InitCmac_ex(Cmac* cmac, const byte* key, word32 keySz,
byte l[AES_BLOCK_SIZE];
XMEMSET(l, 0, AES_BLOCK_SIZE);
wc_AesEncryptDirect(&cmac->aes, l, l);
ShiftAndXorRb(cmac->k1, l);
ShiftAndXorRb(cmac->k2, cmac->k1);
ForceZero(l, AES_BLOCK_SIZE);
#ifdef WOLFSSL_LINUXKM
ret =
#endif
wc_AesEncryptDirect(&cmac->aes, l, l);
#ifdef WOLFSSL_LINUXKM
if (ret == 0) {
#endif
ShiftAndXorRb(cmac->k1, l);
ShiftAndXorRb(cmac->k2, cmac->k1);
ForceZero(l, AES_BLOCK_SIZE);
#ifdef WOLFSSL_LINUXKM
}
#endif
}
return ret;
}
@@ -172,9 +181,18 @@ int wc_CmacUpdate(Cmac* cmac, const byte* in, word32 inSz)
if (cmac->totalSz != 0) {
xorbuf(cmac->buffer, cmac->digest, AES_BLOCK_SIZE);
}
wc_AesEncryptDirect(&cmac->aes, cmac->digest, cmac->buffer);
cmac->totalSz += AES_BLOCK_SIZE;
cmac->bufferSz = 0;
#ifdef WOLFSSL_LINUXKM
ret =
#endif
wc_AesEncryptDirect(&cmac->aes, cmac->digest, cmac->buffer);
#ifdef WOLFSSL_LINUXKM
if (ret == 0) {
#endif
cmac->totalSz += AES_BLOCK_SIZE;
cmac->bufferSz = 0;
#ifdef WOLFSSL_LINUXKM
}
#endif
}
}
@@ -221,9 +239,17 @@ int wc_CmacFinal(Cmac* cmac, byte* out, word32* outSz)
}
xorbuf(cmac->buffer, cmac->digest, AES_BLOCK_SIZE);
xorbuf(cmac->buffer, subKey, AES_BLOCK_SIZE);
wc_AesEncryptDirect(&cmac->aes, cmac->digest, cmac->buffer);
XMEMCPY(out, cmac->digest, *outSz);
#ifdef WOLFSSL_LINUXKM
ret =
#endif
wc_AesEncryptDirect(&cmac->aes, cmac->digest, cmac->buffer);
#ifdef WOLFSSL_LINUXKM
if (ret == 0) {
#endif
XMEMCPY(out, cmac->digest, *outSz);
#ifdef WOLFSSL_LINUXKM
}
#endif
ForceZero(cmac, sizeof(Cmac));

27
wolfcrypt/src/curve25519.c
View File

@@ -128,15 +128,12 @@ int wc_curve25519_make_pub(int public_size, byte* pub, int private_size,
#else
fe_init();
#if defined(USE_INTEL_SPEEDUP) || defined(WOLFSSL_ARMASM)
SAVE_VECTOR_REGISTERS();
#endif
if ((ret = SAVE_VECTOR_REGISTERS()) != 0)
return ret;
ret = curve25519(pub, priv, kCurve25519BasePoint);
#if defined(USE_INTEL_SPEEDUP) || defined(WOLFSSL_ARMASM)
RESTORE_VECTOR_REGISTERS();
#endif
RESTORE_VECTOR_REGISTERS();
#endif
return ret;
@@ -174,15 +171,12 @@ int wc_curve25519_generic(int public_size, byte* pub,
fe_init();
#if defined(USE_INTEL_SPEEDUP) || defined(WOLFSSL_ARMASM)
SAVE_VECTOR_REGISTERS();
#endif
if ((ret = SAVE_VECTOR_REGISTERS()) != 0)
return ret;
ret = curve25519(pub, priv, basepoint);
#if defined(USE_INTEL_SPEEDUP) || defined(WOLFSSL_ARMASM)
RESTORE_VECTOR_REGISTERS();
#endif
RESTORE_VECTOR_REGISTERS();
return ret;
#endif /* FREESCALE_LTC_ECC */
@@ -295,15 +289,12 @@ int wc_curve25519_shared_secret_ex(curve25519_key* private_key,
ret = nxp_ltc_curve25519(&o, private_key->k, &public_key->p,
kLTC_Curve25519);
#else
#if defined(USE_INTEL_SPEEDUP) || defined(WOLFSSL_ARMASM)
SAVE_VECTOR_REGISTERS();
#endif
if ((ret = SAVE_VECTOR_REGISTERS()) != 0)
return ret;
ret = curve25519(o.point, private_key->k, public_key->p.point);
#if defined(USE_INTEL_SPEEDUP) || defined(WOLFSSL_ARMASM)
RESTORE_VECTOR_REGISTERS();
#endif
RESTORE_VECTOR_REGISTERS();
#endif
if (ret != 0) {
ForceZero(&o, sizeof(o));

2
wolfcrypt/src/des3.c
View File

@@ -1422,7 +1422,7 @@
byte* const pc1m = buffer; /* place to modify pc1 into */
byte* const pcr = pc1m + 56; /* place to rotate pc1 into */
byte* const ks = pcr + 56;
register int i, j, l;
int i, j, l;
int m;
for (j = 0; j < 56; j++) { /* convert pc1 to bits of key */

173
wolfcrypt/src/memory.c
View File

@@ -1133,3 +1133,176 @@ void __attribute__((no_instrument_function))
}
#endif
#ifdef WOLFSSL_LINUXKM_SIMD_X86_IRQ_ALLOWED
union fpregs_state **wolfcrypt_irq_fpu_states = NULL;
#endif
#if defined(WOLFSSL_LINUXKM_SIMD_X86) && defined(WOLFSSL_LINUXKM_SIMD_X86_IRQ_ALLOWED)
static WARN_UNUSED_RESULT inline int am_in_hard_interrupt_handler(void)
{
return (preempt_count() & (NMI_MASK | HARDIRQ_MASK)) != 0;
}
WARN_UNUSED_RESULT int allocate_wolfcrypt_irq_fpu_states(void)
{
wolfcrypt_irq_fpu_states =
(union fpregs_state **)kzalloc(nr_cpu_ids
* sizeof(struct fpu_state *),
GFP_KERNEL);
if (! wolfcrypt_irq_fpu_states) {
pr_err("warning, allocation of %lu bytes for "
"wolfcrypt_irq_fpu_states failed.\n",
nr_cpu_ids * sizeof(struct fpu_state *));
return MEMORY_E;
}
{
typeof(nr_cpu_ids) i;
for (i=0; i<nr_cpu_ids; ++i) {
_Static_assert(sizeof(union fpregs_state) <= PAGE_SIZE,
"union fpregs_state is larger than expected.");
wolfcrypt_irq_fpu_states[i] =
(union fpregs_state *)kzalloc(PAGE_SIZE
/* sizeof(union fpregs_state) */,
GFP_KERNEL);
if (! wolfcrypt_irq_fpu_states[i])
break;
/* double-check that the allocation is 64-byte-aligned as needed
* for xsave.
*/
if ((unsigned long)wolfcrypt_irq_fpu_states[i] & 63UL) {
pr_err("warning, allocation for wolfcrypt_irq_fpu_states "
"was not properly aligned (%px).\n",
wolfcrypt_irq_fpu_states[i]);
kfree(wolfcrypt_irq_fpu_states[i]);
wolfcrypt_irq_fpu_states[i] = 0;
break;
}
}
if (i < nr_cpu_ids) {
pr_err("warning, only %u/%u allocations succeeded for "
"wolfcrypt_irq_fpu_states.\n",
i, nr_cpu_ids);
return MEMORY_E;
}
}
return 0;
}
void free_wolfcrypt_irq_fpu_states(void)
{
if (wolfcrypt_irq_fpu_states) {
typeof(nr_cpu_ids) i;
for (i=0; i<nr_cpu_ids; ++i) {
if (wolfcrypt_irq_fpu_states[i])
kfree(wolfcrypt_irq_fpu_states[i]);
}
kfree(wolfcrypt_irq_fpu_states);
wolfcrypt_irq_fpu_states = 0;
}
}
WARN_UNUSED_RESULT int save_vector_registers_x86(void)
{
preempt_disable();
if (! irq_fpu_usable()) {
if (am_in_hard_interrupt_handler()) {
int processor_id;
if (! wolfcrypt_irq_fpu_states) {
static int warned_on_null_wolfcrypt_irq_fpu_states = 0;
preempt_enable();
if (! warned_on_null_wolfcrypt_irq_fpu_states) {
warned_on_null_wolfcrypt_irq_fpu_states = 1;
pr_err("save_vector_registers_x86 with null "
"wolfcrypt_irq_fpu_states.\n");
}
return BAD_STATE_E;
}
processor_id = __smp_processor_id();
if (! wolfcrypt_irq_fpu_states[processor_id]) {
static int _warned_on_null = -1;
preempt_enable();
if (_warned_on_null < processor_id) {
_warned_on_null = processor_id;
pr_err("save_vector_registers_x86 for cpu id %d with "
"null wolfcrypt_irq_fpu_states[id].\n",
processor_id);
}
return BAD_STATE_E;
}
/* check for nested interrupts -- doesn't exist on x86, but make
* sure, in case something changes.
*/
if (((char *)wolfcrypt_irq_fpu_states[processor_id])[PAGE_SIZE-1] != 0) {
preempt_enable();
pr_err("save_vector_registers_x86 called recursively for "
"cpu id %d.\n", processor_id);
return BAD_STATE_E;
}
/* note, fpregs_lock() is not needed here, because
* interrupts/preemptions are already disabled here.
*/
{
/* save_fpregs_to_fpstate() only accesses fpu->state, which
* has stringent alignment requirements (64 byte cache
* line), but takes a pointer to the parent struct. work
* around this.
*/
struct fpu *fake_fpu_pointer =
(struct fpu *)(((char *)wolfcrypt_irq_fpu_states[processor_id])
- offsetof(struct fpu, state));
#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 14, 0)
copy_fpregs_to_fpstate(fake_fpu_pointer);
#else
save_fpregs_to_fpstate(fake_fpu_pointer);
#endif
}
/* mark the slot as used. */
((char *)wolfcrypt_irq_fpu_states[processor_id])[PAGE_SIZE-1] = 1;
/* note, not preempt_enable()ing, mirroring kernel_fpu_begin()
* semantics.
*/
return 0;
}
preempt_enable();
return BAD_STATE_E;
} else {
kernel_fpu_begin();
preempt_enable(); /* kernel_fpu_begin() does its own
* preempt_disable(). decrement ours.
*/
return 0;
}
}
void restore_vector_registers_x86(void)
{
if (am_in_hard_interrupt_handler()) {
int processor_id = __smp_processor_id();
if ((wolfcrypt_irq_fpu_states == NULL) ||
(wolfcrypt_irq_fpu_states[processor_id] == NULL) ||
(((char *)wolfcrypt_irq_fpu_states[processor_id])[PAGE_SIZE-1] == 0))
{
pr_err("restore_vector_registers_x86 called for cpu id %d "
"without saved context.\n", processor_id);
preempt_enable(); /* just in case */
return;
} else {
#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 14, 0)
copy_kernel_to_fpregs(wolfcrypt_irq_fpu_states[processor_id]);
#else
__restore_fpregs_from_fpstate(wolfcrypt_irq_fpu_states[processor_id],
xfeatures_mask_all);
#endif
((char *)wolfcrypt_irq_fpu_states[processor_id])[PAGE_SIZE-1] = 0;
preempt_enable();
return;
}
}
kernel_fpu_end();
}
#endif /* WOLFSSL_LINUXKM_SIMD_X86 && WOLFSSL_LINUXKM_SIMD_X86_IRQ_ALLOWED */

2
wolfcrypt/src/misc.c
View File

@@ -299,7 +299,7 @@ WC_STATIC WC_INLINE void xorbuf(void* buf, const void* mask, word32 count)
#ifndef WOLFSSL_NO_FORCE_ZERO
/* This routine fills the first len bytes of the memory area pointed by mem
with zeros. It ensures compiler optimizations doesn't skip it */
WC_STATIC WC_INLINE void ForceZero(const void* mem, word32 len)
WC_STATIC WC_INLINE void ForceZero(void* mem, word32 len)
{
volatile byte* z = (volatile byte*)mem;

44
wolfcrypt/src/poly1305.c
View File

@@ -262,14 +262,17 @@ static WC_INLINE void u32tole64(const word32 inLe32, byte outLe64[8])
This local function operates on a message with a given number of bytes
with a given ctx pointer to a Poly1305 structure.
*/
static void poly1305_blocks(Poly1305* ctx, const unsigned char *m,
static int poly1305_blocks(Poly1305* ctx, const unsigned char *m,
size_t bytes)
{
#ifdef USE_INTEL_SPEEDUP
/* AVX2 is handled in wc_Poly1305Update. */
SAVE_VECTOR_REGISTERS();
int ret = SAVE_VECTOR_REGISTERS();
if (ret != 0)
return ret;
poly1305_blocks_avx(ctx, m, bytes);
RESTORE_VECTOR_REGISTERS();
return 0;
#elif defined(POLY130564)
const word64 hibit = (ctx->finished) ? 0 : ((word64)1 << 40); /* 1 << 128 */
word64 r0,r1,r2;
@@ -320,6 +323,8 @@ static void poly1305_blocks(Poly1305* ctx, const unsigned char *m,
ctx->h[1] = h1;
ctx->h[2] = h2;
return 0;
#else /* if not 64 bit then use 32 bit */
const word32 hibit = (ctx->finished) ? 0 : ((word32)1 << 24); /* 1 << 128 */
word32 r0,r1,r2,r3,r4;
@@ -385,6 +390,8 @@ static void poly1305_blocks(Poly1305* ctx, const unsigned char *m,
ctx->h[3] = h3;
ctx->h[4] = h4;
return 0;
#endif /* end of 64 bit cpu blocks or 32 bit cpu */
}
@@ -392,15 +399,18 @@ static void poly1305_blocks(Poly1305* ctx, const unsigned char *m,
This local function is used for the last call when a message with a given
number of bytes is less than the block size.
*/
static void poly1305_block(Poly1305* ctx, const unsigned char *m)
static int poly1305_block(Poly1305* ctx, const unsigned char *m)
{
#ifdef USE_INTEL_SPEEDUP
/* No call to poly1305_block when AVX2, AVX2 does 4 blocks at a time. */
SAVE_VECTOR_REGISTERS();
int ret= SAVE_VECTOR_REGISTERS();
if (ret != 0)
return ret;
poly1305_block_avx(ctx, m);
RESTORE_VECTOR_REGISTERS();
return 0;
#else
poly1305_blocks(ctx, m, POLY1305_BLOCK_SIZE);
return poly1305_blocks(ctx, m, POLY1305_BLOCK_SIZE);
#endif
}
#endif /* !defined(WOLFSSL_ARMASM) || !defined(__aarch64__) */
@@ -434,7 +444,11 @@ int wc_Poly1305SetKey(Poly1305* ctx, const byte* key, word32 keySz)
intel_flags = cpuid_get_flags();
cpu_flags_set = 1;
}
SAVE_VECTOR_REGISTERS();
{
int ret = SAVE_VECTOR_REGISTERS();
if (ret != 0)
return ret;
}
#ifdef HAVE_INTEL_AVX2
if (IS_INTEL_AVX2(intel_flags))
poly1305_setkey_avx2(ctx, key);
@@ -497,6 +511,7 @@ int wc_Poly1305SetKey(Poly1305* ctx, const byte* key, word32 keySz)
int wc_Poly1305Final(Poly1305* ctx, byte* mac)
{
#ifdef USE_INTEL_SPEEDUP
int ret;
#elif defined(POLY130564)
word64 h0,h1,h2,c;
@@ -516,7 +531,8 @@ int wc_Poly1305Final(Poly1305* ctx, byte* mac)
return BAD_FUNC_ARG;
#ifdef USE_INTEL_SPEEDUP
SAVE_VECTOR_REGISTERS();
if ((ret = SAVE_VECTOR_REGISTERS()) != 0)
return ret;
#ifdef HAVE_INTEL_AVX2
if (IS_INTEL_AVX2(intel_flags))
poly1305_final_avx2(ctx, mac);
@@ -704,7 +720,12 @@ int wc_Poly1305Update(Poly1305* ctx, const byte* m, word32 bytes)
#ifdef USE_INTEL_SPEEDUP
#ifdef HAVE_INTEL_AVX2
if (IS_INTEL_AVX2(intel_flags)) {
int ret = SAVE_VECTOR_REGISTERS();
if (ret != 0)
return ret;
/* handle leftover */
if (ctx->leftover) {
size_t want = sizeof(ctx->buffer) - ctx->leftover;
if (want > bytes)
@@ -718,15 +739,11 @@ int wc_Poly1305Update(Poly1305* ctx, const byte* m, word32 bytes)
if (ctx->leftover < sizeof(ctx->buffer))
return 0;
SAVE_VECTOR_REGISTERS();
if (!ctx->started)
poly1305_calc_powers_avx2(ctx);
poly1305_blocks_avx2(ctx, ctx->buffer, sizeof(ctx->buffer));
ctx->leftover = 0;
}
else {
SAVE_VECTOR_REGISTERS();
}
/* process full blocks */
if (bytes >= sizeof(ctx->buffer)) {
@@ -769,8 +786,11 @@ int wc_Poly1305Update(Poly1305* ctx, const byte* m, word32 bytes)
/* process full blocks */
if (bytes >= POLY1305_BLOCK_SIZE) {
int ret;
size_t want = (bytes & ~(POLY1305_BLOCK_SIZE - 1));
poly1305_blocks(ctx, m, want);
ret = poly1305_blocks(ctx, m, want);
if (ret != 0)
return ret;
m += want;
bytes -= (word32)want;
}

74
wolfcrypt/src/pwdbased.c
View File

@@ -368,11 +368,17 @@ int wc_PKCS12_PBKDF_ex(byte* output, const byte* passwd, int passLen,
byte* buffer = staticBuffer;
#ifdef WOLFSSL_SMALL_STACK
byte* Ai;
byte* B;
byte* Ai = NULL;
byte* B = NULL;
mp_int *B1 = NULL;
mp_int *i1 = NULL;
mp_int *res = NULL;
#else
byte Ai[WC_MAX_DIGEST_SIZE];
byte B[WC_MAX_BLOCK_SIZE];
mp_int B1[1];
mp_int i1[1];
mp_int res[1];
#endif
enum wc_HashType hashT;
@@ -449,9 +455,20 @@ int wc_PKCS12_PBKDF_ex(byte* output, const byte* passwd, int passLen,
for (i = 0; i < (int)pLen; i++)
P[i] = passwd[i % passLen];
#ifdef WOLFSSL_SMALL_STACK
if (((B1 = (mp_int *)XMALLOC(sizeof(*B1), heap, DYNAMIC_TYPE_TMP_BUFFER))
== NULL) ||
((i1 = (mp_int *)XMALLOC(sizeof(*i1), heap, DYNAMIC_TYPE_TMP_BUFFER))
== NULL) ||
((res = (mp_int *)XMALLOC(sizeof(*res), heap, DYNAMIC_TYPE_TMP_BUFFER))
== NULL)) {
ret = MEMORY_E;
goto out;
}
#endif
while (kLen > 0) {
word32 currentLen;
mp_int B1;
ret = DoPKCS12Hash(hashType, buffer, totalLen, Ai, u, iterations);
if (ret < 0)
@@ -460,55 +477,53 @@ int wc_PKCS12_PBKDF_ex(byte* output, const byte* passwd, int passLen,
for (i = 0; i < (int)v; i++)
B[i] = Ai[i % u];
if (mp_init(&B1) != MP_OKAY)
if (mp_init(B1) != MP_OKAY)
ret = MP_INIT_E;
else if (mp_read_unsigned_bin(&B1, B, v) != MP_OKAY)
else if (mp_read_unsigned_bin(B1, B, v) != MP_OKAY)
ret = MP_READ_E;
else if (mp_add_d(&B1, (mp_digit)1, &B1) != MP_OKAY)
else if (mp_add_d(B1, (mp_digit)1, B1) != MP_OKAY)
ret = MP_ADD_E;
if (ret != 0) {
mp_clear(&B1);
mp_clear(B1);
break;
}
for (i = 0; i < (int)iLen; i += v) {
int outSz;
mp_int i1;
mp_int res;
if (mp_init_multi(&i1, &res, NULL, NULL, NULL, NULL) != MP_OKAY) {
if (mp_init_multi(i1, res, NULL, NULL, NULL, NULL) != MP_OKAY) {
ret = MP_INIT_E;
break;
}
if (mp_read_unsigned_bin(&i1, I + i, v) != MP_OKAY)
if (mp_read_unsigned_bin(i1, I + i, v) != MP_OKAY)
ret = MP_READ_E;
else if (mp_add(&i1, &B1, &res) != MP_OKAY)
else if (mp_add(i1, B1, res) != MP_OKAY)
ret = MP_ADD_E;
else if ( (outSz = mp_unsigned_bin_size(&res)) < 0)
else if ( (outSz = mp_unsigned_bin_size(res)) < 0)
ret = MP_TO_E;
else {
if (outSz > (int)v) {
/* take off MSB */
byte tmp[WC_MAX_BLOCK_SIZE + 1];
ret = mp_to_unsigned_bin(&res, tmp);
ret = mp_to_unsigned_bin(res, tmp);
XMEMCPY(I + i, tmp + 1, v);
}
else if (outSz < (int)v) {
XMEMSET(I + i, 0, v - outSz);
ret = mp_to_unsigned_bin(&res, I + i + v - outSz);
ret = mp_to_unsigned_bin(res, I + i + v - outSz);
}
else
ret = mp_to_unsigned_bin(&res, I + i);
ret = mp_to_unsigned_bin(res, I + i);
}
mp_clear(&i1);
mp_clear(&res);
mp_clear(i1);
mp_clear(res);
if (ret < 0) break;
}
if (ret < 0) {
mp_clear(&B1);
mp_clear(B1);
break;
}
@@ -516,16 +531,27 @@ int wc_PKCS12_PBKDF_ex(byte* output, const byte* passwd, int passLen,
XMEMCPY(output, Ai, currentLen);
output += currentLen;
kLen -= currentLen;
mp_clear(&B1);
mp_clear(B1);
}
if (dynamic) XFREE(buffer, heap, DYNAMIC_TYPE_KEY);
#ifdef WOLFSSL_SMALL_STACK
XFREE(Ai, heap, DYNAMIC_TYPE_TMP_BUFFER);
XFREE(B, heap, DYNAMIC_TYPE_TMP_BUFFER);
out:
if (Ai)
XFREE(Ai, heap, DYNAMIC_TYPE_TMP_BUFFER);
if (B)
XFREE(B, heap, DYNAMIC_TYPE_TMP_BUFFER);
if (B1)
XFREE(B1, heap, DYNAMIC_TYPE_TMP_BUFFER);
if (i1)
XFREE(i1, heap, DYNAMIC_TYPE_TMP_BUFFER);
if (res)
XFREE(res, heap, DYNAMIC_TYPE_TMP_BUFFER);
#endif
if (dynamic)
XFREE(buffer, heap, DYNAMIC_TYPE_KEY);
return ret;
}

48
wolfcrypt/src/rsa.c
View File

@@ -4097,7 +4097,11 @@ int wc_CheckProbablePrime_ex(const byte* pRaw, word32 pRawSz,
const byte* eRaw, word32 eRawSz,
int nlen, int* isPrime, WC_RNG* rng)
{
mp_int p, q, e;
#ifdef WOLFSSL_SMALL_STACK
mp_int *p = NULL, *q = NULL, *e = NULL;
#else
mp_int p[1], q[1], e[1];
#endif
mp_int* Q = NULL;
int ret;
@@ -4111,30 +4115,54 @@ int wc_CheckProbablePrime_ex(const byte* pRaw, word32 pRawSz,
if ((qRaw != NULL && qRawSz == 0) || (qRaw == NULL && qRawSz != 0))
return BAD_FUNC_ARG;
ret = mp_init_multi(&p, &q, &e, NULL, NULL, NULL);
#ifdef WOLFSSL_SMALL_STACK
if (((p = (mp_int *)XMALLOC(sizeof(*p), NULL, DYNAMIC_TYPE_RSA_BUFFER)) == NULL) ||
((q = (mp_int *)XMALLOC(sizeof(*q), NULL, DYNAMIC_TYPE_RSA_BUFFER)) == NULL) ||
((e = (mp_int *)XMALLOC(sizeof(*e), NULL, DYNAMIC_TYPE_RSA_BUFFER)) == NULL))
ret = MEMORY_E;
else
ret = 0;
if (ret == 0)
#endif
ret = mp_init_multi(p, q, e, NULL, NULL, NULL);
if (ret == MP_OKAY)
ret = mp_read_unsigned_bin(&p, pRaw, pRawSz);
ret = mp_read_unsigned_bin(p, pRaw, pRawSz);
if (ret == MP_OKAY) {
if (qRaw != NULL) {
ret = mp_read_unsigned_bin(&q, qRaw, qRawSz);
ret = mp_read_unsigned_bin(q, qRaw, qRawSz);
if (ret == MP_OKAY)
Q = &q;
Q = q;
}
}
if (ret == MP_OKAY)
ret = mp_read_unsigned_bin(&e, eRaw, eRawSz);
ret = mp_read_unsigned_bin(e, eRaw, eRawSz);
if (ret == MP_OKAY)
ret = _CheckProbablePrime(&p, Q, &e, nlen, isPrime, rng);
ret = _CheckProbablePrime(p, Q, e, nlen, isPrime, rng);
ret = (ret == MP_OKAY) ? 0 : PRIME_GEN_E;
mp_clear(&p);
mp_clear(&q);
mp_clear(&e);
#ifdef WOLFSSL_SMALL_STACK
if (p) {
mp_clear(p);
XFREE(p, NULL, DYNAMIC_TYPE_RSA_BUFFER);
}
if (q) {
mp_clear(q);
XFREE(q, NULL, DYNAMIC_TYPE_RSA_BUFFER);
}
if (e) {
mp_clear(e);
XFREE(e, NULL, DYNAMIC_TYPE_RSA_BUFFER);
}
#else
mp_clear(p);
mp_clear(q);
mp_clear(e);
#endif
return ret;
}

12
wolfcrypt/src/sha256.c
View File

@@ -319,8 +319,10 @@ static int InitSha256(wc_Sha256* sha256)
static WC_INLINE int inline_XTRANSFORM(wc_Sha256* S, const byte* D) {
int ret;
if (Transform_Sha256_is_vectorized)
SAVE_VECTOR_REGISTERS();
if (Transform_Sha256_is_vectorized) {
if ((ret = SAVE_VECTOR_REGISTERS()) != 0)
return ret;
}
ret = (*Transform_Sha256_p)(S, D);
if (Transform_Sha256_is_vectorized)
RESTORE_VECTOR_REGISTERS();
@@ -330,8 +332,10 @@ static int InitSha256(wc_Sha256* sha256)
static WC_INLINE int inline_XTRANSFORM_LEN(wc_Sha256* S, const byte* D, word32 L) {
int ret;
if (Transform_Sha256_is_vectorized)
SAVE_VECTOR_REGISTERS();
if (Transform_Sha256_is_vectorized) {
if ((ret = SAVE_VECTOR_REGISTERS()) != 0)
return ret;
}
ret = (*Transform_Sha256_Len_p)(S, D, L);
if (Transform_Sha256_is_vectorized)
RESTORE_VECTOR_REGISTERS();

12
wolfcrypt/src/sha512.c
View File

@@ -448,8 +448,10 @@ static int InitSha512_256(wc_Sha512* sha512)
static WC_INLINE int Transform_Sha512(wc_Sha512 *sha512) {
int ret;
if (Transform_Sha512_is_vectorized)
SAVE_VECTOR_REGISTERS();
if (Transform_Sha512_is_vectorized) {
if ((ret = SAVE_VECTOR_REGISTERS()) != 0)
return ret;
}
ret = (*Transform_Sha512_p)(sha512);
if (Transform_Sha512_is_vectorized)
RESTORE_VECTOR_REGISTERS();
@@ -457,8 +459,10 @@ static int InitSha512_256(wc_Sha512* sha512)
}
static WC_INLINE int Transform_Sha512_Len(wc_Sha512 *sha512, word32 len) {
int ret;
if (Transform_Sha512_is_vectorized)
SAVE_VECTOR_REGISTERS();
if (Transform_Sha512_is_vectorized) {
if ((ret = SAVE_VECTOR_REGISTERS()) != 0)
return ret;
}
ret = (*Transform_Sha512_Len_p)(sha512, len);
if (Transform_Sha512_is_vectorized)
RESTORE_VECTOR_REGISTERS();

23
wolfcrypt/src/wc_port.c
View File

@@ -157,6 +157,15 @@ int wolfCrypt_Init(void)
}
#endif
#if defined(WOLFSSL_LINUXKM_SIMD_X86) \
&& defined(WOLFSSL_LINUXKM_SIMD_X86_IRQ_ALLOWED)
ret = allocate_wolfcrypt_irq_fpu_states();
if (ret != 0) {
WOLFSSL_MSG("allocate_wolfcrypt_irq_fpu_states failed");
return ret;
}
#endif
#if WOLFSSL_CRYPT_HW_MUTEX
/* If crypto hardware mutex protection is enabled, then initialize it */
ret = wolfSSL_CryptHwMutexInit();
@@ -356,6 +365,10 @@ int wolfCrypt_Cleanup(void)
rpcmem_deinit();
wolfSSL_CleanupHandle();
#endif
#if defined(WOLFSSL_LINUXKM_SIMD_X86) \
&& defined(WOLFSSL_LINUXKM_SIMD_X86_IRQ_ALLOWED)
free_wolfcrypt_irq_fpu_states();
#endif
}
return ret;
@@ -2438,9 +2451,15 @@ time_t time(time_t * timer)
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
struct timespec ts;
getnstimeofday(&ts);
ret = ts.tv_sec * 1000000000LL + ts.tv_nsec;
ret = ts.tv_sec;
#else
ret = ktime_get_real_seconds();
struct timespec64 ts;
#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 0, 0)
ts = current_kernel_time64();
#else
ktime_get_coarse_real_ts64(&ts);
#endif
ret = ts.tv_sec;
#endif
if (timer)
*timer = ret;

10
wolfcrypt/test/test.c
View File

@@ -9084,7 +9084,12 @@ WOLFSSL_TEST_SUBROUTINE int aes_test(void)
ret = wc_AesSetKey(enc, niKey, sizeof(niKey), cipher, AES_ENCRYPTION);
if (ret != 0)
ERROR_OUT(-5943, out);
#ifdef WOLFSSL_LINUXKM
if (wc_AesEncryptDirect(enc, cipher, niPlain) != 0)
ERROR_OUT(-5950, out);
#else
wc_AesEncryptDirect(enc, cipher, niPlain);
#endif
if (XMEMCMP(cipher, niCipher, AES_BLOCK_SIZE) != 0)
ERROR_OUT(-5944, out);
@@ -9092,7 +9097,12 @@ WOLFSSL_TEST_SUBROUTINE int aes_test(void)
ret = wc_AesSetKey(dec, niKey, sizeof(niKey), plain, AES_DECRYPTION);
if (ret != 0)
ERROR_OUT(-5945, out);
#ifdef WOLFSSL_LINUXKM
if (wc_AesDecryptDirect(dec, plain, niCipher) != 0)
ERROR_OUT(-5951, out);
#else
wc_AesDecryptDirect(dec, plain, niCipher);
#endif
if (XMEMCMP(plain, niPlain, AES_BLOCK_SIZE) != 0)
ERROR_OUT(-5946, out);
}

5
wolfssl/wolfcrypt/aes.h
View File

@@ -351,8 +351,13 @@ WOLFSSL_API int wc_AesEcbDecrypt(Aes* aes, byte* out,
#endif
/* AES-DIRECT */
#if defined(WOLFSSL_AES_DIRECT)
#ifdef WOLFSSL_LINUXKM
WOLFSSL_API __must_check int wc_AesEncryptDirect(Aes* aes, byte* out, const byte* in);
WOLFSSL_API __must_check int wc_AesDecryptDirect(Aes* aes, byte* out, const byte* in);
#else
WOLFSSL_API void wc_AesEncryptDirect(Aes* aes, byte* out, const byte* in);
WOLFSSL_API void wc_AesDecryptDirect(Aes* aes, byte* out, const byte* in);
#endif
WOLFSSL_API int wc_AesSetKeyDirect(Aes* aes, const byte* key, word32 len,
const byte* iv, int dir);
#endif

4
wolfssl/wolfcrypt/curve25519.h
View File

@@ -73,7 +73,7 @@ typedef struct {
#endif
/* A CURVE25519 Key */
typedef struct curve25519_key {
struct curve25519_key {
int idx; /* Index into the ecc_sets[] for the parameters of
this curve if -1, this key is using user supplied
curve in dp */
@@ -92,7 +92,7 @@ typedef struct curve25519_key {
/* bit fields */
byte pubSet:1;
byte privSet:1;
} curve25519_key;
};
enum {
EC25519_LITTLE_ENDIAN=0,

4
wolfssl/wolfcrypt/curve448.h
View File

@@ -49,7 +49,7 @@
#endif
/* A CURVE448 Key */
typedef struct curve448_key {
struct curve448_key {
byte p[CURVE448_PUB_KEY_SIZE]; /* public key */
byte k[CURVE448_KEY_SIZE]; /* private key */
@@ -60,7 +60,7 @@ typedef struct curve448_key {
/* bit fields */
byte pubSet:1;
byte privSet:1;
} curve448_key;
};
enum {
EC448_LITTLE_ENDIAN = 0,

2
wolfssl/wolfcrypt/misc.h
View File

@@ -66,7 +66,7 @@ WOLFSSL_LOCAL
void xorbuf(void*, const void*, word32);
WOLFSSL_LOCAL
void ForceZero(const void*, word32);
void ForceZero(void*, word32);
WOLFSSL_LOCAL
int ConstantCompare(const byte*, const byte*, int);

10
wolfssl/wolfcrypt/types.h
View File

@@ -320,6 +320,16 @@ decouple library dependencies with standard string, memory and so on.
#define FALL_THROUGH
#endif
#ifndef WARN_UNUSED_RESULT
#if defined(WOLFSSL_LINUXKM) && defined(__must_check)
#define WARN_UNUSED_RESULT __must_check
#elif defined(__GNUC__) && (__GNUC__ >= 4)
#define WARN_UNUSED_RESULT __attribute__((warn_unused_result))
#else
#define WARN_UNUSED_RESULT
#endif
#endif /* WARN_UNUSED_RESULT */
/* Micrium will use Visual Studio for compilation but not the Win32 API */
#if defined(_WIN32) && !defined(MICRIUM) && !defined(FREERTOS) && \
!defined(FREERTOS_TCP) && !defined(EBSNET) && \

172
wolfssl/wolfcrypt/wc_port.h
View File

@@ -122,29 +122,47 @@
#endif
#include <linux/net.h>
#include <linux/slab.h>
#if defined(WOLFSSL_AESNI) || defined(USE_INTEL_SPEEDUP)
#if defined(WOLFSSL_AESNI) || defined(USE_INTEL_SPEEDUP) || defined(WOLFSSL_SP_X86_64_ASM)
#ifndef CONFIG_X86
#error X86 SIMD extensions requested, but CONFIG_X86 is not set.
#endif
#define WOLFSSL_LINUXKM_SIMD
#define WOLFSSL_LINUXKM_SIMD_X86
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
#include <asm/i387.h>
#else
#include <asm/simd.h>
#endif
#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 0, 0)
#include <asm/fpu/internal.h>
#endif
#ifndef SAVE_VECTOR_REGISTERS
#define SAVE_VECTOR_REGISTERS() kernel_fpu_begin()
#define SAVE_VECTOR_REGISTERS() save_vector_registers_x86()
#endif
#ifndef RESTORE_VECTOR_REGISTERS
#define RESTORE_VECTOR_REGISTERS() kernel_fpu_end()
#define RESTORE_VECTOR_REGISTERS() restore_vector_registers_x86()
#endif
#elif defined(WOLFSSL_ARMASM)
#elif defined(WOLFSSL_ARMASM) || defined(WOLFSSL_SP_ARM32_ASM) || \
defined(WOLFSSL_SP_ARM64_ASM) || defined(WOLFSSL_SP_ARM_THUMB_ASM) ||\
defined(WOLFSSL_SP_ARM_CORTEX_M_ASM)
#if !defined(CONFIG_ARM) && !defined(CONFIG_ARM64)
#error ARM SIMD extensions requested, but CONFIG_ARM* is not set.
#endif
#define WOLFSSL_LINUXKM_SIMD
#define WOLFSSL_LINUXKM_SIMD_ARM
#include <asm/fpsimd.h>
#ifndef SAVE_VECTOR_REGISTERS
#define SAVE_VECTOR_REGISTERS() ({ preempt_disable(); fpsimd_preserve_current_state(); })
#define SAVE_VECTOR_REGISTERS() save_vector_registers_arm()
#endif
#ifndef RESTORE_VECTOR_REGISTERS
#define RESTORE_VECTOR_REGISTERS() ({ fpsimd_restore_current_state(); preempt_enable(); })
#define RESTORE_VECTOR_REGISTERS() restore_vector_registers_arm()
#endif
#else
#ifndef WOLFSSL_NO_ASM
#define WOLFSSL_NO_ASM
#endif
#ifndef SAVE_VECTOR_REGISTERS
#define SAVE_VECTOR_REGISTERS() ({})
#define SAVE_VECTOR_REGISTERS() 0
#endif
#ifndef RESTORE_VECTOR_REGISTERS
#define RESTORE_VECTOR_REGISTERS() ({})
@@ -247,10 +265,19 @@
typeof(kmalloc_order_trace) *kmalloc_order_trace;
typeof(get_random_bytes) *get_random_bytes;
typeof(ktime_get_real_seconds) *ktime_get_real_seconds;
typeof(ktime_get_with_offset) *ktime_get_with_offset;
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
typeof(getnstimeofday) *getnstimeofday;
#elif LINUX_VERSION_CODE < KERNEL_VERSION(5, 0, 0)
typeof(current_kernel_time64) *current_kernel_time64;
#else
typeof(ktime_get_coarse_real_ts64) *ktime_get_coarse_real_ts64;
#endif
#if defined(WOLFSSL_AESNI) || defined(USE_INTEL_SPEEDUP)
struct task_struct *(*get_current)(void);
int (*preempt_count)(void);
#ifdef WOLFSSL_LINUXKM_SIMD_X86
typeof(irq_fpu_usable) *irq_fpu_usable;
/* kernel_fpu_begin() replaced by kernel_fpu_begin_mask() in commit e4512289,
* released in kernel 5.11, backported to 5.4.93
*/
@@ -260,11 +287,32 @@
typeof(kernel_fpu_begin) *kernel_fpu_begin;
#endif
typeof(kernel_fpu_end) *kernel_fpu_end;
#ifdef WOLFSSL_LINUXKM_SIMD_X86_IRQ_ALLOWED
#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 14, 0)
typeof(copy_fpregs_to_fpstate) *copy_fpregs_to_fpstate;
typeof(copy_kernel_to_fpregs) *copy_kernel_to_fpregs;
#else
typeof(save_fpregs_to_fpstate) *save_fpregs_to_fpstate;
typeof(__restore_fpregs_from_fpstate) *__restore_fpregs_from_fpstate;
typeof(xfeatures_mask_all) *xfeatures_mask_all;
#endif
typeof(cpu_number) *cpu_number;
typeof(nr_cpu_ids) *nr_cpu_ids;
#endif /* WOLFSSL_LINUXKM_SIMD_X86_IRQ_ALLOWED */
#endif /* WOLFSSL_LINUXKM_SIMD_X86 */
typeof(__mutex_init) *__mutex_init;
typeof(mutex_lock) *mutex_lock;
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
typeof(mutex_lock_nested) *mutex_lock_nested;
#else
typeof(mutex_lock) *mutex_lock;
#endif
typeof(mutex_unlock) *mutex_unlock;
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
typeof(mutex_destroy) *mutex_destroy;
#endif
#ifdef HAVE_FIPS
typeof(wolfCrypt_FIPS_first) *wolfCrypt_FIPS_first;
@@ -326,6 +374,7 @@
#define kfree (wolfssl_linuxkm_get_pie_redirect_table()->kfree)
#define ksize (wolfssl_linuxkm_get_pie_redirect_table()->ksize)
#define krealloc (wolfssl_linuxkm_get_pie_redirect_table()->krealloc)
#define kzalloc(size, flags) kmalloc(size, (flags) | __GFP_ZERO)
#ifdef HAVE_KVMALLOC
#define kvmalloc_node (wolfssl_linuxkm_get_pie_redirect_table()->kvmalloc_node)
#define kvfree (wolfssl_linuxkm_get_pie_redirect_table()->kvfree)
@@ -335,21 +384,51 @@
#define kmalloc_order_trace (wolfssl_linuxkm_get_pie_redirect_table()->kmalloc_order_trace)
#define get_random_bytes (wolfssl_linuxkm_get_pie_redirect_table()->get_random_bytes)
#define ktime_get_real_seconds (wolfssl_linuxkm_get_pie_redirect_table()->ktime_get_real_seconds)
#define ktime_get_with_offset (wolfssl_linuxkm_get_pie_redirect_table()->ktime_get_with_offset)
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
#define getnstimeofday (wolfssl_linuxkm_get_pie_redirect_table()->getnstimeofday)
#elif LINUX_VERSION_CODE < KERNEL_VERSION(5, 0, 0)
#define current_kernel_time64 (wolfssl_linuxkm_get_pie_redirect_table()->current_kernel_time64)
#else
#define ktime_get_coarse_real_ts64 (wolfssl_linuxkm_get_pie_redirect_table()->ktime_get_coarse_real_ts64)
#endif
#if defined(WOLFSSL_AESNI) || defined(USE_INTEL_SPEEDUP)
#undef get_current
#define get_current (wolfssl_linuxkm_get_pie_redirect_table()->get_current)
#undef preempt_count
#define preempt_count (wolfssl_linuxkm_get_pie_redirect_table()->preempt_count)
#ifdef WOLFSSL_LINUXKM_SIMD_X86
#define irq_fpu_usable (wolfssl_linuxkm_get_pie_redirect_table()->irq_fpu_usable)
#ifdef kernel_fpu_begin
#define kernel_fpu_begin_mask (wolfssl_linuxkm_get_pie_redirect_table()->kernel_fpu_begin_mask)
#else
#define kernel_fpu_begin (wolfssl_linuxkm_get_pie_redirect_table()->kernel_fpu_begin)
#endif
#define kernel_fpu_end (wolfssl_linuxkm_get_pie_redirect_table()->kernel_fpu_end)
#ifdef WOLFSSL_LINUXKM_SIMD_X86_IRQ_ALLOWED
#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 14, 0)
#define copy_fpregs_to_fpstate (wolfssl_linuxkm_get_pie_redirect_table()->copy_fpregs_to_fpstate)
#define copy_kernel_to_fpregs (wolfssl_linuxkm_get_pie_redirect_table()->copy_kernel_to_fpregs)
#else
#define save_fpregs_to_fpstate (wolfssl_linuxkm_get_pie_redirect_table()->save_fpregs_to_fpstate)
#define __restore_fpregs_from_fpstate (wolfssl_linuxkm_get_pie_redirect_table()->__restore_fpregs_from_fpstate)
#define xfeatures_mask_all (*(wolfssl_linuxkm_get_pie_redirect_table()->xfeatures_mask_all))
#endif
#define cpu_number (*(wolfssl_linuxkm_get_pie_redirect_table()->cpu_number))
#define nr_cpu_ids (*(wolfssl_linuxkm_get_pie_redirect_table()->nr_cpu_ids))
#endif /* WOLFSSL_LINUXKM_SIMD_X86_IRQ_ALLOWED */
#endif
#define __mutex_init (wolfssl_linuxkm_get_pie_redirect_table()->__mutex_init)
#define mutex_lock (wolfssl_linuxkm_get_pie_redirect_table()->mutex_lock)
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
#define mutex_lock_nested (wolfssl_linuxkm_get_pie_redirect_table()->mutex_lock_nested)
#else
#define mutex_lock (wolfssl_linuxkm_get_pie_redirect_table()->mutex_lock)
#endif
#define mutex_unlock (wolfssl_linuxkm_get_pie_redirect_table()->mutex_unlock)
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
#define mutex_destroy (wolfssl_linuxkm_get_pie_redirect_table()->mutex_destroy)
#endif
/* per linux/ctype.h, tolower() and toupper() are macros bound to static inlines
* that use macros that bring in the _ctype global. for __PIE__, this needs to
@@ -371,6 +450,55 @@
#endif /* USE_WOLFSSL_LINUXKM_PIE_REDIRECT_TABLE */
#ifdef WOLFSSL_LINUXKM_SIMD
#ifdef WOLFSSL_LINUXKM_SIMD_X86
#ifdef WOLFSSL_LINUXKM_SIMD_X86_IRQ_ALLOWED
extern __must_check int allocate_wolfcrypt_irq_fpu_states(void);
extern void free_wolfcrypt_irq_fpu_states(void);
extern __must_check int save_vector_registers_x86(void);
extern void restore_vector_registers_x86(void);
#else /* !WOLFSSL_LINUXKM_SIMD_X86_IRQ_ALLOWED */
#define save_vector_registers_x86() ({ \
int _ret; \
preempt_disable(); \
if (! irq_fpu_usable()) { \
preempt_enable(); \
_ret = BAD_STATE_E; \
} else { \
kernel_fpu_begin(); \
preempt_enable(); /* kernel_fpu_begin() does its own preempt_disable(). decrement ours. */ \
_ret = 0; \
} \
_ret; \
})
#define restore_vector_registers_x86() kernel_fpu_end()
#endif /* !WOLFSSL_LINUXKM_SIMD_X86_IRQ_ALLOWED */
#elif defined(CONFIG_ARM) || defined(CONFIG_ARM64)
static WARN_UNUSED_RESULT inline int save_vector_registers_arm(void)
{
preempt_disable();
if (! may_use_simd()) {
preempt_enable();
return BAD_STATE_E;
} else {
fpsimd_preserve_current_state();
return 0;
}
}
static inline void restore_vector_registers_arm(void)
{
fpsimd_restore_current_state();
preempt_enable();
}
#endif
#endif /* WOLFSSL_LINUXKM_SIMD */
/* Linux headers define these using C expressions, but we need
* them to be evaluable by the preprocessor, for use in sp_int.h.
*/
@@ -466,8 +594,6 @@
extern void fipsEntry(void);
#endif
#endif /* BUILDING_WOLFSSL */
/* needed to suppress inclusion of stdio.h in wolfssl/wolfcrypt/types.h */
#define XSNPRINTF snprintf
@@ -481,10 +607,20 @@
(int)_xatoi_res; \
})
/* suppress false-positive "writing 1 byte into a region of size 0" warnings
* building old kernels with new gcc:
*/
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
_Pragma("GCC diagnostic ignored \"-Wstringop-overflow\"");
#endif
#endif /* BUILDING_WOLFSSL */
#else /* ! WOLFSSL_LINUXKM */
#ifndef SAVE_VECTOR_REGISTERS
#define SAVE_VECTOR_REGISTERS() do{}while(0)
#define SAVE_VECTOR_REGISTERS() 0
#endif
#ifndef RESTORE_VECTOR_REGISTERS
#define RESTORE_VECTOR_REGISTERS() do{}while(0)