mirror of
https://github.com/wolfSSL/wolfssl.git
synced 2026-07-05 12:50:51 +02:00
Merge pull request #9714 from philljj/bsdkm_crypto_accel
bsdkm: x86 crypto acceleration support.
This commit is contained in:
Daniel Pouzzner
GitHub
@@ -239,12 +239,17 @@ linuxkm/linuxkm
|
||||
linuxkm/src
|
||||
linuxkm/patches/src
|
||||
*.nds
|
||||
|
||||
# Generated during FreeBSD kernel module build.
|
||||
bsdkm/export_syms
|
||||
bsdkm/i386
|
||||
bsdkm/libwolfssl.ko
|
||||
bsdkm/machine
|
||||
bsdkm/opt_global.h
|
||||
bsdkm/x86
|
||||
bsdkm/bus_if.h
|
||||
bsdkm/cryptodev_if.h
|
||||
bsdkm/device_if.h
|
||||
|
||||
# autotools generated
|
||||
scripts/unit.test
|
||||
|
||||
@@ -36,6 +36,7 @@ BLAKE2B_SELFTEST
|
||||
BLAKE2S_SELFTEST
|
||||
BLOCKING
|
||||
BSDKM_EXPORT_SYMS
|
||||
ENABLED_BSDKM_REGISTER
|
||||
BSP_DEFAULT_IO_CHANNEL_DEFINED
|
||||
BSP_LED_0
|
||||
BSP_LED_1
|
||||
|
||||
+3
-2
@@ -247,8 +247,9 @@ if BUILD_BSDKM
|
||||
EXTRA_CFLAGS EXTRA_CPPFLAGS EXTRA_CCASFLAGS EXTRA_LDFLAGS \
|
||||
AM_CPPFLAGS CPPFLAGS AM_CFLAGS CFLAGS \
|
||||
AM_CCASFLAGS CCASFLAGS \
|
||||
src_libwolfssl_la_OBJECTS ENABLED_CRYPT_TESTS
|
||||
|
||||
src_libwolfssl_la_OBJECTS ENABLED_CRYPT_TESTS ENABLED_BSDKM_REGISTER \
|
||||
ENABLED_ASM ENABLED_INTELASM ENABLED_AESNI ENABLED_AESNI_WITH_AVX \
|
||||
ENABLED_KERNEL_BENCHMARKS
|
||||
endif
|
||||
|
||||
|
||||
|
||||
+72
-17
@@ -1,19 +1,27 @@
|
||||
# wolfssl kernel module name and source, and root dir.
|
||||
KMOD=libwolfssl
|
||||
SRCS=wolfkmod.c
|
||||
WOLFSSL_DIR=../
|
||||
# wolfssl kernel module name and main source, and wolfssl root dir.
|
||||
KMOD = libwolfssl
|
||||
SRCS = wolfkmod.c
|
||||
WOLFSSL_DIR = ../
|
||||
|
||||
CFLAGS+=-I${WOLFSSL_DIR}
|
||||
CFLAGS+=-DWOLFSSL_IGNORE_FILE_WARN -DHAVE_CONFIG_H -DNO_MAIN_DRIVER
|
||||
CFLAGS += -I${WOLFSSL_DIR}
|
||||
CFLAGS += -DWOLFSSL_IGNORE_FILE_WARN -DHAVE_CONFIG_H -DNO_MAIN_DRIVER
|
||||
#
|
||||
# debug options
|
||||
# verbose printing:
|
||||
# CFLAGS+=-DWOLFSSL_BSDKM_VERBOSE_DEBUG
|
||||
# CFLAGS += -DWOLFSSL_BSDKM_VERBOSE_DEBUG
|
||||
#
|
||||
# print memory mallocs / frees:
|
||||
# CFLAGS+=-DWOLFSSL_BSDKM_MEMORY_DEBUG
|
||||
# CFLAGS += -DWOLFSSL_BSDKM_MEMORY_DEBUG
|
||||
#
|
||||
CFLAGS+=$(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS)
|
||||
# print fpu_kern_enter / leave:
|
||||
# CFLAGS += WOLFSSL_BSDKM_FPU_DEBUG
|
||||
#
|
||||
CFLAGS += $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS)
|
||||
|
||||
.if defined(ENABLED_BSDKM_REGISTER)
|
||||
# These device header files are generated during build.
|
||||
SRCS += bus_if.h cryptodev_if.h device_if.h
|
||||
.endif
|
||||
|
||||
# FreeBSD make does not support GNU make's patsubst and related. Filter
|
||||
# through sed instead.
|
||||
@@ -21,19 +29,26 @@ WOLFSSL_OBJS != echo ${src_libwolfssl_la_OBJECTS} | \
|
||||
sed 's|src_libwolfssl_la-||g' | sed 's|\.lo|.o|g' | \
|
||||
sed 's|wolfcrypt/src/|${WOLFSSL_DIR}/wolfcrypt/src/|g'
|
||||
|
||||
# wolfcrypt test
|
||||
.if ${ENABLED_CRYPT_TESTS} == "yes"
|
||||
WOLFSSL_OBJS += ${WOLFSSL_DIR}/wolfcrypt/test/test.o
|
||||
.else
|
||||
CFLAGS+=-DNO_CRYPT_TEST
|
||||
CFLAGS += -DNO_CRYPT_TEST
|
||||
.endif
|
||||
|
||||
# wolfcrypt benchmark
|
||||
.if ${ENABLED_KERNEL_BENCHMARKS} == "yes"
|
||||
WOLFSSL_OBJS += ${WOLFSSL_DIR}/wolfcrypt/benchmark/benchmark.o
|
||||
CFLAGS += -DWOLFSSL_NO_FLOAT_FMT
|
||||
.endif
|
||||
|
||||
OBJS += ${WOLFSSL_OBJS}
|
||||
|
||||
# Export no public symbols by default.
|
||||
.if !defined(BSDKM_EXPORT_SYMS)
|
||||
EXPORT_SYMS=NO
|
||||
EXPORT_SYMS = NO
|
||||
.else
|
||||
EXPORT_SYMS=${BSDKM_EXPORT_SYMS}
|
||||
EXPORT_SYMS = ${BSDKM_EXPORT_SYMS}
|
||||
.endif
|
||||
|
||||
# Default to live kernel src tree makefile at
|
||||
@@ -45,12 +60,52 @@ OBJS += ${WOLFSSL_OBJS}
|
||||
.endif
|
||||
.include "${SYSDIR}/conf/kmod.mk"
|
||||
|
||||
#
|
||||
# To use aesni and friends in FreeBSD kernel we need to adjust build flags.
|
||||
# See these kernel makefiles for reference:
|
||||
# - /usr/src/sys/modules/aesni/Makefile
|
||||
# - /usr/src/sys/conf/kern.mk
|
||||
#
|
||||
WOLFKMOD_SIMD_BASE = -msse -msse2 -msse4.1
|
||||
WOLFKMOD_SIMD_AES = -maes -mpclmul
|
||||
WOLFKMOD_SIMD_AVX = -mavx -mavx2
|
||||
|
||||
.if ${ENABLED_AESNI} == "yes"
|
||||
CFLAGS.aes.c += ${WOLFKMOD_SIMD_BASE}
|
||||
CFLAGS.aes.c += ${WOLFKMOD_SIMD_AES}
|
||||
.if ${ENABLED_AESNI_WITH_AVX} == "yes"
|
||||
CFLAGS.aes.c += ${WOLFKMOD_SIMD_AVX}
|
||||
.endif # ENABLED_AESNI_WITH_AVX #
|
||||
CFLAGS.aes.c := ${CFLAGS.aes.c:N-nostdinc}
|
||||
CFLAGS.aes.c += -I${SYSDIR}/../contrib/llvm-project/clang/lib/Headers
|
||||
.PATH: ${SYSDIR}/../contrib/llvm-project/clang/lib/Headers
|
||||
.endif # ENABLED_AESNI
|
||||
|
||||
.if ${ENABLED_ASM} == "yes"
|
||||
.for f in chacha dilithium poly1305 sha sha256 sha3 sha512
|
||||
CFLAGS.${f}.c += ${WOLFKMOD_SIMD_BASE}
|
||||
CFLAGS.${f}.c += ${WOLFKMOD_SIMD_AVX}
|
||||
CFLAGS.${f}.c := ${CFLAGS.${f}.c:N-nostdinc}
|
||||
CFLAGS.${f}.c += -I${SYSDIR}/../contrib/llvm-project/clang/lib/Headers
|
||||
.endfor
|
||||
|
||||
.PATH: ${SYSDIR}/../contrib/llvm-project/clang/lib/Headers
|
||||
.endif # ENABLED_ASM == "yes"
|
||||
|
||||
# wolfcrypt benchmark always needs simd for the floating point timings.
|
||||
.if ${ENABLED_KERNEL_BENCHMARKS} == "yes"
|
||||
CFLAGS.benchmark.c += ${WOLFKMOD_SIMD_BASE}
|
||||
CFLAGS.benchmark.c := ${CFLAGS.benchmark.c:N-nostdinc}
|
||||
CFLAGS.benchmark.c += -I${SYSDIR}/../contrib/llvm-project/clang/lib/Headers
|
||||
.PATH: ${SYSDIR}/../contrib/llvm-project/clang/lib/Headers
|
||||
.endif
|
||||
|
||||
# Smooth out a few inconsistencies between FreeBSD default compiler flags
|
||||
# in /usr/src/sys/conf/kern.mk, vs wolfssl harden flags in
|
||||
# m4/ax_harden_compiler_flags.m4. E.g. some FreeBSD header files shorten
|
||||
# 64 to 32 bit, and some wolfcrypt functions cast away const.
|
||||
CFLAGS+= -Wno-unused-function
|
||||
CFLAGS+= -Wno-cast-qual
|
||||
CFLAGS+= -Wno-error=cast-qual
|
||||
CFLAGS+= -Wno-shorten-64-to-32
|
||||
CFLAGS+= -DLIBWOLFSSL_GLOBAL_EXTRA_CFLAGS="\" $(KERNEL_EXTRA_CFLAGS)\""
|
||||
CFLAGS += -Wno-unused-function
|
||||
CFLAGS += -Wno-cast-qual
|
||||
CFLAGS += -Wno-error=cast-qual
|
||||
CFLAGS += -Wno-shorten-64-to-32
|
||||
CFLAGS += -DLIBWOLFSSL_GLOBAL_EXTRA_CFLAGS="\" $(KERNEL_EXTRA_CFLAGS)\""
|
||||
|
||||
+8
-6
@@ -7,10 +7,10 @@ other loadable modules to link to wolfCrypt.
|
||||
Supported features:
|
||||
- wolfCrypt in kernel.
|
||||
- FIPS-wolfcrypt.
|
||||
- crypto acceleration: AES-NI, AVX, etc.
|
||||
|
||||
Planned features:
|
||||
- crypto acceleration: AES-NI, AVX, etc.
|
||||
- kernel opencrypto driver registration.
|
||||
- kernel opencrypto driver registration (supported for internal testing presently).
|
||||
- full wolfSSL in kernel (kernel TLS).
|
||||
|
||||
## Building and Installing
|
||||
@@ -44,10 +44,12 @@ sudo kldunload libwolfssl
|
||||
|
||||
### options
|
||||
|
||||
| freebsdkm option | description |
|
||||
| :------------------------------- | :--------------------------------------- |
|
||||
| --with-bsd-export-syms=LIST | Export list of symbols as global. <br>. Options are 'all', 'none', or <br> comma separated list of symbols. |
|
||||
| --with-kernel-source=PATH | Path to kernel tree root (default `/usr/src/sys`) |
|
||||
| freebsdkm option | description |
|
||||
| :--------------------------------- | :--------------------------------------- |
|
||||
| --with-bsd-export-syms=LIST | Export list of symbols as global. <br>. Options are 'all', 'none', or <br> comma separated list of symbols. |
|
||||
| --with-kernel-source=PATH | Path to kernel tree root (default `/usr/src/sys`) |
|
||||
| --enable-kernel-benchmarks | Run wolfcrypt benchmark at module load |
|
||||
| --enable-freebsdkm-crypto-register | Register with the FreeBSD kernel opencrypto <br>framework (preliminary, for testing) |
|
||||
|
||||
### FIPS
|
||||
|
||||
|
||||
+36
-8
@@ -61,13 +61,13 @@ static inline time_t wolfkmod_time(time_t * tloc) {
|
||||
#define WOLFSSL_DEBUG_PRINTF_FN printf
|
||||
|
||||
/* str and char utility functions */
|
||||
#define XATOI(s) ({ \
|
||||
char * endptr = NULL; \
|
||||
long _xatoi_ret = strtol(s, &endptr, 10); \
|
||||
if ((s) == endptr || *endptr != '\0') { \
|
||||
_xatoi_ret = 0; \
|
||||
} \
|
||||
(int)_xatoi_ret; \
|
||||
#define XATOI(s) ({ \
|
||||
char * endptr = NULL; \
|
||||
long _xatoi_ret = strtol(s, &endptr, 10); \
|
||||
if ((s) == endptr || *endptr != '\0') { \
|
||||
_xatoi_ret = 0; \
|
||||
} \
|
||||
(int)_xatoi_ret; \
|
||||
})
|
||||
|
||||
#if !defined(XMALLOC_OVERRIDE)
|
||||
@@ -103,6 +103,33 @@ extern struct malloc_type M_WOLFSSL[1];
|
||||
})
|
||||
#endif /* WOLFSSL_BSDKM_DEBUG_MEMORY */
|
||||
|
||||
|
||||
#if defined(WOLFSSL_AESNI) || defined(WOLFSSL_KERNEL_BENCHMARKS)
|
||||
int wolfkmod_vecreg_init(void);
|
||||
void wolfkmod_vecreg_exit(void);
|
||||
int wolfkmod_vecreg_save(int flags_unused);
|
||||
void wolfkmod_vecreg_restore(void);
|
||||
/* wrapper defines for FPU_KERN(9).
|
||||
* /usr/src/sys/amd64/amd64/fpu.c
|
||||
* /usr/src/sys/amd64/include/pcb.h
|
||||
* */
|
||||
#ifndef WOLFSSL_USE_SAVE_VECTOR_REGISTERS
|
||||
#define WOLFSSL_USE_SAVE_VECTOR_REGISTERS
|
||||
#endif
|
||||
|
||||
#define SAVE_VECTOR_REGISTERS(fail_clause) { \
|
||||
int _svr_ret = wolfkmod_vecreg_save(0); \
|
||||
if (_svr_ret != 0) { \
|
||||
fail_clause \
|
||||
} \
|
||||
}
|
||||
|
||||
#define SAVE_VECTOR_REGISTERS2() wolfkmod_vecreg_save(0)
|
||||
|
||||
#define RESTORE_VECTOR_REGISTERS() wolfkmod_vecreg_restore()
|
||||
|
||||
#endif /* WOLFSSL_AESNI || WOLFSSL_KERNEL_BENCHMARKS */
|
||||
|
||||
#if !defined(SINGLE_THREADED)
|
||||
#define WC_MUTEX_OPS_INLINE
|
||||
|
||||
@@ -149,7 +176,8 @@ extern struct malloc_type M_WOLFSSL[1];
|
||||
typedef volatile int wolfSSL_Atomic_Int;
|
||||
typedef volatile unsigned int wolfSSL_Atomic_Uint;
|
||||
#define WOLFSSL_ATOMIC_INITIALIZER(x) (x)
|
||||
#define WOLFSSL_ATOMIC_LOAD(x) (int)atomic_load_acq_int(&(x))
|
||||
#define WOLFSSL_ATOMIC_LOAD(x) (int)atomic_load_acq_int(&(x))
|
||||
#define WOLFSSL_ATOMIC_LOAD_UINT(x) atomic_load_acq_int(&(x))
|
||||
#define WOLFSSL_ATOMIC_STORE(x, v) atomic_store_rel_int(&(x), (v))
|
||||
#define WOLFSSL_ATOMIC_OPS
|
||||
|
||||
|
||||
+6
-4
@@ -2,8 +2,10 @@
|
||||
# included from Top Level Makefile.am
|
||||
# All paths should be given relative to the root
|
||||
|
||||
EXTRA_DIST += m4/ax_bsdkm.m4 \
|
||||
bsdkm/Makefile \
|
||||
bsdkm/README.md \
|
||||
bsdkm/wolfkmod.c \
|
||||
EXTRA_DIST += m4/ax_bsdkm.m4 \
|
||||
bsdkm/Makefile \
|
||||
bsdkm/README.md \
|
||||
bsdkm/wolfkmod.c \
|
||||
bsdkm/wolfkmod_aes.c \
|
||||
bsdkm/x86_vecreg.c \
|
||||
bsdkm/bsdkm_wc_port.h
|
||||
|
||||
+780
-17
@@ -26,6 +26,12 @@
|
||||
#include <sys/module.h>
|
||||
#include <sys/kernel.h>
|
||||
|
||||
#if defined(BSDKM_CRYPTO_REGISTER)
|
||||
#include <opencrypto/cryptodev.h>
|
||||
#include <sys/bus.h>
|
||||
#include "cryptodev_if.h"
|
||||
#endif
|
||||
|
||||
/* wolf includes */
|
||||
#include <wolfssl/wolfcrypt/libwolfssl_sources.h>
|
||||
#ifdef WOLFCRYPT_ONLY
|
||||
@@ -44,15 +50,42 @@
|
||||
#if !defined(NO_CRYPT_TEST)
|
||||
#include <wolfcrypt/test/test.h>
|
||||
#endif
|
||||
#if defined(WOLFSSL_KERNEL_BENCHMARKS)
|
||||
#include <wolfcrypt/benchmark/benchmark.h>
|
||||
#endif
|
||||
|
||||
#include <wolfssl/wolfcrypt/random.h>
|
||||
|
||||
MALLOC_DEFINE(M_WOLFSSL, "libwolfssl", "wolfSSL kernel memory");
|
||||
|
||||
static int wolfkmod_init(void);
|
||||
static int wolfkmod_cleanup(void);
|
||||
static int wolfkmod_load(void);
|
||||
static int wolfkmod_unload(void);
|
||||
#if defined(BSDKM_CRYPTO_REGISTER)
|
||||
#include "bsdkm/wolfkmod_aes.c"
|
||||
#endif
|
||||
|
||||
/* common functions. */
|
||||
static int wolfkmod_init(void);
|
||||
static int wolfkmod_cleanup(void);
|
||||
#if !defined(BSDKM_CRYPTO_REGISTER)
|
||||
/* functions specific to a pure kernel module library build. */
|
||||
static int wolfkmod_load(void);
|
||||
static int wolfkmod_unload(void);
|
||||
#else
|
||||
/* functions specific to a kernel crypto driver module build. */
|
||||
static void wolfkdriv_identify(driver_t * driver, device_t parent);
|
||||
static int wolfkdriv_probe(device_t dev);
|
||||
static int wolfkdriv_attach(device_t dev);
|
||||
static int wolfkdriv_detach(device_t dev);
|
||||
static int wolfkdriv_probesession(device_t dev,
|
||||
const struct crypto_session_params *csp);
|
||||
static int wolfkdriv_newsession(device_t dev, crypto_session_t cses,
|
||||
const struct crypto_session_params *csp);
|
||||
static void wolfkdriv_freesession(device_t dev, crypto_session_t cses);
|
||||
static int wolfkdriv_process(device_t dev, struct cryptop *crp, int hint);
|
||||
#endif /* !BSDKM_CRYPTO_REGISTER */
|
||||
|
||||
#if defined(WOLFSSL_AESNI) || defined(WOLFSSL_KERNEL_BENCHMARKS)
|
||||
#include "bsdkm/x86_vecreg.c"
|
||||
#endif /* WOLFSSL_AESNI || WOLFSSL_KERNEL_BENCHMARKS*/
|
||||
|
||||
#ifdef HAVE_FIPS
|
||||
#define WOLFKMOD_FIPS_ERR_MSG(hash) ({ \
|
||||
@@ -82,6 +115,14 @@ static int wolfkmod_init(void)
|
||||
{
|
||||
int error = 0;
|
||||
|
||||
#if defined(WOLFSSL_AESNI) || defined(WOLFSSL_KERNEL_BENCHMARKS)
|
||||
error = wolfkmod_vecreg_init();
|
||||
if (error != 0) {
|
||||
printf("error: wolfkmod_vecreg_init: %d\n", error);
|
||||
return (ECANCELED);
|
||||
}
|
||||
#endif /* WOLFSSL_AESNI || WOLFSSL_KERNEL_BENCHMARKS*/
|
||||
|
||||
#ifdef HAVE_FIPS
|
||||
error = wolfCrypt_SetCb_fips(wolfkmod_fips_cb);
|
||||
if (error != 0) {
|
||||
@@ -174,14 +215,16 @@ static int wolfkmod_cleanup(void)
|
||||
if (error != 0) {
|
||||
printf("error: wolfCrypt_Cleanup failed: %s\n",
|
||||
wc_GetErrorString(error));
|
||||
return (ECANCELED);
|
||||
error = ECANCELED;
|
||||
goto wolfkmod_cleanup_out;
|
||||
}
|
||||
#else
|
||||
error = wolfSSL_Cleanup();
|
||||
if (error != WOLFSSL_SUCCESS) {
|
||||
printf("error: wolfSSL_Cleanup failed: %s\n",
|
||||
wc_GetErrorString(error));
|
||||
return (ECANCELED);
|
||||
error = ECANCELED;
|
||||
goto wolfkmod_cleanup_out;
|
||||
}
|
||||
#endif /* WOLFCRYPT_ONLY */
|
||||
|
||||
@@ -189,10 +232,17 @@ static int wolfkmod_cleanup(void)
|
||||
printf("info: libwolfssl " LIBWOLFSSL_VERSION_STRING
|
||||
" cleanup complete.\n");
|
||||
#endif /* WOLFSSL_BSDKM_VERBOSE_DEBUG */
|
||||
error = 0;
|
||||
|
||||
return (0);
|
||||
wolfkmod_cleanup_out:
|
||||
#if defined(WOLFSSL_AESNI) || defined(WOLFSSL_KERNEL_BENCHMARKS)
|
||||
wolfkmod_vecreg_exit();
|
||||
#endif /* WOLFSSL_AESNI || WOLFSSL_KERNEL_BENCHMARKS*/
|
||||
|
||||
return (error);
|
||||
}
|
||||
|
||||
#if !defined(BSDKM_CRYPTO_REGISTER)
|
||||
static int wolfkmod_load(void)
|
||||
{
|
||||
int error = 0;
|
||||
@@ -212,10 +262,15 @@ static int wolfkmod_load(void)
|
||||
printf("info: wolfCrypt self-test passed.\n");
|
||||
#endif /* NO_CRYPT_TEST */
|
||||
|
||||
/**
|
||||
* todo: register wolfcrypt algs here with crypto_get_driverid
|
||||
* and related.
|
||||
* */
|
||||
#ifdef WOLFSSL_KERNEL_BENCHMARKS
|
||||
error = benchmark_test(NULL);
|
||||
if (error != 0) {
|
||||
printf("error: wolfcrypt benchmark failed: %d\n", error);
|
||||
(void)wolfkmod_cleanup();
|
||||
return (ECANCELED);
|
||||
}
|
||||
printf("info: wolfCrypt benchmark passed.\n");
|
||||
#endif /* WOLFSSL_KERNEL_BENCHMARKS */
|
||||
|
||||
printf("info: libwolfssl loaded\n");
|
||||
|
||||
@@ -239,11 +294,6 @@ static int wolfkmod_unload(void)
|
||||
|
||||
error = wolfkmod_cleanup();
|
||||
|
||||
/**
|
||||
* todo: unregister wolfcrypt algs here with crypto_unregister_all
|
||||
* and related.
|
||||
* */
|
||||
|
||||
if (error == 0) {
|
||||
printf("info: libwolfssl unloaded\n");
|
||||
}
|
||||
@@ -294,7 +344,718 @@ wolfkmod_event(struct module * m, int what, void * arg)
|
||||
|
||||
return (error);
|
||||
}
|
||||
#endif /* !BSDKM_CRYPTO_REGISTER */
|
||||
|
||||
#if defined(BSDKM_CRYPTO_REGISTER)
|
||||
/* wolfkdriv device driver software context. */
|
||||
struct wolfkdriv_softc {
|
||||
int32_t crid;
|
||||
device_t dev;
|
||||
};
|
||||
|
||||
struct km_aes_ctx {
|
||||
Aes aes_encrypt;
|
||||
Aes aes_decrypt;
|
||||
};
|
||||
|
||||
typedef struct km_aes_ctx km_aes_ctx;
|
||||
|
||||
struct wolfkdriv_session {
|
||||
km_aes_ctx aes_ctx;
|
||||
int32_t crid;
|
||||
int type;
|
||||
int ivlen;
|
||||
int klen;
|
||||
};
|
||||
|
||||
typedef struct wolfkdriv_session wolfkdriv_session_t;
|
||||
|
||||
static void km_AesFree(Aes * aes) {
|
||||
if (aes == NULL) {
|
||||
return;
|
||||
}
|
||||
wc_AesFree(aes);
|
||||
#if defined(HAVE_FIPS) && FIPS_VERSION3_LT(6,0,0)
|
||||
ForceZero(aes, sizeof(*aes));
|
||||
#endif
|
||||
}
|
||||
|
||||
static void wolfkdriv_aes_ctx_clear(km_aes_ctx * ctx)
|
||||
{
|
||||
if (ctx != NULL) {
|
||||
km_AesFree(&ctx->aes_encrypt);
|
||||
km_AesFree(&ctx->aes_decrypt);
|
||||
}
|
||||
|
||||
#ifdef WOLFKM_DEBUG_AES
|
||||
printf("info: exiting km_AesExitCommon\n");
|
||||
#endif /* WOLFKM_DEBUG_AES */
|
||||
}
|
||||
|
||||
static void wolfkdriv_identify(driver_t * driver, device_t parent)
|
||||
{
|
||||
(void)driver;
|
||||
|
||||
/* don't double add wolfkdriv child. */
|
||||
if (device_find_child(parent, "libwolf", -1) != NULL) {
|
||||
return;
|
||||
}
|
||||
|
||||
BUS_ADD_CHILD(parent, 10, "libwolf", -1);
|
||||
}
|
||||
|
||||
static int wolfkdriv_probe(device_t dev)
|
||||
{
|
||||
device_set_desc(dev, "wolfSSL crypto");
|
||||
return (BUS_PROBE_DEFAULT);
|
||||
}
|
||||
|
||||
/*
|
||||
* unregister libwolfssl crypto driver
|
||||
*/
|
||||
static void wolfkdriv_unregister(struct wolfkdriv_softc * softc)
|
||||
{
|
||||
if (softc && softc->crid >= 0) {
|
||||
crypto_unregister_all(softc->crid);
|
||||
device_printf(softc->dev, "info: crid unregistered: %d\n", softc->crid);
|
||||
softc->crid = -1;
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
static int wolfkdriv_attach(device_t dev)
|
||||
{
|
||||
struct wolfkdriv_softc * softc = NULL;
|
||||
int flags = CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC |
|
||||
CRYPTOCAP_F_ACCEL_SOFTWARE | CRYPTOCAP_F_HARDWARE;
|
||||
int ret = 0;
|
||||
int crid = 0;
|
||||
int error = 0;
|
||||
|
||||
ret = wolfkmod_init();
|
||||
if (ret != 0) {
|
||||
return (ECANCELED);
|
||||
}
|
||||
|
||||
/**
|
||||
* register wolfcrypt algs here with crypto_get_driverid.
|
||||
*
|
||||
* The crid is the literal index into the kernel crypto_drivers array:
|
||||
* - crid >= 0 is valid.
|
||||
* - crid < 0 is error.
|
||||
* */
|
||||
softc = device_get_softc(dev);
|
||||
softc->dev = dev;
|
||||
|
||||
softc->crid = crypto_get_driverid(dev, sizeof(wolfkdriv_session_t), flags);
|
||||
if (softc->crid < 0) {
|
||||
device_printf(dev, "error: crypto_get_driverid failed: %d\n",
|
||||
softc->crid);
|
||||
return (ENXIO);
|
||||
}
|
||||
|
||||
/*
|
||||
* various sanity checks
|
||||
*/
|
||||
|
||||
/* 1. we should find ourself by name */
|
||||
crid = crypto_find_driver("libwolf");
|
||||
|
||||
if (crid != softc->crid) {
|
||||
device_printf(dev, "error: attach: got crid %d, expected %d\n", crid,
|
||||
softc->crid);
|
||||
error = ENXIO;
|
||||
goto attach_out;
|
||||
}
|
||||
|
||||
/* 2. test various algs */
|
||||
error = wolfkdriv_test_aes(dev, crid);
|
||||
|
||||
if (error) {
|
||||
device_printf(dev, "error: attach: test_aes: %d\n", error);
|
||||
error = ENXIO;
|
||||
goto attach_out;
|
||||
}
|
||||
|
||||
device_printf(dev, "info: driver loaded: %d\n", crid);
|
||||
|
||||
#if defined(WOLFSSL_BSDKM_VERBOSE_DEBUG)
|
||||
device_printf(dev, "info: exiting attach\n");
|
||||
#endif /* WOLFSSL_BSDKM_VERBOSE_DEBUG */
|
||||
|
||||
attach_out:
|
||||
if (error) {
|
||||
wolfkdriv_unregister(softc);
|
||||
error = ENXIO;
|
||||
}
|
||||
|
||||
return (error);
|
||||
}
|
||||
|
||||
static int wolfkdriv_detach(device_t dev)
|
||||
{
|
||||
struct wolfkdriv_softc * softc = NULL;
|
||||
int ret = 0;
|
||||
|
||||
ret = wolfkmod_cleanup();
|
||||
|
||||
if (ret == 0) {
|
||||
/* unregister wolfcrypt algs */
|
||||
softc = device_get_softc(dev);
|
||||
wolfkdriv_unregister(softc);
|
||||
}
|
||||
|
||||
#if defined(WOLFSSL_BSDKM_VERBOSE_DEBUG)
|
||||
device_printf(dev, "info: exiting detach\n");
|
||||
#endif /* WOLFSSL_BSDKM_VERBOSE_DEBUG */
|
||||
|
||||
return (0);
|
||||
}
|
||||
|
||||
static int wolfkdriv_probesession(device_t dev,
|
||||
const struct crypto_session_params *csp)
|
||||
{
|
||||
struct wolfkdriv_softc * softc = NULL;
|
||||
int error = CRYPTODEV_PROBE_ACCEL_SOFTWARE;
|
||||
|
||||
softc = device_get_softc(dev);
|
||||
|
||||
switch (csp->csp_mode) {
|
||||
case CSP_MODE_CIPHER:
|
||||
switch (csp->csp_cipher_alg) {
|
||||
case CRYPTO_AES_CBC:
|
||||
break;
|
||||
default:
|
||||
error = EINVAL;
|
||||
break;
|
||||
}
|
||||
break;
|
||||
|
||||
case CSP_MODE_AEAD:
|
||||
switch (csp->csp_cipher_alg) {
|
||||
case CRYPTO_AES_NIST_GCM_16:
|
||||
break;
|
||||
default:
|
||||
error = EINVAL;
|
||||
break;
|
||||
}
|
||||
break;
|
||||
case CSP_MODE_DIGEST:
|
||||
case CSP_MODE_ETA:
|
||||
default:
|
||||
error = EINVAL;
|
||||
break;
|
||||
}
|
||||
|
||||
(void)softc;
|
||||
(void)csp;
|
||||
|
||||
#if defined(WOLFSSL_BSDKM_VERBOSE_DEBUG)
|
||||
device_printf(dev, "info: probesession: mode=%d, cipher_alg=%d, error=%d\n",
|
||||
csp->csp_mode, csp->csp_cipher_alg, error);
|
||||
#endif /* WOLFSSL_BSDKM_VERBOSE_DEBUG */
|
||||
return (error);
|
||||
}
|
||||
|
||||
static int wolfkdriv_newsession_aes(device_t dev,
|
||||
wolfkdriv_session_t * session,
|
||||
const struct crypto_session_params *csp)
|
||||
{
|
||||
int error = 0;
|
||||
int klen = csp->csp_cipher_klen; /* key len in bytes */
|
||||
|
||||
switch (csp->csp_cipher_alg) {
|
||||
case CRYPTO_AES_NIST_GCM_16:
|
||||
session->type = CRYPTO_AES_NIST_GCM_16;
|
||||
break;
|
||||
case CRYPTO_AES_CBC:
|
||||
session->type = CRYPTO_AES_CBC;
|
||||
break;
|
||||
default:
|
||||
return (EOPNOTSUPP);
|
||||
}
|
||||
|
||||
if (klen != 16 && klen != 24 && klen != 32) {
|
||||
device_printf(dev, "info: newsession_cipher: invalid klen: %d\n", klen);
|
||||
return (EINVAL);
|
||||
}
|
||||
|
||||
session->klen = klen;
|
||||
session->ivlen = csp->csp_ivlen;
|
||||
|
||||
/* encrypt */
|
||||
error = wc_AesInit(&session->aes_ctx.aes_encrypt, NULL, INVALID_DEVID);
|
||||
if (error) {
|
||||
device_printf(dev, "error: newsession_cipher: aes init: %d\n", error);
|
||||
goto newsession_cipher_out;
|
||||
}
|
||||
|
||||
if (session->type == CRYPTO_AES_CBC) {
|
||||
/* Need a separate decrypt structure for aes-cbc. */
|
||||
error = wc_AesInit(&session->aes_ctx.aes_decrypt, NULL, INVALID_DEVID);
|
||||
if (error) {
|
||||
device_printf(dev, "error: newsession_cipher: aes init: %d\n",
|
||||
error);
|
||||
goto newsession_cipher_out;
|
||||
}
|
||||
}
|
||||
|
||||
newsession_cipher_out:
|
||||
|
||||
if (error != 0) {
|
||||
wolfkdriv_aes_ctx_clear(&session->aes_ctx);
|
||||
return (EINVAL);
|
||||
}
|
||||
|
||||
return (error);
|
||||
}
|
||||
|
||||
static int wolfkdriv_newsession(device_t dev, crypto_session_t cses,
|
||||
const struct crypto_session_params *csp)
|
||||
{
|
||||
wolfkdriv_session_t * session = NULL;
|
||||
int error = 0;
|
||||
|
||||
/* get the wolfkdriv_session_t context */
|
||||
session = crypto_get_driver_session(cses);
|
||||
|
||||
switch (csp->csp_mode) {
|
||||
case CSP_MODE_DIGEST:
|
||||
case CSP_MODE_ETA:
|
||||
device_printf(dev, "info: not supported: %d\n", csp->csp_mode);
|
||||
error = EOPNOTSUPP;
|
||||
break;
|
||||
case CSP_MODE_CIPHER:
|
||||
case CSP_MODE_AEAD:
|
||||
error = wolfkdriv_newsession_aes(dev, session, csp);
|
||||
break;
|
||||
default:
|
||||
__assert_unreachable();
|
||||
}
|
||||
|
||||
#if defined(WOLFSSL_BSDKM_VERBOSE_DEBUG)
|
||||
device_printf(dev, "info: newsession: mode=%d, cipher_alg=%d, error=%d\n",
|
||||
csp->csp_mode, csp->csp_cipher_alg, error);
|
||||
#endif /* WOLFSSL_BSDKM_VERBOSE_DEBUG */
|
||||
|
||||
return (error);
|
||||
}
|
||||
|
||||
static void
|
||||
wolfkdriv_freesession(device_t dev, crypto_session_t cses)
|
||||
{
|
||||
wolfkdriv_session_t * session = NULL;
|
||||
(void)dev;
|
||||
|
||||
/* get the wolfkdriv_session_t context */
|
||||
session = crypto_get_driver_session(cses);
|
||||
|
||||
/* clean it up */
|
||||
wolfkdriv_aes_ctx_clear(&session->aes_ctx);
|
||||
|
||||
#if defined(WOLFSSL_BSDKM_VERBOSE_DEBUG)
|
||||
device_printf(dev, "info: exiting freesession\n");
|
||||
#endif /* WOLFSSL_BSDKM_VERBOSE_DEBUG */
|
||||
return;
|
||||
}
|
||||
|
||||
static int wolfkdriv_cbc_work(device_t dev, wolfkdriv_session_t * session,
|
||||
struct cryptop * crp,
|
||||
const struct crypto_session_params * csp)
|
||||
{
|
||||
struct crypto_buffer_cursor cc_in;
|
||||
struct crypto_buffer_cursor cc_out;
|
||||
const unsigned char * in_block = NULL;
|
||||
const unsigned char * in_seg = NULL;
|
||||
unsigned char * out_block = NULL;
|
||||
unsigned char * out_seg = NULL;
|
||||
Aes aes;
|
||||
uint8_t iv[WC_AES_BLOCK_SIZE];
|
||||
uint8_t block[EALG_MAX_BLOCK_LEN];
|
||||
size_t data_len = 0;
|
||||
size_t seg_len = 0;
|
||||
size_t in_len = 0;
|
||||
size_t out_len = 0;
|
||||
int error = 0;
|
||||
int is_encrypt = 0;
|
||||
int type = AES_ENCRYPTION;
|
||||
|
||||
if (csp->csp_cipher_alg != CRYPTO_AES_CBC) {
|
||||
error = EINVAL;
|
||||
goto cbc_work_out;
|
||||
}
|
||||
|
||||
data_len = crp->crp_payload_length;
|
||||
if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
|
||||
is_encrypt = 1;
|
||||
type = AES_ENCRYPTION;
|
||||
memcpy(&aes, &session->aes_ctx.aes_encrypt, sizeof(aes));
|
||||
}
|
||||
else {
|
||||
is_encrypt = 0;
|
||||
type = AES_DECRYPTION;
|
||||
memcpy(&aes, &session->aes_ctx.aes_decrypt, sizeof(aes));
|
||||
}
|
||||
|
||||
/* must be multiple of block size */
|
||||
if (data_len % WC_AES_BLOCK_SIZE) {
|
||||
error = EINVAL;
|
||||
goto cbc_work_out;
|
||||
}
|
||||
|
||||
crypto_read_iv(crp, iv);
|
||||
error = wc_AesSetKey(&aes, csp->csp_cipher_key,
|
||||
csp->csp_cipher_klen, iv, type);
|
||||
if (error) {
|
||||
device_printf(dev, "error: wc_AesSetKey: %d\n", error);
|
||||
goto cbc_work_out;
|
||||
}
|
||||
|
||||
/* set up the crypto buffers */
|
||||
crypto_cursor_init(&cc_in, &crp->crp_buf);
|
||||
crypto_cursor_advance(&cc_in, crp->crp_payload_start);
|
||||
|
||||
in_seg = crypto_cursor_segment(&cc_in, &in_len);
|
||||
|
||||
/* handle if the user supplied a separate out buffer. */
|
||||
if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) {
|
||||
crypto_cursor_init(&cc_out, &crp->crp_obuf);
|
||||
crypto_cursor_advance(&cc_out, crp->crp_payload_output_start);
|
||||
}
|
||||
else {
|
||||
cc_out = cc_in;
|
||||
}
|
||||
|
||||
out_seg = crypto_cursor_segment(&cc_out, &out_len);
|
||||
|
||||
while (data_len) {
|
||||
/* set up input buffers */
|
||||
if (in_len < WC_AES_BLOCK_SIZE) {
|
||||
/* less than a block in segment */
|
||||
crypto_cursor_copydata(&cc_in, WC_AES_BLOCK_SIZE, block);
|
||||
in_block = block;
|
||||
in_len = WC_AES_BLOCK_SIZE;
|
||||
}
|
||||
else {
|
||||
in_block = in_seg;
|
||||
}
|
||||
|
||||
/* set up output buffers */
|
||||
if (out_len < WC_AES_BLOCK_SIZE) {
|
||||
out_block = block;
|
||||
out_len = WC_AES_BLOCK_SIZE;
|
||||
}
|
||||
else {
|
||||
out_block = out_seg;
|
||||
}
|
||||
|
||||
/* choose which of data_len, in_len, out_len, is shorter.
|
||||
* round down to multiple of aes block size. */
|
||||
seg_len = rounddown(MIN(data_len, MIN(in_len, out_len)),
|
||||
WC_AES_BLOCK_SIZE);
|
||||
|
||||
if (is_encrypt) {
|
||||
error = wc_AesCbcEncrypt(&aes, out_block, in_block, seg_len);
|
||||
if (error) {
|
||||
device_printf(dev, "error: wc_AesCbcEncrypt: %d\n", error);
|
||||
goto cbc_work_out;
|
||||
}
|
||||
}
|
||||
else {
|
||||
error = wc_AesCbcDecrypt(&aes, out_block, in_block, seg_len);
|
||||
if (error) {
|
||||
device_printf(dev, "error: wc_AesCbcEncrypt: %d\n", error);
|
||||
goto cbc_work_out;
|
||||
}
|
||||
}
|
||||
|
||||
if (out_block == block) {
|
||||
/* we used the block as local output buffer. copy to cc_out,
|
||||
* and grab the next out cursor segment. */
|
||||
crypto_cursor_copyback(&cc_out, WC_AES_BLOCK_SIZE, block);
|
||||
out_seg = crypto_cursor_segment(&cc_out, &out_len);
|
||||
} else {
|
||||
/* we worked directly in cc_out. advance the cursor. */
|
||||
crypto_cursor_advance(&cc_out, seg_len);
|
||||
out_seg += seg_len;
|
||||
out_len -= seg_len;
|
||||
}
|
||||
|
||||
if (in_block == block) {
|
||||
/* grab a new in cursor segment. */
|
||||
in_seg = crypto_cursor_segment(&cc_in, &in_len);
|
||||
} else {
|
||||
/* else advance existing in cursor. */
|
||||
crypto_cursor_advance(&cc_in, seg_len);
|
||||
in_seg += seg_len;
|
||||
in_len -= seg_len;
|
||||
}
|
||||
|
||||
data_len -= seg_len;
|
||||
}
|
||||
|
||||
cbc_work_out:
|
||||
/* cleanup. */
|
||||
wc_ForceZero(iv, sizeof(iv));
|
||||
wc_ForceZero(block, sizeof(block));
|
||||
|
||||
#if defined(WOLFSSL_BSDKM_VERBOSE_DEBUG)
|
||||
device_printf(dev, "info: cbc_work: mode=%d, cipher_alg=%d, "
|
||||
"payload_length=%d, error=%d\n",
|
||||
csp->csp_mode, csp->csp_cipher_alg, crp->crp_payload_length,
|
||||
error);
|
||||
#endif /* WOLFSSL_BSDKM_VERBOSE_DEBUG */
|
||||
|
||||
return (error);
|
||||
}
|
||||
|
||||
static int wolfkdriv_gcm_work(device_t dev, wolfkdriv_session_t * session,
|
||||
struct cryptop * crp,
|
||||
const struct crypto_session_params * csp)
|
||||
{
|
||||
struct crypto_buffer_cursor cc_in;
|
||||
struct crypto_buffer_cursor cc_out;
|
||||
const unsigned char * in_seg = NULL;
|
||||
unsigned char * out_seg = NULL;
|
||||
Aes aes;
|
||||
uint8_t iv[WC_AES_BLOCK_SIZE];
|
||||
uint8_t auth_tag[WC_AES_BLOCK_SIZE];
|
||||
size_t data_len = 0;
|
||||
size_t seg_len = 0;
|
||||
size_t in_len = 0;
|
||||
size_t out_len = 0;
|
||||
int error = 0;
|
||||
int is_encrypt = 0;
|
||||
|
||||
memcpy(&aes, &session->aes_ctx.aes_encrypt, sizeof(aes));
|
||||
|
||||
if (csp->csp_cipher_alg != CRYPTO_AES_NIST_GCM_16) {
|
||||
error = EINVAL;
|
||||
goto gcm_work_out;
|
||||
}
|
||||
|
||||
data_len = crp->crp_payload_length;
|
||||
if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
|
||||
is_encrypt = 1;
|
||||
}
|
||||
else {
|
||||
is_encrypt = 0;
|
||||
}
|
||||
|
||||
error = wc_AesGcmSetKey(&aes, csp->csp_cipher_key,
|
||||
csp->csp_cipher_klen);
|
||||
if (error) {
|
||||
device_printf(dev, "error: wc_AesGcmSetKey: %d\n", error);
|
||||
goto gcm_work_out;
|
||||
}
|
||||
|
||||
crypto_read_iv(crp, iv);
|
||||
error = wc_AesGcmInit(&aes, NULL /* key */, 0 /* keylen */,
|
||||
iv, csp->csp_ivlen);
|
||||
if (error) {
|
||||
device_printf(dev, "error: wc_AesGcmInit: %d\n", error);
|
||||
goto gcm_work_out;
|
||||
}
|
||||
|
||||
/* process aad first */
|
||||
if (crp->crp_aad != NULL) {
|
||||
/* they passed aad in separate buffer. */
|
||||
if (is_encrypt) {
|
||||
error = wc_AesGcmEncryptUpdate(&aes, NULL, NULL, 0,
|
||||
crp->crp_aad, crp->crp_aad_length);
|
||||
}
|
||||
else {
|
||||
error = wc_AesGcmDecryptUpdate(&aes, NULL, NULL, 0,
|
||||
crp->crp_aad, crp->crp_aad_length);
|
||||
}
|
||||
|
||||
if (error) {
|
||||
error = EINVAL;
|
||||
}
|
||||
}
|
||||
else {
|
||||
/* we need to pull aad out of crp->crp_buf from crp_aad_start. */
|
||||
size_t aad_len = 0;
|
||||
|
||||
crypto_cursor_init(&cc_in, &crp->crp_buf);
|
||||
crypto_cursor_advance(&cc_in, crp->crp_aad_start);
|
||||
|
||||
for (aad_len = crp->crp_aad_length; aad_len > 0; aad_len -= seg_len) {
|
||||
in_seg = crypto_cursor_segment(&cc_in, &in_len);
|
||||
seg_len = MIN(aad_len, in_len);
|
||||
|
||||
if (is_encrypt) {
|
||||
error = wc_AesGcmEncryptUpdate(&aes, NULL, NULL, 0,
|
||||
in_seg, seg_len);
|
||||
}
|
||||
else {
|
||||
error = wc_AesGcmDecryptUpdate(&aes, NULL, NULL, 0,
|
||||
in_seg, seg_len);
|
||||
}
|
||||
|
||||
if (error) {
|
||||
error = EINVAL;
|
||||
goto gcm_work_out;
|
||||
}
|
||||
|
||||
crypto_cursor_advance(&cc_in, seg_len);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* process cipher/plaintext next
|
||||
*/
|
||||
|
||||
/* set up the crypto buffers */
|
||||
crypto_cursor_init(&cc_in, &crp->crp_buf);
|
||||
crypto_cursor_advance(&cc_in, crp->crp_payload_start);
|
||||
|
||||
in_seg = crypto_cursor_segment(&cc_in, &in_len);
|
||||
|
||||
/* handle if the user supplied a separate out buffer. */
|
||||
if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) {
|
||||
crypto_cursor_init(&cc_out, &crp->crp_obuf);
|
||||
crypto_cursor_advance(&cc_out, crp->crp_payload_output_start);
|
||||
}
|
||||
else {
|
||||
cc_out = cc_in;
|
||||
}
|
||||
|
||||
out_seg = crypto_cursor_segment(&cc_out, &out_len);
|
||||
|
||||
while (data_len) {
|
||||
/* process through the available segments. */
|
||||
in_seg = crypto_cursor_segment(&cc_in, &in_len);
|
||||
out_seg = crypto_cursor_segment(&cc_out, &out_len);
|
||||
seg_len = MIN(data_len, MIN(in_len, out_len));
|
||||
|
||||
if (is_encrypt) {
|
||||
error = wc_AesGcmEncryptUpdate(&aes, out_seg, in_seg, seg_len,
|
||||
NULL, 0);
|
||||
if (error) {
|
||||
device_printf(dev, "error: wc_AesGcmEncrypt: %d\n", error);
|
||||
goto gcm_work_out;
|
||||
}
|
||||
}
|
||||
else {
|
||||
error = wc_AesGcmDecryptUpdate(&aes, out_seg, in_seg, seg_len,
|
||||
NULL, 0);
|
||||
if (error) {
|
||||
device_printf(dev, "error: wc_AesGcmDecrypt: %d\n", error);
|
||||
goto gcm_work_out;
|
||||
}
|
||||
}
|
||||
|
||||
/* advance the cursors by amount processed */
|
||||
crypto_cursor_advance(&cc_in, seg_len);
|
||||
crypto_cursor_advance(&cc_out, seg_len);
|
||||
|
||||
data_len -= seg_len;
|
||||
}
|
||||
|
||||
/* process auth tag finally */
|
||||
if (is_encrypt) {
|
||||
error = wc_AesGcmEncryptFinal(&aes, auth_tag, WC_AES_BLOCK_SIZE);
|
||||
if (error == 0) {
|
||||
crypto_copyback(crp, crp->crp_digest_start, WC_AES_BLOCK_SIZE,
|
||||
auth_tag);
|
||||
}
|
||||
}
|
||||
else {
|
||||
crypto_copydata(crp, crp->crp_digest_start, WC_AES_BLOCK_SIZE,
|
||||
auth_tag);
|
||||
error = wc_AesGcmDecryptFinal(&aes, auth_tag, WC_AES_BLOCK_SIZE);
|
||||
if (error) {
|
||||
error = EBADMSG;
|
||||
}
|
||||
}
|
||||
|
||||
gcm_work_out:
|
||||
/* cleanup. */
|
||||
wc_ForceZero(iv, sizeof(iv));
|
||||
wc_ForceZero(auth_tag, sizeof(auth_tag));
|
||||
|
||||
#if defined(WOLFSSL_BSDKM_VERBOSE_DEBUG)
|
||||
device_printf(dev, "info: gcm_work: mode=%d, cipher_alg=%d, "
|
||||
"payload_length=%d, error=%d\n",
|
||||
csp->csp_mode, csp->csp_cipher_alg, crp->crp_payload_length,
|
||||
error);
|
||||
#endif /* WOLFSSL_BSDKM_VERBOSE_DEBUG */
|
||||
|
||||
return (error);
|
||||
}
|
||||
|
||||
static int wolfkdriv_process(device_t dev, struct cryptop * crp, int hint)
|
||||
{
|
||||
const struct crypto_session_params * csp = NULL;
|
||||
wolfkdriv_session_t * session = NULL;
|
||||
int error = 0;
|
||||
(void)hint;
|
||||
|
||||
session = crypto_get_driver_session(crp->crp_session);
|
||||
csp = crypto_get_params(crp->crp_session);
|
||||
|
||||
switch (csp->csp_mode) {
|
||||
case CSP_MODE_CIPHER:
|
||||
error = wolfkdriv_cbc_work(dev, session, crp, csp);
|
||||
break;
|
||||
case CSP_MODE_DIGEST:
|
||||
case CSP_MODE_ETA:
|
||||
error = EINVAL;
|
||||
break;
|
||||
case CSP_MODE_AEAD:
|
||||
error = wolfkdriv_gcm_work(dev, session, crp, csp);
|
||||
break;
|
||||
default:
|
||||
__assert_unreachable();
|
||||
}
|
||||
|
||||
crp->crp_etype = error;
|
||||
crypto_done(crp);
|
||||
|
||||
#if defined(WOLFSSL_BSDKM_VERBOSE_DEBUG)
|
||||
device_printf(dev, "info: process: mode=%d, cipher_alg=%d, error=%d\n",
|
||||
csp->csp_mode, csp->csp_cipher_alg, error);
|
||||
#endif /* WOLFSSL_BSDKM_VERBOSE_DEBUG */
|
||||
|
||||
return (error);
|
||||
}
|
||||
|
||||
/*
|
||||
* wolfkmod as a crypto device driver.
|
||||
*/
|
||||
static device_method_t wolfkdriv_methods[] = {
|
||||
/* device interface methods: called during device setup, etc. */
|
||||
DEVMETHOD(device_identify, wolfkdriv_identify),
|
||||
DEVMETHOD(device_probe, wolfkdriv_probe),
|
||||
DEVMETHOD(device_attach, wolfkdriv_attach),
|
||||
DEVMETHOD(device_detach, wolfkdriv_detach),
|
||||
|
||||
/* crypto device session methods: called during crypto session setup,
|
||||
* work, etc. */
|
||||
DEVMETHOD(cryptodev_probesession, wolfkdriv_probesession),
|
||||
DEVMETHOD(cryptodev_newsession, wolfkdriv_newsession),
|
||||
DEVMETHOD(cryptodev_freesession, wolfkdriv_freesession),
|
||||
DEVMETHOD(cryptodev_process, wolfkdriv_process),
|
||||
|
||||
DEVMETHOD_END
|
||||
};
|
||||
|
||||
static driver_t wolfkdriv_driver = {
|
||||
.name = "libwolf",
|
||||
.methods = wolfkdriv_methods,
|
||||
.size = sizeof(struct wolfkdriv_softc),
|
||||
};
|
||||
|
||||
/* on x86, software-only drivers usually attach to nexus bus. */
|
||||
DRIVER_MODULE(libwolfssl, nexus, wolfkdriv_driver, NULL, NULL);
|
||||
#endif /* BSDKM_CRYPTO_REGISTER */
|
||||
|
||||
#if !defined(BSDKM_CRYPTO_REGISTER)
|
||||
/*
|
||||
* wolfkmod as a pure kernel module.
|
||||
*/
|
||||
static moduledata_t libwolfmod = {
|
||||
#ifdef HAVE_FIPS
|
||||
"libwolfssl_fips", /* module name */
|
||||
@@ -305,6 +1066,8 @@ static moduledata_t libwolfmod = {
|
||||
NULL /* extra data, unused */
|
||||
};
|
||||
|
||||
MODULE_VERSION(libwolfssl, 1);
|
||||
DECLARE_MODULE(libwolfssl, libwolfmod, SI_SUB_DRIVERS, SI_ORDER_MIDDLE);
|
||||
#endif /* !BSDKM_CRYPTO_REGISTER */
|
||||
|
||||
MODULE_VERSION(libwolfssl, 1);
|
||||
#endif /* WOLFSSL_BSDKM */
|
||||
|
||||
@@ -0,0 +1,347 @@
|
||||
#if !defined(WC_SKIP_INCLUDED_C_FILES) && defined(BSDKM_CRYPTO_REGISTER)
|
||||
#include <wolfssl/wolfcrypt/aes.h>
|
||||
|
||||
/*
|
||||
* the cryptodev framework always calls a callback, even when CRYPTOCAP_F_SYNC.
|
||||
*/
|
||||
static int
|
||||
wolfkdriv_test_crp_callback(struct cryptop * crp)
|
||||
{
|
||||
(void)crp;
|
||||
return (0);
|
||||
}
|
||||
|
||||
/* Test aes-cbc with a buffer larger than aes block size.
|
||||
* Verify direct wolfcrypt API and opencrypto framework return
|
||||
* same result. */
|
||||
static int wolfkdriv_test_aes_cbc_big(device_t dev, int crid)
|
||||
{
|
||||
crypto_session_t session = NULL;
|
||||
struct crypto_session_params csp;
|
||||
struct cryptop * crp = NULL;
|
||||
Aes * aes_encrypt = NULL;
|
||||
int error = 0;
|
||||
byte msg[] = {
|
||||
0x6e,0x6f,0x77,0x20,0x69,0x73,0x20,0x74,
|
||||
0x68,0x65,0x20,0x74,0x69,0x6d,0x65,0x20,
|
||||
0x6e,0x6f,0x77,0x20,0x69,0x73,0x20,0x74,
|
||||
0x68,0x65,0x20,0x74,0x69,0x6d,0x65,0x20,
|
||||
0x6e,0x6f,0x77,0x20,0x69,0x73,0x20,0x74,
|
||||
0x68,0x65,0x20,0x74,0x69,0x6d,0x65,0x20
|
||||
};
|
||||
byte work1[WC_AES_BLOCK_SIZE * 3]; /* wolfcrypt buffer */
|
||||
byte work2[WC_AES_BLOCK_SIZE * 3]; /* opencrypto buffer */
|
||||
/* padded to 16-bytes */
|
||||
const byte key[] = "0123456789abcdef ";
|
||||
/* padded to 16-bytes */
|
||||
const byte iv[] = "1234567890abcdef ";
|
||||
|
||||
memset(&csp, 0, sizeof(csp));
|
||||
memcpy(work1, msg, sizeof(msg)); /* wolfcrypt work buffer */
|
||||
memcpy(work2, msg, sizeof(msg)); /* opencrypto work buffer */
|
||||
|
||||
/* wolfcrypt encrypt */
|
||||
aes_encrypt = (Aes *)XMALLOC(sizeof(Aes), NULL, DYNAMIC_TYPE_AES);
|
||||
if (aes_encrypt == NULL) {
|
||||
error = ENOMEM;
|
||||
device_printf(dev, "error: malloc failed\n");
|
||||
goto test_aes_cbc_big_out;
|
||||
}
|
||||
|
||||
error = wc_AesInit(aes_encrypt, NULL, INVALID_DEVID);
|
||||
if (error) {
|
||||
device_printf(dev, "error: newsession_cipher: aes init: %d\n", error);
|
||||
goto test_aes_cbc_big_out;
|
||||
}
|
||||
|
||||
error = wc_AesSetKey(aes_encrypt, key, 16, iv, AES_ENCRYPTION);
|
||||
if (error) {
|
||||
device_printf(dev, "error: wc_AesSetKey: %d\n", error);
|
||||
goto test_aes_cbc_big_out;
|
||||
}
|
||||
|
||||
error = wc_AesCbcEncrypt(aes_encrypt, work1, work1, sizeof(work1));
|
||||
if (error) {
|
||||
device_printf(dev, "error: wc_AesCbcEncrypt: %d\n", error);
|
||||
goto test_aes_cbc_big_out;
|
||||
}
|
||||
|
||||
/* opencrypto encrypt */
|
||||
csp.csp_mode = CSP_MODE_CIPHER;
|
||||
csp.csp_cipher_alg = CRYPTO_AES_CBC;
|
||||
csp.csp_ivlen = WC_AES_BLOCK_SIZE;
|
||||
csp.csp_cipher_key = key;
|
||||
csp.csp_cipher_klen = WC_AES_BLOCK_SIZE;
|
||||
error = crypto_newsession(&session, &csp, crid);
|
||||
if (error || session == NULL) {
|
||||
goto test_aes_cbc_big_out;
|
||||
}
|
||||
|
||||
crp = crypto_getreq(session, M_WAITOK);
|
||||
if (crp == NULL) {
|
||||
device_printf(dev, "error: test_aes: crypto_getreq failed\n");
|
||||
goto test_aes_cbc_big_out;
|
||||
}
|
||||
|
||||
crp->crp_callback = wolfkdriv_test_crp_callback;
|
||||
crp->crp_op = CRYPTO_OP_ENCRYPT;
|
||||
crp->crp_flags = CRYPTO_F_IV_SEPARATE;
|
||||
|
||||
memcpy(crp->crp_iv, iv, WC_AES_BLOCK_SIZE);
|
||||
|
||||
crypto_use_buf(crp, work2, sizeof(work2));
|
||||
crp->crp_payload_start = 0;
|
||||
crp->crp_payload_length = sizeof(work2);
|
||||
|
||||
error = crypto_dispatch(crp);
|
||||
if (error) {
|
||||
goto test_aes_cbc_big_out;
|
||||
}
|
||||
|
||||
error = XMEMCMP(work1, work2, sizeof(work2));
|
||||
if (error) {
|
||||
device_printf(dev, "error: test_aes: enc vectors diff: %d\n", error);
|
||||
goto test_aes_cbc_big_out;
|
||||
}
|
||||
|
||||
/* opencrypto decrypt */
|
||||
crp->crp_op = CRYPTO_OP_DECRYPT;
|
||||
|
||||
error = crypto_dispatch(crp);
|
||||
if (error) {
|
||||
goto test_aes_cbc_big_out;
|
||||
}
|
||||
|
||||
error = XMEMCMP(work2, msg, sizeof(msg));
|
||||
if (error) {
|
||||
device_printf(dev, "error: test_aes: dec vectors diff: %d\n", error);
|
||||
goto test_aes_cbc_big_out;
|
||||
}
|
||||
|
||||
test_aes_cbc_big_out:
|
||||
#if defined(WOLFSSL_BSDKM_VERBOSE_DEBUG)
|
||||
device_printf(dev, "info: test_aes_cbc_big: error=%d, session=%p, crp=%p\n",
|
||||
error, (void *)session, (void*)crp);
|
||||
#endif /* WOLFSSL_BSDKM_VERBOSE_DEBUG */
|
||||
|
||||
if (crp != NULL) {
|
||||
crypto_freereq(crp);
|
||||
crp = NULL;
|
||||
}
|
||||
|
||||
if (session != NULL) {
|
||||
crypto_freesession(session);
|
||||
session = NULL;
|
||||
}
|
||||
|
||||
if (aes_encrypt != NULL) {
|
||||
wc_AesFree(aes_encrypt);
|
||||
XFREE(aes_encrypt, NULL, DYNAMIC_TYPE_AES);
|
||||
aes_encrypt = NULL;
|
||||
}
|
||||
|
||||
return (error);
|
||||
}
|
||||
|
||||
/* Test aes-gcm encrypt and decrypt a small buffer with opencrypto
|
||||
* framework and wolfcrypt.
|
||||
*/
|
||||
static int wolfkdriv_test_aes_gcm(device_t dev, int crid)
|
||||
{
|
||||
crypto_session_t session = NULL;
|
||||
struct crypto_session_params csp;
|
||||
struct cryptop * crp = NULL;
|
||||
Aes * enc = NULL;
|
||||
int error = 0;
|
||||
|
||||
WOLFSSL_SMALL_STACK_STATIC const byte p[] =
|
||||
{
|
||||
0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5,
|
||||
0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a,
|
||||
0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda,
|
||||
0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72,
|
||||
0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53,
|
||||
0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25,
|
||||
0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57,
|
||||
0xba, 0x63, 0x7b, 0x39
|
||||
};
|
||||
|
||||
WOLFSSL_SMALL_STACK_STATIC const byte c1[] =
|
||||
{
|
||||
0x52, 0x2d, 0xc1, 0xf0, 0x99, 0x56, 0x7d, 0x07,
|
||||
0xf4, 0x7f, 0x37, 0xa3, 0x2a, 0x84, 0x42, 0x7d,
|
||||
0x64, 0x3a, 0x8c, 0xdc, 0xbf, 0xe5, 0xc0, 0xc9,
|
||||
0x75, 0x98, 0xa2, 0xbd, 0x25, 0x55, 0xd1, 0xaa,
|
||||
0x8c, 0xb0, 0x8e, 0x48, 0x59, 0x0d, 0xbb, 0x3d,
|
||||
0xa7, 0xb0, 0x8b, 0x10, 0x56, 0x82, 0x88, 0x38,
|
||||
0xc5, 0xf6, 0x1e, 0x63, 0x93, 0xba, 0x7a, 0x0a,
|
||||
0xbc, 0xc9, 0xf6, 0x62
|
||||
};
|
||||
|
||||
WOLFSSL_SMALL_STACK_STATIC byte a[] =
|
||||
{
|
||||
0xfe, 0xed, 0xfa, 0xce, 0xde, 0xad, 0xbe, 0xef,
|
||||
0xfe, 0xed, 0xfa, 0xce, 0xde, 0xad, 0xbe, 0xef,
|
||||
0xab, 0xad, 0xda, 0xd2
|
||||
};
|
||||
|
||||
WOLFSSL_SMALL_STACK_STATIC const byte k1[] =
|
||||
{
|
||||
0xfe, 0xff, 0xe9, 0x92, 0x86, 0x65, 0x73, 0x1c,
|
||||
0x6d, 0x6a, 0x8f, 0x94, 0x67, 0x30, 0x83, 0x08,
|
||||
0xfe, 0xff, 0xe9, 0x92, 0x86, 0x65, 0x73, 0x1c,
|
||||
0x6d, 0x6a, 0x8f, 0x94, 0x67, 0x30, 0x83, 0x08
|
||||
};
|
||||
|
||||
WOLFSSL_SMALL_STACK_STATIC const byte iv1[] =
|
||||
{
|
||||
0xca, 0xfe, 0xba, 0xbe, 0xfa, 0xce, 0xdb, 0xad,
|
||||
0xde, 0xca, 0xf8, 0x88
|
||||
};
|
||||
|
||||
WOLFSSL_SMALL_STACK_STATIC const byte t1[] =
|
||||
{
|
||||
0x76, 0xfc, 0x6e, 0xce, 0x0f, 0x4e, 0x17, 0x68,
|
||||
0xcd, 0xdf, 0x88, 0x53, 0xbb, 0x2d, 0x55, 0x1b
|
||||
};
|
||||
|
||||
byte resultT[sizeof(t1) + WC_AES_BLOCK_SIZE];
|
||||
byte resultC[sizeof(p) + WC_AES_BLOCK_SIZE];
|
||||
byte resultC2[sizeof(p) + WC_AES_BLOCK_SIZE];
|
||||
|
||||
XMEMSET(resultT, 0, sizeof(resultT));
|
||||
XMEMSET(resultC, 0, sizeof(resultC));
|
||||
|
||||
XMEMSET(resultC2, 0, sizeof(resultC));
|
||||
XMEMCPY(resultC2, p, sizeof(p));
|
||||
|
||||
/* wolfcrypt encrypt */
|
||||
enc = (Aes *)XMALLOC(sizeof(Aes), NULL, DYNAMIC_TYPE_AES);
|
||||
if (enc == NULL) {
|
||||
error = ENOMEM;
|
||||
device_printf(dev, "error: malloc failed\n");
|
||||
goto test_aes_gcm_out;
|
||||
}
|
||||
|
||||
error = wc_AesGcmEncryptInit(enc, k1, sizeof(k1), iv1, sizeof(iv1));
|
||||
if (error) { goto test_aes_gcm_out; }
|
||||
|
||||
error = wc_AesGcmEncryptUpdate(enc, resultC, p, sizeof(p), a, sizeof(a));
|
||||
if (error) { goto test_aes_gcm_out; }
|
||||
|
||||
error = wc_AesGcmEncryptFinal(enc, resultT, sizeof(t1));
|
||||
if (error) { goto test_aes_gcm_out; }
|
||||
|
||||
error = XMEMCMP(resultC, c1, sizeof(c1));
|
||||
if (error) { goto test_aes_gcm_out; }
|
||||
|
||||
error = XMEMCMP(resultT, t1, sizeof(t1));
|
||||
if (error) { goto test_aes_gcm_out; }
|
||||
|
||||
/*
|
||||
* opencrypto encrypt
|
||||
* */
|
||||
|
||||
/* set crypto session params */
|
||||
memset(&csp, 0, sizeof(csp));
|
||||
csp.csp_flags |= CSP_F_SEPARATE_AAD;
|
||||
csp.csp_mode = CSP_MODE_AEAD;
|
||||
csp.csp_cipher_alg = CRYPTO_AES_NIST_GCM_16;
|
||||
csp.csp_ivlen = sizeof(iv1);
|
||||
csp.csp_cipher_key = k1;
|
||||
csp.csp_cipher_klen = sizeof(k1);
|
||||
|
||||
/* get crypto session handle */
|
||||
error = crypto_newsession(&session, &csp, crid);
|
||||
if (error || session == NULL) {
|
||||
device_printf(dev, "error: test_aes: crypto_newsession: %d, %p\n",
|
||||
error, (void *)session);
|
||||
goto test_aes_gcm_out;
|
||||
}
|
||||
|
||||
/* get a crypto op handle */
|
||||
crp = crypto_getreq(session, M_WAITOK);
|
||||
if (crp == NULL) {
|
||||
device_printf(dev, "error: test_aes: crypto_getreq failed\n");
|
||||
goto test_aes_gcm_out;
|
||||
}
|
||||
|
||||
/* configure it */
|
||||
crp->crp_callback = wolfkdriv_test_crp_callback;
|
||||
crp->crp_op = (CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST);
|
||||
crp->crp_flags = CRYPTO_F_IV_SEPARATE;
|
||||
|
||||
memcpy(crp->crp_iv, iv1, sizeof(iv1));
|
||||
|
||||
crypto_use_buf(crp, resultC2, sizeof(resultC2));
|
||||
crp->crp_payload_start = 0;
|
||||
crp->crp_payload_length = sizeof(p);
|
||||
|
||||
crp->crp_aad = a;
|
||||
crp->crp_aad_start = 0;
|
||||
crp->crp_aad_length = sizeof(a);
|
||||
crp->crp_digest_start = crp->crp_payload_start + sizeof(p);
|
||||
|
||||
error = crypto_dispatch(crp);
|
||||
if (error) {
|
||||
goto test_aes_gcm_out;
|
||||
}
|
||||
|
||||
error = XMEMCMP(resultC2, c1, sizeof(c1));
|
||||
if (error) { goto test_aes_gcm_out; }
|
||||
|
||||
error = XMEMCMP(resultC2 + sizeof(p), t1, sizeof(t1));
|
||||
if (error) { goto test_aes_gcm_out; }
|
||||
|
||||
/* opencrypto decrypt */
|
||||
crp->crp_op = (CRYPTO_OP_DECRYPT | CRYPTO_OP_VERIFY_DIGEST);
|
||||
|
||||
error = crypto_dispatch(crp);
|
||||
if (error) {
|
||||
goto test_aes_gcm_out;
|
||||
}
|
||||
|
||||
error = XMEMCMP(resultC2, p, sizeof(p));
|
||||
if (error) { goto test_aes_gcm_out; }
|
||||
|
||||
test_aes_gcm_out:
|
||||
#if defined(WOLFSSL_BSDKM_VERBOSE_DEBUG)
|
||||
device_printf(dev, "info: test_aes_gcm: error=%d, session=%p, crp=%p\n",
|
||||
error, (void *)session, (void*)crp);
|
||||
#endif /* WOLFSSL_BSDKM_VERBOSE_DEBUG */
|
||||
|
||||
if (crp != NULL) {
|
||||
crypto_freereq(crp);
|
||||
crp = NULL;
|
||||
}
|
||||
|
||||
if (session != NULL) {
|
||||
crypto_freesession(session);
|
||||
session = NULL;
|
||||
}
|
||||
|
||||
if (enc != NULL) {
|
||||
wc_AesFree(enc);
|
||||
XFREE(enc, NULL, DYNAMIC_TYPE_AES);
|
||||
enc = NULL;
|
||||
}
|
||||
|
||||
return (error);
|
||||
}
|
||||
|
||||
|
||||
static int wolfkdriv_test_aes(device_t dev, int crid)
|
||||
{
|
||||
int error = 0;
|
||||
|
||||
if (error == 0) {
|
||||
error = wolfkdriv_test_aes_cbc_big(dev, crid);
|
||||
}
|
||||
|
||||
if (error == 0) {
|
||||
error = wolfkdriv_test_aes_gcm(dev, crid);
|
||||
}
|
||||
|
||||
return (error);
|
||||
}
|
||||
#endif /* !WC_SKIP_INCLUDED_C_FILES && BSDKM_CRYPTO_REGISTER */
|
||||
@@ -0,0 +1,225 @@
|
||||
/* x86_vecreg.c -- logic to save and restore vector registers
|
||||
* on amd64 in FreeBSD kernel.
|
||||
*
|
||||
* Copyright (C) 2006-2025 wolfSSL Inc.
|
||||
*
|
||||
* This file is part of wolfSSL.
|
||||
*
|
||||
* wolfSSL is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* wolfSSL is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA
|
||||
*/
|
||||
|
||||
/* included by bsdkm/wolfkmod.c */
|
||||
#ifndef WC_SKIP_INCLUDED_C_FILES
|
||||
|
||||
#include <sys/proc.h>
|
||||
#include <sys/smp.h>
|
||||
#include <machine/fpu.h>
|
||||
#include <machine/pcb.h>
|
||||
|
||||
struct wolfkmod_fpu_state_t {
|
||||
volatile lwpid_t td_tid;
|
||||
volatile u_int nest;
|
||||
};
|
||||
|
||||
typedef struct wolfkmod_fpu_state_t wolfkmod_fpu_state_t;
|
||||
|
||||
/* fpu_states array tracks thread id and nesting level of save/restore
|
||||
* and push/pop vector registers macro calls. It is indexed by raw cpu id,
|
||||
* and only accessed after the thread calls fpu_kern_enter(), and before
|
||||
* calling fpu_kern_leave(), and only indexed by the thread's PCPU_GET(cpuid).
|
||||
*
|
||||
* after calling fpu_kern_enter():
|
||||
* - kernel fpu is enabled
|
||||
* - migration is disabled
|
||||
* - soft preempts are disabled
|
||||
* Hard irq are still possible , but hard irq are forbidden from using FPU
|
||||
* in FreeBSD kernel.
|
||||
* */
|
||||
static wolfkmod_fpu_state_t * fpu_states = NULL;
|
||||
|
||||
/* check for active td_tid with atomic before proceeding.
|
||||
* technically not necessary because fpu_kern_enter() gives thread pinning
|
||||
* to cpu, but just to be safe...
|
||||
* */
|
||||
#define wolfkmod_fpu_get_tid() \
|
||||
atomic_load_acq_int(&fpu_states[PCPU_GET(cpuid)].td_tid)
|
||||
|
||||
int wolfkmod_vecreg_init(void)
|
||||
{
|
||||
if (mp_ncpus <= 0) {
|
||||
printf("error: wolfkmod_vecreg_init: mp_ncpus = %d\n", mp_ncpus);
|
||||
return (EINVAL);
|
||||
}
|
||||
|
||||
fpu_states = malloc(mp_ncpus * sizeof(wolfkmod_fpu_state_t),
|
||||
M_WOLFSSL, M_WAITOK | M_ZERO);
|
||||
if (fpu_states == NULL) {
|
||||
printf("error: wolfkmod_vecreg_init: malloc(%lu) failed\n",
|
||||
mp_ncpus * sizeof(wolfkmod_fpu_state_t));
|
||||
return (ENOMEM);
|
||||
}
|
||||
|
||||
return (0);
|
||||
}
|
||||
|
||||
void wolfkmod_vecreg_exit(void)
|
||||
{
|
||||
int i = 0;
|
||||
|
||||
if (fpu_states == NULL) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (i = 0; i < mp_ncpus; ++i) {
|
||||
#if defined(WOLFSSL_BSDKM_FPU_DEBUG)
|
||||
printf("info: wolfkmod_vecreg_exit: fpu_states[%d] = %d, %d\n",
|
||||
i, fpu_states[i].nest, fpu_states[i].td_tid);
|
||||
#endif /* WOLFSSL_BSDKM_FPU_DEBUG */
|
||||
|
||||
if (fpu_states[i].nest != 0 || fpu_states[i].td_tid != 0) {
|
||||
/* Check for orphaned fpu state. There's nothing we can do
|
||||
* but log the event and zero the nesting level. */
|
||||
printf("error: wolfkmod_vecreg_exit: fpu_states[%d] = %d, %d\n",
|
||||
i, fpu_states[i].nest, fpu_states[i].td_tid);
|
||||
fpu_states[i].nest = 0;
|
||||
}
|
||||
}
|
||||
|
||||
free(fpu_states, M_WOLFSSL);
|
||||
fpu_states = NULL;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
/* fpu_kern_enter() and fpu_kern_leave() wrapper defines.
|
||||
* Build with WOLFSSL_BSDKM_FPU_DEBUG to see verbose FPU logging.
|
||||
*/
|
||||
#if defined(WOLFSSL_BSDKM_FPU_DEBUG)
|
||||
#define wolfkmod_print_curthread(what) \
|
||||
printf("%s: cpuid = %d, curthread: td_tid = %d, pid = %d (%s), " \
|
||||
"td_critnest = %d, kernfpu = %02x\n", \
|
||||
(what), PCPU_GET(cpuid), curthread->td_tid, \
|
||||
curthread->td_proc ? curthread->td_proc->p_pid : -1, \
|
||||
curthread->td_proc ? curthread->td_proc->p_comm : "noproc", \
|
||||
curthread->td_critnest, \
|
||||
curthread->td_pcb->pcb_flags & PCB_KERNFPU);
|
||||
|
||||
#define wolfkmod_fpu_kern_enter() \
|
||||
wolfkmod_print_curthread("fpu_kern_enter"); \
|
||||
fpu_kern_enter(curthread, NULL, FPU_KERN_NOCTX);
|
||||
|
||||
#define wolfkmod_fpu_kern_leave() \
|
||||
wolfkmod_print_curthread("fpu_kern_leave"); \
|
||||
fpu_kern_leave(curthread, NULL);
|
||||
#else
|
||||
#define wolfkmod_fpu_kern_enter() \
|
||||
fpu_kern_enter(curthread, NULL, FPU_KERN_NOCTX);
|
||||
|
||||
#define wolfkmod_fpu_kern_leave() \
|
||||
fpu_kern_leave(curthread, NULL);
|
||||
#endif /* WOLFSSL_BSDKM_FPU_DEBUG */
|
||||
|
||||
int wolfkmod_vecreg_save(int flags_unused)
|
||||
{
|
||||
(void)flags_unused;
|
||||
|
||||
#if defined(WOLFSSL_BSDKM_FPU_DEBUG)
|
||||
wolfkmod_print_curthread("wolfkmod_vecreg_save");
|
||||
#endif
|
||||
|
||||
if (is_fpu_kern_thread(0)) {
|
||||
/* kernel fpu threads are special, do nothing. They own a
|
||||
* persistent, dedicated fpu context. */
|
||||
#if defined(WOLFSSL_BSDKM_FPU_DEBUG)
|
||||
printf("info: wolfkmod_vecreg_save: is fpu kern thread\n");
|
||||
#endif
|
||||
return (0);
|
||||
}
|
||||
|
||||
if (curthread->td_pcb->pcb_flags & PCB_KERNFPU) {
|
||||
/* kern fpu is active for this thread. check td_tid and
|
||||
* increment nesting level. */
|
||||
lwpid_t td_tid = wolfkmod_fpu_get_tid();
|
||||
if (td_tid != curthread->td_tid) {
|
||||
printf("error: wolfkmod_vecreg_save: got tid = %d, expected %d\n",
|
||||
td_tid, curthread->td_tid);
|
||||
return (EINVAL);
|
||||
}
|
||||
fpu_states[PCPU_GET(cpuid)].nest++;
|
||||
}
|
||||
else {
|
||||
/* kern fpu not active for this thread, call fpu_kern_enter().
|
||||
* after calling fpu_kern_enter():
|
||||
* - kernel fpu is enabled
|
||||
* - migration is disabled
|
||||
* - soft preempts are disabled */
|
||||
lwpid_t td_tid = 0;
|
||||
wolfkmod_fpu_kern_enter();
|
||||
td_tid = wolfkmod_fpu_get_tid();
|
||||
|
||||
if (fpu_states[PCPU_GET(cpuid)].nest != 0 || td_tid != 0) {
|
||||
printf("error: wolfkmod_fpu_kern_enter() with nest: %d, %d\n",
|
||||
fpu_states[PCPU_GET(cpuid)].nest, td_tid);
|
||||
return (EINVAL);
|
||||
}
|
||||
|
||||
/* increment nest and save td_tid. */
|
||||
fpu_states[PCPU_GET(cpuid)].nest++;
|
||||
fpu_states[PCPU_GET(cpuid)].td_tid = curthread->td_tid;
|
||||
}
|
||||
|
||||
return (0);
|
||||
}
|
||||
|
||||
void wolfkmod_vecreg_restore(void)
|
||||
{
|
||||
#if defined(WOLFSSL_BSDKM_FPU_DEBUG)
|
||||
wolfkmod_print_curthread("wolfkmod_vecreg_restore");
|
||||
#endif
|
||||
|
||||
if (is_fpu_kern_thread(0)) {
|
||||
/* kernel fpu threads are special, do nothing. They own a
|
||||
* persistent, dedicated fpu context. */
|
||||
#if defined(WOLFSSL_BSDKM_FPU_DEBUG)
|
||||
printf("info: wolfkmod_vecreg_restore: is fpu kern thread\n");
|
||||
#endif
|
||||
return;
|
||||
}
|
||||
|
||||
if (curthread->td_pcb->pcb_flags & PCB_KERNFPU) {
|
||||
/* kern fpu is active for this thread. check tid and nesting level. */
|
||||
lwpid_t td_tid = wolfkmod_fpu_get_tid();
|
||||
if (td_tid != curthread->td_tid) {
|
||||
printf("error: wolfkmod_vecreg_restore: got tid = %d, "
|
||||
"expected %d\n", td_tid, curthread->td_tid);
|
||||
return;
|
||||
}
|
||||
|
||||
/* decrement the nesting level. */
|
||||
if (fpu_states[PCPU_GET(cpuid)].nest > 0) {
|
||||
fpu_states[PCPU_GET(cpuid)].nest--;
|
||||
}
|
||||
|
||||
/* if last level, zero the thread id then call fpu_kern_leave */
|
||||
if (fpu_states[PCPU_GET(cpuid)].nest == 0) {
|
||||
fpu_states[PCPU_GET(cpuid)].td_tid = 0;
|
||||
wolfkmod_fpu_kern_leave();
|
||||
}
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
#endif /* !WC_SKIP_INCLUDED_C_FILES */
|
||||
+30
-8
@@ -123,9 +123,18 @@ then
|
||||
AM_CCASFLAGS="$AM_CCASFLAGS -DWOLFSSL_EXPERIMENTAL_SETTINGS"
|
||||
fi
|
||||
|
||||
# Kernel module benchmark
|
||||
ENABLED_KERNEL_BENCHMARKS=""
|
||||
AC_ARG_ENABLE([kernel-benchmarks],
|
||||
[AS_HELP_STRING([--enable-kernel-benchmarks],[Enable crypto benchmarking autorun at module load time for kernel module (default: disabled)])],
|
||||
[ENABLED_KERNEL_BENCHMARKS=$enableval])
|
||||
if test "$ENABLED_KERNEL_BENCHMARKS" = "yes"
|
||||
then
|
||||
AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_KERNEL_BENCHMARKS"
|
||||
fi
|
||||
AC_SUBST([ENABLED_KERNEL_BENCHMARKS])
|
||||
|
||||
# Linux Kernel Module options (more options later)
|
||||
|
||||
AC_ARG_ENABLE([linuxkm],
|
||||
[AS_HELP_STRING([--enable-linuxkm],[Enable Linux Kernel Module (default: disabled)])],
|
||||
[ENABLED_LINUXKM=$enableval],
|
||||
@@ -145,6 +154,12 @@ AC_ARG_ENABLE([freebsdkm],
|
||||
[ENABLED_BSDKM=no]
|
||||
)
|
||||
|
||||
AC_ARG_ENABLE([freebsdkm-crypto-register],
|
||||
[AS_HELP_STRING([--enable-freebsdkm-crypto-register],[Register wolfCrypt implementations with the FreeBSD kernel opencrypto framework. (default: disabled)])],
|
||||
[ENABLED_BSDKM_REGISTER=$enableval],
|
||||
[ENABLED_BSDKM_REGISTER=no]
|
||||
)
|
||||
|
||||
AC_CHECK_HEADERS([arpa/inet.h fcntl.h limits.h netdb.h netinet/in.h stddef.h time.h sys/ioctl.h sys/socket.h sys/time.h errno.h sys/un.h ctype.h sys/random.h])
|
||||
AC_CHECK_LIB([network],[socket])
|
||||
AC_C_BIGENDIAN
|
||||
@@ -727,10 +742,8 @@ AC_SUBST([ENABLED_LINUXKM_PIE])
|
||||
|
||||
AC_ARG_ENABLE([linuxkm-benchmarks],
|
||||
[AS_HELP_STRING([--enable-linuxkm-benchmarks],[Enable crypto benchmarking autorun at module load time for Linux kernel module (default: disabled)])],
|
||||
[ENABLED_KERNEL_BENCHMARKS=$enableval],
|
||||
[ENABLED_KERNEL_BENCHMARKS=no]
|
||||
)
|
||||
if test "$ENABLED_KERNEL_BENCHMARKS" = "yes"
|
||||
[ENABLED_KERNEL_BENCHMARKS=$enableval])
|
||||
if test "$ENABLED_LINUXKM" = "yes" && test "$ENABLED_KERNEL_BENCHMARKS" = "yes"
|
||||
then
|
||||
AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_LINUXKM_BENCHMARKS"
|
||||
fi
|
||||
@@ -819,17 +832,15 @@ AC_ARG_WITH([bsd-export-syms],
|
||||
|
||||
if test "x$ENABLED_BSDKM" = "xyes"
|
||||
then
|
||||
# wolfcrypt only, no-asm supported for now.
|
||||
# note: bsdkm is wolfcrypt only for now.
|
||||
HAVE_KERNEL_MODE=yes
|
||||
KERNEL_MODE_DEFAULTS=yes
|
||||
ENABLED_NO_LIBRARY=yes
|
||||
ENABLED_BENCHMARK=no
|
||||
ENABLED_ASM=no
|
||||
|
||||
output_objdir="$(realpath "$output_objdir")/bsdkm"
|
||||
|
||||
AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_BSDKM -DWC_SIPHASH_NO_ASM"
|
||||
AM_CFLAGS="$AM_CFLAGS -DTFM_NO_ASM -DWOLFSSL_NO_ASM"
|
||||
AM_CFLAGS="$AM_CFLAGS -DNO_DEV_RANDOM -DNO_WRITEV -DNO_STDIO_FILESYSTEM"
|
||||
AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_NO_SOCK -DWOLFSSL_USER_IO"
|
||||
AM_CFLAGS="$AM_CFLAGS -DXMALLOC_OVERRIDE -DWOLFCRYPT_ONLY"
|
||||
@@ -846,7 +857,16 @@ then
|
||||
fi
|
||||
AC_SUBST([KERNEL_ROOT])
|
||||
AC_SUBST([BSDKM_EXPORT_SYMS])
|
||||
fi
|
||||
|
||||
if test "x$ENABLED_BSDKM_REGISTER" = "xyes"
|
||||
then
|
||||
if test "$ENABLED_AESGCM" != "no" && test "$ENABLED_AESGCM_STREAM" = "no" && test "$enable_aesgcm_stream" != "no" && (test "$ENABLED_FIPS" = "no" || test $HAVE_FIPS_VERSION -ge 6); then
|
||||
ENABLED_AESGCM_STREAM=yes
|
||||
fi
|
||||
|
||||
AM_CFLAGS="$AM_CFLAGS -DBSDKM_CRYPTO_REGISTER"
|
||||
AC_SUBST([ENABLED_BSDKM_REGISTER])
|
||||
fi
|
||||
# end FreeBSD configure
|
||||
|
||||
@@ -3969,6 +3989,8 @@ then
|
||||
ENABLED_X86_ASM=yes
|
||||
fi
|
||||
fi
|
||||
AC_SUBST([ENABLED_AESNI])
|
||||
AC_SUBST([ENABLED_AESNI_WITH_AVX])
|
||||
|
||||
AC_ARG_ENABLE([aligndata],
|
||||
[AS_HELP_STRING([--enable-aligndata],[align data for ciphers (default: enabled)])],
|
||||
|
||||
@@ -2683,9 +2683,9 @@ static WC_INLINE void bench_stats_start(int* count, double* start)
|
||||
#endif
|
||||
}
|
||||
|
||||
#ifdef WOLFSSL_USE_SAVE_VECTOR_REGISTERS
|
||||
#if defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS)
|
||||
#define bench_stats_start(count, start) do { \
|
||||
SAVE_VECTOR_REGISTERS(pr_err( \
|
||||
SAVE_VECTOR_REGISTERS(WOLFSSL_DEBUG_PRINTF( \
|
||||
"ERROR: SAVE_VECTOR_REGISTERS failed for benchmark run."); \
|
||||
return; ); \
|
||||
bench_stats_start(count, start); \
|
||||
@@ -3161,7 +3161,7 @@ static void bench_stats_sym_finish(const char* desc, int useDeviceID,
|
||||
(void)useDeviceID;
|
||||
(void)ret;
|
||||
|
||||
#ifdef WOLFSSL_USE_SAVE_VECTOR_REGISTERS
|
||||
#if defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS)
|
||||
RESTORE_VECTOR_REGISTERS();
|
||||
#elif defined(WOLFSSL_LINUXKM)
|
||||
kernel_fpu_end();
|
||||
@@ -3559,7 +3559,7 @@ static void bench_stats_asym_finish_ex(const char* algo, int strength,
|
||||
(void)useDeviceID;
|
||||
(void)ret;
|
||||
|
||||
#ifdef WOLFSSL_USE_SAVE_VECTOR_REGISTERS
|
||||
#if defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS)
|
||||
RESTORE_VECTOR_REGISTERS();
|
||||
#elif defined(WOLFSSL_LINUXKM)
|
||||
kernel_fpu_end();
|
||||
@@ -16024,6 +16024,20 @@ void bench_sphincsKeySign(byte level, byte optim)
|
||||
return (double)ns / 1000000000.0;
|
||||
}
|
||||
|
||||
#elif defined(WOLFSSL_BSDKM)
|
||||
|
||||
#include <sys/timex.h>
|
||||
double current_time(int reset)
|
||||
{
|
||||
(void)reset;
|
||||
struct timespec ts;
|
||||
int64_t result = 0;
|
||||
|
||||
getnanouptime(&ts);
|
||||
result = (int64_t) ts.tv_sec + (int64_t) ts.tv_nsec / NANOSECOND;
|
||||
return (double)result;
|
||||
}
|
||||
|
||||
#elif defined(WOLFSSL_GAISLER_BCC)
|
||||
|
||||
#include <bcc/bcc.h>
|
||||
|
||||
@@ -113,7 +113,11 @@
|
||||
|
||||
static WC_INLINE void cpuid_set_flags(void)
|
||||
{
|
||||
#ifdef WOLFSSL_BSDKM
|
||||
if (WOLFSSL_ATOMIC_LOAD_UINT(cpuid_flags) == WC_CPUID_INITIALIZER) {
|
||||
#else
|
||||
if (WOLFSSL_ATOMIC_LOAD(cpuid_flags) == WC_CPUID_INITIALIZER) {
|
||||
#endif
|
||||
cpuid_flags_t new_cpuid_flags = 0,
|
||||
old_cpuid_flags = WC_CPUID_INITIALIZER;
|
||||
if (cpuid_flag(1, 0, ECX, 28)) { new_cpuid_flags |= CPUID_AVX1 ; }
|
||||
|
||||
@@ -122,7 +122,11 @@ typedef word32 cpuid_flags_t;
|
||||
* accurate.
|
||||
*/
|
||||
static WC_INLINE int cpuid_get_flags_atomic(cpuid_flags_atomic_t *flags) {
|
||||
#ifdef WOLFSSL_BSDKM
|
||||
if (WOLFSSL_ATOMIC_LOAD_UINT(*flags) == WC_CPUID_INITIALIZER) {
|
||||
#else
|
||||
if (WOLFSSL_ATOMIC_LOAD(*flags) == WC_CPUID_INITIALIZER) {
|
||||
#endif /* WOLFSSL_BSDKM */
|
||||
cpuid_flags_t old_cpuid_flags = WC_CPUID_INITIALIZER;
|
||||
return wolfSSL_Atomic_Uint_CompareExchange
|
||||
(flags, &old_cpuid_flags, cpuid_get_flags());
|
||||
|
||||
Reference in New Issue
Block a user