wolfSSL/wolfssl
1
0
Fork 1
mirror of https://github.com/wolfSSL/wolfssl.git synced 2026-01-26 18:32:20 +01:00
Code Issues Packages Projects Releases Wiki Activity
Files
e4e79dd8a39edde00a04f94948070b2fdacc94c0
wolfssl/linuxkm/linuxkm_wc_port.h
Daniel Pouzzner 0059f1647e move WC_RNG_BANK_SUPPORT implementation from wolfcrypt/src/random.c and wolfssl/wolfcrypt/random.h to new files wolfcrypt/src/rng_bank.c and wolfssl/wolfcrypt/rng_bank.h; 
wolfcrypt/src/rng_bank.c:

  * add wc_local_rng_bank_checkout_for_bankref, wc_BankRef_Release(), wc_rng_bank_new(), and wc_rng_bank_free();

  * in wc_rng_bank_checkin(), take a struct wc_rng_bank_inst **rng_inst and NULL it before return;

  * in wc_rng_bank_init(), add a devId arg, and handle devId in wc_rng_bank_inst_reinit();

  * add WC_RNG_BANK_INST_LOCK_* and use them in wc_rng_bank_checkout() and wc_rng_bank_checkin();

  * fix order of operations in wc_rng_bank_checkout() re DISABLE_VECTOR_REGISTERS();

wolfcrypt/src/random.c:

  * refactor per-instance salting for wc_rng_bank_inst: remove changes in Hash_df(), Hash_DRBG_Instantiate(), and _InitRng(), and in wc_rng_bank_init() and wc_rng_bank_inst_reinit(), use wc_InitRngNonce_ex() and pass the wc_rng_bank_inst pointer as the nonce;

  * simplify the WC_RNG_BANK_SUPPORT variant of wc_RNG_GenerateBlock() -- delegate to wc_local_rng_bank_checkout_for_bankref() and remove supplementary error checking;

  * in wc_FreeRng(), call wc_BankRef_Release() when WC_DRBG_BANKREF, and in wc_BankRef_Release(), fix refcount flub (not wolfSSL_RefFree, rather wolfSSL_RefDec);

  * streamline the WOLFSSL_LINUXKM wc_GenerateSeed();

wolfcrypt/test/test.c: add random_bank_test();

linuxkm/lkcapi_sha_glue.c: use WC_RNG_BANK_INST_TO_RNG() opportunistically;

configure.ac: add --enable-amdrdseed as a synonym for --enable-amdrand;

linuxkm/linuxkm_wc_port.h: when LINUXKM_LKCAPI_REGISTER_HASH_DRBG_DEFAULT, don't include get_random_bytes() in struct wolfssl_linuxkm_pie_redirect_table;

add various comments for clarity.
2026-01-07 22:54:07 -06:00

1742 lines
69 KiB
C
Raw Blame History

/* linuxkm_wc_port.h
*
* Copyright (C) 2006-2025 wolfSSL Inc.
*
* This file is part of wolfSSL.
*
* wolfSSL is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* (at your option) any later version.
*
* wolfSSL is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA
*/
/* included by wolfssl/wolfcrypt/wc_port.h */
#ifndef LINUXKM_WC_PORT_H
#define LINUXKM_WC_PORT_H
/*
* CRITICAL: Disable ARM64 LSE atomics for out-of-tree modules.
*
* When CONFIG_ARM64_LSE_ATOMICS is enabled, the kernel uses static keys
* (jump labels) in system_uses_lse_atomics() to choose between LSE and
* LL/SC atomic implementations at runtime. These static keys generate
* asm goto statements that reference .jump_table section symbols which
* cannot be resolved in out-of-tree modules, causing:
* "error: impossible constraint in 'asm'"
*
* By undefining CONFIG_ARM64_LSE_ATOMICS here (before any kernel headers
* that use atomics are included), we force use of the LL/SC fallback path
* which works correctly in out-of-tree modules.
*
* This must appear BEFORE #include <linux/version.h> because that header
* may transitively include headers that use atomics.
*/
#ifdef CONFIG_ARM64_LSE_ATOMICS
#undef CONFIG_ARM64_LSE_ATOMICS
#endif
#include <linux/version.h>
#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 16, 0)
#error Unsupported kernel.
#endif
#if defined(HAVE_FIPS) && defined(LINUXKM_LKCAPI_REGISTER_AESXTS) && defined(CONFIG_CRYPTO_MANAGER_EXTRA_TESTS)
/* CONFIG_CRYPTO_MANAGER_EXTRA_TESTS expects AES-XTS-384 to work, even when CONFIG_CRYPTO_FIPS, but FIPS 140-3 only allows AES-XTS-256 and AES-XTS-512. */
#error CONFIG_CRYPTO_MANAGER_EXTRA_TESTS is incompatible with FIPS wolfCrypt AES-XTS -- please reconfigure the target kernel to disable CONFIG_CRYPTO_MANAGER_EXTRA_TESTS.
#endif
#ifdef HAVE_CONFIG_H
#ifndef PACKAGE_NAME
#error wc_port.h included before config.h
#endif
/* config.h is autogenerated without gating, and is subject to repeat
* inclusions, so gate it out here to keep autodetection masking
* intact:
*/
#undef HAVE_CONFIG_H
#endif
/* suppress inclusion of stdint-gcc.h to avoid conflicts with Linux native
* include/linux/types.h:
*/
#define _GCC_STDINT_H
#define WC_PTR_TYPE uintptr_t
/* needed to suppress inclusion of stdio.h in wolfssl/wolfcrypt/types.h */
#define XSNPRINTF snprintf
/* the rigmarole around kstrtoll() here is to accommodate its
* warn-unused-result attribute.
*
* also needed to suppress inclusion of stdlib.h in
* wolfssl/wolfcrypt/types.h.
*/
#define XATOI(s) ({ \
long long _xatoi_res = 0; \
int _xatoi_ret = kstrtoll(s, 10, &_xatoi_res); \
if (_xatoi_ret != 0) { \
_xatoi_res = 0; \
} \
(int)_xatoi_res; \
})
/* Kbuild+gcc on x86 doesn't consistently honor the default ALIGN16 on stack
* objects, but gives adequate alignment with "32".
*/
#if defined(CONFIG_X86) && !defined(ALIGN16)
#define ALIGN16 __attribute__ ( (aligned (32)))
#endif
/* kvmalloc()/kvfree() and friends added in linux commit a7c3e901, merged for 4.12.
* kvrealloc() added in de2860f463, merged for 5.15, backported to 5.10.137.
* moved to ultimate home (slab.h) in 8587ca6f34, merged for 5.16.
*
* however, until 6.12 (commit 590b9d576c), it took an extra argument,
* oldsize, that makes it incompatible with traditional libc usage patterns,
* so we don't try to use it.
*/
#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 12, 0) && \
!defined(DONT_HAVE_KVMALLOC) && !defined(HAVE_KVMALLOC)
#define HAVE_KVMALLOC
#endif
#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 12, 0) && \
!defined(DONT_HAVE_KVREALLOC) && !defined(HAVE_KVREALLOC)
#define HAVE_KVREALLOC
#endif
#ifdef WOLFCRYPT_ONLY
#if defined(HAVE_KVMALLOC) && \
!defined(DONT_USE_KVMALLOC) && !defined(USE_KVMALLOC)
#define USE_KVMALLOC
#endif
#if defined(HAVE_KVREALLOC) && \
!defined(DONT_USE_KVREALLOC) && !defined(USE_KVREALLOC)
#define USE_KVREALLOC
#endif
#else
/* functioning realloc() is needed for the TLS stack. */
#if defined(HAVE_KVMALLOC) && defined(HAVE_KVREALLOC) && \
!defined(DONT_USE_KVMALLOC) && !defined(DONT_USE_KVREALLOC)
#ifndef USE_KVMALLOC
#define USE_KVMALLOC
#endif
#ifndef USE_KVREALLOC
#define USE_KVREALLOC
#endif
#endif
#endif
#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 1, 0)
/* added by 6bab69c650 */
#define static_assert(expr, ...) __static_assert(expr, ##__VA_ARGS__, #expr)
#define __static_assert(expr, msg, ...) _Static_assert(expr, msg)
#endif
/* kernel printf doesn't implement fp. */
#ifndef WOLFSSL_NO_FLOAT_FMT
#define WOLFSSL_NO_FLOAT_FMT
#endif
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0)) || \
(defined(RHEL_MAJOR) && \
((RHEL_MAJOR > 9) || ((RHEL_MAJOR == 9) && (RHEL_MINOR >= 5))))
#define WOLFSSL_DEBUG_PRINTF_FN _printk
#else
#define WOLFSSL_DEBUG_PRINTF_FN printk
#endif
#ifndef WOLFSSL_LINUXKM_USE_MUTEXES
struct wolfSSL_Mutex;
extern int wc_lkm_LockMutex(struct wolfSSL_Mutex* m);
#endif
#ifndef WC_LINUXKM_INTR_SIGNALS
#define WC_LINUXKM_INTR_SIGNALS { SIGKILL, SIGABRT, SIGHUP, SIGINT }
#endif
extern int wc_linuxkm_sig_ignore_begin(void);
extern int wc_linuxkm_sig_ignore_end(void);
extern int wc_linuxkm_check_for_intr_signals(void);
#ifndef WC_LINUXKM_MAX_NS_WITHOUT_YIELD
#define WC_LINUXKM_MAX_NS_WITHOUT_YIELD 1000000000
#endif
extern void wc_linuxkm_relax_long_loop(void);
#ifndef WC_SIG_IGNORE_BEGIN
#define WC_SIG_IGNORE_BEGIN() wc_linuxkm_sig_ignore_begin()
#endif
#ifndef WC_SIG_IGNORE_END
#define WC_SIG_IGNORE_END() wc_linuxkm_sig_ignore_end()
#endif
#ifndef WC_CHECK_FOR_INTR_SIGNALS
#define WC_CHECK_FOR_INTR_SIGNALS() wc_linuxkm_check_for_intr_signals()
#endif
#ifndef WC_RELAX_LONG_LOOP
#define WC_RELAX_LONG_LOOP() wc_linuxkm_relax_long_loop()
#endif
enum wc_svr_flags {
WC_SVR_FLAG_NONE = 0,
WC_SVR_FLAG_INHIBIT = 1,
};
#if defined(WOLFSSL_AESNI) || defined(USE_INTEL_SPEEDUP) || \
defined(WOLFSSL_SP_X86_64_ASM)
#ifndef CONFIG_X86
#error X86 SIMD extensions requested, but CONFIG_X86 is not set.
#endif
#define WOLFSSL_LINUXKM_SIMD
#define WOLFSSL_LINUXKM_SIMD_X86
#ifndef WOLFSSL_USE_SAVE_VECTOR_REGISTERS
#define WOLFSSL_USE_SAVE_VECTOR_REGISTERS
#endif
#elif defined(WOLFSSL_ARMASM) || defined(WOLFSSL_SP_ARM32_ASM) || \
defined(WOLFSSL_SP_ARM64_ASM) || defined(WOLFSSL_SP_ARM_THUMB_ASM) ||\
defined(WOLFSSL_SP_ARM_CORTEX_M_ASM)
#if !defined(CONFIG_ARM) && !defined(CONFIG_ARM64)
#error ARM SIMD extensions requested, but CONFIG_ARM* is not set.
#endif
#define WOLFSSL_LINUXKM_SIMD
#define WOLFSSL_LINUXKM_SIMD_ARM
#ifndef WOLFSSL_USE_SAVE_VECTOR_REGISTERS
#define WOLFSSL_USE_SAVE_VECTOR_REGISTERS
#endif
#endif
#if defined(HAVE_HASHDRBG) && defined(HAVE_FIPS) && \
defined(HAVE_ENTROPY_MEMUSE) && \
!defined(WC_LINUXKM_WOLFENTROPY_IN_GLUE_LAYER)
#define WC_LINUXKM_WOLFENTROPY_IN_GLUE_LAYER
#elif defined(HAVE_HASHDRBG) && defined(HAVE_FIPS) && \
(defined(HAVE_INTEL_RDSEED) || defined(HAVE_AMD_RDSEED)) && \
!defined(HAVE_ENTROPY_MEMUSE) && \
!defined(WC_LINUXKM_RDSEED_IN_GLUE_LAYER)
#define WC_LINUXKM_RDSEED_IN_GLUE_LAYER
#endif
#if defined(WC_LINUXKM_WOLFENTROPY_IN_GLUE_LAYER)
struct OS_Seed;
extern int wc_linuxkm_GenerateSeed_wolfEntropy(struct OS_Seed* os, unsigned char* output, unsigned int sz);
#define WC_GENERATE_SEED_DEFAULT wc_linuxkm_GenerateSeed_wolfEntropy
#elif defined(WC_LINUXKM_RDSEED_IN_GLUE_LAYER)
struct OS_Seed;
extern int wc_linuxkm_GenerateSeed_IntelRD(struct OS_Seed* os, unsigned char* output, unsigned int sz);
#define WC_GENERATE_SEED_DEFAULT wc_linuxkm_GenerateSeed_IntelRD
#endif
#ifdef BUILDING_WOLFSSL
#if ((LINUX_VERSION_CODE >= KERNEL_VERSION(5, 16, 0)) || \
(defined(RHEL_MAJOR) && \
((RHEL_MAJOR > 9) || ((RHEL_MAJOR == 9) && (RHEL_MINOR >= 5))))) && \
defined(CONFIG_X86)
/* linux/slab.h recursively brings in linux/page-flags.h, bringing in
* non-inline implementations of functions folio_flags() and
* const_folio_flags(). but we can retrofit the attribute.
*/
struct folio;
static __always_inline unsigned long *folio_flags(
struct folio *folio, unsigned n);
#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 9, 0)
static __always_inline const unsigned long *const_folio_flags(
const struct folio *folio, unsigned n);
#endif
#endif
#if defined(CONFIG_MIPS) && defined(WC_SYM_RELOC_TABLES)
/* __ZBOOT__ disables some unhelpful macros around the mem*() funcs in
* legacy arch/mips/include/asm/string.h
*/
#define __ZBOOT__
#define memcmp __builtin_memcmp
#define __ARCH_MEMCMP_NO_REDIRECT
#define __ARCH_MEMCPY_NO_REDIRECT
#define __builtin_memcpy memcpy
extern void *memcpy(void *dest, const void *src, unsigned int n);
#define __ARCH_MEMCPY_NO_REDIRECT
#define __builtin_memset memset
extern void *memset(void *dest, int c, unsigned int n);
#endif
_Pragma("GCC diagnostic push");
/* we include all the needed kernel headers with these masked out. else
* there are profuse warnings, especially on older kernels.
*/
_Pragma("GCC diagnostic ignored \"-Wunused-parameter\"");
_Pragma("GCC diagnostic ignored \"-Wpointer-arith\"");
_Pragma("GCC diagnostic ignored \"-Wshadow\"");
_Pragma("GCC diagnostic ignored \"-Wnested-externs\"");
_Pragma("GCC diagnostic ignored \"-Wredundant-decls\"");
_Pragma("GCC diagnostic ignored \"-Wsign-compare\"");
_Pragma("GCC diagnostic ignored \"-Wpointer-sign\"");
_Pragma("GCC diagnostic ignored \"-Wbad-function-cast\"");
_Pragma("GCC diagnostic ignored \"-Wdiscarded-qualifiers\"");
_Pragma("GCC diagnostic ignored \"-Wtype-limits\"");
_Pragma("GCC diagnostic ignored \"-Wswitch-enum\"");
_Pragma("GCC diagnostic ignored \"-Wcast-function-type\""); /* needed for kernel 4.14.336 */
_Pragma("GCC diagnostic ignored \"-Wformat-nonliteral\""); /* needed for kernel 4.9.282 */
_Pragma("GCC diagnostic ignored \"-Wattributes\"");
#include <linux/kconfig.h>
#ifdef CONFIG_KASAN
#ifndef WC_SANITIZE_DISABLE
#define WC_SANITIZE_DISABLE() kasan_disable_current()
#endif
#ifndef WC_SANITIZE_ENABLE
#define WC_SANITIZE_ENABLE() kasan_enable_current()
#endif
#endif
#if defined(CONFIG_FORTIFY_SOURCE) && \
!defined(WC_FORCE_LINUXKM_FORTIFY_SOURCE) && \
(defined(WC_SYM_RELOC_TABLES) || \
(LINUX_VERSION_CODE < KERNEL_VERSION(5, 18, 0)))
/* fortify-source causes all sorts of awkward problems for the PIE
* build, up to and including stubborn external references and multiple
* definitions of string functions.
*
* fortify-source before kernel 5.18 has similar issues regardless of
* PIE, around our macro-shimming of the string functions.
*/
#undef CONFIG_FORTIFY_SOURCE
#endif
#if defined(WC_FORCE_LINUXKM_FORTIFY_SOURCE) && \
!defined(CONFIG_FORTIFY_SOURCE)
#error WC_FORCE_LINUXKM_FORTIFY_SOURCE without CONFIG_FORTIFY_SOURCE.
#endif
#if defined(WC_CONTAINERIZE_THIS) && defined(CONFIG_ARM64)
#define alt_cb_patch_nops my__alt_cb_patch_nops
#define queued_spin_lock_slowpath my__queued_spin_lock_slowpath
#endif
#include <linux/kernel.h>
#include <linux/ctype.h>
#if defined(CONFIG_FORTIFY_SOURCE) || defined(DEBUG_LINUXKM_FORTIFY_OVERLAY)
#ifdef WC_CONTAINERIZE_THIS
/* the inline definitions in fortify-string.h use non-inline
* fortify_panic().
*/
extern void __my_fortify_panic(const char *name) __noreturn __cold;
#if LINUX_VERSION_CODE >= KERNEL_VERSION(6,9,0)
/* see linux 3d965b33e40d9 */
#define fortify_panic(func, write, avail, size, retfail) \
__my_fortify_panic(#func)
#else
#define fortify_panic __my_fortify_panic
#endif
#endif
/* the _FORTIFY_SOURCE macros and implementations for several string
* functions are incompatible with libwolfssl, so just reimplement with
* inlines and remap with macros.
*/
#define __ARCH_STRLEN_NO_REDIRECT
#define __ARCH_MEMCPY_NO_REDIRECT
#define __ARCH_MEMSET_NO_REDIRECT
#define __ARCH_MEMMOVE_NO_REDIRECT
/* the inline definitions in fortify-string.h use non-inline
* strlen().
*/
static inline size_t strlen(const char *s) {
const char *s_start = s;
while (*s)
++s;
return (size_t)((uintptr_t)s - (uintptr_t)s_start);
}
#include <linux/string.h>
#undef strlen
#define strlen(s) \
((__builtin_constant_p(s) && __builtin_constant_p(*(s))) ? \
(sizeof(s) - 1) : strlen(s))
static inline void *my_memcpy(void *dest, const void *src, size_t n) {
if (! (((uintptr_t)dest | (uintptr_t)src | (uintptr_t)n)
& (uintptr_t)(sizeof(uintptr_t) - 1)))
{
uintptr_t *src_longs = (uintptr_t *)src,
*dest_longs = (uintptr_t *)dest,
*endp = (uintptr_t *)((u8 *)src + n);
while (src_longs < endp)
*dest_longs++ = *src_longs++;
} else {
u8 *src_bytes = (u8 *)src,
*dest_bytes = (u8 *)dest,
*endp = src_bytes + n;
while (src_bytes < endp)
*dest_bytes++ = *src_bytes++;
}
return dest;
}
#undef memcpy
#define memcpy my_memcpy
static inline void *my_memset(void *dest, int c, size_t n) {
if (! (((uintptr_t)dest | (uintptr_t)n)
& (uintptr_t)(sizeof(uintptr_t) - 1)))
{
uintptr_t c_long = __builtin_choose_expr(
sizeof(uintptr_t) == 8,
(uintptr_t)(u8)c * 0x0101010101010101UL,
(uintptr_t)(u8)c * 0x01010101U
);
uintptr_t *dest_longs = (uintptr_t *)dest,
*endp = (uintptr_t *)((u8 *)dest_longs + n);
while (dest_longs < endp)
*dest_longs++ = c_long;
} else {
u8 *dest_bytes = (u8 *)dest, *endp = dest_bytes + n;
while (dest_bytes < endp)
*dest_bytes++ = (u8)c;
}
return dest;
}
#undef memset
#define memset my_memset
static inline void *my_memmove(void *dest, const void *src, size_t n) {
if (! (((uintptr_t)dest | (uintptr_t)src | (uintptr_t)n)
& (uintptr_t)(sizeof(uintptr_t) - 1)))
{
uintptr_t *src_longs = (uintptr_t *)src,
*dest_longs = (uintptr_t *)dest;
n >>= __builtin_choose_expr(
sizeof(uintptr_t) == 8,
3U,
2U);
if (src_longs < dest_longs) {
uintptr_t *startp = src_longs;
src_longs += n - 1;
dest_longs += n - 1;
while (src_longs >= startp)
*dest_longs-- = *src_longs--;
} else if (src_longs > dest_longs) {
uintptr_t *endp = src_longs + n;
while (src_longs < endp)
*dest_longs++ = *src_longs++;
}
} else {
u8 *src_bytes = (u8 *)src, *dest_bytes = (u8 *)dest;
if (src_bytes < dest_bytes) {
u8 *startp = src_bytes;
src_bytes += n - 1;
dest_bytes += n - 1;
while (src_bytes >= startp)
*dest_bytes-- = *src_bytes--;
} else if (src_bytes > dest_bytes) {
u8 *endp = src_bytes + n;
while (src_bytes < endp)
*dest_bytes++ = *src_bytes++;
}
}
return dest;
}
#undef memmove
#define memmove my_memmove
#else /* !CONFIG_FORTIFY_SOURCE */
#include <linux/string.h>
#endif /* !CONFIG_FORTIFY_SOURCE */
#ifndef WC_CONTAINERIZE_THIS
#include <linux/init.h>
#include <linux/module.h>
#include <linux/delay.h>
#endif
#if defined(HAVE_KVMALLOC) && \
(LINUX_VERSION_CODE < KERNEL_VERSION(5, 16, 0)) && \
!(defined(RHEL_MAJOR) && ((RHEL_MAJOR > 9) || \
((RHEL_MAJOR == 9) && (RHEL_MINOR >= 5))))
/* before 5.16, the kvmalloc_node() and kvfree() prototypes were in
* mm.h. however, mm.h brings in static, but not inline, pmd_to_page(),
* with direct references to global vmem variables.
*/
#ifdef WC_CONTAINERIZE_THIS
#include <linux/mm_types.h>
#if USE_SPLIT_PMD_PTLOCKS
static __always_inline struct page *pmd_to_page(pmd_t *pmd);
#endif
#endif
#include <linux/mm.h>
#endif
#ifndef WC_CONTAINERIZE_THIS
#include <linux/kthread.h>
#include <linux/net.h>
#endif
#include <linux/slab.h>
#include <linux/sched.h>
#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)
/* for signal_pending() */
#include <linux/sched/signal.h>
/* for local_clock() */
#include <linux/sched/clock.h>
#endif
#include <linux/random.h>
#if !defined(WC_CONTAINERIZE_THIS) && defined(CONFIG_HAVE_KPROBES)
#include <linux/kprobes.h>
#endif
#ifdef LINUXKM_LKCAPI_REGISTER
/* the LKCAPI assumes that expanded encrypt and decrypt keys will stay
* loaded simultaneously, and the Linux in-tree implementations have two
* AES key structs in each context, one for each direction. in
* linuxkm/lkcapi_aes_glue.c, we do the same
* thing with "struct km_AesCtx". however, wolfCrypt struct AesXts
* already has two AES expanded keys, the main and tweak, and the tweak
* is always used in the encrypt direction regardless of the main
* direction. to avoid allocating and computing a duplicate second
* tweak encrypt key, we set
* WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS, which adds a second
* Aes slot to wolfCrypt's struct AesXts, and activates support for
* AES_ENCRYPTION_AND_DECRYPTION on AES-XTS.
*/
#ifndef WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS
#define WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS
#endif
/* setup for LINUXKM_LKCAPI_REGISTER_HASH_DRBG_DEFAULT needs to be here
* to assure that calls to get_random_bytes() in random.c are gated out
* (they would recurse, potentially infinitely).
*/
#if (defined(LINUXKM_LKCAPI_REGISTER_ALL) && \
!defined(LINUXKM_LKCAPI_DONT_REGISTER_HASH_DRBG) && \
!defined(LINUXKM_LKCAPI_DONT_REGISTER_HASH_DRBG_DEFAULT)) && \
!defined(LINUXKM_LKCAPI_REGISTER_HASH_DRBG_DEFAULT)
#define LINUXKM_LKCAPI_REGISTER_HASH_DRBG_DEFAULT
#endif
#ifndef WC_CONTAINERIZE_THIS
#include <linux/crypto.h>
#include <linux/scatterlist.h>
#include <crypto/scatterwalk.h>
#include <crypto/internal/aead.h>
#include <crypto/internal/hash.h>
#include <crypto/internal/rng.h>
#include <crypto/internal/skcipher.h>
#include <crypto/internal/akcipher.h>
#include <crypto/internal/kpp.h>
#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 13, 0)
#include <crypto/internal/sig.h>
#endif /* linux ver >= 6.13 */
#if defined(_LINUX_REFCOUNT_H) || defined(_LINUX_REFCOUNT_TYPES_H)
static inline int wc_lkm_refcount_to_int(refcount_t *refcount) {
_Pragma("GCC diagnostic push");
_Pragma("GCC diagnostic ignored \"-Wnested-externs\"");
return atomic_read(&(refcount->refs));
_Pragma("GCC diagnostic pop");
}
#else
static inline int wc_lkm_refcount_to_int(atomic_t *refcount) {
_Pragma("GCC diagnostic push");
_Pragma("GCC diagnostic ignored \"-Wnested-externs\"");
return atomic_read(&refcount);
_Pragma("GCC diagnostic pop");
}
#endif
#define WC_LKM_REFCOUNT_TO_INT(refcount) wc_lkm_refcount_to_int(&(refcount))
#endif /* !WC_CONTAINERIZE_THIS */
#endif /* LINUXKM_LKCAPI_REGISTER */
/* benchmarks.c uses floating point math, so needs a working
* SAVE_VECTOR_REGISTERS().
*/
#if defined(WOLFSSL_LINUXKM_BENCHMARKS) && \
!defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS)
#define WOLFSSL_USE_SAVE_VECTOR_REGISTERS
#endif
#if defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) && \
defined(CONFIG_X86)
extern __must_check int allocate_wolfcrypt_linuxkm_fpu_states(void);
extern void free_wolfcrypt_linuxkm_fpu_states(void);
WOLFSSL_API __must_check int wc_can_save_vector_registers_x86(void);
WOLFSSL_API __must_check int wc_save_vector_registers_x86(enum wc_svr_flags flags);
WOLFSSL_API void wc_restore_vector_registers_x86(enum wc_svr_flags flags);
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
#include <asm/i387.h>
#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0)
/* added by a62b01cd6c */
#include <asm-generic/simd.h>
#endif
#else
#include <asm/simd.h>
#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0)
/* added by 266d051601 */
#include <crypto/internal/simd.h>
#endif
#endif
#ifndef CAN_SAVE_VECTOR_REGISTERS
#ifdef DEBUG_VECTOR_REGISTER_ACCESS_FUZZING
#define CAN_SAVE_VECTOR_REGISTERS() (wc_can_save_vector_registers_x86() && (SAVE_VECTOR_REGISTERS2_fuzzer() == 0))
#else
#define CAN_SAVE_VECTOR_REGISTERS() wc_can_save_vector_registers_x86()
#endif
#endif
#ifndef SAVE_VECTOR_REGISTERS
#define SAVE_VECTOR_REGISTERS(fail_clause) { \
int _svr_ret = wc_save_vector_registers_x86(WC_SVR_FLAG_NONE); \
if (_svr_ret != 0) { \
fail_clause \
} \
}
#endif
#ifndef SAVE_VECTOR_REGISTERS2
#ifdef DEBUG_VECTOR_REGISTER_ACCESS_FUZZING
#define SAVE_VECTOR_REGISTERS2() ({ \
int _fuzzer_ret = SAVE_VECTOR_REGISTERS2_fuzzer(); \
(_fuzzer_ret == 0) ? \
wc_save_vector_registers_x86(WC_SVR_FLAG_NONE) : \
_fuzzer_ret; \
})
#else
#define SAVE_VECTOR_REGISTERS2() wc_save_vector_registers_x86(WC_SVR_FLAG_NONE)
#endif
#endif
#ifndef RESTORE_VECTOR_REGISTERS
#define RESTORE_VECTOR_REGISTERS() wc_restore_vector_registers_x86(WC_SVR_FLAG_NONE)
#endif
#ifndef DISABLE_VECTOR_REGISTERS
#define DISABLE_VECTOR_REGISTERS() wc_save_vector_registers_x86(WC_SVR_FLAG_INHIBIT)
#endif
#ifndef REENABLE_VECTOR_REGISTERS
#define REENABLE_VECTOR_REGISTERS() wc_restore_vector_registers_x86(WC_SVR_FLAG_INHIBIT)
#endif
#elif defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) && (defined(CONFIG_ARM) || defined(CONFIG_ARM64))
#error kernel module ARM SIMD is not yet tested or usable.
#include <asm/fpsimd.h>
static WARN_UNUSED_RESULT inline int save_vector_registers_arm(void)
{
preempt_disable();
if (! may_use_simd()) {
preempt_enable();
return BAD_STATE_E;
} else {
fpsimd_preserve_current_state();
return 0;
}
}
static inline void restore_vector_registers_arm(void)
{
fpsimd_restore_current_state();
preempt_enable();
}
#ifndef SAVE_VECTOR_REGISTERS
#define SAVE_VECTOR_REGISTERS(fail_clause) { int _svr_ret = save_vector_registers_arm(); if (_svr_ret != 0) { fail_clause } }
#endif
#ifndef SAVE_VECTOR_REGISTERS2
#define SAVE_VECTOR_REGISTERS2() save_vector_registers_arm()
#endif
#ifndef CAN_SAVE_VECTOR_REGISTERS
#define CAN_SAVE_VECTOR_REGISTERS() can_save_vector_registers_arm()
#endif
#ifndef RESTORE_VECTOR_REGISTERS
#define RESTORE_VECTOR_REGISTERS() restore_vector_registers_arm()
#endif
#elif defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS)
#error WOLFSSL_USE_SAVE_VECTOR_REGISTERS is set for an unimplemented architecture.
#endif /* WOLFSSL_USE_SAVE_VECTOR_REGISTERS */
_Pragma("GCC diagnostic pop");
/* avoid -Wpointer-arith, encountered when -DCONFIG_FORTIFY_SOURCE */
#undef __is_constexpr
#define __is_constexpr(x) __builtin_constant_p(x)
/* the kernel uses -std=c89, but not -pedantic, and makes full use of anon
* structs/unions, so we should too.
*/
#define HAVE_ANONYMOUS_INLINE_AGGREGATES 1
#define NO_THREAD_LS
#define NO_ATTRIBUTE_CONSTRUCTOR
#ifdef HAVE_FIPS
extern int wolfCrypt_FIPS_first(void);
extern int wolfCrypt_FIPS_last(void);
extern const unsigned int wolfCrypt_FIPS_ro_start[];
extern const unsigned int wolfCrypt_FIPS_ro_end[];
#if FIPS_VERSION3_GE(6,0,0)
#ifndef NO_AES
extern int wolfCrypt_FIPS_AES_sanity(void);
extern const unsigned int wolfCrypt_FIPS_aes_ro_sanity[2];
#if defined(WOLFSSL_CMAC) && defined(WOLFSSL_AES_DIRECT)
extern int wolfCrypt_FIPS_CMAC_sanity(void);
extern const unsigned int wolfCrypt_FIPS_cmac_ro_sanity[2];
#endif
#endif
#ifndef NO_DH
extern int wolfCrypt_FIPS_DH_sanity(void);
extern const unsigned int wolfCrypt_FIPS_dh_ro_sanity[2];
#endif
#ifdef HAVE_ECC
extern int wolfCrypt_FIPS_ECC_sanity(void);
extern const unsigned int wolfCrypt_FIPS_ecc_ro_sanity[2];
#endif
#ifdef HAVE_ED25519
extern int wolfCrypt_FIPS_ED25519_sanity(void);
extern const unsigned int wolfCrypt_FIPS_ed25519_ro_sanity[2];
#endif
#ifdef HAVE_ED448
extern int wolfCrypt_FIPS_ED448_sanity(void);
extern const unsigned int wolfCrypt_FIPS_ed448_ro_sanity[2];
#endif
extern int wolfCrypt_FIPS_HMAC_sanity(void);
extern const unsigned int wolfCrypt_FIPS_hmac_ro_sanity[2];
#ifndef NO_KDF
extern int wolfCrypt_FIPS_KDF_sanity(void);
extern const unsigned int wolfCrypt_FIPS_kdf_ro_sanity[2];
#endif
#ifdef HAVE_PBKDF2
extern int wolfCrypt_FIPS_PBKDF_sanity(void);
extern const unsigned int wolfCrypt_FIPS_pbkdf_ro_sanity[2];
#endif
#ifdef HAVE_HASHDRBG
extern int wolfCrypt_FIPS_DRBG_sanity(void);
extern const unsigned int wolfCrypt_FIPS_drbg_ro_sanity[2];
#endif
#ifndef NO_RSA
extern int wolfCrypt_FIPS_RSA_sanity(void);
extern const unsigned int wolfCrypt_FIPS_rsa_ro_sanity[2];
#endif
#ifndef NO_SHA
extern int wolfCrypt_FIPS_SHA_sanity(void);
extern const unsigned int wolfCrypt_FIPS_sha_ro_sanity[2];
#endif
#ifndef NO_SHA256
extern int wolfCrypt_FIPS_SHA256_sanity(void);
extern const unsigned int wolfCrypt_FIPS_sha256_ro_sanity[2];
#endif
#ifdef WOLFSSL_SHA512
extern int wolfCrypt_FIPS_SHA512_sanity(void);
extern const unsigned int wolfCrypt_FIPS_sha512_ro_sanity[2];
#endif
#ifdef WOLFSSL_SHA3
extern int wolfCrypt_FIPS_SHA3_sanity(void);
extern const unsigned int wolfCrypt_FIPS_sha3_ro_sanity[2];
#endif
extern int wolfCrypt_FIPS_FT_sanity(void);
extern const unsigned int wolfCrypt_FIPS_ft_ro_sanity[2];
extern const unsigned int wolfCrypt_FIPS_f_ro_sanity[2];
extern int wc_RunAllCast_fips(void);
#endif
#endif
#if !defined(WOLFCRYPT_ONLY) && !defined(NO_CERTS)
/* work around backward dependency of asn.c on ssl.c. */
struct Signer;
struct Signer *GetCA(void *signers, unsigned char *hash);
#ifndef NO_SKID
struct Signer *GetCAByName(void* signers, unsigned char *hash);
#ifdef HAVE_OCSP
struct Signer* GetCAByKeyHash(void* vp, const unsigned char* keyHash);
#endif /* HAVE_OCSP */
#ifdef WOLFSSL_AKID_NAME
#ifdef WOLFSSL_API_PREFIX_MAP
#define GetCAByAKID wolfSSL_GetCAByAKID
#endif
struct Signer* GetCAByAKID(void* vp, const unsigned char* issuer,
unsigned int issuerSz,
const unsigned char* serial,
unsigned int serialSz);
#endif
#endif /* NO_SKID */
#if defined(OPENSSL_EXTRA) || defined(OPENSSL_EXTRA_X509_SMALL)
struct WOLFSSL_X509_NAME;
extern int wolfSSL_X509_NAME_add_entry_by_NID(struct WOLFSSL_X509_NAME *name, int nid,
int type, const unsigned char *bytes,
int len, int loc, int set);
extern void wolfSSL_X509_NAME_free(struct WOLFSSL_X509_NAME* name);
extern struct WOLFSSL_X509_NAME* wolfSSL_X509_NAME_new_ex(void *heap);
#endif /* OPENSSL_EXTRA || OPENSSL_EXTRA_X509_SMALL */
#endif /* !WOLFCRYPT_ONLY && !NO_CERTS */
#if defined(WC_CONTAINERIZE_THIS) && !defined(WC_SYM_RELOC_TABLES)
#error "compiling -DWC_CONTAINERIZE_THIS requires relocation tables."
#endif
#ifdef WC_SYM_RELOC_TABLES
#ifndef WOLFSSL_TEXT_SEGMENT_CANONICALIZER
#define WOLFSSL_TEXT_SEGMENT_CANONICALIZER(text_in, text_in_len, text_out, cur_index_p) \
wc_linuxkm_normalize_relocations(text_in, text_in_len, text_out, cur_index_p)
#define WOLFSSL_TEXT_SEGMENT_CANONICALIZER_BUFSIZ 8192
#endif
extern __attribute__((error("uncallable fencepost"))) int __wc_text_start(void);
extern __attribute__((error("uncallable fencepost"))) int __wc_text_end(void);
#ifdef HAVE_FIPS
static_assert(__builtin_types_compatible_p(typeof(__wc_text_start), typeof(wolfCrypt_FIPS_first)));
static_assert(__builtin_types_compatible_p(typeof(__wc_text_end), typeof(wolfCrypt_FIPS_last)));
#endif
extern const unsigned int __wc_rodata_start[], __wc_rodata_end[];
#ifdef HAVE_FIPS
static_assert(__builtin_types_compatible_p(typeof(__wc_rodata_start), typeof(wolfCrypt_FIPS_ro_start)));
static_assert(__builtin_types_compatible_p(typeof(__wc_rodata_end), typeof(wolfCrypt_FIPS_ro_end)));
#endif
extern const u8
__wc_rwdata_start[],
__wc_rwdata_end[],
__wc_bss_start[],
__wc_bss_end[];
extern const unsigned int wc_linuxkm_pie_reloc_tab[];
extern const unsigned long wc_linuxkm_pie_reloc_tab_length;
extern ssize_t wc_linuxkm_normalize_relocations(
const u8 *text_in,
size_t text_in_len,
u8 *text_out,
ssize_t *cur_index_p);
#ifdef CONFIG_MIPS
#undef __ARCH_MEMCMP_NO_REDIRECT
#undef memcmp
extern int memcmp(const void *s1, const void *s2, size_t n);
#endif
struct wolfssl_linuxkm_pie_redirect_table {
typeof(wc_linuxkm_normalize_relocations) *wc_linuxkm_normalize_relocations;
#ifndef __ARCH_MEMCMP_NO_REDIRECT
typeof(memcmp) *memcmp;
#endif
#ifndef __ARCH_MEMCPY_NO_REDIRECT
typeof(memcpy) *memcpy;
#endif
#ifndef __ARCH_MEMSET_NO_REDIRECT
typeof(memset) *memset;
#endif
#ifndef __ARCH_MEMMOVE_NO_REDIRECT
typeof(memmove) *memmove;
#endif
#ifndef __ARCH_STRCMP_NO_REDIRECT
typeof(strcmp) *strcmp;
#endif
#ifndef __ARCH_STRNCMP_NO_REDIRECT
typeof(strncmp) *strncmp;
#endif
#ifndef __ARCH_STRCASECMP_NO_REDIRECT
typeof(strcasecmp) *strcasecmp;
#endif
#ifndef __ARCH_STRNCASECMP_NO_REDIRECT
typeof(strncasecmp) *strncasecmp;
#endif
#ifndef __ARCH_STRLEN_NO_REDIRECT
typeof(strlen) *strlen;
#endif
#ifndef __ARCH_STRSTR_NO_REDIRECT
typeof(strstr) *strstr;
#endif
#ifndef __ARCH_STRNCPY_NO_REDIRECT
typeof(strncpy) *strncpy;
#endif
#ifndef __ARCH_STRNCAT_NO_REDIRECT
typeof(strncat) *strncat;
#endif
typeof(kstrtoll) *kstrtoll;
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0)) || \
(defined(RHEL_MAJOR) && \
((RHEL_MAJOR > 9) || ((RHEL_MAJOR == 9) && (RHEL_MINOR >= 5))))
typeof(_printk) *_printk;
#else
typeof(printk) *printk;
#endif
#ifdef CONFIG_FORTIFY_SOURCE
typeof(__warn_printk) *__warn_printk;
#endif
typeof(snprintf) *snprintf;
const unsigned char *_ctype;
#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0)
typeof(kmalloc_noprof) *kmalloc_noprof;
typeof(krealloc_node_align_noprof) *krealloc_node_align_noprof;
typeof(kzalloc_noprof) *kzalloc_noprof;
typeof(__kvmalloc_node_noprof) *__kvmalloc_node_noprof;
typeof(__kmalloc_cache_noprof) *__kmalloc_cache_noprof;
#ifdef HAVE_KVREALLOC
typeof(kvrealloc_node_align_noprof) *kvrealloc_node_align_noprof;
#endif
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(6, 11, 0)
typeof(kmalloc_noprof) *kmalloc_noprof;
typeof(krealloc_noprof) *krealloc_noprof;
typeof(kzalloc_noprof) *kzalloc_noprof;
typeof(__kvmalloc_node_noprof) *__kvmalloc_node_noprof;
typeof(__kmalloc_cache_noprof) *__kmalloc_cache_noprof;
#ifdef HAVE_KVREALLOC
typeof(kvrealloc_noprof) *kvrealloc_noprof;
#endif
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(6, 10, 0)
typeof(kmalloc_noprof) *kmalloc_noprof;
typeof(krealloc_noprof) *krealloc_noprof;
typeof(kzalloc_noprof) *kzalloc_noprof;
typeof(kvmalloc_node_noprof) *kvmalloc_node_noprof;
typeof(kmalloc_trace_noprof) *kmalloc_trace_noprof;
#ifdef HAVE_KVREALLOC
typeof(kvrealloc_noprof) *kvrealloc_noprof;
#endif
#else /* <6.10.0 */
typeof(kmalloc) *kmalloc;
typeof(krealloc) *krealloc;
#ifdef HAVE_KVMALLOC
typeof(kvmalloc_node) *kvmalloc_node;
#endif
#ifdef HAVE_KVREALLOC
typeof(kvrealloc) *kvrealloc;
#endif
#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 1, 0) || \
(defined(RHEL_MAJOR) && \
((RHEL_MAJOR > 9) || ((RHEL_MAJOR == 9) && (RHEL_MINOR >= 5))))
typeof(kmalloc_trace) *kmalloc_trace;
#else
typeof(kmem_cache_alloc_trace) *kmem_cache_alloc_trace;
typeof(kmalloc_order_trace) *kmalloc_order_trace;
#endif
#endif /* <6.10.0 */
#ifdef HAVE_KVMALLOC
typeof(kvfree) *kvfree;
#endif
typeof(kfree) *kfree;
typeof(ksize) *ksize;
#ifndef LINUXKM_LKCAPI_REGISTER_HASH_DRBG_DEFAULT
typeof(get_random_bytes) *get_random_bytes;
#endif
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
typeof(getnstimeofday) *getnstimeofday;
#elif LINUX_VERSION_CODE < KERNEL_VERSION(5, 0, 0)
typeof(current_kernel_time64) *current_kernel_time64;
#else
typeof(ktime_get_coarse_real_ts64) *ktime_get_coarse_real_ts64;
#endif
struct task_struct *(*get_current)(void);
#ifdef WOLFSSL_USE_SAVE_VECTOR_REGISTERS
#ifdef CONFIG_X86
typeof(allocate_wolfcrypt_linuxkm_fpu_states) *allocate_wolfcrypt_linuxkm_fpu_states;
typeof(wc_can_save_vector_registers_x86) *wc_can_save_vector_registers_x86;
typeof(free_wolfcrypt_linuxkm_fpu_states) *free_wolfcrypt_linuxkm_fpu_states;
typeof(wc_restore_vector_registers_x86) *wc_restore_vector_registers_x86;
typeof(wc_save_vector_registers_x86) *wc_save_vector_registers_x86;
#else /* !CONFIG_X86 */
#error WOLFSSL_USE_SAVE_VECTOR_REGISTERS is set for an unimplemented architecture.
#endif /* arch */
#endif /* WOLFSSL_USE_SAVE_VECTOR_REGISTERS */
#ifndef CONFIG_PREEMPT_RT
typeof(__mutex_init) *__mutex_init;
#else
typeof(__rt_mutex_init) *__rt_mutex_init;
typeof(rt_mutex_base_init) *rt_mutex_base_init;
typeof(rt_spin_lock) *rt_spin_lock;
typeof(rt_spin_unlock) *rt_spin_unlock;
#endif
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
typeof(mutex_lock_nested) *mutex_lock_nested;
#else
typeof(mutex_lock) *mutex_lock;
#endif
typeof(mutex_unlock) *mutex_unlock;
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
typeof(mutex_destroy) *mutex_destroy;
#endif
#ifdef HAVE_FIPS
typeof(wolfCrypt_FIPS_first) *wolfCrypt_FIPS_first;
typeof(wolfCrypt_FIPS_last) *wolfCrypt_FIPS_last;
typeof(wolfCrypt_FIPS_ro_start) *wolfCrypt_FIPS_ro_start;
typeof(wolfCrypt_FIPS_ro_end) *wolfCrypt_FIPS_ro_end;
#if FIPS_VERSION3_GE(6,0,0)
#ifndef NO_AES
typeof(wolfCrypt_FIPS_AES_sanity) *wolfCrypt_FIPS_AES_sanity;
typeof(wolfCrypt_FIPS_aes_ro_sanity) *wolfCrypt_FIPS_aes_ro_sanity;
#if defined(WOLFSSL_CMAC) && defined(WOLFSSL_AES_DIRECT)
typeof(wolfCrypt_FIPS_CMAC_sanity) *wolfCrypt_FIPS_CMAC_sanity;
typeof(wolfCrypt_FIPS_cmac_ro_sanity) *wolfCrypt_FIPS_cmac_ro_sanity;
#endif
#endif
#ifndef NO_DH
typeof(wolfCrypt_FIPS_DH_sanity) *wolfCrypt_FIPS_DH_sanity;
typeof(wolfCrypt_FIPS_dh_ro_sanity) *wolfCrypt_FIPS_dh_ro_sanity;
#endif
#ifdef HAVE_ECC
typeof(wolfCrypt_FIPS_ECC_sanity) *wolfCrypt_FIPS_ECC_sanity;
typeof(wolfCrypt_FIPS_ecc_ro_sanity) *wolfCrypt_FIPS_ecc_ro_sanity;
#endif
#ifdef HAVE_ED25519
typeof(wolfCrypt_FIPS_ED25519_sanity) *wolfCrypt_FIPS_ED25519_sanity;
typeof(wolfCrypt_FIPS_ed25519_ro_sanity) *wolfCrypt_FIPS_ed25519_ro_sanity;
#endif
#ifdef HAVE_ED448
typeof(wolfCrypt_FIPS_ED448_sanity) *wolfCrypt_FIPS_ED448_sanity;
typeof(wolfCrypt_FIPS_ed448_ro_sanity) *wolfCrypt_FIPS_ed448_ro_sanity;
#endif
typeof(wolfCrypt_FIPS_HMAC_sanity) *wolfCrypt_FIPS_HMAC_sanity;
typeof(wolfCrypt_FIPS_hmac_ro_sanity) *wolfCrypt_FIPS_hmac_ro_sanity;
#ifndef NO_KDF
typeof(wolfCrypt_FIPS_KDF_sanity) *wolfCrypt_FIPS_KDF_sanity;
typeof(wolfCrypt_FIPS_kdf_ro_sanity) *wolfCrypt_FIPS_kdf_ro_sanity;
#endif
#ifdef HAVE_PBKDF2
typeof(wolfCrypt_FIPS_PBKDF_sanity) *wolfCrypt_FIPS_PBKDF_sanity;
typeof(wolfCrypt_FIPS_pbkdf_ro_sanity) *wolfCrypt_FIPS_pbkdf_ro_sanity;
#endif
#ifdef HAVE_HASHDRBG
typeof(wolfCrypt_FIPS_DRBG_sanity) *wolfCrypt_FIPS_DRBG_sanity;
typeof(wolfCrypt_FIPS_drbg_ro_sanity) *wolfCrypt_FIPS_drbg_ro_sanity;
#endif
#ifndef NO_RSA
typeof(wolfCrypt_FIPS_RSA_sanity) *wolfCrypt_FIPS_RSA_sanity;
typeof(wolfCrypt_FIPS_rsa_ro_sanity) *wolfCrypt_FIPS_rsa_ro_sanity;
#endif
#ifndef NO_SHA
typeof(wolfCrypt_FIPS_SHA_sanity) *wolfCrypt_FIPS_SHA_sanity;
typeof(wolfCrypt_FIPS_sha_ro_sanity) *wolfCrypt_FIPS_sha_ro_sanity;
#endif
#ifndef NO_SHA256
typeof(wolfCrypt_FIPS_SHA256_sanity) *wolfCrypt_FIPS_SHA256_sanity;
typeof(wolfCrypt_FIPS_sha256_ro_sanity) *wolfCrypt_FIPS_sha256_ro_sanity;
#endif
#ifdef WOLFSSL_SHA512
typeof(wolfCrypt_FIPS_SHA512_sanity) *wolfCrypt_FIPS_SHA512_sanity;
typeof(wolfCrypt_FIPS_sha512_ro_sanity) *wolfCrypt_FIPS_sha512_ro_sanity;
#endif
#ifdef WOLFSSL_SHA3
typeof(wolfCrypt_FIPS_SHA3_sanity) *wolfCrypt_FIPS_SHA3_sanity;
typeof(wolfCrypt_FIPS_sha3_ro_sanity) *wolfCrypt_FIPS_sha3_ro_sanity;
#endif
typeof(wolfCrypt_FIPS_FT_sanity) *wolfCrypt_FIPS_FT_sanity;
typeof(wolfCrypt_FIPS_ft_ro_sanity) *wolfCrypt_FIPS_ft_ro_sanity;
typeof(wolfCrypt_FIPS_f_ro_sanity) *wolfCrypt_FIPS_f_ro_sanity;
typeof(wc_RunAllCast_fips) *wc_RunAllCast_fips;
#endif
#endif
#if !defined(WOLFCRYPT_ONLY) && !defined(NO_CERTS)
typeof(GetCA) *GetCA;
#ifndef NO_SKID
typeof(GetCAByName) *GetCAByName;
#ifdef HAVE_OCSP
typeof(GetCAByKeyHash) *GetCAByKeyHash;
#endif /* HAVE_OCSP */
#endif /* NO_SKID */
#ifdef WOLFSSL_AKID_NAME
typeof(GetCAByAKID) *GetCAByAKID;
#endif /* WOLFSSL_AKID_NAME */
#if defined(OPENSSL_EXTRA) || defined(OPENSSL_EXTRA_X509_SMALL)
typeof(wolfSSL_X509_NAME_add_entry_by_NID) *wolfSSL_X509_NAME_add_entry_by_NID;
typeof(wolfSSL_X509_NAME_free) *wolfSSL_X509_NAME_free;
typeof(wolfSSL_X509_NAME_new_ex) *wolfSSL_X509_NAME_new_ex;
#endif /* OPENSSL_EXTRA || OPENSSL_EXTRA_X509_SMALL */
#endif /* !WOLFCRYPT_ONLY && !NO_CERTS */
typeof(dump_stack) *dump_stack;
#ifdef CONFIG_ARM64
#ifndef CONFIG_ARCH_TEGRA
#ifdef WC_CONTAINERIZE_THIS
/* alt_cb_patch_nops and queued_spin_lock_slowpath are defined early
* to allow shimming in system headers, but now we need the native
* ones.
*/
#undef alt_cb_patch_nops
typeof(my__alt_cb_patch_nops) *alt_cb_patch_nops;
#undef queued_spin_lock_slowpath
typeof(my__queued_spin_lock_slowpath) *queued_spin_lock_slowpath;
#else
typeof(alt_cb_patch_nops) *alt_cb_patch_nops;
typeof(queued_spin_lock_slowpath) *queued_spin_lock_slowpath;
#endif
#endif
#endif
typeof(preempt_count) *preempt_count;
#ifndef _raw_spin_lock_irqsave
typeof(_raw_spin_lock_irqsave) *_raw_spin_lock_irqsave;
#endif
#ifndef _raw_spin_trylock
typeof(_raw_spin_trylock) *_raw_spin_trylock;
#endif
#ifndef _raw_spin_unlock_irqrestore
typeof(_raw_spin_unlock_irqrestore) *_raw_spin_unlock_irqrestore;
#endif
typeof(_cond_resched) *_cond_resched;
#ifndef WOLFSSL_LINUXKM_USE_MUTEXES
typeof(wc_lkm_LockMutex) *wc_lkm_LockMutex;
#endif
typeof(wc_linuxkm_sig_ignore_begin) *wc_linuxkm_sig_ignore_begin;
typeof(wc_linuxkm_sig_ignore_end) *wc_linuxkm_sig_ignore_end;
typeof(wc_linuxkm_check_for_intr_signals) *wc_linuxkm_check_for_intr_signals;
typeof(wc_linuxkm_relax_long_loop) *wc_linuxkm_relax_long_loop;
const void *_last_slot;
};
extern const struct wolfssl_linuxkm_pie_redirect_table *wolfssl_linuxkm_get_pie_redirect_table(void);
extern struct wolfssl_linuxkm_pie_redirect_table wolfssl_linuxkm_pie_redirect_table;
#if defined(WC_PIE_INDIRECT_SYM)
/* keep user-supplied override definition. */
#elif defined(WC_PIE_INDIRECT_SYM_BY_FUNC_ONLY) || \
defined(WC_PIE_INDIRECT_SYM_BY_DIRECT_TABLE_READ)
/* keep user-supplied override method. */
#elif defined(CONFIG_X86)
#define WC_PIE_INDIRECT_SYM_BY_DIRECT_TABLE_READ
#elif defined(CONFIG_ARM64)
/* direct access to wolfssl_linuxkm_pie_redirect_table.x on aarch64
* produces GOT relocations, e.g. R_AARCH64_LD64_GOT_LO12_NC.
*/
#define WC_PIE_INDIRECT_SYM_BY_FUNC_ONLY
#else
/* for other archs, by default use the safe way. */
#define WC_PIE_INDIRECT_SYM_BY_FUNC_ONLY
#endif
#if defined(WC_PIE_INDIRECT_SYM)
/* keep user-supplied override definition. */
#elif defined(WC_PIE_INDIRECT_SYM_BY_FUNC_ONLY)
static const struct wolfssl_linuxkm_pie_redirect_table *wolfssl_linuxkm_pie_redirect_table_local_ptr = NULL;
static inline const struct wolfssl_linuxkm_pie_redirect_table *wolfssl_linuxkm_get_pie_redirect_table_local(void) {
if (wolfssl_linuxkm_pie_redirect_table_local_ptr == NULL)
wolfssl_linuxkm_pie_redirect_table_local_ptr = wolfssl_linuxkm_get_pie_redirect_table();
return wolfssl_linuxkm_pie_redirect_table_local_ptr;
}
#define WC_PIE_INDIRECT_SYM(x) (wolfssl_linuxkm_get_pie_redirect_table_local()->x)
#elif defined(WC_PIE_INDIRECT_SYM_BY_DIRECT_TABLE_READ)
#define WC_PIE_INDIRECT_SYM(x) (wolfssl_linuxkm_pie_redirect_table.x)
#else
#error no WC_PIE_INDIRECT_SYM method defined.
#endif
#ifdef WC_CONTAINERIZE_THIS
#define wc_linuxkm_normalize_relocations \
WC_PIE_INDIRECT_SYM(wc_linuxkm_normalize_relocations)
#ifndef __ARCH_MEMCMP_NO_REDIRECT
#define memcmp WC_PIE_INDIRECT_SYM(memcmp)
#endif
#ifndef __ARCH_MEMCPY_NO_REDIRECT
#define memcpy WC_PIE_INDIRECT_SYM(memcpy)
#endif
#ifndef __ARCH_MEMSET_NO_REDIRECT
#define memset WC_PIE_INDIRECT_SYM(memset)
#endif
#ifndef __ARCH_MEMMOVE_NO_REDIRECT
#define memmove WC_PIE_INDIRECT_SYM(memmove)
#endif
#ifndef __ARCH_STRCMP_NO_REDIRECT
#define strcmp WC_PIE_INDIRECT_SYM(strcmp)
#endif
#ifndef __ARCH_STRNCMP_NO_REDIRECT
#define strncmp WC_PIE_INDIRECT_SYM(strncmp)
#endif
#ifndef __ARCH_STRCASECMP_NO_REDIRECT
#define strcasecmp WC_PIE_INDIRECT_SYM(strcasecmp)
#endif
#ifndef __ARCH_STRNCASECMP_NO_REDIRECT
#define strncasecmp WC_PIE_INDIRECT_SYM(strncasecmp)
#endif
#ifndef __ARCH_STRLEN_NO_REDIRECT
#define strlen WC_PIE_INDIRECT_SYM(strlen)
#endif
#ifndef __ARCH_STRSTR_NO_REDIRECT
#define strstr WC_PIE_INDIRECT_SYM(strstr)
#endif
#ifndef __ARCH_STRNCPY_NO_REDIRECT
#define strncpy WC_PIE_INDIRECT_SYM(strncpy)
#endif
#ifndef __ARCH_STRNCAT_NO_REDIRECT
#define strncat WC_PIE_INDIRECT_SYM(strncat)
#endif
#define kstrtoll WC_PIE_INDIRECT_SYM(kstrtoll)
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0)) || \
(defined(RHEL_MAJOR) && \
((RHEL_MAJOR > 9) || ((RHEL_MAJOR == 9) && (RHEL_MINOR >= 5))))
#define _printk WC_PIE_INDIRECT_SYM(_printk)
#else
#define printk WC_PIE_INDIRECT_SYM(printk)
#endif
#ifdef CONFIG_FORTIFY_SOURCE
#define __warn_printk WC_PIE_INDIRECT_SYM(__warn_printk)
#endif
#define snprintf WC_PIE_INDIRECT_SYM(snprintf)
#define _ctype WC_PIE_INDIRECT_SYM(_ctype)
#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0)
/* see include/linux/alloc_tag.h and include/linux/slab.h */
#define kmalloc_noprof WC_PIE_INDIRECT_SYM(kmalloc_noprof)
#define krealloc_node_align_noprof WC_PIE_INDIRECT_SYM(krealloc_node_align_noprof)
#define kzalloc_noprof WC_PIE_INDIRECT_SYM(kzalloc_noprof)
#define __kvmalloc_node_noprof WC_PIE_INDIRECT_SYM(__kvmalloc_node_noprof)
#define __kmalloc_cache_noprof WC_PIE_INDIRECT_SYM(__kmalloc_cache_noprof)
#ifdef HAVE_KVREALLOC
#define kvrealloc_node_align_noprof WC_PIE_INDIRECT_SYM(kvrealloc_node_align_noprof)
#endif
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(6, 11, 0)
/* see include/linux/alloc_tag.h and include/linux/slab.h */
#define kmalloc_noprof WC_PIE_INDIRECT_SYM(kmalloc_noprof)
#define krealloc_noprof WC_PIE_INDIRECT_SYM(krealloc_noprof)
#define kzalloc_noprof WC_PIE_INDIRECT_SYM(kzalloc_noprof)
#define __kvmalloc_node_noprof WC_PIE_INDIRECT_SYM(__kvmalloc_node_noprof)
#define __kmalloc_cache_noprof WC_PIE_INDIRECT_SYM(__kmalloc_cache_noprof)
#ifdef HAVE_KVREALLOC
#define kvrealloc_noprof WC_PIE_INDIRECT_SYM(kvrealloc_noprof)
#endif
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(6, 10, 0)
/* see include/linux/alloc_tag.h and include/linux/slab.h */
#define kmalloc_noprof WC_PIE_INDIRECT_SYM(kmalloc_noprof)
#define krealloc_noprof WC_PIE_INDIRECT_SYM(krealloc_noprof)
#define kzalloc_noprof WC_PIE_INDIRECT_SYM(kzalloc_noprof)
#define kvmalloc_node_noprof WC_PIE_INDIRECT_SYM(kvmalloc_node_noprof)
#define kmalloc_trace_noprof WC_PIE_INDIRECT_SYM(kmalloc_trace_noprof)
#ifdef HAVE_KVREALLOC
#define kvrealloc_noprof WC_PIE_INDIRECT_SYM(kvrealloc_noprof)
#endif
#else /* <6.10.0 */
#define kmalloc WC_PIE_INDIRECT_SYM(kmalloc)
#define krealloc WC_PIE_INDIRECT_SYM(krealloc)
#define kzalloc(size, flags) kmalloc(size, (flags) | __GFP_ZERO)
#ifdef HAVE_KVMALLOC
#define kvmalloc_node WC_PIE_INDIRECT_SYM(kvmalloc_node)
#endif
#ifdef HAVE_KVREALLOC
#define kvrealloc WC_PIE_INDIRECT_SYM(kvrealloc)
#endif
#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 1, 0) || \
(defined(RHEL_MAJOR) && \
((RHEL_MAJOR > 9) || ((RHEL_MAJOR == 9) && (RHEL_MINOR >= 5))))
#define kmalloc_trace WC_PIE_INDIRECT_SYM(kmalloc_trace)
#else
#define kmem_cache_alloc_trace WC_PIE_INDIRECT_SYM(kmem_cache_alloc_trace)
#define kmalloc_order_trace WC_PIE_INDIRECT_SYM(kmalloc_order_trace)
#endif
#endif /* <6.10.0 */
#define kfree WC_PIE_INDIRECT_SYM(kfree)
#ifdef HAVE_KVMALLOC
#define kvfree WC_PIE_INDIRECT_SYM(kvfree)
#endif
#define ksize WC_PIE_INDIRECT_SYM(ksize)
#ifndef LINUXKM_LKCAPI_REGISTER_HASH_DRBG_DEFAULT
#define get_random_bytes WC_PIE_INDIRECT_SYM(get_random_bytes)
#endif
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
#define getnstimeofday WC_PIE_INDIRECT_SYM(getnstimeofday)
#elif LINUX_VERSION_CODE < KERNEL_VERSION(5, 0, 0)
#define current_kernel_time64 WC_PIE_INDIRECT_SYM(current_kernel_time64)
#else
#define ktime_get_coarse_real_ts64 WC_PIE_INDIRECT_SYM(ktime_get_coarse_real_ts64)
#endif
#undef get_current
#define get_current WC_PIE_INDIRECT_SYM(get_current)
#if defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) && defined(CONFIG_X86)
#define allocate_wolfcrypt_linuxkm_fpu_states WC_PIE_INDIRECT_SYM(allocate_wolfcrypt_linuxkm_fpu_states)
#define wc_can_save_vector_registers_x86 WC_PIE_INDIRECT_SYM(wc_can_save_vector_registers_x86)
#define free_wolfcrypt_linuxkm_fpu_states WC_PIE_INDIRECT_SYM(free_wolfcrypt_linuxkm_fpu_states)
#define wc_restore_vector_registers_x86 WC_PIE_INDIRECT_SYM(wc_restore_vector_registers_x86)
#define wc_save_vector_registers_x86 WC_PIE_INDIRECT_SYM(wc_save_vector_registers_x86)
#elif defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS)
#error WOLFSSL_USE_SAVE_VECTOR_REGISTERS is set for an unimplemented architecture.
#endif /* WOLFSSL_USE_SAVE_VECTOR_REGISTERS */
#ifndef CONFIG_PREEMPT_RT
#define __mutex_init WC_PIE_INDIRECT_SYM(__mutex_init)
#else
/* On RT kernels, __mutex_init is a macro pointing to __rt_mutex_init */
#undef __mutex_init
#define __rt_mutex_init WC_PIE_INDIRECT_SYM(__rt_mutex_init)
#define __mutex_init(mutex, name, key) __rt_mutex_init(mutex, name, key)
#define rt_mutex_base_init WC_PIE_INDIRECT_SYM(rt_mutex_base_init)
#define rt_spin_lock WC_PIE_INDIRECT_SYM(rt_spin_lock)
#define rt_spin_unlock WC_PIE_INDIRECT_SYM(rt_spin_unlock)
#endif
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
#define mutex_lock_nested WC_PIE_INDIRECT_SYM(mutex_lock_nested)
#else
#define mutex_lock WC_PIE_INDIRECT_SYM(mutex_lock)
#endif
#define mutex_unlock WC_PIE_INDIRECT_SYM(mutex_unlock)
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
#define mutex_destroy WC_PIE_INDIRECT_SYM(mutex_destroy)
#endif
/* per linux/ctype.h, tolower() and toupper() are macros bound to static inlines
* that use macros that bring in the _ctype global. for WC_CONTAINERIZE_THIS,
* this needs to be masked out.
*/
#undef tolower
#undef toupper
#define tolower(c) (islower(c) ? (c) : ((c) + ('a'-'A')))
#define toupper(c) (isupper(c) ? (c) : ((c) - ('a'-'A')))
#if !defined(WOLFCRYPT_ONLY) && !defined(NO_CERTS)
#define GetCA WC_PIE_INDIRECT_SYM(GetCA)
#ifndef NO_SKID
#define GetCAByName WC_PIE_INDIRECT_SYM(GetCAByName)
#ifdef HAVE_OCSP
#define GetCAByKeyHash WC_PIE_INDIRECT_SYM(GetCAByKeyHash)
#endif /* HAVE_OCSP */
#endif /* NO_SKID */
#ifdef WOLFSSL_AKID_NAME
#ifdef WOLFSSL_API_PREFIX_MAP
#define wolfSSL_GetCAByAKID WC_PIE_INDIRECT_SYM(wolfSSL_GetCAByAKID)
#else
#define GetCAByAKID WC_PIE_INDIRECT_SYM(GetCAByAKID)
#endif
#endif
#if defined(OPENSSL_EXTRA) || defined(OPENSSL_EXTRA_X509_SMALL)
#define wolfSSL_X509_NAME_add_entry_by_NID WC_PIE_INDIRECT_SYM(wolfSSL_X509_NAME_add_entry_by_NID)
#define wolfSSL_X509_NAME_free WC_PIE_INDIRECT_SYM(wolfSSL_X509_NAME_free)
#define wolfSSL_X509_NAME_new_ex WC_PIE_INDIRECT_SYM(wolfSSL_X509_NAME_new_ex)
#endif /* OPENSSL_EXTRA || OPENSSL_EXTRA_X509_SMALL */
#endif /* !WOLFCRYPT_ONLY && !NO_CERTS */
#define dump_stack WC_PIE_INDIRECT_SYM(dump_stack)
#undef preempt_count /* just in case -- not a macro on x86. */
#define preempt_count WC_PIE_INDIRECT_SYM(preempt_count)
#ifndef WOLFSSL_LINUXKM_USE_MUTEXES
#ifndef _raw_spin_lock_irqsave
#define _raw_spin_lock_irqsave WC_PIE_INDIRECT_SYM(_raw_spin_lock_irqsave)
#endif
#ifndef _raw_spin_trylock
#define _raw_spin_trylock WC_PIE_INDIRECT_SYM(_raw_spin_trylock)
#endif
#ifndef _raw_spin_unlock_irqrestore
#define _raw_spin_unlock_irqrestore WC_PIE_INDIRECT_SYM(_raw_spin_unlock_irqrestore)
#endif
#endif
#define _cond_resched WC_PIE_INDIRECT_SYM(_cond_resched)
/* this is defined in linux/spinlock.h as an inline that calls the unshimmed
* raw_spin_unlock_irqrestore(). use a macro here to supersede it.
* Note: On PREEMPT_RT kernels, spinlock_t doesn't have rlock member,
* so we skip this redefinition and use the kernel's native implementation.
*/
#ifndef CONFIG_PREEMPT_RT
#define spin_unlock_irqrestore(lock, flags) raw_spin_unlock_irqrestore(&((lock)->rlock), flags)
#else
/* Undo internal wolfSSL PIE macro rewriting */
#ifdef rt_spin_unlock
#undef rt_spin_unlock
#endif
#ifdef rt_spin_lock
#undef rt_spin_lock
#endif
static inline int wolfssl_spin_unlock_irqrestore_rt(spinlock_t *lock,
unsigned long flags)
{
(void)flags; /* rt_spin_unlock ignores flags */
WC_PIE_INDIRECT_SYM(rt_spin_unlock)(lock);
return 0;
}
#undef spin_unlock_irqrestore
#define spin_unlock_irqrestore(lock, flags) \
wolfssl_spin_unlock_irqrestore_rt((lock), (flags))
#endif
#define wc_linuxkm_sig_ignore_begin WC_PIE_INDIRECT_SYM(wc_linuxkm_sig_ignore_begin);
#define wc_linuxkm_sig_ignore_end WC_PIE_INDIRECT_SYM(wc_linuxkm_sig_ignore_end);
#define wc_linuxkm_check_for_intr_signals WC_PIE_INDIRECT_SYM(wc_linuxkm_check_for_intr_signals)
#define wc_linuxkm_relax_long_loop WC_PIE_INDIRECT_SYM(wc_linuxkm_relax_long_loop)
#endif /* WC_CONTAINERIZE_THIS */
#endif /* WC_SYM_RELOC_TABLES */
/* remove this multifariously conflicting macro, picked up from
* Linux arch/<arch>/include/asm/current.h.
*/
#ifndef WOLFSSL_LINUXKM_NEED_LINUX_CURRENT
#undef current
#endif
/* min() and max() in linux/kernel.h over-aggressively type-check, producing
* myriad spurious -Werrors throughout the codebase.
*/
#undef min
#undef max
/* work around namespace conflict between wolfssl/internal.h (enum HandShakeType)
* and linux/key.h (extern int()).
*/
#define key_update wc_key_update
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0)) || \
(defined(RHEL_MAJOR) && \
((RHEL_MAJOR > 9) || ((RHEL_MAJOR == 9) && (RHEL_MINOR >= 5))))
#define wc_km_printf(format, args...) _printk(KERN_INFO "wolfssl: %s(): " format, __func__, ## args)
#else
#define wc_km_printf(format, args...) printk(KERN_INFO "wolfssl: %s(): " format, __func__, ## args)
#endif
#define printf(...) wc_km_printf(__VA_ARGS__)
#ifdef HAVE_FIPS
extern void fipsEntry(void);
#endif
/* suppress false-positive "writing 1 byte into a region of size 0" warnings
* building old kernels with new gcc:
*/
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
_Pragma("GCC diagnostic ignored \"-Wstringop-overflow\"");
#endif
/* includes are all above, with incompatible warnings masked out. */
#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 5, 0)
typedef __kernel_time_t time_t;
#else
typedef __kernel_time64_t time_t;
#endif
extern time_t time(time_t * timer);
#define XTIME time
#define WOLFSSL_GMTIME
#define XGMTIME(c, t) gmtime(c)
#define NO_TIMEVAL 1
#endif /* BUILDING_WOLFSSL */
#if !defined(BUILDING_WOLFSSL)
/* some caller code needs these. */
#if defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS)
#if defined(CONFIG_X86)
WOLFSSL_API __must_check int wc_can_save_vector_registers_x86(void);
WOLFSSL_API __must_check int wc_save_vector_registers_x86(enum wc_svr_flags flags);
WOLFSSL_API void wc_restore_vector_registers_x86(enum wc_svr_flags flags);
#ifndef DISABLE_VECTOR_REGISTERS
#define DISABLE_VECTOR_REGISTERS() wc_save_vector_registers_x86(WC_SVR_FLAG_INHIBIT)
#endif
#ifndef REENABLE_VECTOR_REGISTERS
#define REENABLE_VECTOR_REGISTERS() wc_restore_vector_registers_x86(WC_SVR_FLAG_INHIBIT)
#endif
#else /* !CONFIG_X86 */
#error WOLFSSL_USE_SAVE_VECTOR_REGISTERS is set for an unimplemented architecture.
#endif /* !CONFIG_X86 */
#endif /* WOLFSSL_USE_SAVE_VECTOR_REGISTERS */
#endif /* !BUILDING_WOLFSSL */
/* Copied from wc_port.h */
#if defined(HAVE_FIPS) && !defined(WOLFSSL_API_PREFIX_MAP)
/* For FIPS keep the function names the same */
#define wc_InitMutex InitMutex
#define wc_FreeMutex FreeMutex
#define wc_LockMutex LockMutex
#define wc_UnLockMutex UnLockMutex
#endif /* HAVE_FIPS */
#ifdef WOLFSSL_LINUXKM_USE_MUTEXES
#define WC_MUTEX_OPS_INLINE
#ifdef LINUXKM_LKCAPI_REGISTER
/* must use spin locks when registering implementations with the
* kernel, because mutexes are forbidden when calling with nonzero
* irq_count().
*/
#error WOLFSSL_LINUXKM_USE_MUTEXES is incompatible with LINUXKM_LKCAPI_REGISTER.
#endif
/* if BUILDING_WOLFSSL, mutex.h will have already been included
* recursively above, with the bevy of warnings suppressed, and the
* below include will be a redundant no-op.
*/
#include <linux/mutex.h>
typedef struct mutex wolfSSL_Mutex;
#define WOLFSSL_MUTEX_INITIALIZER(lockname) __MUTEX_INITIALIZER(lockname)
/* Linux kernel mutex routines are voids, alas. */
static inline int wc_InitMutex(wolfSSL_Mutex* m)
{
mutex_init(m);
return 0;
}
static inline int wc_FreeMutex(wolfSSL_Mutex* m)
{
mutex_destroy(m);
return 0;
}
static __must_check inline int wc_LockMutex(wolfSSL_Mutex* m)
{
if (in_nmi() || hardirq_count() || in_softirq())
return -1;
mutex_lock(m);
return 0;
}
static inline int wc_UnLockMutex(wolfSSL_Mutex* m)
{
mutex_unlock(m);
return 0;
}
#else
#define WC_MUTEX_OPS_INLINE
/* if BUILDING_WOLFSSL, spinlock.h will have already been included
* recursively above, with the bevy of warnings suppressed, and the
* below include will be a redundant no-op.
*/
#include <linux/spinlock.h>
typedef struct wolfSSL_Mutex {
spinlock_t lock;
unsigned long irq_flags;
} wolfSSL_Mutex;
#define WOLFSSL_MUTEX_INITIALIZER(lockname) { .lock =__SPIN_LOCK_UNLOCKED(lockname), .irq_flags = 0 }
static __always_inline int wc_InitMutex(wolfSSL_Mutex* m)
{
/* Tegra vendor kernels do not support assignment of __SPIN_LOCK_UNLOCKED() */
# ifndef CONFIG_ARCH_TEGRA
m->lock = __SPIN_LOCK_UNLOCKED(m);
# else
spin_lock_init(&m->lock);
#endif
m->irq_flags = 0;
return 0;
}
static __always_inline int wc_FreeMutex(wolfSSL_Mutex* m)
{
(void)m;
return 0;
}
#ifdef WC_CONTAINERIZE_THIS
/* wc_lkm_LockMutex() can't be used inline in WC_CONTAINERIZE_THIS objects, due to
* direct access to pv_ops.
*/
static __must_check __always_inline int wc_LockMutex(wolfSSL_Mutex *m)
{
return WC_PIE_INDIRECT_SYM(wc_lkm_LockMutex)(m);
}
#else /* !WC_CONTAINERIZE_THIS */
static __must_check __always_inline int wc_LockMutex(wolfSSL_Mutex *m)
{
return wc_lkm_LockMutex(m);
}
#endif /* !WC_CONTAINERIZE_THIS */
static __always_inline int wc_UnLockMutex(wolfSSL_Mutex* m)
{
spin_unlock_irqrestore(&m->lock, m->irq_flags);
return 0;
}
#endif
/* Undo copied defines from wc_port.h, to avoid redefinition warnings. */
#ifdef HAVE_FIPS
#undef wc_InitMutex
#undef wc_FreeMutex
#undef wc_LockMutex
#undef wc_UnLockMutex
#endif /* HAVE_FIPS */
/* prevent gcc's mm_malloc.h from being included, since it unconditionally
* includes stdlib.h, which is kernel-incompatible.
*/
#define _MM_MALLOC_H_INCLUDED
#ifndef BUILDING_WOLFSSL
#include <linux/slab.h>
#if defined(USE_KVMALLOC) && (LINUX_VERSION_CODE < KERNEL_VERSION(5, 16, 0))
#include <linux/mm.h>
#endif
#endif
/* fun fact: since linux commit 59bb47985c, kmalloc with power-of-2 size is
* aligned to the size.
*/
#define WC_LINUXKM_ROUND_UP_P_OF_2(x) ( \
{ \
size_t _alloc_sz = (x); \
if ((_alloc_sz < 8192) && (_alloc_sz != 0)) \
_alloc_sz = 1UL << \
((sizeof(_alloc_sz) * 8UL) - __builtin_clzl(_alloc_sz - 1)); \
_alloc_sz; \
})
#ifdef USE_KVMALLOC
#define malloc(size) kvmalloc_node(WC_LINUXKM_ROUND_UP_P_OF_2(size), (preempt_count() == 0 ? GFP_KERNEL : GFP_ATOMIC), NUMA_NO_NODE)
#define free(ptr) kvfree(ptr)
#ifdef USE_KVREALLOC
#define realloc(ptr, newsize) kvrealloc(ptr, WC_LINUXKM_ROUND_UP_P_OF_2(newsize), (preempt_count() == 0 ? GFP_KERNEL : GFP_ATOMIC))
#else
#define realloc(ptr, newsize) ((void)(ptr), (void)(newsize), NULL)
#endif
#else
#define malloc(size) kmalloc(WC_LINUXKM_ROUND_UP_P_OF_2(size), (preempt_count() == 0 ? GFP_KERNEL : GFP_ATOMIC))
#define free(ptr) kfree(ptr)
#define realloc(ptr, newsize) krealloc(ptr, WC_LINUXKM_ROUND_UP_P_OF_2(newsize), (preempt_count() == 0 ? GFP_KERNEL : GFP_ATOMIC))
#endif
#ifndef static_assert
#define static_assert(expr, ...) __static_assert(expr, ##__VA_ARGS__, #expr)
#define __static_assert(expr, msg, ...) _Static_assert(expr, msg)
#endif
#ifdef WOLFSSL_TRACK_MEMORY
#define XMALLOC(s, h, t) ({(void)(h); (void)(t); wolfSSL_Malloc(s);})
#ifdef WOLFSSL_XFREE_NO_NULLNESS_CHECK
#define XFREE(p, h, t) ({(void)(h); (void)(t); wolfSSL_Free(p);})
#else
#define XFREE(p, h, t) ({void* _xp; (void)(h); _xp = (p); if(_xp) wolfSSL_Free(_xp);})
#endif
#define XREALLOC(p, n, h, t) ({(void)(h); (void)(t); wolfSSL_Realloc(p, n);})
#else
#if !defined(XMALLOC_USER) && !defined(XMALLOC_OVERRIDE)
#define XMALLOC(s, h, t) ({(void)(h); (void)(t); malloc(s);})
#ifdef WOLFSSL_XFREE_NO_NULLNESS_CHECK
#define XFREE(p, h, t) ({(void)(h); (void)(t); free(p);})
#else
#define XFREE(p, h, t) ({void* _xp; (void)(h); (void)(t); _xp = (p); if(_xp) free(_xp);})
#endif
#if defined(USE_KVREALLOC) || !defined(USE_KVMALLOC)
#define XREALLOC(p, n, h, t) ({(void)(h); (void)(t); realloc(p, n);})
#endif
#endif /* !XMALLOC_USER && !XMALLOC_OVERRIDE */
#endif
#include <linux/limits.h>
#ifndef INT32_MAX
#define INT32_MAX INT_MAX
#endif
#ifndef UINT32_MAX
#define UINT32_MAX UINT_MAX
#endif
/* Linux headers define these using C expressions, but we need
* them to be evaluable by the preprocessor, for use in sp_int.h.
*/
#if BITS_PER_LONG == 64
static_assert(sizeof(ULONG_MAX) == 8,
"BITS_PER_LONG is 64, but ULONG_MAX is not.");
#undef UCHAR_MAX
#define UCHAR_MAX 255
#undef USHRT_MAX
#define USHRT_MAX 65535
#undef UINT_MAX
#define UINT_MAX 4294967295U
#undef ULONG_MAX
#define ULONG_MAX 18446744073709551615UL
#undef ULLONG_MAX
#define ULLONG_MAX ULONG_MAX
#undef INT_MAX
#define INT_MAX 2147483647
#undef LONG_MAX
#define LONG_MAX 9223372036854775807L
#undef LLONG_MAX
#define LLONG_MAX LONG_MAX
#elif BITS_PER_LONG == 32
static_assert(sizeof(ULONG_MAX) == 4,
"BITS_PER_LONG is 32, but ULONG_MAX is not.");
#undef UCHAR_MAX
#define UCHAR_MAX 255
#undef USHRT_MAX
#define USHRT_MAX 65535
#undef UINT_MAX
#define UINT_MAX 4294967295U
#undef ULONG_MAX
#define ULONG_MAX 4294967295UL
#undef INT_MAX
#define INT_MAX 2147483647
#undef LONG_MAX
#define LONG_MAX 2147483647L
#undef ULLONG_MAX
#undef LLONG_MAX
#if BITS_PER_LONG_LONG == 64
#define ULLONG_MAX 18446744073709551615UL
#define LLONG_MAX 9223372036854775807L
#else
#undef NO_64BIT
#define NO_64BIT
#define ULLONG_MAX ULONG_MAX
#define LLONG_MAX LONG_MAX
#endif
#else
#error unexpected BITS_PER_LONG value.
#endif
/* WC_DUMP_BACKTRACE_NONDEBUG is intended to dump a backtrace only if it hasn't
* already been dumped by the called function.
*/
#if defined(WOLFSSL_DEBUG_TRACE_ERROR_CODES) && defined(WOLFSSL_DEBUG_BACKTRACE_ERROR_CODES)
#define WC_DUMP_BACKTRACE_NONDEBUG WC_DO_NOTHING
#else
#define WC_DUMP_BACKTRACE_NONDEBUG dump_stack()
#endif
#endif /* LINUXKM_WC_PORT_H */
Reference in New Issue View Git Blame Copy Permalink