diff --git a/linuxkm/Kbuild b/linuxkm/Kbuild index 57ad487cf..3133ea8fc 100644 --- a/linuxkm/Kbuild +++ b/linuxkm/Kbuild @@ -90,6 +90,11 @@ ifeq "$(ENABLED_LINUXKM_PIE)" "yes" $(obj)/linuxkm/module_hooks.o: ccflags-y += $(PIE_SUPPORT_FLAGS) endif +ifeq "$(ENABLED_LINUXKM_BENCHMARKS)" "yes" + $(obj)/linuxkm/module_hooks.o: ccflags-y = $(WOLFSSL_CFLAGS) $(CFLAGS_FPU_ENABLE) $(CFLAGS_SIMD_ENABLE) $(PIE_SUPPORT_FLAGS) + $(obj)/linuxkm/module_hooks.o: asflags-y = $(WOLFSSL_ASFLAGS) $(ASFLAGS_FPU_ENABLE_SIMD_DISABLE) +endif + asflags-y := $(WOLFSSL_ASFLAGS) $(ASFLAGS_FPUSIMD_DISABLE) # vectorized implementations that are kernel-safe are listed here. diff --git a/linuxkm/linuxkm_memory.c b/linuxkm/linuxkm_memory.c index ee30af987..31a7b9355 100644 --- a/linuxkm/linuxkm_memory.c +++ b/linuxkm/linuxkm_memory.c @@ -21,320 +21,262 @@ /* included by wolfcrypt/src/memory.c */ -#if defined(WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS) && defined(CONFIG_X86) - #ifdef LINUXKM_SIMD_IRQ - #if LINUX_VERSION_CODE < KERNEL_VERSION(5, 16, 0) - static union fpregs_state **wolfcrypt_linuxkm_fpu_states = NULL; - #else - static struct fpstate **wolfcrypt_linuxkm_fpu_states = NULL; - #endif - #else - static unsigned int *wolfcrypt_linuxkm_fpu_states = NULL; - #define WC_FPU_COUNT_MASK 0x7fffffffU - #define WC_FPU_SAVED_MASK 0x80000000U - #endif +#ifdef HAVE_KVMALLOC +/* adapted from kvrealloc() draft by Changli Gao, 2010-05-13 */ +void *lkm_realloc(void *ptr, size_t newsize) { + void *nptr; + size_t oldsize; - static WARN_UNUSED_RESULT inline int am_in_hard_interrupt_handler(void) - { - return (preempt_count() & (NMI_MASK | HARDIRQ_MASK)) != 0; + if (unlikely(newsize == 0)) { + kvfree(ptr); + return ZERO_SIZE_PTR; } - WARN_UNUSED_RESULT int allocate_wolfcrypt_linuxkm_fpu_states(void) - { - #ifdef LINUXKM_SIMD_IRQ - #if LINUX_VERSION_CODE < KERNEL_VERSION(5, 16, 0) - wolfcrypt_linuxkm_fpu_states = - (union fpregs_state **)kzalloc(nr_cpu_ids - * sizeof(struct fpu_state *), - GFP_KERNEL); - #else - wolfcrypt_linuxkm_fpu_states = - (struct fpstate **)kzalloc(nr_cpu_ids - * sizeof(struct fpstate *), - GFP_KERNEL); - #endif - #else - wolfcrypt_linuxkm_fpu_states = - (unsigned int *)kzalloc(nr_cpu_ids * sizeof(unsigned int), - GFP_KERNEL); - #endif + if (unlikely(ptr == NULL)) + return kvmalloc_node(newsize, GFP_KERNEL, NUMA_NO_NODE); - if (! wolfcrypt_linuxkm_fpu_states) { - pr_err("warning, allocation of %lu bytes for " - "wolfcrypt_linuxkm_fpu_states failed.\n", - nr_cpu_ids * sizeof(struct fpu_state *)); - return MEMORY_E; - } -#ifdef LINUXKM_SIMD_IRQ - { - typeof(nr_cpu_ids) i; - for (i=0; istate, which - * has stringent alignment requirements (64 byte cache - * line), but takes a pointer to the parent struct. work - * around this. - */ - #if LINUX_VERSION_CODE < KERNEL_VERSION(5, 14, 0) - struct fpu *fake_fpu_pointer = - (struct fpu *)(((char *)wolfcrypt_linuxkm_fpu_states[processor_id]) - - offsetof(struct fpu, state)); - copy_fpregs_to_fpstate(fake_fpu_pointer); - #elif LINUX_VERSION_CODE < KERNEL_VERSION(5, 16, 0) - struct fpu *fake_fpu_pointer = - (struct fpu *)(((char *)wolfcrypt_linuxkm_fpu_states[processor_id]) - - offsetof(struct fpu, state)); - save_fpregs_to_fpstate(fake_fpu_pointer); - #else - struct fpu *fake_fpu_pointer = - (struct fpu *)(((char *)wolfcrypt_linuxkm_fpu_states[processor_id]) - - offsetof(struct fpu, fpstate)); - save_fpregs_to_fpstate(fake_fpu_pointer); - #endif - } - /* mark the slot as used. */ - ((unsigned char *)wolfcrypt_linuxkm_fpu_states[processor_id])[PAGE_SIZE-1] = 1; - /* note, not preempt_enable()ing, mirroring kernel_fpu_begin() - * semantics, even though routine will have been entered already - * non-preemptable. - */ - return 0; - } else -#endif /* LINUXKM_SIMD_IRQ */ - { - preempt_enable(); - return BAD_STATE_E; - } + page = virt_to_head_page(ptr); + if (PageSlab(page) || PageCompound(page)) { + if (newsize < PAGE_SIZE) +#endif /* ! __PIE__ */ + return krealloc(ptr, newsize, GFP_KERNEL); +#ifndef __PIE__ + oldsize = ksize(ptr); } else { + oldsize = page->private; + if (newsize <= oldsize) + return ptr; + } +#endif /* ! __PIE__ */ + } - /* allow for nested calls */ -#ifdef LINUXKM_SIMD_IRQ - if (((unsigned char *)wolfcrypt_linuxkm_fpu_states[processor_id])[PAGE_SIZE-1] != 0) { - if (((unsigned char *)wolfcrypt_linuxkm_fpu_states[processor_id])[PAGE_SIZE-1] == 255) { - preempt_enable(); - pr_err("save_vector_registers_x86 recursion register overflow for " - "cpu id %d.\n", processor_id); - return BAD_STATE_E; - } else { - ++((unsigned char *)wolfcrypt_linuxkm_fpu_states[processor_id])[PAGE_SIZE-1]; - return 0; - } - } - kernel_fpu_begin(); - preempt_enable(); /* kernel_fpu_begin() does its own - * preempt_disable(). decrement ours. - */ - ((unsigned char *)wolfcrypt_linuxkm_fpu_states[processor_id])[PAGE_SIZE-1] = 1; -#else /* !LINUXKM_SIMD_IRQ */ - if (wolfcrypt_linuxkm_fpu_states[processor_id] != 0U) { - if ((wolfcrypt_linuxkm_fpu_states[processor_id] & WC_FPU_COUNT_MASK) - == WC_FPU_COUNT_MASK) - { - preempt_enable(); - pr_err("save_vector_registers_x86 recursion register overflow for " - "cpu id %d.\n", processor_id); - return BAD_STATE_E; - } else { - ++wolfcrypt_linuxkm_fpu_states[processor_id]; - return 0; - } - } + nptr = kvmalloc_node(newsize, GFP_KERNEL, NUMA_NO_NODE); + if (nptr != NULL) { + memcpy(nptr, ptr, oldsize); + kvfree(ptr); + } - /* if kernel_fpu_begin() won't actually save the reg file (because - * it was already saved and invalidated, or because we're in a - * kernel thread), don't call kernel_fpu_begin() here, nor call - * kernel_fpu_end() in cleanup. this avoids pointless overhead. in - * kernels >=5.17.12 (from changes to irq_fpu_usable() in linux - * commit 59f5ede3bc0f, backported somewhere >5.17.5), this also - * fixes register corruption. - */ - if ((current->flags & PF_KTHREAD) || - test_thread_flag(TIF_NEED_FPU_LOAD)) - { - wolfcrypt_linuxkm_fpu_states[processor_id] = - WC_FPU_SAVED_MASK + 1U; /* set msb 1 to inhibit kernel_fpu_end() at cleanup. */ - /* keep preempt_disable()d from above. */ - } else { - kernel_fpu_begin(); - preempt_enable(); /* kernel_fpu_begin() does its own - * preempt_disable(). decrement ours. - */ - wolfcrypt_linuxkm_fpu_states[processor_id] = 1U; /* set msb 0 to trigger kernel_fpu_end() at cleanup. */ - } -#endif /* !LINUXKM_SIMD_IRQ */ + return nptr; +} +#endif /* HAVE_KVMALLOC */ +#if defined(WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS) && defined(CONFIG_X86) + +static unsigned int wc_linuxkm_fpu_states_n_tracked = 0; + +struct wc_thread_fpu_count_ent { + volatile pid_t pid; + unsigned int fpu_state; +}; +struct wc_thread_fpu_count_ent *wc_linuxkm_fpu_states = NULL; +#define WC_FPU_COUNT_MASK 0x7fffffffU +#define WC_FPU_SAVED_MASK 0x80000000U + +WARN_UNUSED_RESULT int allocate_wolfcrypt_linuxkm_fpu_states(void) +{ + if (wc_linuxkm_fpu_states != NULL) { + static int warned_for_repeat_alloc = 0; + if (! warned_for_repeat_alloc) { + pr_err("attempt at repeat allocation" + " in allocate_wolfcrypt_linuxkm_fpu_states\n"); + warned_for_repeat_alloc = 1; + } + return BAD_STATE_E; + } + + if (nr_cpu_ids >= 16) + wc_linuxkm_fpu_states_n_tracked = nr_cpu_ids * 2; + else + wc_linuxkm_fpu_states_n_tracked = 32; + + wc_linuxkm_fpu_states = + (struct wc_thread_fpu_count_ent *)malloc( + wc_linuxkm_fpu_states_n_tracked * sizeof(wc_linuxkm_fpu_states[0])); + + if (! wc_linuxkm_fpu_states) { + pr_err("allocation of %lu bytes for " + "wc_linuxkm_fpu_states failed.\n", + nr_cpu_ids * sizeof(struct fpu_state *)); + return MEMORY_E; + } + + memset(wc_linuxkm_fpu_states, 0, wc_linuxkm_fpu_states_n_tracked * sizeof(wc_linuxkm_fpu_states[0])); + + return 0; +} + +void free_wolfcrypt_linuxkm_fpu_states(void) { + struct wc_thread_fpu_count_ent *i, *i_endptr; + pid_t i_pid; + + if (wc_linuxkm_fpu_states == NULL) { + pr_err("free_wolfcrypt_linuxkm_fpu_states called" + " before allocate_wolfcrypt_linuxkm_fpu_states.\n"); + return; + } + + for (i = wc_linuxkm_fpu_states, + i_endptr = &wc_linuxkm_fpu_states[wc_linuxkm_fpu_states_n_tracked]; + i < i_endptr; + ++i) + { + i_pid = __atomic_load_n(&i->pid, __ATOMIC_CONSUME); + if (i_pid == 0) + continue; + if (i->fpu_state != 0) { + pr_err("free_wolfcrypt_linuxkm_fpu_states called" + " with nonzero state 0x%x for pid %d.\n", i->fpu_state, i_pid); + i->fpu_state = 0; + } + } + + free(wc_linuxkm_fpu_states); + wc_linuxkm_fpu_states = NULL; +} + +/* lock-(mostly)-free thread-local storage facility for tracking recursive fpu pushing/popping */ +static struct wc_thread_fpu_count_ent *wc_linuxkm_fpu_state_assoc(int create_p) { + struct wc_thread_fpu_count_ent *i, *i_endptr, *i_empty; + pid_t my_pid = task_pid_nr(current), i_pid; + + { + static int _warned_on_null = 0; + if (wc_linuxkm_fpu_states == NULL) + { + if (_warned_on_null == 0) { + pr_err("wc_linuxkm_fpu_state_assoc called by pid %d" + " before allocate_wolfcrypt_linuxkm_fpu_states.\n", my_pid); + _warned_on_null = 1; + } + return NULL; + } + } + + i_endptr = &wc_linuxkm_fpu_states[wc_linuxkm_fpu_states_n_tracked]; + + for (;;) { + for (i = wc_linuxkm_fpu_states, + i_empty = NULL; + i < i_endptr; + ++i) + { + i_pid = __atomic_load_n(&i->pid, __ATOMIC_CONSUME); + if (i_pid == my_pid) + return i; + if ((i_empty == NULL) && (i_pid == 0)) + i_empty = i; + } + if ((i_empty == NULL) || (! create_p)) + return NULL; + + i_pid = 0; + if (__atomic_compare_exchange_n( + &(i_empty->pid), + &i_pid, + my_pid, + 0 /* weak */, + __ATOMIC_SEQ_CST /* success_memmodel */, + __ATOMIC_SEQ_CST /* failure_memmodel */)) + { + return i_empty; + } + } +} + +static void wc_linuxkm_fpu_state_free(struct wc_thread_fpu_count_ent *ent) { + if (ent->fpu_state != 0) { + static int warned_nonzero_fpu_state = 0; + if (! warned_nonzero_fpu_state) { + pr_err("wc_linuxkm_fpu_state_free for pid %d" + " with nonzero fpu_state 0x%x.\n", ent->pid, ent->fpu_state); + warned_nonzero_fpu_state = 1; + } + ent->fpu_state = 0; + } + __atomic_store_n(&ent->pid, 0, __ATOMIC_RELEASE); +} + +WARN_UNUSED_RESULT int save_vector_registers_x86(void) +{ + struct wc_thread_fpu_count_ent *pstate = wc_linuxkm_fpu_state_assoc(1); + if (pstate == NULL) + return ENOMEM; + + /* allow for nested calls */ + if (pstate->fpu_state != 0U) { + if ((pstate->fpu_state & WC_FPU_COUNT_MASK) + == WC_FPU_COUNT_MASK) + { + pr_err("save_vector_registers_x86 recursion register overflow for " + "pid %d.\n", pstate->pid); + return BAD_STATE_E; + } else { + ++pstate->fpu_state; return 0; } } - void restore_vector_registers_x86(void) - { - int processor_id = smp_processor_id(); - if ((wolfcrypt_linuxkm_fpu_states == NULL) -#ifdef LINUXKM_SIMD_IRQ - || (wolfcrypt_linuxkm_fpu_states[processor_id] == NULL) + if (irq_fpu_usable()) { +#if defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_COUNT) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)) + /* inhibit migration, which gums up the algorithm in kernel_fpu_{begin,end}(). */ + migrate_disable(); #endif - ) - { - pr_err("restore_vector_registers_x86 called for cpu id %d " - "with null context buffer.\n", processor_id); - return; - } + kernel_fpu_begin(); + pstate->fpu_state = 1U; /* set msb 0 to trigger kernel_fpu_end() at cleanup. */ + } else if (in_nmi() || (hardirq_count() > 0) || (softirq_count() > 0)) { + static int warned_fpu_forbidden = 0; + if (! warned_fpu_forbidden) + pr_err("save_vector_registers_x86 called from IRQ handler.\n"); + wc_linuxkm_fpu_state_free(pstate); + return EPERM; + } else { +#if defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_COUNT) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)) + migrate_disable(); +#endif + /* assume already safely in_kernel_fpu. */ + pstate->fpu_state = + WC_FPU_SAVED_MASK + 1U; /* set msb 1 to inhibit kernel_fpu_end() at cleanup. */ + } -#ifdef LINUXKM_SIMD_IRQ - if (((unsigned char *)wolfcrypt_linuxkm_fpu_states[processor_id])[PAGE_SIZE-1] == 0) - { - pr_err("restore_vector_registers_x86 called for cpu id %d " - "without saved context.\n", processor_id); - return; - } - - if (--((unsigned char *)wolfcrypt_linuxkm_fpu_states[processor_id])[PAGE_SIZE-1] > 0) { - preempt_enable(); /* preempt_disable count will still be nonzero after this decrement. */ - return; - } - - if (am_in_hard_interrupt_handler()) { - #if LINUX_VERSION_CODE < KERNEL_VERSION(5, 14, 0) - copy_kernel_to_fpregs(wolfcrypt_linuxkm_fpu_states[processor_id]); - #elif LINUX_VERSION_CODE < KERNEL_VERSION(5, 16, 0) - __restore_fpregs_from_fpstate(wolfcrypt_linuxkm_fpu_states[processor_id], - xfeatures_mask_all); - #else - restore_fpregs_from_fpstate(wolfcrypt_linuxkm_fpu_states[processor_id], - fpu_kernel_cfg.max_features); - #endif - preempt_enable(); - } else { - kernel_fpu_end(); - } -#else /* !LINUXKM_SIMD_IRQ */ - if ((wolfcrypt_linuxkm_fpu_states[processor_id] & WC_FPU_COUNT_MASK) == 0U) - { - pr_err("restore_vector_registers_x86 called for cpu id %d " - "without saved context.\n", processor_id); - return; - } - - if ((--wolfcrypt_linuxkm_fpu_states[processor_id] & WC_FPU_COUNT_MASK) > 0U) { - preempt_enable(); /* preempt_disable count may still be nonzero - * after this decrement, but any remaining - * count(s) aren't ours. - */ - return; - } - - if (wolfcrypt_linuxkm_fpu_states[processor_id] == 0U) { - kernel_fpu_end(); - } else { - preempt_enable(); /* preempt_disable count will still be nonzero - * after this decrement. - */ - wolfcrypt_linuxkm_fpu_states[processor_id] = 0U; - } -#endif /* !LINUXKM_SIMD_IRQ */ + return 0; +} +void restore_vector_registers_x86(void) +{ + struct wc_thread_fpu_count_ent *pstate = wc_linuxkm_fpu_state_assoc(0); + if (pstate == NULL) { + pr_err("restore_vector_registers_x86 called by pid %d " + "with no saved state.\n", task_pid_nr(current)); return; } + + if ((--pstate->fpu_state & WC_FPU_COUNT_MASK) > 0U) { + return; + } + + if (pstate->fpu_state == 0U) + kernel_fpu_end(); + else + pstate->fpu_state = 0U; +#if defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_COUNT) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)) + migrate_enable(); +#endif + + wc_linuxkm_fpu_state_free(pstate); + + return; +} #endif /* WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS && CONFIG_X86 */ #if defined(__PIE__) && (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 1, 0)) diff --git a/linuxkm/linuxkm_wc_port.h b/linuxkm/linuxkm_wc_port.h index 8b839161c..bc17aa57f 100644 --- a/linuxkm/linuxkm_wc_port.h +++ b/linuxkm/linuxkm_wc_port.h @@ -24,6 +24,12 @@ #ifndef LINUXKM_WC_PORT_H #define LINUXKM_WC_PORT_H + #include + + #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 16, 0) + #error Unsupported kernel. + #endif + #ifdef HAVE_CONFIG_H #ifndef PACKAGE_NAME #error wc_port.h included before config.h @@ -59,6 +65,23 @@ (int)_xatoi_res; \ }) + /* Kbuild+gcc on x86 doesn't consistently honor the default ALIGN16 on stack objects, + * but gives adequate alignment with "32". + */ + #if defined(CONFIG_X86) && !defined(ALIGN16) + #define ALIGN16 __attribute__ ( (aligned (32))) + #endif + + /* kvmalloc()/kvfree() and friends added in linux commit a7c3e901 */ + #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 12, 0) + #define HAVE_KVMALLOC + #endif + + /* kernel printf doesn't implement fp. */ + #ifndef WOLFSSL_NO_FLOAT_FMT + #define WOLFSSL_NO_FLOAT_FMT + #endif + #ifdef BUILDING_WOLFSSL #if defined(CONFIG_MIPS) && defined(HAVE_LINUXKM_PIE_SUPPORT) @@ -95,7 +118,6 @@ #include #include - #include #include #include #include @@ -124,6 +146,8 @@ #ifndef CONFIG_X86 #error X86 SIMD extensions requested, but CONFIG_X86 is not set. #endif + #define WOLFSSL_LINUXKM_SIMD + #define WOLFSSL_LINUXKM_SIMD_X86 #ifndef WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS #define WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS #endif @@ -133,6 +157,8 @@ #if !defined(CONFIG_ARM) && !defined(CONFIG_ARM64) #error ARM SIMD extensions requested, but CONFIG_ARM* is not set. #endif + #define WOLFSSL_LINUXKM_SIMD + #define WOLFSSL_LINUXKM_SIMD_ARM #ifndef WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS #define WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS #endif @@ -142,26 +168,17 @@ #endif #endif + /* benchmarks.c uses floating point math, so needs a working SAVE_VECTOR_REGISTERS(). */ + #if defined(WOLFSSL_LINUXKM_BENCHMARKS) && !defined(WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS) + #define WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS + #endif + #if defined(WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS) && defined(CONFIG_X86) - #define WOLFSSL_LINUXKM_SIMD - #define WOLFSSL_LINUXKM_SIMD_X86 #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0) #include #else #include #endif - #ifdef LINUXKM_SIMD_IRQ - #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 0, 0) - #include - #endif - #if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 16, 0) - #error LINUXKM_SIMD_IRQ is unavailable on linux >= 5.16 (missing exports around fpregs) - /* - * #include - * #include - */ - #endif - #endif #ifndef SAVE_VECTOR_REGISTERS #define SAVE_VECTOR_REGISTERS(fail_clause) { int _svr_ret = save_vector_registers_x86(); if (_svr_ret != 0) { fail_clause } } #endif @@ -169,12 +186,7 @@ #define RESTORE_VECTOR_REGISTERS() restore_vector_registers_x86() #endif #elif defined(WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS) && (defined(CONFIG_ARM) || defined(CONFIG_ARM64)) - #define WOLFSSL_LINUXKM_SIMD - #define WOLFSSL_LINUXKM_SIMD_ARM #include - #ifdef LINUXKM_SIMD_IRQ - #error LINUXKM_SIMD_IRQ is unavailable on ARM (not implemented) - #endif #ifndef SAVE_VECTOR_REGISTERS #define SAVE_VECTOR_REGISTERS(fail_clause) { int _svr_ret = save_vector_registers_arm(); if (_svr_ret != 0) { fail_clause } } #endif @@ -195,11 +207,6 @@ #define NO_THREAD_LS #define NO_ATTRIBUTE_CONSTRUCTOR - /* kvmalloc()/kvfree() and friends added in linux commit a7c3e901 */ - #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 12, 0) - #define HAVE_KVMALLOC - #endif - #ifdef HAVE_FIPS extern int wolfCrypt_FIPS_first(void); extern int wolfCrypt_FIPS_last(void); @@ -215,7 +222,7 @@ #endif #if defined(__PIE__) && !defined(USE_WOLFSSL_LINUXKM_PIE_REDIRECT_TABLE) - #error "compiling -fPIE without PIE support." + #error "compiling -fPIE requires PIE redirect table." #endif #if defined(HAVE_FIPS) && !defined(HAVE_LINUXKM_PIE_SUPPORT) @@ -307,42 +314,37 @@ struct task_struct *(*get_current)(void); int (*preempt_count)(void); - #ifdef WOLFSSL_LINUXKM_SIMD_X86 - typeof(irq_fpu_usable) *irq_fpu_usable; - /* kernel_fpu_begin() replaced by kernel_fpu_begin_mask() in commit e4512289, - * released in kernel 5.11, backported to 5.4.93 - */ - #ifdef kernel_fpu_begin - typeof(kernel_fpu_begin_mask) *kernel_fpu_begin_mask; - #else - typeof(kernel_fpu_begin) *kernel_fpu_begin; - #endif - typeof(kernel_fpu_end) *kernel_fpu_end; + #ifdef WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS - #ifdef LINUXKM_SIMD_IRQ - #if LINUX_VERSION_CODE < KERNEL_VERSION(5, 14, 0) - typeof(copy_fpregs_to_fpstate) *copy_fpregs_to_fpstate; - typeof(copy_kernel_to_fpregs) *copy_kernel_to_fpregs; - #elif LINUX_VERSION_CODE < KERNEL_VERSION(5, 16, 0) - typeof(save_fpregs_to_fpstate) *save_fpregs_to_fpstate; - typeof(__restore_fpregs_from_fpstate) *__restore_fpregs_from_fpstate; - typeof(xfeatures_mask_all) *xfeatures_mask_all; - /* - * #else - * typeof(save_fpregs_to_fpstate) *save_fpregs_to_fpstate; - * typeof(restore_fpregs_from_fpstate) *restore_fpregs_from_fpstate; - * typeof(fpu_kernel_cfg) *fpu_kernel_cfg; - */ + #if LINUX_VERSION_CODE < KERNEL_VERSION(6, 2, 0) + typeof(cpu_number) *cpu_number; + #else + typeof(pcpu_hot) *pcpu_hot; #endif - #endif - #if LINUX_VERSION_CODE < KERNEL_VERSION(6, 2, 0) - typeof(cpu_number) *cpu_number; - #else - typeof(pcpu_hot) *pcpu_hot; - #endif - typeof(nr_cpu_ids) *nr_cpu_ids; + typeof(nr_cpu_ids) *nr_cpu_ids; - #endif /* WOLFSSL_LINUXKM_SIMD_X86 */ + #if defined(CONFIG_SMP) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)) + /* note the current and needed version of these were added in af449901b8 (2020-Sep-17) */ + typeof(migrate_disable) *migrate_disable; + typeof(migrate_enable) *migrate_enable; + #endif + + #ifdef CONFIG_X86 + typeof(irq_fpu_usable) *irq_fpu_usable; + /* kernel_fpu_begin() replaced by kernel_fpu_begin_mask() in commit e4512289, + * released in kernel 5.11, backported to 5.4.93 + */ + #ifdef kernel_fpu_begin + typeof(kernel_fpu_begin_mask) *kernel_fpu_begin_mask; + #else + typeof(kernel_fpu_begin) *kernel_fpu_begin; + #endif + typeof(kernel_fpu_end) *kernel_fpu_end; + #else /* !CONFIG_X86 */ + #error WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS is set for an unsupported architecture. + #endif /* arch */ + + #endif /* WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS */ typeof(__mutex_init) *__mutex_init; #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0) @@ -452,37 +454,31 @@ #undef preempt_count #define preempt_count (wolfssl_linuxkm_get_pie_redirect_table()->preempt_count) - #ifdef WOLFSSL_LINUXKM_SIMD_X86 - #define irq_fpu_usable (wolfssl_linuxkm_get_pie_redirect_table()->irq_fpu_usable) - #ifdef kernel_fpu_begin - #define kernel_fpu_begin_mask (wolfssl_linuxkm_get_pie_redirect_table()->kernel_fpu_begin_mask) - #else - #define kernel_fpu_begin (wolfssl_linuxkm_get_pie_redirect_table()->kernel_fpu_begin) - #endif - #define kernel_fpu_end (wolfssl_linuxkm_get_pie_redirect_table()->kernel_fpu_end) - #ifdef LINUXKM_SIMD_IRQ - #if LINUX_VERSION_CODE < KERNEL_VERSION(5, 14, 0) - #define copy_fpregs_to_fpstate (wolfssl_linuxkm_get_pie_redirect_table()->copy_fpregs_to_fpstate) - #define copy_kernel_to_fpregs (wolfssl_linuxkm_get_pie_redirect_table()->copy_kernel_to_fpregs) - #elif LINUX_VERSION_CODE < KERNEL_VERSION(5, 16, 0) - #define save_fpregs_to_fpstate (wolfssl_linuxkm_get_pie_redirect_table()->save_fpregs_to_fpstate) - #define __restore_fpregs_from_fpstate (wolfssl_linuxkm_get_pie_redirect_table()->__restore_fpregs_from_fpstate) - #define xfeatures_mask_all (*(wolfssl_linuxkm_get_pie_redirect_table()->xfeatures_mask_all)) - /* - * #else - * #define save_fpregs_to_fpstate (wolfssl_linuxkm_get_pie_redirect_table()->save_fpregs_to_fpstate) - * #define restore_fpregs_from_fpstate (wolfssl_linuxkm_get_pie_redirect_table()->restore_fpregs_from_fpstate) - * #define fpu_kernel_cfg (*(wolfssl_linuxkm_get_pie_redirect_table()->fpu_kernel_cfg)) - */ - #endif - #endif + #ifdef WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS #if LINUX_VERSION_CODE < KERNEL_VERSION(6, 2, 0) #define cpu_number (*(wolfssl_linuxkm_get_pie_redirect_table()->cpu_number)) #else #define pcpu_hot (*(wolfssl_linuxkm_get_pie_redirect_table()->pcpu_hot)) #endif #define nr_cpu_ids (*(wolfssl_linuxkm_get_pie_redirect_table()->nr_cpu_ids)) - #endif + + #if defined(CONFIG_SMP) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)) + #define migrate_disable (*(wolfssl_linuxkm_get_pie_redirect_table()->migrate_disable)) + #define migrate_enable (*(wolfssl_linuxkm_get_pie_redirect_table()->migrate_enable)) + #endif + + #ifdef CONFIG_X86 + #define irq_fpu_usable (wolfssl_linuxkm_get_pie_redirect_table()->irq_fpu_usable) + #ifdef kernel_fpu_begin + #define kernel_fpu_begin_mask (wolfssl_linuxkm_get_pie_redirect_table()->kernel_fpu_begin_mask) + #else + #define kernel_fpu_begin (wolfssl_linuxkm_get_pie_redirect_table()->kernel_fpu_begin) + #endif + #define kernel_fpu_end (wolfssl_linuxkm_get_pie_redirect_table()->kernel_fpu_end) + #else /* !CONFIG_X86 */ + #error WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS is set for an unsupported architecture. + #endif /* archs */ + #endif /* WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS */ #define __mutex_init (wolfssl_linuxkm_get_pie_redirect_table()->__mutex_init) #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0) @@ -515,9 +511,9 @@ #endif /* USE_WOLFSSL_LINUXKM_PIE_REDIRECT_TABLE */ -#ifdef WOLFSSL_LINUXKM_SIMD +#ifdef WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS -#ifdef WOLFSSL_LINUXKM_SIMD_X86 +#ifdef CONFIG_X86 extern __must_check int allocate_wolfcrypt_linuxkm_fpu_states(void); extern void free_wolfcrypt_linuxkm_fpu_states(void); @@ -547,7 +543,7 @@ #endif -#endif /* WOLFSSL_LINUXKM_SIMD */ +#endif /* WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS */ /* remove this multifariously conflicting macro, picked up from * Linux arch//include/asm/current.h. @@ -556,22 +552,6 @@ #undef current #endif - /* prevent gcc's mm_malloc.h from being included, since it unconditionally - * includes stdlib.h, which is kernel-incompatible. - */ - #define _MM_MALLOC_H_INCLUDED - - #ifdef HAVE_KVMALLOC - #define malloc(x) kvmalloc_node(x, GFP_KERNEL, NUMA_NO_NODE) - #define free(x) kvfree(x) - void *lkm_realloc(void *ptr, size_t newsize); - #define realloc(x, y) lkm_realloc(x, y) - #else - #define malloc(x) kmalloc(x, GFP_KERNEL) - #define free(x) kfree(x) - #define realloc(x,y) krealloc(x, y, GFP_KERNEL) - #endif - /* min() and max() in linux/kernel.h over-aggressively type-check, producing * myriad spurious -Werrors throughout the codebase. */ @@ -618,9 +598,41 @@ #include typedef struct mutex wolfSSL_Mutex; - #define XMALLOC(s, h, t) ({(void)(h); (void)(t); kmalloc(s, GFP_KERNEL);}) - #define XFREE(p, h, t) ({void* _xp; (void)(h); _xp = (p); if(_xp) kfree(_xp);}) - #define XREALLOC(p, n, h, t) ({(void)(h); (void)(t); krealloc((p), (n), GFP_KERNEL);}) + /* prevent gcc's mm_malloc.h from being included, since it unconditionally + * includes stdlib.h, which is kernel-incompatible. + */ + #define _MM_MALLOC_H_INCLUDED + + /* fun fact: since linux commit 59bb47985c, kmalloc with power-of-2 size is + * aligned to the size. + */ + #define WC_LINUXKM_ROUND_UP_P_OF_2(x) ( \ + { \ + size_t _alloc_sz = (x); \ + _alloc_sz = 1UL << ((sizeof(_alloc_sz) * 8UL) - __builtin_clzl(_alloc_sz)); \ + _alloc_sz; \ + }) + #ifdef HAVE_KVMALLOC + #define malloc(size) kvmalloc_node(WC_LINUXKM_ROUND_UP_P_OF_2(size), GFP_KERNEL, NUMA_NO_NODE) + #define free(ptr) kvfree(ptr) + void *lkm_realloc(void *ptr, size_t newsize); + #define realloc(ptr, newsize) lkm_realloc(ptr, WC_LINUXKM_ROUND_UP_P_OF_2(newsize)) + #else + #define malloc(size) kmalloc(WC_LINUXKM_ROUND_UP_P_OF_2(size), GFP_KERNEL) + #define free(ptr) kfree(ptr) + #define realloc(ptr, newsize) krealloc(ptr, WC_LINUXKM_ROUND_UP_P_OF_2(newsize), GFP_KERNEL) + #endif + +#ifdef WOLFSSL_TRACK_MEMORY + #include + #define XMALLOC(s, h, t) ({(void)(h); (void)(t); wolfSSL_Malloc(s);}) + #define XFREE(p, h, t) ({void* _xp; (void)(h); _xp = (p); if(_xp) wolfSSL_Free(_xp);}) + #define XREALLOC(p, n, h, t) ({(void)(h); (void)(t); wolfSSL_Realloc(p, n);}) +#else + #define XMALLOC(s, h, t) ({(void)(h); (void)(t); malloc(s);}) + #define XFREE(p, h, t) ({void* _xp; (void)(h); _xp = (p); if(_xp) free(_xp);}) + #define XREALLOC(p, n, h, t) ({(void)(h); (void)(t); realloc(p, n);}) +#endif #include diff --git a/linuxkm/module_hooks.c b/linuxkm/module_hooks.c index 3f17217d3..7f9f83994 100644 --- a/linuxkm/module_hooks.c +++ b/linuxkm/module_hooks.c @@ -113,6 +113,15 @@ static void lkmFipsCb(int ok, int err, const char* hash) static int updateFipsHash(void); #endif +#ifdef WOLFSSL_LINUXKM_BENCHMARKS +#undef HAVE_PTHREAD +#define STRING_USER +#define NO_MAIN_FUNCTION +#define current_time benchmark_current_time +#define WOLFSSL_NO_FLOAT_FMT +#include "/home/douzzer/com/wolfssl/src/wolfssl/wolfcrypt/benchmark/benchmark.c" +#endif /* WOLFSSL_LINUXKM_BENCHMARKS */ + #if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 0, 0) static int __init wolfssl_init(void) #else @@ -202,10 +211,9 @@ static int wolfssl_init(void) * the true module start address, which is potentially useful to an * attacker. */ - pr_info("wolfCrypt container hashes (spans): %x (%lu) %x (%lu), text base %pK, ro base %pK\n", + pr_info("wolfCrypt container hashes (spans): text 0x%x (%lu), rodata 0x%x (%lu)\n", text_hash, pie_text_end-pie_text_start, - rodata_hash, pie_rodata_end-pie_rodata_start, - THIS_MODULE_TEXT_BASE, THIS_MODULE_RO_BASE); + rodata_hash, pie_rodata_end-pie_rodata_start); } #endif /* HAVE_LINUXKM_PIE_SUPPORT */ @@ -277,6 +285,10 @@ static int wolfssl_init(void) pr_info("wolfCrypt self-test passed.\n"); #endif +#ifdef WOLFSSL_LINUXKM_BENCHMARKS + wolfcrypt_benchmark_main(0, (char**)NULL); +#endif + #ifdef WOLFCRYPT_ONLY pr_info("wolfCrypt " LIBWOLFSSL_VERSION_STRING " loaded%s" ".\nSee https://www.wolfssl.com/ for more information.\n" @@ -334,15 +346,6 @@ static int my_preempt_count(void) { return preempt_count(); } -#if defined(WOLFSSL_LINUXKM_SIMD_X86) && (LINUX_VERSION_CODE < KERNEL_VERSION(5, 14, 0)) -static int my_copy_fpregs_to_fpstate(struct fpu *fpu) { - return copy_fpregs_to_fpstate(fpu); -} -static void my_copy_kernel_to_fpregs(union fpregs_state *fpstate) { - copy_kernel_to_fpregs(fpstate); -} -#endif - static int set_up_wolfssl_linuxkm_pie_redirect_table(void) { memset( &wolfssl_linuxkm_pie_redirect_table, @@ -430,6 +433,20 @@ static int set_up_wolfssl_linuxkm_pie_redirect_table(void) { wolfssl_linuxkm_pie_redirect_table.get_current = my_get_current_thread; wolfssl_linuxkm_pie_redirect_table.preempt_count = my_preempt_count; +#ifdef WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS + + #if LINUX_VERSION_CODE < KERNEL_VERSION(6, 2, 0) + wolfssl_linuxkm_pie_redirect_table.cpu_number = &cpu_number; + #else + wolfssl_linuxkm_pie_redirect_table.pcpu_hot = &pcpu_hot; + #endif + wolfssl_linuxkm_pie_redirect_table.nr_cpu_ids = &nr_cpu_ids; + + #if defined(CONFIG_SMP) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)) + wolfssl_linuxkm_pie_redirect_table.migrate_disable = &migrate_disable; + wolfssl_linuxkm_pie_redirect_table.migrate_enable = &migrate_enable; + #endif + #ifdef WOLFSSL_LINUXKM_SIMD_X86 wolfssl_linuxkm_pie_redirect_table.irq_fpu_usable = irq_fpu_usable; #ifdef kernel_fpu_begin @@ -440,29 +457,9 @@ static int set_up_wolfssl_linuxkm_pie_redirect_table(void) { kernel_fpu_begin; #endif wolfssl_linuxkm_pie_redirect_table.kernel_fpu_end = kernel_fpu_end; - #ifdef LINUXKM_SIMD_IRQ - #if LINUX_VERSION_CODE < KERNEL_VERSION(5, 14, 0) - wolfssl_linuxkm_pie_redirect_table.copy_fpregs_to_fpstate = my_copy_fpregs_to_fpstate; - wolfssl_linuxkm_pie_redirect_table.copy_kernel_to_fpregs = my_copy_kernel_to_fpregs; - #elif LINUX_VERSION_CODE < KERNEL_VERSION(5, 16, 0) - wolfssl_linuxkm_pie_redirect_table.save_fpregs_to_fpstate = save_fpregs_to_fpstate; - wolfssl_linuxkm_pie_redirect_table.__restore_fpregs_from_fpstate = __restore_fpregs_from_fpstate; - wolfssl_linuxkm_pie_redirect_table.xfeatures_mask_all = &xfeatures_mask_all; - /* - * #else - * wolfssl_linuxkm_pie_redirect_table.save_fpregs_to_fpstate = save_fpregs_to_fpstate; - * wolfssl_linuxkm_pie_redirect_table.restore_fpregs_from_fpstate = restore_fpregs_from_fpstate; - * wolfssl_linuxkm_pie_redirect_table.fpu_kernel_cfg = &fpu_kernel_cfg; - */ - #endif - #endif - #if LINUX_VERSION_CODE < KERNEL_VERSION(6, 2, 0) - wolfssl_linuxkm_pie_redirect_table.cpu_number = &cpu_number; - #else - wolfssl_linuxkm_pie_redirect_table.pcpu_hot = &pcpu_hot; - #endif - wolfssl_linuxkm_pie_redirect_table.nr_cpu_ids = &nr_cpu_ids; -#endif +#endif /* WOLFSSL_LINUXKM_SIMD_X86 */ + +#endif /* WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS */ wolfssl_linuxkm_pie_redirect_table.__mutex_init = __mutex_init; #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0) diff --git a/wolfcrypt/src/wc_port.c b/wolfcrypt/src/wc_port.c index b6736e501..ff1a817b0 100644 --- a/wolfcrypt/src/wc_port.c +++ b/wolfcrypt/src/wc_port.c @@ -207,7 +207,7 @@ int wolfCrypt_Init(void) } #endif - #if defined(WOLFSSL_LINUXKM_SIMD_X86) + #ifdef WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS ret = allocate_wolfcrypt_linuxkm_fpu_states(); if (ret != 0) { WOLFSSL_MSG("allocate_wolfcrypt_linuxkm_fpu_states failed"); @@ -466,7 +466,7 @@ int wolfCrypt_Cleanup(void) rpcmem_deinit(); wolfSSL_CleanupHandle(); #endif - #if defined(WOLFSSL_LINUXKM_SIMD_X86) + #ifdef WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS free_wolfcrypt_linuxkm_fpu_states(); #endif @@ -3207,56 +3207,6 @@ char* mystrnstr(const char* s1, const char* s2, unsigned int n) #endif /* WOLFSSL_NUCLEUS_1_2 */ -#if defined(WOLFSSL_LINUXKM) && defined(HAVE_KVMALLOC) - /* adapted from kvrealloc() draft by Changli Gao, 2010-05-13 */ - void *lkm_realloc(void *ptr, size_t newsize) { - void *nptr; - size_t oldsize; - - if (unlikely(newsize == 0)) { - kvfree(ptr); - return ZERO_SIZE_PTR; - } - - if (unlikely(ptr == NULL)) - return kvmalloc_node(newsize, GFP_KERNEL, NUMA_NO_NODE); - - if (is_vmalloc_addr(ptr)) { - /* no way to discern the size of the old allocation, - * because the kernel doesn't export find_vm_area(). if - * it did, we could then call get_vm_area_size() on the - * returned struct vm_struct. - */ - return NULL; - } else { -#ifndef __PIE__ - struct page *page; - - page = virt_to_head_page(ptr); - if (PageSlab(page) || PageCompound(page)) { - if (newsize < PAGE_SIZE) -#endif /* ! __PIE__ */ - return krealloc(ptr, newsize, GFP_KERNEL); -#ifndef __PIE__ - oldsize = ksize(ptr); - } else { - oldsize = page->private; - if (newsize <= oldsize) - return ptr; - } -#endif /* ! __PIE__ */ - } - - nptr = kvmalloc_node(newsize, GFP_KERNEL, NUMA_NO_NODE); - if (nptr != NULL) { - memcpy(nptr, ptr, oldsize); - kvfree(ptr); - } - - return nptr; - } -#endif /* WOLFSSL_LINUXKM && HAVE_KVMALLOC */ - #if defined(WOLFSSL_TI_CRYPT) || defined(WOLFSSL_TI_HASH) #include /* initialize and Mutex for TI Crypt Engine */ #include /* md5, sha1, sha224, sha256 */