Browse Source

linuxkm/linuxkm_memory.c: refactor SAVE/RESTORE_VECTOR_REGISTERS() to be per-process rather than per-CPU, and add migrate_disable/enable() to kernel_fpu_begin/end() because preempt_disable() is just a barrier on _PREEMPT_VOLUNTARY kernels;

linuxkm/linuxkm_wc_port.h: activate SAVE/RESTORE_VECTOR_REGISTERS() whenever defined(WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS) for benchmark.c support, independent of vector crypto features;

fix and optimize various alignment issues with stack and heap allocations;

fix macro definitions for XMALLOC/XREALLOC/XFREE to correctly use kvmalloc and friends when defined(HAVE_KVMALLOC), and to use wolfSSL_Malloc() and friends when defined(WOLFSSL_TRACK_MEMORY);

purge stale LINUXKM_SIMD_IRQ code.
Daniel Pouzzner 10 months ago
parent
commit
4f4842fce6
5 changed files with 380 additions and 474 deletions
  1. 5 0
      linuxkm/Kbuild
  2. 222 280
      linuxkm/linuxkm_memory.c
  3. 119 107
      linuxkm/linuxkm_wc_port.h
  4. 32 35
      linuxkm/module_hooks.c
  5. 2 52
      wolfcrypt/src/wc_port.c

+ 5 - 0
linuxkm/Kbuild

@@ -90,6 +90,11 @@ ifeq "$(ENABLED_LINUXKM_PIE)" "yes"
     $(obj)/linuxkm/module_hooks.o: ccflags-y += $(PIE_SUPPORT_FLAGS)
 endif
 
+ifeq "$(ENABLED_LINUXKM_BENCHMARKS)" "yes"
+    $(obj)/linuxkm/module_hooks.o: ccflags-y = $(WOLFSSL_CFLAGS) $(CFLAGS_FPU_ENABLE) $(CFLAGS_SIMD_ENABLE) $(PIE_SUPPORT_FLAGS)
+    $(obj)/linuxkm/module_hooks.o: asflags-y = $(WOLFSSL_ASFLAGS) $(ASFLAGS_FPU_ENABLE_SIMD_DISABLE)
+endif
+
 asflags-y := $(WOLFSSL_ASFLAGS) $(ASFLAGS_FPUSIMD_DISABLE)
 
 # vectorized implementations that are kernel-safe are listed here.

+ 222 - 280
linuxkm/linuxkm_memory.c

@@ -21,320 +21,262 @@
 
 /* included by wolfcrypt/src/memory.c */
 
-#if defined(WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS) && defined(CONFIG_X86)
-    #ifdef LINUXKM_SIMD_IRQ
-        #if LINUX_VERSION_CODE < KERNEL_VERSION(5, 16, 0)
-            static union fpregs_state **wolfcrypt_linuxkm_fpu_states = NULL;
-        #else
-            static struct fpstate **wolfcrypt_linuxkm_fpu_states = NULL;
-        #endif
-    #else
-        static unsigned int *wolfcrypt_linuxkm_fpu_states = NULL;
-        #define WC_FPU_COUNT_MASK 0x7fffffffU
-        #define WC_FPU_SAVED_MASK 0x80000000U
-    #endif
-
-    static WARN_UNUSED_RESULT inline int am_in_hard_interrupt_handler(void)
-    {
-        return (preempt_count() & (NMI_MASK | HARDIRQ_MASK)) != 0;
+#ifdef HAVE_KVMALLOC
+/* adapted from kvrealloc() draft by Changli Gao, 2010-05-13 */
+void *lkm_realloc(void *ptr, size_t newsize) {
+    void *nptr;
+    size_t oldsize;
+
+    if (unlikely(newsize == 0)) {
+        kvfree(ptr);
+        return ZERO_SIZE_PTR;
     }
 
-    WARN_UNUSED_RESULT int allocate_wolfcrypt_linuxkm_fpu_states(void)
-    {
-        #ifdef LINUXKM_SIMD_IRQ
-            #if LINUX_VERSION_CODE < KERNEL_VERSION(5, 16, 0)
-            wolfcrypt_linuxkm_fpu_states =
-                (union fpregs_state **)kzalloc(nr_cpu_ids
-                                               * sizeof(struct fpu_state *),
-                                               GFP_KERNEL);
-            #else
-            wolfcrypt_linuxkm_fpu_states =
-                (struct fpstate **)kzalloc(nr_cpu_ids
-                                           * sizeof(struct fpstate *),
-                                           GFP_KERNEL);
-            #endif
-        #else
-            wolfcrypt_linuxkm_fpu_states =
-                (unsigned int *)kzalloc(nr_cpu_ids * sizeof(unsigned int),
-                                        GFP_KERNEL);
-        #endif
-
-        if (! wolfcrypt_linuxkm_fpu_states) {
-            pr_err("warning, allocation of %lu bytes for "
-                   "wolfcrypt_linuxkm_fpu_states failed.\n",
-                   nr_cpu_ids * sizeof(struct fpu_state *));
-            return MEMORY_E;
-        }
-#ifdef LINUXKM_SIMD_IRQ
-        {
-            typeof(nr_cpu_ids) i;
-            for (i=0; i<nr_cpu_ids; ++i) {
-                _Static_assert(sizeof(union fpregs_state) <= PAGE_SIZE,
-                               "union fpregs_state is larger than expected.");
-                #if LINUX_VERSION_CODE < KERNEL_VERSION(5, 16, 0)
-                wolfcrypt_linuxkm_fpu_states[i] =
-                    (union fpregs_state *)kzalloc(PAGE_SIZE
-                                                  /* sizeof(union fpregs_state) */,
-                                                  GFP_KERNEL);
-                #else
-                wolfcrypt_linuxkm_fpu_states[i] =
-                    (struct fpstate *)kzalloc(PAGE_SIZE
-                                              /* sizeof(struct fpstate) */,
-                                              GFP_KERNEL);
-                #endif
-                if (! wolfcrypt_linuxkm_fpu_states[i])
-                    break;
-                /* double-check that the allocation is 64-byte-aligned as needed
-                 * for xsave.
-                 */
-                if ((unsigned long)wolfcrypt_linuxkm_fpu_states[i] & 63UL) {
-                    pr_err("warning, allocation for wolfcrypt_linuxkm_fpu_states "
-                           "was not properly aligned (%px).\n",
-                           wolfcrypt_linuxkm_fpu_states[i]);
-                    kfree(wolfcrypt_linuxkm_fpu_states[i]);
-                    wolfcrypt_linuxkm_fpu_states[i] = 0;
-                    break;
-                }
-            }
-            if (i < nr_cpu_ids) {
-                pr_err("warning, only %u/%u allocations succeeded for "
-                       "wolfcrypt_linuxkm_fpu_states.\n",
-                       i, nr_cpu_ids);
-                return MEMORY_E;
-            }
+    if (unlikely(ptr == NULL))
+        return kvmalloc_node(newsize, GFP_KERNEL, NUMA_NO_NODE);
+
+    if (is_vmalloc_addr(ptr)) {
+        /* no way to discern the size of the old allocation,
+         * because the kernel doesn't export find_vm_area().  if
+         * it did, we could then call get_vm_area_size() on the
+         * returned struct vm_struct.
+         */
+        return NULL;
+    } else {
+#ifndef __PIE__
+        struct page *page;
+
+        page = virt_to_head_page(ptr);
+        if (PageSlab(page) || PageCompound(page)) {
+            if (newsize < PAGE_SIZE)
+#endif /* ! __PIE__ */
+                return krealloc(ptr, newsize, GFP_KERNEL);
+#ifndef __PIE__
+            oldsize = ksize(ptr);
+        } else {
+            oldsize = page->private;
+            if (newsize <= oldsize)
+                return ptr;
         }
-#endif /* LINUXKM_SIMD_IRQ */
-        return 0;
+#endif /* ! __PIE__ */
     }
 
-    void free_wolfcrypt_linuxkm_fpu_states(void)
-    {
-        if (wolfcrypt_linuxkm_fpu_states) {
-#ifdef LINUXKM_SIMD_IRQ
-            typeof(nr_cpu_ids) i;
-            for (i=0; i<nr_cpu_ids; ++i) {
-                if (wolfcrypt_linuxkm_fpu_states[i])
-                    kfree(wolfcrypt_linuxkm_fpu_states[i]);
-            }
-#endif /* LINUXKM_SIMD_IRQ */
-            kfree(wolfcrypt_linuxkm_fpu_states);
-            wolfcrypt_linuxkm_fpu_states = 0;
-        }
+    nptr = kvmalloc_node(newsize, GFP_KERNEL, NUMA_NO_NODE);
+    if (nptr != NULL) {
+        memcpy(nptr, ptr, oldsize);
+        kvfree(ptr);
     }
 
-    WARN_UNUSED_RESULT int save_vector_registers_x86(void)
-    {
-        int processor_id;
+    return nptr;
+}
+#endif /* HAVE_KVMALLOC */
 
-        preempt_disable();
+#if defined(WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS) && defined(CONFIG_X86)
 
-        processor_id = smp_processor_id();
+static unsigned int wc_linuxkm_fpu_states_n_tracked = 0;
 
-        {
-            static int _warned_on_null = -1;
-            if ((wolfcrypt_linuxkm_fpu_states == NULL)
-#ifdef LINUXKM_SIMD_IRQ
-                || (wolfcrypt_linuxkm_fpu_states[processor_id] == NULL)
-#endif
-                )
-            {
-                preempt_enable();
-                if (_warned_on_null < processor_id) {
-                    _warned_on_null = processor_id;
-                    pr_err("save_vector_registers_x86 called for cpu id %d "
-                           "with null context buffer.\n", processor_id);
-                }
-                return BAD_STATE_E;
-            }
+struct wc_thread_fpu_count_ent {
+    volatile pid_t pid;
+    unsigned int fpu_state;
+};
+struct wc_thread_fpu_count_ent *wc_linuxkm_fpu_states = NULL;
+#define WC_FPU_COUNT_MASK 0x7fffffffU
+#define WC_FPU_SAVED_MASK 0x80000000U
+
+WARN_UNUSED_RESULT int allocate_wolfcrypt_linuxkm_fpu_states(void)
+{
+    if (wc_linuxkm_fpu_states != NULL) {
+        static int warned_for_repeat_alloc = 0;
+        if (! warned_for_repeat_alloc) {
+            pr_err("attempt at repeat allocation"
+                   " in allocate_wolfcrypt_linuxkm_fpu_states\n");
+            warned_for_repeat_alloc = 1;
         }
+        return BAD_STATE_E;
+    }
 
-        if (! irq_fpu_usable()) {
-
-#ifdef LINUXKM_SIMD_IRQ
-            if (am_in_hard_interrupt_handler()) {
-
-                /* allow for nested calls */
-                if (((unsigned char *)wolfcrypt_linuxkm_fpu_states[processor_id])[PAGE_SIZE-1] != 0) {
-                    if (((unsigned char *)wolfcrypt_linuxkm_fpu_states[processor_id])[PAGE_SIZE-1] == 255) {
-                        preempt_enable();
-                        pr_err("save_vector_registers_x86 recursion register overflow for "
-                               "cpu id %d.\n", processor_id);
-                        return BAD_STATE_E;
-                    } else {
-                        ++((unsigned char *)wolfcrypt_linuxkm_fpu_states[processor_id])[PAGE_SIZE-1];
-                        return 0;
-                    }
-                }
-                /* note, fpregs_lock() is not needed here, because
-                 * interrupts/preemptions are already disabled here.
-                 */
-                {
-                    /* save_fpregs_to_fpstate() only accesses fpu->state, which
-                     * has stringent alignment requirements (64 byte cache
-                     * line), but takes a pointer to the parent struct.  work
-                     * around this.
-                     */
-                #if LINUX_VERSION_CODE < KERNEL_VERSION(5, 14, 0)
-                    struct fpu *fake_fpu_pointer =
-                        (struct fpu *)(((char *)wolfcrypt_linuxkm_fpu_states[processor_id])
-                                       - offsetof(struct fpu, state));
-                    copy_fpregs_to_fpstate(fake_fpu_pointer);
-                #elif LINUX_VERSION_CODE < KERNEL_VERSION(5, 16, 0)
-                    struct fpu *fake_fpu_pointer =
-                        (struct fpu *)(((char *)wolfcrypt_linuxkm_fpu_states[processor_id])
-                                       - offsetof(struct fpu, state));
-                    save_fpregs_to_fpstate(fake_fpu_pointer);
-                #else
-                    struct fpu *fake_fpu_pointer =
-                        (struct fpu *)(((char *)wolfcrypt_linuxkm_fpu_states[processor_id])
-                                       - offsetof(struct fpu, fpstate));
-                    save_fpregs_to_fpstate(fake_fpu_pointer);
-                #endif
-                }
-                /* mark the slot as used. */
-                ((unsigned char *)wolfcrypt_linuxkm_fpu_states[processor_id])[PAGE_SIZE-1] = 1;
-                /* note, not preempt_enable()ing, mirroring kernel_fpu_begin()
-                 * semantics, even though routine will have been entered already
-                 * non-preemptable.
-                 */
-                return 0;
-            } else
-#endif /* LINUXKM_SIMD_IRQ */
-            {
-                preempt_enable();
-                return BAD_STATE_E;
-            }
-        } else {
+    if (nr_cpu_ids >= 16)
+        wc_linuxkm_fpu_states_n_tracked = nr_cpu_ids * 2;
+    else
+        wc_linuxkm_fpu_states_n_tracked = 32;
 
-            /* allow for nested calls */
-#ifdef LINUXKM_SIMD_IRQ
-            if (((unsigned char *)wolfcrypt_linuxkm_fpu_states[processor_id])[PAGE_SIZE-1] != 0) {
-                if (((unsigned char *)wolfcrypt_linuxkm_fpu_states[processor_id])[PAGE_SIZE-1] == 255) {
-                    preempt_enable();
-                    pr_err("save_vector_registers_x86 recursion register overflow for "
-                           "cpu id %d.\n", processor_id);
-                    return BAD_STATE_E;
-                } else {
-                    ++((unsigned char *)wolfcrypt_linuxkm_fpu_states[processor_id])[PAGE_SIZE-1];
-                    return 0;
-                }
-            }
-            kernel_fpu_begin();
-            preempt_enable(); /* kernel_fpu_begin() does its own
-                               * preempt_disable().  decrement ours.
-                               */
-            ((unsigned char *)wolfcrypt_linuxkm_fpu_states[processor_id])[PAGE_SIZE-1] = 1;
-#else /* !LINUXKM_SIMD_IRQ */
-            if (wolfcrypt_linuxkm_fpu_states[processor_id] != 0U) {
-                if ((wolfcrypt_linuxkm_fpu_states[processor_id] & WC_FPU_COUNT_MASK)
-                    == WC_FPU_COUNT_MASK)
-                {
-                    preempt_enable();
-                    pr_err("save_vector_registers_x86 recursion register overflow for "
-                           "cpu id %d.\n", processor_id);
-                    return BAD_STATE_E;
-                } else {
-                    ++wolfcrypt_linuxkm_fpu_states[processor_id];
-                    return 0;
-                }
-            }
+    wc_linuxkm_fpu_states =
+        (struct wc_thread_fpu_count_ent *)malloc(
+            wc_linuxkm_fpu_states_n_tracked * sizeof(wc_linuxkm_fpu_states[0]));
 
-            /* if kernel_fpu_begin() won't actually save the reg file (because
-             * it was already saved and invalidated, or because we're in a
-             * kernel thread), don't call kernel_fpu_begin() here, nor call
-             * kernel_fpu_end() in cleanup.  this avoids pointless overhead.  in
-             * kernels >=5.17.12 (from changes to irq_fpu_usable() in linux
-             * commit 59f5ede3bc0f, backported somewhere >5.17.5), this also
-             * fixes register corruption.
-             */
-            if ((current->flags & PF_KTHREAD) ||
-                test_thread_flag(TIF_NEED_FPU_LOAD))
-            {
-                wolfcrypt_linuxkm_fpu_states[processor_id] =
-                    WC_FPU_SAVED_MASK + 1U; /* set msb 1 to inhibit kernel_fpu_end() at cleanup. */
-                /* keep preempt_disable()d from above. */
-            } else {
-              kernel_fpu_begin();
-              preempt_enable(); /* kernel_fpu_begin() does its own
-                                 * preempt_disable().  decrement ours.
-                                 */
-              wolfcrypt_linuxkm_fpu_states[processor_id] = 1U; /* set msb 0 to trigger kernel_fpu_end() at cleanup. */
-            }
-#endif /* !LINUXKM_SIMD_IRQ */
+    if (! wc_linuxkm_fpu_states) {
+        pr_err("allocation of %lu bytes for "
+               "wc_linuxkm_fpu_states failed.\n",
+               nr_cpu_ids * sizeof(struct fpu_state *));
+        return MEMORY_E;
+    }
 
-            return 0;
-        }
+    memset(wc_linuxkm_fpu_states, 0, wc_linuxkm_fpu_states_n_tracked * sizeof(wc_linuxkm_fpu_states[0]));
+
+    return 0;
+}
+
+void free_wolfcrypt_linuxkm_fpu_states(void) {
+    struct wc_thread_fpu_count_ent *i, *i_endptr;
+    pid_t i_pid;
+
+    if (wc_linuxkm_fpu_states == NULL) {
+        pr_err("free_wolfcrypt_linuxkm_fpu_states called"
+               " before allocate_wolfcrypt_linuxkm_fpu_states.\n");
+        return;
     }
-    void restore_vector_registers_x86(void)
-    {
-        int processor_id = smp_processor_id();
 
-        if ((wolfcrypt_linuxkm_fpu_states == NULL)
-#ifdef LINUXKM_SIMD_IRQ
-            || (wolfcrypt_linuxkm_fpu_states[processor_id] == NULL)
-#endif
-           )
-        {
-                pr_err("restore_vector_registers_x86 called for cpu id %d "
-                       "with null context buffer.\n", processor_id);
-                return;
+    for (i = wc_linuxkm_fpu_states,
+             i_endptr = &wc_linuxkm_fpu_states[wc_linuxkm_fpu_states_n_tracked];
+         i < i_endptr;
+         ++i)
+    {
+        i_pid = __atomic_load_n(&i->pid, __ATOMIC_CONSUME);
+        if (i_pid == 0)
+            continue;
+        if (i->fpu_state != 0) {
+            pr_err("free_wolfcrypt_linuxkm_fpu_states called"
+                   " with nonzero state 0x%x for pid %d.\n", i->fpu_state, i_pid);
+            i->fpu_state = 0;
         }
+    }
+
+    free(wc_linuxkm_fpu_states);
+    wc_linuxkm_fpu_states = NULL;
+}
 
-#ifdef LINUXKM_SIMD_IRQ
-        if (((unsigned char *)wolfcrypt_linuxkm_fpu_states[processor_id])[PAGE_SIZE-1] == 0)
+/* lock-(mostly)-free thread-local storage facility for tracking recursive fpu pushing/popping */
+static struct wc_thread_fpu_count_ent *wc_linuxkm_fpu_state_assoc(int create_p) {
+    struct wc_thread_fpu_count_ent *i, *i_endptr, *i_empty;
+    pid_t my_pid = task_pid_nr(current), i_pid;
+
+    {
+        static int _warned_on_null = 0;
+        if (wc_linuxkm_fpu_states == NULL)
         {
-            pr_err("restore_vector_registers_x86 called for cpu id %d "
-                   "without saved context.\n", processor_id);
-            return;
+            if (_warned_on_null == 0) {
+                pr_err("wc_linuxkm_fpu_state_assoc called by pid %d"
+                       " before allocate_wolfcrypt_linuxkm_fpu_states.\n", my_pid);
+                _warned_on_null = 1;
+            }
+            return NULL;
         }
+    }
 
-        if (--((unsigned char *)wolfcrypt_linuxkm_fpu_states[processor_id])[PAGE_SIZE-1] > 0) {
-            preempt_enable(); /* preempt_disable count will still be nonzero after this decrement. */
-            return;
-        }
+    i_endptr = &wc_linuxkm_fpu_states[wc_linuxkm_fpu_states_n_tracked];
 
-        if (am_in_hard_interrupt_handler()) {
-        #if LINUX_VERSION_CODE < KERNEL_VERSION(5, 14, 0)
-            copy_kernel_to_fpregs(wolfcrypt_linuxkm_fpu_states[processor_id]);
-        #elif LINUX_VERSION_CODE < KERNEL_VERSION(5, 16, 0)
-            __restore_fpregs_from_fpstate(wolfcrypt_linuxkm_fpu_states[processor_id],
-                                          xfeatures_mask_all);
-        #else
-            restore_fpregs_from_fpstate(wolfcrypt_linuxkm_fpu_states[processor_id],
-                                          fpu_kernel_cfg.max_features);
-        #endif
-            preempt_enable();
-        } else {
-            kernel_fpu_end();
+    for (;;) {
+        for (i = wc_linuxkm_fpu_states,
+                 i_empty = NULL;
+             i < i_endptr;
+             ++i)
+        {
+            i_pid = __atomic_load_n(&i->pid, __ATOMIC_CONSUME);
+            if (i_pid == my_pid)
+                return i;
+            if ((i_empty == NULL) && (i_pid == 0))
+                i_empty = i;
         }
-#else /* !LINUXKM_SIMD_IRQ */
-        if ((wolfcrypt_linuxkm_fpu_states[processor_id] & WC_FPU_COUNT_MASK) == 0U)
+        if ((i_empty == NULL) || (! create_p))
+            return NULL;
+
+        i_pid = 0;
+        if (__atomic_compare_exchange_n(
+                &(i_empty->pid),
+                &i_pid,
+                my_pid,
+                0 /* weak */,
+                __ATOMIC_SEQ_CST /* success_memmodel */,
+                __ATOMIC_SEQ_CST /* failure_memmodel */))
         {
-            pr_err("restore_vector_registers_x86 called for cpu id %d "
-                   "without saved context.\n", processor_id);
-            return;
+            return i_empty;
         }
+    }
+}
 
-        if ((--wolfcrypt_linuxkm_fpu_states[processor_id] & WC_FPU_COUNT_MASK) > 0U) {
-            preempt_enable(); /* preempt_disable count may still be nonzero
-                               * after this decrement, but any remaining
-                               * count(s) aren't ours.
-                               */
-            return;
+static void wc_linuxkm_fpu_state_free(struct wc_thread_fpu_count_ent *ent) {
+    if (ent->fpu_state != 0) {
+        static int warned_nonzero_fpu_state = 0;
+        if (! warned_nonzero_fpu_state) {
+            pr_err("wc_linuxkm_fpu_state_free for pid %d"
+                   " with nonzero fpu_state 0x%x.\n", ent->pid, ent->fpu_state);
+            warned_nonzero_fpu_state = 1;
         }
+        ent->fpu_state = 0;
+    }
+    __atomic_store_n(&ent->pid, 0, __ATOMIC_RELEASE);
+}
+
+WARN_UNUSED_RESULT int save_vector_registers_x86(void)
+{
+    struct wc_thread_fpu_count_ent *pstate = wc_linuxkm_fpu_state_assoc(1);
+    if (pstate == NULL)
+        return ENOMEM;
 
-        if (wolfcrypt_linuxkm_fpu_states[processor_id] == 0U) {
-            kernel_fpu_end();
+    /* allow for nested calls */
+    if (pstate->fpu_state != 0U) {
+        if ((pstate->fpu_state & WC_FPU_COUNT_MASK)
+            == WC_FPU_COUNT_MASK)
+        {
+            pr_err("save_vector_registers_x86 recursion register overflow for "
+                   "pid %d.\n", pstate->pid);
+            return BAD_STATE_E;
         } else {
-            preempt_enable(); /* preempt_disable count will still be nonzero
-                               * after this decrement.
-                               */
-            wolfcrypt_linuxkm_fpu_states[processor_id] = 0U;
+            ++pstate->fpu_state;
+            return 0;
         }
-#endif /* !LINUXKM_SIMD_IRQ */
+    }
+
+    if (irq_fpu_usable()) {
+#if defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_COUNT) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0))
+        /* inhibit migration, which gums up the algorithm in kernel_fpu_{begin,end}(). */
+        migrate_disable();
+#endif
+        kernel_fpu_begin();
+        pstate->fpu_state = 1U; /* set msb 0 to trigger kernel_fpu_end() at cleanup. */
+    } else if (in_nmi() || (hardirq_count() > 0) || (softirq_count() > 0)) {
+        static int warned_fpu_forbidden = 0;
+        if (! warned_fpu_forbidden)
+            pr_err("save_vector_registers_x86 called from IRQ handler.\n");
+        wc_linuxkm_fpu_state_free(pstate);
+        return EPERM;
+    } else {
+#if defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_COUNT) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0))
+        migrate_disable();
+#endif
+        /* assume already safely in_kernel_fpu. */
+        pstate->fpu_state =
+            WC_FPU_SAVED_MASK + 1U; /* set msb 1 to inhibit kernel_fpu_end() at cleanup. */
+    }
 
+    return 0;
+}
+
+void restore_vector_registers_x86(void)
+{
+    struct wc_thread_fpu_count_ent *pstate = wc_linuxkm_fpu_state_assoc(0);
+    if (pstate == NULL) {
+        pr_err("restore_vector_registers_x86 called by pid %d "
+               "with no saved state.\n", task_pid_nr(current));
         return;
     }
+
+    if ((--pstate->fpu_state & WC_FPU_COUNT_MASK) > 0U) {
+        return;
+    }
+
+    if (pstate->fpu_state == 0U)
+        kernel_fpu_end();
+    else
+        pstate->fpu_state = 0U;
+#if defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_COUNT) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0))
+    migrate_enable();
+#endif
+
+    wc_linuxkm_fpu_state_free(pstate);
+
+    return;
+}
 #endif /* WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS && CONFIG_X86 */
 
 #if defined(__PIE__) && (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 1, 0))

+ 119 - 107
linuxkm/linuxkm_wc_port.h

@@ -24,6 +24,12 @@
 #ifndef LINUXKM_WC_PORT_H
 #define LINUXKM_WC_PORT_H
 
+    #include <linux/version.h>
+
+    #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 16, 0)
+        #error Unsupported kernel.
+    #endif
+
     #ifdef HAVE_CONFIG_H
         #ifndef PACKAGE_NAME
             #error wc_port.h included before config.h
@@ -59,6 +65,23 @@
           (int)_xatoi_res;                              \
         })
 
+    /* Kbuild+gcc on x86 doesn't consistently honor the default ALIGN16 on stack objects,
+     *   but gives adequate alignment with "32".
+     */
+    #if defined(CONFIG_X86) && !defined(ALIGN16)
+        #define ALIGN16 __attribute__ ( (aligned (32)))
+    #endif
+
+    /* kvmalloc()/kvfree() and friends added in linux commit a7c3e901 */
+    #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 12, 0)
+        #define HAVE_KVMALLOC
+    #endif
+
+    /* kernel printf doesn't implement fp. */
+    #ifndef WOLFSSL_NO_FLOAT_FMT
+        #define WOLFSSL_NO_FLOAT_FMT
+    #endif
+
     #ifdef BUILDING_WOLFSSL
 
     #if defined(CONFIG_MIPS) && defined(HAVE_LINUXKM_PIE_SUPPORT)
@@ -95,7 +118,6 @@
 
     #include <linux/kconfig.h>
     #include <linux/kernel.h>
-    #include <linux/version.h>
     #include <linux/ctype.h>
     #include <linux/init.h>
     #include <linux/module.h>
@@ -124,6 +146,8 @@
         #ifndef CONFIG_X86
             #error X86 SIMD extensions requested, but CONFIG_X86 is not set.
         #endif
+        #define WOLFSSL_LINUXKM_SIMD
+        #define WOLFSSL_LINUXKM_SIMD_X86
         #ifndef WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS
             #define WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS
         #endif
@@ -133,6 +157,8 @@
         #if !defined(CONFIG_ARM) && !defined(CONFIG_ARM64)
             #error ARM SIMD extensions requested, but CONFIG_ARM* is not set.
         #endif
+        #define WOLFSSL_LINUXKM_SIMD
+        #define WOLFSSL_LINUXKM_SIMD_ARM
         #ifndef WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS
             #define WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS
         #endif
@@ -142,26 +168,17 @@
         #endif
     #endif
 
+    /* benchmarks.c uses floating point math, so needs a working SAVE_VECTOR_REGISTERS(). */
+    #if defined(WOLFSSL_LINUXKM_BENCHMARKS) && !defined(WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS)
+        #define WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS
+    #endif
+
     #if defined(WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS) && defined(CONFIG_X86)
-        #define WOLFSSL_LINUXKM_SIMD
-        #define WOLFSSL_LINUXKM_SIMD_X86
         #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
             #include <asm/i387.h>
         #else
             #include <asm/simd.h>
         #endif
-        #ifdef LINUXKM_SIMD_IRQ
-            #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 0, 0)
-                #include <asm/fpu/internal.h>
-            #endif
-            #if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 16, 0)
-                #error LINUXKM_SIMD_IRQ is unavailable on linux >= 5.16 (missing exports around fpregs)
-                /*
-                 * #include <asm/fpu/sched.h>
-                 * #include <asm/fpu/signal.h>
-                 */
-            #endif
-        #endif
         #ifndef SAVE_VECTOR_REGISTERS
             #define SAVE_VECTOR_REGISTERS(fail_clause) { int _svr_ret = save_vector_registers_x86(); if (_svr_ret != 0) { fail_clause } }
         #endif
@@ -169,12 +186,7 @@
             #define RESTORE_VECTOR_REGISTERS() restore_vector_registers_x86()
         #endif
     #elif defined(WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS) && (defined(CONFIG_ARM) || defined(CONFIG_ARM64))
-        #define WOLFSSL_LINUXKM_SIMD
-        #define WOLFSSL_LINUXKM_SIMD_ARM
         #include <asm/fpsimd.h>
-        #ifdef LINUXKM_SIMD_IRQ
-            #error LINUXKM_SIMD_IRQ is unavailable on ARM (not implemented)
-        #endif
         #ifndef SAVE_VECTOR_REGISTERS
             #define SAVE_VECTOR_REGISTERS(fail_clause) { int _svr_ret = save_vector_registers_arm(); if (_svr_ret != 0) { fail_clause } }
         #endif
@@ -195,11 +207,6 @@
     #define NO_THREAD_LS
     #define NO_ATTRIBUTE_CONSTRUCTOR
 
-    /* kvmalloc()/kvfree() and friends added in linux commit a7c3e901 */
-    #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 12, 0)
-        #define HAVE_KVMALLOC
-    #endif
-
     #ifdef HAVE_FIPS
         extern int wolfCrypt_FIPS_first(void);
         extern int wolfCrypt_FIPS_last(void);
@@ -215,7 +222,7 @@
     #endif
 
     #if defined(__PIE__) && !defined(USE_WOLFSSL_LINUXKM_PIE_REDIRECT_TABLE)
-        #error "compiling -fPIE without PIE support."
+        #error "compiling -fPIE requires PIE redirect table."
     #endif
 
     #if defined(HAVE_FIPS) && !defined(HAVE_LINUXKM_PIE_SUPPORT)
@@ -307,42 +314,37 @@
         struct task_struct *(*get_current)(void);
         int (*preempt_count)(void);
 
-        #ifdef WOLFSSL_LINUXKM_SIMD_X86
-        typeof(irq_fpu_usable) *irq_fpu_usable;
-        /* kernel_fpu_begin() replaced by kernel_fpu_begin_mask() in commit e4512289,
-         * released in kernel 5.11, backported to 5.4.93
-         */
-        #ifdef kernel_fpu_begin
-            typeof(kernel_fpu_begin_mask) *kernel_fpu_begin_mask;
-        #else
-            typeof(kernel_fpu_begin) *kernel_fpu_begin;
-        #endif
-        typeof(kernel_fpu_end) *kernel_fpu_end;
-
-        #ifdef LINUXKM_SIMD_IRQ
-            #if LINUX_VERSION_CODE < KERNEL_VERSION(5, 14, 0)
-                typeof(copy_fpregs_to_fpstate) *copy_fpregs_to_fpstate;
-                typeof(copy_kernel_to_fpregs) *copy_kernel_to_fpregs;
-            #elif LINUX_VERSION_CODE < KERNEL_VERSION(5, 16, 0)
-                typeof(save_fpregs_to_fpstate) *save_fpregs_to_fpstate;
-                typeof(__restore_fpregs_from_fpstate) *__restore_fpregs_from_fpstate;
-                typeof(xfeatures_mask_all) *xfeatures_mask_all;
-            /*
-             * #else
-             *  typeof(save_fpregs_to_fpstate) *save_fpregs_to_fpstate;
-             *  typeof(restore_fpregs_from_fpstate) *restore_fpregs_from_fpstate;
-             *  typeof(fpu_kernel_cfg) *fpu_kernel_cfg;
-             */
+        #ifdef WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS
+
+            #if LINUX_VERSION_CODE < KERNEL_VERSION(6, 2, 0)
+                typeof(cpu_number) *cpu_number;
+            #else
+                typeof(pcpu_hot) *pcpu_hot;
+            #endif
+            typeof(nr_cpu_ids) *nr_cpu_ids;
+
+            #if defined(CONFIG_SMP) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0))
+                /* note the current and needed version of these were added in af449901b8 (2020-Sep-17) */
+                typeof(migrate_disable) *migrate_disable;
+                typeof(migrate_enable) *migrate_enable;
             #endif
-        #endif
-        #if LINUX_VERSION_CODE < KERNEL_VERSION(6, 2, 0)
-            typeof(cpu_number) *cpu_number;
-        #else
-            typeof(pcpu_hot) *pcpu_hot;
-        #endif
-        typeof(nr_cpu_ids) *nr_cpu_ids;
 
-        #endif /* WOLFSSL_LINUXKM_SIMD_X86 */
+            #ifdef CONFIG_X86
+                typeof(irq_fpu_usable) *irq_fpu_usable;
+                /* kernel_fpu_begin() replaced by kernel_fpu_begin_mask() in commit e4512289,
+                 * released in kernel 5.11, backported to 5.4.93
+                 */
+                #ifdef kernel_fpu_begin
+                    typeof(kernel_fpu_begin_mask) *kernel_fpu_begin_mask;
+                #else
+                    typeof(kernel_fpu_begin) *kernel_fpu_begin;
+                #endif
+                typeof(kernel_fpu_end) *kernel_fpu_end;
+            #else /* !CONFIG_X86 */
+                #error WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS is set for an unsupported architecture.
+            #endif /* arch */
+
+        #endif /* WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS */
 
         typeof(__mutex_init) *__mutex_init;
         #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
@@ -452,37 +454,31 @@
     #undef preempt_count
     #define preempt_count (wolfssl_linuxkm_get_pie_redirect_table()->preempt_count)
 
-    #ifdef WOLFSSL_LINUXKM_SIMD_X86
-        #define irq_fpu_usable (wolfssl_linuxkm_get_pie_redirect_table()->irq_fpu_usable)
-        #ifdef kernel_fpu_begin
-            #define kernel_fpu_begin_mask (wolfssl_linuxkm_get_pie_redirect_table()->kernel_fpu_begin_mask)
-        #else
-            #define kernel_fpu_begin (wolfssl_linuxkm_get_pie_redirect_table()->kernel_fpu_begin)
-        #endif
-        #define kernel_fpu_end (wolfssl_linuxkm_get_pie_redirect_table()->kernel_fpu_end)
-        #ifdef LINUXKM_SIMD_IRQ
-            #if LINUX_VERSION_CODE < KERNEL_VERSION(5, 14, 0)
-                #define copy_fpregs_to_fpstate (wolfssl_linuxkm_get_pie_redirect_table()->copy_fpregs_to_fpstate)
-                #define copy_kernel_to_fpregs (wolfssl_linuxkm_get_pie_redirect_table()->copy_kernel_to_fpregs)
-            #elif LINUX_VERSION_CODE < KERNEL_VERSION(5, 16, 0)
-                #define save_fpregs_to_fpstate (wolfssl_linuxkm_get_pie_redirect_table()->save_fpregs_to_fpstate)
-                #define __restore_fpregs_from_fpstate (wolfssl_linuxkm_get_pie_redirect_table()->__restore_fpregs_from_fpstate)
-                #define xfeatures_mask_all (*(wolfssl_linuxkm_get_pie_redirect_table()->xfeatures_mask_all))
-            /*
-             * #else
-             *  #define save_fpregs_to_fpstate (wolfssl_linuxkm_get_pie_redirect_table()->save_fpregs_to_fpstate)
-             *  #define restore_fpregs_from_fpstate (wolfssl_linuxkm_get_pie_redirect_table()->restore_fpregs_from_fpstate)
-             *  #define fpu_kernel_cfg (*(wolfssl_linuxkm_get_pie_redirect_table()->fpu_kernel_cfg))
-             */
-            #endif
-        #endif
+    #ifdef WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS
         #if LINUX_VERSION_CODE < KERNEL_VERSION(6, 2, 0)
             #define cpu_number (*(wolfssl_linuxkm_get_pie_redirect_table()->cpu_number))
         #else
             #define pcpu_hot (*(wolfssl_linuxkm_get_pie_redirect_table()->pcpu_hot))
         #endif
         #define nr_cpu_ids (*(wolfssl_linuxkm_get_pie_redirect_table()->nr_cpu_ids))
-    #endif
+
+        #if defined(CONFIG_SMP) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0))
+            #define migrate_disable (*(wolfssl_linuxkm_get_pie_redirect_table()->migrate_disable))
+            #define migrate_enable (*(wolfssl_linuxkm_get_pie_redirect_table()->migrate_enable))
+        #endif
+
+        #ifdef CONFIG_X86
+            #define irq_fpu_usable (wolfssl_linuxkm_get_pie_redirect_table()->irq_fpu_usable)
+            #ifdef kernel_fpu_begin
+                #define kernel_fpu_begin_mask (wolfssl_linuxkm_get_pie_redirect_table()->kernel_fpu_begin_mask)
+            #else
+                #define kernel_fpu_begin (wolfssl_linuxkm_get_pie_redirect_table()->kernel_fpu_begin)
+            #endif
+            #define kernel_fpu_end (wolfssl_linuxkm_get_pie_redirect_table()->kernel_fpu_end)
+        #else /* !CONFIG_X86 */
+            #error WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS is set for an unsupported architecture.
+        #endif /* archs */
+    #endif /* WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS */
 
     #define __mutex_init (wolfssl_linuxkm_get_pie_redirect_table()->__mutex_init)
     #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
@@ -515,9 +511,9 @@
 
     #endif /* USE_WOLFSSL_LINUXKM_PIE_REDIRECT_TABLE */
 
-#ifdef WOLFSSL_LINUXKM_SIMD
+#ifdef WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS
 
-#ifdef WOLFSSL_LINUXKM_SIMD_X86
+#ifdef CONFIG_X86
 
     extern __must_check int allocate_wolfcrypt_linuxkm_fpu_states(void);
     extern void free_wolfcrypt_linuxkm_fpu_states(void);
@@ -547,7 +543,7 @@
 
 #endif
 
-#endif /* WOLFSSL_LINUXKM_SIMD */
+#endif /* WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS */
 
     /* remove this multifariously conflicting macro, picked up from
      * Linux arch/<arch>/include/asm/current.h.
@@ -556,22 +552,6 @@
         #undef current
     #endif
 
-    /* prevent gcc's mm_malloc.h from being included, since it unconditionally
-     * includes stdlib.h, which is kernel-incompatible.
-     */
-    #define _MM_MALLOC_H_INCLUDED
-
-    #ifdef HAVE_KVMALLOC
-        #define malloc(x) kvmalloc_node(x, GFP_KERNEL, NUMA_NO_NODE)
-        #define free(x) kvfree(x)
-        void *lkm_realloc(void *ptr, size_t newsize);
-        #define realloc(x, y) lkm_realloc(x, y)
-    #else
-        #define malloc(x) kmalloc(x, GFP_KERNEL)
-        #define free(x) kfree(x)
-        #define realloc(x,y) krealloc(x, y, GFP_KERNEL)
-    #endif
-
     /* min() and max() in linux/kernel.h over-aggressively type-check, producing
      * myriad spurious -Werrors throughout the codebase.
      */
@@ -618,9 +598,41 @@
     #include <linux/mutex.h>
     typedef struct mutex wolfSSL_Mutex;
 
-    #define XMALLOC(s, h, t)     ({(void)(h); (void)(t); kmalloc(s, GFP_KERNEL);})
-    #define XFREE(p, h, t)       ({void* _xp; (void)(h); _xp = (p); if(_xp) kfree(_xp);})
-    #define XREALLOC(p, n, h, t) ({(void)(h); (void)(t); krealloc((p), (n), GFP_KERNEL);})
+    /* prevent gcc's mm_malloc.h from being included, since it unconditionally
+     * includes stdlib.h, which is kernel-incompatible.
+     */
+    #define _MM_MALLOC_H_INCLUDED
+
+    /* fun fact: since linux commit 59bb47985c, kmalloc with power-of-2 size is
+     * aligned to the size.
+     */
+    #define WC_LINUXKM_ROUND_UP_P_OF_2(x) (                                         \
+    {                                                                               \
+        size_t _alloc_sz = (x);                                                     \
+        _alloc_sz = 1UL << ((sizeof(_alloc_sz) * 8UL) - __builtin_clzl(_alloc_sz)); \
+        _alloc_sz;                                                                  \
+    })
+    #ifdef HAVE_KVMALLOC
+        #define malloc(size) kvmalloc_node(WC_LINUXKM_ROUND_UP_P_OF_2(size), GFP_KERNEL, NUMA_NO_NODE)
+        #define free(ptr) kvfree(ptr)
+        void *lkm_realloc(void *ptr, size_t newsize);
+        #define realloc(ptr, newsize) lkm_realloc(ptr, WC_LINUXKM_ROUND_UP_P_OF_2(newsize))
+    #else
+        #define malloc(size) kmalloc(WC_LINUXKM_ROUND_UP_P_OF_2(size), GFP_KERNEL)
+        #define free(ptr) kfree(ptr)
+        #define realloc(ptr, newsize) krealloc(ptr, WC_LINUXKM_ROUND_UP_P_OF_2(newsize), GFP_KERNEL)
+    #endif
+
+#ifdef WOLFSSL_TRACK_MEMORY
+    #include <wolfssl/wolfcrypt/memory.h>
+    #define XMALLOC(s, h, t)     ({(void)(h); (void)(t); wolfSSL_Malloc(s);})
+    #define XFREE(p, h, t)       ({void* _xp; (void)(h); _xp = (p); if(_xp) wolfSSL_Free(_xp);})
+    #define XREALLOC(p, n, h, t) ({(void)(h); (void)(t); wolfSSL_Realloc(p, n);})
+#else
+    #define XMALLOC(s, h, t)     ({(void)(h); (void)(t); malloc(s);})
+    #define XFREE(p, h, t)       ({void* _xp; (void)(h); _xp = (p); if(_xp) free(_xp);})
+    #define XREALLOC(p, n, h, t) ({(void)(h); (void)(t); realloc(p, n);})
+#endif
 
     #include <linux/limits.h>
 

+ 32 - 35
linuxkm/module_hooks.c

@@ -113,6 +113,15 @@ static void lkmFipsCb(int ok, int err, const char* hash)
 static int updateFipsHash(void);
 #endif
 
+#ifdef WOLFSSL_LINUXKM_BENCHMARKS
+#undef HAVE_PTHREAD
+#define STRING_USER
+#define NO_MAIN_FUNCTION
+#define current_time benchmark_current_time
+#define WOLFSSL_NO_FLOAT_FMT
+#include "/home/douzzer/com/wolfssl/src/wolfssl/wolfcrypt/benchmark/benchmark.c"
+#endif /* WOLFSSL_LINUXKM_BENCHMARKS */
+
 #if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 0, 0)
 static int __init wolfssl_init(void)
 #else
@@ -202,10 +211,9 @@ static int wolfssl_init(void)
          * the true module start address, which is potentially useful to an
          * attacker.
          */
-        pr_info("wolfCrypt container hashes (spans): %x (%lu) %x (%lu), text base %pK, ro base %pK\n",
+        pr_info("wolfCrypt container hashes (spans): text 0x%x (%lu), rodata 0x%x (%lu)\n",
                 text_hash, pie_text_end-pie_text_start,
-                rodata_hash, pie_rodata_end-pie_rodata_start,
-                THIS_MODULE_TEXT_BASE, THIS_MODULE_RO_BASE);
+                rodata_hash, pie_rodata_end-pie_rodata_start);
     }
 #endif /* HAVE_LINUXKM_PIE_SUPPORT */
 
@@ -277,6 +285,10 @@ static int wolfssl_init(void)
     pr_info("wolfCrypt self-test passed.\n");
 #endif
 
+#ifdef WOLFSSL_LINUXKM_BENCHMARKS
+    wolfcrypt_benchmark_main(0, (char**)NULL);
+#endif
+
 #ifdef WOLFCRYPT_ONLY
     pr_info("wolfCrypt " LIBWOLFSSL_VERSION_STRING " loaded%s"
             ".\nSee https://www.wolfssl.com/ for more information.\n"
@@ -334,15 +346,6 @@ static int my_preempt_count(void) {
     return preempt_count();
 }
 
-#if defined(WOLFSSL_LINUXKM_SIMD_X86) && (LINUX_VERSION_CODE < KERNEL_VERSION(5, 14, 0))
-static int my_copy_fpregs_to_fpstate(struct fpu *fpu) {
-    return copy_fpregs_to_fpstate(fpu);
-}
-static void my_copy_kernel_to_fpregs(union fpregs_state *fpstate) {
-    copy_kernel_to_fpregs(fpstate);
-}
-#endif
-
 static int set_up_wolfssl_linuxkm_pie_redirect_table(void) {
     memset(
         &wolfssl_linuxkm_pie_redirect_table,
@@ -430,6 +433,20 @@ static int set_up_wolfssl_linuxkm_pie_redirect_table(void) {
     wolfssl_linuxkm_pie_redirect_table.get_current = my_get_current_thread;
     wolfssl_linuxkm_pie_redirect_table.preempt_count = my_preempt_count;
 
+#ifdef WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS
+
+    #if LINUX_VERSION_CODE < KERNEL_VERSION(6, 2, 0)
+        wolfssl_linuxkm_pie_redirect_table.cpu_number = &cpu_number;
+    #else
+        wolfssl_linuxkm_pie_redirect_table.pcpu_hot = &pcpu_hot;
+    #endif
+    wolfssl_linuxkm_pie_redirect_table.nr_cpu_ids = &nr_cpu_ids;
+
+    #if defined(CONFIG_SMP) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0))
+        wolfssl_linuxkm_pie_redirect_table.migrate_disable = &migrate_disable;
+        wolfssl_linuxkm_pie_redirect_table.migrate_enable = &migrate_enable;
+    #endif
+
 #ifdef WOLFSSL_LINUXKM_SIMD_X86
     wolfssl_linuxkm_pie_redirect_table.irq_fpu_usable = irq_fpu_usable;
     #ifdef kernel_fpu_begin
@@ -440,29 +457,9 @@ static int set_up_wolfssl_linuxkm_pie_redirect_table(void) {
         kernel_fpu_begin;
     #endif
     wolfssl_linuxkm_pie_redirect_table.kernel_fpu_end = kernel_fpu_end;
-    #ifdef LINUXKM_SIMD_IRQ
-        #if LINUX_VERSION_CODE < KERNEL_VERSION(5, 14, 0)
-            wolfssl_linuxkm_pie_redirect_table.copy_fpregs_to_fpstate = my_copy_fpregs_to_fpstate;
-            wolfssl_linuxkm_pie_redirect_table.copy_kernel_to_fpregs = my_copy_kernel_to_fpregs;
-        #elif LINUX_VERSION_CODE < KERNEL_VERSION(5, 16, 0)
-            wolfssl_linuxkm_pie_redirect_table.save_fpregs_to_fpstate = save_fpregs_to_fpstate;
-            wolfssl_linuxkm_pie_redirect_table.__restore_fpregs_from_fpstate = __restore_fpregs_from_fpstate;
-            wolfssl_linuxkm_pie_redirect_table.xfeatures_mask_all = &xfeatures_mask_all;
-        /*
-         * #else
-         *  wolfssl_linuxkm_pie_redirect_table.save_fpregs_to_fpstate = save_fpregs_to_fpstate;
-         *  wolfssl_linuxkm_pie_redirect_table.restore_fpregs_from_fpstate = restore_fpregs_from_fpstate;
-         *  wolfssl_linuxkm_pie_redirect_table.fpu_kernel_cfg = &fpu_kernel_cfg;
-         */
-        #endif
-    #endif
-    #if LINUX_VERSION_CODE < KERNEL_VERSION(6, 2, 0)
-        wolfssl_linuxkm_pie_redirect_table.cpu_number = &cpu_number;
-    #else
-        wolfssl_linuxkm_pie_redirect_table.pcpu_hot = &pcpu_hot;
-    #endif
-    wolfssl_linuxkm_pie_redirect_table.nr_cpu_ids = &nr_cpu_ids;
-#endif
+#endif /* WOLFSSL_LINUXKM_SIMD_X86 */
+
+#endif /* WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS */
 
     wolfssl_linuxkm_pie_redirect_table.__mutex_init = __mutex_init;
     #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)

+ 2 - 52
wolfcrypt/src/wc_port.c

@@ -207,7 +207,7 @@ int wolfCrypt_Init(void)
         }
     #endif
 
-    #if defined(WOLFSSL_LINUXKM_SIMD_X86)
+    #ifdef WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS
         ret = allocate_wolfcrypt_linuxkm_fpu_states();
         if (ret != 0) {
             WOLFSSL_MSG("allocate_wolfcrypt_linuxkm_fpu_states failed");
@@ -466,7 +466,7 @@ int wolfCrypt_Cleanup(void)
         rpcmem_deinit();
         wolfSSL_CleanupHandle();
     #endif
-    #if defined(WOLFSSL_LINUXKM_SIMD_X86)
+    #ifdef WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS
         free_wolfcrypt_linuxkm_fpu_states();
     #endif
 
@@ -3207,56 +3207,6 @@ char* mystrnstr(const char* s1, const char* s2, unsigned int n)
 
 #endif /* WOLFSSL_NUCLEUS_1_2 */
 
-#if defined(WOLFSSL_LINUXKM) && defined(HAVE_KVMALLOC)
-    /* adapted from kvrealloc() draft by Changli Gao, 2010-05-13 */
-    void *lkm_realloc(void *ptr, size_t newsize) {
-        void *nptr;
-        size_t oldsize;
-
-        if (unlikely(newsize == 0)) {
-            kvfree(ptr);
-            return ZERO_SIZE_PTR;
-        }
-
-        if (unlikely(ptr == NULL))
-            return kvmalloc_node(newsize, GFP_KERNEL, NUMA_NO_NODE);
-
-        if (is_vmalloc_addr(ptr)) {
-            /* no way to discern the size of the old allocation,
-             * because the kernel doesn't export find_vm_area().  if
-             * it did, we could then call get_vm_area_size() on the
-             * returned struct vm_struct.
-             */
-            return NULL;
-        } else {
-#ifndef __PIE__
-            struct page *page;
-
-            page = virt_to_head_page(ptr);
-            if (PageSlab(page) || PageCompound(page)) {
-                if (newsize < PAGE_SIZE)
-#endif /* ! __PIE__ */
-                    return krealloc(ptr, newsize, GFP_KERNEL);
-#ifndef __PIE__
-                oldsize = ksize(ptr);
-            } else {
-                oldsize = page->private;
-                if (newsize <= oldsize)
-                    return ptr;
-            }
-#endif /* ! __PIE__ */
-        }
-
-        nptr = kvmalloc_node(newsize, GFP_KERNEL, NUMA_NO_NODE);
-        if (nptr != NULL) {
-            memcpy(nptr, ptr, oldsize);
-            kvfree(ptr);
-        }
-
-        return nptr;
-    }
-#endif /* WOLFSSL_LINUXKM && HAVE_KVMALLOC */
-
 #if defined(WOLFSSL_TI_CRYPT) || defined(WOLFSSL_TI_HASH)
     #include <wolfcrypt/src/port/ti/ti-ccm.c>  /* initialize and Mutex for TI Crypt Engine */
     #include <wolfcrypt/src/port/ti/ti-hash.c> /* md5, sha1, sha224, sha256 */