|
|
@@ -0,0 +1,575 @@
|
|
|
+/* x86_vector_register_glue.c -- glue logic to save and restore vector registers
|
|
|
+ * on x86
|
|
|
+ *
|
|
|
+ * Copyright (C) 2006-2024 wolfSSL Inc.
|
|
|
+ *
|
|
|
+ * This file is part of wolfSSL.
|
|
|
+ *
|
|
|
+ * wolfSSL is free software; you can redistribute it and/or modify
|
|
|
+ * it under the terms of the GNU General Public License as published by
|
|
|
+ * the Free Software Foundation; either version 2 of the License, or
|
|
|
+ * (at your option) any later version.
|
|
|
+ *
|
|
|
+ * wolfSSL is distributed in the hope that it will be useful,
|
|
|
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
+ * GNU General Public License for more details.
|
|
|
+ *
|
|
|
+ * You should have received a copy of the GNU General Public License
|
|
|
+ * along with this program; if not, write to the Free Software
|
|
|
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA
|
|
|
+ */
|
|
|
+
|
|
|
+/* included by linuxkm/module_hooks.c */
|
|
|
+
|
|
|
+#if !defined(WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS) || !defined(CONFIG_X86)
|
|
|
+ #error x86_vector_register_glue.c included in non-vectorized/non-x86 project.
|
|
|
+#endif
|
|
|
+
|
|
|
+/* kernel 4.19 -- the most recent LTS before 5.4 -- lacks the necessary safety
|
|
|
+ * checks in __kernel_fpu_begin(), and lacks TIF_NEED_FPU_LOAD.
|
|
|
+ */
|
|
|
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0))
|
|
|
+ #error WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS on x86 requires kernel 5.4.0 or higher.
|
|
|
+#endif
|
|
|
+
|
|
|
+static unsigned int wc_linuxkm_fpu_states_n_tracked = 0;
|
|
|
+
|
|
|
+struct wc_thread_fpu_count_ent {
|
|
|
+ volatile pid_t pid;
|
|
|
+ unsigned int fpu_state;
|
|
|
+};
|
|
|
+struct wc_thread_fpu_count_ent *wc_linuxkm_fpu_states = NULL;
|
|
|
+
|
|
|
+#ifdef WOLFSSL_COMMERCIAL_LICENSE
|
|
|
+
|
|
|
+#ifndef LINUXKM_FPU_STATES_FOLLOW_THREADS
|
|
|
+ #error WOLFSSL_COMMERCIAL_LICENSE requires LINUXKM_FPU_STATES_FOLLOW_THREADS
|
|
|
+#endif
|
|
|
+
|
|
|
+#pragma GCC diagnostic push
|
|
|
+#pragma GCC diagnostic ignored "-Wunused-parameter"
|
|
|
+#pragma GCC diagnostic ignored "-Wnested-externs"
|
|
|
+/* avoid dependence on "alternatives_patched" and "xfd_validate_state()". */
|
|
|
+#undef CONFIG_X86_DEBUG_FPU
|
|
|
+#include "../kernel/fpu/internal.h"
|
|
|
+#include "../kernel/fpu/xstate.h"
|
|
|
+#pragma GCC diagnostic pop
|
|
|
+
|
|
|
+static union wc_linuxkm_fpu_savebuf {
|
|
|
+ byte buf[1024]; /* must be 64-byte-aligned */
|
|
|
+ struct fpstate fpstate;
|
|
|
+} *wc_linuxkm_fpu_savebufs = NULL;
|
|
|
+
|
|
|
+#endif /* WOLFSSL_COMMERCIAL_LICENSE */
|
|
|
+
|
|
|
+#define WC_FPU_COUNT_MASK 0x7fffffffU
|
|
|
+#define WC_FPU_SAVED_MASK 0x80000000U
|
|
|
+
|
|
|
+WARN_UNUSED_RESULT int allocate_wolfcrypt_linuxkm_fpu_states(void)
|
|
|
+{
|
|
|
+ if (wc_linuxkm_fpu_states != NULL) {
|
|
|
+#ifdef HAVE_FIPS
|
|
|
+ /* see note below in wc_linuxkm_fpu_state_assoc_unlikely(). */
|
|
|
+ return 0;
|
|
|
+#else
|
|
|
+ static int warned_for_repeat_alloc = 0;
|
|
|
+ if (! warned_for_repeat_alloc) {
|
|
|
+ pr_err("attempt at repeat allocation"
|
|
|
+ " in allocate_wolfcrypt_linuxkm_fpu_states\n");
|
|
|
+ warned_for_repeat_alloc = 1;
|
|
|
+ }
|
|
|
+ return BAD_STATE_E;
|
|
|
+#endif
|
|
|
+ }
|
|
|
+
|
|
|
+#ifdef LINUXKM_FPU_STATES_FOLLOW_THREADS
|
|
|
+ if (nr_cpu_ids >= 16)
|
|
|
+ wc_linuxkm_fpu_states_n_tracked = nr_cpu_ids * 2;
|
|
|
+ else
|
|
|
+ wc_linuxkm_fpu_states_n_tracked = 32;
|
|
|
+#else
|
|
|
+ wc_linuxkm_fpu_states_n_tracked = nr_cpu_ids;
|
|
|
+#endif
|
|
|
+
|
|
|
+ wc_linuxkm_fpu_states =
|
|
|
+ (struct wc_thread_fpu_count_ent *)malloc(
|
|
|
+ wc_linuxkm_fpu_states_n_tracked * sizeof(wc_linuxkm_fpu_states[0]));
|
|
|
+
|
|
|
+ if (! wc_linuxkm_fpu_states) {
|
|
|
+ pr_err("allocation of %lu bytes for "
|
|
|
+ "wc_linuxkm_fpu_states failed.\n",
|
|
|
+ nr_cpu_ids * sizeof(struct fpu_state *));
|
|
|
+ return MEMORY_E;
|
|
|
+ }
|
|
|
+
|
|
|
+ memset(wc_linuxkm_fpu_states, 0, wc_linuxkm_fpu_states_n_tracked
|
|
|
+ * sizeof(wc_linuxkm_fpu_states[0]));
|
|
|
+
|
|
|
+#ifdef WOLFSSL_COMMERCIAL_LICENSE
|
|
|
+ wc_linuxkm_fpu_savebufs = (union wc_linuxkm_fpu_savebuf *)malloc(
|
|
|
+ wc_linuxkm_fpu_states_n_tracked * sizeof(*wc_linuxkm_fpu_savebufs));
|
|
|
+ if (! wc_linuxkm_fpu_savebufs) {
|
|
|
+ pr_err("allocation of %lu bytes for "
|
|
|
+ "wc_linuxkm_fpu_savebufs failed.\n",
|
|
|
+ WC_LINUXKM_ROUND_UP_P_OF_2(wc_linuxkm_fpu_states_n_tracked)
|
|
|
+ * sizeof(*wc_linuxkm_fpu_savebufs));
|
|
|
+ free(wc_linuxkm_fpu_states);
|
|
|
+ wc_linuxkm_fpu_states = NULL;
|
|
|
+ return MEMORY_E;
|
|
|
+ }
|
|
|
+ if ((uintptr_t)wc_linuxkm_fpu_savebufs
|
|
|
+ & (WC_LINUXKM_ROUND_UP_P_OF_2(sizeof(*wc_linuxkm_fpu_savebufs)) - 1))
|
|
|
+ {
|
|
|
+ pr_err("allocation of %lu bytes for "
|
|
|
+ "wc_linuxkm_fpu_savebufs allocated with wrong alignment 0x%lx.\n",
|
|
|
+ WC_LINUXKM_ROUND_UP_P_OF_2(wc_linuxkm_fpu_states_n_tracked)
|
|
|
+ * sizeof(*wc_linuxkm_fpu_savebufs),
|
|
|
+ (uintptr_t)wc_linuxkm_fpu_savebufs);
|
|
|
+ free(wc_linuxkm_fpu_savebufs);
|
|
|
+ wc_linuxkm_fpu_savebufs = NULL;
|
|
|
+ free(wc_linuxkm_fpu_states);
|
|
|
+ wc_linuxkm_fpu_states = NULL;
|
|
|
+ return MEMORY_E;
|
|
|
+ }
|
|
|
+
|
|
|
+#endif
|
|
|
+
|
|
|
+ return 0;
|
|
|
+}
|
|
|
+
|
|
|
+void free_wolfcrypt_linuxkm_fpu_states(void) {
|
|
|
+ struct wc_thread_fpu_count_ent *i, *i_endptr;
|
|
|
+ pid_t i_pid;
|
|
|
+
|
|
|
+ if (wc_linuxkm_fpu_states == NULL) {
|
|
|
+ pr_err("free_wolfcrypt_linuxkm_fpu_states called"
|
|
|
+ " before allocate_wolfcrypt_linuxkm_fpu_states.\n");
|
|
|
+ return;
|
|
|
+ }
|
|
|
+
|
|
|
+ for (i = wc_linuxkm_fpu_states,
|
|
|
+ i_endptr = &wc_linuxkm_fpu_states[wc_linuxkm_fpu_states_n_tracked];
|
|
|
+ i < i_endptr;
|
|
|
+ ++i)
|
|
|
+ {
|
|
|
+ i_pid = __atomic_load_n(&i->pid, __ATOMIC_CONSUME);
|
|
|
+ if (i_pid == 0)
|
|
|
+ continue;
|
|
|
+ if (i->fpu_state != 0) {
|
|
|
+ pr_err("free_wolfcrypt_linuxkm_fpu_states called"
|
|
|
+ " with nonzero state 0x%x for pid %d.\n", i->fpu_state, i_pid);
|
|
|
+ i->fpu_state = 0;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+#ifdef WOLFSSL_COMMERCIAL_LICENSE
|
|
|
+ free(wc_linuxkm_fpu_savebufs);
|
|
|
+ wc_linuxkm_fpu_savebufs = NULL;
|
|
|
+#endif
|
|
|
+ free(wc_linuxkm_fpu_states);
|
|
|
+ wc_linuxkm_fpu_states = NULL;
|
|
|
+}
|
|
|
+
|
|
|
+#ifdef LINUXKM_FPU_STATES_FOLLOW_THREADS
|
|
|
+/* legacy thread-local storage facility for tracking recursive fpu
|
|
|
+ * pushing/popping
|
|
|
+ */
|
|
|
+static struct wc_thread_fpu_count_ent *wc_linuxkm_fpu_state_assoc(int create_p) {
|
|
|
+ struct wc_thread_fpu_count_ent *i, *i_endptr, *i_empty;
|
|
|
+ pid_t my_pid = task_pid_nr(current), i_pid;
|
|
|
+
|
|
|
+ {
|
|
|
+ static int _warned_on_null = 0;
|
|
|
+ if (wc_linuxkm_fpu_states == NULL)
|
|
|
+ {
|
|
|
+#ifdef HAVE_FIPS
|
|
|
+ /* FIPS needs to use SHA256 for the core verify HMAC, before
|
|
|
+ * reaching the regular wolfCrypt_Init() logic. to break the
|
|
|
+ * dependency loop on intelasm builds, we allocate here.
|
|
|
+ * this is not thread-safe and doesn't need to be.
|
|
|
+ */
|
|
|
+ int ret = allocate_wolfcrypt_linuxkm_fpu_states();
|
|
|
+ if (ret != 0)
|
|
|
+#endif
|
|
|
+ {
|
|
|
+ if (_warned_on_null == 0) {
|
|
|
+ pr_err("wc_linuxkm_fpu_state_assoc called by pid %d"
|
|
|
+ " before allocate_wolfcrypt_linuxkm_fpu_states.\n", my_pid);
|
|
|
+ _warned_on_null = 1;
|
|
|
+ }
|
|
|
+ return NULL;
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ i_endptr = &wc_linuxkm_fpu_states[wc_linuxkm_fpu_states_n_tracked];
|
|
|
+
|
|
|
+ for (;;) {
|
|
|
+ for (i = wc_linuxkm_fpu_states,
|
|
|
+ i_empty = NULL;
|
|
|
+ i < i_endptr;
|
|
|
+ ++i)
|
|
|
+ {
|
|
|
+ i_pid = __atomic_load_n(&i->pid, __ATOMIC_CONSUME);
|
|
|
+ if (i_pid == my_pid)
|
|
|
+ return i;
|
|
|
+ if ((i_empty == NULL) && (i_pid == 0))
|
|
|
+ i_empty = i;
|
|
|
+ }
|
|
|
+ if ((i_empty == NULL) || (! create_p))
|
|
|
+ return NULL;
|
|
|
+
|
|
|
+ i_pid = 0;
|
|
|
+ if (__atomic_compare_exchange_n(
|
|
|
+ &(i_empty->pid),
|
|
|
+ &i_pid,
|
|
|
+ my_pid,
|
|
|
+ 0 /* weak */,
|
|
|
+ __ATOMIC_SEQ_CST /* success_memmodel */,
|
|
|
+ __ATOMIC_SEQ_CST /* failure_memmodel */))
|
|
|
+ {
|
|
|
+ return i_empty;
|
|
|
+ }
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+#else /* !LINUXKM_FPU_STATES_FOLLOW_THREADS */
|
|
|
+
|
|
|
+/* lock-free O(1)-lookup CPU-local storage facility for tracking recursive fpu
|
|
|
+ * pushing/popping.
|
|
|
+ *
|
|
|
+ * caller must have already called kernel_fpu_begin() or preempt_disable()
|
|
|
+ * before entering this or the streamlined inline version of it below.
|
|
|
+ */
|
|
|
+static struct wc_thread_fpu_count_ent *wc_linuxkm_fpu_state_assoc_unlikely(int create_p) {
|
|
|
+ int my_cpu = raw_smp_processor_id();
|
|
|
+ pid_t my_pid = task_pid_nr(current), slot_pid;
|
|
|
+ struct wc_thread_fpu_count_ent *slot;
|
|
|
+
|
|
|
+ {
|
|
|
+ static int _warned_on_null = 0;
|
|
|
+ if (wc_linuxkm_fpu_states == NULL)
|
|
|
+ {
|
|
|
+#ifdef HAVE_FIPS
|
|
|
+ /* FIPS needs to use SHA256 for the core verify HMAC, before
|
|
|
+ * reaching the regular wolfCrypt_Init() logic. to break the
|
|
|
+ * dependency loop on intelasm builds, we allocate here.
|
|
|
+ * this is not thread-safe and doesn't need to be.
|
|
|
+ */
|
|
|
+ int ret = allocate_wolfcrypt_linuxkm_fpu_states();
|
|
|
+ if (ret != 0)
|
|
|
+#endif
|
|
|
+ {
|
|
|
+ if (_warned_on_null == 0) {
|
|
|
+ pr_err("wc_linuxkm_fpu_state_assoc called by pid %d"
|
|
|
+ " before allocate_wolfcrypt_linuxkm_fpu_states.\n", my_pid);
|
|
|
+ _warned_on_null = 1;
|
|
|
+ }
|
|
|
+ return NULL;
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ slot = &wc_linuxkm_fpu_states[my_cpu];
|
|
|
+ slot_pid = __atomic_load_n(&slot->pid, __ATOMIC_CONSUME);
|
|
|
+ if (slot_pid == my_pid) {
|
|
|
+ if (create_p) {
|
|
|
+ static int _warned_on_redundant_create_p = 0;
|
|
|
+ if (_warned_on_redundant_create_p < 10) {
|
|
|
+ pr_err("wc_linuxkm_fpu_state_assoc called with create_p=1 by"
|
|
|
+ " pid %d on cpu %d with cpu slot already reserved by"
|
|
|
+ " said pid.\n", my_pid, my_cpu);
|
|
|
+ ++_warned_on_redundant_create_p;
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return slot;
|
|
|
+ }
|
|
|
+ if (create_p) {
|
|
|
+ if (slot_pid == 0) {
|
|
|
+ __atomic_store_n(&slot->pid, my_pid, __ATOMIC_RELEASE);
|
|
|
+ return slot;
|
|
|
+ } else {
|
|
|
+ /* if the slot is already occupied, that can be benign due to a
|
|
|
+ * migration, but it will require fixup by the thread that owns the
|
|
|
+ * slot, which will happen when it releases its lock, or sooner (see
|
|
|
+ * below).
|
|
|
+ */
|
|
|
+ static int _warned_on_mismatched_pid = 0;
|
|
|
+ if (_warned_on_mismatched_pid < 10) {
|
|
|
+ pr_warn("wc_linuxkm_fpu_state_assoc called by pid %d on cpu %d"
|
|
|
+ " but cpu slot already reserved by pid %d.\n",
|
|
|
+ my_pid, my_cpu, slot_pid);
|
|
|
+ ++_warned_on_mismatched_pid;
|
|
|
+ }
|
|
|
+ return NULL;
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+ /* check for migration. this can happen despite our best efforts if any
|
|
|
+ * I/O occured while locked, e.g. kernel messages like "uninitialized
|
|
|
+ * urandom read". since we're locked now, we can safely migrate the
|
|
|
+ * entry in wc_linuxkm_fpu_states[], freeing up the slot on the previous
|
|
|
+ * cpu.
|
|
|
+ */
|
|
|
+ unsigned int cpu_i;
|
|
|
+ for (cpu_i = 0; cpu_i < wc_linuxkm_fpu_states_n_tracked; ++cpu_i) {
|
|
|
+ if (__atomic_load_n(
|
|
|
+ &wc_linuxkm_fpu_states[cpu_i].pid,
|
|
|
+ __ATOMIC_CONSUME)
|
|
|
+ == my_pid)
|
|
|
+ {
|
|
|
+ wc_linuxkm_fpu_states[my_cpu] = wc_linuxkm_fpu_states[cpu_i];
|
|
|
+ __atomic_store_n(&wc_linuxkm_fpu_states[cpu_i].fpu_state, 0,
|
|
|
+ __ATOMIC_RELEASE);
|
|
|
+ __atomic_store_n(&wc_linuxkm_fpu_states[cpu_i].pid, 0,
|
|
|
+ __ATOMIC_RELEASE);
|
|
|
+ return &wc_linuxkm_fpu_states[my_cpu];
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return NULL;
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+static inline struct wc_thread_fpu_count_ent *wc_linuxkm_fpu_state_assoc(
|
|
|
+ int create_p)
|
|
|
+{
|
|
|
+ int my_cpu = raw_smp_processor_id(); /* my_cpu is only trustworthy if we're
|
|
|
+ * already nonpreemptible -- we'll
|
|
|
+ * determine that soon enough by
|
|
|
+ * checking if the pid matches or,
|
|
|
+ * failing that, if create_p.
|
|
|
+ */
|
|
|
+ pid_t my_pid = task_pid_nr(current), slot_pid;
|
|
|
+ struct wc_thread_fpu_count_ent *slot;
|
|
|
+
|
|
|
+ if (unlikely(wc_linuxkm_fpu_states == NULL))
|
|
|
+ return wc_linuxkm_fpu_state_assoc_unlikely(create_p);
|
|
|
+
|
|
|
+ slot = &wc_linuxkm_fpu_states[my_cpu];
|
|
|
+ slot_pid = __atomic_load_n(&slot->pid, __ATOMIC_CONSUME);
|
|
|
+ if (slot_pid == my_pid) {
|
|
|
+ if (unlikely(create_p))
|
|
|
+ return wc_linuxkm_fpu_state_assoc_unlikely(create_p);
|
|
|
+ else
|
|
|
+ return slot;
|
|
|
+ }
|
|
|
+ if (likely(create_p)) {
|
|
|
+ if (likely(slot_pid == 0)) {
|
|
|
+ __atomic_store_n(&slot->pid, my_pid, __ATOMIC_RELEASE);
|
|
|
+ return slot;
|
|
|
+ } else {
|
|
|
+ return wc_linuxkm_fpu_state_assoc_unlikely(create_p);
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+ return wc_linuxkm_fpu_state_assoc_unlikely(create_p);
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+#endif /* !LINUXKM_FPU_STATES_FOLLOW_THREADS */
|
|
|
+
|
|
|
+#ifdef WOLFSSL_COMMERCIAL_LICENSE
|
|
|
+static struct fpstate *wc_linuxkm_fpstate_buf_from_fpu_state(
|
|
|
+ struct wc_thread_fpu_count_ent *state)
|
|
|
+{
|
|
|
+ size_t i = (size_t)(state - wc_linuxkm_fpu_states) / sizeof(*state);
|
|
|
+ return &wc_linuxkm_fpu_savebufs[i].fpstate;
|
|
|
+}
|
|
|
+#endif
|
|
|
+
|
|
|
+static void wc_linuxkm_fpu_state_release_unlikely(
|
|
|
+ struct wc_thread_fpu_count_ent *ent)
|
|
|
+{
|
|
|
+ if (ent->fpu_state != 0) {
|
|
|
+ static int warned_nonzero_fpu_state = 0;
|
|
|
+ if (! warned_nonzero_fpu_state) {
|
|
|
+ pr_err("wc_linuxkm_fpu_state_free for pid %d"
|
|
|
+ " with nonzero fpu_state 0x%x.\n", ent->pid, ent->fpu_state);
|
|
|
+ warned_nonzero_fpu_state = 1;
|
|
|
+ }
|
|
|
+ ent->fpu_state = 0;
|
|
|
+ }
|
|
|
+ __atomic_store_n(&ent->pid, 0, __ATOMIC_RELEASE);
|
|
|
+}
|
|
|
+
|
|
|
+static inline void wc_linuxkm_fpu_state_release(
|
|
|
+ struct wc_thread_fpu_count_ent *ent)
|
|
|
+{
|
|
|
+ if (unlikely(ent->fpu_state != 0))
|
|
|
+ return wc_linuxkm_fpu_state_release_unlikely(ent);
|
|
|
+ __atomic_store_n(&ent->pid, 0, __ATOMIC_RELEASE);
|
|
|
+}
|
|
|
+
|
|
|
+WARN_UNUSED_RESULT int can_save_vector_registers_x86(void)
|
|
|
+{
|
|
|
+ if (irq_fpu_usable())
|
|
|
+ return 1;
|
|
|
+ else if (in_nmi() || (hardirq_count() > 0) || (softirq_count() > 0))
|
|
|
+ return 0;
|
|
|
+ else if (test_thread_flag(TIF_NEED_FPU_LOAD))
|
|
|
+ return 1;
|
|
|
+ return 0;
|
|
|
+}
|
|
|
+
|
|
|
+WARN_UNUSED_RESULT int save_vector_registers_x86(void)
|
|
|
+{
|
|
|
+#ifdef LINUXKM_FPU_STATES_FOLLOW_THREADS
|
|
|
+ struct wc_thread_fpu_count_ent *pstate = wc_linuxkm_fpu_state_assoc(1);
|
|
|
+#else
|
|
|
+ struct wc_thread_fpu_count_ent *pstate = wc_linuxkm_fpu_state_assoc(0);
|
|
|
+#endif
|
|
|
+
|
|
|
+ /* allow for nested calls */
|
|
|
+#ifdef LINUXKM_FPU_STATES_FOLLOW_THREADS
|
|
|
+ if (pstate == NULL)
|
|
|
+ return MEMORY_E;
|
|
|
+#endif
|
|
|
+ if (
|
|
|
+#ifndef LINUXKM_FPU_STATES_FOLLOW_THREADS
|
|
|
+ (pstate != NULL) &&
|
|
|
+#endif
|
|
|
+ (pstate->fpu_state != 0U))
|
|
|
+ {
|
|
|
+ if (unlikely((pstate->fpu_state & WC_FPU_COUNT_MASK)
|
|
|
+ == WC_FPU_COUNT_MASK))
|
|
|
+ {
|
|
|
+ pr_err("save_vector_registers_x86 recursion register overflow for "
|
|
|
+ "pid %d.\n", pstate->pid);
|
|
|
+ return BAD_STATE_E;
|
|
|
+ } else {
|
|
|
+ ++pstate->fpu_state;
|
|
|
+ return 0;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ if (irq_fpu_usable()
|
|
|
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 17, 0))
|
|
|
+ /* work around a kernel bug -- see linux commit 59f5ede3bc0f0.
|
|
|
+ * what we really want here is this_cpu_read(in_kernel_fpu), but
|
|
|
+ * in_kernel_fpu is an unexported static array.
|
|
|
+ */
|
|
|
+ && !test_thread_flag(TIF_NEED_FPU_LOAD)
|
|
|
+#endif
|
|
|
+ )
|
|
|
+ {
|
|
|
+#ifdef WOLFSSL_COMMERCIAL_LICENSE
|
|
|
+ struct fpstate *fpstate = wc_linuxkm_fpstate_buf_from_fpu_state(pstate);
|
|
|
+ fpregs_lock();
|
|
|
+ fpstate->xfeatures = ~0UL;
|
|
|
+ os_xsave(fpstate);
|
|
|
+#else /* !WOLFSSL_COMMERCIAL_LICENSE */
|
|
|
+ #if defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_COUNT) && \
|
|
|
+ (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0))
|
|
|
+ /* inhibit migration, which gums up the algorithm in
|
|
|
+ * kernel_fpu_{begin,end}().
|
|
|
+ */
|
|
|
+ migrate_disable();
|
|
|
+ #endif
|
|
|
+ kernel_fpu_begin();
|
|
|
+
|
|
|
+#ifndef LINUXKM_FPU_STATES_FOLLOW_THREADS
|
|
|
+ pstate = wc_linuxkm_fpu_state_assoc(1);
|
|
|
+ if (pstate == NULL) {
|
|
|
+ kernel_fpu_end();
|
|
|
+ #if defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_COUNT) && \
|
|
|
+ (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)) && \
|
|
|
+ !defined(WOLFSSL_COMMERCIAL_LICENSE)
|
|
|
+ migrate_enable();
|
|
|
+ #endif
|
|
|
+ return BAD_STATE_E;
|
|
|
+ }
|
|
|
+#endif
|
|
|
+
|
|
|
+#endif /* !WOLFSSL_COMMERCIAL_LICENSE */
|
|
|
+ /* set msb to 0 to trigger kernel_fpu_end() at cleanup. */
|
|
|
+ pstate->fpu_state = 1U;
|
|
|
+ } else if (in_nmi() || (hardirq_count() > 0) || (softirq_count() > 0)) {
|
|
|
+ static int warned_fpu_forbidden = 0;
|
|
|
+ if (! warned_fpu_forbidden)
|
|
|
+ pr_err("save_vector_registers_x86 called from IRQ handler.\n");
|
|
|
+#ifdef LINUXKM_FPU_STATES_FOLLOW_THREADS
|
|
|
+ wc_linuxkm_fpu_state_release(pstate);
|
|
|
+#endif
|
|
|
+ return BAD_STATE_E;
|
|
|
+ } else if (!test_thread_flag(TIF_NEED_FPU_LOAD)) {
|
|
|
+ static int warned_fpu_forbidden = 0;
|
|
|
+ if (! warned_fpu_forbidden)
|
|
|
+ pr_err("save_vector_registers_x86 called with !irq_fpu_usable from"
|
|
|
+ " thread without previous FPU save.\n");
|
|
|
+#ifdef LINUXKM_FPU_STATES_FOLLOW_THREADS
|
|
|
+ wc_linuxkm_fpu_state_release(pstate);
|
|
|
+#endif
|
|
|
+ return BAD_STATE_E;
|
|
|
+ } else {
|
|
|
+ /* assume already safely in_kernel_fpu from caller, but recursively
|
|
|
+ * preempt_disable() to be extra-safe.
|
|
|
+ */
|
|
|
+ preempt_disable();
|
|
|
+#if defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_COUNT) && \
|
|
|
+ (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)) && \
|
|
|
+ !defined(WOLFSSL_COMMERCIAL_LICENSE)
|
|
|
+ migrate_disable();
|
|
|
+#endif
|
|
|
+#ifndef LINUXKM_FPU_STATES_FOLLOW_THREADS
|
|
|
+ pstate = wc_linuxkm_fpu_state_assoc(1);
|
|
|
+ if (pstate == NULL) {
|
|
|
+ #if defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_COUNT) && \
|
|
|
+ (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)) && \
|
|
|
+ !defined(WOLFSSL_COMMERCIAL_LICENSE)
|
|
|
+ migrate_enable();
|
|
|
+ #endif
|
|
|
+ preempt_enable();
|
|
|
+ return BAD_STATE_E;
|
|
|
+ }
|
|
|
+#endif
|
|
|
+ /* set msb to 1 to inhibit kernel_fpu_end() at cleanup. */
|
|
|
+ pstate->fpu_state =
|
|
|
+ WC_FPU_SAVED_MASK + 1U;
|
|
|
+ }
|
|
|
+
|
|
|
+ return 0;
|
|
|
+}
|
|
|
+
|
|
|
+void restore_vector_registers_x86(void)
|
|
|
+{
|
|
|
+ struct wc_thread_fpu_count_ent *pstate = wc_linuxkm_fpu_state_assoc(0);
|
|
|
+ if (unlikely(pstate == NULL)) {
|
|
|
+ pr_err("restore_vector_registers_x86 called by pid %d on CPU %d "
|
|
|
+ "with no saved state.\n", task_pid_nr(current),
|
|
|
+ raw_smp_processor_id());
|
|
|
+ return;
|
|
|
+ }
|
|
|
+
|
|
|
+ if ((--pstate->fpu_state & WC_FPU_COUNT_MASK) > 0U) {
|
|
|
+ return;
|
|
|
+ }
|
|
|
+
|
|
|
+ if (pstate->fpu_state == 0U) {
|
|
|
+#ifdef WOLFSSL_COMMERCIAL_LICENSE
|
|
|
+ struct fpstate *fpstate = wc_linuxkm_fpstate_buf_from_fpu_state(pstate);
|
|
|
+ os_xrstor(fpstate, fpstate->xfeatures);
|
|
|
+ fpregs_unlock();
|
|
|
+#else
|
|
|
+ #ifndef LINUXKM_FPU_STATES_FOLLOW_THREADS
|
|
|
+ wc_linuxkm_fpu_state_release(pstate);
|
|
|
+ #endif
|
|
|
+ kernel_fpu_end();
|
|
|
+#endif
|
|
|
+ } else {
|
|
|
+ pstate->fpu_state = 0U;
|
|
|
+ #ifndef LINUXKM_FPU_STATES_FOLLOW_THREADS
|
|
|
+ wc_linuxkm_fpu_state_release(pstate);
|
|
|
+ #endif
|
|
|
+ preempt_enable();
|
|
|
+ }
|
|
|
+#if defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_COUNT) && \
|
|
|
+ (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)) && \
|
|
|
+ !defined(WOLFSSL_COMMERCIAL_LICENSE)
|
|
|
+ migrate_enable();
|
|
|
+#endif
|
|
|
+
|
|
|
+#ifdef LINUXKM_FPU_STATES_FOLLOW_THREADS
|
|
|
+ wc_linuxkm_fpu_state_release(pstate);
|
|
|
+#endif
|
|
|
+
|
|
|
+ return;
|
|
|
+}
|
Daniel Pouzzner