1
0

linuxkm_memory.c 9.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296
  1. /* linuxkm_memory.c
  2. *
  3. * Copyright (C) 2006-2023 wolfSSL Inc.
  4. *
  5. * This file is part of wolfSSL.
  6. *
  7. * wolfSSL is free software; you can redistribute it and/or modify
  8. * it under the terms of the GNU General Public License as published by
  9. * the Free Software Foundation; either version 2 of the License, or
  10. * (at your option) any later version.
  11. *
  12. * wolfSSL is distributed in the hope that it will be useful,
  13. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  15. * GNU General Public License for more details.
  16. *
  17. * You should have received a copy of the GNU General Public License
  18. * along with this program; if not, write to the Free Software
  19. * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA
  20. */
  21. /* included by wolfcrypt/src/memory.c */
  22. #ifdef HAVE_KVMALLOC
  23. /* adapted from kvrealloc() draft by Changli Gao, 2010-05-13 */
  24. void *lkm_realloc(void *ptr, size_t newsize) {
  25. void *nptr;
  26. size_t oldsize;
  27. if (unlikely(newsize == 0)) {
  28. kvfree(ptr);
  29. return ZERO_SIZE_PTR;
  30. }
  31. if (unlikely(ptr == NULL))
  32. return kvmalloc_node(newsize, GFP_KERNEL, NUMA_NO_NODE);
  33. if (is_vmalloc_addr(ptr)) {
  34. /* no way to discern the size of the old allocation,
  35. * because the kernel doesn't export find_vm_area(). if
  36. * it did, we could then call get_vm_area_size() on the
  37. * returned struct vm_struct.
  38. */
  39. return NULL;
  40. } else {
  41. #ifndef __PIE__
  42. struct page *page;
  43. page = virt_to_head_page(ptr);
  44. if (PageSlab(page) || PageCompound(page)) {
  45. if (newsize < PAGE_SIZE)
  46. #endif /* ! __PIE__ */
  47. return krealloc(ptr, newsize, GFP_KERNEL);
  48. #ifndef __PIE__
  49. oldsize = ksize(ptr);
  50. } else {
  51. oldsize = page->private;
  52. if (newsize <= oldsize)
  53. return ptr;
  54. }
  55. #endif /* ! __PIE__ */
  56. }
  57. nptr = kvmalloc_node(newsize, GFP_KERNEL, NUMA_NO_NODE);
  58. if (nptr != NULL) {
  59. memcpy(nptr, ptr, oldsize);
  60. kvfree(ptr);
  61. }
  62. return nptr;
  63. }
  64. #endif /* HAVE_KVMALLOC */
  65. #if defined(WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS) && defined(CONFIG_X86)
  66. static unsigned int wc_linuxkm_fpu_states_n_tracked = 0;
  67. struct wc_thread_fpu_count_ent {
  68. volatile pid_t pid;
  69. unsigned int fpu_state;
  70. };
  71. struct wc_thread_fpu_count_ent *wc_linuxkm_fpu_states = NULL;
  72. #define WC_FPU_COUNT_MASK 0x7fffffffU
  73. #define WC_FPU_SAVED_MASK 0x80000000U
  74. WARN_UNUSED_RESULT int allocate_wolfcrypt_linuxkm_fpu_states(void)
  75. {
  76. if (wc_linuxkm_fpu_states != NULL) {
  77. static int warned_for_repeat_alloc = 0;
  78. if (! warned_for_repeat_alloc) {
  79. pr_err("attempt at repeat allocation"
  80. " in allocate_wolfcrypt_linuxkm_fpu_states\n");
  81. warned_for_repeat_alloc = 1;
  82. }
  83. return BAD_STATE_E;
  84. }
  85. if (nr_cpu_ids >= 16)
  86. wc_linuxkm_fpu_states_n_tracked = nr_cpu_ids * 2;
  87. else
  88. wc_linuxkm_fpu_states_n_tracked = 32;
  89. wc_linuxkm_fpu_states =
  90. (struct wc_thread_fpu_count_ent *)malloc(
  91. wc_linuxkm_fpu_states_n_tracked * sizeof(wc_linuxkm_fpu_states[0]));
  92. if (! wc_linuxkm_fpu_states) {
  93. pr_err("allocation of %lu bytes for "
  94. "wc_linuxkm_fpu_states failed.\n",
  95. nr_cpu_ids * sizeof(struct fpu_state *));
  96. return MEMORY_E;
  97. }
  98. memset(wc_linuxkm_fpu_states, 0, wc_linuxkm_fpu_states_n_tracked * sizeof(wc_linuxkm_fpu_states[0]));
  99. return 0;
  100. }
  101. void free_wolfcrypt_linuxkm_fpu_states(void) {
  102. struct wc_thread_fpu_count_ent *i, *i_endptr;
  103. pid_t i_pid;
  104. if (wc_linuxkm_fpu_states == NULL) {
  105. pr_err("free_wolfcrypt_linuxkm_fpu_states called"
  106. " before allocate_wolfcrypt_linuxkm_fpu_states.\n");
  107. return;
  108. }
  109. for (i = wc_linuxkm_fpu_states,
  110. i_endptr = &wc_linuxkm_fpu_states[wc_linuxkm_fpu_states_n_tracked];
  111. i < i_endptr;
  112. ++i)
  113. {
  114. i_pid = __atomic_load_n(&i->pid, __ATOMIC_CONSUME);
  115. if (i_pid == 0)
  116. continue;
  117. if (i->fpu_state != 0) {
  118. pr_err("free_wolfcrypt_linuxkm_fpu_states called"
  119. " with nonzero state 0x%x for pid %d.\n", i->fpu_state, i_pid);
  120. i->fpu_state = 0;
  121. }
  122. }
  123. free(wc_linuxkm_fpu_states);
  124. wc_linuxkm_fpu_states = NULL;
  125. }
  126. /* lock-(mostly)-free thread-local storage facility for tracking recursive fpu pushing/popping */
  127. static struct wc_thread_fpu_count_ent *wc_linuxkm_fpu_state_assoc(int create_p) {
  128. struct wc_thread_fpu_count_ent *i, *i_endptr, *i_empty;
  129. pid_t my_pid = task_pid_nr(current), i_pid;
  130. {
  131. static int _warned_on_null = 0;
  132. if (wc_linuxkm_fpu_states == NULL)
  133. {
  134. if (_warned_on_null == 0) {
  135. pr_err("wc_linuxkm_fpu_state_assoc called by pid %d"
  136. " before allocate_wolfcrypt_linuxkm_fpu_states.\n", my_pid);
  137. _warned_on_null = 1;
  138. }
  139. return NULL;
  140. }
  141. }
  142. i_endptr = &wc_linuxkm_fpu_states[wc_linuxkm_fpu_states_n_tracked];
  143. for (;;) {
  144. for (i = wc_linuxkm_fpu_states,
  145. i_empty = NULL;
  146. i < i_endptr;
  147. ++i)
  148. {
  149. i_pid = __atomic_load_n(&i->pid, __ATOMIC_CONSUME);
  150. if (i_pid == my_pid)
  151. return i;
  152. if ((i_empty == NULL) && (i_pid == 0))
  153. i_empty = i;
  154. }
  155. if ((i_empty == NULL) || (! create_p))
  156. return NULL;
  157. i_pid = 0;
  158. if (__atomic_compare_exchange_n(
  159. &(i_empty->pid),
  160. &i_pid,
  161. my_pid,
  162. 0 /* weak */,
  163. __ATOMIC_SEQ_CST /* success_memmodel */,
  164. __ATOMIC_SEQ_CST /* failure_memmodel */))
  165. {
  166. return i_empty;
  167. }
  168. }
  169. }
  170. static void wc_linuxkm_fpu_state_free(struct wc_thread_fpu_count_ent *ent) {
  171. if (ent->fpu_state != 0) {
  172. static int warned_nonzero_fpu_state = 0;
  173. if (! warned_nonzero_fpu_state) {
  174. pr_err("wc_linuxkm_fpu_state_free for pid %d"
  175. " with nonzero fpu_state 0x%x.\n", ent->pid, ent->fpu_state);
  176. warned_nonzero_fpu_state = 1;
  177. }
  178. ent->fpu_state = 0;
  179. }
  180. __atomic_store_n(&ent->pid, 0, __ATOMIC_RELEASE);
  181. }
  182. WARN_UNUSED_RESULT int save_vector_registers_x86(void)
  183. {
  184. struct wc_thread_fpu_count_ent *pstate = wc_linuxkm_fpu_state_assoc(1);
  185. if (pstate == NULL)
  186. return ENOMEM;
  187. /* allow for nested calls */
  188. if (pstate->fpu_state != 0U) {
  189. if ((pstate->fpu_state & WC_FPU_COUNT_MASK)
  190. == WC_FPU_COUNT_MASK)
  191. {
  192. pr_err("save_vector_registers_x86 recursion register overflow for "
  193. "pid %d.\n", pstate->pid);
  194. return BAD_STATE_E;
  195. } else {
  196. ++pstate->fpu_state;
  197. return 0;
  198. }
  199. }
  200. if (irq_fpu_usable()) {
  201. #if defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_COUNT) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0))
  202. /* inhibit migration, which gums up the algorithm in kernel_fpu_{begin,end}(). */
  203. migrate_disable();
  204. #endif
  205. kernel_fpu_begin();
  206. pstate->fpu_state = 1U; /* set msb 0 to trigger kernel_fpu_end() at cleanup. */
  207. } else if (in_nmi() || (hardirq_count() > 0) || (softirq_count() > 0)) {
  208. static int warned_fpu_forbidden = 0;
  209. if (! warned_fpu_forbidden)
  210. pr_err("save_vector_registers_x86 called from IRQ handler.\n");
  211. wc_linuxkm_fpu_state_free(pstate);
  212. return EPERM;
  213. } else {
  214. #if defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_COUNT) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0))
  215. migrate_disable();
  216. #endif
  217. /* assume already safely in_kernel_fpu. */
  218. pstate->fpu_state =
  219. WC_FPU_SAVED_MASK + 1U; /* set msb 1 to inhibit kernel_fpu_end() at cleanup. */
  220. }
  221. return 0;
  222. }
  223. void restore_vector_registers_x86(void)
  224. {
  225. struct wc_thread_fpu_count_ent *pstate = wc_linuxkm_fpu_state_assoc(0);
  226. if (pstate == NULL) {
  227. pr_err("restore_vector_registers_x86 called by pid %d "
  228. "with no saved state.\n", task_pid_nr(current));
  229. return;
  230. }
  231. if ((--pstate->fpu_state & WC_FPU_COUNT_MASK) > 0U) {
  232. return;
  233. }
  234. if (pstate->fpu_state == 0U)
  235. kernel_fpu_end();
  236. else
  237. pstate->fpu_state = 0U;
  238. #if defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_COUNT) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0))
  239. migrate_enable();
  240. #endif
  241. wc_linuxkm_fpu_state_free(pstate);
  242. return;
  243. }
  244. #endif /* WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS && CONFIG_X86 */
  245. #if defined(__PIE__) && (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 1, 0))
  246. /* needed in 6.1+ because show_free_areas() static definition in mm.h calls
  247. * __show_free_areas(), which isn't exported (neither was show_free_areas()).
  248. */
  249. void my__show_free_areas(
  250. unsigned int flags,
  251. nodemask_t *nodemask,
  252. int max_zone_idx)
  253. {
  254. (void)flags;
  255. (void)nodemask;
  256. (void)max_zone_idx;
  257. return;
  258. }
  259. #endif