linuxkm_memory.c 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531
  1. /* linuxkm_memory.c
  2. *
  3. * Copyright (C) 2006-2023 wolfSSL Inc.
  4. *
  5. * This file is part of wolfSSL.
  6. *
  7. * wolfSSL is free software; you can redistribute it and/or modify
  8. * it under the terms of the GNU General Public License as published by
  9. * the Free Software Foundation; either version 2 of the License, or
  10. * (at your option) any later version.
  11. *
  12. * wolfSSL is distributed in the hope that it will be useful,
  13. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  15. * GNU General Public License for more details.
  16. *
  17. * You should have received a copy of the GNU General Public License
  18. * along with this program; if not, write to the Free Software
  19. * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA
  20. */
  21. /* included by wolfcrypt/src/memory.c */
  22. #ifdef HAVE_KVMALLOC
  23. /* adapted from kvrealloc() draft by Changli Gao, 2010-05-13 */
  24. void *lkm_realloc(void *ptr, size_t newsize) {
  25. void *nptr;
  26. size_t oldsize;
  27. if (unlikely(newsize == 0)) {
  28. kvfree(ptr);
  29. return ZERO_SIZE_PTR;
  30. }
  31. if (unlikely(ptr == NULL))
  32. return kvmalloc_node(newsize, GFP_KERNEL, NUMA_NO_NODE);
  33. if (is_vmalloc_addr(ptr)) {
  34. /* no way to discern the size of the old allocation,
  35. * because the kernel doesn't export find_vm_area(). if
  36. * it did, we could then call get_vm_area_size() on the
  37. * returned struct vm_struct.
  38. */
  39. return NULL;
  40. } else {
  41. #ifndef __PIE__
  42. struct page *page;
  43. page = virt_to_head_page(ptr);
  44. if (PageSlab(page) || PageCompound(page)) {
  45. if (newsize < PAGE_SIZE)
  46. #endif /* ! __PIE__ */
  47. return krealloc(ptr, newsize, GFP_KERNEL);
  48. #ifndef __PIE__
  49. oldsize = ksize(ptr);
  50. } else {
  51. oldsize = page->private;
  52. if (newsize <= oldsize)
  53. return ptr;
  54. }
  55. #endif /* ! __PIE__ */
  56. }
  57. nptr = kvmalloc_node(newsize, GFP_KERNEL, NUMA_NO_NODE);
  58. if (nptr != NULL) {
  59. memcpy(nptr, ptr, oldsize);
  60. kvfree(ptr);
  61. }
  62. return nptr;
  63. }
  64. #endif /* HAVE_KVMALLOC */
  65. #if defined(WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS) && defined(CONFIG_X86)
  66. static unsigned int wc_linuxkm_fpu_states_n_tracked = 0;
  67. struct wc_thread_fpu_count_ent {
  68. volatile pid_t pid;
  69. unsigned int fpu_state;
  70. };
  71. struct wc_thread_fpu_count_ent *wc_linuxkm_fpu_states = NULL;
  72. #ifdef WOLFSSL_COMMERCIAL_LICENSE
  73. #ifndef LINUXKM_FPU_STATES_FOLLOW_THREADS
  74. #error WOLFSSL_COMMERCIAL_LICENSE requires LINUXKM_FPU_STATES_FOLLOW_THREADS
  75. #endif
  76. #pragma GCC diagnostic push
  77. #pragma GCC diagnostic ignored "-Wunused-parameter"
  78. #pragma GCC diagnostic ignored "-Wnested-externs"
  79. /* avoid dependence on "alternatives_patched" and "xfd_validate_state()". */
  80. #undef CONFIG_X86_DEBUG_FPU
  81. #include "../kernel/fpu/internal.h"
  82. #include "../kernel/fpu/xstate.h"
  83. #pragma GCC diagnostic pop
  84. static union wc_linuxkm_fpu_savebuf {
  85. byte buf[1024]; /* must be 64-byte-aligned */
  86. struct fpstate fpstate;
  87. } *wc_linuxkm_fpu_savebufs = NULL;
  88. #endif /* WOLFSSL_COMMERCIAL_LICENSE */
  89. #define WC_FPU_COUNT_MASK 0x7fffffffU
  90. #define WC_FPU_SAVED_MASK 0x80000000U
  91. WARN_UNUSED_RESULT int allocate_wolfcrypt_linuxkm_fpu_states(void)
  92. {
  93. if (wc_linuxkm_fpu_states != NULL) {
  94. static int warned_for_repeat_alloc = 0;
  95. if (! warned_for_repeat_alloc) {
  96. pr_err("attempt at repeat allocation"
  97. " in allocate_wolfcrypt_linuxkm_fpu_states\n");
  98. warned_for_repeat_alloc = 1;
  99. }
  100. return BAD_STATE_E;
  101. }
  102. #ifdef LINUXKM_FPU_STATES_FOLLOW_THREADS
  103. if (nr_cpu_ids >= 16)
  104. wc_linuxkm_fpu_states_n_tracked = nr_cpu_ids * 2;
  105. else
  106. wc_linuxkm_fpu_states_n_tracked = 32;
  107. #else
  108. wc_linuxkm_fpu_states_n_tracked = nr_cpu_ids;
  109. #endif
  110. wc_linuxkm_fpu_states =
  111. (struct wc_thread_fpu_count_ent *)malloc(
  112. wc_linuxkm_fpu_states_n_tracked * sizeof(wc_linuxkm_fpu_states[0]));
  113. if (! wc_linuxkm_fpu_states) {
  114. pr_err("allocation of %lu bytes for "
  115. "wc_linuxkm_fpu_states failed.\n",
  116. nr_cpu_ids * sizeof(struct fpu_state *));
  117. return MEMORY_E;
  118. }
  119. memset(wc_linuxkm_fpu_states, 0, wc_linuxkm_fpu_states_n_tracked
  120. * sizeof(wc_linuxkm_fpu_states[0]));
  121. #ifdef WOLFSSL_COMMERCIAL_LICENSE
  122. wc_linuxkm_fpu_savebufs = (union wc_linuxkm_fpu_savebuf *)malloc(
  123. wc_linuxkm_fpu_states_n_tracked * sizeof(*wc_linuxkm_fpu_savebufs));
  124. if (! wc_linuxkm_fpu_savebufs) {
  125. pr_err("allocation of %lu bytes for "
  126. "wc_linuxkm_fpu_savebufs failed.\n",
  127. WC_LINUXKM_ROUND_UP_P_OF_2(wc_linuxkm_fpu_states_n_tracked)
  128. * sizeof(*wc_linuxkm_fpu_savebufs));
  129. free(wc_linuxkm_fpu_states);
  130. wc_linuxkm_fpu_states = NULL;
  131. return MEMORY_E;
  132. }
  133. if ((uintptr_t)wc_linuxkm_fpu_savebufs
  134. & (WC_LINUXKM_ROUND_UP_P_OF_2(sizeof(*wc_linuxkm_fpu_savebufs)) - 1))
  135. {
  136. pr_err("allocation of %lu bytes for "
  137. "wc_linuxkm_fpu_savebufs allocated with wrong alignment 0x%lx.\n",
  138. WC_LINUXKM_ROUND_UP_P_OF_2(wc_linuxkm_fpu_states_n_tracked)
  139. * sizeof(*wc_linuxkm_fpu_savebufs),
  140. (uintptr_t)wc_linuxkm_fpu_savebufs);
  141. free(wc_linuxkm_fpu_savebufs);
  142. wc_linuxkm_fpu_savebufs = NULL;
  143. free(wc_linuxkm_fpu_states);
  144. wc_linuxkm_fpu_states = NULL;
  145. return MEMORY_E;
  146. }
  147. #endif
  148. return 0;
  149. }
  150. void free_wolfcrypt_linuxkm_fpu_states(void) {
  151. struct wc_thread_fpu_count_ent *i, *i_endptr;
  152. pid_t i_pid;
  153. if (wc_linuxkm_fpu_states == NULL) {
  154. pr_err("free_wolfcrypt_linuxkm_fpu_states called"
  155. " before allocate_wolfcrypt_linuxkm_fpu_states.\n");
  156. return;
  157. }
  158. for (i = wc_linuxkm_fpu_states,
  159. i_endptr = &wc_linuxkm_fpu_states[wc_linuxkm_fpu_states_n_tracked];
  160. i < i_endptr;
  161. ++i)
  162. {
  163. i_pid = __atomic_load_n(&i->pid, __ATOMIC_CONSUME);
  164. if (i_pid == 0)
  165. continue;
  166. if (i->fpu_state != 0) {
  167. pr_err("free_wolfcrypt_linuxkm_fpu_states called"
  168. " with nonzero state 0x%x for pid %d.\n", i->fpu_state, i_pid);
  169. i->fpu_state = 0;
  170. }
  171. }
  172. #ifdef WOLFSSL_COMMERCIAL_LICENSE
  173. free(wc_linuxkm_fpu_savebufs);
  174. wc_linuxkm_fpu_savebufs = NULL;
  175. #endif
  176. free(wc_linuxkm_fpu_states);
  177. wc_linuxkm_fpu_states = NULL;
  178. }
  179. #ifdef LINUXKM_FPU_STATES_FOLLOW_THREADS
  180. /* legacy thread-local storage facility for tracking recursive fpu
  181. * pushing/popping
  182. */
  183. static struct wc_thread_fpu_count_ent *wc_linuxkm_fpu_state_assoc(int create_p) {
  184. struct wc_thread_fpu_count_ent *i, *i_endptr, *i_empty;
  185. pid_t my_pid = task_pid_nr(current), i_pid;
  186. {
  187. static int _warned_on_null = 0;
  188. if (wc_linuxkm_fpu_states == NULL)
  189. {
  190. if (_warned_on_null == 0) {
  191. pr_err("wc_linuxkm_fpu_state_assoc called by pid %d"
  192. " before allocate_wolfcrypt_linuxkm_fpu_states.\n", my_pid);
  193. _warned_on_null = 1;
  194. }
  195. return NULL;
  196. }
  197. }
  198. i_endptr = &wc_linuxkm_fpu_states[wc_linuxkm_fpu_states_n_tracked];
  199. for (;;) {
  200. for (i = wc_linuxkm_fpu_states,
  201. i_empty = NULL;
  202. i < i_endptr;
  203. ++i)
  204. {
  205. i_pid = __atomic_load_n(&i->pid, __ATOMIC_CONSUME);
  206. if (i_pid == my_pid)
  207. return i;
  208. if ((i_empty == NULL) && (i_pid == 0))
  209. i_empty = i;
  210. }
  211. if ((i_empty == NULL) || (! create_p))
  212. return NULL;
  213. i_pid = 0;
  214. if (__atomic_compare_exchange_n(
  215. &(i_empty->pid),
  216. &i_pid,
  217. my_pid,
  218. 0 /* weak */,
  219. __ATOMIC_SEQ_CST /* success_memmodel */,
  220. __ATOMIC_SEQ_CST /* failure_memmodel */))
  221. {
  222. return i_empty;
  223. }
  224. }
  225. }
  226. #else /* !LINUXKM_FPU_STATES_FOLLOW_THREADS */
  227. /* lock-free O(1)-lookup CPU-local storage facility for tracking recursive fpu
  228. * pushing/popping
  229. */
  230. static struct wc_thread_fpu_count_ent *wc_linuxkm_fpu_state_assoc_unlikely(int create_p) {
  231. int my_cpu = raw_smp_processor_id();
  232. pid_t my_pid = task_pid_nr(current), slot_pid;
  233. struct wc_thread_fpu_count_ent *slot;
  234. {
  235. static int _warned_on_null = 0;
  236. if (wc_linuxkm_fpu_states == NULL)
  237. {
  238. if (_warned_on_null == 0) {
  239. pr_err("wc_linuxkm_fpu_state_assoc called by pid %d"
  240. " before allocate_wolfcrypt_linuxkm_fpu_states.\n", my_pid);
  241. _warned_on_null = 1;
  242. }
  243. return NULL;
  244. }
  245. }
  246. slot = &wc_linuxkm_fpu_states[my_cpu];
  247. slot_pid = __atomic_load_n(&slot->pid, __ATOMIC_CONSUME);
  248. if (slot_pid == my_pid)
  249. return slot;
  250. if (create_p) {
  251. /* caller must have already called kernel_fpu_begin() if create_p. */
  252. if (slot_pid == 0) {
  253. __atomic_store_n(&slot->pid, my_pid, __ATOMIC_RELEASE);
  254. return slot;
  255. } else {
  256. static int _warned_on_mismatched_pid = 0;
  257. if (_warned_on_mismatched_pid < 10) {
  258. pr_err("wc_linuxkm_fpu_state_assoc called by pid %d on cpu %d"
  259. " but cpu slot already reserved by pid %d.\n", my_pid, my_cpu, slot_pid);
  260. ++_warned_on_mismatched_pid;
  261. }
  262. return NULL;
  263. }
  264. } else {
  265. return NULL;
  266. }
  267. }
  268. static inline struct wc_thread_fpu_count_ent *wc_linuxkm_fpu_state_assoc(int create_p) {
  269. int my_cpu = raw_smp_processor_id(); /* my_cpu is only trustworthy if we're
  270. * already nonpreemptible -- we'll
  271. * determine that soon enough by
  272. * checking if the pid matches or,
  273. * failing that, if create_p.
  274. */
  275. pid_t my_pid = task_pid_nr(current), slot_pid;
  276. struct wc_thread_fpu_count_ent *slot;
  277. if (wc_linuxkm_fpu_states == NULL)
  278. return wc_linuxkm_fpu_state_assoc_unlikely(create_p);
  279. slot = &wc_linuxkm_fpu_states[my_cpu];
  280. slot_pid = __atomic_load_n(&slot->pid, __ATOMIC_CONSUME);
  281. if (slot_pid == my_pid)
  282. return slot;
  283. if (create_p) {
  284. /* caller must have already called kernel_fpu_begin() if create_p. */
  285. if (slot_pid == 0) {
  286. __atomic_store_n(&slot->pid, my_pid, __ATOMIC_RELEASE);
  287. return slot;
  288. } else {
  289. return wc_linuxkm_fpu_state_assoc_unlikely(create_p);
  290. }
  291. } else {
  292. return NULL;
  293. }
  294. }
  295. #endif /* !LINUXKM_FPU_STATES_FOLLOW_THREADS */
  296. #ifdef WOLFSSL_COMMERCIAL_LICENSE
  297. static struct fpstate *wc_linuxkm_fpstate_buf_from_fpu_state(
  298. struct wc_thread_fpu_count_ent *state)
  299. {
  300. size_t i = (size_t)(state - wc_linuxkm_fpu_states) / sizeof(*state);
  301. return &wc_linuxkm_fpu_savebufs[i].fpstate;
  302. }
  303. #endif
  304. static void wc_linuxkm_fpu_state_release_unlikely(struct wc_thread_fpu_count_ent *ent) {
  305. if (ent->fpu_state != 0) {
  306. static int warned_nonzero_fpu_state = 0;
  307. if (! warned_nonzero_fpu_state) {
  308. pr_err("wc_linuxkm_fpu_state_free for pid %d"
  309. " with nonzero fpu_state 0x%x.\n", ent->pid, ent->fpu_state);
  310. warned_nonzero_fpu_state = 1;
  311. }
  312. ent->fpu_state = 0;
  313. }
  314. __atomic_store_n(&ent->pid, 0, __ATOMIC_RELEASE);
  315. }
  316. static inline void wc_linuxkm_fpu_state_release(struct wc_thread_fpu_count_ent *ent) {
  317. if (unlikely(ent->fpu_state != 0))
  318. return wc_linuxkm_fpu_state_release_unlikely(ent);
  319. __atomic_store_n(&ent->pid, 0, __ATOMIC_RELEASE);
  320. }
  321. WARN_UNUSED_RESULT int save_vector_registers_x86(void)
  322. {
  323. #ifdef LINUXKM_FPU_STATES_FOLLOW_THREADS
  324. struct wc_thread_fpu_count_ent *pstate = wc_linuxkm_fpu_state_assoc(1);
  325. #else
  326. struct wc_thread_fpu_count_ent *pstate = wc_linuxkm_fpu_state_assoc(0);
  327. #endif
  328. /* allow for nested calls */
  329. #ifdef LINUXKM_FPU_STATES_FOLLOW_THREADS
  330. if (pstate == NULL)
  331. return MEMORY_E;
  332. #endif
  333. if (
  334. #ifndef LINUXKM_FPU_STATES_FOLLOW_THREADS
  335. (pstate != NULL) &&
  336. #endif
  337. (pstate->fpu_state != 0U))
  338. {
  339. if (unlikely((pstate->fpu_state & WC_FPU_COUNT_MASK)
  340. == WC_FPU_COUNT_MASK))
  341. {
  342. pr_err("save_vector_registers_x86 recursion register overflow for "
  343. "pid %d.\n", pstate->pid);
  344. return BAD_STATE_E;
  345. } else {
  346. ++pstate->fpu_state;
  347. return 0;
  348. }
  349. }
  350. if (irq_fpu_usable()) {
  351. #ifdef WOLFSSL_COMMERCIAL_LICENSE
  352. struct fpstate *fpstate = wc_linuxkm_fpstate_buf_from_fpu_state(pstate);
  353. fpregs_lock();
  354. fpstate->xfeatures = ~0UL;
  355. os_xsave(fpstate);
  356. #else /* !WOLFSSL_COMMERCIAL_LICENSE */
  357. #if defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_COUNT) && \
  358. (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0))
  359. /* inhibit migration, which gums up the algorithm in
  360. * kernel_fpu_{begin,end}().
  361. */
  362. migrate_disable();
  363. #endif
  364. kernel_fpu_begin();
  365. #ifndef LINUXKM_FPU_STATES_FOLLOW_THREADS
  366. pstate = wc_linuxkm_fpu_state_assoc(1);
  367. if (pstate == NULL) {
  368. kernel_fpu_end();
  369. #if defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_COUNT) && \
  370. (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)) && \
  371. !defined(WOLFSSL_COMMERCIAL_LICENSE)
  372. migrate_enable();
  373. #endif
  374. return BAD_STATE_E;
  375. }
  376. #endif
  377. #endif /* !WOLFSSL_COMMERCIAL_LICENSE */
  378. /* set msb to 0 to trigger kernel_fpu_end() at cleanup. */
  379. pstate->fpu_state = 1U;
  380. } else if (in_nmi() || (hardirq_count() > 0) || (softirq_count() > 0)) {
  381. static int warned_fpu_forbidden = 0;
  382. if (! warned_fpu_forbidden)
  383. pr_err("save_vector_registers_x86 called from IRQ handler.\n");
  384. #ifdef LINUXKM_FPU_STATES_FOLLOW_THREADS
  385. wc_linuxkm_fpu_state_release(pstate);
  386. #endif
  387. return BAD_STATE_E;
  388. } else {
  389. /* assume already safely in_kernel_fpu. */
  390. #if defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_COUNT) && \
  391. (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)) && \
  392. !defined(WOLFSSL_COMMERCIAL_LICENSE)
  393. migrate_disable();
  394. #endif
  395. #ifndef LINUXKM_FPU_STATES_FOLLOW_THREADS
  396. pstate = wc_linuxkm_fpu_state_assoc(1);
  397. if (pstate == NULL) {
  398. #if defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_COUNT) && \
  399. (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)) && \
  400. !defined(WOLFSSL_COMMERCIAL_LICENSE)
  401. migrate_enable();
  402. #endif
  403. return BAD_STATE_E;
  404. }
  405. #endif
  406. /* set msb to 1 to inhibit kernel_fpu_end() at cleanup. */
  407. pstate->fpu_state =
  408. WC_FPU_SAVED_MASK + 1U;
  409. }
  410. return 0;
  411. }
  412. void restore_vector_registers_x86(void)
  413. {
  414. struct wc_thread_fpu_count_ent *pstate = wc_linuxkm_fpu_state_assoc(0);
  415. if (unlikely(pstate == NULL)) {
  416. pr_err("restore_vector_registers_x86 called by pid %d on CPU %d "
  417. "with no saved state.\n", task_pid_nr(current),
  418. raw_smp_processor_id());
  419. return;
  420. }
  421. if ((--pstate->fpu_state & WC_FPU_COUNT_MASK) > 0U) {
  422. return;
  423. }
  424. if (pstate->fpu_state == 0U) {
  425. #ifdef WOLFSSL_COMMERCIAL_LICENSE
  426. struct fpstate *fpstate = wc_linuxkm_fpstate_buf_from_fpu_state(pstate);
  427. os_xrstor(fpstate, fpstate->xfeatures);
  428. fpregs_unlock();
  429. #else
  430. #ifndef LINUXKM_FPU_STATES_FOLLOW_THREADS
  431. wc_linuxkm_fpu_state_release(pstate);
  432. #endif
  433. kernel_fpu_end();
  434. #endif
  435. } else {
  436. pstate->fpu_state = 0U;
  437. #ifndef LINUXKM_FPU_STATES_FOLLOW_THREADS
  438. wc_linuxkm_fpu_state_release(pstate);
  439. #endif
  440. }
  441. #if defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_COUNT) && \
  442. (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)) && \
  443. !defined(WOLFSSL_COMMERCIAL_LICENSE)
  444. migrate_enable();
  445. #endif
  446. #ifdef LINUXKM_FPU_STATES_FOLLOW_THREADS
  447. wc_linuxkm_fpu_state_release(pstate);
  448. #endif
  449. return;
  450. }
  451. #endif /* WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS && CONFIG_X86 */
  452. #if defined(__PIE__) && (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 1, 0))
  453. /* needed in 6.1+ because show_free_areas() static definition in mm.h calls
  454. * __show_free_areas(), which isn't exported (neither was show_free_areas()).
  455. */
  456. void my__show_free_areas(
  457. unsigned int flags,
  458. nodemask_t *nodemask,
  459. int max_zone_idx)
  460. {
  461. (void)flags;
  462. (void)nodemask;
  463. (void)max_zone_idx;
  464. return;
  465. }
  466. #endif
  467. #if defined(__PIE__) && defined(CONFIG_FORTIFY_SOURCE)
  468. /* needed because FORTIFY_SOURCE inline implementations call fortify_panic(). */
  469. void __my_fortify_panic(const char *name) {
  470. pr_emerg("__my_fortify_panic in %s\n", name);
  471. BUG();
  472. }
  473. #endif