/* linuxkm_wc_port.h * * Copyright (C) 2006-2023 wolfSSL Inc. * * This file is part of wolfSSL. * * wolfSSL is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * wolfSSL is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA */ /* included by wolfssl/wolfcrypt/wc_port.h */ #ifndef LINUXKM_WC_PORT_H #define LINUXKM_WC_PORT_H #include #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 16, 0) #error Unsupported kernel. #endif #ifdef HAVE_CONFIG_H #ifndef PACKAGE_NAME #error wc_port.h included before config.h #endif /* config.h is autogenerated without gating, and is subject to repeat * inclusions, so gate it out here to keep autodetection masking * intact: */ #undef HAVE_CONFIG_H #endif /* suppress inclusion of stdint-gcc.h to avoid conflicts with Linux native * include/linux/types.h: */ #define _GCC_STDINT_H #define WC_PTR_TYPE uintptr_t /* needed to suppress inclusion of stdio.h in wolfssl/wolfcrypt/types.h */ #define XSNPRINTF snprintf /* the rigmarole around kstrtoll() here is to accommodate its * warn-unused-result attribute. * * also needed to suppress inclusion of stdlib.h in * wolfssl/wolfcrypt/types.h. */ #define XATOI(s) ({ \ long long _xatoi_res = 0; \ int _xatoi_ret = kstrtoll(s, 10, &_xatoi_res); \ if (_xatoi_ret != 0) { \ _xatoi_res = 0; \ } \ (int)_xatoi_res; \ }) /* Kbuild+gcc on x86 doesn't consistently honor the default ALIGN16 on stack * objects, but gives adequate alignment with "32". */ #if defined(CONFIG_X86) && !defined(ALIGN16) #define ALIGN16 __attribute__ ( (aligned (32))) #endif /* kvmalloc()/kvfree() and friends added in linux commit a7c3e901 */ #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 12, 0) #define HAVE_KVMALLOC #endif /* kernel printf doesn't implement fp. */ #ifndef WOLFSSL_NO_FLOAT_FMT #define WOLFSSL_NO_FLOAT_FMT #endif #ifdef BUILDING_WOLFSSL #if defined(CONFIG_MIPS) && defined(HAVE_LINUXKM_PIE_SUPPORT) /* __ZBOOT__ disables some unhelpful macros around the mem*() funcs in * legacy arch/mips/include/asm/string.h */ #define __ZBOOT__ #define memcmp __builtin_memcmp #define __ARCH_MEMCMP_NO_REDIRECT #define __ARCH_MEMCPY_NO_REDIRECT #define __builtin_memcpy memcpy extern void *memcpy(void *dest, const void *src, unsigned int n); #define __ARCH_MEMCPY_NO_REDIRECT #define __builtin_memset memset extern void *memset(void *dest, int c, unsigned int n); #endif _Pragma("GCC diagnostic push"); /* we include all the needed kernel headers with these masked out. else * there are profuse warnings. */ _Pragma("GCC diagnostic ignored \"-Wunused-parameter\""); _Pragma("GCC diagnostic ignored \"-Wpointer-arith\""); _Pragma("GCC diagnostic ignored \"-Wshadow\""); _Pragma("GCC diagnostic ignored \"-Wnested-externs\""); _Pragma("GCC diagnostic ignored \"-Wredundant-decls\""); _Pragma("GCC diagnostic ignored \"-Wsign-compare\""); _Pragma("GCC diagnostic ignored \"-Wpointer-sign\""); _Pragma("GCC diagnostic ignored \"-Wbad-function-cast\""); _Pragma("GCC diagnostic ignored \"-Wdiscarded-qualifiers\""); _Pragma("GCC diagnostic ignored \"-Wtype-limits\""); _Pragma("GCC diagnostic ignored \"-Wswitch-enum\""); #include #include #include #if defined(CONFIG_FORTIFY_SOURCE) || defined(DEBUG_LINUXKM_FORTIFY_OVERLAY) #ifdef __PIE__ /* the inline definitions in fortify-string.h use non-inline * fortify_panic(). */ extern void __my_fortify_panic(const char *name) __noreturn __cold; #define fortify_panic __my_fortify_panic #endif /* the _FORTIFY_SOURCE macros and implementations for several string * functions are incompatible with libwolfssl, so just reimplement with * inlines and remap with macros. */ #define __ARCH_STRLEN_NO_REDIRECT #define __ARCH_MEMCPY_NO_REDIRECT #define __ARCH_MEMSET_NO_REDIRECT #define __ARCH_MEMMOVE_NO_REDIRECT /* the inline definitions in fortify-string.h use non-inline * strlen(). */ static inline size_t strlen(const char *s) { const char *s_start = s; while (*s) ++s; return (size_t)((uintptr_t)s - (uintptr_t)s_start); } #include #undef strlen #define strlen(s) \ ((__builtin_constant_p(s) && __builtin_constant_p(*(s))) ? \ (sizeof(s) - 1) : strlen(s)) static inline void *my_memcpy(void *dest, const void *src, size_t n) { if (! (((uintptr_t)dest | (uintptr_t)src | (uintptr_t)n) & (uintptr_t)(sizeof(uintptr_t) - 1))) { uintptr_t *src_longs = (uintptr_t *)src, *dest_longs = (uintptr_t *)dest, *endp = (uintptr_t *)((u8 *)src + n); while (src_longs < endp) *dest_longs++ = *src_longs++; } else { u8 *src_bytes = (u8 *)src, *dest_bytes = (u8 *)dest, *endp = src_bytes + n; while (src_bytes < endp) *dest_bytes++ = *src_bytes++; } return dest; } #undef memcpy #define memcpy my_memcpy static inline void *my_memset(void *dest, int c, size_t n) { if (! (((uintptr_t)dest | (uintptr_t)n) & (uintptr_t)(sizeof(uintptr_t) - 1))) { uintptr_t c_long = __builtin_choose_expr( sizeof(uintptr_t) == 8, (uintptr_t)(u8)c * 0x0101010101010101UL, (uintptr_t)(u8)c * 0x01010101U ); uintptr_t *dest_longs = (uintptr_t *)dest, *endp = (uintptr_t *)((u8 *)dest_longs + n); while (dest_longs < endp) *dest_longs++ = c_long; } else { u8 *dest_bytes = (u8 *)dest, *endp = dest_bytes + n; while (dest_bytes < endp) *dest_bytes++ = (u8)c; } return dest; } #undef memset #define memset my_memset static inline void *my_memmove(void *dest, const void *src, size_t n) { if (! (((uintptr_t)dest | (uintptr_t)src | (uintptr_t)n) & (uintptr_t)(sizeof(uintptr_t) - 1))) { uintptr_t *src_longs = (uintptr_t *)src, *dest_longs = (uintptr_t *)dest; n >>= __builtin_choose_expr( sizeof(uintptr_t) == 8, 3U, 2U); if (src_longs < dest_longs) { uintptr_t *startp = src_longs; src_longs += n - 1; dest_longs += n - 1; while (src_longs >= startp) *dest_longs-- = *src_longs--; } else if (src_longs > dest_longs) { uintptr_t *endp = src_longs + n; while (src_longs < endp) *dest_longs++ = *src_longs++; } } else { u8 *src_bytes = (u8 *)src, *dest_bytes = (u8 *)dest; if (src_bytes < dest_bytes) { u8 *startp = src_bytes; src_bytes += n - 1; dest_bytes += n - 1; while (src_bytes >= startp) *dest_bytes-- = *src_bytes--; } else if (src_bytes > dest_bytes) { u8 *endp = src_bytes + n; while (src_bytes < endp) *dest_bytes++ = *src_bytes++; } } return dest; } #undef memmove #define memmove my_memmove #endif /* CONFIG_FORTIFY_SOURCE */ #include #include #include #ifdef __PIE__ /* without this, mm.h brings in static, but not inline, pmd_to_page(), * with direct references to global vmem variables. */ #undef USE_SPLIT_PMD_PTLOCKS #define USE_SPLIT_PMD_PTLOCKS 0 #if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 1, 0) /* without this, static show_free_areas() mm.h brings in direct * reference to unexported __show_free_areas(). */ #define __show_free_areas my__show_free_areas void my__show_free_areas( unsigned int flags, nodemask_t *nodemask, int max_zone_idx); #endif #endif #include #ifndef SINGLE_THREADED #include #endif #include #include #ifdef LINUXKM_LKCAPI_REGISTER #include #include #include #include #include /* the LKCAPI assumes that expanded encrypt and decrypt keys will stay * loaded simultaneously, and the Linux in-tree implementations have two * AES key structs in each context, one for each direction. in * linuxkm/lkcapi_glue.c (used for CBC, CFB, and GCM), we do the same * thing with "struct km_AesCtx". however, wolfCrypt struct AesXts * already has two AES expanded keys, the main and tweak, and the tweak * is always used in the encrypt direction regardless of the main * direction. to avoid allocating and computing a duplicate second * tweak encrypt key, we set * WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS, which adds a second * Aes slot to wolfCrypt's struct AesXts, and activates support for * AES_ENCRYPTION_AND_DECRYPTION on AES-XTS. */ #ifndef WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS #define WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS #endif #endif #if defined(WOLFSSL_AESNI) || defined(USE_INTEL_SPEEDUP) || \ defined(WOLFSSL_SP_X86_64_ASM) #ifndef CONFIG_X86 #error X86 SIMD extensions requested, but CONFIG_X86 is not set. #endif #define WOLFSSL_LINUXKM_SIMD #define WOLFSSL_LINUXKM_SIMD_X86 #ifndef WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS #define WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS #endif #elif defined(WOLFSSL_ARMASM) || defined(WOLFSSL_SP_ARM32_ASM) || \ defined(WOLFSSL_SP_ARM64_ASM) || defined(WOLFSSL_SP_ARM_THUMB_ASM) ||\ defined(WOLFSSL_SP_ARM_CORTEX_M_ASM) #if !defined(CONFIG_ARM) && !defined(CONFIG_ARM64) #error ARM SIMD extensions requested, but CONFIG_ARM* is not set. #endif #define WOLFSSL_LINUXKM_SIMD #define WOLFSSL_LINUXKM_SIMD_ARM #ifndef WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS #define WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS #endif #else #ifndef WOLFSSL_NO_ASM #define WOLFSSL_NO_ASM #endif #endif /* benchmarks.c uses floating point math, so needs a working * SAVE_VECTOR_REGISTERS(). */ #if defined(WOLFSSL_LINUXKM_BENCHMARKS) && \ !defined(WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS) #define WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS #endif #if defined(WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS) && \ defined(CONFIG_X86) #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0) #include #else #include #endif #ifndef SAVE_VECTOR_REGISTERS #define SAVE_VECTOR_REGISTERS(fail_clause) { \ int _svr_ret = save_vector_registers_x86(); \ if (_svr_ret != 0) { \ fail_clause \ } \ } #endif #ifndef SAVE_VECTOR_REGISTERS2 #ifdef DEBUG_VECTOR_REGISTER_ACCESS_FUZZING #define SAVE_VECTOR_REGISTERS2() ({ \ int _fuzzer_ret = SAVE_VECTOR_REGISTERS2_fuzzer(); \ (_fuzzer_ret == 0) ? \ save_vector_registers_x86() : \ _fuzzer_ret; \ }) #else #define SAVE_VECTOR_REGISTERS2() save_vector_registers_x86() #endif #endif #ifndef RESTORE_VECTOR_REGISTERS #define RESTORE_VECTOR_REGISTERS() restore_vector_registers_x86() #endif #elif defined(WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS) && (defined(CONFIG_ARM) || defined(CONFIG_ARM64)) #include #ifndef SAVE_VECTOR_REGISTERS #define SAVE_VECTOR_REGISTERS(fail_clause) { int _svr_ret = save_vector_registers_arm(); if (_svr_ret != 0) { fail_clause } } #endif #ifndef SAVE_VECTOR_REGISTERS2 #define SAVE_VECTOR_REGISTERS2() save_vector_registers_arm() #endif #ifndef RESTORE_VECTOR_REGISTERS #define RESTORE_VECTOR_REGISTERS() restore_vector_registers_arm() #endif #elif defined(WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS) #error WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS is set for an unsupported architecture. #endif _Pragma("GCC diagnostic pop"); /* avoid -Wpointer-arith, encountered when -DCONFIG_FORTIFY_SOURCE */ #undef __is_constexpr #define __is_constexpr(x) __builtin_constant_p(x) /* the kernel uses -std=c89, but not -pedantic, and makes full use of anon * structs/unions, so we should too. */ #define HAVE_ANONYMOUS_INLINE_AGGREGATES 1 #define NO_THREAD_LS #define NO_ATTRIBUTE_CONSTRUCTOR #ifdef HAVE_FIPS extern int wolfCrypt_FIPS_first(void); extern int wolfCrypt_FIPS_last(void); #endif #if !defined(WOLFCRYPT_ONLY) && !defined(NO_CERTS) /* work around backward dependency of asn.c on ssl.c. */ struct Signer; struct Signer *GetCA(void *signers, unsigned char *hash); #ifndef NO_SKID struct Signer *GetCAByName(void* signers, unsigned char *hash); #endif #endif #if defined(__PIE__) && !defined(USE_WOLFSSL_LINUXKM_PIE_REDIRECT_TABLE) #error "compiling -fPIE requires PIE redirect table." #endif #if defined(HAVE_FIPS) && !defined(HAVE_LINUXKM_PIE_SUPPORT) #error "FIPS build requires PIE support." #endif #ifdef USE_WOLFSSL_LINUXKM_PIE_REDIRECT_TABLE #ifdef CONFIG_MIPS #undef __ARCH_MEMCMP_NO_REDIRECT #undef memcmp extern int memcmp(const void *s1, const void *s2, size_t n); #endif struct wolfssl_linuxkm_pie_redirect_table { #ifndef __ARCH_MEMCMP_NO_REDIRECT typeof(memcmp) *memcmp; #endif #ifndef __ARCH_MEMCPY_NO_REDIRECT typeof(memcpy) *memcpy; #endif #ifndef __ARCH_MEMSET_NO_REDIRECT typeof(memset) *memset; #endif #ifndef __ARCH_MEMMOVE_NO_REDIRECT typeof(memmove) *memmove; #endif #ifndef __ARCH_STRCMP_NO_REDIRECT typeof(strcmp) *strcmp; #endif #ifndef __ARCH_STRNCMP_NO_REDIRECT typeof(strncmp) *strncmp; #endif #ifndef __ARCH_STRCASECMP_NO_REDIRECT typeof(strcasecmp) *strcasecmp; #endif #ifndef __ARCH_STRNCASECMP_NO_REDIRECT typeof(strncasecmp) *strncasecmp; #endif #ifndef __ARCH_STRLEN_NO_REDIRECT typeof(strlen) *strlen; #endif #ifndef __ARCH_STRSTR_NO_REDIRECT typeof(strstr) *strstr; #endif #ifndef __ARCH_STRNCPY_NO_REDIRECT typeof(strncpy) *strncpy; #endif #ifndef __ARCH_STRNCAT_NO_REDIRECT typeof(strncat) *strncat; #endif typeof(kstrtoll) *kstrtoll; #if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0) typeof(_printk) *_printk; #else typeof(printk) *printk; #endif #ifdef CONFIG_FORTIFY_SOURCE typeof(__warn_printk) *__warn_printk; #endif typeof(snprintf) *snprintf; const unsigned char *_ctype; typeof(kmalloc) *kmalloc; typeof(kfree) *kfree; typeof(ksize) *ksize; typeof(krealloc) *krealloc; #ifdef HAVE_KVMALLOC typeof(kvmalloc_node) *kvmalloc_node; typeof(kvfree) *kvfree; #endif typeof(is_vmalloc_addr) *is_vmalloc_addr; #if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 1, 0) typeof(kmalloc_trace) *kmalloc_trace; #else typeof(kmem_cache_alloc_trace) *kmem_cache_alloc_trace; typeof(kmalloc_order_trace) *kmalloc_order_trace; #endif typeof(get_random_bytes) *get_random_bytes; #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0) typeof(getnstimeofday) *getnstimeofday; #elif LINUX_VERSION_CODE < KERNEL_VERSION(5, 0, 0) typeof(current_kernel_time64) *current_kernel_time64; #else typeof(ktime_get_coarse_real_ts64) *ktime_get_coarse_real_ts64; #endif struct task_struct *(*get_current)(void); int (*preempt_count)(void); #ifdef WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS #if LINUX_VERSION_CODE < KERNEL_VERSION(6, 2, 0) typeof(cpu_number) *cpu_number; #else typeof(pcpu_hot) *pcpu_hot; #endif typeof(nr_cpu_ids) *nr_cpu_ids; #if defined(CONFIG_SMP) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)) && !defined(WOLFSSL_COMMERCIAL_LICENSE) /* note the current and needed version of these were added in af449901b8 (2020-Sep-17) */ typeof(migrate_disable) *migrate_disable; typeof(migrate_enable) *migrate_enable; #endif #ifdef CONFIG_X86 typeof(irq_fpu_usable) *irq_fpu_usable; #ifdef WOLFSSL_COMMERCIAL_LICENSE typeof(fpregs_lock) *fpregs_lock; typeof(fpregs_lock) *fpregs_unlock; #else /* !WOLFSSL_COMMERCIAL_LICENSE */ /* kernel_fpu_begin() replaced by kernel_fpu_begin_mask() in commit e4512289, * released in kernel 5.11, backported to 5.4.93 */ #ifdef kernel_fpu_begin typeof(kernel_fpu_begin_mask) *kernel_fpu_begin_mask; #else typeof(kernel_fpu_begin) *kernel_fpu_begin; #endif typeof(kernel_fpu_end) *kernel_fpu_end; #endif /* !defined(WOLFSSL_COMMERCIAL_LICENSE) */ #else /* !CONFIG_X86 */ #error WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS is set for an unsupported architecture. #endif /* arch */ #endif /* WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS */ typeof(__mutex_init) *__mutex_init; #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0) typeof(mutex_lock_nested) *mutex_lock_nested; #else typeof(mutex_lock) *mutex_lock; #endif typeof(mutex_unlock) *mutex_unlock; #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0) typeof(mutex_destroy) *mutex_destroy; #endif #ifdef HAVE_FIPS typeof(wolfCrypt_FIPS_first) *wolfCrypt_FIPS_first; typeof(wolfCrypt_FIPS_last) *wolfCrypt_FIPS_last; #endif #if !defined(WOLFCRYPT_ONLY) && !defined(NO_CERTS) typeof(GetCA) *GetCA; #ifndef NO_SKID typeof(GetCAByName) *GetCAByName; #endif #endif const void *_last_slot; }; extern const struct wolfssl_linuxkm_pie_redirect_table *wolfssl_linuxkm_get_pie_redirect_table(void); #ifdef __PIE__ #ifndef __ARCH_MEMCMP_NO_REDIRECT #define memcmp (wolfssl_linuxkm_get_pie_redirect_table()->memcmp) #endif #ifndef __ARCH_MEMCPY_NO_REDIRECT #define memcpy (wolfssl_linuxkm_get_pie_redirect_table()->memcpy) #endif #ifndef __ARCH_MEMSET_NO_REDIRECT #define memset (wolfssl_linuxkm_get_pie_redirect_table()->memset) #endif #ifndef __ARCH_MEMMOVE_NO_REDIRECT #define memmove (wolfssl_linuxkm_get_pie_redirect_table()->memmove) #endif #ifndef __ARCH_STRCMP_NO_REDIRECT #define strcmp (wolfssl_linuxkm_get_pie_redirect_table()->strcmp) #endif #ifndef __ARCH_STRNCMP_NO_REDIRECT #define strncmp (wolfssl_linuxkm_get_pie_redirect_table()->strncmp) #endif #ifndef __ARCH_STRCASECMP_NO_REDIRECT #define strcasecmp (wolfssl_linuxkm_get_pie_redirect_table()->strcasecmp) #endif #ifndef __ARCH_STRNCASECMP_NO_REDIRECT #define strncasecmp (wolfssl_linuxkm_get_pie_redirect_table()->strncasecmp) #endif #ifndef __ARCH_STRLEN_NO_REDIRECT #define strlen (wolfssl_linuxkm_get_pie_redirect_table()->strlen) #endif #ifndef __ARCH_STRSTR_NO_REDIRECT #define strstr (wolfssl_linuxkm_get_pie_redirect_table()->strstr) #endif #ifndef __ARCH_STRNCPY_NO_REDIRECT #define strncpy (wolfssl_linuxkm_get_pie_redirect_table()->strncpy) #endif #ifndef __ARCH_STRNCAT_NO_REDIRECT #define strncat (wolfssl_linuxkm_get_pie_redirect_table()->strncat) #endif #define kstrtoll (wolfssl_linuxkm_get_pie_redirect_table()->kstrtoll) #if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0) #define _printk (wolfssl_linuxkm_get_pie_redirect_table()->_printk) #else #define printk (wolfssl_linuxkm_get_pie_redirect_table()->printk) #endif #ifdef CONFIG_FORTIFY_SOURCE #define __warn_printk (wolfssl_linuxkm_get_pie_redirect_table()->__warn_printk) #endif #define snprintf (wolfssl_linuxkm_get_pie_redirect_table()->snprintf) #define _ctype (wolfssl_linuxkm_get_pie_redirect_table()->_ctype) #define kmalloc (wolfssl_linuxkm_get_pie_redirect_table()->kmalloc) #define kfree (wolfssl_linuxkm_get_pie_redirect_table()->kfree) #define ksize (wolfssl_linuxkm_get_pie_redirect_table()->ksize) #define krealloc (wolfssl_linuxkm_get_pie_redirect_table()->krealloc) #define kzalloc(size, flags) kmalloc(size, (flags) | __GFP_ZERO) #ifdef HAVE_KVMALLOC #define kvmalloc_node (wolfssl_linuxkm_get_pie_redirect_table()->kvmalloc_node) #define kvfree (wolfssl_linuxkm_get_pie_redirect_table()->kvfree) #endif #define is_vmalloc_addr (wolfssl_linuxkm_get_pie_redirect_table()->is_vmalloc_addr) #if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 1, 0) #define kmalloc_trace (wolfssl_linuxkm_get_pie_redirect_table()->kmalloc_trace) #else #define kmem_cache_alloc_trace (wolfssl_linuxkm_get_pie_redirect_table()->kmem_cache_alloc_trace) #define kmalloc_order_trace (wolfssl_linuxkm_get_pie_redirect_table()->kmalloc_order_trace) #endif #define get_random_bytes (wolfssl_linuxkm_get_pie_redirect_table()->get_random_bytes) #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0) #define getnstimeofday (wolfssl_linuxkm_get_pie_redirect_table()->getnstimeofday) #elif LINUX_VERSION_CODE < KERNEL_VERSION(5, 0, 0) #define current_kernel_time64 (wolfssl_linuxkm_get_pie_redirect_table()->current_kernel_time64) #else #define ktime_get_coarse_real_ts64 (wolfssl_linuxkm_get_pie_redirect_table()->ktime_get_coarse_real_ts64) #endif #undef get_current #define get_current (wolfssl_linuxkm_get_pie_redirect_table()->get_current) #undef preempt_count #define preempt_count (wolfssl_linuxkm_get_pie_redirect_table()->preempt_count) #ifdef WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS #if LINUX_VERSION_CODE < KERNEL_VERSION(6, 2, 0) #define cpu_number (*(wolfssl_linuxkm_get_pie_redirect_table()->cpu_number)) #else #define pcpu_hot (*(wolfssl_linuxkm_get_pie_redirect_table()->pcpu_hot)) #endif #define nr_cpu_ids (*(wolfssl_linuxkm_get_pie_redirect_table()->nr_cpu_ids)) #if defined(CONFIG_SMP) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)) && !defined(WOLFSSL_COMMERCIAL_LICENSE) #define migrate_disable (*(wolfssl_linuxkm_get_pie_redirect_table()->migrate_disable)) #define migrate_enable (*(wolfssl_linuxkm_get_pie_redirect_table()->migrate_enable)) #endif #ifdef CONFIG_X86 #define irq_fpu_usable (wolfssl_linuxkm_get_pie_redirect_table()->irq_fpu_usable) #ifdef WOLFSSL_COMMERCIAL_LICENSE #define fpregs_lock() (wolfssl_linuxkm_get_pie_redirect_table()->fpregs_lock()) #define fpregs_unlock() (wolfssl_linuxkm_get_pie_redirect_table()->fpregs_unlock()) #else /* !defined(WOLFSSL_COMMERCIAL_LICENSE) */ #ifdef kernel_fpu_begin #define kernel_fpu_begin_mask (wolfssl_linuxkm_get_pie_redirect_table()->kernel_fpu_begin_mask) #else #define kernel_fpu_begin (wolfssl_linuxkm_get_pie_redirect_table()->kernel_fpu_begin) #endif #define kernel_fpu_end (wolfssl_linuxkm_get_pie_redirect_table()->kernel_fpu_end) #endif /* !defined(WOLFSSL_COMMERCIAL_LICENSE) */ #else /* !CONFIG_X86 */ #error WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS is set for an unsupported architecture. #endif /* archs */ #endif /* WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS */ #define __mutex_init (wolfssl_linuxkm_get_pie_redirect_table()->__mutex_init) #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0) #define mutex_lock_nested (wolfssl_linuxkm_get_pie_redirect_table()->mutex_lock_nested) #else #define mutex_lock (wolfssl_linuxkm_get_pie_redirect_table()->mutex_lock) #endif #define mutex_unlock (wolfssl_linuxkm_get_pie_redirect_table()->mutex_unlock) #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0) #define mutex_destroy (wolfssl_linuxkm_get_pie_redirect_table()->mutex_destroy) #endif /* per linux/ctype.h, tolower() and toupper() are macros bound to static inlines * that use macros that bring in the _ctype global. for __PIE__, this needs to * be masked out. */ #undef tolower #undef toupper #define tolower(c) (islower(c) ? (c) : ((c) + ('a'-'A'))) #define toupper(c) (isupper(c) ? (c) : ((c) - ('a'-'A'))) #if !defined(WOLFCRYPT_ONLY) && !defined(NO_CERTS) #define GetCA (wolfssl_linuxkm_get_pie_redirect_table()->GetCA) #ifndef NO_SKID #define GetCAByName (wolfssl_linuxkm_get_pie_redirect_table()->GetCAByName) #endif #endif #endif /* __PIE__ */ #endif /* USE_WOLFSSL_LINUXKM_PIE_REDIRECT_TABLE */ #ifdef WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS #ifdef CONFIG_X86 extern __must_check int allocate_wolfcrypt_linuxkm_fpu_states(void); extern void free_wolfcrypt_linuxkm_fpu_states(void); extern __must_check int save_vector_registers_x86(void); extern void restore_vector_registers_x86(void); #elif defined(CONFIG_ARM) || defined(CONFIG_ARM64) #error kernel module ARM SIMD is not yet tested or usable. static WARN_UNUSED_RESULT inline int save_vector_registers_arm(void) { preempt_disable(); if (! may_use_simd()) { preempt_enable(); return BAD_STATE_E; } else { fpsimd_preserve_current_state(); return 0; } } static inline void restore_vector_registers_arm(void) { fpsimd_restore_current_state(); preempt_enable(); } #endif #endif /* WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS */ /* remove this multifariously conflicting macro, picked up from * Linux arch//include/asm/current.h. */ #ifndef WOLFSSL_NEED_LINUX_CURRENT #undef current #endif /* min() and max() in linux/kernel.h over-aggressively type-check, producing * myriad spurious -Werrors throughout the codebase. */ #undef min #undef max /* work around namespace conflict between wolfssl/internal.h (enum HandShakeType) * and linux/key.h (extern int()). */ #define key_update wc_key_update #define lkm_printf(format, args...) printk(KERN_INFO "wolfssl: %s(): " format, __func__, ## args) #define printf(...) lkm_printf(__VA_ARGS__) #ifdef HAVE_FIPS extern void fipsEntry(void); #endif /* suppress false-positive "writing 1 byte into a region of size 0" warnings * building old kernels with new gcc: */ #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0) _Pragma("GCC diagnostic ignored \"-Wstringop-overflow\""); #endif /* includes are all above, with incompatible warnings masked out. */ #if LINUX_VERSION_CODE < KERNEL_VERSION(5, 5, 0) typedef __kernel_time_t time_t; #else typedef __kernel_time64_t time_t; #endif extern time_t time(time_t * timer); #define XTIME time #define WOLFSSL_GMTIME #define XGMTIME(c, t) gmtime(c) #define NO_TIMEVAL 1 #endif /* BUILDING_WOLFSSL */ /* if BUILDING_WOLFSSL, mutex.h will have already been included recursively * above, with the bevy of warnings suppressed, and the below include will * be a redundant no-op. */ #include typedef struct mutex wolfSSL_Mutex; /* prevent gcc's mm_malloc.h from being included, since it unconditionally * includes stdlib.h, which is kernel-incompatible. */ #define _MM_MALLOC_H_INCLUDED /* fun fact: since linux commit 59bb47985c, kmalloc with power-of-2 size is * aligned to the size. */ #define WC_LINUXKM_ROUND_UP_P_OF_2(x) ( \ { \ size_t _alloc_sz = (x); \ _alloc_sz = 1UL << ((sizeof(_alloc_sz) * 8UL) - __builtin_clzl(_alloc_sz)); \ _alloc_sz; \ }) #ifdef HAVE_KVMALLOC #define malloc(size) kvmalloc_node(WC_LINUXKM_ROUND_UP_P_OF_2(size), GFP_KERNEL, NUMA_NO_NODE) #define free(ptr) kvfree(ptr) void *lkm_realloc(void *ptr, size_t newsize); #define realloc(ptr, newsize) lkm_realloc(ptr, WC_LINUXKM_ROUND_UP_P_OF_2(newsize)) #else #define malloc(size) kmalloc(WC_LINUXKM_ROUND_UP_P_OF_2(size), GFP_KERNEL) #define free(ptr) kfree(ptr) #define realloc(ptr, newsize) krealloc(ptr, WC_LINUXKM_ROUND_UP_P_OF_2(newsize), GFP_KERNEL) #endif #ifndef static_assert #define static_assert(expr, ...) __static_assert(expr, ##__VA_ARGS__, #expr) #define __static_assert(expr, msg, ...) _Static_assert(expr, msg) #endif #include #ifdef WOLFSSL_TRACK_MEMORY #define XMALLOC(s, h, t) ({(void)(h); (void)(t); wolfSSL_Malloc(s);}) #ifdef WOLFSSL_XFREE_NO_NULLNESS_CHECK #define XFREE(p, h, t) ({(void)(h); (void)(t); wolfSSL_Free(p);}) #else #define XFREE(p, h, t) ({void* _xp; (void)(h); _xp = (p); if(_xp) wolfSSL_Free(_xp);}) #endif #define XREALLOC(p, n, h, t) ({(void)(h); (void)(t); wolfSSL_Realloc(p, n);}) #else #define XMALLOC(s, h, t) ({(void)(h); (void)(t); malloc(s);}) #ifdef WOLFSSL_XFREE_NO_NULLNESS_CHECK #define XFREE(p, h, t) ({(void)(h); (void)(t); free(p);}) #else #define XFREE(p, h, t) ({void* _xp; (void)(h); (void)(t); _xp = (p); if(_xp) free(_xp);}) #endif #define XREALLOC(p, n, h, t) ({(void)(h); (void)(t); realloc(p, n);}) #endif #include /* Linux headers define these using C expressions, but we need * them to be evaluable by the preprocessor, for use in sp_int.h. */ #if BITS_PER_LONG == 64 static_assert(sizeof(ULONG_MAX) == 8, "BITS_PER_LONG is 64, but ULONG_MAX is not."); #undef UCHAR_MAX #define UCHAR_MAX 255 #undef USHRT_MAX #define USHRT_MAX 65535 #undef UINT_MAX #define UINT_MAX 4294967295U #undef ULONG_MAX #define ULONG_MAX 18446744073709551615UL #undef ULLONG_MAX #define ULLONG_MAX ULONG_MAX #undef INT_MAX #define INT_MAX 2147483647 #undef LONG_MAX #define LONG_MAX 9223372036854775807L #undef LLONG_MAX #define LLONG_MAX LONG_MAX #elif BITS_PER_LONG == 32 static_assert(sizeof(ULONG_MAX) == 4, "BITS_PER_LONG is 32, but ULONG_MAX is not."); #undef UCHAR_MAX #define UCHAR_MAX 255 #undef USHRT_MAX #define USHRT_MAX 65535 #undef UINT_MAX #define UINT_MAX 4294967295U #undef ULONG_MAX #define ULONG_MAX 4294967295UL #undef INT_MAX #define INT_MAX 2147483647 #undef LONG_MAX #define LONG_MAX 2147483647L #undef ULLONG_MAX #undef LLONG_MAX #if BITS_PER_LONG_LONG == 64 #define ULLONG_MAX 18446744073709551615UL #define LLONG_MAX 9223372036854775807L #else #undef NO_64BIT #define NO_64BIT #define ULLONG_MAX ULONG_MAX #define LLONG_MAX LONG_MAX #endif #else #error unexpected BITS_PER_LONG value. #endif #endif /* LINUXKM_WC_PORT_H */