armcap.c 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439
  1. /*
  2. * Copyright 2011-2023 The OpenSSL Project Authors. All Rights Reserved.
  3. *
  4. * Licensed under the Apache License 2.0 (the "License"). You may not use
  5. * this file except in compliance with the License. You can obtain a copy
  6. * in the file LICENSE in the source distribution or at
  7. * https://www.openssl.org/source/license.html
  8. */
  9. #include <stdio.h>
  10. #include <stdlib.h>
  11. #include <string.h>
  12. #include <openssl/crypto.h>
  13. #ifdef __APPLE__
  14. #include <sys/sysctl.h>
  15. #else
  16. #include <setjmp.h>
  17. #include <signal.h>
  18. #endif
  19. #include "internal/cryptlib.h"
  20. #ifdef _WIN32
  21. #include <windows.h>
  22. #else
  23. #include <unistd.h>
  24. #endif
  25. #include "arm_arch.h"
  26. unsigned int OPENSSL_armcap_P = 0;
  27. unsigned int OPENSSL_arm_midr = 0;
  28. unsigned int OPENSSL_armv8_rsa_neonized = 0;
  29. #ifdef _WIN32
  30. void OPENSSL_cpuid_setup(void)
  31. {
  32. OPENSSL_armcap_P |= ARMV7_NEON;
  33. OPENSSL_armv8_rsa_neonized = 1;
  34. if (IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE)) {
  35. // These are all covered by one call in Windows
  36. OPENSSL_armcap_P |= ARMV8_AES;
  37. OPENSSL_armcap_P |= ARMV8_PMULL;
  38. OPENSSL_armcap_P |= ARMV8_SHA1;
  39. OPENSSL_armcap_P |= ARMV8_SHA256;
  40. }
  41. }
  42. uint32_t OPENSSL_rdtsc(void)
  43. {
  44. return 0;
  45. }
  46. #elif __ARM_MAX_ARCH__ < 7
  47. void OPENSSL_cpuid_setup(void)
  48. {
  49. }
  50. uint32_t OPENSSL_rdtsc(void)
  51. {
  52. return 0;
  53. }
  54. #else /* !_WIN32 && __ARM_MAX_ARCH__ >= 7 */
  55. /* 3 ways of handling things here: __APPLE__, getauxval() or SIGILL detect */
  56. /* First determine if getauxval() is available (OSSL_IMPLEMENT_GETAUXVAL) */
  57. # if defined(__GNUC__) && __GNUC__>=2
  58. void OPENSSL_cpuid_setup(void) __attribute__ ((constructor));
  59. # endif
  60. # if defined(__GLIBC__) && defined(__GLIBC_PREREQ)
  61. # if __GLIBC_PREREQ(2, 16)
  62. # include <sys/auxv.h>
  63. # define OSSL_IMPLEMENT_GETAUXVAL
  64. # endif
  65. # elif defined(__ANDROID_API__)
  66. /* see https://developer.android.google.cn/ndk/guides/cpu-features */
  67. # if __ANDROID_API__ >= 18
  68. # include <sys/auxv.h>
  69. # define OSSL_IMPLEMENT_GETAUXVAL
  70. # endif
  71. # endif
  72. # if defined(__FreeBSD__)
  73. # include <sys/param.h>
  74. # if __FreeBSD_version >= 1200000
  75. # include <sys/auxv.h>
  76. # define OSSL_IMPLEMENT_GETAUXVAL
  77. static unsigned long getauxval(unsigned long key)
  78. {
  79. unsigned long val = 0ul;
  80. if (elf_aux_info((int)key, &val, sizeof(val)) != 0)
  81. return 0ul;
  82. return val;
  83. }
  84. # endif
  85. # endif
  86. /*
  87. * Android: according to https://developer.android.com/ndk/guides/cpu-features,
  88. * getauxval is supported starting with API level 18
  89. */
  90. # if defined(__ANDROID__) && defined(__ANDROID_API__) && __ANDROID_API__ >= 18
  91. # include <sys/auxv.h>
  92. # define OSSL_IMPLEMENT_GETAUXVAL
  93. # endif
  94. /*
  95. * ARM puts the feature bits for Crypto Extensions in AT_HWCAP2, whereas
  96. * AArch64 used AT_HWCAP.
  97. */
  98. # ifndef AT_HWCAP
  99. # define AT_HWCAP 16
  100. # endif
  101. # ifndef AT_HWCAP2
  102. # define AT_HWCAP2 26
  103. # endif
  104. # if defined(__arm__) || defined (__arm)
  105. # define OSSL_HWCAP AT_HWCAP
  106. # define OSSL_HWCAP_NEON (1 << 12)
  107. # define OSSL_HWCAP_CE AT_HWCAP2
  108. # define OSSL_HWCAP_CE_AES (1 << 0)
  109. # define OSSL_HWCAP_CE_PMULL (1 << 1)
  110. # define OSSL_HWCAP_CE_SHA1 (1 << 2)
  111. # define OSSL_HWCAP_CE_SHA256 (1 << 3)
  112. # elif defined(__aarch64__)
  113. # define OSSL_HWCAP AT_HWCAP
  114. # define OSSL_HWCAP_NEON (1 << 1)
  115. # define OSSL_HWCAP_CE AT_HWCAP
  116. # define OSSL_HWCAP_CE_AES (1 << 3)
  117. # define OSSL_HWCAP_CE_PMULL (1 << 4)
  118. # define OSSL_HWCAP_CE_SHA1 (1 << 5)
  119. # define OSSL_HWCAP_CE_SHA256 (1 << 6)
  120. # define OSSL_HWCAP_CPUID (1 << 11)
  121. # define OSSL_HWCAP_SHA3 (1 << 17)
  122. # define OSSL_HWCAP_CE_SM3 (1 << 18)
  123. # define OSSL_HWCAP_CE_SM4 (1 << 19)
  124. # define OSSL_HWCAP_CE_SHA512 (1 << 21)
  125. # define OSSL_HWCAP_SVE (1 << 22)
  126. /* AT_HWCAP2 */
  127. # define OSSL_HWCAP2 26
  128. # define OSSL_HWCAP2_SVE2 (1 << 1)
  129. # define OSSL_HWCAP2_RNG (1 << 16)
  130. # endif
  131. uint32_t _armv7_tick(void);
  132. uint32_t OPENSSL_rdtsc(void)
  133. {
  134. if (OPENSSL_armcap_P & ARMV7_TICK)
  135. return _armv7_tick();
  136. else
  137. return 0;
  138. }
  139. # ifdef __aarch64__
  140. size_t OPENSSL_rndr_asm(unsigned char *buf, size_t len);
  141. size_t OPENSSL_rndrrs_asm(unsigned char *buf, size_t len);
  142. size_t OPENSSL_rndr_bytes(unsigned char *buf, size_t len);
  143. size_t OPENSSL_rndrrs_bytes(unsigned char *buf, size_t len);
  144. static size_t OPENSSL_rndr_wrapper(size_t (*func)(unsigned char *, size_t), unsigned char *buf, size_t len)
  145. {
  146. size_t buffer_size = 0;
  147. int i;
  148. for (i = 0; i < 8; i++) {
  149. buffer_size = func(buf, len);
  150. if (buffer_size == len)
  151. break;
  152. usleep(5000); /* 5000 microseconds (5 milliseconds) */
  153. }
  154. return buffer_size;
  155. }
  156. size_t OPENSSL_rndr_bytes(unsigned char *buf, size_t len)
  157. {
  158. return OPENSSL_rndr_wrapper(OPENSSL_rndr_asm, buf, len);
  159. }
  160. size_t OPENSSL_rndrrs_bytes(unsigned char *buf, size_t len)
  161. {
  162. return OPENSSL_rndr_wrapper(OPENSSL_rndrrs_asm, buf, len);
  163. }
  164. # endif
  165. # if !defined(__APPLE__) && !defined(OSSL_IMPLEMENT_GETAUXVAL)
  166. static sigset_t all_masked;
  167. static sigjmp_buf ill_jmp;
  168. static void ill_handler(int sig)
  169. {
  170. siglongjmp(ill_jmp, sig);
  171. }
  172. /*
  173. * Following subroutines could have been inlined, but not all
  174. * ARM compilers support inline assembler, and we'd then have to
  175. * worry about the compiler optimising out the detection code...
  176. */
  177. void _armv7_neon_probe(void);
  178. void _armv8_aes_probe(void);
  179. void _armv8_sha1_probe(void);
  180. void _armv8_sha256_probe(void);
  181. void _armv8_pmull_probe(void);
  182. # ifdef __aarch64__
  183. void _armv8_sm3_probe(void);
  184. void _armv8_sm4_probe(void);
  185. void _armv8_sha512_probe(void);
  186. void _armv8_eor3_probe(void);
  187. void _armv8_sve_probe(void);
  188. void _armv8_sve2_probe(void);
  189. void _armv8_rng_probe(void);
  190. # endif
  191. # endif /* !__APPLE__ && !OSSL_IMPLEMENT_GETAUXVAL */
  192. /* We only call _armv8_cpuid_probe() if (OPENSSL_armcap_P & ARMV8_CPUID) != 0 */
  193. unsigned int _armv8_cpuid_probe(void);
  194. # if defined(__APPLE__)
  195. /*
  196. * Checks the specified integer sysctl, returning `value` if it's 1, otherwise returning 0.
  197. */
  198. static unsigned int sysctl_query(const char *name, unsigned int value)
  199. {
  200. unsigned int sys_value = 0;
  201. size_t len = sizeof(sys_value);
  202. return (sysctlbyname(name, &sys_value, &len, NULL, 0) == 0 && sys_value == 1) ? value : 0;
  203. }
  204. # elif !defined(OSSL_IMPLEMENT_GETAUXVAL)
  205. /*
  206. * Calls a provided probe function, which may SIGILL. If it doesn't, return `value`, otherwise return 0.
  207. */
  208. static unsigned int arm_probe_for(void (*probe)(void), volatile unsigned int value)
  209. {
  210. if (sigsetjmp(ill_jmp, 1) == 0) {
  211. probe();
  212. return value;
  213. } else {
  214. /* The probe function gave us SIGILL */
  215. return 0;
  216. }
  217. }
  218. # endif
  219. void OPENSSL_cpuid_setup(void)
  220. {
  221. const char *e;
  222. # if !defined(__APPLE__) && !defined(OSSL_IMPLEMENT_GETAUXVAL)
  223. struct sigaction ill_oact, ill_act;
  224. sigset_t oset;
  225. # endif
  226. static int trigger = 0;
  227. if (trigger)
  228. return;
  229. trigger = 1;
  230. OPENSSL_armcap_P = 0;
  231. if ((e = getenv("OPENSSL_armcap"))) {
  232. OPENSSL_armcap_P = (unsigned int)strtoul(e, NULL, 0);
  233. return;
  234. }
  235. # if defined(__APPLE__)
  236. # if !defined(__aarch64__)
  237. /*
  238. * Capability probing by catching SIGILL appears to be problematic
  239. * on iOS. But since Apple universe is "monocultural", it's actually
  240. * possible to simply set pre-defined processor capability mask.
  241. */
  242. if (1) {
  243. OPENSSL_armcap_P = ARMV7_NEON;
  244. return;
  245. }
  246. # else
  247. {
  248. /*
  249. * From
  250. * https://github.com/llvm/llvm-project/blob/412237dcd07e5a2afbb1767858262a5f037149a3/llvm/lib/Target/AArch64/AArch64.td#L719
  251. * all of these have been available on 64-bit Apple Silicon from the
  252. * beginning (the A7).
  253. */
  254. OPENSSL_armcap_P |= ARMV7_NEON | ARMV8_PMULL | ARMV8_AES | ARMV8_SHA1 | ARMV8_SHA256;
  255. /* More recent extensions are indicated by sysctls */
  256. OPENSSL_armcap_P |= sysctl_query("hw.optional.armv8_2_sha512", ARMV8_SHA512);
  257. OPENSSL_armcap_P |= sysctl_query("hw.optional.armv8_2_sha3", ARMV8_SHA3);
  258. if (OPENSSL_armcap_P & ARMV8_SHA3) {
  259. char uarch[64];
  260. size_t len = sizeof(uarch);
  261. if ((sysctlbyname("machdep.cpu.brand_string", uarch, &len, NULL, 0) == 0) &&
  262. ((strncmp(uarch, "Apple M1", 8) == 0) ||
  263. (strncmp(uarch, "Apple M2", 8) == 0))) {
  264. OPENSSL_armcap_P |= ARMV8_UNROLL8_EOR3;
  265. OPENSSL_armcap_P |= ARMV8_HAVE_SHA3_AND_WORTH_USING;
  266. }
  267. }
  268. }
  269. # endif /* __aarch64__ */
  270. # elif defined(OSSL_IMPLEMENT_GETAUXVAL)
  271. if (getauxval(OSSL_HWCAP) & OSSL_HWCAP_NEON) {
  272. unsigned long hwcap = getauxval(OSSL_HWCAP_CE);
  273. OPENSSL_armcap_P |= ARMV7_NEON;
  274. if (hwcap & OSSL_HWCAP_CE_AES)
  275. OPENSSL_armcap_P |= ARMV8_AES;
  276. if (hwcap & OSSL_HWCAP_CE_PMULL)
  277. OPENSSL_armcap_P |= ARMV8_PMULL;
  278. if (hwcap & OSSL_HWCAP_CE_SHA1)
  279. OPENSSL_armcap_P |= ARMV8_SHA1;
  280. if (hwcap & OSSL_HWCAP_CE_SHA256)
  281. OPENSSL_armcap_P |= ARMV8_SHA256;
  282. # ifdef __aarch64__
  283. if (hwcap & OSSL_HWCAP_CE_SM4)
  284. OPENSSL_armcap_P |= ARMV8_SM4;
  285. if (hwcap & OSSL_HWCAP_CE_SHA512)
  286. OPENSSL_armcap_P |= ARMV8_SHA512;
  287. if (hwcap & OSSL_HWCAP_CPUID)
  288. OPENSSL_armcap_P |= ARMV8_CPUID;
  289. if (hwcap & OSSL_HWCAP_CE_SM3)
  290. OPENSSL_armcap_P |= ARMV8_SM3;
  291. if (hwcap & OSSL_HWCAP_SHA3)
  292. OPENSSL_armcap_P |= ARMV8_SHA3;
  293. # endif
  294. }
  295. # ifdef __aarch64__
  296. if (getauxval(OSSL_HWCAP) & OSSL_HWCAP_SVE)
  297. OPENSSL_armcap_P |= ARMV8_SVE;
  298. if (getauxval(OSSL_HWCAP2) & OSSL_HWCAP2_SVE2)
  299. OPENSSL_armcap_P |= ARMV8_SVE2;
  300. if (getauxval(OSSL_HWCAP2) & OSSL_HWCAP2_RNG)
  301. OPENSSL_armcap_P |= ARMV8_RNG;
  302. # endif
  303. # else /* !__APPLE__ && !OSSL_IMPLEMENT_GETAUXVAL */
  304. /* If all else fails, do brute force SIGILL-based feature detection */
  305. sigfillset(&all_masked);
  306. sigdelset(&all_masked, SIGILL);
  307. sigdelset(&all_masked, SIGTRAP);
  308. sigdelset(&all_masked, SIGFPE);
  309. sigdelset(&all_masked, SIGBUS);
  310. sigdelset(&all_masked, SIGSEGV);
  311. memset(&ill_act, 0, sizeof(ill_act));
  312. ill_act.sa_handler = ill_handler;
  313. ill_act.sa_mask = all_masked;
  314. sigprocmask(SIG_SETMASK, &ill_act.sa_mask, &oset);
  315. sigaction(SIGILL, &ill_act, &ill_oact);
  316. OPENSSL_armcap_P |= arm_probe_for(_armv7_neon_probe, ARMV7_NEON);
  317. if (OPENSSL_armcap_P & ARMV7_NEON) {
  318. OPENSSL_armcap_P |= arm_probe_for(_armv8_pmull_probe, ARMV8_PMULL | ARMV8_AES);
  319. if (!(OPENSSL_armcap_P & ARMV8_AES)) {
  320. OPENSSL_armcap_P |= arm_probe_for(_armv8_aes_probe, ARMV8_AES);
  321. }
  322. OPENSSL_armcap_P |= arm_probe_for(_armv8_sha1_probe, ARMV8_SHA1);
  323. OPENSSL_armcap_P |= arm_probe_for(_armv8_sha256_probe, ARMV8_SHA256);
  324. # if defined(__aarch64__)
  325. OPENSSL_armcap_P |= arm_probe_for(_armv8_sm3_probe, ARMV8_SM3);
  326. OPENSSL_armcap_P |= arm_probe_for(_armv8_sm4_probe, ARMV8_SM4);
  327. OPENSSL_armcap_P |= arm_probe_for(_armv8_sha512_probe, ARMV8_SHA512);
  328. OPENSSL_armcap_P |= arm_probe_for(_armv8_eor3_probe, ARMV8_SHA3);
  329. # endif
  330. }
  331. # ifdef __aarch64__
  332. OPENSSL_armcap_P |= arm_probe_for(_armv8_sve_probe, ARMV8_SVE);
  333. OPENSSL_armcap_P |= arm_probe_for(_armv8_sve2_probe, ARMV8_SVE2);
  334. OPENSSL_armcap_P |= arm_probe_for(_armv8_rng_probe, ARMV8_RNG);
  335. # endif
  336. /*
  337. * Probing for ARMV7_TICK is known to produce unreliable results,
  338. * so we only use the feature when the user explicitly enables it
  339. * with OPENSSL_armcap.
  340. */
  341. sigaction(SIGILL, &ill_oact, NULL);
  342. sigprocmask(SIG_SETMASK, &oset, NULL);
  343. # endif /* __APPLE__, OSSL_IMPLEMENT_GETAUXVAL */
  344. # ifdef __aarch64__
  345. if (OPENSSL_armcap_P & ARMV8_CPUID)
  346. OPENSSL_arm_midr = _armv8_cpuid_probe();
  347. if ((MIDR_IS_CPU_MODEL(OPENSSL_arm_midr, ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A72) ||
  348. MIDR_IS_CPU_MODEL(OPENSSL_arm_midr, ARM_CPU_IMP_ARM, ARM_CPU_PART_N1)) &&
  349. (OPENSSL_armcap_P & ARMV7_NEON)) {
  350. OPENSSL_armv8_rsa_neonized = 1;
  351. }
  352. if ((MIDR_IS_CPU_MODEL(OPENSSL_arm_midr, ARM_CPU_IMP_ARM, ARM_CPU_PART_V1) ||
  353. MIDR_IS_CPU_MODEL(OPENSSL_arm_midr, ARM_CPU_IMP_ARM, ARM_CPU_PART_N2) ||
  354. MIDR_IS_CPU_MODEL(OPENSSL_arm_midr, ARM_CPU_IMP_ARM, ARM_CPU_PART_V2)) &&
  355. (OPENSSL_armcap_P & ARMV8_SHA3))
  356. OPENSSL_armcap_P |= ARMV8_UNROLL8_EOR3;
  357. if ((MIDR_IS_CPU_MODEL(OPENSSL_arm_midr, ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M1_FIRESTORM) ||
  358. MIDR_IS_CPU_MODEL(OPENSSL_arm_midr, ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M1_ICESTORM) ||
  359. MIDR_IS_CPU_MODEL(OPENSSL_arm_midr, ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M1_FIRESTORM_PRO) ||
  360. MIDR_IS_CPU_MODEL(OPENSSL_arm_midr, ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M1_ICESTORM_PRO) ||
  361. MIDR_IS_CPU_MODEL(OPENSSL_arm_midr, ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M1_FIRESTORM_MAX) ||
  362. MIDR_IS_CPU_MODEL(OPENSSL_arm_midr, ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M1_ICESTORM_MAX) ||
  363. MIDR_IS_CPU_MODEL(OPENSSL_arm_midr, ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M2_AVALANCHE) ||
  364. MIDR_IS_CPU_MODEL(OPENSSL_arm_midr, ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M2_BLIZZARD) ||
  365. MIDR_IS_CPU_MODEL(OPENSSL_arm_midr, ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M2_AVALANCHE_PRO) ||
  366. MIDR_IS_CPU_MODEL(OPENSSL_arm_midr, ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M2_BLIZZARD_PRO) ||
  367. MIDR_IS_CPU_MODEL(OPENSSL_arm_midr, ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M2_AVALANCHE_MAX) ||
  368. MIDR_IS_CPU_MODEL(OPENSSL_arm_midr, ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M2_BLIZZARD_MAX)) &&
  369. (OPENSSL_armcap_P & ARMV8_SHA3))
  370. OPENSSL_armcap_P |= ARMV8_HAVE_SHA3_AND_WORTH_USING;
  371. # endif
  372. }
  373. #endif /* _WIN32, __ARM_MAX_ARCH__ >= 7 */