armcap.c 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384
  1. /*
  2. * Copyright 2011-2022 The OpenSSL Project Authors. All Rights Reserved.
  3. *
  4. * Licensed under the Apache License 2.0 (the "License"). You may not use
  5. * this file except in compliance with the License. You can obtain a copy
  6. * in the file LICENSE in the source distribution or at
  7. * https://www.openssl.org/source/license.html
  8. */
  9. #include <stdio.h>
  10. #include <stdlib.h>
  11. #include <string.h>
  12. #include <setjmp.h>
  13. #include <signal.h>
  14. #include <openssl/crypto.h>
  15. #ifdef __APPLE__
  16. #include <sys/sysctl.h>
  17. #endif
  18. #include "internal/cryptlib.h"
  19. #include <unistd.h>
  20. #include "arm_arch.h"
  21. unsigned int OPENSSL_armcap_P = 0;
  22. unsigned int OPENSSL_arm_midr = 0;
  23. unsigned int OPENSSL_armv8_rsa_neonized = 0;
  24. #if __ARM_MAX_ARCH__<7
  25. void OPENSSL_cpuid_setup(void)
  26. {
  27. }
  28. uint32_t OPENSSL_rdtsc(void)
  29. {
  30. return 0;
  31. }
  32. #else
  33. static sigset_t all_masked;
  34. static sigjmp_buf ill_jmp;
  35. static void ill_handler(int sig)
  36. {
  37. siglongjmp(ill_jmp, sig);
  38. }
  39. /*
  40. * Following subroutines could have been inlined, but it's not all
  41. * ARM compilers support inline assembler...
  42. */
  43. void _armv7_neon_probe(void);
  44. void _armv8_aes_probe(void);
  45. void _armv8_sha1_probe(void);
  46. void _armv8_sha256_probe(void);
  47. void _armv8_pmull_probe(void);
  48. # ifdef __aarch64__
  49. void _armv8_sm3_probe(void);
  50. void _armv8_sm4_probe(void);
  51. void _armv8_sha512_probe(void);
  52. unsigned int _armv8_cpuid_probe(void);
  53. void _armv8_sve_probe(void);
  54. void _armv8_sve2_probe(void);
  55. void _armv8_rng_probe(void);
  56. size_t OPENSSL_rndr_asm(unsigned char *buf, size_t len);
  57. size_t OPENSSL_rndrrs_asm(unsigned char *buf, size_t len);
  58. size_t OPENSSL_rndr_bytes(unsigned char *buf, size_t len);
  59. size_t OPENSSL_rndrrs_bytes(unsigned char *buf, size_t len);
  60. static size_t OPENSSL_rndr_wrapper(size_t (*func)(unsigned char *, size_t), unsigned char *buf, size_t len)
  61. {
  62. size_t buffer_size = 0;
  63. int i;
  64. for (i = 0; i < 8; i++) {
  65. buffer_size = func(buf, len);
  66. if (buffer_size == len)
  67. break;
  68. usleep(5000); /* 5000 microseconds (5 milliseconds) */
  69. }
  70. return buffer_size;
  71. }
  72. size_t OPENSSL_rndr_bytes(unsigned char *buf, size_t len)
  73. {
  74. return OPENSSL_rndr_wrapper(OPENSSL_rndr_asm, buf, len);
  75. }
  76. size_t OPENSSL_rndrrs_bytes(unsigned char *buf, size_t len)
  77. {
  78. return OPENSSL_rndr_wrapper(OPENSSL_rndrrs_asm, buf, len);
  79. }
  80. # endif
  81. uint32_t _armv7_tick(void);
  82. uint32_t OPENSSL_rdtsc(void)
  83. {
  84. if (OPENSSL_armcap_P & ARMV7_TICK)
  85. return _armv7_tick();
  86. else
  87. return 0;
  88. }
  89. # if defined(__GNUC__) && __GNUC__>=2
  90. void OPENSSL_cpuid_setup(void) __attribute__ ((constructor));
  91. # endif
  92. # if defined(__GLIBC__) && defined(__GLIBC_PREREQ)
  93. # if __GLIBC_PREREQ(2, 16)
  94. # include <sys/auxv.h>
  95. # define OSSL_IMPLEMENT_GETAUXVAL
  96. # endif
  97. # elif defined(__ANDROID_API__)
  98. /* see https://developer.android.google.cn/ndk/guides/cpu-features */
  99. # if __ANDROID_API__ >= 18
  100. # include <sys/auxv.h>
  101. # define OSSL_IMPLEMENT_GETAUXVAL
  102. # endif
  103. # endif
  104. # if defined(__FreeBSD__)
  105. # include <sys/param.h>
  106. # if __FreeBSD_version >= 1200000
  107. # include <sys/auxv.h>
  108. # define OSSL_IMPLEMENT_GETAUXVAL
  109. static unsigned long getauxval(unsigned long key)
  110. {
  111. unsigned long val = 0ul;
  112. if (elf_aux_info((int)key, &val, sizeof(val)) != 0)
  113. return 0ul;
  114. return val;
  115. }
  116. # endif
  117. # endif
  118. /*
  119. * Android: according to https://developer.android.com/ndk/guides/cpu-features,
  120. * getauxval is supported starting with API level 18
  121. */
  122. # if defined(__ANDROID__) && defined(__ANDROID_API__) && __ANDROID_API__ >= 18
  123. # include <sys/auxv.h>
  124. # define OSSL_IMPLEMENT_GETAUXVAL
  125. # endif
  126. /*
  127. * ARM puts the feature bits for Crypto Extensions in AT_HWCAP2, whereas
  128. * AArch64 used AT_HWCAP.
  129. */
  130. # ifndef AT_HWCAP
  131. # define AT_HWCAP 16
  132. # endif
  133. # ifndef AT_HWCAP2
  134. # define AT_HWCAP2 26
  135. # endif
  136. # if defined(__arm__) || defined (__arm)
  137. # define HWCAP AT_HWCAP
  138. # define HWCAP_NEON (1 << 12)
  139. # define HWCAP_CE AT_HWCAP2
  140. # define HWCAP_CE_AES (1 << 0)
  141. # define HWCAP_CE_PMULL (1 << 1)
  142. # define HWCAP_CE_SHA1 (1 << 2)
  143. # define HWCAP_CE_SHA256 (1 << 3)
  144. # elif defined(__aarch64__)
  145. # define HWCAP AT_HWCAP
  146. # define HWCAP_NEON (1 << 1)
  147. # define HWCAP_CE HWCAP
  148. # define HWCAP_CE_AES (1 << 3)
  149. # define HWCAP_CE_PMULL (1 << 4)
  150. # define HWCAP_CE_SHA1 (1 << 5)
  151. # define HWCAP_CE_SHA256 (1 << 6)
  152. # define HWCAP_CPUID (1 << 11)
  153. # define HWCAP_SHA3 (1 << 17)
  154. # define HWCAP_CE_SM3 (1 << 18)
  155. # define HWCAP_CE_SM4 (1 << 19)
  156. # define HWCAP_CE_SHA512 (1 << 21)
  157. # define HWCAP_SVE (1 << 22)
  158. /* AT_HWCAP2 */
  159. # define HWCAP2 26
  160. # define HWCAP2_SVE2 (1 << 1)
  161. # define HWCAP2_RNG (1 << 16)
  162. # endif
  163. void OPENSSL_cpuid_setup(void)
  164. {
  165. const char *e;
  166. struct sigaction ill_oact, ill_act;
  167. sigset_t oset;
  168. static int trigger = 0;
  169. if (trigger)
  170. return;
  171. trigger = 1;
  172. OPENSSL_armcap_P = 0;
  173. if ((e = getenv("OPENSSL_armcap"))) {
  174. OPENSSL_armcap_P = (unsigned int)strtoul(e, NULL, 0);
  175. return;
  176. }
  177. # if defined(__APPLE__)
  178. # if !defined(__aarch64__)
  179. /*
  180. * Capability probing by catching SIGILL appears to be problematic
  181. * on iOS. But since Apple universe is "monocultural", it's actually
  182. * possible to simply set pre-defined processor capability mask.
  183. */
  184. if (1) {
  185. OPENSSL_armcap_P = ARMV7_NEON;
  186. return;
  187. }
  188. /*
  189. * One could do same even for __aarch64__ iOS builds. It's not done
  190. * exclusively for reasons of keeping code unified across platforms.
  191. * Unified code works because it never triggers SIGILL on Apple
  192. * devices...
  193. */
  194. # else
  195. {
  196. unsigned int feature;
  197. size_t len = sizeof(feature);
  198. char uarch[64];
  199. if (sysctlbyname("hw.optional.armv8_2_sha512", &feature, &len, NULL, 0) == 0 && feature == 1)
  200. OPENSSL_armcap_P |= ARMV8_SHA512;
  201. feature = 0;
  202. if (sysctlbyname("hw.optional.armv8_2_sha3", &feature, &len, NULL, 0) == 0 && feature == 1) {
  203. OPENSSL_armcap_P |= ARMV8_SHA3;
  204. len = sizeof(uarch);
  205. if ((sysctlbyname("machdep.cpu.brand_string", uarch, &len, NULL, 0) == 0) &&
  206. (strncmp(uarch, "Apple M1", 8) == 0))
  207. OPENSSL_armcap_P |= ARMV8_UNROLL8_EOR3;
  208. }
  209. }
  210. # endif
  211. # endif
  212. # ifdef OSSL_IMPLEMENT_GETAUXVAL
  213. if (getauxval(HWCAP) & HWCAP_NEON) {
  214. unsigned long hwcap = getauxval(HWCAP_CE);
  215. OPENSSL_armcap_P |= ARMV7_NEON;
  216. if (hwcap & HWCAP_CE_AES)
  217. OPENSSL_armcap_P |= ARMV8_AES;
  218. if (hwcap & HWCAP_CE_PMULL)
  219. OPENSSL_armcap_P |= ARMV8_PMULL;
  220. if (hwcap & HWCAP_CE_SHA1)
  221. OPENSSL_armcap_P |= ARMV8_SHA1;
  222. if (hwcap & HWCAP_CE_SHA256)
  223. OPENSSL_armcap_P |= ARMV8_SHA256;
  224. # ifdef __aarch64__
  225. if (hwcap & HWCAP_CE_SM4)
  226. OPENSSL_armcap_P |= ARMV8_SM4;
  227. if (hwcap & HWCAP_CE_SHA512)
  228. OPENSSL_armcap_P |= ARMV8_SHA512;
  229. if (hwcap & HWCAP_CPUID)
  230. OPENSSL_armcap_P |= ARMV8_CPUID;
  231. if (hwcap & HWCAP_CE_SM3)
  232. OPENSSL_armcap_P |= ARMV8_SM3;
  233. if (hwcap & HWCAP_SHA3)
  234. OPENSSL_armcap_P |= ARMV8_SHA3;
  235. # endif
  236. }
  237. # ifdef __aarch64__
  238. if (getauxval(HWCAP) & HWCAP_SVE)
  239. OPENSSL_armcap_P |= ARMV8_SVE;
  240. if (getauxval(HWCAP2) & HWCAP2_SVE2)
  241. OPENSSL_armcap_P |= ARMV8_SVE2;
  242. if (getauxval(HWCAP2) & HWCAP2_RNG)
  243. OPENSSL_armcap_P |= ARMV8_RNG;
  244. # endif
  245. # endif
  246. sigfillset(&all_masked);
  247. sigdelset(&all_masked, SIGILL);
  248. sigdelset(&all_masked, SIGTRAP);
  249. sigdelset(&all_masked, SIGFPE);
  250. sigdelset(&all_masked, SIGBUS);
  251. sigdelset(&all_masked, SIGSEGV);
  252. memset(&ill_act, 0, sizeof(ill_act));
  253. ill_act.sa_handler = ill_handler;
  254. ill_act.sa_mask = all_masked;
  255. sigprocmask(SIG_SETMASK, &ill_act.sa_mask, &oset);
  256. sigaction(SIGILL, &ill_act, &ill_oact);
  257. /* If we used getauxval, we already have all the values */
  258. # ifndef OSSL_IMPLEMENT_GETAUXVAL
  259. if (sigsetjmp(ill_jmp, 1) == 0) {
  260. _armv7_neon_probe();
  261. OPENSSL_armcap_P |= ARMV7_NEON;
  262. if (sigsetjmp(ill_jmp, 1) == 0) {
  263. _armv8_pmull_probe();
  264. OPENSSL_armcap_P |= ARMV8_PMULL | ARMV8_AES;
  265. } else if (sigsetjmp(ill_jmp, 1) == 0) {
  266. _armv8_aes_probe();
  267. OPENSSL_armcap_P |= ARMV8_AES;
  268. }
  269. if (sigsetjmp(ill_jmp, 1) == 0) {
  270. _armv8_sha1_probe();
  271. OPENSSL_armcap_P |= ARMV8_SHA1;
  272. }
  273. if (sigsetjmp(ill_jmp, 1) == 0) {
  274. _armv8_sha256_probe();
  275. OPENSSL_armcap_P |= ARMV8_SHA256;
  276. }
  277. # if defined(__aarch64__) && !defined(__APPLE__)
  278. if (sigsetjmp(ill_jmp, 1) == 0) {
  279. _armv8_sm4_probe();
  280. OPENSSL_armcap_P |= ARMV8_SM4;
  281. }
  282. if (sigsetjmp(ill_jmp, 1) == 0) {
  283. _armv8_sha512_probe();
  284. OPENSSL_armcap_P |= ARMV8_SHA512;
  285. }
  286. if (sigsetjmp(ill_jmp, 1) == 0) {
  287. _armv8_sm3_probe();
  288. OPENSSL_armcap_P |= ARMV8_SM3;
  289. if (sigsetjmp(ill_jmp, 1) == 0) {
  290. _armv8_eor3_probe();
  291. OPENSSL_armcap_P |= ARMV8_SHA3;
  292. }
  293. # endif
  294. }
  295. # ifdef __aarch64__
  296. if (sigsetjmp(ill_jmp, 1) == 0) {
  297. _armv8_sve_probe();
  298. OPENSSL_armcap_P |= ARMV8_SVE;
  299. }
  300. if (sigsetjmp(ill_jmp, 1) == 0) {
  301. _armv8_sve2_probe();
  302. OPENSSL_armcap_P |= ARMV8_SVE2;
  303. }
  304. if (sigsetjmp(ill_jmp, 1) == 0) {
  305. _armv8_rng_probe();
  306. OPENSSL_armcap_P |= ARMV8_RNG;
  307. }
  308. # endif
  309. # endif
  310. /* Things that getauxval didn't tell us */
  311. if (sigsetjmp(ill_jmp, 1) == 0) {
  312. _armv7_tick();
  313. OPENSSL_armcap_P |= ARMV7_TICK;
  314. }
  315. sigaction(SIGILL, &ill_oact, NULL);
  316. sigprocmask(SIG_SETMASK, &oset, NULL);
  317. # ifdef __aarch64__
  318. if (OPENSSL_armcap_P & ARMV8_CPUID)
  319. OPENSSL_arm_midr = _armv8_cpuid_probe();
  320. if ((MIDR_IS_CPU_MODEL(OPENSSL_arm_midr, ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A72) ||
  321. MIDR_IS_CPU_MODEL(OPENSSL_arm_midr, ARM_CPU_IMP_ARM, ARM_CPU_PART_N1)) &&
  322. (OPENSSL_armcap_P & ARMV7_NEON)) {
  323. OPENSSL_armv8_rsa_neonized = 1;
  324. }
  325. if ((MIDR_IS_CPU_MODEL(OPENSSL_arm_midr, ARM_CPU_IMP_ARM, ARM_CPU_PART_V1) ||
  326. MIDR_IS_CPU_MODEL(OPENSSL_arm_midr, ARM_CPU_IMP_ARM, ARM_CPU_PART_N2)) &&
  327. (OPENSSL_armcap_P & ARMV8_SHA3))
  328. OPENSSL_armcap_P |= ARMV8_UNROLL8_EOR3;
  329. # endif
  330. }
  331. #endif