ppccap.c 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424
  1. /*
  2. * Copyright 2009-2018 The OpenSSL Project Authors. All Rights Reserved.
  3. *
  4. * Licensed under the Apache License 2.0 (the "License"). You may not use
  5. * this file except in compliance with the License. You can obtain a copy
  6. * in the file LICENSE in the source distribution or at
  7. * https://www.openssl.org/source/license.html
  8. */
  9. #include <stdio.h>
  10. #include <stdlib.h>
  11. #include <string.h>
  12. #include <setjmp.h>
  13. #include <signal.h>
  14. #include <unistd.h>
  15. #if defined(__linux) || defined(_AIX)
  16. # include <sys/utsname.h>
  17. #endif
  18. #if defined(_AIX53) /* defined even on post-5.3 */
  19. # include <sys/systemcfg.h>
  20. # if !defined(__power_set)
  21. # define __power_set(a) (_system_configuration.implementation & (a))
  22. # endif
  23. #endif
  24. #if defined(__APPLE__) && defined(__MACH__)
  25. # include <sys/types.h>
  26. # include <sys/sysctl.h>
  27. #endif
  28. #include <openssl/crypto.h>
  29. #include <openssl/bn.h>
  30. #include <internal/cryptlib.h>
  31. #include <crypto/chacha.h>
  32. #include "bn/bn_local.h"
  33. #include "ppc_arch.h"
  34. unsigned int OPENSSL_ppccap_P = 0;
  35. static sigset_t all_masked;
  36. #ifdef OPENSSL_BN_ASM_MONT
  37. int bn_mul_mont(BN_ULONG *rp, const BN_ULONG *ap, const BN_ULONG *bp,
  38. const BN_ULONG *np, const BN_ULONG *n0, int num)
  39. {
  40. int bn_mul_mont_int(BN_ULONG *rp, const BN_ULONG *ap, const BN_ULONG *bp,
  41. const BN_ULONG *np, const BN_ULONG *n0, int num);
  42. int bn_mul4x_mont_int(BN_ULONG *rp, const BN_ULONG *ap, const BN_ULONG *bp,
  43. const BN_ULONG *np, const BN_ULONG *n0, int num);
  44. if (num < 4)
  45. return 0;
  46. if ((num & 3) == 0)
  47. return bn_mul4x_mont_int(rp, ap, bp, np, n0, num);
  48. /*
  49. * There used to be [optional] call to bn_mul_mont_fpu64 here,
  50. * but above subroutine is faster on contemporary processors.
  51. * Formulation means that there might be old processors where
  52. * FPU code path would be faster, POWER6 perhaps, but there was
  53. * no opportunity to figure it out...
  54. */
  55. return bn_mul_mont_int(rp, ap, bp, np, n0, num);
  56. }
  57. #endif
  58. void sha256_block_p8(void *ctx, const void *inp, size_t len);
  59. void sha256_block_ppc(void *ctx, const void *inp, size_t len);
  60. void sha256_block_data_order(void *ctx, const void *inp, size_t len);
  61. void sha256_block_data_order(void *ctx, const void *inp, size_t len)
  62. {
  63. OPENSSL_ppccap_P & PPC_CRYPTO207 ? sha256_block_p8(ctx, inp, len) :
  64. sha256_block_ppc(ctx, inp, len);
  65. }
  66. void sha512_block_p8(void *ctx, const void *inp, size_t len);
  67. void sha512_block_ppc(void *ctx, const void *inp, size_t len);
  68. void sha512_block_data_order(void *ctx, const void *inp, size_t len);
  69. void sha512_block_data_order(void *ctx, const void *inp, size_t len)
  70. {
  71. OPENSSL_ppccap_P & PPC_CRYPTO207 ? sha512_block_p8(ctx, inp, len) :
  72. sha512_block_ppc(ctx, inp, len);
  73. }
  74. /*
  75. * TODO(3.0): Temporarily disabled some assembler that hasn't been brought into
  76. * the FIPS module yet.
  77. */
  78. #ifndef FIPS_MODE
  79. # ifndef OPENSSL_NO_CHACHA
  80. void ChaCha20_ctr32_int(unsigned char *out, const unsigned char *inp,
  81. size_t len, const unsigned int key[8],
  82. const unsigned int counter[4]);
  83. void ChaCha20_ctr32_vmx(unsigned char *out, const unsigned char *inp,
  84. size_t len, const unsigned int key[8],
  85. const unsigned int counter[4]);
  86. void ChaCha20_ctr32_vsx(unsigned char *out, const unsigned char *inp,
  87. size_t len, const unsigned int key[8],
  88. const unsigned int counter[4]);
  89. void ChaCha20_ctr32(unsigned char *out, const unsigned char *inp,
  90. size_t len, const unsigned int key[8],
  91. const unsigned int counter[4])
  92. {
  93. OPENSSL_ppccap_P & PPC_CRYPTO207
  94. ? ChaCha20_ctr32_vsx(out, inp, len, key, counter)
  95. : OPENSSL_ppccap_P & PPC_ALTIVEC
  96. ? ChaCha20_ctr32_vmx(out, inp, len, key, counter)
  97. : ChaCha20_ctr32_int(out, inp, len, key, counter);
  98. }
  99. # endif
  100. # ifndef OPENSSL_NO_POLY1305
  101. void poly1305_init_int(void *ctx, const unsigned char key[16]);
  102. void poly1305_blocks(void *ctx, const unsigned char *inp, size_t len,
  103. unsigned int padbit);
  104. void poly1305_emit(void *ctx, unsigned char mac[16],
  105. const unsigned int nonce[4]);
  106. void poly1305_init_fpu(void *ctx, const unsigned char key[16]);
  107. void poly1305_blocks_fpu(void *ctx, const unsigned char *inp, size_t len,
  108. unsigned int padbit);
  109. void poly1305_emit_fpu(void *ctx, unsigned char mac[16],
  110. const unsigned int nonce[4]);
  111. void poly1305_init_vsx(void *ctx, const unsigned char key[16]);
  112. void poly1305_blocks_vsx(void *ctx, const unsigned char *inp, size_t len,
  113. unsigned int padbit);
  114. void poly1305_emit_vsx(void *ctx, unsigned char mac[16],
  115. const unsigned int nonce[4]);
  116. int poly1305_init(void *ctx, const unsigned char key[16], void *func[2]);
  117. int poly1305_init(void *ctx, const unsigned char key[16], void *func[2])
  118. {
  119. if (OPENSSL_ppccap_P & PPC_CRYPTO207) {
  120. poly1305_init_int(ctx, key);
  121. func[0] = (void*)(uintptr_t)poly1305_blocks_vsx;
  122. func[1] = (void*)(uintptr_t)poly1305_emit;
  123. } else if (sizeof(size_t) == 4 && (OPENSSL_ppccap_P & PPC_FPU)) {
  124. poly1305_init_fpu(ctx, key);
  125. func[0] = (void*)(uintptr_t)poly1305_blocks_fpu;
  126. func[1] = (void*)(uintptr_t)poly1305_emit_fpu;
  127. } else {
  128. poly1305_init_int(ctx, key);
  129. func[0] = (void*)(uintptr_t)poly1305_blocks;
  130. func[1] = (void*)(uintptr_t)poly1305_emit;
  131. }
  132. return 1;
  133. }
  134. # endif
  135. #endif /* FIPS_MODE */
  136. #ifdef ECP_NISTZ256_ASM
  137. void ecp_nistz256_mul_mont(unsigned long res[4], const unsigned long a[4],
  138. const unsigned long b[4]);
  139. void ecp_nistz256_to_mont(unsigned long res[4], const unsigned long in[4]);
  140. void ecp_nistz256_to_mont(unsigned long res[4], const unsigned long in[4])
  141. {
  142. static const unsigned long RR[] = { 0x0000000000000003U,
  143. 0xfffffffbffffffffU,
  144. 0xfffffffffffffffeU,
  145. 0x00000004fffffffdU };
  146. ecp_nistz256_mul_mont(res, in, RR);
  147. }
  148. void ecp_nistz256_from_mont(unsigned long res[4], const unsigned long in[4]);
  149. void ecp_nistz256_from_mont(unsigned long res[4], const unsigned long in[4])
  150. {
  151. static const unsigned long one[] = { 1, 0, 0, 0 };
  152. ecp_nistz256_mul_mont(res, in, one);
  153. }
  154. #endif
  155. static sigjmp_buf ill_jmp;
  156. static void ill_handler(int sig)
  157. {
  158. siglongjmp(ill_jmp, sig);
  159. }
  160. void OPENSSL_fpu_probe(void);
  161. void OPENSSL_ppc64_probe(void);
  162. void OPENSSL_altivec_probe(void);
  163. void OPENSSL_crypto207_probe(void);
  164. void OPENSSL_madd300_probe(void);
  165. long OPENSSL_rdtsc_mftb(void);
  166. long OPENSSL_rdtsc_mfspr268(void);
  167. uint32_t OPENSSL_rdtsc(void)
  168. {
  169. if (OPENSSL_ppccap_P & PPC_MFTB)
  170. return OPENSSL_rdtsc_mftb();
  171. else if (OPENSSL_ppccap_P & PPC_MFSPR268)
  172. return OPENSSL_rdtsc_mfspr268();
  173. else
  174. return 0;
  175. }
  176. size_t OPENSSL_instrument_bus_mftb(unsigned int *, size_t);
  177. size_t OPENSSL_instrument_bus_mfspr268(unsigned int *, size_t);
  178. size_t OPENSSL_instrument_bus(unsigned int *out, size_t cnt)
  179. {
  180. if (OPENSSL_ppccap_P & PPC_MFTB)
  181. return OPENSSL_instrument_bus_mftb(out, cnt);
  182. else if (OPENSSL_ppccap_P & PPC_MFSPR268)
  183. return OPENSSL_instrument_bus_mfspr268(out, cnt);
  184. else
  185. return 0;
  186. }
  187. size_t OPENSSL_instrument_bus2_mftb(unsigned int *, size_t, size_t);
  188. size_t OPENSSL_instrument_bus2_mfspr268(unsigned int *, size_t, size_t);
  189. size_t OPENSSL_instrument_bus2(unsigned int *out, size_t cnt, size_t max)
  190. {
  191. if (OPENSSL_ppccap_P & PPC_MFTB)
  192. return OPENSSL_instrument_bus2_mftb(out, cnt, max);
  193. else if (OPENSSL_ppccap_P & PPC_MFSPR268)
  194. return OPENSSL_instrument_bus2_mfspr268(out, cnt, max);
  195. else
  196. return 0;
  197. }
  198. #if defined(__GLIBC__) && defined(__GLIBC_PREREQ)
  199. # if __GLIBC_PREREQ(2, 16)
  200. # include <sys/auxv.h>
  201. # define OSSL_IMPLEMENT_GETAUXVAL
  202. # endif
  203. #endif
  204. /* I wish <sys/auxv.h> was universally available */
  205. #define HWCAP 16 /* AT_HWCAP */
  206. #define HWCAP_PPC64 (1U << 30)
  207. #define HWCAP_ALTIVEC (1U << 28)
  208. #define HWCAP_FPU (1U << 27)
  209. #define HWCAP_POWER6_EXT (1U << 9)
  210. #define HWCAP_VSX (1U << 7)
  211. #define HWCAP2 26 /* AT_HWCAP2 */
  212. #define HWCAP_VEC_CRYPTO (1U << 25)
  213. #define HWCAP_ARCH_3_00 (1U << 23)
  214. # if defined(__GNUC__) && __GNUC__>=2
  215. __attribute__ ((constructor))
  216. # endif
  217. void OPENSSL_cpuid_setup(void)
  218. {
  219. char *e;
  220. struct sigaction ill_oact, ill_act;
  221. sigset_t oset;
  222. static int trigger = 0;
  223. if (trigger)
  224. return;
  225. trigger = 1;
  226. if ((e = getenv("OPENSSL_ppccap"))) {
  227. OPENSSL_ppccap_P = strtoul(e, NULL, 0);
  228. return;
  229. }
  230. OPENSSL_ppccap_P = 0;
  231. #if defined(_AIX)
  232. OPENSSL_ppccap_P |= PPC_FPU;
  233. if (sizeof(size_t) == 4) {
  234. struct utsname uts;
  235. # if defined(_SC_AIX_KERNEL_BITMODE)
  236. if (sysconf(_SC_AIX_KERNEL_BITMODE) != 64)
  237. return;
  238. # endif
  239. if (uname(&uts) != 0 || atoi(uts.version) < 6)
  240. return;
  241. }
  242. # if defined(__power_set)
  243. /*
  244. * Value used in __power_set is a single-bit 1<<n one denoting
  245. * specific processor class. Incidentally 0xffffffff<<n can be
  246. * used to denote specific processor and its successors.
  247. */
  248. if (sizeof(size_t) == 4) {
  249. /* In 32-bit case PPC_FPU64 is always fastest [if option] */
  250. if (__power_set(0xffffffffU<<13)) /* POWER5 and later */
  251. OPENSSL_ppccap_P |= PPC_FPU64;
  252. } else {
  253. /* In 64-bit case PPC_FPU64 is fastest only on POWER6 */
  254. if (__power_set(0x1U<<14)) /* POWER6 */
  255. OPENSSL_ppccap_P |= PPC_FPU64;
  256. }
  257. if (__power_set(0xffffffffU<<14)) /* POWER6 and later */
  258. OPENSSL_ppccap_P |= PPC_ALTIVEC;
  259. if (__power_set(0xffffffffU<<16)) /* POWER8 and later */
  260. OPENSSL_ppccap_P |= PPC_CRYPTO207;
  261. if (__power_set(0xffffffffU<<17)) /* POWER9 and later */
  262. OPENSSL_ppccap_P |= PPC_MADD300;
  263. return;
  264. # endif
  265. #endif
  266. #if defined(__APPLE__) && defined(__MACH__)
  267. OPENSSL_ppccap_P |= PPC_FPU;
  268. {
  269. int val;
  270. size_t len = sizeof(val);
  271. if (sysctlbyname("hw.optional.64bitops", &val, &len, NULL, 0) == 0) {
  272. if (val)
  273. OPENSSL_ppccap_P |= PPC_FPU64;
  274. }
  275. len = sizeof(val);
  276. if (sysctlbyname("hw.optional.altivec", &val, &len, NULL, 0) == 0) {
  277. if (val)
  278. OPENSSL_ppccap_P |= PPC_ALTIVEC;
  279. }
  280. return;
  281. }
  282. #endif
  283. #ifdef OSSL_IMPLEMENT_GETAUXVAL
  284. {
  285. unsigned long hwcap = getauxval(HWCAP);
  286. unsigned long hwcap2 = getauxval(HWCAP2);
  287. if (hwcap & HWCAP_FPU) {
  288. OPENSSL_ppccap_P |= PPC_FPU;
  289. if (sizeof(size_t) == 4) {
  290. /* In 32-bit case PPC_FPU64 is always fastest [if option] */
  291. if (hwcap & HWCAP_PPC64)
  292. OPENSSL_ppccap_P |= PPC_FPU64;
  293. } else {
  294. /* In 64-bit case PPC_FPU64 is fastest only on POWER6 */
  295. if (hwcap & HWCAP_POWER6_EXT)
  296. OPENSSL_ppccap_P |= PPC_FPU64;
  297. }
  298. }
  299. if (hwcap & HWCAP_ALTIVEC) {
  300. OPENSSL_ppccap_P |= PPC_ALTIVEC;
  301. if ((hwcap & HWCAP_VSX) && (hwcap2 & HWCAP_VEC_CRYPTO))
  302. OPENSSL_ppccap_P |= PPC_CRYPTO207;
  303. }
  304. if (hwcap2 & HWCAP_ARCH_3_00) {
  305. OPENSSL_ppccap_P |= PPC_MADD300;
  306. }
  307. }
  308. #endif
  309. sigfillset(&all_masked);
  310. sigdelset(&all_masked, SIGILL);
  311. sigdelset(&all_masked, SIGTRAP);
  312. #ifdef SIGEMT
  313. sigdelset(&all_masked, SIGEMT);
  314. #endif
  315. sigdelset(&all_masked, SIGFPE);
  316. sigdelset(&all_masked, SIGBUS);
  317. sigdelset(&all_masked, SIGSEGV);
  318. memset(&ill_act, 0, sizeof(ill_act));
  319. ill_act.sa_handler = ill_handler;
  320. ill_act.sa_mask = all_masked;
  321. sigprocmask(SIG_SETMASK, &ill_act.sa_mask, &oset);
  322. sigaction(SIGILL, &ill_act, &ill_oact);
  323. #ifndef OSSL_IMPLEMENT_GETAUXVAL
  324. if (sigsetjmp(ill_jmp,1) == 0) {
  325. OPENSSL_fpu_probe();
  326. OPENSSL_ppccap_P |= PPC_FPU;
  327. if (sizeof(size_t) == 4) {
  328. # ifdef __linux
  329. struct utsname uts;
  330. if (uname(&uts) == 0 && strcmp(uts.machine, "ppc64") == 0)
  331. # endif
  332. if (sigsetjmp(ill_jmp, 1) == 0) {
  333. OPENSSL_ppc64_probe();
  334. OPENSSL_ppccap_P |= PPC_FPU64;
  335. }
  336. } else {
  337. /*
  338. * Wanted code detecting POWER6 CPU and setting PPC_FPU64
  339. */
  340. }
  341. }
  342. if (sigsetjmp(ill_jmp, 1) == 0) {
  343. OPENSSL_altivec_probe();
  344. OPENSSL_ppccap_P |= PPC_ALTIVEC;
  345. if (sigsetjmp(ill_jmp, 1) == 0) {
  346. OPENSSL_crypto207_probe();
  347. OPENSSL_ppccap_P |= PPC_CRYPTO207;
  348. }
  349. }
  350. if (sigsetjmp(ill_jmp, 1) == 0) {
  351. OPENSSL_madd300_probe();
  352. OPENSSL_ppccap_P |= PPC_MADD300;
  353. }
  354. #endif
  355. if (sigsetjmp(ill_jmp, 1) == 0) {
  356. OPENSSL_rdtsc_mftb();
  357. OPENSSL_ppccap_P |= PPC_MFTB;
  358. } else if (sigsetjmp(ill_jmp, 1) == 0) {
  359. OPENSSL_rdtsc_mfspr268();
  360. OPENSSL_ppccap_P |= PPC_MFSPR268;
  361. }
  362. sigaction(SIGILL, &ill_oact, NULL);
  363. sigprocmask(SIG_SETMASK, &oset, NULL);
  364. }