x86_64-gcc.c 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642
  1. /*
  2. * Copyright 2002-2018 The OpenSSL Project Authors. All Rights Reserved.
  3. *
  4. * Licensed under the Apache License 2.0 (the "License"). You may not use
  5. * this file except in compliance with the License. You can obtain a copy
  6. * in the file LICENSE in the source distribution or at
  7. * https://www.openssl.org/source/license.html
  8. */
  9. #include "../bn_local.h"
  10. #if !(defined(__GNUC__) && __GNUC__>=2)
  11. # include "../bn_asm.c" /* kind of dirty hack for Sun Studio */
  12. #else
  13. /*-
  14. * x86_64 BIGNUM accelerator version 0.1, December 2002.
  15. *
  16. * Implemented by Andy Polyakov <appro@openssl.org> for the OpenSSL
  17. * project.
  18. *
  19. * Rights for redistribution and usage in source and binary forms are
  20. * granted according to the License. Warranty of any kind is disclaimed.
  21. *
  22. * Q. Version 0.1? It doesn't sound like Andy, he used to assign real
  23. * versions, like 1.0...
  24. * A. Well, that's because this code is basically a quick-n-dirty
  25. * proof-of-concept hack. As you can see it's implemented with
  26. * inline assembler, which means that you're bound to GCC and that
  27. * there might be enough room for further improvement.
  28. *
  29. * Q. Why inline assembler?
  30. * A. x86_64 features own ABI which I'm not familiar with. This is
  31. * why I decided to let the compiler take care of subroutine
  32. * prologue/epilogue as well as register allocation. For reference.
  33. * Win64 implements different ABI for AMD64, different from Linux.
  34. *
  35. * Q. How much faster does it get?
  36. * A. 'apps/openssl speed rsa dsa' output with no-asm:
  37. *
  38. * sign verify sign/s verify/s
  39. * rsa 512 bits 0.0006s 0.0001s 1683.8 18456.2
  40. * rsa 1024 bits 0.0028s 0.0002s 356.0 6407.0
  41. * rsa 2048 bits 0.0172s 0.0005s 58.0 1957.8
  42. * rsa 4096 bits 0.1155s 0.0018s 8.7 555.6
  43. * sign verify sign/s verify/s
  44. * dsa 512 bits 0.0005s 0.0006s 2100.8 1768.3
  45. * dsa 1024 bits 0.0014s 0.0018s 692.3 559.2
  46. * dsa 2048 bits 0.0049s 0.0061s 204.7 165.0
  47. *
  48. * 'apps/openssl speed rsa dsa' output with this module:
  49. *
  50. * sign verify sign/s verify/s
  51. * rsa 512 bits 0.0004s 0.0000s 2767.1 33297.9
  52. * rsa 1024 bits 0.0012s 0.0001s 867.4 14674.7
  53. * rsa 2048 bits 0.0061s 0.0002s 164.0 5270.0
  54. * rsa 4096 bits 0.0384s 0.0006s 26.1 1650.8
  55. * sign verify sign/s verify/s
  56. * dsa 512 bits 0.0002s 0.0003s 4442.2 3786.3
  57. * dsa 1024 bits 0.0005s 0.0007s 1835.1 1497.4
  58. * dsa 2048 bits 0.0016s 0.0020s 620.4 504.6
  59. *
  60. * For the reference. IA-32 assembler implementation performs
  61. * very much like 64-bit code compiled with no-asm on the same
  62. * machine.
  63. */
  64. # undef mul
  65. # undef mul_add
  66. /*-
  67. * "m"(a), "+m"(r) is the way to favor DirectPath µ-code;
  68. * "g"(0) let the compiler to decide where does it
  69. * want to keep the value of zero;
  70. */
  71. # define mul_add(r,a,word,carry) do { \
  72. register BN_ULONG high,low; \
  73. asm ("mulq %3" \
  74. : "=a"(low),"=d"(high) \
  75. : "a"(word),"m"(a) \
  76. : "cc"); \
  77. asm ("addq %2,%0; adcq %3,%1" \
  78. : "+r"(carry),"+d"(high)\
  79. : "a"(low),"g"(0) \
  80. : "cc"); \
  81. asm ("addq %2,%0; adcq %3,%1" \
  82. : "+m"(r),"+d"(high) \
  83. : "r"(carry),"g"(0) \
  84. : "cc"); \
  85. carry=high; \
  86. } while (0)
  87. # define mul(r,a,word,carry) do { \
  88. register BN_ULONG high,low; \
  89. asm ("mulq %3" \
  90. : "=a"(low),"=d"(high) \
  91. : "a"(word),"g"(a) \
  92. : "cc"); \
  93. asm ("addq %2,%0; adcq %3,%1" \
  94. : "+r"(carry),"+d"(high)\
  95. : "a"(low),"g"(0) \
  96. : "cc"); \
  97. (r)=carry, carry=high; \
  98. } while (0)
  99. # undef sqr
  100. # define sqr(r0,r1,a) \
  101. asm ("mulq %2" \
  102. : "=a"(r0),"=d"(r1) \
  103. : "a"(a) \
  104. : "cc");
  105. BN_ULONG bn_mul_add_words(BN_ULONG *rp, const BN_ULONG *ap, int num,
  106. BN_ULONG w)
  107. {
  108. BN_ULONG c1 = 0;
  109. if (num <= 0)
  110. return c1;
  111. while (num & ~3) {
  112. mul_add(rp[0], ap[0], w, c1);
  113. mul_add(rp[1], ap[1], w, c1);
  114. mul_add(rp[2], ap[2], w, c1);
  115. mul_add(rp[3], ap[3], w, c1);
  116. ap += 4;
  117. rp += 4;
  118. num -= 4;
  119. }
  120. if (num) {
  121. mul_add(rp[0], ap[0], w, c1);
  122. if (--num == 0)
  123. return c1;
  124. mul_add(rp[1], ap[1], w, c1);
  125. if (--num == 0)
  126. return c1;
  127. mul_add(rp[2], ap[2], w, c1);
  128. return c1;
  129. }
  130. return c1;
  131. }
  132. BN_ULONG bn_mul_words(BN_ULONG *rp, const BN_ULONG *ap, int num, BN_ULONG w)
  133. {
  134. BN_ULONG c1 = 0;
  135. if (num <= 0)
  136. return c1;
  137. while (num & ~3) {
  138. mul(rp[0], ap[0], w, c1);
  139. mul(rp[1], ap[1], w, c1);
  140. mul(rp[2], ap[2], w, c1);
  141. mul(rp[3], ap[3], w, c1);
  142. ap += 4;
  143. rp += 4;
  144. num -= 4;
  145. }
  146. if (num) {
  147. mul(rp[0], ap[0], w, c1);
  148. if (--num == 0)
  149. return c1;
  150. mul(rp[1], ap[1], w, c1);
  151. if (--num == 0)
  152. return c1;
  153. mul(rp[2], ap[2], w, c1);
  154. }
  155. return c1;
  156. }
  157. void bn_sqr_words(BN_ULONG *r, const BN_ULONG *a, int n)
  158. {
  159. if (n <= 0)
  160. return;
  161. while (n & ~3) {
  162. sqr(r[0], r[1], a[0]);
  163. sqr(r[2], r[3], a[1]);
  164. sqr(r[4], r[5], a[2]);
  165. sqr(r[6], r[7], a[3]);
  166. a += 4;
  167. r += 8;
  168. n -= 4;
  169. }
  170. if (n) {
  171. sqr(r[0], r[1], a[0]);
  172. if (--n == 0)
  173. return;
  174. sqr(r[2], r[3], a[1]);
  175. if (--n == 0)
  176. return;
  177. sqr(r[4], r[5], a[2]);
  178. }
  179. }
  180. BN_ULONG bn_div_words(BN_ULONG h, BN_ULONG l, BN_ULONG d)
  181. {
  182. BN_ULONG ret, waste;
  183. asm("divq %4":"=a"(ret), "=d"(waste)
  184. : "a"(l), "d"(h), "r"(d)
  185. : "cc");
  186. return ret;
  187. }
  188. BN_ULONG bn_add_words(BN_ULONG *rp, const BN_ULONG *ap, const BN_ULONG *bp,
  189. int n)
  190. {
  191. BN_ULONG ret;
  192. size_t i = 0;
  193. if (n <= 0)
  194. return 0;
  195. asm volatile (" subq %0,%0 \n" /* clear carry */
  196. " jmp 1f \n"
  197. ".p2align 4 \n"
  198. "1: movq (%4,%2,8),%0 \n"
  199. " adcq (%5,%2,8),%0 \n"
  200. " movq %0,(%3,%2,8) \n"
  201. " lea 1(%2),%2 \n"
  202. " dec %1 \n"
  203. " jnz 1b \n"
  204. " sbbq %0,%0 \n"
  205. :"=&r" (ret), "+c"(n), "+r"(i)
  206. :"r"(rp), "r"(ap), "r"(bp)
  207. :"cc", "memory");
  208. return ret & 1;
  209. }
  210. # ifndef SIMICS
  211. BN_ULONG bn_sub_words(BN_ULONG *rp, const BN_ULONG *ap, const BN_ULONG *bp,
  212. int n)
  213. {
  214. BN_ULONG ret;
  215. size_t i = 0;
  216. if (n <= 0)
  217. return 0;
  218. asm volatile (" subq %0,%0 \n" /* clear borrow */
  219. " jmp 1f \n"
  220. ".p2align 4 \n"
  221. "1: movq (%4,%2,8),%0 \n"
  222. " sbbq (%5,%2,8),%0 \n"
  223. " movq %0,(%3,%2,8) \n"
  224. " lea 1(%2),%2 \n"
  225. " dec %1 \n"
  226. " jnz 1b \n"
  227. " sbbq %0,%0 \n"
  228. :"=&r" (ret), "+c"(n), "+r"(i)
  229. :"r"(rp), "r"(ap), "r"(bp)
  230. :"cc", "memory");
  231. return ret & 1;
  232. }
  233. # else
  234. /* Simics 1.4<7 has buggy sbbq:-( */
  235. # define BN_MASK2 0xffffffffffffffffL
  236. BN_ULONG bn_sub_words(BN_ULONG *r, BN_ULONG *a, BN_ULONG *b, int n)
  237. {
  238. BN_ULONG t1, t2;
  239. int c = 0;
  240. if (n <= 0)
  241. return (BN_ULONG)0;
  242. for (;;) {
  243. t1 = a[0];
  244. t2 = b[0];
  245. r[0] = (t1 - t2 - c) & BN_MASK2;
  246. if (t1 != t2)
  247. c = (t1 < t2);
  248. if (--n <= 0)
  249. break;
  250. t1 = a[1];
  251. t2 = b[1];
  252. r[1] = (t1 - t2 - c) & BN_MASK2;
  253. if (t1 != t2)
  254. c = (t1 < t2);
  255. if (--n <= 0)
  256. break;
  257. t1 = a[2];
  258. t2 = b[2];
  259. r[2] = (t1 - t2 - c) & BN_MASK2;
  260. if (t1 != t2)
  261. c = (t1 < t2);
  262. if (--n <= 0)
  263. break;
  264. t1 = a[3];
  265. t2 = b[3];
  266. r[3] = (t1 - t2 - c) & BN_MASK2;
  267. if (t1 != t2)
  268. c = (t1 < t2);
  269. if (--n <= 0)
  270. break;
  271. a += 4;
  272. b += 4;
  273. r += 4;
  274. }
  275. return c;
  276. }
  277. # endif
  278. /* mul_add_c(a,b,c0,c1,c2) -- c+=a*b for three word number c=(c2,c1,c0) */
  279. /* mul_add_c2(a,b,c0,c1,c2) -- c+=2*a*b for three word number c=(c2,c1,c0) */
  280. /* sqr_add_c(a,i,c0,c1,c2) -- c+=a[i]^2 for three word number c=(c2,c1,c0) */
  281. /*
  282. * sqr_add_c2(a,i,c0,c1,c2) -- c+=2*a[i]*a[j] for three word number
  283. * c=(c2,c1,c0)
  284. */
  285. /*
  286. * Keep in mind that carrying into high part of multiplication result
  287. * can not overflow, because it cannot be all-ones.
  288. */
  289. # if 0
  290. /* original macros are kept for reference purposes */
  291. # define mul_add_c(a,b,c0,c1,c2) do { \
  292. BN_ULONG ta = (a), tb = (b); \
  293. BN_ULONG lo, hi; \
  294. BN_UMULT_LOHI(lo,hi,ta,tb); \
  295. c0 += lo; hi += (c0<lo)?1:0; \
  296. c1 += hi; c2 += (c1<hi)?1:0; \
  297. } while(0)
  298. # define mul_add_c2(a,b,c0,c1,c2) do { \
  299. BN_ULONG ta = (a), tb = (b); \
  300. BN_ULONG lo, hi, tt; \
  301. BN_UMULT_LOHI(lo,hi,ta,tb); \
  302. c0 += lo; tt = hi+((c0<lo)?1:0); \
  303. c1 += tt; c2 += (c1<tt)?1:0; \
  304. c0 += lo; hi += (c0<lo)?1:0; \
  305. c1 += hi; c2 += (c1<hi)?1:0; \
  306. } while(0)
  307. # define sqr_add_c(a,i,c0,c1,c2) do { \
  308. BN_ULONG ta = (a)[i]; \
  309. BN_ULONG lo, hi; \
  310. BN_UMULT_LOHI(lo,hi,ta,ta); \
  311. c0 += lo; hi += (c0<lo)?1:0; \
  312. c1 += hi; c2 += (c1<hi)?1:0; \
  313. } while(0)
  314. # else
  315. # define mul_add_c(a,b,c0,c1,c2) do { \
  316. BN_ULONG t1,t2; \
  317. asm ("mulq %3" \
  318. : "=a"(t1),"=d"(t2) \
  319. : "a"(a),"m"(b) \
  320. : "cc"); \
  321. asm ("addq %3,%0; adcq %4,%1; adcq %5,%2" \
  322. : "+r"(c0),"+r"(c1),"+r"(c2) \
  323. : "r"(t1),"r"(t2),"g"(0) \
  324. : "cc"); \
  325. } while (0)
  326. # define sqr_add_c(a,i,c0,c1,c2) do { \
  327. BN_ULONG t1,t2; \
  328. asm ("mulq %2" \
  329. : "=a"(t1),"=d"(t2) \
  330. : "a"(a[i]) \
  331. : "cc"); \
  332. asm ("addq %3,%0; adcq %4,%1; adcq %5,%2" \
  333. : "+r"(c0),"+r"(c1),"+r"(c2) \
  334. : "r"(t1),"r"(t2),"g"(0) \
  335. : "cc"); \
  336. } while (0)
  337. # define mul_add_c2(a,b,c0,c1,c2) do { \
  338. BN_ULONG t1,t2; \
  339. asm ("mulq %3" \
  340. : "=a"(t1),"=d"(t2) \
  341. : "a"(a),"m"(b) \
  342. : "cc"); \
  343. asm ("addq %3,%0; adcq %4,%1; adcq %5,%2" \
  344. : "+r"(c0),"+r"(c1),"+r"(c2) \
  345. : "r"(t1),"r"(t2),"g"(0) \
  346. : "cc"); \
  347. asm ("addq %3,%0; adcq %4,%1; adcq %5,%2" \
  348. : "+r"(c0),"+r"(c1),"+r"(c2) \
  349. : "r"(t1),"r"(t2),"g"(0) \
  350. : "cc"); \
  351. } while (0)
  352. # endif
  353. # define sqr_add_c2(a,i,j,c0,c1,c2) \
  354. mul_add_c2((a)[i],(a)[j],c0,c1,c2)
  355. void bn_mul_comba8(BN_ULONG *r, BN_ULONG *a, BN_ULONG *b)
  356. {
  357. BN_ULONG c1, c2, c3;
  358. c1 = 0;
  359. c2 = 0;
  360. c3 = 0;
  361. mul_add_c(a[0], b[0], c1, c2, c3);
  362. r[0] = c1;
  363. c1 = 0;
  364. mul_add_c(a[0], b[1], c2, c3, c1);
  365. mul_add_c(a[1], b[0], c2, c3, c1);
  366. r[1] = c2;
  367. c2 = 0;
  368. mul_add_c(a[2], b[0], c3, c1, c2);
  369. mul_add_c(a[1], b[1], c3, c1, c2);
  370. mul_add_c(a[0], b[2], c3, c1, c2);
  371. r[2] = c3;
  372. c3 = 0;
  373. mul_add_c(a[0], b[3], c1, c2, c3);
  374. mul_add_c(a[1], b[2], c1, c2, c3);
  375. mul_add_c(a[2], b[1], c1, c2, c3);
  376. mul_add_c(a[3], b[0], c1, c2, c3);
  377. r[3] = c1;
  378. c1 = 0;
  379. mul_add_c(a[4], b[0], c2, c3, c1);
  380. mul_add_c(a[3], b[1], c2, c3, c1);
  381. mul_add_c(a[2], b[2], c2, c3, c1);
  382. mul_add_c(a[1], b[3], c2, c3, c1);
  383. mul_add_c(a[0], b[4], c2, c3, c1);
  384. r[4] = c2;
  385. c2 = 0;
  386. mul_add_c(a[0], b[5], c3, c1, c2);
  387. mul_add_c(a[1], b[4], c3, c1, c2);
  388. mul_add_c(a[2], b[3], c3, c1, c2);
  389. mul_add_c(a[3], b[2], c3, c1, c2);
  390. mul_add_c(a[4], b[1], c3, c1, c2);
  391. mul_add_c(a[5], b[0], c3, c1, c2);
  392. r[5] = c3;
  393. c3 = 0;
  394. mul_add_c(a[6], b[0], c1, c2, c3);
  395. mul_add_c(a[5], b[1], c1, c2, c3);
  396. mul_add_c(a[4], b[2], c1, c2, c3);
  397. mul_add_c(a[3], b[3], c1, c2, c3);
  398. mul_add_c(a[2], b[4], c1, c2, c3);
  399. mul_add_c(a[1], b[5], c1, c2, c3);
  400. mul_add_c(a[0], b[6], c1, c2, c3);
  401. r[6] = c1;
  402. c1 = 0;
  403. mul_add_c(a[0], b[7], c2, c3, c1);
  404. mul_add_c(a[1], b[6], c2, c3, c1);
  405. mul_add_c(a[2], b[5], c2, c3, c1);
  406. mul_add_c(a[3], b[4], c2, c3, c1);
  407. mul_add_c(a[4], b[3], c2, c3, c1);
  408. mul_add_c(a[5], b[2], c2, c3, c1);
  409. mul_add_c(a[6], b[1], c2, c3, c1);
  410. mul_add_c(a[7], b[0], c2, c3, c1);
  411. r[7] = c2;
  412. c2 = 0;
  413. mul_add_c(a[7], b[1], c3, c1, c2);
  414. mul_add_c(a[6], b[2], c3, c1, c2);
  415. mul_add_c(a[5], b[3], c3, c1, c2);
  416. mul_add_c(a[4], b[4], c3, c1, c2);
  417. mul_add_c(a[3], b[5], c3, c1, c2);
  418. mul_add_c(a[2], b[6], c3, c1, c2);
  419. mul_add_c(a[1], b[7], c3, c1, c2);
  420. r[8] = c3;
  421. c3 = 0;
  422. mul_add_c(a[2], b[7], c1, c2, c3);
  423. mul_add_c(a[3], b[6], c1, c2, c3);
  424. mul_add_c(a[4], b[5], c1, c2, c3);
  425. mul_add_c(a[5], b[4], c1, c2, c3);
  426. mul_add_c(a[6], b[3], c1, c2, c3);
  427. mul_add_c(a[7], b[2], c1, c2, c3);
  428. r[9] = c1;
  429. c1 = 0;
  430. mul_add_c(a[7], b[3], c2, c3, c1);
  431. mul_add_c(a[6], b[4], c2, c3, c1);
  432. mul_add_c(a[5], b[5], c2, c3, c1);
  433. mul_add_c(a[4], b[6], c2, c3, c1);
  434. mul_add_c(a[3], b[7], c2, c3, c1);
  435. r[10] = c2;
  436. c2 = 0;
  437. mul_add_c(a[4], b[7], c3, c1, c2);
  438. mul_add_c(a[5], b[6], c3, c1, c2);
  439. mul_add_c(a[6], b[5], c3, c1, c2);
  440. mul_add_c(a[7], b[4], c3, c1, c2);
  441. r[11] = c3;
  442. c3 = 0;
  443. mul_add_c(a[7], b[5], c1, c2, c3);
  444. mul_add_c(a[6], b[6], c1, c2, c3);
  445. mul_add_c(a[5], b[7], c1, c2, c3);
  446. r[12] = c1;
  447. c1 = 0;
  448. mul_add_c(a[6], b[7], c2, c3, c1);
  449. mul_add_c(a[7], b[6], c2, c3, c1);
  450. r[13] = c2;
  451. c2 = 0;
  452. mul_add_c(a[7], b[7], c3, c1, c2);
  453. r[14] = c3;
  454. r[15] = c1;
  455. }
  456. void bn_mul_comba4(BN_ULONG *r, BN_ULONG *a, BN_ULONG *b)
  457. {
  458. BN_ULONG c1, c2, c3;
  459. c1 = 0;
  460. c2 = 0;
  461. c3 = 0;
  462. mul_add_c(a[0], b[0], c1, c2, c3);
  463. r[0] = c1;
  464. c1 = 0;
  465. mul_add_c(a[0], b[1], c2, c3, c1);
  466. mul_add_c(a[1], b[0], c2, c3, c1);
  467. r[1] = c2;
  468. c2 = 0;
  469. mul_add_c(a[2], b[0], c3, c1, c2);
  470. mul_add_c(a[1], b[1], c3, c1, c2);
  471. mul_add_c(a[0], b[2], c3, c1, c2);
  472. r[2] = c3;
  473. c3 = 0;
  474. mul_add_c(a[0], b[3], c1, c2, c3);
  475. mul_add_c(a[1], b[2], c1, c2, c3);
  476. mul_add_c(a[2], b[1], c1, c2, c3);
  477. mul_add_c(a[3], b[0], c1, c2, c3);
  478. r[3] = c1;
  479. c1 = 0;
  480. mul_add_c(a[3], b[1], c2, c3, c1);
  481. mul_add_c(a[2], b[2], c2, c3, c1);
  482. mul_add_c(a[1], b[3], c2, c3, c1);
  483. r[4] = c2;
  484. c2 = 0;
  485. mul_add_c(a[2], b[3], c3, c1, c2);
  486. mul_add_c(a[3], b[2], c3, c1, c2);
  487. r[5] = c3;
  488. c3 = 0;
  489. mul_add_c(a[3], b[3], c1, c2, c3);
  490. r[6] = c1;
  491. r[7] = c2;
  492. }
  493. void bn_sqr_comba8(BN_ULONG *r, const BN_ULONG *a)
  494. {
  495. BN_ULONG c1, c2, c3;
  496. c1 = 0;
  497. c2 = 0;
  498. c3 = 0;
  499. sqr_add_c(a, 0, c1, c2, c3);
  500. r[0] = c1;
  501. c1 = 0;
  502. sqr_add_c2(a, 1, 0, c2, c3, c1);
  503. r[1] = c2;
  504. c2 = 0;
  505. sqr_add_c(a, 1, c3, c1, c2);
  506. sqr_add_c2(a, 2, 0, c3, c1, c2);
  507. r[2] = c3;
  508. c3 = 0;
  509. sqr_add_c2(a, 3, 0, c1, c2, c3);
  510. sqr_add_c2(a, 2, 1, c1, c2, c3);
  511. r[3] = c1;
  512. c1 = 0;
  513. sqr_add_c(a, 2, c2, c3, c1);
  514. sqr_add_c2(a, 3, 1, c2, c3, c1);
  515. sqr_add_c2(a, 4, 0, c2, c3, c1);
  516. r[4] = c2;
  517. c2 = 0;
  518. sqr_add_c2(a, 5, 0, c3, c1, c2);
  519. sqr_add_c2(a, 4, 1, c3, c1, c2);
  520. sqr_add_c2(a, 3, 2, c3, c1, c2);
  521. r[5] = c3;
  522. c3 = 0;
  523. sqr_add_c(a, 3, c1, c2, c3);
  524. sqr_add_c2(a, 4, 2, c1, c2, c3);
  525. sqr_add_c2(a, 5, 1, c1, c2, c3);
  526. sqr_add_c2(a, 6, 0, c1, c2, c3);
  527. r[6] = c1;
  528. c1 = 0;
  529. sqr_add_c2(a, 7, 0, c2, c3, c1);
  530. sqr_add_c2(a, 6, 1, c2, c3, c1);
  531. sqr_add_c2(a, 5, 2, c2, c3, c1);
  532. sqr_add_c2(a, 4, 3, c2, c3, c1);
  533. r[7] = c2;
  534. c2 = 0;
  535. sqr_add_c(a, 4, c3, c1, c2);
  536. sqr_add_c2(a, 5, 3, c3, c1, c2);
  537. sqr_add_c2(a, 6, 2, c3, c1, c2);
  538. sqr_add_c2(a, 7, 1, c3, c1, c2);
  539. r[8] = c3;
  540. c3 = 0;
  541. sqr_add_c2(a, 7, 2, c1, c2, c3);
  542. sqr_add_c2(a, 6, 3, c1, c2, c3);
  543. sqr_add_c2(a, 5, 4, c1, c2, c3);
  544. r[9] = c1;
  545. c1 = 0;
  546. sqr_add_c(a, 5, c2, c3, c1);
  547. sqr_add_c2(a, 6, 4, c2, c3, c1);
  548. sqr_add_c2(a, 7, 3, c2, c3, c1);
  549. r[10] = c2;
  550. c2 = 0;
  551. sqr_add_c2(a, 7, 4, c3, c1, c2);
  552. sqr_add_c2(a, 6, 5, c3, c1, c2);
  553. r[11] = c3;
  554. c3 = 0;
  555. sqr_add_c(a, 6, c1, c2, c3);
  556. sqr_add_c2(a, 7, 5, c1, c2, c3);
  557. r[12] = c1;
  558. c1 = 0;
  559. sqr_add_c2(a, 7, 6, c2, c3, c1);
  560. r[13] = c2;
  561. c2 = 0;
  562. sqr_add_c(a, 7, c3, c1, c2);
  563. r[14] = c3;
  564. r[15] = c1;
  565. }
  566. void bn_sqr_comba4(BN_ULONG *r, const BN_ULONG *a)
  567. {
  568. BN_ULONG c1, c2, c3;
  569. c1 = 0;
  570. c2 = 0;
  571. c3 = 0;
  572. sqr_add_c(a, 0, c1, c2, c3);
  573. r[0] = c1;
  574. c1 = 0;
  575. sqr_add_c2(a, 1, 0, c2, c3, c1);
  576. r[1] = c2;
  577. c2 = 0;
  578. sqr_add_c(a, 1, c3, c1, c2);
  579. sqr_add_c2(a, 2, 0, c3, c1, c2);
  580. r[2] = c3;
  581. c3 = 0;
  582. sqr_add_c2(a, 3, 0, c1, c2, c3);
  583. sqr_add_c2(a, 2, 1, c1, c2, c3);
  584. r[3] = c1;
  585. c1 = 0;
  586. sqr_add_c(a, 2, c2, c3, c1);
  587. sqr_add_c2(a, 3, 1, c2, c3, c1);
  588. r[4] = c2;
  589. c2 = 0;
  590. sqr_add_c2(a, 3, 2, c3, c1, c2);
  591. r[5] = c3;
  592. c3 = 0;
  593. sqr_add_c(a, 3, c1, c2, c3);
  594. r[6] = c1;
  595. r[7] = c2;
  596. }
  597. #endif