x86_64-mont.pl 33 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594
  1. #! /usr/bin/env perl
  2. # Copyright 2005-2020 The OpenSSL Project Authors. All Rights Reserved.
  3. #
  4. # Licensed under the Apache License 2.0 (the "License"). You may not use
  5. # this file except in compliance with the License. You can obtain a copy
  6. # in the file LICENSE in the source distribution or at
  7. # https://www.openssl.org/source/license.html
  8. # ====================================================================
  9. # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
  10. # project. The module is, however, dual licensed under OpenSSL and
  11. # CRYPTOGAMS licenses depending on where you obtain it. For further
  12. # details see http://www.openssl.org/~appro/cryptogams/.
  13. # ====================================================================
  14. # October 2005.
  15. #
  16. # Montgomery multiplication routine for x86_64. While it gives modest
  17. # 9% improvement of rsa4096 sign on Opteron, rsa512 sign runs more
  18. # than twice, >2x, as fast. Most common rsa1024 sign is improved by
  19. # respectful 50%. It remains to be seen if loop unrolling and
  20. # dedicated squaring routine can provide further improvement...
  21. # July 2011.
  22. #
  23. # Add dedicated squaring procedure. Performance improvement varies
  24. # from platform to platform, but in average it's ~5%/15%/25%/33%
  25. # for 512-/1024-/2048-/4096-bit RSA *sign* benchmarks respectively.
  26. # August 2011.
  27. #
  28. # Unroll and modulo-schedule inner loops in such manner that they
  29. # are "fallen through" for input lengths of 8, which is critical for
  30. # 1024-bit RSA *sign*. Average performance improvement in comparison
  31. # to *initial* version of this module from 2005 is ~0%/30%/40%/45%
  32. # for 512-/1024-/2048-/4096-bit RSA *sign* benchmarks respectively.
  33. # June 2013.
  34. #
  35. # Optimize reduction in squaring procedure and improve 1024+-bit RSA
  36. # sign performance by 10-16% on Intel Sandy Bridge and later
  37. # (virtually same on non-Intel processors).
  38. # August 2013.
  39. #
  40. # Add MULX/ADOX/ADCX code path.
  41. # $output is the last argument if it looks like a file (it has an extension)
  42. # $flavour is the first argument if it doesn't look like a file
  43. $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
  44. $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
  45. $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
  46. $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
  47. ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
  48. ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
  49. die "can't locate x86_64-xlate.pl";
  50. open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\""
  51. or die "can't call $xlate: $!";
  52. *STDOUT=*OUT;
  53. if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
  54. =~ /GNU assembler version ([2-9]\.[0-9]+)/) {
  55. $addx = ($1>=2.23);
  56. }
  57. if (!$addx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) &&
  58. `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) {
  59. $addx = ($1>=2.10);
  60. }
  61. if (!$addx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) &&
  62. `ml64 2>&1` =~ /Version ([0-9]+)\./) {
  63. $addx = ($1>=12);
  64. }
  65. if (!$addx && `$ENV{CC} -v 2>&1` =~ /((?:clang|LLVM) version|.*based on LLVM) ([0-9]+)\.([0-9]+)/) {
  66. my $ver = $2 + $3/100.0; # 3.1->3.01, 3.10->3.10
  67. $addx = ($ver>=3.03);
  68. }
  69. # int bn_mul_mont(
  70. $rp="%rdi"; # BN_ULONG *rp,
  71. $ap="%rsi"; # const BN_ULONG *ap,
  72. $bp="%rdx"; # const BN_ULONG *bp,
  73. $np="%rcx"; # const BN_ULONG *np,
  74. $n0="%r8"; # const BN_ULONG *n0,
  75. $num="%r9"; # int num);
  76. $lo0="%r10";
  77. $hi0="%r11";
  78. $hi1="%r13";
  79. $i="%r14";
  80. $j="%r15";
  81. $m0="%rbx";
  82. $m1="%rbp";
  83. $code=<<___;
  84. .text
  85. .extern OPENSSL_ia32cap_P
  86. .globl bn_mul_mont
  87. .type bn_mul_mont,\@function,6
  88. .align 16
  89. bn_mul_mont:
  90. .cfi_startproc
  91. mov ${num}d,${num}d
  92. mov %rsp,%rax
  93. .cfi_def_cfa_register %rax
  94. test \$3,${num}d
  95. jnz .Lmul_enter
  96. cmp \$8,${num}d
  97. jb .Lmul_enter
  98. ___
  99. $code.=<<___ if ($addx);
  100. mov OPENSSL_ia32cap_P+8(%rip),%r11d
  101. ___
  102. $code.=<<___;
  103. cmp $ap,$bp
  104. jne .Lmul4x_enter
  105. test \$7,${num}d
  106. jz .Lsqr8x_enter
  107. jmp .Lmul4x_enter
  108. .align 16
  109. .Lmul_enter:
  110. push %rbx
  111. .cfi_push %rbx
  112. push %rbp
  113. .cfi_push %rbp
  114. push %r12
  115. .cfi_push %r12
  116. push %r13
  117. .cfi_push %r13
  118. push %r14
  119. .cfi_push %r14
  120. push %r15
  121. .cfi_push %r15
  122. neg $num
  123. mov %rsp,%r11
  124. lea -16(%rsp,$num,8),%r10 # future alloca(8*(num+2))
  125. neg $num # restore $num
  126. and \$-1024,%r10 # minimize TLB usage
  127. # An OS-agnostic version of __chkstk.
  128. #
  129. # Some OSes (Windows) insist on stack being "wired" to
  130. # physical memory in strictly sequential manner, i.e. if stack
  131. # allocation spans two pages, then reference to farmost one can
  132. # be punishable by SEGV. But page walking can do good even on
  133. # other OSes, because it guarantees that villain thread hits
  134. # the guard page before it can make damage to innocent one...
  135. sub %r10,%r11
  136. and \$-4096,%r11
  137. lea (%r10,%r11),%rsp
  138. mov (%rsp),%r11
  139. cmp %r10,%rsp
  140. ja .Lmul_page_walk
  141. jmp .Lmul_page_walk_done
  142. .align 16
  143. .Lmul_page_walk:
  144. lea -4096(%rsp),%rsp
  145. mov (%rsp),%r11
  146. cmp %r10,%rsp
  147. ja .Lmul_page_walk
  148. .Lmul_page_walk_done:
  149. mov %rax,8(%rsp,$num,8) # tp[num+1]=%rsp
  150. .cfi_cfa_expression %rsp+8,$num,8,mul,plus,deref,+8
  151. .Lmul_body:
  152. mov $bp,%r12 # reassign $bp
  153. ___
  154. $bp="%r12";
  155. $code.=<<___;
  156. mov ($n0),$n0 # pull n0[0] value
  157. mov ($bp),$m0 # m0=bp[0]
  158. mov ($ap),%rax
  159. xor $i,$i # i=0
  160. xor $j,$j # j=0
  161. mov $n0,$m1
  162. mulq $m0 # ap[0]*bp[0]
  163. mov %rax,$lo0
  164. mov ($np),%rax
  165. imulq $lo0,$m1 # "tp[0]"*n0
  166. mov %rdx,$hi0
  167. mulq $m1 # np[0]*m1
  168. add %rax,$lo0 # discarded
  169. mov 8($ap),%rax
  170. adc \$0,%rdx
  171. mov %rdx,$hi1
  172. lea 1($j),$j # j++
  173. jmp .L1st_enter
  174. .align 16
  175. .L1st:
  176. add %rax,$hi1
  177. mov ($ap,$j,8),%rax
  178. adc \$0,%rdx
  179. add $hi0,$hi1 # np[j]*m1+ap[j]*bp[0]
  180. mov $lo0,$hi0
  181. adc \$0,%rdx
  182. mov $hi1,-16(%rsp,$j,8) # tp[j-1]
  183. mov %rdx,$hi1
  184. .L1st_enter:
  185. mulq $m0 # ap[j]*bp[0]
  186. add %rax,$hi0
  187. mov ($np,$j,8),%rax
  188. adc \$0,%rdx
  189. lea 1($j),$j # j++
  190. mov %rdx,$lo0
  191. mulq $m1 # np[j]*m1
  192. cmp $num,$j
  193. jne .L1st
  194. add %rax,$hi1
  195. mov ($ap),%rax # ap[0]
  196. adc \$0,%rdx
  197. add $hi0,$hi1 # np[j]*m1+ap[j]*bp[0]
  198. adc \$0,%rdx
  199. mov $hi1,-16(%rsp,$j,8) # tp[j-1]
  200. mov %rdx,$hi1
  201. mov $lo0,$hi0
  202. xor %rdx,%rdx
  203. add $hi0,$hi1
  204. adc \$0,%rdx
  205. mov $hi1,-8(%rsp,$num,8)
  206. mov %rdx,(%rsp,$num,8) # store upmost overflow bit
  207. lea 1($i),$i # i++
  208. jmp .Louter
  209. .align 16
  210. .Louter:
  211. mov ($bp,$i,8),$m0 # m0=bp[i]
  212. xor $j,$j # j=0
  213. mov $n0,$m1
  214. mov (%rsp),$lo0
  215. mulq $m0 # ap[0]*bp[i]
  216. add %rax,$lo0 # ap[0]*bp[i]+tp[0]
  217. mov ($np),%rax
  218. adc \$0,%rdx
  219. imulq $lo0,$m1 # tp[0]*n0
  220. mov %rdx,$hi0
  221. mulq $m1 # np[0]*m1
  222. add %rax,$lo0 # discarded
  223. mov 8($ap),%rax
  224. adc \$0,%rdx
  225. mov 8(%rsp),$lo0 # tp[1]
  226. mov %rdx,$hi1
  227. lea 1($j),$j # j++
  228. jmp .Linner_enter
  229. .align 16
  230. .Linner:
  231. add %rax,$hi1
  232. mov ($ap,$j,8),%rax
  233. adc \$0,%rdx
  234. add $lo0,$hi1 # np[j]*m1+ap[j]*bp[i]+tp[j]
  235. mov (%rsp,$j,8),$lo0
  236. adc \$0,%rdx
  237. mov $hi1,-16(%rsp,$j,8) # tp[j-1]
  238. mov %rdx,$hi1
  239. .Linner_enter:
  240. mulq $m0 # ap[j]*bp[i]
  241. add %rax,$hi0
  242. mov ($np,$j,8),%rax
  243. adc \$0,%rdx
  244. add $hi0,$lo0 # ap[j]*bp[i]+tp[j]
  245. mov %rdx,$hi0
  246. adc \$0,$hi0
  247. lea 1($j),$j # j++
  248. mulq $m1 # np[j]*m1
  249. cmp $num,$j
  250. jne .Linner
  251. add %rax,$hi1
  252. mov ($ap),%rax # ap[0]
  253. adc \$0,%rdx
  254. add $lo0,$hi1 # np[j]*m1+ap[j]*bp[i]+tp[j]
  255. mov (%rsp,$j,8),$lo0
  256. adc \$0,%rdx
  257. mov $hi1,-16(%rsp,$j,8) # tp[j-1]
  258. mov %rdx,$hi1
  259. xor %rdx,%rdx
  260. add $hi0,$hi1
  261. adc \$0,%rdx
  262. add $lo0,$hi1 # pull upmost overflow bit
  263. adc \$0,%rdx
  264. mov $hi1,-8(%rsp,$num,8)
  265. mov %rdx,(%rsp,$num,8) # store upmost overflow bit
  266. lea 1($i),$i # i++
  267. cmp $num,$i
  268. jb .Louter
  269. xor $i,$i # i=0 and clear CF!
  270. mov (%rsp),%rax # tp[0]
  271. mov $num,$j # j=num
  272. .align 16
  273. .Lsub: sbb ($np,$i,8),%rax
  274. mov %rax,($rp,$i,8) # rp[i]=tp[i]-np[i]
  275. mov 8(%rsp,$i,8),%rax # tp[i+1]
  276. lea 1($i),$i # i++
  277. dec $j # doesn't affect CF!
  278. jnz .Lsub
  279. sbb \$0,%rax # handle upmost overflow bit
  280. mov \$-1,%rbx
  281. xor %rax,%rbx # not %rax
  282. xor $i,$i
  283. mov $num,$j # j=num
  284. .Lcopy: # conditional copy
  285. mov ($rp,$i,8),%rcx
  286. mov (%rsp,$i,8),%rdx
  287. and %rbx,%rcx
  288. and %rax,%rdx
  289. mov $num,(%rsp,$i,8) # zap temporary vector
  290. or %rcx,%rdx
  291. mov %rdx,($rp,$i,8) # rp[i]=tp[i]
  292. lea 1($i),$i
  293. sub \$1,$j
  294. jnz .Lcopy
  295. mov 8(%rsp,$num,8),%rsi # restore %rsp
  296. .cfi_def_cfa %rsi,8
  297. mov \$1,%rax
  298. mov -48(%rsi),%r15
  299. .cfi_restore %r15
  300. mov -40(%rsi),%r14
  301. .cfi_restore %r14
  302. mov -32(%rsi),%r13
  303. .cfi_restore %r13
  304. mov -24(%rsi),%r12
  305. .cfi_restore %r12
  306. mov -16(%rsi),%rbp
  307. .cfi_restore %rbp
  308. mov -8(%rsi),%rbx
  309. .cfi_restore %rbx
  310. lea (%rsi),%rsp
  311. .cfi_def_cfa_register %rsp
  312. .Lmul_epilogue:
  313. ret
  314. .cfi_endproc
  315. .size bn_mul_mont,.-bn_mul_mont
  316. ___
  317. {{{
  318. my @A=("%r10","%r11");
  319. my @N=("%r13","%rdi");
  320. $code.=<<___;
  321. .type bn_mul4x_mont,\@function,6
  322. .align 16
  323. bn_mul4x_mont:
  324. .cfi_startproc
  325. mov ${num}d,${num}d
  326. mov %rsp,%rax
  327. .cfi_def_cfa_register %rax
  328. .Lmul4x_enter:
  329. ___
  330. $code.=<<___ if ($addx);
  331. and \$0x80100,%r11d
  332. cmp \$0x80100,%r11d
  333. je .Lmulx4x_enter
  334. ___
  335. $code.=<<___;
  336. push %rbx
  337. .cfi_push %rbx
  338. push %rbp
  339. .cfi_push %rbp
  340. push %r12
  341. .cfi_push %r12
  342. push %r13
  343. .cfi_push %r13
  344. push %r14
  345. .cfi_push %r14
  346. push %r15
  347. .cfi_push %r15
  348. neg $num
  349. mov %rsp,%r11
  350. lea -32(%rsp,$num,8),%r10 # future alloca(8*(num+4))
  351. neg $num # restore
  352. and \$-1024,%r10 # minimize TLB usage
  353. sub %r10,%r11
  354. and \$-4096,%r11
  355. lea (%r10,%r11),%rsp
  356. mov (%rsp),%r11
  357. cmp %r10,%rsp
  358. ja .Lmul4x_page_walk
  359. jmp .Lmul4x_page_walk_done
  360. .Lmul4x_page_walk:
  361. lea -4096(%rsp),%rsp
  362. mov (%rsp),%r11
  363. cmp %r10,%rsp
  364. ja .Lmul4x_page_walk
  365. .Lmul4x_page_walk_done:
  366. mov %rax,8(%rsp,$num,8) # tp[num+1]=%rsp
  367. .cfi_cfa_expression %rsp+8,$num,8,mul,plus,deref,+8
  368. .Lmul4x_body:
  369. mov $rp,16(%rsp,$num,8) # tp[num+2]=$rp
  370. mov %rdx,%r12 # reassign $bp
  371. ___
  372. $bp="%r12";
  373. $code.=<<___;
  374. mov ($n0),$n0 # pull n0[0] value
  375. mov ($bp),$m0 # m0=bp[0]
  376. mov ($ap),%rax
  377. xor $i,$i # i=0
  378. xor $j,$j # j=0
  379. mov $n0,$m1
  380. mulq $m0 # ap[0]*bp[0]
  381. mov %rax,$A[0]
  382. mov ($np),%rax
  383. imulq $A[0],$m1 # "tp[0]"*n0
  384. mov %rdx,$A[1]
  385. mulq $m1 # np[0]*m1
  386. add %rax,$A[0] # discarded
  387. mov 8($ap),%rax
  388. adc \$0,%rdx
  389. mov %rdx,$N[1]
  390. mulq $m0
  391. add %rax,$A[1]
  392. mov 8($np),%rax
  393. adc \$0,%rdx
  394. mov %rdx,$A[0]
  395. mulq $m1
  396. add %rax,$N[1]
  397. mov 16($ap),%rax
  398. adc \$0,%rdx
  399. add $A[1],$N[1]
  400. lea 4($j),$j # j++
  401. adc \$0,%rdx
  402. mov $N[1],(%rsp)
  403. mov %rdx,$N[0]
  404. jmp .L1st4x
  405. .align 16
  406. .L1st4x:
  407. mulq $m0 # ap[j]*bp[0]
  408. add %rax,$A[0]
  409. mov -16($np,$j,8),%rax
  410. adc \$0,%rdx
  411. mov %rdx,$A[1]
  412. mulq $m1 # np[j]*m1
  413. add %rax,$N[0]
  414. mov -8($ap,$j,8),%rax
  415. adc \$0,%rdx
  416. add $A[0],$N[0] # np[j]*m1+ap[j]*bp[0]
  417. adc \$0,%rdx
  418. mov $N[0],-24(%rsp,$j,8) # tp[j-1]
  419. mov %rdx,$N[1]
  420. mulq $m0 # ap[j]*bp[0]
  421. add %rax,$A[1]
  422. mov -8($np,$j,8),%rax
  423. adc \$0,%rdx
  424. mov %rdx,$A[0]
  425. mulq $m1 # np[j]*m1
  426. add %rax,$N[1]
  427. mov ($ap,$j,8),%rax
  428. adc \$0,%rdx
  429. add $A[1],$N[1] # np[j]*m1+ap[j]*bp[0]
  430. adc \$0,%rdx
  431. mov $N[1],-16(%rsp,$j,8) # tp[j-1]
  432. mov %rdx,$N[0]
  433. mulq $m0 # ap[j]*bp[0]
  434. add %rax,$A[0]
  435. mov ($np,$j,8),%rax
  436. adc \$0,%rdx
  437. mov %rdx,$A[1]
  438. mulq $m1 # np[j]*m1
  439. add %rax,$N[0]
  440. mov 8($ap,$j,8),%rax
  441. adc \$0,%rdx
  442. add $A[0],$N[0] # np[j]*m1+ap[j]*bp[0]
  443. adc \$0,%rdx
  444. mov $N[0],-8(%rsp,$j,8) # tp[j-1]
  445. mov %rdx,$N[1]
  446. mulq $m0 # ap[j]*bp[0]
  447. add %rax,$A[1]
  448. mov 8($np,$j,8),%rax
  449. adc \$0,%rdx
  450. lea 4($j),$j # j++
  451. mov %rdx,$A[0]
  452. mulq $m1 # np[j]*m1
  453. add %rax,$N[1]
  454. mov -16($ap,$j,8),%rax
  455. adc \$0,%rdx
  456. add $A[1],$N[1] # np[j]*m1+ap[j]*bp[0]
  457. adc \$0,%rdx
  458. mov $N[1],-32(%rsp,$j,8) # tp[j-1]
  459. mov %rdx,$N[0]
  460. cmp $num,$j
  461. jb .L1st4x
  462. mulq $m0 # ap[j]*bp[0]
  463. add %rax,$A[0]
  464. mov -16($np,$j,8),%rax
  465. adc \$0,%rdx
  466. mov %rdx,$A[1]
  467. mulq $m1 # np[j]*m1
  468. add %rax,$N[0]
  469. mov -8($ap,$j,8),%rax
  470. adc \$0,%rdx
  471. add $A[0],$N[0] # np[j]*m1+ap[j]*bp[0]
  472. adc \$0,%rdx
  473. mov $N[0],-24(%rsp,$j,8) # tp[j-1]
  474. mov %rdx,$N[1]
  475. mulq $m0 # ap[j]*bp[0]
  476. add %rax,$A[1]
  477. mov -8($np,$j,8),%rax
  478. adc \$0,%rdx
  479. mov %rdx,$A[0]
  480. mulq $m1 # np[j]*m1
  481. add %rax,$N[1]
  482. mov ($ap),%rax # ap[0]
  483. adc \$0,%rdx
  484. add $A[1],$N[1] # np[j]*m1+ap[j]*bp[0]
  485. adc \$0,%rdx
  486. mov $N[1],-16(%rsp,$j,8) # tp[j-1]
  487. mov %rdx,$N[0]
  488. xor $N[1],$N[1]
  489. add $A[0],$N[0]
  490. adc \$0,$N[1]
  491. mov $N[0],-8(%rsp,$j,8)
  492. mov $N[1],(%rsp,$j,8) # store upmost overflow bit
  493. lea 1($i),$i # i++
  494. .align 4
  495. .Louter4x:
  496. mov ($bp,$i,8),$m0 # m0=bp[i]
  497. xor $j,$j # j=0
  498. mov (%rsp),$A[0]
  499. mov $n0,$m1
  500. mulq $m0 # ap[0]*bp[i]
  501. add %rax,$A[0] # ap[0]*bp[i]+tp[0]
  502. mov ($np),%rax
  503. adc \$0,%rdx
  504. imulq $A[0],$m1 # tp[0]*n0
  505. mov %rdx,$A[1]
  506. mulq $m1 # np[0]*m1
  507. add %rax,$A[0] # "$N[0]", discarded
  508. mov 8($ap),%rax
  509. adc \$0,%rdx
  510. mov %rdx,$N[1]
  511. mulq $m0 # ap[j]*bp[i]
  512. add %rax,$A[1]
  513. mov 8($np),%rax
  514. adc \$0,%rdx
  515. add 8(%rsp),$A[1] # +tp[1]
  516. adc \$0,%rdx
  517. mov %rdx,$A[0]
  518. mulq $m1 # np[j]*m1
  519. add %rax,$N[1]
  520. mov 16($ap),%rax
  521. adc \$0,%rdx
  522. add $A[1],$N[1] # np[j]*m1+ap[j]*bp[i]+tp[j]
  523. lea 4($j),$j # j+=2
  524. adc \$0,%rdx
  525. mov $N[1],(%rsp) # tp[j-1]
  526. mov %rdx,$N[0]
  527. jmp .Linner4x
  528. .align 16
  529. .Linner4x:
  530. mulq $m0 # ap[j]*bp[i]
  531. add %rax,$A[0]
  532. mov -16($np,$j,8),%rax
  533. adc \$0,%rdx
  534. add -16(%rsp,$j,8),$A[0] # ap[j]*bp[i]+tp[j]
  535. adc \$0,%rdx
  536. mov %rdx,$A[1]
  537. mulq $m1 # np[j]*m1
  538. add %rax,$N[0]
  539. mov -8($ap,$j,8),%rax
  540. adc \$0,%rdx
  541. add $A[0],$N[0]
  542. adc \$0,%rdx
  543. mov $N[0],-24(%rsp,$j,8) # tp[j-1]
  544. mov %rdx,$N[1]
  545. mulq $m0 # ap[j]*bp[i]
  546. add %rax,$A[1]
  547. mov -8($np,$j,8),%rax
  548. adc \$0,%rdx
  549. add -8(%rsp,$j,8),$A[1]
  550. adc \$0,%rdx
  551. mov %rdx,$A[0]
  552. mulq $m1 # np[j]*m1
  553. add %rax,$N[1]
  554. mov ($ap,$j,8),%rax
  555. adc \$0,%rdx
  556. add $A[1],$N[1]
  557. adc \$0,%rdx
  558. mov $N[1],-16(%rsp,$j,8) # tp[j-1]
  559. mov %rdx,$N[0]
  560. mulq $m0 # ap[j]*bp[i]
  561. add %rax,$A[0]
  562. mov ($np,$j,8),%rax
  563. adc \$0,%rdx
  564. add (%rsp,$j,8),$A[0] # ap[j]*bp[i]+tp[j]
  565. adc \$0,%rdx
  566. mov %rdx,$A[1]
  567. mulq $m1 # np[j]*m1
  568. add %rax,$N[0]
  569. mov 8($ap,$j,8),%rax
  570. adc \$0,%rdx
  571. add $A[0],$N[0]
  572. adc \$0,%rdx
  573. mov $N[0],-8(%rsp,$j,8) # tp[j-1]
  574. mov %rdx,$N[1]
  575. mulq $m0 # ap[j]*bp[i]
  576. add %rax,$A[1]
  577. mov 8($np,$j,8),%rax
  578. adc \$0,%rdx
  579. add 8(%rsp,$j,8),$A[1]
  580. adc \$0,%rdx
  581. lea 4($j),$j # j++
  582. mov %rdx,$A[0]
  583. mulq $m1 # np[j]*m1
  584. add %rax,$N[1]
  585. mov -16($ap,$j,8),%rax
  586. adc \$0,%rdx
  587. add $A[1],$N[1]
  588. adc \$0,%rdx
  589. mov $N[1],-32(%rsp,$j,8) # tp[j-1]
  590. mov %rdx,$N[0]
  591. cmp $num,$j
  592. jb .Linner4x
  593. mulq $m0 # ap[j]*bp[i]
  594. add %rax,$A[0]
  595. mov -16($np,$j,8),%rax
  596. adc \$0,%rdx
  597. add -16(%rsp,$j,8),$A[0] # ap[j]*bp[i]+tp[j]
  598. adc \$0,%rdx
  599. mov %rdx,$A[1]
  600. mulq $m1 # np[j]*m1
  601. add %rax,$N[0]
  602. mov -8($ap,$j,8),%rax
  603. adc \$0,%rdx
  604. add $A[0],$N[0]
  605. adc \$0,%rdx
  606. mov $N[0],-24(%rsp,$j,8) # tp[j-1]
  607. mov %rdx,$N[1]
  608. mulq $m0 # ap[j]*bp[i]
  609. add %rax,$A[1]
  610. mov -8($np,$j,8),%rax
  611. adc \$0,%rdx
  612. add -8(%rsp,$j,8),$A[1]
  613. adc \$0,%rdx
  614. lea 1($i),$i # i++
  615. mov %rdx,$A[0]
  616. mulq $m1 # np[j]*m1
  617. add %rax,$N[1]
  618. mov ($ap),%rax # ap[0]
  619. adc \$0,%rdx
  620. add $A[1],$N[1]
  621. adc \$0,%rdx
  622. mov $N[1],-16(%rsp,$j,8) # tp[j-1]
  623. mov %rdx,$N[0]
  624. xor $N[1],$N[1]
  625. add $A[0],$N[0]
  626. adc \$0,$N[1]
  627. add (%rsp,$num,8),$N[0] # pull upmost overflow bit
  628. adc \$0,$N[1]
  629. mov $N[0],-8(%rsp,$j,8)
  630. mov $N[1],(%rsp,$j,8) # store upmost overflow bit
  631. cmp $num,$i
  632. jb .Louter4x
  633. ___
  634. {
  635. my @ri=("%rax","%rdx",$m0,$m1);
  636. $code.=<<___;
  637. mov 16(%rsp,$num,8),$rp # restore $rp
  638. lea -4($num),$j
  639. mov 0(%rsp),@ri[0] # tp[0]
  640. mov 8(%rsp),@ri[1] # tp[1]
  641. shr \$2,$j # j=num/4-1
  642. lea (%rsp),$ap # borrow ap for tp
  643. xor $i,$i # i=0 and clear CF!
  644. sub 0($np),@ri[0]
  645. mov 16($ap),@ri[2] # tp[2]
  646. mov 24($ap),@ri[3] # tp[3]
  647. sbb 8($np),@ri[1]
  648. .Lsub4x:
  649. mov @ri[0],0($rp,$i,8) # rp[i]=tp[i]-np[i]
  650. mov @ri[1],8($rp,$i,8) # rp[i]=tp[i]-np[i]
  651. sbb 16($np,$i,8),@ri[2]
  652. mov 32($ap,$i,8),@ri[0] # tp[i+1]
  653. mov 40($ap,$i,8),@ri[1]
  654. sbb 24($np,$i,8),@ri[3]
  655. mov @ri[2],16($rp,$i,8) # rp[i]=tp[i]-np[i]
  656. mov @ri[3],24($rp,$i,8) # rp[i]=tp[i]-np[i]
  657. sbb 32($np,$i,8),@ri[0]
  658. mov 48($ap,$i,8),@ri[2]
  659. mov 56($ap,$i,8),@ri[3]
  660. sbb 40($np,$i,8),@ri[1]
  661. lea 4($i),$i # i++
  662. dec $j # doesn't affect CF!
  663. jnz .Lsub4x
  664. mov @ri[0],0($rp,$i,8) # rp[i]=tp[i]-np[i]
  665. mov 32($ap,$i,8),@ri[0] # load overflow bit
  666. sbb 16($np,$i,8),@ri[2]
  667. mov @ri[1],8($rp,$i,8) # rp[i]=tp[i]-np[i]
  668. sbb 24($np,$i,8),@ri[3]
  669. mov @ri[2],16($rp,$i,8) # rp[i]=tp[i]-np[i]
  670. sbb \$0,@ri[0] # handle upmost overflow bit
  671. mov @ri[3],24($rp,$i,8) # rp[i]=tp[i]-np[i]
  672. pxor %xmm0,%xmm0
  673. movq @ri[0],%xmm4
  674. pcmpeqd %xmm5,%xmm5
  675. pshufd \$0,%xmm4,%xmm4
  676. mov $num,$j
  677. pxor %xmm4,%xmm5
  678. shr \$2,$j # j=num/4
  679. xor %eax,%eax # i=0
  680. jmp .Lcopy4x
  681. .align 16
  682. .Lcopy4x: # conditional copy
  683. movdqa (%rsp,%rax),%xmm1
  684. movdqu ($rp,%rax),%xmm2
  685. pand %xmm4,%xmm1
  686. pand %xmm5,%xmm2
  687. movdqa 16(%rsp,%rax),%xmm3
  688. movdqa %xmm0,(%rsp,%rax)
  689. por %xmm2,%xmm1
  690. movdqu 16($rp,%rax),%xmm2
  691. movdqu %xmm1,($rp,%rax)
  692. pand %xmm4,%xmm3
  693. pand %xmm5,%xmm2
  694. movdqa %xmm0,16(%rsp,%rax)
  695. por %xmm2,%xmm3
  696. movdqu %xmm3,16($rp,%rax)
  697. lea 32(%rax),%rax
  698. dec $j
  699. jnz .Lcopy4x
  700. ___
  701. }
  702. $code.=<<___;
  703. mov 8(%rsp,$num,8),%rsi # restore %rsp
  704. .cfi_def_cfa %rsi, 8
  705. mov \$1,%rax
  706. mov -48(%rsi),%r15
  707. .cfi_restore %r15
  708. mov -40(%rsi),%r14
  709. .cfi_restore %r14
  710. mov -32(%rsi),%r13
  711. .cfi_restore %r13
  712. mov -24(%rsi),%r12
  713. .cfi_restore %r12
  714. mov -16(%rsi),%rbp
  715. .cfi_restore %rbp
  716. mov -8(%rsi),%rbx
  717. .cfi_restore %rbx
  718. lea (%rsi),%rsp
  719. .cfi_def_cfa_register %rsp
  720. .Lmul4x_epilogue:
  721. ret
  722. .cfi_endproc
  723. .size bn_mul4x_mont,.-bn_mul4x_mont
  724. ___
  725. }}}
  726. {{{
  727. ######################################################################
  728. # void bn_sqr8x_mont(
  729. my $rptr="%rdi"; # const BN_ULONG *rptr,
  730. my $aptr="%rsi"; # const BN_ULONG *aptr,
  731. my $bptr="%rdx"; # not used
  732. my $nptr="%rcx"; # const BN_ULONG *nptr,
  733. my $n0 ="%r8"; # const BN_ULONG *n0);
  734. my $num ="%r9"; # int num, has to be divisible by 8
  735. my ($i,$j,$tptr)=("%rbp","%rcx",$rptr);
  736. my @A0=("%r10","%r11");
  737. my @A1=("%r12","%r13");
  738. my ($a0,$a1,$ai)=("%r14","%r15","%rbx");
  739. $code.=<<___ if ($addx);
  740. .extern bn_sqrx8x_internal # see x86_64-mont5 module
  741. ___
  742. $code.=<<___;
  743. .extern bn_sqr8x_internal # see x86_64-mont5 module
  744. .type bn_sqr8x_mont,\@function,6
  745. .align 32
  746. bn_sqr8x_mont:
  747. .cfi_startproc
  748. mov %rsp,%rax
  749. .cfi_def_cfa_register %rax
  750. .Lsqr8x_enter:
  751. push %rbx
  752. .cfi_push %rbx
  753. push %rbp
  754. .cfi_push %rbp
  755. push %r12
  756. .cfi_push %r12
  757. push %r13
  758. .cfi_push %r13
  759. push %r14
  760. .cfi_push %r14
  761. push %r15
  762. .cfi_push %r15
  763. .Lsqr8x_prologue:
  764. mov ${num}d,%r10d
  765. shl \$3,${num}d # convert $num to bytes
  766. shl \$3+2,%r10 # 4*$num
  767. neg $num
  768. ##############################################################
  769. # ensure that stack frame doesn't alias with $aptr modulo
  770. # 4096. this is done to allow memory disambiguation logic
  771. # do its job.
  772. #
  773. lea -64(%rsp,$num,2),%r11
  774. mov %rsp,%rbp
  775. mov ($n0),$n0 # *n0
  776. sub $aptr,%r11
  777. and \$4095,%r11
  778. cmp %r11,%r10
  779. jb .Lsqr8x_sp_alt
  780. sub %r11,%rbp # align with $aptr
  781. lea -64(%rbp,$num,2),%rbp # future alloca(frame+2*$num)
  782. jmp .Lsqr8x_sp_done
  783. .align 32
  784. .Lsqr8x_sp_alt:
  785. lea 4096-64(,$num,2),%r10 # 4096-frame-2*$num
  786. lea -64(%rbp,$num,2),%rbp # future alloca(frame+2*$num)
  787. sub %r10,%r11
  788. mov \$0,%r10
  789. cmovc %r10,%r11
  790. sub %r11,%rbp
  791. .Lsqr8x_sp_done:
  792. and \$-64,%rbp
  793. mov %rsp,%r11
  794. sub %rbp,%r11
  795. and \$-4096,%r11
  796. lea (%rbp,%r11),%rsp
  797. mov (%rsp),%r10
  798. cmp %rbp,%rsp
  799. ja .Lsqr8x_page_walk
  800. jmp .Lsqr8x_page_walk_done
  801. .align 16
  802. .Lsqr8x_page_walk:
  803. lea -4096(%rsp),%rsp
  804. mov (%rsp),%r10
  805. cmp %rbp,%rsp
  806. ja .Lsqr8x_page_walk
  807. .Lsqr8x_page_walk_done:
  808. mov $num,%r10
  809. neg $num
  810. mov $n0, 32(%rsp)
  811. mov %rax, 40(%rsp) # save original %rsp
  812. .cfi_cfa_expression %rsp+40,deref,+8
  813. .Lsqr8x_body:
  814. movq $nptr, %xmm2 # save pointer to modulus
  815. pxor %xmm0,%xmm0
  816. movq $rptr,%xmm1 # save $rptr
  817. movq %r10, %xmm3 # -$num
  818. ___
  819. $code.=<<___ if ($addx);
  820. mov OPENSSL_ia32cap_P+8(%rip),%eax
  821. and \$0x80100,%eax
  822. cmp \$0x80100,%eax
  823. jne .Lsqr8x_nox
  824. call bn_sqrx8x_internal # see x86_64-mont5 module
  825. # %rax top-most carry
  826. # %rbp nptr
  827. # %rcx -8*num
  828. # %r8 end of tp[2*num]
  829. lea (%r8,%rcx),%rbx
  830. mov %rcx,$num
  831. mov %rcx,%rdx
  832. movq %xmm1,$rptr
  833. sar \$3+2,%rcx # %cf=0
  834. jmp .Lsqr8x_sub
  835. .align 32
  836. .Lsqr8x_nox:
  837. ___
  838. $code.=<<___;
  839. call bn_sqr8x_internal # see x86_64-mont5 module
  840. # %rax top-most carry
  841. # %rbp nptr
  842. # %r8 -8*num
  843. # %rdi end of tp[2*num]
  844. lea (%rdi,$num),%rbx
  845. mov $num,%rcx
  846. mov $num,%rdx
  847. movq %xmm1,$rptr
  848. sar \$3+2,%rcx # %cf=0
  849. jmp .Lsqr8x_sub
  850. .align 32
  851. .Lsqr8x_sub:
  852. mov 8*0(%rbx),%r12
  853. mov 8*1(%rbx),%r13
  854. mov 8*2(%rbx),%r14
  855. mov 8*3(%rbx),%r15
  856. lea 8*4(%rbx),%rbx
  857. sbb 8*0(%rbp),%r12
  858. sbb 8*1(%rbp),%r13
  859. sbb 8*2(%rbp),%r14
  860. sbb 8*3(%rbp),%r15
  861. lea 8*4(%rbp),%rbp
  862. mov %r12,8*0($rptr)
  863. mov %r13,8*1($rptr)
  864. mov %r14,8*2($rptr)
  865. mov %r15,8*3($rptr)
  866. lea 8*4($rptr),$rptr
  867. inc %rcx # preserves %cf
  868. jnz .Lsqr8x_sub
  869. sbb \$0,%rax # top-most carry
  870. lea (%rbx,$num),%rbx # rewind
  871. lea ($rptr,$num),$rptr # rewind
  872. movq %rax,%xmm1
  873. pxor %xmm0,%xmm0
  874. pshufd \$0,%xmm1,%xmm1
  875. mov 40(%rsp),%rsi # restore %rsp
  876. .cfi_def_cfa %rsi,8
  877. jmp .Lsqr8x_cond_copy
  878. .align 32
  879. .Lsqr8x_cond_copy:
  880. movdqa 16*0(%rbx),%xmm2
  881. movdqa 16*1(%rbx),%xmm3
  882. lea 16*2(%rbx),%rbx
  883. movdqu 16*0($rptr),%xmm4
  884. movdqu 16*1($rptr),%xmm5
  885. lea 16*2($rptr),$rptr
  886. movdqa %xmm0,-16*2(%rbx) # zero tp
  887. movdqa %xmm0,-16*1(%rbx)
  888. movdqa %xmm0,-16*2(%rbx,%rdx)
  889. movdqa %xmm0,-16*1(%rbx,%rdx)
  890. pcmpeqd %xmm1,%xmm0
  891. pand %xmm1,%xmm2
  892. pand %xmm1,%xmm3
  893. pand %xmm0,%xmm4
  894. pand %xmm0,%xmm5
  895. pxor %xmm0,%xmm0
  896. por %xmm2,%xmm4
  897. por %xmm3,%xmm5
  898. movdqu %xmm4,-16*2($rptr)
  899. movdqu %xmm5,-16*1($rptr)
  900. add \$32,$num
  901. jnz .Lsqr8x_cond_copy
  902. mov \$1,%rax
  903. mov -48(%rsi),%r15
  904. .cfi_restore %r15
  905. mov -40(%rsi),%r14
  906. .cfi_restore %r14
  907. mov -32(%rsi),%r13
  908. .cfi_restore %r13
  909. mov -24(%rsi),%r12
  910. .cfi_restore %r12
  911. mov -16(%rsi),%rbp
  912. .cfi_restore %rbp
  913. mov -8(%rsi),%rbx
  914. .cfi_restore %rbx
  915. lea (%rsi),%rsp
  916. .cfi_def_cfa_register %rsp
  917. .Lsqr8x_epilogue:
  918. ret
  919. .cfi_endproc
  920. .size bn_sqr8x_mont,.-bn_sqr8x_mont
  921. ___
  922. }}}
  923. if ($addx) {{{
  924. my $bp="%rdx"; # original value
  925. $code.=<<___;
  926. .type bn_mulx4x_mont,\@function,6
  927. .align 32
  928. bn_mulx4x_mont:
  929. .cfi_startproc
  930. mov %rsp,%rax
  931. .cfi_def_cfa_register %rax
  932. .Lmulx4x_enter:
  933. push %rbx
  934. .cfi_push %rbx
  935. push %rbp
  936. .cfi_push %rbp
  937. push %r12
  938. .cfi_push %r12
  939. push %r13
  940. .cfi_push %r13
  941. push %r14
  942. .cfi_push %r14
  943. push %r15
  944. .cfi_push %r15
  945. .Lmulx4x_prologue:
  946. shl \$3,${num}d # convert $num to bytes
  947. xor %r10,%r10
  948. sub $num,%r10 # -$num
  949. mov ($n0),$n0 # *n0
  950. lea -72(%rsp,%r10),%rbp # future alloca(frame+$num+8)
  951. and \$-128,%rbp
  952. mov %rsp,%r11
  953. sub %rbp,%r11
  954. and \$-4096,%r11
  955. lea (%rbp,%r11),%rsp
  956. mov (%rsp),%r10
  957. cmp %rbp,%rsp
  958. ja .Lmulx4x_page_walk
  959. jmp .Lmulx4x_page_walk_done
  960. .align 16
  961. .Lmulx4x_page_walk:
  962. lea -4096(%rsp),%rsp
  963. mov (%rsp),%r10
  964. cmp %rbp,%rsp
  965. ja .Lmulx4x_page_walk
  966. .Lmulx4x_page_walk_done:
  967. lea ($bp,$num),%r10
  968. ##############################################################
  969. # Stack layout
  970. # +0 num
  971. # +8 off-loaded &b[i]
  972. # +16 end of b[num]
  973. # +24 saved n0
  974. # +32 saved rp
  975. # +40 saved %rsp
  976. # +48 inner counter
  977. # +56
  978. # +64 tmp[num+1]
  979. #
  980. mov $num,0(%rsp) # save $num
  981. shr \$5,$num
  982. mov %r10,16(%rsp) # end of b[num]
  983. sub \$1,$num
  984. mov $n0, 24(%rsp) # save *n0
  985. mov $rp, 32(%rsp) # save $rp
  986. mov %rax,40(%rsp) # save original %rsp
  987. .cfi_cfa_expression %rsp+40,deref,+8
  988. mov $num,48(%rsp) # inner counter
  989. jmp .Lmulx4x_body
  990. .align 32
  991. .Lmulx4x_body:
  992. ___
  993. my ($aptr, $bptr, $nptr, $tptr, $mi, $bi, $zero, $num)=
  994. ("%rsi","%rdi","%rcx","%rbx","%r8","%r9","%rbp","%rax");
  995. my $rptr=$bptr;
  996. $code.=<<___;
  997. lea 8($bp),$bptr
  998. mov ($bp),%rdx # b[0], $bp==%rdx actually
  999. lea 64+32(%rsp),$tptr
  1000. mov %rdx,$bi
  1001. mulx 0*8($aptr),$mi,%rax # a[0]*b[0]
  1002. mulx 1*8($aptr),%r11,%r14 # a[1]*b[0]
  1003. add %rax,%r11
  1004. mov $bptr,8(%rsp) # off-load &b[i]
  1005. mulx 2*8($aptr),%r12,%r13 # ...
  1006. adc %r14,%r12
  1007. adc \$0,%r13
  1008. mov $mi,$bptr # borrow $bptr
  1009. imulq 24(%rsp),$mi # "t[0]"*n0
  1010. xor $zero,$zero # cf=0, of=0
  1011. mulx 3*8($aptr),%rax,%r14
  1012. mov $mi,%rdx
  1013. lea 4*8($aptr),$aptr
  1014. adcx %rax,%r13
  1015. adcx $zero,%r14 # cf=0
  1016. mulx 0*8($nptr),%rax,%r10
  1017. adcx %rax,$bptr # discarded
  1018. adox %r11,%r10
  1019. mulx 1*8($nptr),%rax,%r11
  1020. adcx %rax,%r10
  1021. adox %r12,%r11
  1022. .byte 0xc4,0x62,0xfb,0xf6,0xa1,0x10,0x00,0x00,0x00 # mulx 2*8($nptr),%rax,%r12
  1023. mov 48(%rsp),$bptr # counter value
  1024. mov %r10,-4*8($tptr)
  1025. adcx %rax,%r11
  1026. adox %r13,%r12
  1027. mulx 3*8($nptr),%rax,%r15
  1028. mov $bi,%rdx
  1029. mov %r11,-3*8($tptr)
  1030. adcx %rax,%r12
  1031. adox $zero,%r15 # of=0
  1032. lea 4*8($nptr),$nptr
  1033. mov %r12,-2*8($tptr)
  1034. jmp .Lmulx4x_1st
  1035. .align 32
  1036. .Lmulx4x_1st:
  1037. adcx $zero,%r15 # cf=0, modulo-scheduled
  1038. mulx 0*8($aptr),%r10,%rax # a[4]*b[0]
  1039. adcx %r14,%r10
  1040. mulx 1*8($aptr),%r11,%r14 # a[5]*b[0]
  1041. adcx %rax,%r11
  1042. mulx 2*8($aptr),%r12,%rax # ...
  1043. adcx %r14,%r12
  1044. mulx 3*8($aptr),%r13,%r14
  1045. .byte 0x67,0x67
  1046. mov $mi,%rdx
  1047. adcx %rax,%r13
  1048. adcx $zero,%r14 # cf=0
  1049. lea 4*8($aptr),$aptr
  1050. lea 4*8($tptr),$tptr
  1051. adox %r15,%r10
  1052. mulx 0*8($nptr),%rax,%r15
  1053. adcx %rax,%r10
  1054. adox %r15,%r11
  1055. mulx 1*8($nptr),%rax,%r15
  1056. adcx %rax,%r11
  1057. adox %r15,%r12
  1058. mulx 2*8($nptr),%rax,%r15
  1059. mov %r10,-5*8($tptr)
  1060. adcx %rax,%r12
  1061. mov %r11,-4*8($tptr)
  1062. adox %r15,%r13
  1063. mulx 3*8($nptr),%rax,%r15
  1064. mov $bi,%rdx
  1065. mov %r12,-3*8($tptr)
  1066. adcx %rax,%r13
  1067. adox $zero,%r15
  1068. lea 4*8($nptr),$nptr
  1069. mov %r13,-2*8($tptr)
  1070. dec $bptr # of=0, pass cf
  1071. jnz .Lmulx4x_1st
  1072. mov 0(%rsp),$num # load num
  1073. mov 8(%rsp),$bptr # re-load &b[i]
  1074. adc $zero,%r15 # modulo-scheduled
  1075. add %r15,%r14
  1076. sbb %r15,%r15 # top-most carry
  1077. mov %r14,-1*8($tptr)
  1078. jmp .Lmulx4x_outer
  1079. .align 32
  1080. .Lmulx4x_outer:
  1081. mov ($bptr),%rdx # b[i]
  1082. lea 8($bptr),$bptr # b++
  1083. sub $num,$aptr # rewind $aptr
  1084. mov %r15,($tptr) # save top-most carry
  1085. lea 64+4*8(%rsp),$tptr
  1086. sub $num,$nptr # rewind $nptr
  1087. mulx 0*8($aptr),$mi,%r11 # a[0]*b[i]
  1088. xor %ebp,%ebp # xor $zero,$zero # cf=0, of=0
  1089. mov %rdx,$bi
  1090. mulx 1*8($aptr),%r14,%r12 # a[1]*b[i]
  1091. adox -4*8($tptr),$mi
  1092. adcx %r14,%r11
  1093. mulx 2*8($aptr),%r15,%r13 # ...
  1094. adox -3*8($tptr),%r11
  1095. adcx %r15,%r12
  1096. adox -2*8($tptr),%r12
  1097. adcx $zero,%r13
  1098. adox $zero,%r13
  1099. mov $bptr,8(%rsp) # off-load &b[i]
  1100. mov $mi,%r15
  1101. imulq 24(%rsp),$mi # "t[0]"*n0
  1102. xor %ebp,%ebp # xor $zero,$zero # cf=0, of=0
  1103. mulx 3*8($aptr),%rax,%r14
  1104. mov $mi,%rdx
  1105. adcx %rax,%r13
  1106. adox -1*8($tptr),%r13
  1107. adcx $zero,%r14
  1108. lea 4*8($aptr),$aptr
  1109. adox $zero,%r14
  1110. mulx 0*8($nptr),%rax,%r10
  1111. adcx %rax,%r15 # discarded
  1112. adox %r11,%r10
  1113. mulx 1*8($nptr),%rax,%r11
  1114. adcx %rax,%r10
  1115. adox %r12,%r11
  1116. mulx 2*8($nptr),%rax,%r12
  1117. mov %r10,-4*8($tptr)
  1118. adcx %rax,%r11
  1119. adox %r13,%r12
  1120. mulx 3*8($nptr),%rax,%r15
  1121. mov $bi,%rdx
  1122. mov %r11,-3*8($tptr)
  1123. lea 4*8($nptr),$nptr
  1124. adcx %rax,%r12
  1125. adox $zero,%r15 # of=0
  1126. mov 48(%rsp),$bptr # counter value
  1127. mov %r12,-2*8($tptr)
  1128. jmp .Lmulx4x_inner
  1129. .align 32
  1130. .Lmulx4x_inner:
  1131. mulx 0*8($aptr),%r10,%rax # a[4]*b[i]
  1132. adcx $zero,%r15 # cf=0, modulo-scheduled
  1133. adox %r14,%r10
  1134. mulx 1*8($aptr),%r11,%r14 # a[5]*b[i]
  1135. adcx 0*8($tptr),%r10
  1136. adox %rax,%r11
  1137. mulx 2*8($aptr),%r12,%rax # ...
  1138. adcx 1*8($tptr),%r11
  1139. adox %r14,%r12
  1140. mulx 3*8($aptr),%r13,%r14
  1141. mov $mi,%rdx
  1142. adcx 2*8($tptr),%r12
  1143. adox %rax,%r13
  1144. adcx 3*8($tptr),%r13
  1145. adox $zero,%r14 # of=0
  1146. lea 4*8($aptr),$aptr
  1147. lea 4*8($tptr),$tptr
  1148. adcx $zero,%r14 # cf=0
  1149. adox %r15,%r10
  1150. mulx 0*8($nptr),%rax,%r15
  1151. adcx %rax,%r10
  1152. adox %r15,%r11
  1153. mulx 1*8($nptr),%rax,%r15
  1154. adcx %rax,%r11
  1155. adox %r15,%r12
  1156. mulx 2*8($nptr),%rax,%r15
  1157. mov %r10,-5*8($tptr)
  1158. adcx %rax,%r12
  1159. adox %r15,%r13
  1160. mulx 3*8($nptr),%rax,%r15
  1161. mov $bi,%rdx
  1162. mov %r11,-4*8($tptr)
  1163. mov %r12,-3*8($tptr)
  1164. adcx %rax,%r13
  1165. adox $zero,%r15
  1166. lea 4*8($nptr),$nptr
  1167. mov %r13,-2*8($tptr)
  1168. dec $bptr # of=0, pass cf
  1169. jnz .Lmulx4x_inner
  1170. mov 0(%rsp),$num # load num
  1171. mov 8(%rsp),$bptr # re-load &b[i]
  1172. adc $zero,%r15 # modulo-scheduled
  1173. sub 0*8($tptr),$zero # pull top-most carry
  1174. adc %r15,%r14
  1175. sbb %r15,%r15 # top-most carry
  1176. mov %r14,-1*8($tptr)
  1177. cmp 16(%rsp),$bptr
  1178. jne .Lmulx4x_outer
  1179. lea 64(%rsp),$tptr
  1180. sub $num,$nptr # rewind $nptr
  1181. neg %r15
  1182. mov $num,%rdx
  1183. shr \$3+2,$num # %cf=0
  1184. mov 32(%rsp),$rptr # restore rp
  1185. jmp .Lmulx4x_sub
  1186. .align 32
  1187. .Lmulx4x_sub:
  1188. mov 8*0($tptr),%r11
  1189. mov 8*1($tptr),%r12
  1190. mov 8*2($tptr),%r13
  1191. mov 8*3($tptr),%r14
  1192. lea 8*4($tptr),$tptr
  1193. sbb 8*0($nptr),%r11
  1194. sbb 8*1($nptr),%r12
  1195. sbb 8*2($nptr),%r13
  1196. sbb 8*3($nptr),%r14
  1197. lea 8*4($nptr),$nptr
  1198. mov %r11,8*0($rptr)
  1199. mov %r12,8*1($rptr)
  1200. mov %r13,8*2($rptr)
  1201. mov %r14,8*3($rptr)
  1202. lea 8*4($rptr),$rptr
  1203. dec $num # preserves %cf
  1204. jnz .Lmulx4x_sub
  1205. sbb \$0,%r15 # top-most carry
  1206. lea 64(%rsp),$tptr
  1207. sub %rdx,$rptr # rewind
  1208. movq %r15,%xmm1
  1209. pxor %xmm0,%xmm0
  1210. pshufd \$0,%xmm1,%xmm1
  1211. mov 40(%rsp),%rsi # restore %rsp
  1212. .cfi_def_cfa %rsi,8
  1213. jmp .Lmulx4x_cond_copy
  1214. .align 32
  1215. .Lmulx4x_cond_copy:
  1216. movdqa 16*0($tptr),%xmm2
  1217. movdqa 16*1($tptr),%xmm3
  1218. lea 16*2($tptr),$tptr
  1219. movdqu 16*0($rptr),%xmm4
  1220. movdqu 16*1($rptr),%xmm5
  1221. lea 16*2($rptr),$rptr
  1222. movdqa %xmm0,-16*2($tptr) # zero tp
  1223. movdqa %xmm0,-16*1($tptr)
  1224. pcmpeqd %xmm1,%xmm0
  1225. pand %xmm1,%xmm2
  1226. pand %xmm1,%xmm3
  1227. pand %xmm0,%xmm4
  1228. pand %xmm0,%xmm5
  1229. pxor %xmm0,%xmm0
  1230. por %xmm2,%xmm4
  1231. por %xmm3,%xmm5
  1232. movdqu %xmm4,-16*2($rptr)
  1233. movdqu %xmm5,-16*1($rptr)
  1234. sub \$32,%rdx
  1235. jnz .Lmulx4x_cond_copy
  1236. mov %rdx,($tptr)
  1237. mov \$1,%rax
  1238. mov -48(%rsi),%r15
  1239. .cfi_restore %r15
  1240. mov -40(%rsi),%r14
  1241. .cfi_restore %r14
  1242. mov -32(%rsi),%r13
  1243. .cfi_restore %r13
  1244. mov -24(%rsi),%r12
  1245. .cfi_restore %r12
  1246. mov -16(%rsi),%rbp
  1247. .cfi_restore %rbp
  1248. mov -8(%rsi),%rbx
  1249. .cfi_restore %rbx
  1250. lea (%rsi),%rsp
  1251. .cfi_def_cfa_register %rsp
  1252. .Lmulx4x_epilogue:
  1253. ret
  1254. .cfi_endproc
  1255. .size bn_mulx4x_mont,.-bn_mulx4x_mont
  1256. ___
  1257. }}}
  1258. $code.=<<___;
  1259. .asciz "Montgomery Multiplication for x86_64, CRYPTOGAMS by <appro\@openssl.org>"
  1260. .align 16
  1261. ___
  1262. # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame,
  1263. # CONTEXT *context,DISPATCHER_CONTEXT *disp)
  1264. if ($win64) {
  1265. $rec="%rcx";
  1266. $frame="%rdx";
  1267. $context="%r8";
  1268. $disp="%r9";
  1269. $code.=<<___;
  1270. .extern __imp_RtlVirtualUnwind
  1271. .type mul_handler,\@abi-omnipotent
  1272. .align 16
  1273. mul_handler:
  1274. push %rsi
  1275. push %rdi
  1276. push %rbx
  1277. push %rbp
  1278. push %r12
  1279. push %r13
  1280. push %r14
  1281. push %r15
  1282. pushfq
  1283. sub \$64,%rsp
  1284. mov 120($context),%rax # pull context->Rax
  1285. mov 248($context),%rbx # pull context->Rip
  1286. mov 8($disp),%rsi # disp->ImageBase
  1287. mov 56($disp),%r11 # disp->HandlerData
  1288. mov 0(%r11),%r10d # HandlerData[0]
  1289. lea (%rsi,%r10),%r10 # end of prologue label
  1290. cmp %r10,%rbx # context->Rip<end of prologue label
  1291. jb .Lcommon_seh_tail
  1292. mov 152($context),%rax # pull context->Rsp
  1293. mov 4(%r11),%r10d # HandlerData[1]
  1294. lea (%rsi,%r10),%r10 # epilogue label
  1295. cmp %r10,%rbx # context->Rip>=epilogue label
  1296. jae .Lcommon_seh_tail
  1297. mov 192($context),%r10 # pull $num
  1298. mov 8(%rax,%r10,8),%rax # pull saved stack pointer
  1299. jmp .Lcommon_pop_regs
  1300. .size mul_handler,.-mul_handler
  1301. .type sqr_handler,\@abi-omnipotent
  1302. .align 16
  1303. sqr_handler:
  1304. push %rsi
  1305. push %rdi
  1306. push %rbx
  1307. push %rbp
  1308. push %r12
  1309. push %r13
  1310. push %r14
  1311. push %r15
  1312. pushfq
  1313. sub \$64,%rsp
  1314. mov 120($context),%rax # pull context->Rax
  1315. mov 248($context),%rbx # pull context->Rip
  1316. mov 8($disp),%rsi # disp->ImageBase
  1317. mov 56($disp),%r11 # disp->HandlerData
  1318. mov 0(%r11),%r10d # HandlerData[0]
  1319. lea (%rsi,%r10),%r10 # end of prologue label
  1320. cmp %r10,%rbx # context->Rip<.Lsqr_prologue
  1321. jb .Lcommon_seh_tail
  1322. mov 4(%r11),%r10d # HandlerData[1]
  1323. lea (%rsi,%r10),%r10 # body label
  1324. cmp %r10,%rbx # context->Rip<.Lsqr_body
  1325. jb .Lcommon_pop_regs
  1326. mov 152($context),%rax # pull context->Rsp
  1327. mov 8(%r11),%r10d # HandlerData[2]
  1328. lea (%rsi,%r10),%r10 # epilogue label
  1329. cmp %r10,%rbx # context->Rip>=.Lsqr_epilogue
  1330. jae .Lcommon_seh_tail
  1331. mov 40(%rax),%rax # pull saved stack pointer
  1332. .Lcommon_pop_regs:
  1333. mov -8(%rax),%rbx
  1334. mov -16(%rax),%rbp
  1335. mov -24(%rax),%r12
  1336. mov -32(%rax),%r13
  1337. mov -40(%rax),%r14
  1338. mov -48(%rax),%r15
  1339. mov %rbx,144($context) # restore context->Rbx
  1340. mov %rbp,160($context) # restore context->Rbp
  1341. mov %r12,216($context) # restore context->R12
  1342. mov %r13,224($context) # restore context->R13
  1343. mov %r14,232($context) # restore context->R14
  1344. mov %r15,240($context) # restore context->R15
  1345. .Lcommon_seh_tail:
  1346. mov 8(%rax),%rdi
  1347. mov 16(%rax),%rsi
  1348. mov %rax,152($context) # restore context->Rsp
  1349. mov %rsi,168($context) # restore context->Rsi
  1350. mov %rdi,176($context) # restore context->Rdi
  1351. mov 40($disp),%rdi # disp->ContextRecord
  1352. mov $context,%rsi # context
  1353. mov \$154,%ecx # sizeof(CONTEXT)
  1354. .long 0xa548f3fc # cld; rep movsq
  1355. mov $disp,%rsi
  1356. xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER
  1357. mov 8(%rsi),%rdx # arg2, disp->ImageBase
  1358. mov 0(%rsi),%r8 # arg3, disp->ControlPc
  1359. mov 16(%rsi),%r9 # arg4, disp->FunctionEntry
  1360. mov 40(%rsi),%r10 # disp->ContextRecord
  1361. lea 56(%rsi),%r11 # &disp->HandlerData
  1362. lea 24(%rsi),%r12 # &disp->EstablisherFrame
  1363. mov %r10,32(%rsp) # arg5
  1364. mov %r11,40(%rsp) # arg6
  1365. mov %r12,48(%rsp) # arg7
  1366. mov %rcx,56(%rsp) # arg8, (NULL)
  1367. call *__imp_RtlVirtualUnwind(%rip)
  1368. mov \$1,%eax # ExceptionContinueSearch
  1369. add \$64,%rsp
  1370. popfq
  1371. pop %r15
  1372. pop %r14
  1373. pop %r13
  1374. pop %r12
  1375. pop %rbp
  1376. pop %rbx
  1377. pop %rdi
  1378. pop %rsi
  1379. ret
  1380. .size sqr_handler,.-sqr_handler
  1381. .section .pdata
  1382. .align 4
  1383. .rva .LSEH_begin_bn_mul_mont
  1384. .rva .LSEH_end_bn_mul_mont
  1385. .rva .LSEH_info_bn_mul_mont
  1386. .rva .LSEH_begin_bn_mul4x_mont
  1387. .rva .LSEH_end_bn_mul4x_mont
  1388. .rva .LSEH_info_bn_mul4x_mont
  1389. .rva .LSEH_begin_bn_sqr8x_mont
  1390. .rva .LSEH_end_bn_sqr8x_mont
  1391. .rva .LSEH_info_bn_sqr8x_mont
  1392. ___
  1393. $code.=<<___ if ($addx);
  1394. .rva .LSEH_begin_bn_mulx4x_mont
  1395. .rva .LSEH_end_bn_mulx4x_mont
  1396. .rva .LSEH_info_bn_mulx4x_mont
  1397. ___
  1398. $code.=<<___;
  1399. .section .xdata
  1400. .align 8
  1401. .LSEH_info_bn_mul_mont:
  1402. .byte 9,0,0,0
  1403. .rva mul_handler
  1404. .rva .Lmul_body,.Lmul_epilogue # HandlerData[]
  1405. .LSEH_info_bn_mul4x_mont:
  1406. .byte 9,0,0,0
  1407. .rva mul_handler
  1408. .rva .Lmul4x_body,.Lmul4x_epilogue # HandlerData[]
  1409. .LSEH_info_bn_sqr8x_mont:
  1410. .byte 9,0,0,0
  1411. .rva sqr_handler
  1412. .rva .Lsqr8x_prologue,.Lsqr8x_body,.Lsqr8x_epilogue # HandlerData[]
  1413. .align 8
  1414. ___
  1415. $code.=<<___ if ($addx);
  1416. .LSEH_info_bn_mulx4x_mont:
  1417. .byte 9,0,0,0
  1418. .rva sqr_handler
  1419. .rva .Lmulx4x_prologue,.Lmulx4x_body,.Lmulx4x_epilogue # HandlerData[]
  1420. .align 8
  1421. ___
  1422. }
  1423. print $code;
  1424. close STDOUT or die "error closing STDOUT: $!";