ppc64-mont.pl 26 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088
  1. #!/usr/bin/env perl
  2. # ====================================================================
  3. # Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
  4. # project. The module is, however, dual licensed under OpenSSL and
  5. # CRYPTOGAMS licenses depending on where you obtain it. For further
  6. # details see http://www.openssl.org/~appro/cryptogams/.
  7. # ====================================================================
  8. # December 2007
  9. # The reason for undertaken effort is basically following. Even though
  10. # Power 6 CPU operates at incredible 4.7GHz clock frequency, its PKI
  11. # performance was observed to be less than impressive, essentially as
  12. # fast as 1.8GHz PPC970, or 2.6 times(!) slower than one would hope.
  13. # Well, it's not surprising that IBM had to make some sacrifices to
  14. # boost the clock frequency that much, but no overall improvement?
  15. # Having observed how much difference did switching to FPU make on
  16. # UltraSPARC, playing same stunt on Power 6 appeared appropriate...
  17. # Unfortunately the resulting performance improvement is not as
  18. # impressive, ~30%, and in absolute terms is still very far from what
  19. # one would expect from 4.7GHz CPU. There is a chance that I'm doing
  20. # something wrong, but in the lack of assembler level micro-profiling
  21. # data or at least decent platform guide I can't tell... Or better
  22. # results might be achieved with VMX... Anyway, this module provides
  23. # *worse* performance on other PowerPC implementations, ~40-15% slower
  24. # on PPC970 depending on key length and ~40% slower on Power 5 for all
  25. # key lengths. As it's obviously inappropriate as "best all-round"
  26. # alternative, it has to be complemented with run-time CPU family
  27. # detection. Oh! It should also be noted that unlike other PowerPC
  28. # implementation IALU ppc-mont.pl module performs *suboptimaly* on
  29. # >=1024-bit key lengths on Power 6. It should also be noted that
  30. # *everything* said so far applies to 64-bit builds! As far as 32-bit
  31. # application executed on 64-bit CPU goes, this module is likely to
  32. # become preferred choice, because it's easy to adapt it for such
  33. # case and *is* faster than 32-bit ppc-mont.pl on *all* processors.
  34. # February 2008
  35. # Micro-profiling assisted optimization results in ~15% improvement
  36. # over original ppc64-mont.pl version, or overall ~50% improvement
  37. # over ppc.pl module on Power 6. If compared to ppc-mont.pl on same
  38. # Power 6 CPU, this module is 5-150% faster depending on key length,
  39. # [hereafter] more for longer keys. But if compared to ppc-mont.pl
  40. # on 1.8GHz PPC970, it's only 5-55% faster. Still far from impressive
  41. # in absolute terms, but it's apparently the way Power 6 is...
  42. # December 2009
  43. # Adapted for 32-bit build this module delivers 25-120%, yes, more
  44. # than *twice* for longer keys, performance improvement over 32-bit
  45. # ppc-mont.pl on 1.8GHz PPC970. However! This implementation utilizes
  46. # even 64-bit integer operations and the trouble is that most PPC
  47. # operating systems don't preserve upper halves of general purpose
  48. # registers upon 32-bit signal delivery. They do preserve them upon
  49. # context switch, but not signalling:-( This means that asynchronous
  50. # signals have to be blocked upon entry to this subroutine. Signal
  51. # masking (and of course complementary unmasking) has quite an impact
  52. # on performance, naturally larger for shorter keys. It's so severe
  53. # that 512-bit key performance can be as low as 1/3 of expected one.
  54. # This is why this routine can be engaged for longer key operations
  55. # only on these OSes, see crypto/ppccap.c for further details. MacOS X
  56. # is an exception from this and doesn't require signal masking, and
  57. # that's where above improvement coefficients were collected. For
  58. # others alternative would be to break dependence on upper halves of
  59. # GPRs by sticking to 32-bit integer operations...
  60. $flavour = shift;
  61. if ($flavour =~ /32/) {
  62. $SIZE_T=4;
  63. $RZONE= 224;
  64. $fname= "bn_mul_mont_fpu64";
  65. $STUX= "stwux"; # store indexed and update
  66. $PUSH= "stw";
  67. $POP= "lwz";
  68. } elsif ($flavour =~ /64/) {
  69. $SIZE_T=8;
  70. $RZONE= 288;
  71. $fname= "bn_mul_mont_fpu64";
  72. # same as above, but 64-bit mnemonics...
  73. $STUX= "stdux"; # store indexed and update
  74. $PUSH= "std";
  75. $POP= "ld";
  76. } else { die "nonsense $flavour"; }
  77. $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
  78. ( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
  79. ( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
  80. die "can't locate ppc-xlate.pl";
  81. open STDOUT,"| $^X $xlate $flavour ".shift || die "can't call $xlate: $!";
  82. $FRAME=64; # padded frame header
  83. $TRANSFER=16*8;
  84. $carry="r0";
  85. $sp="r1";
  86. $toc="r2";
  87. $rp="r3"; $ovf="r3";
  88. $ap="r4";
  89. $bp="r5";
  90. $np="r6";
  91. $n0="r7";
  92. $num="r8";
  93. $rp="r9"; # $rp is reassigned
  94. $tp="r10";
  95. $j="r11";
  96. $i="r12";
  97. # non-volatile registers
  98. $nap_d="r22"; # interleaved ap and np in double format
  99. $a0="r23"; # ap[0]
  100. $t0="r24"; # temporary registers
  101. $t1="r25";
  102. $t2="r26";
  103. $t3="r27";
  104. $t4="r28";
  105. $t5="r29";
  106. $t6="r30";
  107. $t7="r31";
  108. # PPC offers enough register bank capacity to unroll inner loops twice
  109. #
  110. # ..A3A2A1A0
  111. # dcba
  112. # -----------
  113. # A0a
  114. # A0b
  115. # A0c
  116. # A0d
  117. # A1a
  118. # A1b
  119. # A1c
  120. # A1d
  121. # A2a
  122. # A2b
  123. # A2c
  124. # A2d
  125. # A3a
  126. # A3b
  127. # A3c
  128. # A3d
  129. # ..a
  130. # ..b
  131. #
  132. $ba="f0"; $bb="f1"; $bc="f2"; $bd="f3";
  133. $na="f4"; $nb="f5"; $nc="f6"; $nd="f7";
  134. $dota="f8"; $dotb="f9";
  135. $A0="f10"; $A1="f11"; $A2="f12"; $A3="f13";
  136. $N0="f20"; $N1="f21"; $N2="f22"; $N3="f23";
  137. $T0a="f24"; $T0b="f25";
  138. $T1a="f26"; $T1b="f27";
  139. $T2a="f28"; $T2b="f29";
  140. $T3a="f30"; $T3b="f31";
  141. # sp----------->+-------------------------------+
  142. # | saved sp |
  143. # +-------------------------------+
  144. # . .
  145. # +64 +-------------------------------+
  146. # | 16 gpr<->fpr transfer zone |
  147. # . .
  148. # . .
  149. # +16*8 +-------------------------------+
  150. # | __int64 tmp[-1] |
  151. # +-------------------------------+
  152. # | __int64 tmp[num] |
  153. # . .
  154. # . .
  155. # . .
  156. # +(num+1)*8 +-------------------------------+
  157. # | padding to 64 byte boundary |
  158. # . .
  159. # +X +-------------------------------+
  160. # | double nap_d[4*num] |
  161. # . .
  162. # . .
  163. # . .
  164. # +-------------------------------+
  165. # . .
  166. # -12*size_t +-------------------------------+
  167. # | 10 saved gpr, r22-r31 |
  168. # . .
  169. # . .
  170. # -12*8 +-------------------------------+
  171. # | 12 saved fpr, f20-f31 |
  172. # . .
  173. # . .
  174. # +-------------------------------+
  175. $code=<<___;
  176. .machine "any"
  177. .text
  178. .globl .$fname
  179. .align 5
  180. .$fname:
  181. cmpwi $num,`3*8/$SIZE_T`
  182. mr $rp,r3 ; $rp is reassigned
  183. li r3,0 ; possible "not handled" return code
  184. bltlr-
  185. andi. r0,$num,`16/$SIZE_T-1` ; $num has to be "even"
  186. bnelr-
  187. slwi $num,$num,`log($SIZE_T)/log(2)` ; num*=sizeof(BN_LONG)
  188. li $i,-4096
  189. slwi $tp,$num,2 ; place for {an}p_{lh}[num], i.e. 4*num
  190. add $tp,$tp,$num ; place for tp[num+1]
  191. addi $tp,$tp,`$FRAME+$TRANSFER+8+64+$RZONE`
  192. subf $tp,$tp,$sp ; $sp-$tp
  193. and $tp,$tp,$i ; minimize TLB usage
  194. subf $tp,$sp,$tp ; $tp-$sp
  195. mr $i,$sp
  196. $STUX $sp,$sp,$tp ; alloca
  197. $PUSH r22,`-12*8-10*$SIZE_T`($i)
  198. $PUSH r23,`-12*8-9*$SIZE_T`($i)
  199. $PUSH r24,`-12*8-8*$SIZE_T`($i)
  200. $PUSH r25,`-12*8-7*$SIZE_T`($i)
  201. $PUSH r26,`-12*8-6*$SIZE_T`($i)
  202. $PUSH r27,`-12*8-5*$SIZE_T`($i)
  203. $PUSH r28,`-12*8-4*$SIZE_T`($i)
  204. $PUSH r29,`-12*8-3*$SIZE_T`($i)
  205. $PUSH r30,`-12*8-2*$SIZE_T`($i)
  206. $PUSH r31,`-12*8-1*$SIZE_T`($i)
  207. stfd f20,`-12*8`($i)
  208. stfd f21,`-11*8`($i)
  209. stfd f22,`-10*8`($i)
  210. stfd f23,`-9*8`($i)
  211. stfd f24,`-8*8`($i)
  212. stfd f25,`-7*8`($i)
  213. stfd f26,`-6*8`($i)
  214. stfd f27,`-5*8`($i)
  215. stfd f28,`-4*8`($i)
  216. stfd f29,`-3*8`($i)
  217. stfd f30,`-2*8`($i)
  218. stfd f31,`-1*8`($i)
  219. ___
  220. $code.=<<___ if ($SIZE_T==8);
  221. ld $a0,0($ap) ; pull ap[0] value
  222. ld $n0,0($n0) ; pull n0[0] value
  223. ld $t3,0($bp) ; bp[0]
  224. ___
  225. $code.=<<___ if ($SIZE_T==4);
  226. mr $t1,$n0
  227. lwz $a0,0($ap) ; pull ap[0,1] value
  228. lwz $t0,4($ap)
  229. lwz $n0,0($t1) ; pull n0[0,1] value
  230. lwz $t1,4($t1)
  231. lwz $t3,0($bp) ; bp[0,1]
  232. lwz $t2,4($bp)
  233. insrdi $a0,$t0,32,0
  234. insrdi $n0,$t1,32,0
  235. insrdi $t3,$t2,32,0
  236. ___
  237. $code.=<<___;
  238. addi $tp,$sp,`$FRAME+$TRANSFER+8+64`
  239. li $i,-64
  240. add $nap_d,$tp,$num
  241. and $nap_d,$nap_d,$i ; align to 64 bytes
  242. mulld $t7,$a0,$t3 ; ap[0]*bp[0]
  243. ; nap_d is off by 1, because it's used with stfdu/lfdu
  244. addi $nap_d,$nap_d,-8
  245. srwi $j,$num,`3+1` ; counter register, num/2
  246. mulld $t7,$t7,$n0 ; tp[0]*n0
  247. addi $j,$j,-1
  248. addi $tp,$sp,`$FRAME+$TRANSFER-8`
  249. li $carry,0
  250. mtctr $j
  251. ; transfer bp[0] to FPU as 4x16-bit values
  252. extrdi $t0,$t3,16,48
  253. extrdi $t1,$t3,16,32
  254. extrdi $t2,$t3,16,16
  255. extrdi $t3,$t3,16,0
  256. std $t0,`$FRAME+0`($sp)
  257. std $t1,`$FRAME+8`($sp)
  258. std $t2,`$FRAME+16`($sp)
  259. std $t3,`$FRAME+24`($sp)
  260. ; transfer (ap[0]*bp[0])*n0 to FPU as 4x16-bit values
  261. extrdi $t4,$t7,16,48
  262. extrdi $t5,$t7,16,32
  263. extrdi $t6,$t7,16,16
  264. extrdi $t7,$t7,16,0
  265. std $t4,`$FRAME+32`($sp)
  266. std $t5,`$FRAME+40`($sp)
  267. std $t6,`$FRAME+48`($sp)
  268. std $t7,`$FRAME+56`($sp)
  269. ___
  270. $code.=<<___ if ($SIZE_T==8);
  271. lwz $t0,4($ap) ; load a[j] as 32-bit word pair
  272. lwz $t1,0($ap)
  273. lwz $t2,12($ap) ; load a[j+1] as 32-bit word pair
  274. lwz $t3,8($ap)
  275. lwz $t4,4($np) ; load n[j] as 32-bit word pair
  276. lwz $t5,0($np)
  277. lwz $t6,12($np) ; load n[j+1] as 32-bit word pair
  278. lwz $t7,8($np)
  279. ___
  280. $code.=<<___ if ($SIZE_T==4);
  281. lwz $t0,0($ap) ; load a[j..j+3] as 32-bit word pairs
  282. lwz $t1,4($ap)
  283. lwz $t2,8($ap)
  284. lwz $t3,12($ap)
  285. lwz $t4,0($np) ; load n[j..j+3] as 32-bit word pairs
  286. lwz $t5,4($np)
  287. lwz $t6,8($np)
  288. lwz $t7,12($np)
  289. ___
  290. $code.=<<___;
  291. lfd $ba,`$FRAME+0`($sp)
  292. lfd $bb,`$FRAME+8`($sp)
  293. lfd $bc,`$FRAME+16`($sp)
  294. lfd $bd,`$FRAME+24`($sp)
  295. lfd $na,`$FRAME+32`($sp)
  296. lfd $nb,`$FRAME+40`($sp)
  297. lfd $nc,`$FRAME+48`($sp)
  298. lfd $nd,`$FRAME+56`($sp)
  299. std $t0,`$FRAME+64`($sp)
  300. std $t1,`$FRAME+72`($sp)
  301. std $t2,`$FRAME+80`($sp)
  302. std $t3,`$FRAME+88`($sp)
  303. std $t4,`$FRAME+96`($sp)
  304. std $t5,`$FRAME+104`($sp)
  305. std $t6,`$FRAME+112`($sp)
  306. std $t7,`$FRAME+120`($sp)
  307. fcfid $ba,$ba
  308. fcfid $bb,$bb
  309. fcfid $bc,$bc
  310. fcfid $bd,$bd
  311. fcfid $na,$na
  312. fcfid $nb,$nb
  313. fcfid $nc,$nc
  314. fcfid $nd,$nd
  315. lfd $A0,`$FRAME+64`($sp)
  316. lfd $A1,`$FRAME+72`($sp)
  317. lfd $A2,`$FRAME+80`($sp)
  318. lfd $A3,`$FRAME+88`($sp)
  319. lfd $N0,`$FRAME+96`($sp)
  320. lfd $N1,`$FRAME+104`($sp)
  321. lfd $N2,`$FRAME+112`($sp)
  322. lfd $N3,`$FRAME+120`($sp)
  323. fcfid $A0,$A0
  324. fcfid $A1,$A1
  325. fcfid $A2,$A2
  326. fcfid $A3,$A3
  327. fcfid $N0,$N0
  328. fcfid $N1,$N1
  329. fcfid $N2,$N2
  330. fcfid $N3,$N3
  331. addi $ap,$ap,16
  332. addi $np,$np,16
  333. fmul $T1a,$A1,$ba
  334. fmul $T1b,$A1,$bb
  335. stfd $A0,8($nap_d) ; save a[j] in double format
  336. stfd $A1,16($nap_d)
  337. fmul $T2a,$A2,$ba
  338. fmul $T2b,$A2,$bb
  339. stfd $A2,24($nap_d) ; save a[j+1] in double format
  340. stfd $A3,32($nap_d)
  341. fmul $T3a,$A3,$ba
  342. fmul $T3b,$A3,$bb
  343. stfd $N0,40($nap_d) ; save n[j] in double format
  344. stfd $N1,48($nap_d)
  345. fmul $T0a,$A0,$ba
  346. fmul $T0b,$A0,$bb
  347. stfd $N2,56($nap_d) ; save n[j+1] in double format
  348. stfdu $N3,64($nap_d)
  349. fmadd $T1a,$A0,$bc,$T1a
  350. fmadd $T1b,$A0,$bd,$T1b
  351. fmadd $T2a,$A1,$bc,$T2a
  352. fmadd $T2b,$A1,$bd,$T2b
  353. fmadd $T3a,$A2,$bc,$T3a
  354. fmadd $T3b,$A2,$bd,$T3b
  355. fmul $dota,$A3,$bc
  356. fmul $dotb,$A3,$bd
  357. fmadd $T1a,$N1,$na,$T1a
  358. fmadd $T1b,$N1,$nb,$T1b
  359. fmadd $T2a,$N2,$na,$T2a
  360. fmadd $T2b,$N2,$nb,$T2b
  361. fmadd $T3a,$N3,$na,$T3a
  362. fmadd $T3b,$N3,$nb,$T3b
  363. fmadd $T0a,$N0,$na,$T0a
  364. fmadd $T0b,$N0,$nb,$T0b
  365. fmadd $T1a,$N0,$nc,$T1a
  366. fmadd $T1b,$N0,$nd,$T1b
  367. fmadd $T2a,$N1,$nc,$T2a
  368. fmadd $T2b,$N1,$nd,$T2b
  369. fmadd $T3a,$N2,$nc,$T3a
  370. fmadd $T3b,$N2,$nd,$T3b
  371. fmadd $dota,$N3,$nc,$dota
  372. fmadd $dotb,$N3,$nd,$dotb
  373. fctid $T0a,$T0a
  374. fctid $T0b,$T0b
  375. fctid $T1a,$T1a
  376. fctid $T1b,$T1b
  377. fctid $T2a,$T2a
  378. fctid $T2b,$T2b
  379. fctid $T3a,$T3a
  380. fctid $T3b,$T3b
  381. stfd $T0a,`$FRAME+0`($sp)
  382. stfd $T0b,`$FRAME+8`($sp)
  383. stfd $T1a,`$FRAME+16`($sp)
  384. stfd $T1b,`$FRAME+24`($sp)
  385. stfd $T2a,`$FRAME+32`($sp)
  386. stfd $T2b,`$FRAME+40`($sp)
  387. stfd $T3a,`$FRAME+48`($sp)
  388. stfd $T3b,`$FRAME+56`($sp)
  389. .align 5
  390. L1st:
  391. ___
  392. $code.=<<___ if ($SIZE_T==8);
  393. lwz $t0,4($ap) ; load a[j] as 32-bit word pair
  394. lwz $t1,0($ap)
  395. lwz $t2,12($ap) ; load a[j+1] as 32-bit word pair
  396. lwz $t3,8($ap)
  397. lwz $t4,4($np) ; load n[j] as 32-bit word pair
  398. lwz $t5,0($np)
  399. lwz $t6,12($np) ; load n[j+1] as 32-bit word pair
  400. lwz $t7,8($np)
  401. ___
  402. $code.=<<___ if ($SIZE_T==4);
  403. lwz $t0,0($ap) ; load a[j..j+3] as 32-bit word pairs
  404. lwz $t1,4($ap)
  405. lwz $t2,8($ap)
  406. lwz $t3,12($ap)
  407. lwz $t4,0($np) ; load n[j..j+3] as 32-bit word pairs
  408. lwz $t5,4($np)
  409. lwz $t6,8($np)
  410. lwz $t7,12($np)
  411. ___
  412. $code.=<<___;
  413. std $t0,`$FRAME+64`($sp)
  414. std $t1,`$FRAME+72`($sp)
  415. std $t2,`$FRAME+80`($sp)
  416. std $t3,`$FRAME+88`($sp)
  417. std $t4,`$FRAME+96`($sp)
  418. std $t5,`$FRAME+104`($sp)
  419. std $t6,`$FRAME+112`($sp)
  420. std $t7,`$FRAME+120`($sp)
  421. ld $t0,`$FRAME+0`($sp)
  422. ld $t1,`$FRAME+8`($sp)
  423. ld $t2,`$FRAME+16`($sp)
  424. ld $t3,`$FRAME+24`($sp)
  425. ld $t4,`$FRAME+32`($sp)
  426. ld $t5,`$FRAME+40`($sp)
  427. ld $t6,`$FRAME+48`($sp)
  428. ld $t7,`$FRAME+56`($sp)
  429. lfd $A0,`$FRAME+64`($sp)
  430. lfd $A1,`$FRAME+72`($sp)
  431. lfd $A2,`$FRAME+80`($sp)
  432. lfd $A3,`$FRAME+88`($sp)
  433. lfd $N0,`$FRAME+96`($sp)
  434. lfd $N1,`$FRAME+104`($sp)
  435. lfd $N2,`$FRAME+112`($sp)
  436. lfd $N3,`$FRAME+120`($sp)
  437. fcfid $A0,$A0
  438. fcfid $A1,$A1
  439. fcfid $A2,$A2
  440. fcfid $A3,$A3
  441. fcfid $N0,$N0
  442. fcfid $N1,$N1
  443. fcfid $N2,$N2
  444. fcfid $N3,$N3
  445. addi $ap,$ap,16
  446. addi $np,$np,16
  447. fmul $T1a,$A1,$ba
  448. fmul $T1b,$A1,$bb
  449. fmul $T2a,$A2,$ba
  450. fmul $T2b,$A2,$bb
  451. stfd $A0,8($nap_d) ; save a[j] in double format
  452. stfd $A1,16($nap_d)
  453. fmul $T3a,$A3,$ba
  454. fmul $T3b,$A3,$bb
  455. fmadd $T0a,$A0,$ba,$dota
  456. fmadd $T0b,$A0,$bb,$dotb
  457. stfd $A2,24($nap_d) ; save a[j+1] in double format
  458. stfd $A3,32($nap_d)
  459. fmadd $T1a,$A0,$bc,$T1a
  460. fmadd $T1b,$A0,$bd,$T1b
  461. fmadd $T2a,$A1,$bc,$T2a
  462. fmadd $T2b,$A1,$bd,$T2b
  463. stfd $N0,40($nap_d) ; save n[j] in double format
  464. stfd $N1,48($nap_d)
  465. fmadd $T3a,$A2,$bc,$T3a
  466. fmadd $T3b,$A2,$bd,$T3b
  467. add $t0,$t0,$carry ; can not overflow
  468. fmul $dota,$A3,$bc
  469. fmul $dotb,$A3,$bd
  470. stfd $N2,56($nap_d) ; save n[j+1] in double format
  471. stfdu $N3,64($nap_d)
  472. srdi $carry,$t0,16
  473. add $t1,$t1,$carry
  474. srdi $carry,$t1,16
  475. fmadd $T1a,$N1,$na,$T1a
  476. fmadd $T1b,$N1,$nb,$T1b
  477. insrdi $t0,$t1,16,32
  478. fmadd $T2a,$N2,$na,$T2a
  479. fmadd $T2b,$N2,$nb,$T2b
  480. add $t2,$t2,$carry
  481. fmadd $T3a,$N3,$na,$T3a
  482. fmadd $T3b,$N3,$nb,$T3b
  483. srdi $carry,$t2,16
  484. fmadd $T0a,$N0,$na,$T0a
  485. fmadd $T0b,$N0,$nb,$T0b
  486. insrdi $t0,$t2,16,16
  487. add $t3,$t3,$carry
  488. srdi $carry,$t3,16
  489. fmadd $T1a,$N0,$nc,$T1a
  490. fmadd $T1b,$N0,$nd,$T1b
  491. insrdi $t0,$t3,16,0 ; 0..63 bits
  492. fmadd $T2a,$N1,$nc,$T2a
  493. fmadd $T2b,$N1,$nd,$T2b
  494. add $t4,$t4,$carry
  495. fmadd $T3a,$N2,$nc,$T3a
  496. fmadd $T3b,$N2,$nd,$T3b
  497. srdi $carry,$t4,16
  498. fmadd $dota,$N3,$nc,$dota
  499. fmadd $dotb,$N3,$nd,$dotb
  500. add $t5,$t5,$carry
  501. srdi $carry,$t5,16
  502. insrdi $t4,$t5,16,32
  503. fctid $T0a,$T0a
  504. fctid $T0b,$T0b
  505. add $t6,$t6,$carry
  506. fctid $T1a,$T1a
  507. fctid $T1b,$T1b
  508. srdi $carry,$t6,16
  509. fctid $T2a,$T2a
  510. fctid $T2b,$T2b
  511. insrdi $t4,$t6,16,16
  512. fctid $T3a,$T3a
  513. fctid $T3b,$T3b
  514. add $t7,$t7,$carry
  515. insrdi $t4,$t7,16,0 ; 64..127 bits
  516. srdi $carry,$t7,16 ; upper 33 bits
  517. stfd $T0a,`$FRAME+0`($sp)
  518. stfd $T0b,`$FRAME+8`($sp)
  519. stfd $T1a,`$FRAME+16`($sp)
  520. stfd $T1b,`$FRAME+24`($sp)
  521. stfd $T2a,`$FRAME+32`($sp)
  522. stfd $T2b,`$FRAME+40`($sp)
  523. stfd $T3a,`$FRAME+48`($sp)
  524. stfd $T3b,`$FRAME+56`($sp)
  525. std $t0,8($tp) ; tp[j-1]
  526. stdu $t4,16($tp) ; tp[j]
  527. bdnz L1st
  528. fctid $dota,$dota
  529. fctid $dotb,$dotb
  530. ld $t0,`$FRAME+0`($sp)
  531. ld $t1,`$FRAME+8`($sp)
  532. ld $t2,`$FRAME+16`($sp)
  533. ld $t3,`$FRAME+24`($sp)
  534. ld $t4,`$FRAME+32`($sp)
  535. ld $t5,`$FRAME+40`($sp)
  536. ld $t6,`$FRAME+48`($sp)
  537. ld $t7,`$FRAME+56`($sp)
  538. stfd $dota,`$FRAME+64`($sp)
  539. stfd $dotb,`$FRAME+72`($sp)
  540. add $t0,$t0,$carry ; can not overflow
  541. srdi $carry,$t0,16
  542. add $t1,$t1,$carry
  543. srdi $carry,$t1,16
  544. insrdi $t0,$t1,16,32
  545. add $t2,$t2,$carry
  546. srdi $carry,$t2,16
  547. insrdi $t0,$t2,16,16
  548. add $t3,$t3,$carry
  549. srdi $carry,$t3,16
  550. insrdi $t0,$t3,16,0 ; 0..63 bits
  551. add $t4,$t4,$carry
  552. srdi $carry,$t4,16
  553. add $t5,$t5,$carry
  554. srdi $carry,$t5,16
  555. insrdi $t4,$t5,16,32
  556. add $t6,$t6,$carry
  557. srdi $carry,$t6,16
  558. insrdi $t4,$t6,16,16
  559. add $t7,$t7,$carry
  560. insrdi $t4,$t7,16,0 ; 64..127 bits
  561. srdi $carry,$t7,16 ; upper 33 bits
  562. ld $t6,`$FRAME+64`($sp)
  563. ld $t7,`$FRAME+72`($sp)
  564. std $t0,8($tp) ; tp[j-1]
  565. stdu $t4,16($tp) ; tp[j]
  566. add $t6,$t6,$carry ; can not overflow
  567. srdi $carry,$t6,16
  568. add $t7,$t7,$carry
  569. insrdi $t6,$t7,48,0
  570. srdi $ovf,$t7,48
  571. std $t6,8($tp) ; tp[num-1]
  572. slwi $t7,$num,2
  573. subf $nap_d,$t7,$nap_d ; rewind pointer
  574. li $i,8 ; i=1
  575. .align 5
  576. Louter:
  577. ___
  578. $code.=<<___ if ($SIZE_T==8);
  579. ldx $t3,$bp,$i ; bp[i]
  580. ___
  581. $code.=<<___ if ($SIZE_T==4);
  582. add $t0,$bp,$i
  583. lwz $t3,0($t0) ; bp[i,i+1]
  584. lwz $t0,4($t0)
  585. insrdi $t3,$t0,32,0
  586. ___
  587. $code.=<<___;
  588. ld $t6,`$FRAME+$TRANSFER+8`($sp) ; tp[0]
  589. mulld $t7,$a0,$t3 ; ap[0]*bp[i]
  590. addi $tp,$sp,`$FRAME+$TRANSFER`
  591. add $t7,$t7,$t6 ; ap[0]*bp[i]+tp[0]
  592. li $carry,0
  593. mulld $t7,$t7,$n0 ; tp[0]*n0
  594. mtctr $j
  595. ; transfer bp[i] to FPU as 4x16-bit values
  596. extrdi $t0,$t3,16,48
  597. extrdi $t1,$t3,16,32
  598. extrdi $t2,$t3,16,16
  599. extrdi $t3,$t3,16,0
  600. std $t0,`$FRAME+0`($sp)
  601. std $t1,`$FRAME+8`($sp)
  602. std $t2,`$FRAME+16`($sp)
  603. std $t3,`$FRAME+24`($sp)
  604. ; transfer (ap[0]*bp[i]+tp[0])*n0 to FPU as 4x16-bit values
  605. extrdi $t4,$t7,16,48
  606. extrdi $t5,$t7,16,32
  607. extrdi $t6,$t7,16,16
  608. extrdi $t7,$t7,16,0
  609. std $t4,`$FRAME+32`($sp)
  610. std $t5,`$FRAME+40`($sp)
  611. std $t6,`$FRAME+48`($sp)
  612. std $t7,`$FRAME+56`($sp)
  613. lfd $A0,8($nap_d) ; load a[j] in double format
  614. lfd $A1,16($nap_d)
  615. lfd $A2,24($nap_d) ; load a[j+1] in double format
  616. lfd $A3,32($nap_d)
  617. lfd $N0,40($nap_d) ; load n[j] in double format
  618. lfd $N1,48($nap_d)
  619. lfd $N2,56($nap_d) ; load n[j+1] in double format
  620. lfdu $N3,64($nap_d)
  621. lfd $ba,`$FRAME+0`($sp)
  622. lfd $bb,`$FRAME+8`($sp)
  623. lfd $bc,`$FRAME+16`($sp)
  624. lfd $bd,`$FRAME+24`($sp)
  625. lfd $na,`$FRAME+32`($sp)
  626. lfd $nb,`$FRAME+40`($sp)
  627. lfd $nc,`$FRAME+48`($sp)
  628. lfd $nd,`$FRAME+56`($sp)
  629. fcfid $ba,$ba
  630. fcfid $bb,$bb
  631. fcfid $bc,$bc
  632. fcfid $bd,$bd
  633. fcfid $na,$na
  634. fcfid $nb,$nb
  635. fcfid $nc,$nc
  636. fcfid $nd,$nd
  637. fmul $T1a,$A1,$ba
  638. fmul $T1b,$A1,$bb
  639. fmul $T2a,$A2,$ba
  640. fmul $T2b,$A2,$bb
  641. fmul $T3a,$A3,$ba
  642. fmul $T3b,$A3,$bb
  643. fmul $T0a,$A0,$ba
  644. fmul $T0b,$A0,$bb
  645. fmadd $T1a,$A0,$bc,$T1a
  646. fmadd $T1b,$A0,$bd,$T1b
  647. fmadd $T2a,$A1,$bc,$T2a
  648. fmadd $T2b,$A1,$bd,$T2b
  649. fmadd $T3a,$A2,$bc,$T3a
  650. fmadd $T3b,$A2,$bd,$T3b
  651. fmul $dota,$A3,$bc
  652. fmul $dotb,$A3,$bd
  653. fmadd $T1a,$N1,$na,$T1a
  654. fmadd $T1b,$N1,$nb,$T1b
  655. lfd $A0,8($nap_d) ; load a[j] in double format
  656. lfd $A1,16($nap_d)
  657. fmadd $T2a,$N2,$na,$T2a
  658. fmadd $T2b,$N2,$nb,$T2b
  659. lfd $A2,24($nap_d) ; load a[j+1] in double format
  660. lfd $A3,32($nap_d)
  661. fmadd $T3a,$N3,$na,$T3a
  662. fmadd $T3b,$N3,$nb,$T3b
  663. fmadd $T0a,$N0,$na,$T0a
  664. fmadd $T0b,$N0,$nb,$T0b
  665. fmadd $T1a,$N0,$nc,$T1a
  666. fmadd $T1b,$N0,$nd,$T1b
  667. fmadd $T2a,$N1,$nc,$T2a
  668. fmadd $T2b,$N1,$nd,$T2b
  669. fmadd $T3a,$N2,$nc,$T3a
  670. fmadd $T3b,$N2,$nd,$T3b
  671. fmadd $dota,$N3,$nc,$dota
  672. fmadd $dotb,$N3,$nd,$dotb
  673. fctid $T0a,$T0a
  674. fctid $T0b,$T0b
  675. fctid $T1a,$T1a
  676. fctid $T1b,$T1b
  677. fctid $T2a,$T2a
  678. fctid $T2b,$T2b
  679. fctid $T3a,$T3a
  680. fctid $T3b,$T3b
  681. stfd $T0a,`$FRAME+0`($sp)
  682. stfd $T0b,`$FRAME+8`($sp)
  683. stfd $T1a,`$FRAME+16`($sp)
  684. stfd $T1b,`$FRAME+24`($sp)
  685. stfd $T2a,`$FRAME+32`($sp)
  686. stfd $T2b,`$FRAME+40`($sp)
  687. stfd $T3a,`$FRAME+48`($sp)
  688. stfd $T3b,`$FRAME+56`($sp)
  689. .align 5
  690. Linner:
  691. fmul $T1a,$A1,$ba
  692. fmul $T1b,$A1,$bb
  693. fmul $T2a,$A2,$ba
  694. fmul $T2b,$A2,$bb
  695. lfd $N0,40($nap_d) ; load n[j] in double format
  696. lfd $N1,48($nap_d)
  697. fmul $T3a,$A3,$ba
  698. fmul $T3b,$A3,$bb
  699. fmadd $T0a,$A0,$ba,$dota
  700. fmadd $T0b,$A0,$bb,$dotb
  701. lfd $N2,56($nap_d) ; load n[j+1] in double format
  702. lfdu $N3,64($nap_d)
  703. fmadd $T1a,$A0,$bc,$T1a
  704. fmadd $T1b,$A0,$bd,$T1b
  705. fmadd $T2a,$A1,$bc,$T2a
  706. fmadd $T2b,$A1,$bd,$T2b
  707. lfd $A0,8($nap_d) ; load a[j] in double format
  708. lfd $A1,16($nap_d)
  709. fmadd $T3a,$A2,$bc,$T3a
  710. fmadd $T3b,$A2,$bd,$T3b
  711. fmul $dota,$A3,$bc
  712. fmul $dotb,$A3,$bd
  713. lfd $A2,24($nap_d) ; load a[j+1] in double format
  714. lfd $A3,32($nap_d)
  715. fmadd $T1a,$N1,$na,$T1a
  716. fmadd $T1b,$N1,$nb,$T1b
  717. ld $t0,`$FRAME+0`($sp)
  718. ld $t1,`$FRAME+8`($sp)
  719. fmadd $T2a,$N2,$na,$T2a
  720. fmadd $T2b,$N2,$nb,$T2b
  721. ld $t2,`$FRAME+16`($sp)
  722. ld $t3,`$FRAME+24`($sp)
  723. fmadd $T3a,$N3,$na,$T3a
  724. fmadd $T3b,$N3,$nb,$T3b
  725. add $t0,$t0,$carry ; can not overflow
  726. ld $t4,`$FRAME+32`($sp)
  727. ld $t5,`$FRAME+40`($sp)
  728. fmadd $T0a,$N0,$na,$T0a
  729. fmadd $T0b,$N0,$nb,$T0b
  730. srdi $carry,$t0,16
  731. add $t1,$t1,$carry
  732. srdi $carry,$t1,16
  733. ld $t6,`$FRAME+48`($sp)
  734. ld $t7,`$FRAME+56`($sp)
  735. fmadd $T1a,$N0,$nc,$T1a
  736. fmadd $T1b,$N0,$nd,$T1b
  737. insrdi $t0,$t1,16,32
  738. ld $t1,8($tp) ; tp[j]
  739. fmadd $T2a,$N1,$nc,$T2a
  740. fmadd $T2b,$N1,$nd,$T2b
  741. add $t2,$t2,$carry
  742. fmadd $T3a,$N2,$nc,$T3a
  743. fmadd $T3b,$N2,$nd,$T3b
  744. srdi $carry,$t2,16
  745. insrdi $t0,$t2,16,16
  746. fmadd $dota,$N3,$nc,$dota
  747. fmadd $dotb,$N3,$nd,$dotb
  748. add $t3,$t3,$carry
  749. ldu $t2,16($tp) ; tp[j+1]
  750. srdi $carry,$t3,16
  751. insrdi $t0,$t3,16,0 ; 0..63 bits
  752. add $t4,$t4,$carry
  753. fctid $T0a,$T0a
  754. fctid $T0b,$T0b
  755. srdi $carry,$t4,16
  756. fctid $T1a,$T1a
  757. fctid $T1b,$T1b
  758. add $t5,$t5,$carry
  759. fctid $T2a,$T2a
  760. fctid $T2b,$T2b
  761. srdi $carry,$t5,16
  762. insrdi $t4,$t5,16,32
  763. fctid $T3a,$T3a
  764. fctid $T3b,$T3b
  765. add $t6,$t6,$carry
  766. srdi $carry,$t6,16
  767. insrdi $t4,$t6,16,16
  768. stfd $T0a,`$FRAME+0`($sp)
  769. stfd $T0b,`$FRAME+8`($sp)
  770. add $t7,$t7,$carry
  771. addc $t3,$t0,$t1
  772. ___
  773. $code.=<<___ if ($SIZE_T==4); # adjust XER[CA]
  774. extrdi $t0,$t0,32,0
  775. extrdi $t1,$t1,32,0
  776. adde $t0,$t0,$t1
  777. ___
  778. $code.=<<___;
  779. stfd $T1a,`$FRAME+16`($sp)
  780. stfd $T1b,`$FRAME+24`($sp)
  781. insrdi $t4,$t7,16,0 ; 64..127 bits
  782. srdi $carry,$t7,16 ; upper 33 bits
  783. stfd $T2a,`$FRAME+32`($sp)
  784. stfd $T2b,`$FRAME+40`($sp)
  785. adde $t5,$t4,$t2
  786. ___
  787. $code.=<<___ if ($SIZE_T==4); # adjust XER[CA]
  788. extrdi $t4,$t4,32,0
  789. extrdi $t2,$t2,32,0
  790. adde $t4,$t4,$t2
  791. ___
  792. $code.=<<___;
  793. stfd $T3a,`$FRAME+48`($sp)
  794. stfd $T3b,`$FRAME+56`($sp)
  795. addze $carry,$carry
  796. std $t3,-16($tp) ; tp[j-1]
  797. std $t5,-8($tp) ; tp[j]
  798. bdnz Linner
  799. fctid $dota,$dota
  800. fctid $dotb,$dotb
  801. ld $t0,`$FRAME+0`($sp)
  802. ld $t1,`$FRAME+8`($sp)
  803. ld $t2,`$FRAME+16`($sp)
  804. ld $t3,`$FRAME+24`($sp)
  805. ld $t4,`$FRAME+32`($sp)
  806. ld $t5,`$FRAME+40`($sp)
  807. ld $t6,`$FRAME+48`($sp)
  808. ld $t7,`$FRAME+56`($sp)
  809. stfd $dota,`$FRAME+64`($sp)
  810. stfd $dotb,`$FRAME+72`($sp)
  811. add $t0,$t0,$carry ; can not overflow
  812. srdi $carry,$t0,16
  813. add $t1,$t1,$carry
  814. srdi $carry,$t1,16
  815. insrdi $t0,$t1,16,32
  816. add $t2,$t2,$carry
  817. ld $t1,8($tp) ; tp[j]
  818. srdi $carry,$t2,16
  819. insrdi $t0,$t2,16,16
  820. add $t3,$t3,$carry
  821. ldu $t2,16($tp) ; tp[j+1]
  822. srdi $carry,$t3,16
  823. insrdi $t0,$t3,16,0 ; 0..63 bits
  824. add $t4,$t4,$carry
  825. srdi $carry,$t4,16
  826. add $t5,$t5,$carry
  827. srdi $carry,$t5,16
  828. insrdi $t4,$t5,16,32
  829. add $t6,$t6,$carry
  830. srdi $carry,$t6,16
  831. insrdi $t4,$t6,16,16
  832. add $t7,$t7,$carry
  833. insrdi $t4,$t7,16,0 ; 64..127 bits
  834. srdi $carry,$t7,16 ; upper 33 bits
  835. ld $t6,`$FRAME+64`($sp)
  836. ld $t7,`$FRAME+72`($sp)
  837. addc $t3,$t0,$t1
  838. ___
  839. $code.=<<___ if ($SIZE_T==4); # adjust XER[CA]
  840. extrdi $t0,$t0,32,0
  841. extrdi $t1,$t1,32,0
  842. adde $t0,$t0,$t1
  843. ___
  844. $code.=<<___;
  845. adde $t5,$t4,$t2
  846. ___
  847. $code.=<<___ if ($SIZE_T==4); # adjust XER[CA]
  848. extrdi $t4,$t4,32,0
  849. extrdi $t2,$t2,32,0
  850. adde $t4,$t4,$t2
  851. ___
  852. $code.=<<___;
  853. addze $carry,$carry
  854. std $t3,-16($tp) ; tp[j-1]
  855. std $t5,-8($tp) ; tp[j]
  856. add $carry,$carry,$ovf ; comsume upmost overflow
  857. add $t6,$t6,$carry ; can not overflow
  858. srdi $carry,$t6,16
  859. add $t7,$t7,$carry
  860. insrdi $t6,$t7,48,0
  861. srdi $ovf,$t7,48
  862. std $t6,0($tp) ; tp[num-1]
  863. slwi $t7,$num,2
  864. addi $i,$i,8
  865. subf $nap_d,$t7,$nap_d ; rewind pointer
  866. cmpw $i,$num
  867. blt- Louter
  868. ___
  869. $code.=<<___ if ($SIZE_T==8);
  870. subf $np,$num,$np ; rewind np
  871. addi $j,$j,1 ; restore counter
  872. subfc $i,$i,$i ; j=0 and "clear" XER[CA]
  873. addi $tp,$sp,`$FRAME+$TRANSFER+8`
  874. addi $t4,$sp,`$FRAME+$TRANSFER+16`
  875. addi $t5,$np,8
  876. addi $t6,$rp,8
  877. mtctr $j
  878. .align 4
  879. Lsub: ldx $t0,$tp,$i
  880. ldx $t1,$np,$i
  881. ldx $t2,$t4,$i
  882. ldx $t3,$t5,$i
  883. subfe $t0,$t1,$t0 ; tp[j]-np[j]
  884. subfe $t2,$t3,$t2 ; tp[j+1]-np[j+1]
  885. stdx $t0,$rp,$i
  886. stdx $t2,$t6,$i
  887. addi $i,$i,16
  888. bdnz Lsub
  889. li $i,0
  890. subfe $ovf,$i,$ovf ; handle upmost overflow bit
  891. and $ap,$tp,$ovf
  892. andc $np,$rp,$ovf
  893. or $ap,$ap,$np ; ap=borrow?tp:rp
  894. addi $t7,$ap,8
  895. mtctr $j
  896. .align 4
  897. Lcopy: ; copy or in-place refresh
  898. ldx $t0,$ap,$i
  899. ldx $t1,$t7,$i
  900. std $i,8($nap_d) ; zap nap_d
  901. std $i,16($nap_d)
  902. std $i,24($nap_d)
  903. std $i,32($nap_d)
  904. std $i,40($nap_d)
  905. std $i,48($nap_d)
  906. std $i,56($nap_d)
  907. stdu $i,64($nap_d)
  908. stdx $t0,$rp,$i
  909. stdx $t1,$t6,$i
  910. stdx $i,$tp,$i ; zap tp at once
  911. stdx $i,$t4,$i
  912. addi $i,$i,16
  913. bdnz Lcopy
  914. ___
  915. $code.=<<___ if ($SIZE_T==4);
  916. subf $np,$num,$np ; rewind np
  917. addi $j,$j,1 ; restore counter
  918. subfc $i,$i,$i ; j=0 and "clear" XER[CA]
  919. addi $tp,$sp,`$FRAME+$TRANSFER`
  920. addi $np,$np,-4
  921. addi $rp,$rp,-4
  922. addi $ap,$sp,`$FRAME+$TRANSFER+4`
  923. mtctr $j
  924. .align 4
  925. Lsub: ld $t0,8($tp) ; load tp[j..j+3] in 64-bit word order
  926. ldu $t2,16($tp)
  927. lwz $t4,4($np) ; load np[j..j+3] in 32-bit word order
  928. lwz $t5,8($np)
  929. lwz $t6,12($np)
  930. lwzu $t7,16($np)
  931. extrdi $t1,$t0,32,0
  932. extrdi $t3,$t2,32,0
  933. subfe $t4,$t4,$t0 ; tp[j]-np[j]
  934. stw $t0,4($ap) ; save tp[j..j+3] in 32-bit word order
  935. subfe $t5,$t5,$t1 ; tp[j+1]-np[j+1]
  936. stw $t1,8($ap)
  937. subfe $t6,$t6,$t2 ; tp[j+2]-np[j+2]
  938. stw $t2,12($ap)
  939. subfe $t7,$t7,$t3 ; tp[j+3]-np[j+3]
  940. stwu $t3,16($ap)
  941. stw $t4,4($rp)
  942. stw $t5,8($rp)
  943. stw $t6,12($rp)
  944. stwu $t7,16($rp)
  945. bdnz Lsub
  946. li $i,0
  947. subfe $ovf,$i,$ovf ; handle upmost overflow bit
  948. addi $tp,$sp,`$FRAME+$TRANSFER+4`
  949. subf $rp,$num,$rp ; rewind rp
  950. and $ap,$tp,$ovf
  951. andc $np,$rp,$ovf
  952. or $ap,$ap,$np ; ap=borrow?tp:rp
  953. addi $tp,$sp,`$FRAME+$TRANSFER`
  954. mtctr $j
  955. .align 4
  956. Lcopy: ; copy or in-place refresh
  957. lwz $t0,4($ap)
  958. lwz $t1,8($ap)
  959. lwz $t2,12($ap)
  960. lwzu $t3,16($ap)
  961. std $i,8($nap_d) ; zap nap_d
  962. std $i,16($nap_d)
  963. std $i,24($nap_d)
  964. std $i,32($nap_d)
  965. std $i,40($nap_d)
  966. std $i,48($nap_d)
  967. std $i,56($nap_d)
  968. stdu $i,64($nap_d)
  969. stw $t0,4($rp)
  970. stw $t1,8($rp)
  971. stw $t2,12($rp)
  972. stwu $t3,16($rp)
  973. std $i,8($tp) ; zap tp at once
  974. stdu $i,16($tp)
  975. bdnz Lcopy
  976. ___
  977. $code.=<<___;
  978. $POP $i,0($sp)
  979. li r3,1 ; signal "handled"
  980. $POP r22,`-12*8-10*$SIZE_T`($i)
  981. $POP r23,`-12*8-9*$SIZE_T`($i)
  982. $POP r24,`-12*8-8*$SIZE_T`($i)
  983. $POP r25,`-12*8-7*$SIZE_T`($i)
  984. $POP r26,`-12*8-6*$SIZE_T`($i)
  985. $POP r27,`-12*8-5*$SIZE_T`($i)
  986. $POP r28,`-12*8-4*$SIZE_T`($i)
  987. $POP r29,`-12*8-3*$SIZE_T`($i)
  988. $POP r30,`-12*8-2*$SIZE_T`($i)
  989. $POP r31,`-12*8-1*$SIZE_T`($i)
  990. lfd f20,`-12*8`($i)
  991. lfd f21,`-11*8`($i)
  992. lfd f22,`-10*8`($i)
  993. lfd f23,`-9*8`($i)
  994. lfd f24,`-8*8`($i)
  995. lfd f25,`-7*8`($i)
  996. lfd f26,`-6*8`($i)
  997. lfd f27,`-5*8`($i)
  998. lfd f28,`-4*8`($i)
  999. lfd f29,`-3*8`($i)
  1000. lfd f30,`-2*8`($i)
  1001. lfd f31,`-1*8`($i)
  1002. mr $sp,$i
  1003. blr
  1004. .long 0
  1005. .byte 0,12,4,0,0x8c,10,6,0
  1006. .long 0
  1007. .asciz "Montgomery Multiplication for PPC64, CRYPTOGAMS by <appro\@openssl.org>"
  1008. ___
  1009. $code =~ s/\`([^\`]*)\`/eval $1/gem;
  1010. print $code;
  1011. close STDOUT;