ia64.S 44 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555
  1. .explicit
  2. .text
  3. .ident "ia64.S, Version 2.1"
  4. .ident "IA-64 ISA artwork by Andy Polyakov <appro@fy.chalmers.se>"
  5. //
  6. // ====================================================================
  7. // Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
  8. // project.
  9. //
  10. // Rights for redistribution and usage in source and binary forms are
  11. // granted according to the OpenSSL license. Warranty of any kind is
  12. // disclaimed.
  13. // ====================================================================
  14. //
  15. // Version 2.x is Itanium2 re-tune. Few words about how Itanum2 is
  16. // different from Itanium to this module viewpoint. Most notably, is it
  17. // "wider" than Itanium? Can you experience loop scalability as
  18. // discussed in commentary sections? Not really:-( Itanium2 has 6
  19. // integer ALU ports, i.e. it's 2 ports wider, but it's not enough to
  20. // spin twice as fast, as I need 8 IALU ports. Amount of floating point
  21. // ports is the same, i.e. 2, while I need 4. In other words, to this
  22. // module Itanium2 remains effectively as "wide" as Itanium. Yet it's
  23. // essentially different in respect to this module, and a re-tune was
  24. // required. Well, because some intruction latencies has changed. Most
  25. // noticeably those intensively used:
  26. //
  27. // Itanium Itanium2
  28. // ldf8 9 6 L2 hit
  29. // ld8 2 1 L1 hit
  30. // getf 2 5
  31. // xma[->getf] 7[+1] 4[+0]
  32. // add[->st8] 1[+1] 1[+0]
  33. //
  34. // What does it mean? You might ratiocinate that the original code
  35. // should run just faster... Because sum of latencies is smaller...
  36. // Wrong! Note that getf latency increased. This means that if a loop is
  37. // scheduled for lower latency (as they were), then it will suffer from
  38. // stall condition and the code will therefore turn anti-scalable, e.g.
  39. // original bn_mul_words spun at 5*n or 2.5 times slower than expected
  40. // on Itanium2! What to do? Reschedule loops for Itanium2? But then
  41. // Itanium would exhibit anti-scalability. So I've chosen to reschedule
  42. // for worst latency for every instruction aiming for best *all-round*
  43. // performance.
  44. // Q. How much faster does it get?
  45. // A. Here is the output from 'openssl speed rsa dsa' for vanilla
  46. // 0.9.6a compiled with gcc version 2.96 20000731 (Red Hat
  47. // Linux 7.1 2.96-81):
  48. //
  49. // sign verify sign/s verify/s
  50. // rsa 512 bits 0.0036s 0.0003s 275.3 2999.2
  51. // rsa 1024 bits 0.0203s 0.0011s 49.3 894.1
  52. // rsa 2048 bits 0.1331s 0.0040s 7.5 250.9
  53. // rsa 4096 bits 0.9270s 0.0147s 1.1 68.1
  54. // sign verify sign/s verify/s
  55. // dsa 512 bits 0.0035s 0.0043s 288.3 234.8
  56. // dsa 1024 bits 0.0111s 0.0135s 90.0 74.2
  57. //
  58. // And here is similar output but for this assembler
  59. // implementation:-)
  60. //
  61. // sign verify sign/s verify/s
  62. // rsa 512 bits 0.0021s 0.0001s 549.4 9638.5
  63. // rsa 1024 bits 0.0055s 0.0002s 183.8 4481.1
  64. // rsa 2048 bits 0.0244s 0.0006s 41.4 1726.3
  65. // rsa 4096 bits 0.1295s 0.0018s 7.7 561.5
  66. // sign verify sign/s verify/s
  67. // dsa 512 bits 0.0012s 0.0013s 891.9 756.6
  68. // dsa 1024 bits 0.0023s 0.0028s 440.4 376.2
  69. //
  70. // Yes, you may argue that it's not fair comparison as it's
  71. // possible to craft the C implementation with BN_UMULT_HIGH
  72. // inline assembler macro. But of course! Here is the output
  73. // with the macro:
  74. //
  75. // sign verify sign/s verify/s
  76. // rsa 512 bits 0.0020s 0.0002s 495.0 6561.0
  77. // rsa 1024 bits 0.0086s 0.0004s 116.2 2235.7
  78. // rsa 2048 bits 0.0519s 0.0015s 19.3 667.3
  79. // rsa 4096 bits 0.3464s 0.0053s 2.9 187.7
  80. // sign verify sign/s verify/s
  81. // dsa 512 bits 0.0016s 0.0020s 613.1 510.5
  82. // dsa 1024 bits 0.0045s 0.0054s 221.0 183.9
  83. //
  84. // My code is still way faster, huh:-) And I believe that even
  85. // higher performance can be achieved. Note that as keys get
  86. // longer, performance gain is larger. Why? According to the
  87. // profiler there is another player in the field, namely
  88. // BN_from_montgomery consuming larger and larger portion of CPU
  89. // time as keysize decreases. I therefore consider putting effort
  90. // to assembler implementation of the following routine:
  91. //
  92. // void bn_mul_add_mont (BN_ULONG *rp,BN_ULONG *np,int nl,BN_ULONG n0)
  93. // {
  94. // int i,j;
  95. // BN_ULONG v;
  96. //
  97. // for (i=0; i<nl; i++)
  98. // {
  99. // v=bn_mul_add_words(rp,np,nl,(rp[0]*n0)&BN_MASK2);
  100. // nrp++;
  101. // rp++;
  102. // if (((nrp[-1]+=v)&BN_MASK2) < v)
  103. // for (j=0; ((++nrp[j])&BN_MASK2) == 0; j++) ;
  104. // }
  105. // }
  106. //
  107. // It might as well be beneficial to implement even combaX
  108. // variants, as it appears as it can literally unleash the
  109. // performance (see comment section to bn_mul_comba8 below).
  110. //
  111. // And finally for your reference the output for 0.9.6a compiled
  112. // with SGIcc version 0.01.0-12 (keep in mind that for the moment
  113. // of this writing it's not possible to convince SGIcc to use
  114. // BN_UMULT_HIGH inline assembler macro, yet the code is fast,
  115. // i.e. for a compiler generated one:-):
  116. //
  117. // sign verify sign/s verify/s
  118. // rsa 512 bits 0.0022s 0.0002s 452.7 5894.3
  119. // rsa 1024 bits 0.0097s 0.0005s 102.7 2002.9
  120. // rsa 2048 bits 0.0578s 0.0017s 17.3 600.2
  121. // rsa 4096 bits 0.3838s 0.0061s 2.6 164.5
  122. // sign verify sign/s verify/s
  123. // dsa 512 bits 0.0018s 0.0022s 547.3 459.6
  124. // dsa 1024 bits 0.0051s 0.0062s 196.6 161.3
  125. //
  126. // Oh! Benchmarks were performed on 733MHz Lion-class Itanium
  127. // system running Redhat Linux 7.1 (very special thanks to Ray
  128. // McCaffity of Williams Communications for providing an account).
  129. //
  130. // Q. What's the heck with 'rum 1<<5' at the end of every function?
  131. // A. Well, by clearing the "upper FP registers written" bit of the
  132. // User Mask I want to excuse the kernel from preserving upper
  133. // (f32-f128) FP register bank over process context switch, thus
  134. // minimizing bus bandwidth consumption during the switch (i.e.
  135. // after PKI opration completes and the program is off doing
  136. // something else like bulk symmetric encryption). Having said
  137. // this, I also want to point out that it might be good idea
  138. // to compile the whole toolkit (as well as majority of the
  139. // programs for that matter) with -mfixed-range=f32-f127 command
  140. // line option. No, it doesn't prevent the compiler from writing
  141. // to upper bank, but at least discourages to do so. If you don't
  142. // like the idea you have the option to compile the module with
  143. // -Drum=nop.m in command line.
  144. //
  145. #if defined(_HPUX_SOURCE) && !defined(_LP64)
  146. #define ADDP addp4
  147. #else
  148. #define ADDP add
  149. #endif
  150. #if 1
  151. //
  152. // bn_[add|sub]_words routines.
  153. //
  154. // Loops are spinning in 2*(n+5) ticks on Itanuim (provided that the
  155. // data reside in L1 cache, i.e. 2 ticks away). It's possible to
  156. // compress the epilogue and get down to 2*n+6, but at the cost of
  157. // scalability (the neat feature of this implementation is that it
  158. // shall automagically spin in n+5 on "wider" IA-64 implementations:-)
  159. // I consider that the epilogue is short enough as it is to trade tiny
  160. // performance loss on Itanium for scalability.
  161. //
  162. // BN_ULONG bn_add_words(BN_ULONG *rp, BN_ULONG *ap, BN_ULONG *bp,int num)
  163. //
  164. .global bn_add_words#
  165. .proc bn_add_words#
  166. .align 64
  167. .skip 32 // makes the loop body aligned at 64-byte boundary
  168. bn_add_words:
  169. .prologue
  170. .save ar.pfs,r2
  171. { .mii; alloc r2=ar.pfs,4,12,0,16
  172. cmp4.le p6,p0=r35,r0 };;
  173. { .mfb; mov r8=r0 // return value
  174. (p6) br.ret.spnt.many b0 };;
  175. { .mib; sub r10=r35,r0,1
  176. .save ar.lc,r3
  177. mov r3=ar.lc
  178. brp.loop.imp .L_bn_add_words_ctop,.L_bn_add_words_cend-16
  179. }
  180. { .mib; ADDP r14=0,r32 // rp
  181. .save pr,r9
  182. mov r9=pr };;
  183. .body
  184. { .mii; ADDP r15=0,r33 // ap
  185. mov ar.lc=r10
  186. mov ar.ec=6 }
  187. { .mib; ADDP r16=0,r34 // bp
  188. mov pr.rot=1<<16 };;
  189. .L_bn_add_words_ctop:
  190. { .mii; (p16) ld8 r32=[r16],8 // b=*(bp++)
  191. (p18) add r39=r37,r34
  192. (p19) cmp.ltu.unc p56,p0=r40,r38 }
  193. { .mfb; (p0) nop.m 0x0
  194. (p0) nop.f 0x0
  195. (p0) nop.b 0x0 }
  196. { .mii; (p16) ld8 r35=[r15],8 // a=*(ap++)
  197. (p58) cmp.eq.or p57,p0=-1,r41 // (p20)
  198. (p58) add r41=1,r41 } // (p20)
  199. { .mfb; (p21) st8 [r14]=r42,8 // *(rp++)=r
  200. (p0) nop.f 0x0
  201. br.ctop.sptk .L_bn_add_words_ctop };;
  202. .L_bn_add_words_cend:
  203. { .mii;
  204. (p59) add r8=1,r8 // return value
  205. mov pr=r9,0x1ffff
  206. mov ar.lc=r3 }
  207. { .mbb; nop.b 0x0
  208. br.ret.sptk.many b0 };;
  209. .endp bn_add_words#
  210. //
  211. // BN_ULONG bn_sub_words(BN_ULONG *rp, BN_ULONG *ap, BN_ULONG *bp,int num)
  212. //
  213. .global bn_sub_words#
  214. .proc bn_sub_words#
  215. .align 64
  216. .skip 32 // makes the loop body aligned at 64-byte boundary
  217. bn_sub_words:
  218. .prologue
  219. .save ar.pfs,r2
  220. { .mii; alloc r2=ar.pfs,4,12,0,16
  221. cmp4.le p6,p0=r35,r0 };;
  222. { .mfb; mov r8=r0 // return value
  223. (p6) br.ret.spnt.many b0 };;
  224. { .mib; sub r10=r35,r0,1
  225. .save ar.lc,r3
  226. mov r3=ar.lc
  227. brp.loop.imp .L_bn_sub_words_ctop,.L_bn_sub_words_cend-16
  228. }
  229. { .mib; ADDP r14=0,r32 // rp
  230. .save pr,r9
  231. mov r9=pr };;
  232. .body
  233. { .mii; ADDP r15=0,r33 // ap
  234. mov ar.lc=r10
  235. mov ar.ec=6 }
  236. { .mib; ADDP r16=0,r34 // bp
  237. mov pr.rot=1<<16 };;
  238. .L_bn_sub_words_ctop:
  239. { .mii; (p16) ld8 r32=[r16],8 // b=*(bp++)
  240. (p18) sub r39=r37,r34
  241. (p19) cmp.gtu.unc p56,p0=r40,r38 }
  242. { .mfb; (p0) nop.m 0x0
  243. (p0) nop.f 0x0
  244. (p0) nop.b 0x0 }
  245. { .mii; (p16) ld8 r35=[r15],8 // a=*(ap++)
  246. (p58) cmp.eq.or p57,p0=0,r41 // (p20)
  247. (p58) add r41=-1,r41 } // (p20)
  248. { .mbb; (p21) st8 [r14]=r42,8 // *(rp++)=r
  249. (p0) nop.b 0x0
  250. br.ctop.sptk .L_bn_sub_words_ctop };;
  251. .L_bn_sub_words_cend:
  252. { .mii;
  253. (p59) add r8=1,r8 // return value
  254. mov pr=r9,0x1ffff
  255. mov ar.lc=r3 }
  256. { .mbb; nop.b 0x0
  257. br.ret.sptk.many b0 };;
  258. .endp bn_sub_words#
  259. #endif
  260. #if 0
  261. #define XMA_TEMPTATION
  262. #endif
  263. #if 1
  264. //
  265. // BN_ULONG bn_mul_words(BN_ULONG *rp, BN_ULONG *ap, int num, BN_ULONG w)
  266. //
  267. .global bn_mul_words#
  268. .proc bn_mul_words#
  269. .align 64
  270. .skip 32 // makes the loop body aligned at 64-byte boundary
  271. bn_mul_words:
  272. .prologue
  273. .save ar.pfs,r2
  274. #ifdef XMA_TEMPTATION
  275. { .mfi; alloc r2=ar.pfs,4,0,0,0 };;
  276. #else
  277. { .mfi; alloc r2=ar.pfs,4,12,0,16 };;
  278. #endif
  279. { .mib; mov r8=r0 // return value
  280. cmp4.le p6,p0=r34,r0
  281. (p6) br.ret.spnt.many b0 };;
  282. { .mii; sub r10=r34,r0,1
  283. .save ar.lc,r3
  284. mov r3=ar.lc
  285. .save pr,r9
  286. mov r9=pr };;
  287. .body
  288. { .mib; setf.sig f8=r35 // w
  289. mov pr.rot=0x800001<<16
  290. // ------^----- serves as (p50) at first (p27)
  291. brp.loop.imp .L_bn_mul_words_ctop,.L_bn_mul_words_cend-16
  292. }
  293. #ifndef XMA_TEMPTATION
  294. { .mmi; ADDP r14=0,r32 // rp
  295. ADDP r15=0,r33 // ap
  296. mov ar.lc=r10 }
  297. { .mmi; mov r40=0 // serves as r35 at first (p27)
  298. mov ar.ec=13 };;
  299. // This loop spins in 2*(n+12) ticks. It's scheduled for data in Itanium
  300. // L2 cache (i.e. 9 ticks away) as floating point load/store instructions
  301. // bypass L1 cache and L2 latency is actually best-case scenario for
  302. // ldf8. The loop is not scalable and shall run in 2*(n+12) even on
  303. // "wider" IA-64 implementations. It's a trade-off here. n+24 loop
  304. // would give us ~5% in *overall* performance improvement on "wider"
  305. // IA-64, but would hurt Itanium for about same because of longer
  306. // epilogue. As it's a matter of few percents in either case I've
  307. // chosen to trade the scalability for development time (you can see
  308. // this very instruction sequence in bn_mul_add_words loop which in
  309. // turn is scalable).
  310. .L_bn_mul_words_ctop:
  311. { .mfi; (p25) getf.sig r36=f52 // low
  312. (p21) xmpy.lu f48=f37,f8
  313. (p28) cmp.ltu p54,p50=r41,r39 }
  314. { .mfi; (p16) ldf8 f32=[r15],8
  315. (p21) xmpy.hu f40=f37,f8
  316. (p0) nop.i 0x0 };;
  317. { .mii; (p25) getf.sig r32=f44 // high
  318. .pred.rel "mutex",p50,p54
  319. (p50) add r40=r38,r35 // (p27)
  320. (p54) add r40=r38,r35,1 } // (p27)
  321. { .mfb; (p28) st8 [r14]=r41,8
  322. (p0) nop.f 0x0
  323. br.ctop.sptk .L_bn_mul_words_ctop };;
  324. .L_bn_mul_words_cend:
  325. { .mii; nop.m 0x0
  326. .pred.rel "mutex",p51,p55
  327. (p51) add r8=r36,r0
  328. (p55) add r8=r36,r0,1 }
  329. { .mfb; nop.m 0x0
  330. nop.f 0x0
  331. nop.b 0x0 }
  332. #else // XMA_TEMPTATION
  333. setf.sig f37=r0 // serves as carry at (p18) tick
  334. mov ar.lc=r10
  335. mov ar.ec=5;;
  336. // Most of you examining this code very likely wonder why in the name
  337. // of Intel the following loop is commented out? Indeed, it looks so
  338. // neat that you find it hard to believe that it's something wrong
  339. // with it, right? The catch is that every iteration depends on the
  340. // result from previous one and the latter isn't available instantly.
  341. // The loop therefore spins at the latency of xma minus 1, or in other
  342. // words at 6*(n+4) ticks:-( Compare to the "production" loop above
  343. // that runs in 2*(n+11) where the low latency problem is worked around
  344. // by moving the dependency to one-tick latent interger ALU. Note that
  345. // "distance" between ldf8 and xma is not latency of ldf8, but the
  346. // *difference* between xma and ldf8 latencies.
  347. .L_bn_mul_words_ctop:
  348. { .mfi; (p16) ldf8 f32=[r33],8
  349. (p18) xma.hu f38=f34,f8,f39 }
  350. { .mfb; (p20) stf8 [r32]=f37,8
  351. (p18) xma.lu f35=f34,f8,f39
  352. br.ctop.sptk .L_bn_mul_words_ctop };;
  353. .L_bn_mul_words_cend:
  354. getf.sig r8=f41 // the return value
  355. #endif // XMA_TEMPTATION
  356. { .mii; nop.m 0x0
  357. mov pr=r9,0x1ffff
  358. mov ar.lc=r3 }
  359. { .mfb; rum 1<<5 // clear um.mfh
  360. nop.f 0x0
  361. br.ret.sptk.many b0 };;
  362. .endp bn_mul_words#
  363. #endif
  364. #if 1
  365. //
  366. // BN_ULONG bn_mul_add_words(BN_ULONG *rp, BN_ULONG *ap, int num, BN_ULONG w)
  367. //
  368. .global bn_mul_add_words#
  369. .proc bn_mul_add_words#
  370. .align 64
  371. .skip 48 // makes the loop body aligned at 64-byte boundary
  372. bn_mul_add_words:
  373. .prologue
  374. .save ar.pfs,r2
  375. { .mmi; alloc r2=ar.pfs,4,4,0,8
  376. cmp4.le p6,p0=r34,r0
  377. .save ar.lc,r3
  378. mov r3=ar.lc };;
  379. { .mib; mov r8=r0 // return value
  380. sub r10=r34,r0,1
  381. (p6) br.ret.spnt.many b0 };;
  382. { .mib; setf.sig f8=r35 // w
  383. .save pr,r9
  384. mov r9=pr
  385. brp.loop.imp .L_bn_mul_add_words_ctop,.L_bn_mul_add_words_cend-16
  386. }
  387. .body
  388. { .mmi; ADDP r14=0,r32 // rp
  389. ADDP r15=0,r33 // ap
  390. mov ar.lc=r10 }
  391. { .mii; ADDP r16=0,r32 // rp copy
  392. mov pr.rot=0x2001<<16
  393. // ------^----- serves as (p40) at first (p27)
  394. mov ar.ec=11 };;
  395. // This loop spins in 3*(n+10) ticks on Itanium and in 2*(n+10) on
  396. // Itanium 2. Yes, unlike previous versions it scales:-) Previous
  397. // version was performing *all* additions in IALU and was starving
  398. // for those even on Itanium 2. In this version one addition is
  399. // moved to FPU and is folded with multiplication. This is at cost
  400. // of propogating the result from previous call to this subroutine
  401. // to L2 cache... In other words negligible even for shorter keys.
  402. // *Overall* performance improvement [over previous version] varies
  403. // from 11 to 22 percent depending on key length.
  404. .L_bn_mul_add_words_ctop:
  405. .pred.rel "mutex",p40,p42
  406. { .mfi; (p23) getf.sig r36=f45 // low
  407. (p20) xma.lu f42=f36,f8,f50 // low
  408. (p40) add r39=r39,r35 } // (p27)
  409. { .mfi; (p16) ldf8 f32=[r15],8 // *(ap++)
  410. (p20) xma.hu f36=f36,f8,f50 // high
  411. (p42) add r39=r39,r35,1 };; // (p27)
  412. { .mmi; (p24) getf.sig r32=f40 // high
  413. (p16) ldf8 f46=[r16],8 // *(rp1++)
  414. (p40) cmp.ltu p41,p39=r39,r35 } // (p27)
  415. { .mib; (p26) st8 [r14]=r39,8 // *(rp2++)
  416. (p42) cmp.leu p41,p39=r39,r35 // (p27)
  417. br.ctop.sptk .L_bn_mul_add_words_ctop};;
  418. .L_bn_mul_add_words_cend:
  419. { .mmi; .pred.rel "mutex",p40,p42
  420. (p40) add r8=r35,r0
  421. (p42) add r8=r35,r0,1
  422. mov pr=r9,0x1ffff }
  423. { .mib; rum 1<<5 // clear um.mfh
  424. mov ar.lc=r3
  425. br.ret.sptk.many b0 };;
  426. .endp bn_mul_add_words#
  427. #endif
  428. #if 1
  429. //
  430. // void bn_sqr_words(BN_ULONG *rp, BN_ULONG *ap, int num)
  431. //
  432. .global bn_sqr_words#
  433. .proc bn_sqr_words#
  434. .align 64
  435. .skip 32 // makes the loop body aligned at 64-byte boundary
  436. bn_sqr_words:
  437. .prologue
  438. .save ar.pfs,r2
  439. { .mii; alloc r2=ar.pfs,3,0,0,0
  440. sxt4 r34=r34 };;
  441. { .mii; cmp.le p6,p0=r34,r0
  442. mov r8=r0 } // return value
  443. { .mfb; ADDP r32=0,r32
  444. nop.f 0x0
  445. (p6) br.ret.spnt.many b0 };;
  446. { .mii; sub r10=r34,r0,1
  447. .save ar.lc,r3
  448. mov r3=ar.lc
  449. .save pr,r9
  450. mov r9=pr };;
  451. .body
  452. { .mib; ADDP r33=0,r33
  453. mov pr.rot=1<<16
  454. brp.loop.imp .L_bn_sqr_words_ctop,.L_bn_sqr_words_cend-16
  455. }
  456. { .mii; add r34=8,r32
  457. mov ar.lc=r10
  458. mov ar.ec=18 };;
  459. // 2*(n+17) on Itanium, (n+17) on "wider" IA-64 implementations. It's
  460. // possible to compress the epilogue (I'm getting tired to write this
  461. // comment over and over) and get down to 2*n+16 at the cost of
  462. // scalability. The decision will very likely be reconsidered after the
  463. // benchmark program is profiled. I.e. if perfomance gain on Itanium
  464. // will appear larger than loss on "wider" IA-64, then the loop should
  465. // be explicitely split and the epilogue compressed.
  466. .L_bn_sqr_words_ctop:
  467. { .mfi; (p16) ldf8 f32=[r33],8
  468. (p25) xmpy.lu f42=f41,f41
  469. (p0) nop.i 0x0 }
  470. { .mib; (p33) stf8 [r32]=f50,16
  471. (p0) nop.i 0x0
  472. (p0) nop.b 0x0 }
  473. { .mfi; (p0) nop.m 0x0
  474. (p25) xmpy.hu f52=f41,f41
  475. (p0) nop.i 0x0 }
  476. { .mib; (p33) stf8 [r34]=f60,16
  477. (p0) nop.i 0x0
  478. br.ctop.sptk .L_bn_sqr_words_ctop };;
  479. .L_bn_sqr_words_cend:
  480. { .mii; nop.m 0x0
  481. mov pr=r9,0x1ffff
  482. mov ar.lc=r3 }
  483. { .mfb; rum 1<<5 // clear um.mfh
  484. nop.f 0x0
  485. br.ret.sptk.many b0 };;
  486. .endp bn_sqr_words#
  487. #endif
  488. #if 1
  489. // Apparently we win nothing by implementing special bn_sqr_comba8.
  490. // Yes, it is possible to reduce the number of multiplications by
  491. // almost factor of two, but then the amount of additions would
  492. // increase by factor of two (as we would have to perform those
  493. // otherwise performed by xma ourselves). Normally we would trade
  494. // anyway as multiplications are way more expensive, but not this
  495. // time... Multiplication kernel is fully pipelined and as we drain
  496. // one 128-bit multiplication result per clock cycle multiplications
  497. // are effectively as inexpensive as additions. Special implementation
  498. // might become of interest for "wider" IA-64 implementation as you'll
  499. // be able to get through the multiplication phase faster (there won't
  500. // be any stall issues as discussed in the commentary section below and
  501. // you therefore will be able to employ all 4 FP units)... But these
  502. // Itanium days it's simply too hard to justify the effort so I just
  503. // drop down to bn_mul_comba8 code:-)
  504. //
  505. // void bn_sqr_comba8(BN_ULONG *r, BN_ULONG *a)
  506. //
  507. .global bn_sqr_comba8#
  508. .proc bn_sqr_comba8#
  509. .align 64
  510. bn_sqr_comba8:
  511. .prologue
  512. .save ar.pfs,r2
  513. #if defined(_HPUX_SOURCE) && !defined(_LP64)
  514. { .mii; alloc r2=ar.pfs,2,1,0,0
  515. addp4 r33=0,r33
  516. addp4 r32=0,r32 };;
  517. { .mii;
  518. #else
  519. { .mii; alloc r2=ar.pfs,2,1,0,0
  520. #endif
  521. mov r34=r33
  522. add r14=8,r33 };;
  523. .body
  524. { .mii; add r17=8,r34
  525. add r15=16,r33
  526. add r18=16,r34 }
  527. { .mfb; add r16=24,r33
  528. br .L_cheat_entry_point8 };;
  529. .endp bn_sqr_comba8#
  530. #endif
  531. #if 1
  532. // I've estimated this routine to run in ~120 ticks, but in reality
  533. // (i.e. according to ar.itc) it takes ~160 ticks. Are those extra
  534. // cycles consumed for instructions fetch? Or did I misinterpret some
  535. // clause in Itanium µ-architecture manual? Comments are welcomed and
  536. // highly appreciated.
  537. //
  538. // On Itanium 2 it takes ~190 ticks. This is because of stalls on
  539. // result from getf.sig. I do nothing about it at this point for
  540. // reasons depicted below.
  541. //
  542. // However! It should be noted that even 160 ticks is darn good result
  543. // as it's over 10 (yes, ten, spelled as t-e-n) times faster than the
  544. // C version (compiled with gcc with inline assembler). I really
  545. // kicked compiler's butt here, didn't I? Yeah! This brings us to the
  546. // following statement. It's damn shame that this routine isn't called
  547. // very often nowadays! According to the profiler most CPU time is
  548. // consumed by bn_mul_add_words called from BN_from_montgomery. In
  549. // order to estimate what we're missing, I've compared the performance
  550. // of this routine against "traditional" implementation, i.e. against
  551. // following routine:
  552. //
  553. // void bn_mul_comba8(BN_ULONG *r, BN_ULONG *a, BN_ULONG *b)
  554. // { r[ 8]=bn_mul_words( &(r[0]),a,8,b[0]);
  555. // r[ 9]=bn_mul_add_words(&(r[1]),a,8,b[1]);
  556. // r[10]=bn_mul_add_words(&(r[2]),a,8,b[2]);
  557. // r[11]=bn_mul_add_words(&(r[3]),a,8,b[3]);
  558. // r[12]=bn_mul_add_words(&(r[4]),a,8,b[4]);
  559. // r[13]=bn_mul_add_words(&(r[5]),a,8,b[5]);
  560. // r[14]=bn_mul_add_words(&(r[6]),a,8,b[6]);
  561. // r[15]=bn_mul_add_words(&(r[7]),a,8,b[7]);
  562. // }
  563. //
  564. // The one below is over 8 times faster than the one above:-( Even
  565. // more reasons to "combafy" bn_mul_add_mont...
  566. //
  567. // And yes, this routine really made me wish there were an optimizing
  568. // assembler! It also feels like it deserves a dedication.
  569. //
  570. // To my wife for being there and to my kids...
  571. //
  572. // void bn_mul_comba8(BN_ULONG *r, BN_ULONG *a, BN_ULONG *b)
  573. //
  574. #define carry1 r14
  575. #define carry2 r15
  576. #define carry3 r34
  577. .global bn_mul_comba8#
  578. .proc bn_mul_comba8#
  579. .align 64
  580. bn_mul_comba8:
  581. .prologue
  582. .save ar.pfs,r2
  583. #if defined(_HPUX_SOURCE) && !defined(_LP64)
  584. { .mii; alloc r2=ar.pfs,3,0,0,0
  585. addp4 r33=0,r33
  586. addp4 r34=0,r34 };;
  587. { .mii; addp4 r32=0,r32
  588. #else
  589. { .mii; alloc r2=ar.pfs,3,0,0,0
  590. #endif
  591. add r14=8,r33
  592. add r17=8,r34 }
  593. .body
  594. { .mii; add r15=16,r33
  595. add r18=16,r34
  596. add r16=24,r33 }
  597. .L_cheat_entry_point8:
  598. { .mmi; add r19=24,r34
  599. ldf8 f32=[r33],32 };;
  600. { .mmi; ldf8 f120=[r34],32
  601. ldf8 f121=[r17],32 }
  602. { .mmi; ldf8 f122=[r18],32
  603. ldf8 f123=[r19],32 };;
  604. { .mmi; ldf8 f124=[r34]
  605. ldf8 f125=[r17] }
  606. { .mmi; ldf8 f126=[r18]
  607. ldf8 f127=[r19] }
  608. { .mmi; ldf8 f33=[r14],32
  609. ldf8 f34=[r15],32 }
  610. { .mmi; ldf8 f35=[r16],32;;
  611. ldf8 f36=[r33] }
  612. { .mmi; ldf8 f37=[r14]
  613. ldf8 f38=[r15] }
  614. { .mfi; ldf8 f39=[r16]
  615. // -------\ Entering multiplier's heaven /-------
  616. // ------------\ /------------
  617. // -----------------\ /-----------------
  618. // ----------------------\/----------------------
  619. xma.hu f41=f32,f120,f0 }
  620. { .mfi; xma.lu f40=f32,f120,f0 };; // (*)
  621. { .mfi; xma.hu f51=f32,f121,f0 }
  622. { .mfi; xma.lu f50=f32,f121,f0 };;
  623. { .mfi; xma.hu f61=f32,f122,f0 }
  624. { .mfi; xma.lu f60=f32,f122,f0 };;
  625. { .mfi; xma.hu f71=f32,f123,f0 }
  626. { .mfi; xma.lu f70=f32,f123,f0 };;
  627. { .mfi; xma.hu f81=f32,f124,f0 }
  628. { .mfi; xma.lu f80=f32,f124,f0 };;
  629. { .mfi; xma.hu f91=f32,f125,f0 }
  630. { .mfi; xma.lu f90=f32,f125,f0 };;
  631. { .mfi; xma.hu f101=f32,f126,f0 }
  632. { .mfi; xma.lu f100=f32,f126,f0 };;
  633. { .mfi; xma.hu f111=f32,f127,f0 }
  634. { .mfi; xma.lu f110=f32,f127,f0 };;//
  635. // (*) You can argue that splitting at every second bundle would
  636. // prevent "wider" IA-64 implementations from achieving the peak
  637. // performance. Well, not really... The catch is that if you
  638. // intend to keep 4 FP units busy by splitting at every fourth
  639. // bundle and thus perform these 16 multiplications in 4 ticks,
  640. // the first bundle *below* would stall because the result from
  641. // the first xma bundle *above* won't be available for another 3
  642. // ticks (if not more, being an optimist, I assume that "wider"
  643. // implementation will have same latency:-). This stall will hold
  644. // you back and the performance would be as if every second bundle
  645. // were split *anyway*...
  646. { .mfi; getf.sig r16=f40
  647. xma.hu f42=f33,f120,f41
  648. add r33=8,r32 }
  649. { .mfi; xma.lu f41=f33,f120,f41 };;
  650. { .mfi; getf.sig r24=f50
  651. xma.hu f52=f33,f121,f51 }
  652. { .mfi; xma.lu f51=f33,f121,f51 };;
  653. { .mfi; st8 [r32]=r16,16
  654. xma.hu f62=f33,f122,f61 }
  655. { .mfi; xma.lu f61=f33,f122,f61 };;
  656. { .mfi; xma.hu f72=f33,f123,f71 }
  657. { .mfi; xma.lu f71=f33,f123,f71 };;
  658. { .mfi; xma.hu f82=f33,f124,f81 }
  659. { .mfi; xma.lu f81=f33,f124,f81 };;
  660. { .mfi; xma.hu f92=f33,f125,f91 }
  661. { .mfi; xma.lu f91=f33,f125,f91 };;
  662. { .mfi; xma.hu f102=f33,f126,f101 }
  663. { .mfi; xma.lu f101=f33,f126,f101 };;
  664. { .mfi; xma.hu f112=f33,f127,f111 }
  665. { .mfi; xma.lu f111=f33,f127,f111 };;//
  666. //-------------------------------------------------//
  667. { .mfi; getf.sig r25=f41
  668. xma.hu f43=f34,f120,f42 }
  669. { .mfi; xma.lu f42=f34,f120,f42 };;
  670. { .mfi; getf.sig r16=f60
  671. xma.hu f53=f34,f121,f52 }
  672. { .mfi; xma.lu f52=f34,f121,f52 };;
  673. { .mfi; getf.sig r17=f51
  674. xma.hu f63=f34,f122,f62
  675. add r25=r25,r24 }
  676. { .mfi; xma.lu f62=f34,f122,f62
  677. mov carry1=0 };;
  678. { .mfi; cmp.ltu p6,p0=r25,r24
  679. xma.hu f73=f34,f123,f72 }
  680. { .mfi; xma.lu f72=f34,f123,f72 };;
  681. { .mfi; st8 [r33]=r25,16
  682. xma.hu f83=f34,f124,f82
  683. (p6) add carry1=1,carry1 }
  684. { .mfi; xma.lu f82=f34,f124,f82 };;
  685. { .mfi; xma.hu f93=f34,f125,f92 }
  686. { .mfi; xma.lu f92=f34,f125,f92 };;
  687. { .mfi; xma.hu f103=f34,f126,f102 }
  688. { .mfi; xma.lu f102=f34,f126,f102 };;
  689. { .mfi; xma.hu f113=f34,f127,f112 }
  690. { .mfi; xma.lu f112=f34,f127,f112 };;//
  691. //-------------------------------------------------//
  692. { .mfi; getf.sig r18=f42
  693. xma.hu f44=f35,f120,f43
  694. add r17=r17,r16 }
  695. { .mfi; xma.lu f43=f35,f120,f43 };;
  696. { .mfi; getf.sig r24=f70
  697. xma.hu f54=f35,f121,f53 }
  698. { .mfi; mov carry2=0
  699. xma.lu f53=f35,f121,f53 };;
  700. { .mfi; getf.sig r25=f61
  701. xma.hu f64=f35,f122,f63
  702. cmp.ltu p7,p0=r17,r16 }
  703. { .mfi; add r18=r18,r17
  704. xma.lu f63=f35,f122,f63 };;
  705. { .mfi; getf.sig r26=f52
  706. xma.hu f74=f35,f123,f73
  707. (p7) add carry2=1,carry2 }
  708. { .mfi; cmp.ltu p7,p0=r18,r17
  709. xma.lu f73=f35,f123,f73
  710. add r18=r18,carry1 };;
  711. { .mfi;
  712. xma.hu f84=f35,f124,f83
  713. (p7) add carry2=1,carry2 }
  714. { .mfi; cmp.ltu p7,p0=r18,carry1
  715. xma.lu f83=f35,f124,f83 };;
  716. { .mfi; st8 [r32]=r18,16
  717. xma.hu f94=f35,f125,f93
  718. (p7) add carry2=1,carry2 }
  719. { .mfi; xma.lu f93=f35,f125,f93 };;
  720. { .mfi; xma.hu f104=f35,f126,f103 }
  721. { .mfi; xma.lu f103=f35,f126,f103 };;
  722. { .mfi; xma.hu f114=f35,f127,f113 }
  723. { .mfi; mov carry1=0
  724. xma.lu f113=f35,f127,f113
  725. add r25=r25,r24 };;//
  726. //-------------------------------------------------//
  727. { .mfi; getf.sig r27=f43
  728. xma.hu f45=f36,f120,f44
  729. cmp.ltu p6,p0=r25,r24 }
  730. { .mfi; xma.lu f44=f36,f120,f44
  731. add r26=r26,r25 };;
  732. { .mfi; getf.sig r16=f80
  733. xma.hu f55=f36,f121,f54
  734. (p6) add carry1=1,carry1 }
  735. { .mfi; xma.lu f54=f36,f121,f54 };;
  736. { .mfi; getf.sig r17=f71
  737. xma.hu f65=f36,f122,f64
  738. cmp.ltu p6,p0=r26,r25 }
  739. { .mfi; xma.lu f64=f36,f122,f64
  740. add r27=r27,r26 };;
  741. { .mfi; getf.sig r18=f62
  742. xma.hu f75=f36,f123,f74
  743. (p6) add carry1=1,carry1 }
  744. { .mfi; cmp.ltu p6,p0=r27,r26
  745. xma.lu f74=f36,f123,f74
  746. add r27=r27,carry2 };;
  747. { .mfi; getf.sig r19=f53
  748. xma.hu f85=f36,f124,f84
  749. (p6) add carry1=1,carry1 }
  750. { .mfi; xma.lu f84=f36,f124,f84
  751. cmp.ltu p6,p0=r27,carry2 };;
  752. { .mfi; st8 [r33]=r27,16
  753. xma.hu f95=f36,f125,f94
  754. (p6) add carry1=1,carry1 }
  755. { .mfi; xma.lu f94=f36,f125,f94 };;
  756. { .mfi; xma.hu f105=f36,f126,f104 }
  757. { .mfi; mov carry2=0
  758. xma.lu f104=f36,f126,f104
  759. add r17=r17,r16 };;
  760. { .mfi; xma.hu f115=f36,f127,f114
  761. cmp.ltu p7,p0=r17,r16 }
  762. { .mfi; xma.lu f114=f36,f127,f114
  763. add r18=r18,r17 };;//
  764. //-------------------------------------------------//
  765. { .mfi; getf.sig r20=f44
  766. xma.hu f46=f37,f120,f45
  767. (p7) add carry2=1,carry2 }
  768. { .mfi; cmp.ltu p7,p0=r18,r17
  769. xma.lu f45=f37,f120,f45
  770. add r19=r19,r18 };;
  771. { .mfi; getf.sig r24=f90
  772. xma.hu f56=f37,f121,f55 }
  773. { .mfi; xma.lu f55=f37,f121,f55 };;
  774. { .mfi; getf.sig r25=f81
  775. xma.hu f66=f37,f122,f65
  776. (p7) add carry2=1,carry2 }
  777. { .mfi; cmp.ltu p7,p0=r19,r18
  778. xma.lu f65=f37,f122,f65
  779. add r20=r20,r19 };;
  780. { .mfi; getf.sig r26=f72
  781. xma.hu f76=f37,f123,f75
  782. (p7) add carry2=1,carry2 }
  783. { .mfi; cmp.ltu p7,p0=r20,r19
  784. xma.lu f75=f37,f123,f75
  785. add r20=r20,carry1 };;
  786. { .mfi; getf.sig r27=f63
  787. xma.hu f86=f37,f124,f85
  788. (p7) add carry2=1,carry2 }
  789. { .mfi; xma.lu f85=f37,f124,f85
  790. cmp.ltu p7,p0=r20,carry1 };;
  791. { .mfi; getf.sig r28=f54
  792. xma.hu f96=f37,f125,f95
  793. (p7) add carry2=1,carry2 }
  794. { .mfi; st8 [r32]=r20,16
  795. xma.lu f95=f37,f125,f95 };;
  796. { .mfi; xma.hu f106=f37,f126,f105 }
  797. { .mfi; mov carry1=0
  798. xma.lu f105=f37,f126,f105
  799. add r25=r25,r24 };;
  800. { .mfi; xma.hu f116=f37,f127,f115
  801. cmp.ltu p6,p0=r25,r24 }
  802. { .mfi; xma.lu f115=f37,f127,f115
  803. add r26=r26,r25 };;//
  804. //-------------------------------------------------//
  805. { .mfi; getf.sig r29=f45
  806. xma.hu f47=f38,f120,f46
  807. (p6) add carry1=1,carry1 }
  808. { .mfi; cmp.ltu p6,p0=r26,r25
  809. xma.lu f46=f38,f120,f46
  810. add r27=r27,r26 };;
  811. { .mfi; getf.sig r16=f100
  812. xma.hu f57=f38,f121,f56
  813. (p6) add carry1=1,carry1 }
  814. { .mfi; cmp.ltu p6,p0=r27,r26
  815. xma.lu f56=f38,f121,f56
  816. add r28=r28,r27 };;
  817. { .mfi; getf.sig r17=f91
  818. xma.hu f67=f38,f122,f66
  819. (p6) add carry1=1,carry1 }
  820. { .mfi; cmp.ltu p6,p0=r28,r27
  821. xma.lu f66=f38,f122,f66
  822. add r29=r29,r28 };;
  823. { .mfi; getf.sig r18=f82
  824. xma.hu f77=f38,f123,f76
  825. (p6) add carry1=1,carry1 }
  826. { .mfi; cmp.ltu p6,p0=r29,r28
  827. xma.lu f76=f38,f123,f76
  828. add r29=r29,carry2 };;
  829. { .mfi; getf.sig r19=f73
  830. xma.hu f87=f38,f124,f86
  831. (p6) add carry1=1,carry1 }
  832. { .mfi; xma.lu f86=f38,f124,f86
  833. cmp.ltu p6,p0=r29,carry2 };;
  834. { .mfi; getf.sig r20=f64
  835. xma.hu f97=f38,f125,f96
  836. (p6) add carry1=1,carry1 }
  837. { .mfi; st8 [r33]=r29,16
  838. xma.lu f96=f38,f125,f96 };;
  839. { .mfi; getf.sig r21=f55
  840. xma.hu f107=f38,f126,f106 }
  841. { .mfi; mov carry2=0
  842. xma.lu f106=f38,f126,f106
  843. add r17=r17,r16 };;
  844. { .mfi; xma.hu f117=f38,f127,f116
  845. cmp.ltu p7,p0=r17,r16 }
  846. { .mfi; xma.lu f116=f38,f127,f116
  847. add r18=r18,r17 };;//
  848. //-------------------------------------------------//
  849. { .mfi; getf.sig r22=f46
  850. xma.hu f48=f39,f120,f47
  851. (p7) add carry2=1,carry2 }
  852. { .mfi; cmp.ltu p7,p0=r18,r17
  853. xma.lu f47=f39,f120,f47
  854. add r19=r19,r18 };;
  855. { .mfi; getf.sig r24=f110
  856. xma.hu f58=f39,f121,f57
  857. (p7) add carry2=1,carry2 }
  858. { .mfi; cmp.ltu p7,p0=r19,r18
  859. xma.lu f57=f39,f121,f57
  860. add r20=r20,r19 };;
  861. { .mfi; getf.sig r25=f101
  862. xma.hu f68=f39,f122,f67
  863. (p7) add carry2=1,carry2 }
  864. { .mfi; cmp.ltu p7,p0=r20,r19
  865. xma.lu f67=f39,f122,f67
  866. add r21=r21,r20 };;
  867. { .mfi; getf.sig r26=f92
  868. xma.hu f78=f39,f123,f77
  869. (p7) add carry2=1,carry2 }
  870. { .mfi; cmp.ltu p7,p0=r21,r20
  871. xma.lu f77=f39,f123,f77
  872. add r22=r22,r21 };;
  873. { .mfi; getf.sig r27=f83
  874. xma.hu f88=f39,f124,f87
  875. (p7) add carry2=1,carry2 }
  876. { .mfi; cmp.ltu p7,p0=r22,r21
  877. xma.lu f87=f39,f124,f87
  878. add r22=r22,carry1 };;
  879. { .mfi; getf.sig r28=f74
  880. xma.hu f98=f39,f125,f97
  881. (p7) add carry2=1,carry2 }
  882. { .mfi; xma.lu f97=f39,f125,f97
  883. cmp.ltu p7,p0=r22,carry1 };;
  884. { .mfi; getf.sig r29=f65
  885. xma.hu f108=f39,f126,f107
  886. (p7) add carry2=1,carry2 }
  887. { .mfi; st8 [r32]=r22,16
  888. xma.lu f107=f39,f126,f107 };;
  889. { .mfi; getf.sig r30=f56
  890. xma.hu f118=f39,f127,f117 }
  891. { .mfi; xma.lu f117=f39,f127,f117 };;//
  892. //-------------------------------------------------//
  893. // Leaving muliplier's heaven... Quite a ride, huh?
  894. { .mii; getf.sig r31=f47
  895. add r25=r25,r24
  896. mov carry1=0 };;
  897. { .mii; getf.sig r16=f111
  898. cmp.ltu p6,p0=r25,r24
  899. add r26=r26,r25 };;
  900. { .mfb; getf.sig r17=f102 }
  901. { .mii;
  902. (p6) add carry1=1,carry1
  903. cmp.ltu p6,p0=r26,r25
  904. add r27=r27,r26 };;
  905. { .mfb; nop.m 0x0 }
  906. { .mii;
  907. (p6) add carry1=1,carry1
  908. cmp.ltu p6,p0=r27,r26
  909. add r28=r28,r27 };;
  910. { .mii; getf.sig r18=f93
  911. add r17=r17,r16
  912. mov carry3=0 }
  913. { .mii;
  914. (p6) add carry1=1,carry1
  915. cmp.ltu p6,p0=r28,r27
  916. add r29=r29,r28 };;
  917. { .mii; getf.sig r19=f84
  918. cmp.ltu p7,p0=r17,r16 }
  919. { .mii;
  920. (p6) add carry1=1,carry1
  921. cmp.ltu p6,p0=r29,r28
  922. add r30=r30,r29 };;
  923. { .mii; getf.sig r20=f75
  924. add r18=r18,r17 }
  925. { .mii;
  926. (p6) add carry1=1,carry1
  927. cmp.ltu p6,p0=r30,r29
  928. add r31=r31,r30 };;
  929. { .mfb; getf.sig r21=f66 }
  930. { .mii; (p7) add carry3=1,carry3
  931. cmp.ltu p7,p0=r18,r17
  932. add r19=r19,r18 }
  933. { .mfb; nop.m 0x0 }
  934. { .mii;
  935. (p6) add carry1=1,carry1
  936. cmp.ltu p6,p0=r31,r30
  937. add r31=r31,carry2 };;
  938. { .mfb; getf.sig r22=f57 }
  939. { .mii; (p7) add carry3=1,carry3
  940. cmp.ltu p7,p0=r19,r18
  941. add r20=r20,r19 }
  942. { .mfb; nop.m 0x0 }
  943. { .mii;
  944. (p6) add carry1=1,carry1
  945. cmp.ltu p6,p0=r31,carry2 };;
  946. { .mfb; getf.sig r23=f48 }
  947. { .mii; (p7) add carry3=1,carry3
  948. cmp.ltu p7,p0=r20,r19
  949. add r21=r21,r20 }
  950. { .mii;
  951. (p6) add carry1=1,carry1 }
  952. { .mfb; st8 [r33]=r31,16 };;
  953. { .mfb; getf.sig r24=f112 }
  954. { .mii; (p7) add carry3=1,carry3
  955. cmp.ltu p7,p0=r21,r20
  956. add r22=r22,r21 };;
  957. { .mfb; getf.sig r25=f103 }
  958. { .mii; (p7) add carry3=1,carry3
  959. cmp.ltu p7,p0=r22,r21
  960. add r23=r23,r22 };;
  961. { .mfb; getf.sig r26=f94 }
  962. { .mii; (p7) add carry3=1,carry3
  963. cmp.ltu p7,p0=r23,r22
  964. add r23=r23,carry1 };;
  965. { .mfb; getf.sig r27=f85 }
  966. { .mii; (p7) add carry3=1,carry3
  967. cmp.ltu p7,p8=r23,carry1};;
  968. { .mii; getf.sig r28=f76
  969. add r25=r25,r24
  970. mov carry1=0 }
  971. { .mii; st8 [r32]=r23,16
  972. (p7) add carry2=1,carry3
  973. (p8) add carry2=0,carry3 };;
  974. { .mfb; nop.m 0x0 }
  975. { .mii; getf.sig r29=f67
  976. cmp.ltu p6,p0=r25,r24
  977. add r26=r26,r25 };;
  978. { .mfb; getf.sig r30=f58 }
  979. { .mii;
  980. (p6) add carry1=1,carry1
  981. cmp.ltu p6,p0=r26,r25
  982. add r27=r27,r26 };;
  983. { .mfb; getf.sig r16=f113 }
  984. { .mii;
  985. (p6) add carry1=1,carry1
  986. cmp.ltu p6,p0=r27,r26
  987. add r28=r28,r27 };;
  988. { .mfb; getf.sig r17=f104 }
  989. { .mii;
  990. (p6) add carry1=1,carry1
  991. cmp.ltu p6,p0=r28,r27
  992. add r29=r29,r28 };;
  993. { .mfb; getf.sig r18=f95 }
  994. { .mii;
  995. (p6) add carry1=1,carry1
  996. cmp.ltu p6,p0=r29,r28
  997. add r30=r30,r29 };;
  998. { .mii; getf.sig r19=f86
  999. add r17=r17,r16
  1000. mov carry3=0 }
  1001. { .mii;
  1002. (p6) add carry1=1,carry1
  1003. cmp.ltu p6,p0=r30,r29
  1004. add r30=r30,carry2 };;
  1005. { .mii; getf.sig r20=f77
  1006. cmp.ltu p7,p0=r17,r16
  1007. add r18=r18,r17 }
  1008. { .mii;
  1009. (p6) add carry1=1,carry1
  1010. cmp.ltu p6,p0=r30,carry2 };;
  1011. { .mfb; getf.sig r21=f68 }
  1012. { .mii; st8 [r33]=r30,16
  1013. (p6) add carry1=1,carry1 };;
  1014. { .mfb; getf.sig r24=f114 }
  1015. { .mii; (p7) add carry3=1,carry3
  1016. cmp.ltu p7,p0=r18,r17
  1017. add r19=r19,r18 };;
  1018. { .mfb; getf.sig r25=f105 }
  1019. { .mii; (p7) add carry3=1,carry3
  1020. cmp.ltu p7,p0=r19,r18
  1021. add r20=r20,r19 };;
  1022. { .mfb; getf.sig r26=f96 }
  1023. { .mii; (p7) add carry3=1,carry3
  1024. cmp.ltu p7,p0=r20,r19
  1025. add r21=r21,r20 };;
  1026. { .mfb; getf.sig r27=f87 }
  1027. { .mii; (p7) add carry3=1,carry3
  1028. cmp.ltu p7,p0=r21,r20
  1029. add r21=r21,carry1 };;
  1030. { .mib; getf.sig r28=f78
  1031. add r25=r25,r24 }
  1032. { .mib; (p7) add carry3=1,carry3
  1033. cmp.ltu p7,p8=r21,carry1};;
  1034. { .mii; st8 [r32]=r21,16
  1035. (p7) add carry2=1,carry3
  1036. (p8) add carry2=0,carry3 }
  1037. { .mii; mov carry1=0
  1038. cmp.ltu p6,p0=r25,r24
  1039. add r26=r26,r25 };;
  1040. { .mfb; getf.sig r16=f115 }
  1041. { .mii;
  1042. (p6) add carry1=1,carry1
  1043. cmp.ltu p6,p0=r26,r25
  1044. add r27=r27,r26 };;
  1045. { .mfb; getf.sig r17=f106 }
  1046. { .mii;
  1047. (p6) add carry1=1,carry1
  1048. cmp.ltu p6,p0=r27,r26
  1049. add r28=r28,r27 };;
  1050. { .mfb; getf.sig r18=f97 }
  1051. { .mii;
  1052. (p6) add carry1=1,carry1
  1053. cmp.ltu p6,p0=r28,r27
  1054. add r28=r28,carry2 };;
  1055. { .mib; getf.sig r19=f88
  1056. add r17=r17,r16 }
  1057. { .mib;
  1058. (p6) add carry1=1,carry1
  1059. cmp.ltu p6,p0=r28,carry2 };;
  1060. { .mii; st8 [r33]=r28,16
  1061. (p6) add carry1=1,carry1 }
  1062. { .mii; mov carry2=0
  1063. cmp.ltu p7,p0=r17,r16
  1064. add r18=r18,r17 };;
  1065. { .mfb; getf.sig r24=f116 }
  1066. { .mii; (p7) add carry2=1,carry2
  1067. cmp.ltu p7,p0=r18,r17
  1068. add r19=r19,r18 };;
  1069. { .mfb; getf.sig r25=f107 }
  1070. { .mii; (p7) add carry2=1,carry2
  1071. cmp.ltu p7,p0=r19,r18
  1072. add r19=r19,carry1 };;
  1073. { .mfb; getf.sig r26=f98 }
  1074. { .mii; (p7) add carry2=1,carry2
  1075. cmp.ltu p7,p0=r19,carry1};;
  1076. { .mii; st8 [r32]=r19,16
  1077. (p7) add carry2=1,carry2 }
  1078. { .mfb; add r25=r25,r24 };;
  1079. { .mfb; getf.sig r16=f117 }
  1080. { .mii; mov carry1=0
  1081. cmp.ltu p6,p0=r25,r24
  1082. add r26=r26,r25 };;
  1083. { .mfb; getf.sig r17=f108 }
  1084. { .mii;
  1085. (p6) add carry1=1,carry1
  1086. cmp.ltu p6,p0=r26,r25
  1087. add r26=r26,carry2 };;
  1088. { .mfb; nop.m 0x0 }
  1089. { .mii;
  1090. (p6) add carry1=1,carry1
  1091. cmp.ltu p6,p0=r26,carry2 };;
  1092. { .mii; st8 [r33]=r26,16
  1093. (p6) add carry1=1,carry1 }
  1094. { .mfb; add r17=r17,r16 };;
  1095. { .mfb; getf.sig r24=f118 }
  1096. { .mii; mov carry2=0
  1097. cmp.ltu p7,p0=r17,r16
  1098. add r17=r17,carry1 };;
  1099. { .mii; (p7) add carry2=1,carry2
  1100. cmp.ltu p7,p0=r17,carry1};;
  1101. { .mii; st8 [r32]=r17
  1102. (p7) add carry2=1,carry2 };;
  1103. { .mfb; add r24=r24,carry2 };;
  1104. { .mib; st8 [r33]=r24 }
  1105. { .mib; rum 1<<5 // clear um.mfh
  1106. br.ret.sptk.many b0 };;
  1107. .endp bn_mul_comba8#
  1108. #undef carry3
  1109. #undef carry2
  1110. #undef carry1
  1111. #endif
  1112. #if 1
  1113. // It's possible to make it faster (see comment to bn_sqr_comba8), but
  1114. // I reckon it doesn't worth the effort. Basically because the routine
  1115. // (actually both of them) practically never called... So I just play
  1116. // same trick as with bn_sqr_comba8.
  1117. //
  1118. // void bn_sqr_comba4(BN_ULONG *r, BN_ULONG *a)
  1119. //
  1120. .global bn_sqr_comba4#
  1121. .proc bn_sqr_comba4#
  1122. .align 64
  1123. bn_sqr_comba4:
  1124. .prologue
  1125. .save ar.pfs,r2
  1126. #if defined(_HPUX_SOURCE) && !defined(_LP64)
  1127. { .mii; alloc r2=ar.pfs,2,1,0,0
  1128. addp4 r32=0,r32
  1129. addp4 r33=0,r33 };;
  1130. { .mii;
  1131. #else
  1132. { .mii; alloc r2=ar.pfs,2,1,0,0
  1133. #endif
  1134. mov r34=r33
  1135. add r14=8,r33 };;
  1136. .body
  1137. { .mii; add r17=8,r34
  1138. add r15=16,r33
  1139. add r18=16,r34 }
  1140. { .mfb; add r16=24,r33
  1141. br .L_cheat_entry_point4 };;
  1142. .endp bn_sqr_comba4#
  1143. #endif
  1144. #if 1
  1145. // Runs in ~115 cycles and ~4.5 times faster than C. Well, whatever...
  1146. //
  1147. // void bn_mul_comba4(BN_ULONG *r, BN_ULONG *a, BN_ULONG *b)
  1148. //
  1149. #define carry1 r14
  1150. #define carry2 r15
  1151. .global bn_mul_comba4#
  1152. .proc bn_mul_comba4#
  1153. .align 64
  1154. bn_mul_comba4:
  1155. .prologue
  1156. .save ar.pfs,r2
  1157. #if defined(_HPUX_SOURCE) && !defined(_LP64)
  1158. { .mii; alloc r2=ar.pfs,3,0,0,0
  1159. addp4 r33=0,r33
  1160. addp4 r34=0,r34 };;
  1161. { .mii; addp4 r32=0,r32
  1162. #else
  1163. { .mii; alloc r2=ar.pfs,3,0,0,0
  1164. #endif
  1165. add r14=8,r33
  1166. add r17=8,r34 }
  1167. .body
  1168. { .mii; add r15=16,r33
  1169. add r18=16,r34
  1170. add r16=24,r33 };;
  1171. .L_cheat_entry_point4:
  1172. { .mmi; add r19=24,r34
  1173. ldf8 f32=[r33] }
  1174. { .mmi; ldf8 f120=[r34]
  1175. ldf8 f121=[r17] };;
  1176. { .mmi; ldf8 f122=[r18]
  1177. ldf8 f123=[r19] }
  1178. { .mmi; ldf8 f33=[r14]
  1179. ldf8 f34=[r15] }
  1180. { .mfi; ldf8 f35=[r16]
  1181. xma.hu f41=f32,f120,f0 }
  1182. { .mfi; xma.lu f40=f32,f120,f0 };;
  1183. { .mfi; xma.hu f51=f32,f121,f0 }
  1184. { .mfi; xma.lu f50=f32,f121,f0 };;
  1185. { .mfi; xma.hu f61=f32,f122,f0 }
  1186. { .mfi; xma.lu f60=f32,f122,f0 };;
  1187. { .mfi; xma.hu f71=f32,f123,f0 }
  1188. { .mfi; xma.lu f70=f32,f123,f0 };;//
  1189. // Major stall takes place here, and 3 more places below. Result from
  1190. // first xma is not available for another 3 ticks.
  1191. { .mfi; getf.sig r16=f40
  1192. xma.hu f42=f33,f120,f41
  1193. add r33=8,r32 }
  1194. { .mfi; xma.lu f41=f33,f120,f41 };;
  1195. { .mfi; getf.sig r24=f50
  1196. xma.hu f52=f33,f121,f51 }
  1197. { .mfi; xma.lu f51=f33,f121,f51 };;
  1198. { .mfi; st8 [r32]=r16,16
  1199. xma.hu f62=f33,f122,f61 }
  1200. { .mfi; xma.lu f61=f33,f122,f61 };;
  1201. { .mfi; xma.hu f72=f33,f123,f71 }
  1202. { .mfi; xma.lu f71=f33,f123,f71 };;//
  1203. //-------------------------------------------------//
  1204. { .mfi; getf.sig r25=f41
  1205. xma.hu f43=f34,f120,f42 }
  1206. { .mfi; xma.lu f42=f34,f120,f42 };;
  1207. { .mfi; getf.sig r16=f60
  1208. xma.hu f53=f34,f121,f52 }
  1209. { .mfi; xma.lu f52=f34,f121,f52 };;
  1210. { .mfi; getf.sig r17=f51
  1211. xma.hu f63=f34,f122,f62
  1212. add r25=r25,r24 }
  1213. { .mfi; mov carry1=0
  1214. xma.lu f62=f34,f122,f62 };;
  1215. { .mfi; st8 [r33]=r25,16
  1216. xma.hu f73=f34,f123,f72
  1217. cmp.ltu p6,p0=r25,r24 }
  1218. { .mfi; xma.lu f72=f34,f123,f72 };;//
  1219. //-------------------------------------------------//
  1220. { .mfi; getf.sig r18=f42
  1221. xma.hu f44=f35,f120,f43
  1222. (p6) add carry1=1,carry1 }
  1223. { .mfi; add r17=r17,r16
  1224. xma.lu f43=f35,f120,f43
  1225. mov carry2=0 };;
  1226. { .mfi; getf.sig r24=f70
  1227. xma.hu f54=f35,f121,f53
  1228. cmp.ltu p7,p0=r17,r16 }
  1229. { .mfi; xma.lu f53=f35,f121,f53 };;
  1230. { .mfi; getf.sig r25=f61
  1231. xma.hu f64=f35,f122,f63
  1232. add r18=r18,r17 }
  1233. { .mfi; xma.lu f63=f35,f122,f63
  1234. (p7) add carry2=1,carry2 };;
  1235. { .mfi; getf.sig r26=f52
  1236. xma.hu f74=f35,f123,f73
  1237. cmp.ltu p7,p0=r18,r17 }
  1238. { .mfi; xma.lu f73=f35,f123,f73
  1239. add r18=r18,carry1 };;
  1240. //-------------------------------------------------//
  1241. { .mii; st8 [r32]=r18,16
  1242. (p7) add carry2=1,carry2
  1243. cmp.ltu p7,p0=r18,carry1 };;
  1244. { .mfi; getf.sig r27=f43 // last major stall
  1245. (p7) add carry2=1,carry2 };;
  1246. { .mii; getf.sig r16=f71
  1247. add r25=r25,r24
  1248. mov carry1=0 };;
  1249. { .mii; getf.sig r17=f62
  1250. cmp.ltu p6,p0=r25,r24
  1251. add r26=r26,r25 };;
  1252. { .mii;
  1253. (p6) add carry1=1,carry1
  1254. cmp.ltu p6,p0=r26,r25
  1255. add r27=r27,r26 };;
  1256. { .mii;
  1257. (p6) add carry1=1,carry1
  1258. cmp.ltu p6,p0=r27,r26
  1259. add r27=r27,carry2 };;
  1260. { .mii; getf.sig r18=f53
  1261. (p6) add carry1=1,carry1
  1262. cmp.ltu p6,p0=r27,carry2 };;
  1263. { .mfi; st8 [r33]=r27,16
  1264. (p6) add carry1=1,carry1 }
  1265. { .mii; getf.sig r19=f44
  1266. add r17=r17,r16
  1267. mov carry2=0 };;
  1268. { .mii; getf.sig r24=f72
  1269. cmp.ltu p7,p0=r17,r16
  1270. add r18=r18,r17 };;
  1271. { .mii; (p7) add carry2=1,carry2
  1272. cmp.ltu p7,p0=r18,r17
  1273. add r19=r19,r18 };;
  1274. { .mii; (p7) add carry2=1,carry2
  1275. cmp.ltu p7,p0=r19,r18
  1276. add r19=r19,carry1 };;
  1277. { .mii; getf.sig r25=f63
  1278. (p7) add carry2=1,carry2
  1279. cmp.ltu p7,p0=r19,carry1};;
  1280. { .mii; st8 [r32]=r19,16
  1281. (p7) add carry2=1,carry2 }
  1282. { .mii; getf.sig r26=f54
  1283. add r25=r25,r24
  1284. mov carry1=0 };;
  1285. { .mii; getf.sig r16=f73
  1286. cmp.ltu p6,p0=r25,r24
  1287. add r26=r26,r25 };;
  1288. { .mii;
  1289. (p6) add carry1=1,carry1
  1290. cmp.ltu p6,p0=r26,r25
  1291. add r26=r26,carry2 };;
  1292. { .mii; getf.sig r17=f64
  1293. (p6) add carry1=1,carry1
  1294. cmp.ltu p6,p0=r26,carry2 };;
  1295. { .mii; st8 [r33]=r26,16
  1296. (p6) add carry1=1,carry1 }
  1297. { .mii; getf.sig r24=f74
  1298. add r17=r17,r16
  1299. mov carry2=0 };;
  1300. { .mii; cmp.ltu p7,p0=r17,r16
  1301. add r17=r17,carry1 };;
  1302. { .mii; (p7) add carry2=1,carry2
  1303. cmp.ltu p7,p0=r17,carry1};;
  1304. { .mii; st8 [r32]=r17,16
  1305. (p7) add carry2=1,carry2 };;
  1306. { .mii; add r24=r24,carry2 };;
  1307. { .mii; st8 [r33]=r24 }
  1308. { .mib; rum 1<<5 // clear um.mfh
  1309. br.ret.sptk.many b0 };;
  1310. .endp bn_mul_comba4#
  1311. #undef carry2
  1312. #undef carry1
  1313. #endif
  1314. #if 1
  1315. //
  1316. // BN_ULONG bn_div_words(BN_ULONG h, BN_ULONG l, BN_ULONG d)
  1317. //
  1318. // In the nutshell it's a port of my MIPS III/IV implementation.
  1319. //
  1320. #define AT r14
  1321. #define H r16
  1322. #define HH r20
  1323. #define L r17
  1324. #define D r18
  1325. #define DH r22
  1326. #define I r21
  1327. #if 0
  1328. // Some preprocessors (most notably HP-UX) appear to be allergic to
  1329. // macros enclosed to parenthesis [as these three were].
  1330. #define cont p16
  1331. #define break p0 // p20
  1332. #define equ p24
  1333. #else
  1334. cont=p16
  1335. break=p0
  1336. equ=p24
  1337. #endif
  1338. .global abort#
  1339. .global bn_div_words#
  1340. .proc bn_div_words#
  1341. .align 64
  1342. bn_div_words:
  1343. .prologue
  1344. .save ar.pfs,r2
  1345. { .mii; alloc r2=ar.pfs,3,5,0,8
  1346. .save b0,r3
  1347. mov r3=b0
  1348. .save pr,r10
  1349. mov r10=pr };;
  1350. { .mmb; cmp.eq p6,p0=r34,r0
  1351. mov r8=-1
  1352. (p6) br.ret.spnt.many b0 };;
  1353. .body
  1354. { .mii; mov H=r32 // save h
  1355. mov ar.ec=0 // don't rotate at exit
  1356. mov pr.rot=0 }
  1357. { .mii; mov L=r33 // save l
  1358. mov r36=r0 };;
  1359. .L_divw_shift: // -vv- note signed comparison
  1360. { .mfi; (p0) cmp.lt p16,p0=r0,r34 // d
  1361. (p0) shladd r33=r34,1,r0 }
  1362. { .mfb; (p0) add r35=1,r36
  1363. (p0) nop.f 0x0
  1364. (p16) br.wtop.dpnt .L_divw_shift };;
  1365. { .mii; mov D=r34
  1366. shr.u DH=r34,32
  1367. sub r35=64,r36 };;
  1368. { .mii; setf.sig f7=DH
  1369. shr.u AT=H,r35
  1370. mov I=r36 };;
  1371. { .mib; cmp.ne p6,p0=r0,AT
  1372. shl H=H,r36
  1373. (p6) br.call.spnt.clr b0=abort };; // overflow, die...
  1374. { .mfi; fcvt.xuf.s1 f7=f7
  1375. shr.u AT=L,r35 };;
  1376. { .mii; shl L=L,r36
  1377. or H=H,AT };;
  1378. { .mii; nop.m 0x0
  1379. cmp.leu p6,p0=D,H;;
  1380. (p6) sub H=H,D }
  1381. { .mlx; setf.sig f14=D
  1382. movl AT=0xffffffff };;
  1383. ///////////////////////////////////////////////////////////
  1384. { .mii; setf.sig f6=H
  1385. shr.u HH=H,32;;
  1386. cmp.eq p6,p7=HH,DH };;
  1387. { .mfb;
  1388. (p6) setf.sig f8=AT
  1389. (p7) fcvt.xuf.s1 f6=f6
  1390. (p7) br.call.sptk b6=.L_udiv64_32_b6 };;
  1391. { .mfi; getf.sig r33=f8 // q
  1392. xmpy.lu f9=f8,f14 }
  1393. { .mfi; xmpy.hu f10=f8,f14
  1394. shrp H=H,L,32 };;
  1395. { .mmi; getf.sig r35=f9 // tl
  1396. getf.sig r31=f10 };; // th
  1397. .L_divw_1st_iter:
  1398. { .mii; (p0) add r32=-1,r33
  1399. (p0) cmp.eq equ,cont=HH,r31 };;
  1400. { .mii; (p0) cmp.ltu p8,p0=r35,D
  1401. (p0) sub r34=r35,D
  1402. (equ) cmp.leu break,cont=r35,H };;
  1403. { .mib; (cont) cmp.leu cont,break=HH,r31
  1404. (p8) add r31=-1,r31
  1405. (cont) br.wtop.spnt .L_divw_1st_iter };;
  1406. ///////////////////////////////////////////////////////////
  1407. { .mii; sub H=H,r35
  1408. shl r8=r33,32
  1409. shl L=L,32 };;
  1410. ///////////////////////////////////////////////////////////
  1411. { .mii; setf.sig f6=H
  1412. shr.u HH=H,32;;
  1413. cmp.eq p6,p7=HH,DH };;
  1414. { .mfb;
  1415. (p6) setf.sig f8=AT
  1416. (p7) fcvt.xuf.s1 f6=f6
  1417. (p7) br.call.sptk b6=.L_udiv64_32_b6 };;
  1418. { .mfi; getf.sig r33=f8 // q
  1419. xmpy.lu f9=f8,f14 }
  1420. { .mfi; xmpy.hu f10=f8,f14
  1421. shrp H=H,L,32 };;
  1422. { .mmi; getf.sig r35=f9 // tl
  1423. getf.sig r31=f10 };; // th
  1424. .L_divw_2nd_iter:
  1425. { .mii; (p0) add r32=-1,r33
  1426. (p0) cmp.eq equ,cont=HH,r31 };;
  1427. { .mii; (p0) cmp.ltu p8,p0=r35,D
  1428. (p0) sub r34=r35,D
  1429. (equ) cmp.leu break,cont=r35,H };;
  1430. { .mib; (cont) cmp.leu cont,break=HH,r31
  1431. (p8) add r31=-1,r31
  1432. (cont) br.wtop.spnt .L_divw_2nd_iter };;
  1433. ///////////////////////////////////////////////////////////
  1434. { .mii; sub H=H,r35
  1435. or r8=r8,r33
  1436. mov ar.pfs=r2 };;
  1437. { .mii; shr.u r9=H,I // remainder if anybody wants it
  1438. mov pr=r10,0x1ffff }
  1439. { .mfb; br.ret.sptk.many b0 };;
  1440. // Unsigned 64 by 32 (well, by 64 for the moment) bit integer division
  1441. // procedure.
  1442. //
  1443. // inputs: f6 = (double)a, f7 = (double)b
  1444. // output: f8 = (int)(a/b)
  1445. // clobbered: f8,f9,f10,f11,pred
  1446. pred=p15
  1447. // One can argue that this snippet is copyrighted to Intel
  1448. // Corporation, as it's essentially identical to one of those
  1449. // found in "Divide, Square Root and Remainder" section at
  1450. // http://www.intel.com/software/products/opensource/libraries/num.htm.
  1451. // Yes, I admit that the referred code was used as template,
  1452. // but after I realized that there hardly is any other instruction
  1453. // sequence which would perform this operation. I mean I figure that
  1454. // any independent attempt to implement high-performance division
  1455. // will result in code virtually identical to the Intel code. It
  1456. // should be noted though that below division kernel is 1 cycle
  1457. // faster than Intel one (note commented splits:-), not to mention
  1458. // original prologue (rather lack of one) and epilogue.
  1459. .align 32
  1460. .skip 16
  1461. .L_udiv64_32_b6:
  1462. frcpa.s1 f8,pred=f6,f7;; // [0] y0 = 1 / b
  1463. (pred) fnma.s1 f9=f7,f8,f1 // [5] e0 = 1 - b * y0
  1464. (pred) fmpy.s1 f10=f6,f8;; // [5] q0 = a * y0
  1465. (pred) fmpy.s1 f11=f9,f9 // [10] e1 = e0 * e0
  1466. (pred) fma.s1 f10=f9,f10,f10;; // [10] q1 = q0 + e0 * q0
  1467. (pred) fma.s1 f8=f9,f8,f8 //;; // [15] y1 = y0 + e0 * y0
  1468. (pred) fma.s1 f9=f11,f10,f10;; // [15] q2 = q1 + e1 * q1
  1469. (pred) fma.s1 f8=f11,f8,f8 //;; // [20] y2 = y1 + e1 * y1
  1470. (pred) fnma.s1 f10=f7,f9,f6;; // [20] r2 = a - b * q2
  1471. (pred) fma.s1 f8=f10,f8,f9;; // [25] q3 = q2 + r2 * y2
  1472. fcvt.fxu.trunc.s1 f8=f8 // [30] q = trunc(q3)
  1473. br.ret.sptk.many b6;;
  1474. .endp bn_div_words#
  1475. #endif