x86_64-mont5.pl 85 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942
  1. #! /usr/bin/env perl
  2. # Copyright 2011-2016 The OpenSSL Project Authors. All Rights Reserved.
  3. #
  4. # Licensed under the OpenSSL license (the "License"). You may not use
  5. # this file except in compliance with the License. You can obtain a copy
  6. # in the file LICENSE in the source distribution or at
  7. # https://www.openssl.org/source/license.html
  8. # ====================================================================
  9. # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
  10. # project. The module is, however, dual licensed under OpenSSL and
  11. # CRYPTOGAMS licenses depending on where you obtain it. For further
  12. # details see http://www.openssl.org/~appro/cryptogams/.
  13. # ====================================================================
  14. # August 2011.
  15. #
  16. # Companion to x86_64-mont.pl that optimizes cache-timing attack
  17. # countermeasures. The subroutines are produced by replacing bp[i]
  18. # references in their x86_64-mont.pl counterparts with cache-neutral
  19. # references to powers table computed in BN_mod_exp_mont_consttime.
  20. # In addition subroutine that scatters elements of the powers table
  21. # is implemented, so that scatter-/gathering can be tuned without
  22. # bn_exp.c modifications.
  23. # August 2013.
  24. #
  25. # Add MULX/AD*X code paths and additional interfaces to optimize for
  26. # branch prediction unit. For input lengths that are multiples of 8
  27. # the np argument is not just modulus value, but one interleaved
  28. # with 0. This is to optimize post-condition...
  29. $flavour = shift;
  30. $output = shift;
  31. if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
  32. $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
  33. $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
  34. ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
  35. ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
  36. die "can't locate x86_64-xlate.pl";
  37. open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\"";
  38. *STDOUT=*OUT;
  39. if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
  40. =~ /GNU assembler version ([2-9]\.[0-9]+)/) {
  41. $addx = ($1>=2.23);
  42. }
  43. if (!$addx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) &&
  44. `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) {
  45. $addx = ($1>=2.10);
  46. }
  47. if (!$addx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) &&
  48. `ml64 2>&1` =~ /Version ([0-9]+)\./) {
  49. $addx = ($1>=12);
  50. }
  51. if (!$addx && `$ENV{CC} -v 2>&1` =~ /((?:^clang|LLVM) version|.*based on LLVM) ([3-9])\.([0-9]+)/) {
  52. my $ver = $2 + $3/100.0; # 3.1->3.01, 3.10->3.10
  53. $addx = ($ver>=3.03);
  54. }
  55. # int bn_mul_mont_gather5(
  56. $rp="%rdi"; # BN_ULONG *rp,
  57. $ap="%rsi"; # const BN_ULONG *ap,
  58. $bp="%rdx"; # const BN_ULONG *bp,
  59. $np="%rcx"; # const BN_ULONG *np,
  60. $n0="%r8"; # const BN_ULONG *n0,
  61. $num="%r9"; # int num,
  62. # int idx); # 0 to 2^5-1, "index" in $bp holding
  63. # pre-computed powers of a', interlaced
  64. # in such manner that b[0] is $bp[idx],
  65. # b[1] is [2^5+idx], etc.
  66. $lo0="%r10";
  67. $hi0="%r11";
  68. $hi1="%r13";
  69. $i="%r14";
  70. $j="%r15";
  71. $m0="%rbx";
  72. $m1="%rbp";
  73. $code=<<___;
  74. .text
  75. .extern OPENSSL_ia32cap_P
  76. .globl bn_mul_mont_gather5
  77. .type bn_mul_mont_gather5,\@function,6
  78. .align 64
  79. bn_mul_mont_gather5:
  80. .cfi_startproc
  81. mov ${num}d,${num}d
  82. mov %rsp,%rax
  83. .cfi_def_cfa_register %rax
  84. test \$7,${num}d
  85. jnz .Lmul_enter
  86. ___
  87. $code.=<<___ if ($addx);
  88. mov OPENSSL_ia32cap_P+8(%rip),%r11d
  89. ___
  90. $code.=<<___;
  91. jmp .Lmul4x_enter
  92. .align 16
  93. .Lmul_enter:
  94. movd `($win64?56:8)`(%rsp),%xmm5 # load 7th argument
  95. push %rbx
  96. .cfi_push %rbx
  97. push %rbp
  98. .cfi_push %rbp
  99. push %r12
  100. .cfi_push %r12
  101. push %r13
  102. .cfi_push %r13
  103. push %r14
  104. .cfi_push %r14
  105. push %r15
  106. .cfi_push %r15
  107. neg $num
  108. mov %rsp,%r11
  109. lea -280(%rsp,$num,8),%r10 # future alloca(8*(num+2)+256+8)
  110. neg $num # restore $num
  111. and \$-1024,%r10 # minimize TLB usage
  112. # An OS-agnostic version of __chkstk.
  113. #
  114. # Some OSes (Windows) insist on stack being "wired" to
  115. # physical memory in strictly sequential manner, i.e. if stack
  116. # allocation spans two pages, then reference to farmost one can
  117. # be punishable by SEGV. But page walking can do good even on
  118. # other OSes, because it guarantees that villain thread hits
  119. # the guard page before it can make damage to innocent one...
  120. sub %r10,%r11
  121. and \$-4096,%r11
  122. lea (%r10,%r11),%rsp
  123. mov (%rsp),%r11
  124. cmp %r10,%rsp
  125. ja .Lmul_page_walk
  126. jmp .Lmul_page_walk_done
  127. .Lmul_page_walk:
  128. lea -4096(%rsp),%rsp
  129. mov (%rsp),%r11
  130. cmp %r10,%rsp
  131. ja .Lmul_page_walk
  132. .Lmul_page_walk_done:
  133. lea .Linc(%rip),%r10
  134. mov %rax,8(%rsp,$num,8) # tp[num+1]=%rsp
  135. .cfi_cfa_expression %rsp+8,$num,8,mul,plus,deref,+8
  136. .Lmul_body:
  137. lea 128($bp),%r12 # reassign $bp (+size optimization)
  138. ___
  139. $bp="%r12";
  140. $STRIDE=2**5*8; # 5 is "window size"
  141. $N=$STRIDE/4; # should match cache line size
  142. $code.=<<___;
  143. movdqa 0(%r10),%xmm0 # 00000001000000010000000000000000
  144. movdqa 16(%r10),%xmm1 # 00000002000000020000000200000002
  145. lea 24-112(%rsp,$num,8),%r10# place the mask after tp[num+3] (+ICache optimization)
  146. and \$-16,%r10
  147. pshufd \$0,%xmm5,%xmm5 # broadcast index
  148. movdqa %xmm1,%xmm4
  149. movdqa %xmm1,%xmm2
  150. ___
  151. ########################################################################
  152. # calculate mask by comparing 0..31 to index and save result to stack
  153. #
  154. $code.=<<___;
  155. paddd %xmm0,%xmm1
  156. pcmpeqd %xmm5,%xmm0 # compare to 1,0
  157. .byte 0x67
  158. movdqa %xmm4,%xmm3
  159. ___
  160. for($k=0;$k<$STRIDE/16-4;$k+=4) {
  161. $code.=<<___;
  162. paddd %xmm1,%xmm2
  163. pcmpeqd %xmm5,%xmm1 # compare to 3,2
  164. movdqa %xmm0,`16*($k+0)+112`(%r10)
  165. movdqa %xmm4,%xmm0
  166. paddd %xmm2,%xmm3
  167. pcmpeqd %xmm5,%xmm2 # compare to 5,4
  168. movdqa %xmm1,`16*($k+1)+112`(%r10)
  169. movdqa %xmm4,%xmm1
  170. paddd %xmm3,%xmm0
  171. pcmpeqd %xmm5,%xmm3 # compare to 7,6
  172. movdqa %xmm2,`16*($k+2)+112`(%r10)
  173. movdqa %xmm4,%xmm2
  174. paddd %xmm0,%xmm1
  175. pcmpeqd %xmm5,%xmm0
  176. movdqa %xmm3,`16*($k+3)+112`(%r10)
  177. movdqa %xmm4,%xmm3
  178. ___
  179. }
  180. $code.=<<___; # last iteration can be optimized
  181. paddd %xmm1,%xmm2
  182. pcmpeqd %xmm5,%xmm1
  183. movdqa %xmm0,`16*($k+0)+112`(%r10)
  184. paddd %xmm2,%xmm3
  185. .byte 0x67
  186. pcmpeqd %xmm5,%xmm2
  187. movdqa %xmm1,`16*($k+1)+112`(%r10)
  188. pcmpeqd %xmm5,%xmm3
  189. movdqa %xmm2,`16*($k+2)+112`(%r10)
  190. pand `16*($k+0)-128`($bp),%xmm0 # while it's still in register
  191. pand `16*($k+1)-128`($bp),%xmm1
  192. pand `16*($k+2)-128`($bp),%xmm2
  193. movdqa %xmm3,`16*($k+3)+112`(%r10)
  194. pand `16*($k+3)-128`($bp),%xmm3
  195. por %xmm2,%xmm0
  196. por %xmm3,%xmm1
  197. ___
  198. for($k=0;$k<$STRIDE/16-4;$k+=4) {
  199. $code.=<<___;
  200. movdqa `16*($k+0)-128`($bp),%xmm4
  201. movdqa `16*($k+1)-128`($bp),%xmm5
  202. movdqa `16*($k+2)-128`($bp),%xmm2
  203. pand `16*($k+0)+112`(%r10),%xmm4
  204. movdqa `16*($k+3)-128`($bp),%xmm3
  205. pand `16*($k+1)+112`(%r10),%xmm5
  206. por %xmm4,%xmm0
  207. pand `16*($k+2)+112`(%r10),%xmm2
  208. por %xmm5,%xmm1
  209. pand `16*($k+3)+112`(%r10),%xmm3
  210. por %xmm2,%xmm0
  211. por %xmm3,%xmm1
  212. ___
  213. }
  214. $code.=<<___;
  215. por %xmm1,%xmm0
  216. pshufd \$0x4e,%xmm0,%xmm1
  217. por %xmm1,%xmm0
  218. lea $STRIDE($bp),$bp
  219. movq %xmm0,$m0 # m0=bp[0]
  220. mov ($n0),$n0 # pull n0[0] value
  221. mov ($ap),%rax
  222. xor $i,$i # i=0
  223. xor $j,$j # j=0
  224. mov $n0,$m1
  225. mulq $m0 # ap[0]*bp[0]
  226. mov %rax,$lo0
  227. mov ($np),%rax
  228. imulq $lo0,$m1 # "tp[0]"*n0
  229. mov %rdx,$hi0
  230. mulq $m1 # np[0]*m1
  231. add %rax,$lo0 # discarded
  232. mov 8($ap),%rax
  233. adc \$0,%rdx
  234. mov %rdx,$hi1
  235. lea 1($j),$j # j++
  236. jmp .L1st_enter
  237. .align 16
  238. .L1st:
  239. add %rax,$hi1
  240. mov ($ap,$j,8),%rax
  241. adc \$0,%rdx
  242. add $hi0,$hi1 # np[j]*m1+ap[j]*bp[0]
  243. mov $lo0,$hi0
  244. adc \$0,%rdx
  245. mov $hi1,-16(%rsp,$j,8) # tp[j-1]
  246. mov %rdx,$hi1
  247. .L1st_enter:
  248. mulq $m0 # ap[j]*bp[0]
  249. add %rax,$hi0
  250. mov ($np,$j,8),%rax
  251. adc \$0,%rdx
  252. lea 1($j),$j # j++
  253. mov %rdx,$lo0
  254. mulq $m1 # np[j]*m1
  255. cmp $num,$j
  256. jne .L1st # note that upon exit $j==$num, so
  257. # they can be used interchangeably
  258. add %rax,$hi1
  259. adc \$0,%rdx
  260. add $hi0,$hi1 # np[j]*m1+ap[j]*bp[0]
  261. adc \$0,%rdx
  262. mov $hi1,-16(%rsp,$num,8) # tp[num-1]
  263. mov %rdx,$hi1
  264. mov $lo0,$hi0
  265. xor %rdx,%rdx
  266. add $hi0,$hi1
  267. adc \$0,%rdx
  268. mov $hi1,-8(%rsp,$num,8)
  269. mov %rdx,(%rsp,$num,8) # store upmost overflow bit
  270. lea 1($i),$i # i++
  271. jmp .Louter
  272. .align 16
  273. .Louter:
  274. lea 24+128(%rsp,$num,8),%rdx # where 256-byte mask is (+size optimization)
  275. and \$-16,%rdx
  276. pxor %xmm4,%xmm4
  277. pxor %xmm5,%xmm5
  278. ___
  279. for($k=0;$k<$STRIDE/16;$k+=4) {
  280. $code.=<<___;
  281. movdqa `16*($k+0)-128`($bp),%xmm0
  282. movdqa `16*($k+1)-128`($bp),%xmm1
  283. movdqa `16*($k+2)-128`($bp),%xmm2
  284. movdqa `16*($k+3)-128`($bp),%xmm3
  285. pand `16*($k+0)-128`(%rdx),%xmm0
  286. pand `16*($k+1)-128`(%rdx),%xmm1
  287. por %xmm0,%xmm4
  288. pand `16*($k+2)-128`(%rdx),%xmm2
  289. por %xmm1,%xmm5
  290. pand `16*($k+3)-128`(%rdx),%xmm3
  291. por %xmm2,%xmm4
  292. por %xmm3,%xmm5
  293. ___
  294. }
  295. $code.=<<___;
  296. por %xmm5,%xmm4
  297. pshufd \$0x4e,%xmm4,%xmm0
  298. por %xmm4,%xmm0
  299. lea $STRIDE($bp),$bp
  300. mov ($ap),%rax # ap[0]
  301. movq %xmm0,$m0 # m0=bp[i]
  302. xor $j,$j # j=0
  303. mov $n0,$m1
  304. mov (%rsp),$lo0
  305. mulq $m0 # ap[0]*bp[i]
  306. add %rax,$lo0 # ap[0]*bp[i]+tp[0]
  307. mov ($np),%rax
  308. adc \$0,%rdx
  309. imulq $lo0,$m1 # tp[0]*n0
  310. mov %rdx,$hi0
  311. mulq $m1 # np[0]*m1
  312. add %rax,$lo0 # discarded
  313. mov 8($ap),%rax
  314. adc \$0,%rdx
  315. mov 8(%rsp),$lo0 # tp[1]
  316. mov %rdx,$hi1
  317. lea 1($j),$j # j++
  318. jmp .Linner_enter
  319. .align 16
  320. .Linner:
  321. add %rax,$hi1
  322. mov ($ap,$j,8),%rax
  323. adc \$0,%rdx
  324. add $lo0,$hi1 # np[j]*m1+ap[j]*bp[i]+tp[j]
  325. mov (%rsp,$j,8),$lo0
  326. adc \$0,%rdx
  327. mov $hi1,-16(%rsp,$j,8) # tp[j-1]
  328. mov %rdx,$hi1
  329. .Linner_enter:
  330. mulq $m0 # ap[j]*bp[i]
  331. add %rax,$hi0
  332. mov ($np,$j,8),%rax
  333. adc \$0,%rdx
  334. add $hi0,$lo0 # ap[j]*bp[i]+tp[j]
  335. mov %rdx,$hi0
  336. adc \$0,$hi0
  337. lea 1($j),$j # j++
  338. mulq $m1 # np[j]*m1
  339. cmp $num,$j
  340. jne .Linner # note that upon exit $j==$num, so
  341. # they can be used interchangeably
  342. add %rax,$hi1
  343. adc \$0,%rdx
  344. add $lo0,$hi1 # np[j]*m1+ap[j]*bp[i]+tp[j]
  345. mov (%rsp,$num,8),$lo0
  346. adc \$0,%rdx
  347. mov $hi1,-16(%rsp,$num,8) # tp[num-1]
  348. mov %rdx,$hi1
  349. xor %rdx,%rdx
  350. add $hi0,$hi1
  351. adc \$0,%rdx
  352. add $lo0,$hi1 # pull upmost overflow bit
  353. adc \$0,%rdx
  354. mov $hi1,-8(%rsp,$num,8)
  355. mov %rdx,(%rsp,$num,8) # store upmost overflow bit
  356. lea 1($i),$i # i++
  357. cmp $num,$i
  358. jb .Louter
  359. xor $i,$i # i=0 and clear CF!
  360. mov (%rsp),%rax # tp[0]
  361. lea (%rsp),$ap # borrow ap for tp
  362. mov $num,$j # j=num
  363. jmp .Lsub
  364. .align 16
  365. .Lsub: sbb ($np,$i,8),%rax
  366. mov %rax,($rp,$i,8) # rp[i]=tp[i]-np[i]
  367. mov 8($ap,$i,8),%rax # tp[i+1]
  368. lea 1($i),$i # i++
  369. dec $j # doesn't affect CF!
  370. jnz .Lsub
  371. sbb \$0,%rax # handle upmost overflow bit
  372. xor $i,$i
  373. and %rax,$ap
  374. not %rax
  375. mov $rp,$np
  376. and %rax,$np
  377. mov $num,$j # j=num
  378. or $np,$ap # ap=borrow?tp:rp
  379. .align 16
  380. .Lcopy: # copy or in-place refresh
  381. mov ($ap,$i,8),%rax
  382. mov $i,(%rsp,$i,8) # zap temporary vector
  383. mov %rax,($rp,$i,8) # rp[i]=tp[i]
  384. lea 1($i),$i
  385. sub \$1,$j
  386. jnz .Lcopy
  387. mov 8(%rsp,$num,8),%rsi # restore %rsp
  388. .cfi_def_cfa %rsi,8
  389. mov \$1,%rax
  390. mov -48(%rsi),%r15
  391. .cfi_restore %r15
  392. mov -40(%rsi),%r14
  393. .cfi_restore %r14
  394. mov -32(%rsi),%r13
  395. .cfi_restore %r13
  396. mov -24(%rsi),%r12
  397. .cfi_restore %r12
  398. mov -16(%rsi),%rbp
  399. .cfi_restore %rbp
  400. mov -8(%rsi),%rbx
  401. .cfi_restore %rbx
  402. lea (%rsi),%rsp
  403. .cfi_def_cfa_register %rsp
  404. .Lmul_epilogue:
  405. ret
  406. .cfi_endproc
  407. .size bn_mul_mont_gather5,.-bn_mul_mont_gather5
  408. ___
  409. {{{
  410. my @A=("%r10","%r11");
  411. my @N=("%r13","%rdi");
  412. $code.=<<___;
  413. .type bn_mul4x_mont_gather5,\@function,6
  414. .align 32
  415. bn_mul4x_mont_gather5:
  416. .cfi_startproc
  417. .byte 0x67
  418. mov %rsp,%rax
  419. .cfi_def_cfa_register %rax
  420. .Lmul4x_enter:
  421. ___
  422. $code.=<<___ if ($addx);
  423. and \$0x80108,%r11d
  424. cmp \$0x80108,%r11d # check for AD*X+BMI2+BMI1
  425. je .Lmulx4x_enter
  426. ___
  427. $code.=<<___;
  428. push %rbx
  429. .cfi_push %rbx
  430. push %rbp
  431. .cfi_push %rbp
  432. push %r12
  433. .cfi_push %r12
  434. push %r13
  435. .cfi_push %r13
  436. push %r14
  437. .cfi_push %r14
  438. push %r15
  439. .cfi_push %r15
  440. .Lmul4x_prologue:
  441. .byte 0x67
  442. shl \$3,${num}d # convert $num to bytes
  443. lea ($num,$num,2),%r10 # 3*$num in bytes
  444. neg $num # -$num
  445. ##############################################################
  446. # Ensure that stack frame doesn't alias with $rptr+3*$num
  447. # modulo 4096, which covers ret[num], am[num] and n[num]
  448. # (see bn_exp.c). This is done to allow memory disambiguation
  449. # logic do its magic. [Extra [num] is allocated in order
  450. # to align with bn_power5's frame, which is cleansed after
  451. # completing exponentiation. Extra 256 bytes is for power mask
  452. # calculated from 7th argument, the index.]
  453. #
  454. lea -320(%rsp,$num,2),%r11
  455. mov %rsp,%rbp
  456. sub $rp,%r11
  457. and \$4095,%r11
  458. cmp %r11,%r10
  459. jb .Lmul4xsp_alt
  460. sub %r11,%rbp # align with $rp
  461. lea -320(%rbp,$num,2),%rbp # future alloca(frame+2*num*8+256)
  462. jmp .Lmul4xsp_done
  463. .align 32
  464. .Lmul4xsp_alt:
  465. lea 4096-320(,$num,2),%r10
  466. lea -320(%rbp,$num,2),%rbp # future alloca(frame+2*num*8+256)
  467. sub %r10,%r11
  468. mov \$0,%r10
  469. cmovc %r10,%r11
  470. sub %r11,%rbp
  471. .Lmul4xsp_done:
  472. and \$-64,%rbp
  473. mov %rsp,%r11
  474. sub %rbp,%r11
  475. and \$-4096,%r11
  476. lea (%rbp,%r11),%rsp
  477. mov (%rsp),%r10
  478. cmp %rbp,%rsp
  479. ja .Lmul4x_page_walk
  480. jmp .Lmul4x_page_walk_done
  481. .Lmul4x_page_walk:
  482. lea -4096(%rsp),%rsp
  483. mov (%rsp),%r10
  484. cmp %rbp,%rsp
  485. ja .Lmul4x_page_walk
  486. .Lmul4x_page_walk_done:
  487. neg $num
  488. mov %rax,40(%rsp)
  489. .cfi_cfa_expression %rsp+40,deref,+8
  490. .Lmul4x_body:
  491. call mul4x_internal
  492. mov 40(%rsp),%rsi # restore %rsp
  493. .cfi_def_cfa %rsi,8
  494. mov \$1,%rax
  495. mov -48(%rsi),%r15
  496. .cfi_restore %r15
  497. mov -40(%rsi),%r14
  498. .cfi_restore %r14
  499. mov -32(%rsi),%r13
  500. .cfi_restore %r13
  501. mov -24(%rsi),%r12
  502. .cfi_restore %r12
  503. mov -16(%rsi),%rbp
  504. .cfi_restore %rbp
  505. mov -8(%rsi),%rbx
  506. .cfi_restore %rbx
  507. lea (%rsi),%rsp
  508. .cfi_def_cfa_register %rsp
  509. .Lmul4x_epilogue:
  510. ret
  511. .cfi_endproc
  512. .size bn_mul4x_mont_gather5,.-bn_mul4x_mont_gather5
  513. .type mul4x_internal,\@abi-omnipotent
  514. .align 32
  515. mul4x_internal:
  516. shl \$5,$num # $num was in bytes
  517. movd `($win64?56:8)`(%rax),%xmm5 # load 7th argument, index
  518. lea .Linc(%rip),%rax
  519. lea 128(%rdx,$num),%r13 # end of powers table (+size optimization)
  520. shr \$5,$num # restore $num
  521. ___
  522. $bp="%r12";
  523. $STRIDE=2**5*8; # 5 is "window size"
  524. $N=$STRIDE/4; # should match cache line size
  525. $tp=$i;
  526. $code.=<<___;
  527. movdqa 0(%rax),%xmm0 # 00000001000000010000000000000000
  528. movdqa 16(%rax),%xmm1 # 00000002000000020000000200000002
  529. lea 88-112(%rsp,$num),%r10 # place the mask after tp[num+1] (+ICache optimization)
  530. lea 128(%rdx),$bp # size optimization
  531. pshufd \$0,%xmm5,%xmm5 # broadcast index
  532. movdqa %xmm1,%xmm4
  533. .byte 0x67,0x67
  534. movdqa %xmm1,%xmm2
  535. ___
  536. ########################################################################
  537. # calculate mask by comparing 0..31 to index and save result to stack
  538. #
  539. $code.=<<___;
  540. paddd %xmm0,%xmm1
  541. pcmpeqd %xmm5,%xmm0 # compare to 1,0
  542. .byte 0x67
  543. movdqa %xmm4,%xmm3
  544. ___
  545. for($i=0;$i<$STRIDE/16-4;$i+=4) {
  546. $code.=<<___;
  547. paddd %xmm1,%xmm2
  548. pcmpeqd %xmm5,%xmm1 # compare to 3,2
  549. movdqa %xmm0,`16*($i+0)+112`(%r10)
  550. movdqa %xmm4,%xmm0
  551. paddd %xmm2,%xmm3
  552. pcmpeqd %xmm5,%xmm2 # compare to 5,4
  553. movdqa %xmm1,`16*($i+1)+112`(%r10)
  554. movdqa %xmm4,%xmm1
  555. paddd %xmm3,%xmm0
  556. pcmpeqd %xmm5,%xmm3 # compare to 7,6
  557. movdqa %xmm2,`16*($i+2)+112`(%r10)
  558. movdqa %xmm4,%xmm2
  559. paddd %xmm0,%xmm1
  560. pcmpeqd %xmm5,%xmm0
  561. movdqa %xmm3,`16*($i+3)+112`(%r10)
  562. movdqa %xmm4,%xmm3
  563. ___
  564. }
  565. $code.=<<___; # last iteration can be optimized
  566. paddd %xmm1,%xmm2
  567. pcmpeqd %xmm5,%xmm1
  568. movdqa %xmm0,`16*($i+0)+112`(%r10)
  569. paddd %xmm2,%xmm3
  570. .byte 0x67
  571. pcmpeqd %xmm5,%xmm2
  572. movdqa %xmm1,`16*($i+1)+112`(%r10)
  573. pcmpeqd %xmm5,%xmm3
  574. movdqa %xmm2,`16*($i+2)+112`(%r10)
  575. pand `16*($i+0)-128`($bp),%xmm0 # while it's still in register
  576. pand `16*($i+1)-128`($bp),%xmm1
  577. pand `16*($i+2)-128`($bp),%xmm2
  578. movdqa %xmm3,`16*($i+3)+112`(%r10)
  579. pand `16*($i+3)-128`($bp),%xmm3
  580. por %xmm2,%xmm0
  581. por %xmm3,%xmm1
  582. ___
  583. for($i=0;$i<$STRIDE/16-4;$i+=4) {
  584. $code.=<<___;
  585. movdqa `16*($i+0)-128`($bp),%xmm4
  586. movdqa `16*($i+1)-128`($bp),%xmm5
  587. movdqa `16*($i+2)-128`($bp),%xmm2
  588. pand `16*($i+0)+112`(%r10),%xmm4
  589. movdqa `16*($i+3)-128`($bp),%xmm3
  590. pand `16*($i+1)+112`(%r10),%xmm5
  591. por %xmm4,%xmm0
  592. pand `16*($i+2)+112`(%r10),%xmm2
  593. por %xmm5,%xmm1
  594. pand `16*($i+3)+112`(%r10),%xmm3
  595. por %xmm2,%xmm0
  596. por %xmm3,%xmm1
  597. ___
  598. }
  599. $code.=<<___;
  600. por %xmm1,%xmm0
  601. pshufd \$0x4e,%xmm0,%xmm1
  602. por %xmm1,%xmm0
  603. lea $STRIDE($bp),$bp
  604. movq %xmm0,$m0 # m0=bp[0]
  605. mov %r13,16+8(%rsp) # save end of b[num]
  606. mov $rp, 56+8(%rsp) # save $rp
  607. mov ($n0),$n0 # pull n0[0] value
  608. mov ($ap),%rax
  609. lea ($ap,$num),$ap # end of a[num]
  610. neg $num
  611. mov $n0,$m1
  612. mulq $m0 # ap[0]*bp[0]
  613. mov %rax,$A[0]
  614. mov ($np),%rax
  615. imulq $A[0],$m1 # "tp[0]"*n0
  616. lea 64+8(%rsp),$tp
  617. mov %rdx,$A[1]
  618. mulq $m1 # np[0]*m1
  619. add %rax,$A[0] # discarded
  620. mov 8($ap,$num),%rax
  621. adc \$0,%rdx
  622. mov %rdx,$N[1]
  623. mulq $m0
  624. add %rax,$A[1]
  625. mov 8*1($np),%rax
  626. adc \$0,%rdx
  627. mov %rdx,$A[0]
  628. mulq $m1
  629. add %rax,$N[1]
  630. mov 16($ap,$num),%rax
  631. adc \$0,%rdx
  632. add $A[1],$N[1]
  633. lea 4*8($num),$j # j=4
  634. lea 8*4($np),$np
  635. adc \$0,%rdx
  636. mov $N[1],($tp)
  637. mov %rdx,$N[0]
  638. jmp .L1st4x
  639. .align 32
  640. .L1st4x:
  641. mulq $m0 # ap[j]*bp[0]
  642. add %rax,$A[0]
  643. mov -8*2($np),%rax
  644. lea 32($tp),$tp
  645. adc \$0,%rdx
  646. mov %rdx,$A[1]
  647. mulq $m1 # np[j]*m1
  648. add %rax,$N[0]
  649. mov -8($ap,$j),%rax
  650. adc \$0,%rdx
  651. add $A[0],$N[0] # np[j]*m1+ap[j]*bp[0]
  652. adc \$0,%rdx
  653. mov $N[0],-24($tp) # tp[j-1]
  654. mov %rdx,$N[1]
  655. mulq $m0 # ap[j]*bp[0]
  656. add %rax,$A[1]
  657. mov -8*1($np),%rax
  658. adc \$0,%rdx
  659. mov %rdx,$A[0]
  660. mulq $m1 # np[j]*m1
  661. add %rax,$N[1]
  662. mov ($ap,$j),%rax
  663. adc \$0,%rdx
  664. add $A[1],$N[1] # np[j]*m1+ap[j]*bp[0]
  665. adc \$0,%rdx
  666. mov $N[1],-16($tp) # tp[j-1]
  667. mov %rdx,$N[0]
  668. mulq $m0 # ap[j]*bp[0]
  669. add %rax,$A[0]
  670. mov 8*0($np),%rax
  671. adc \$0,%rdx
  672. mov %rdx,$A[1]
  673. mulq $m1 # np[j]*m1
  674. add %rax,$N[0]
  675. mov 8($ap,$j),%rax
  676. adc \$0,%rdx
  677. add $A[0],$N[0] # np[j]*m1+ap[j]*bp[0]
  678. adc \$0,%rdx
  679. mov $N[0],-8($tp) # tp[j-1]
  680. mov %rdx,$N[1]
  681. mulq $m0 # ap[j]*bp[0]
  682. add %rax,$A[1]
  683. mov 8*1($np),%rax
  684. adc \$0,%rdx
  685. mov %rdx,$A[0]
  686. mulq $m1 # np[j]*m1
  687. add %rax,$N[1]
  688. mov 16($ap,$j),%rax
  689. adc \$0,%rdx
  690. add $A[1],$N[1] # np[j]*m1+ap[j]*bp[0]
  691. lea 8*4($np),$np
  692. adc \$0,%rdx
  693. mov $N[1],($tp) # tp[j-1]
  694. mov %rdx,$N[0]
  695. add \$32,$j # j+=4
  696. jnz .L1st4x
  697. mulq $m0 # ap[j]*bp[0]
  698. add %rax,$A[0]
  699. mov -8*2($np),%rax
  700. lea 32($tp),$tp
  701. adc \$0,%rdx
  702. mov %rdx,$A[1]
  703. mulq $m1 # np[j]*m1
  704. add %rax,$N[0]
  705. mov -8($ap),%rax
  706. adc \$0,%rdx
  707. add $A[0],$N[0] # np[j]*m1+ap[j]*bp[0]
  708. adc \$0,%rdx
  709. mov $N[0],-24($tp) # tp[j-1]
  710. mov %rdx,$N[1]
  711. mulq $m0 # ap[j]*bp[0]
  712. add %rax,$A[1]
  713. mov -8*1($np),%rax
  714. adc \$0,%rdx
  715. mov %rdx,$A[0]
  716. mulq $m1 # np[j]*m1
  717. add %rax,$N[1]
  718. mov ($ap,$num),%rax # ap[0]
  719. adc \$0,%rdx
  720. add $A[1],$N[1] # np[j]*m1+ap[j]*bp[0]
  721. adc \$0,%rdx
  722. mov $N[1],-16($tp) # tp[j-1]
  723. mov %rdx,$N[0]
  724. lea ($np,$num),$np # rewind $np
  725. xor $N[1],$N[1]
  726. add $A[0],$N[0]
  727. adc \$0,$N[1]
  728. mov $N[0],-8($tp)
  729. jmp .Louter4x
  730. .align 32
  731. .Louter4x:
  732. lea 16+128($tp),%rdx # where 256-byte mask is (+size optimization)
  733. pxor %xmm4,%xmm4
  734. pxor %xmm5,%xmm5
  735. ___
  736. for($i=0;$i<$STRIDE/16;$i+=4) {
  737. $code.=<<___;
  738. movdqa `16*($i+0)-128`($bp),%xmm0
  739. movdqa `16*($i+1)-128`($bp),%xmm1
  740. movdqa `16*($i+2)-128`($bp),%xmm2
  741. movdqa `16*($i+3)-128`($bp),%xmm3
  742. pand `16*($i+0)-128`(%rdx),%xmm0
  743. pand `16*($i+1)-128`(%rdx),%xmm1
  744. por %xmm0,%xmm4
  745. pand `16*($i+2)-128`(%rdx),%xmm2
  746. por %xmm1,%xmm5
  747. pand `16*($i+3)-128`(%rdx),%xmm3
  748. por %xmm2,%xmm4
  749. por %xmm3,%xmm5
  750. ___
  751. }
  752. $code.=<<___;
  753. por %xmm5,%xmm4
  754. pshufd \$0x4e,%xmm4,%xmm0
  755. por %xmm4,%xmm0
  756. lea $STRIDE($bp),$bp
  757. movq %xmm0,$m0 # m0=bp[i]
  758. mov ($tp,$num),$A[0]
  759. mov $n0,$m1
  760. mulq $m0 # ap[0]*bp[i]
  761. add %rax,$A[0] # ap[0]*bp[i]+tp[0]
  762. mov ($np),%rax
  763. adc \$0,%rdx
  764. imulq $A[0],$m1 # tp[0]*n0
  765. mov %rdx,$A[1]
  766. mov $N[1],($tp) # store upmost overflow bit
  767. lea ($tp,$num),$tp # rewind $tp
  768. mulq $m1 # np[0]*m1
  769. add %rax,$A[0] # "$N[0]", discarded
  770. mov 8($ap,$num),%rax
  771. adc \$0,%rdx
  772. mov %rdx,$N[1]
  773. mulq $m0 # ap[j]*bp[i]
  774. add %rax,$A[1]
  775. mov 8*1($np),%rax
  776. adc \$0,%rdx
  777. add 8($tp),$A[1] # +tp[1]
  778. adc \$0,%rdx
  779. mov %rdx,$A[0]
  780. mulq $m1 # np[j]*m1
  781. add %rax,$N[1]
  782. mov 16($ap,$num),%rax
  783. adc \$0,%rdx
  784. add $A[1],$N[1] # np[j]*m1+ap[j]*bp[i]+tp[j]
  785. lea 4*8($num),$j # j=4
  786. lea 8*4($np),$np
  787. adc \$0,%rdx
  788. mov %rdx,$N[0]
  789. jmp .Linner4x
  790. .align 32
  791. .Linner4x:
  792. mulq $m0 # ap[j]*bp[i]
  793. add %rax,$A[0]
  794. mov -8*2($np),%rax
  795. adc \$0,%rdx
  796. add 16($tp),$A[0] # ap[j]*bp[i]+tp[j]
  797. lea 32($tp),$tp
  798. adc \$0,%rdx
  799. mov %rdx,$A[1]
  800. mulq $m1 # np[j]*m1
  801. add %rax,$N[0]
  802. mov -8($ap,$j),%rax
  803. adc \$0,%rdx
  804. add $A[0],$N[0]
  805. adc \$0,%rdx
  806. mov $N[1],-32($tp) # tp[j-1]
  807. mov %rdx,$N[1]
  808. mulq $m0 # ap[j]*bp[i]
  809. add %rax,$A[1]
  810. mov -8*1($np),%rax
  811. adc \$0,%rdx
  812. add -8($tp),$A[1]
  813. adc \$0,%rdx
  814. mov %rdx,$A[0]
  815. mulq $m1 # np[j]*m1
  816. add %rax,$N[1]
  817. mov ($ap,$j),%rax
  818. adc \$0,%rdx
  819. add $A[1],$N[1]
  820. adc \$0,%rdx
  821. mov $N[0],-24($tp) # tp[j-1]
  822. mov %rdx,$N[0]
  823. mulq $m0 # ap[j]*bp[i]
  824. add %rax,$A[0]
  825. mov 8*0($np),%rax
  826. adc \$0,%rdx
  827. add ($tp),$A[0] # ap[j]*bp[i]+tp[j]
  828. adc \$0,%rdx
  829. mov %rdx,$A[1]
  830. mulq $m1 # np[j]*m1
  831. add %rax,$N[0]
  832. mov 8($ap,$j),%rax
  833. adc \$0,%rdx
  834. add $A[0],$N[0]
  835. adc \$0,%rdx
  836. mov $N[1],-16($tp) # tp[j-1]
  837. mov %rdx,$N[1]
  838. mulq $m0 # ap[j]*bp[i]
  839. add %rax,$A[1]
  840. mov 8*1($np),%rax
  841. adc \$0,%rdx
  842. add 8($tp),$A[1]
  843. adc \$0,%rdx
  844. mov %rdx,$A[0]
  845. mulq $m1 # np[j]*m1
  846. add %rax,$N[1]
  847. mov 16($ap,$j),%rax
  848. adc \$0,%rdx
  849. add $A[1],$N[1]
  850. lea 8*4($np),$np
  851. adc \$0,%rdx
  852. mov $N[0],-8($tp) # tp[j-1]
  853. mov %rdx,$N[0]
  854. add \$32,$j # j+=4
  855. jnz .Linner4x
  856. mulq $m0 # ap[j]*bp[i]
  857. add %rax,$A[0]
  858. mov -8*2($np),%rax
  859. adc \$0,%rdx
  860. add 16($tp),$A[0] # ap[j]*bp[i]+tp[j]
  861. lea 32($tp),$tp
  862. adc \$0,%rdx
  863. mov %rdx,$A[1]
  864. mulq $m1 # np[j]*m1
  865. add %rax,$N[0]
  866. mov -8($ap),%rax
  867. adc \$0,%rdx
  868. add $A[0],$N[0]
  869. adc \$0,%rdx
  870. mov $N[1],-32($tp) # tp[j-1]
  871. mov %rdx,$N[1]
  872. mulq $m0 # ap[j]*bp[i]
  873. add %rax,$A[1]
  874. mov $m1,%rax
  875. mov -8*1($np),$m1
  876. adc \$0,%rdx
  877. add -8($tp),$A[1]
  878. adc \$0,%rdx
  879. mov %rdx,$A[0]
  880. mulq $m1 # np[j]*m1
  881. add %rax,$N[1]
  882. mov ($ap,$num),%rax # ap[0]
  883. adc \$0,%rdx
  884. add $A[1],$N[1]
  885. adc \$0,%rdx
  886. mov $N[0],-24($tp) # tp[j-1]
  887. mov %rdx,$N[0]
  888. mov $N[1],-16($tp) # tp[j-1]
  889. lea ($np,$num),$np # rewind $np
  890. xor $N[1],$N[1]
  891. add $A[0],$N[0]
  892. adc \$0,$N[1]
  893. add ($tp),$N[0] # pull upmost overflow bit
  894. adc \$0,$N[1] # upmost overflow bit
  895. mov $N[0],-8($tp)
  896. cmp 16+8(%rsp),$bp
  897. jb .Louter4x
  898. ___
  899. if (1) {
  900. $code.=<<___;
  901. xor %rax,%rax
  902. sub $N[0],$m1 # compare top-most words
  903. adc $j,$j # $j is zero
  904. or $j,$N[1]
  905. sub $N[1],%rax # %rax=-$N[1]
  906. lea ($tp,$num),%rbx # tptr in .sqr4x_sub
  907. mov ($np),%r12
  908. lea ($np),%rbp # nptr in .sqr4x_sub
  909. mov %r9,%rcx
  910. sar \$3+2,%rcx
  911. mov 56+8(%rsp),%rdi # rptr in .sqr4x_sub
  912. dec %r12 # so that after 'not' we get -n[0]
  913. xor %r10,%r10
  914. mov 8*1(%rbp),%r13
  915. mov 8*2(%rbp),%r14
  916. mov 8*3(%rbp),%r15
  917. jmp .Lsqr4x_sub_entry
  918. ___
  919. } else {
  920. my @ri=("%rax",$bp,$m0,$m1);
  921. my $rp="%rdx";
  922. $code.=<<___
  923. xor \$1,$N[1]
  924. lea ($tp,$num),$tp # rewind $tp
  925. sar \$5,$num # cf=0
  926. lea ($np,$N[1],8),$np
  927. mov 56+8(%rsp),$rp # restore $rp
  928. jmp .Lsub4x
  929. .align 32
  930. .Lsub4x:
  931. .byte 0x66
  932. mov 8*0($tp),@ri[0]
  933. mov 8*1($tp),@ri[1]
  934. .byte 0x66
  935. sbb 16*0($np),@ri[0]
  936. mov 8*2($tp),@ri[2]
  937. sbb 16*1($np),@ri[1]
  938. mov 3*8($tp),@ri[3]
  939. lea 4*8($tp),$tp
  940. sbb 16*2($np),@ri[2]
  941. mov @ri[0],8*0($rp)
  942. sbb 16*3($np),@ri[3]
  943. lea 16*4($np),$np
  944. mov @ri[1],8*1($rp)
  945. mov @ri[2],8*2($rp)
  946. mov @ri[3],8*3($rp)
  947. lea 8*4($rp),$rp
  948. inc $num
  949. jnz .Lsub4x
  950. ret
  951. ___
  952. }
  953. $code.=<<___;
  954. .size mul4x_internal,.-mul4x_internal
  955. ___
  956. }}}
  957. {{{
  958. ######################################################################
  959. # void bn_power5(
  960. my $rptr="%rdi"; # BN_ULONG *rptr,
  961. my $aptr="%rsi"; # const BN_ULONG *aptr,
  962. my $bptr="%rdx"; # const void *table,
  963. my $nptr="%rcx"; # const BN_ULONG *nptr,
  964. my $n0 ="%r8"; # const BN_ULONG *n0);
  965. my $num ="%r9"; # int num, has to be divisible by 8
  966. # int pwr
  967. my ($i,$j,$tptr)=("%rbp","%rcx",$rptr);
  968. my @A0=("%r10","%r11");
  969. my @A1=("%r12","%r13");
  970. my ($a0,$a1,$ai)=("%r14","%r15","%rbx");
  971. $code.=<<___;
  972. .globl bn_power5
  973. .type bn_power5,\@function,6
  974. .align 32
  975. bn_power5:
  976. .cfi_startproc
  977. mov %rsp,%rax
  978. .cfi_def_cfa_register %rax
  979. ___
  980. $code.=<<___ if ($addx);
  981. mov OPENSSL_ia32cap_P+8(%rip),%r11d
  982. and \$0x80108,%r11d
  983. cmp \$0x80108,%r11d # check for AD*X+BMI2+BMI1
  984. je .Lpowerx5_enter
  985. ___
  986. $code.=<<___;
  987. push %rbx
  988. .cfi_push %rbx
  989. push %rbp
  990. .cfi_push %rbp
  991. push %r12
  992. .cfi_push %r12
  993. push %r13
  994. .cfi_push %r13
  995. push %r14
  996. .cfi_push %r14
  997. push %r15
  998. .cfi_push %r15
  999. .Lpower5_prologue:
  1000. shl \$3,${num}d # convert $num to bytes
  1001. lea ($num,$num,2),%r10d # 3*$num
  1002. neg $num
  1003. mov ($n0),$n0 # *n0
  1004. ##############################################################
  1005. # Ensure that stack frame doesn't alias with $rptr+3*$num
  1006. # modulo 4096, which covers ret[num], am[num] and n[num]
  1007. # (see bn_exp.c). This is done to allow memory disambiguation
  1008. # logic do its magic. [Extra 256 bytes is for power mask
  1009. # calculated from 7th argument, the index.]
  1010. #
  1011. lea -320(%rsp,$num,2),%r11
  1012. mov %rsp,%rbp
  1013. sub $rptr,%r11
  1014. and \$4095,%r11
  1015. cmp %r11,%r10
  1016. jb .Lpwr_sp_alt
  1017. sub %r11,%rbp # align with $aptr
  1018. lea -320(%rbp,$num,2),%rbp # future alloca(frame+2*num*8+256)
  1019. jmp .Lpwr_sp_done
  1020. .align 32
  1021. .Lpwr_sp_alt:
  1022. lea 4096-320(,$num,2),%r10
  1023. lea -320(%rbp,$num,2),%rbp # future alloca(frame+2*num*8+256)
  1024. sub %r10,%r11
  1025. mov \$0,%r10
  1026. cmovc %r10,%r11
  1027. sub %r11,%rbp
  1028. .Lpwr_sp_done:
  1029. and \$-64,%rbp
  1030. mov %rsp,%r11
  1031. sub %rbp,%r11
  1032. and \$-4096,%r11
  1033. lea (%rbp,%r11),%rsp
  1034. mov (%rsp),%r10
  1035. cmp %rbp,%rsp
  1036. ja .Lpwr_page_walk
  1037. jmp .Lpwr_page_walk_done
  1038. .Lpwr_page_walk:
  1039. lea -4096(%rsp),%rsp
  1040. mov (%rsp),%r10
  1041. cmp %rbp,%rsp
  1042. ja .Lpwr_page_walk
  1043. .Lpwr_page_walk_done:
  1044. mov $num,%r10
  1045. neg $num
  1046. ##############################################################
  1047. # Stack layout
  1048. #
  1049. # +0 saved $num, used in reduction section
  1050. # +8 &t[2*$num], used in reduction section
  1051. # +32 saved *n0
  1052. # +40 saved %rsp
  1053. # +48 t[2*$num]
  1054. #
  1055. mov $n0, 32(%rsp)
  1056. mov %rax, 40(%rsp) # save original %rsp
  1057. .cfi_cfa_expression %rsp+40,deref,+8
  1058. .Lpower5_body:
  1059. movq $rptr,%xmm1 # save $rptr, used in sqr8x
  1060. movq $nptr,%xmm2 # save $nptr
  1061. movq %r10, %xmm3 # -$num, used in sqr8x
  1062. movq $bptr,%xmm4
  1063. call __bn_sqr8x_internal
  1064. call __bn_post4x_internal
  1065. call __bn_sqr8x_internal
  1066. call __bn_post4x_internal
  1067. call __bn_sqr8x_internal
  1068. call __bn_post4x_internal
  1069. call __bn_sqr8x_internal
  1070. call __bn_post4x_internal
  1071. call __bn_sqr8x_internal
  1072. call __bn_post4x_internal
  1073. movq %xmm2,$nptr
  1074. movq %xmm4,$bptr
  1075. mov $aptr,$rptr
  1076. mov 40(%rsp),%rax
  1077. lea 32(%rsp),$n0
  1078. call mul4x_internal
  1079. mov 40(%rsp),%rsi # restore %rsp
  1080. .cfi_def_cfa %rsi,8
  1081. mov \$1,%rax
  1082. mov -48(%rsi),%r15
  1083. .cfi_restore %r15
  1084. mov -40(%rsi),%r14
  1085. .cfi_restore %r14
  1086. mov -32(%rsi),%r13
  1087. .cfi_restore %r13
  1088. mov -24(%rsi),%r12
  1089. .cfi_restore %r12
  1090. mov -16(%rsi),%rbp
  1091. .cfi_restore %rbp
  1092. mov -8(%rsi),%rbx
  1093. .cfi_restore %rbx
  1094. lea (%rsi),%rsp
  1095. .cfi_def_cfa_register %rsp
  1096. .Lpower5_epilogue:
  1097. ret
  1098. .cfi_endproc
  1099. .size bn_power5,.-bn_power5
  1100. .globl bn_sqr8x_internal
  1101. .hidden bn_sqr8x_internal
  1102. .type bn_sqr8x_internal,\@abi-omnipotent
  1103. .align 32
  1104. bn_sqr8x_internal:
  1105. __bn_sqr8x_internal:
  1106. ##############################################################
  1107. # Squaring part:
  1108. #
  1109. # a) multiply-n-add everything but a[i]*a[i];
  1110. # b) shift result of a) by 1 to the left and accumulate
  1111. # a[i]*a[i] products;
  1112. #
  1113. ##############################################################
  1114. # a[1]a[0]
  1115. # a[2]a[0]
  1116. # a[3]a[0]
  1117. # a[2]a[1]
  1118. # a[4]a[0]
  1119. # a[3]a[1]
  1120. # a[5]a[0]
  1121. # a[4]a[1]
  1122. # a[3]a[2]
  1123. # a[6]a[0]
  1124. # a[5]a[1]
  1125. # a[4]a[2]
  1126. # a[7]a[0]
  1127. # a[6]a[1]
  1128. # a[5]a[2]
  1129. # a[4]a[3]
  1130. # a[7]a[1]
  1131. # a[6]a[2]
  1132. # a[5]a[3]
  1133. # a[7]a[2]
  1134. # a[6]a[3]
  1135. # a[5]a[4]
  1136. # a[7]a[3]
  1137. # a[6]a[4]
  1138. # a[7]a[4]
  1139. # a[6]a[5]
  1140. # a[7]a[5]
  1141. # a[7]a[6]
  1142. # a[1]a[0]
  1143. # a[2]a[0]
  1144. # a[3]a[0]
  1145. # a[4]a[0]
  1146. # a[5]a[0]
  1147. # a[6]a[0]
  1148. # a[7]a[0]
  1149. # a[2]a[1]
  1150. # a[3]a[1]
  1151. # a[4]a[1]
  1152. # a[5]a[1]
  1153. # a[6]a[1]
  1154. # a[7]a[1]
  1155. # a[3]a[2]
  1156. # a[4]a[2]
  1157. # a[5]a[2]
  1158. # a[6]a[2]
  1159. # a[7]a[2]
  1160. # a[4]a[3]
  1161. # a[5]a[3]
  1162. # a[6]a[3]
  1163. # a[7]a[3]
  1164. # a[5]a[4]
  1165. # a[6]a[4]
  1166. # a[7]a[4]
  1167. # a[6]a[5]
  1168. # a[7]a[5]
  1169. # a[7]a[6]
  1170. # a[0]a[0]
  1171. # a[1]a[1]
  1172. # a[2]a[2]
  1173. # a[3]a[3]
  1174. # a[4]a[4]
  1175. # a[5]a[5]
  1176. # a[6]a[6]
  1177. # a[7]a[7]
  1178. lea 32(%r10),$i # $i=-($num-32)
  1179. lea ($aptr,$num),$aptr # end of a[] buffer, ($aptr,$i)=&ap[2]
  1180. mov $num,$j # $j=$num
  1181. # comments apply to $num==8 case
  1182. mov -32($aptr,$i),$a0 # a[0]
  1183. lea 48+8(%rsp,$num,2),$tptr # end of tp[] buffer, &tp[2*$num]
  1184. mov -24($aptr,$i),%rax # a[1]
  1185. lea -32($tptr,$i),$tptr # end of tp[] window, &tp[2*$num-"$i"]
  1186. mov -16($aptr,$i),$ai # a[2]
  1187. mov %rax,$a1
  1188. mul $a0 # a[1]*a[0]
  1189. mov %rax,$A0[0] # a[1]*a[0]
  1190. mov $ai,%rax # a[2]
  1191. mov %rdx,$A0[1]
  1192. mov $A0[0],-24($tptr,$i) # t[1]
  1193. mul $a0 # a[2]*a[0]
  1194. add %rax,$A0[1]
  1195. mov $ai,%rax
  1196. adc \$0,%rdx
  1197. mov $A0[1],-16($tptr,$i) # t[2]
  1198. mov %rdx,$A0[0]
  1199. mov -8($aptr,$i),$ai # a[3]
  1200. mul $a1 # a[2]*a[1]
  1201. mov %rax,$A1[0] # a[2]*a[1]+t[3]
  1202. mov $ai,%rax
  1203. mov %rdx,$A1[1]
  1204. lea ($i),$j
  1205. mul $a0 # a[3]*a[0]
  1206. add %rax,$A0[0] # a[3]*a[0]+a[2]*a[1]+t[3]
  1207. mov $ai,%rax
  1208. mov %rdx,$A0[1]
  1209. adc \$0,$A0[1]
  1210. add $A1[0],$A0[0]
  1211. adc \$0,$A0[1]
  1212. mov $A0[0],-8($tptr,$j) # t[3]
  1213. jmp .Lsqr4x_1st
  1214. .align 32
  1215. .Lsqr4x_1st:
  1216. mov ($aptr,$j),$ai # a[4]
  1217. mul $a1 # a[3]*a[1]
  1218. add %rax,$A1[1] # a[3]*a[1]+t[4]
  1219. mov $ai,%rax
  1220. mov %rdx,$A1[0]
  1221. adc \$0,$A1[0]
  1222. mul $a0 # a[4]*a[0]
  1223. add %rax,$A0[1] # a[4]*a[0]+a[3]*a[1]+t[4]
  1224. mov $ai,%rax # a[3]
  1225. mov 8($aptr,$j),$ai # a[5]
  1226. mov %rdx,$A0[0]
  1227. adc \$0,$A0[0]
  1228. add $A1[1],$A0[1]
  1229. adc \$0,$A0[0]
  1230. mul $a1 # a[4]*a[3]
  1231. add %rax,$A1[0] # a[4]*a[3]+t[5]
  1232. mov $ai,%rax
  1233. mov $A0[1],($tptr,$j) # t[4]
  1234. mov %rdx,$A1[1]
  1235. adc \$0,$A1[1]
  1236. mul $a0 # a[5]*a[2]
  1237. add %rax,$A0[0] # a[5]*a[2]+a[4]*a[3]+t[5]
  1238. mov $ai,%rax
  1239. mov 16($aptr,$j),$ai # a[6]
  1240. mov %rdx,$A0[1]
  1241. adc \$0,$A0[1]
  1242. add $A1[0],$A0[0]
  1243. adc \$0,$A0[1]
  1244. mul $a1 # a[5]*a[3]
  1245. add %rax,$A1[1] # a[5]*a[3]+t[6]
  1246. mov $ai,%rax
  1247. mov $A0[0],8($tptr,$j) # t[5]
  1248. mov %rdx,$A1[0]
  1249. adc \$0,$A1[0]
  1250. mul $a0 # a[6]*a[2]
  1251. add %rax,$A0[1] # a[6]*a[2]+a[5]*a[3]+t[6]
  1252. mov $ai,%rax # a[3]
  1253. mov 24($aptr,$j),$ai # a[7]
  1254. mov %rdx,$A0[0]
  1255. adc \$0,$A0[0]
  1256. add $A1[1],$A0[1]
  1257. adc \$0,$A0[0]
  1258. mul $a1 # a[6]*a[5]
  1259. add %rax,$A1[0] # a[6]*a[5]+t[7]
  1260. mov $ai,%rax
  1261. mov $A0[1],16($tptr,$j) # t[6]
  1262. mov %rdx,$A1[1]
  1263. adc \$0,$A1[1]
  1264. lea 32($j),$j
  1265. mul $a0 # a[7]*a[4]
  1266. add %rax,$A0[0] # a[7]*a[4]+a[6]*a[5]+t[6]
  1267. mov $ai,%rax
  1268. mov %rdx,$A0[1]
  1269. adc \$0,$A0[1]
  1270. add $A1[0],$A0[0]
  1271. adc \$0,$A0[1]
  1272. mov $A0[0],-8($tptr,$j) # t[7]
  1273. cmp \$0,$j
  1274. jne .Lsqr4x_1st
  1275. mul $a1 # a[7]*a[5]
  1276. add %rax,$A1[1]
  1277. lea 16($i),$i
  1278. adc \$0,%rdx
  1279. add $A0[1],$A1[1]
  1280. adc \$0,%rdx
  1281. mov $A1[1],($tptr) # t[8]
  1282. mov %rdx,$A1[0]
  1283. mov %rdx,8($tptr) # t[9]
  1284. jmp .Lsqr4x_outer
  1285. .align 32
  1286. .Lsqr4x_outer: # comments apply to $num==6 case
  1287. mov -32($aptr,$i),$a0 # a[0]
  1288. lea 48+8(%rsp,$num,2),$tptr # end of tp[] buffer, &tp[2*$num]
  1289. mov -24($aptr,$i),%rax # a[1]
  1290. lea -32($tptr,$i),$tptr # end of tp[] window, &tp[2*$num-"$i"]
  1291. mov -16($aptr,$i),$ai # a[2]
  1292. mov %rax,$a1
  1293. mul $a0 # a[1]*a[0]
  1294. mov -24($tptr,$i),$A0[0] # t[1]
  1295. add %rax,$A0[0] # a[1]*a[0]+t[1]
  1296. mov $ai,%rax # a[2]
  1297. adc \$0,%rdx
  1298. mov $A0[0],-24($tptr,$i) # t[1]
  1299. mov %rdx,$A0[1]
  1300. mul $a0 # a[2]*a[0]
  1301. add %rax,$A0[1]
  1302. mov $ai,%rax
  1303. adc \$0,%rdx
  1304. add -16($tptr,$i),$A0[1] # a[2]*a[0]+t[2]
  1305. mov %rdx,$A0[0]
  1306. adc \$0,$A0[0]
  1307. mov $A0[1],-16($tptr,$i) # t[2]
  1308. xor $A1[0],$A1[0]
  1309. mov -8($aptr,$i),$ai # a[3]
  1310. mul $a1 # a[2]*a[1]
  1311. add %rax,$A1[0] # a[2]*a[1]+t[3]
  1312. mov $ai,%rax
  1313. adc \$0,%rdx
  1314. add -8($tptr,$i),$A1[0]
  1315. mov %rdx,$A1[1]
  1316. adc \$0,$A1[1]
  1317. mul $a0 # a[3]*a[0]
  1318. add %rax,$A0[0] # a[3]*a[0]+a[2]*a[1]+t[3]
  1319. mov $ai,%rax
  1320. adc \$0,%rdx
  1321. add $A1[0],$A0[0]
  1322. mov %rdx,$A0[1]
  1323. adc \$0,$A0[1]
  1324. mov $A0[0],-8($tptr,$i) # t[3]
  1325. lea ($i),$j
  1326. jmp .Lsqr4x_inner
  1327. .align 32
  1328. .Lsqr4x_inner:
  1329. mov ($aptr,$j),$ai # a[4]
  1330. mul $a1 # a[3]*a[1]
  1331. add %rax,$A1[1] # a[3]*a[1]+t[4]
  1332. mov $ai,%rax
  1333. mov %rdx,$A1[0]
  1334. adc \$0,$A1[0]
  1335. add ($tptr,$j),$A1[1]
  1336. adc \$0,$A1[0]
  1337. .byte 0x67
  1338. mul $a0 # a[4]*a[0]
  1339. add %rax,$A0[1] # a[4]*a[0]+a[3]*a[1]+t[4]
  1340. mov $ai,%rax # a[3]
  1341. mov 8($aptr,$j),$ai # a[5]
  1342. mov %rdx,$A0[0]
  1343. adc \$0,$A0[0]
  1344. add $A1[1],$A0[1]
  1345. adc \$0,$A0[0]
  1346. mul $a1 # a[4]*a[3]
  1347. add %rax,$A1[0] # a[4]*a[3]+t[5]
  1348. mov $A0[1],($tptr,$j) # t[4]
  1349. mov $ai,%rax
  1350. mov %rdx,$A1[1]
  1351. adc \$0,$A1[1]
  1352. add 8($tptr,$j),$A1[0]
  1353. lea 16($j),$j # j++
  1354. adc \$0,$A1[1]
  1355. mul $a0 # a[5]*a[2]
  1356. add %rax,$A0[0] # a[5]*a[2]+a[4]*a[3]+t[5]
  1357. mov $ai,%rax
  1358. adc \$0,%rdx
  1359. add $A1[0],$A0[0]
  1360. mov %rdx,$A0[1]
  1361. adc \$0,$A0[1]
  1362. mov $A0[0],-8($tptr,$j) # t[5], "preloaded t[1]" below
  1363. cmp \$0,$j
  1364. jne .Lsqr4x_inner
  1365. .byte 0x67
  1366. mul $a1 # a[5]*a[3]
  1367. add %rax,$A1[1]
  1368. adc \$0,%rdx
  1369. add $A0[1],$A1[1]
  1370. adc \$0,%rdx
  1371. mov $A1[1],($tptr) # t[6], "preloaded t[2]" below
  1372. mov %rdx,$A1[0]
  1373. mov %rdx,8($tptr) # t[7], "preloaded t[3]" below
  1374. add \$16,$i
  1375. jnz .Lsqr4x_outer
  1376. # comments apply to $num==4 case
  1377. mov -32($aptr),$a0 # a[0]
  1378. lea 48+8(%rsp,$num,2),$tptr # end of tp[] buffer, &tp[2*$num]
  1379. mov -24($aptr),%rax # a[1]
  1380. lea -32($tptr,$i),$tptr # end of tp[] window, &tp[2*$num-"$i"]
  1381. mov -16($aptr),$ai # a[2]
  1382. mov %rax,$a1
  1383. mul $a0 # a[1]*a[0]
  1384. add %rax,$A0[0] # a[1]*a[0]+t[1], preloaded t[1]
  1385. mov $ai,%rax # a[2]
  1386. mov %rdx,$A0[1]
  1387. adc \$0,$A0[1]
  1388. mul $a0 # a[2]*a[0]
  1389. add %rax,$A0[1]
  1390. mov $ai,%rax
  1391. mov $A0[0],-24($tptr) # t[1]
  1392. mov %rdx,$A0[0]
  1393. adc \$0,$A0[0]
  1394. add $A1[1],$A0[1] # a[2]*a[0]+t[2], preloaded t[2]
  1395. mov -8($aptr),$ai # a[3]
  1396. adc \$0,$A0[0]
  1397. mul $a1 # a[2]*a[1]
  1398. add %rax,$A1[0] # a[2]*a[1]+t[3], preloaded t[3]
  1399. mov $ai,%rax
  1400. mov $A0[1],-16($tptr) # t[2]
  1401. mov %rdx,$A1[1]
  1402. adc \$0,$A1[1]
  1403. mul $a0 # a[3]*a[0]
  1404. add %rax,$A0[0] # a[3]*a[0]+a[2]*a[1]+t[3]
  1405. mov $ai,%rax
  1406. mov %rdx,$A0[1]
  1407. adc \$0,$A0[1]
  1408. add $A1[0],$A0[0]
  1409. adc \$0,$A0[1]
  1410. mov $A0[0],-8($tptr) # t[3]
  1411. mul $a1 # a[3]*a[1]
  1412. add %rax,$A1[1]
  1413. mov -16($aptr),%rax # a[2]
  1414. adc \$0,%rdx
  1415. add $A0[1],$A1[1]
  1416. adc \$0,%rdx
  1417. mov $A1[1],($tptr) # t[4]
  1418. mov %rdx,$A1[0]
  1419. mov %rdx,8($tptr) # t[5]
  1420. mul $ai # a[2]*a[3]
  1421. ___
  1422. {
  1423. my ($shift,$carry)=($a0,$a1);
  1424. my @S=(@A1,$ai,$n0);
  1425. $code.=<<___;
  1426. add \$16,$i
  1427. xor $shift,$shift
  1428. sub $num,$i # $i=16-$num
  1429. xor $carry,$carry
  1430. add $A1[0],%rax # t[5]
  1431. adc \$0,%rdx
  1432. mov %rax,8($tptr) # t[5]
  1433. mov %rdx,16($tptr) # t[6]
  1434. mov $carry,24($tptr) # t[7]
  1435. mov -16($aptr,$i),%rax # a[0]
  1436. lea 48+8(%rsp),$tptr
  1437. xor $A0[0],$A0[0] # t[0]
  1438. mov 8($tptr),$A0[1] # t[1]
  1439. lea ($shift,$A0[0],2),$S[0] # t[2*i]<<1 | shift
  1440. shr \$63,$A0[0]
  1441. lea ($j,$A0[1],2),$S[1] # t[2*i+1]<<1 |
  1442. shr \$63,$A0[1]
  1443. or $A0[0],$S[1] # | t[2*i]>>63
  1444. mov 16($tptr),$A0[0] # t[2*i+2] # prefetch
  1445. mov $A0[1],$shift # shift=t[2*i+1]>>63
  1446. mul %rax # a[i]*a[i]
  1447. neg $carry # mov $carry,cf
  1448. mov 24($tptr),$A0[1] # t[2*i+2+1] # prefetch
  1449. adc %rax,$S[0]
  1450. mov -8($aptr,$i),%rax # a[i+1] # prefetch
  1451. mov $S[0],($tptr)
  1452. adc %rdx,$S[1]
  1453. lea ($shift,$A0[0],2),$S[2] # t[2*i]<<1 | shift
  1454. mov $S[1],8($tptr)
  1455. sbb $carry,$carry # mov cf,$carry
  1456. shr \$63,$A0[0]
  1457. lea ($j,$A0[1],2),$S[3] # t[2*i+1]<<1 |
  1458. shr \$63,$A0[1]
  1459. or $A0[0],$S[3] # | t[2*i]>>63
  1460. mov 32($tptr),$A0[0] # t[2*i+2] # prefetch
  1461. mov $A0[1],$shift # shift=t[2*i+1]>>63
  1462. mul %rax # a[i]*a[i]
  1463. neg $carry # mov $carry,cf
  1464. mov 40($tptr),$A0[1] # t[2*i+2+1] # prefetch
  1465. adc %rax,$S[2]
  1466. mov 0($aptr,$i),%rax # a[i+1] # prefetch
  1467. mov $S[2],16($tptr)
  1468. adc %rdx,$S[3]
  1469. lea 16($i),$i
  1470. mov $S[3],24($tptr)
  1471. sbb $carry,$carry # mov cf,$carry
  1472. lea 64($tptr),$tptr
  1473. jmp .Lsqr4x_shift_n_add
  1474. .align 32
  1475. .Lsqr4x_shift_n_add:
  1476. lea ($shift,$A0[0],2),$S[0] # t[2*i]<<1 | shift
  1477. shr \$63,$A0[0]
  1478. lea ($j,$A0[1],2),$S[1] # t[2*i+1]<<1 |
  1479. shr \$63,$A0[1]
  1480. or $A0[0],$S[1] # | t[2*i]>>63
  1481. mov -16($tptr),$A0[0] # t[2*i+2] # prefetch
  1482. mov $A0[1],$shift # shift=t[2*i+1]>>63
  1483. mul %rax # a[i]*a[i]
  1484. neg $carry # mov $carry,cf
  1485. mov -8($tptr),$A0[1] # t[2*i+2+1] # prefetch
  1486. adc %rax,$S[0]
  1487. mov -8($aptr,$i),%rax # a[i+1] # prefetch
  1488. mov $S[0],-32($tptr)
  1489. adc %rdx,$S[1]
  1490. lea ($shift,$A0[0],2),$S[2] # t[2*i]<<1 | shift
  1491. mov $S[1],-24($tptr)
  1492. sbb $carry,$carry # mov cf,$carry
  1493. shr \$63,$A0[0]
  1494. lea ($j,$A0[1],2),$S[3] # t[2*i+1]<<1 |
  1495. shr \$63,$A0[1]
  1496. or $A0[0],$S[3] # | t[2*i]>>63
  1497. mov 0($tptr),$A0[0] # t[2*i+2] # prefetch
  1498. mov $A0[1],$shift # shift=t[2*i+1]>>63
  1499. mul %rax # a[i]*a[i]
  1500. neg $carry # mov $carry,cf
  1501. mov 8($tptr),$A0[1] # t[2*i+2+1] # prefetch
  1502. adc %rax,$S[2]
  1503. mov 0($aptr,$i),%rax # a[i+1] # prefetch
  1504. mov $S[2],-16($tptr)
  1505. adc %rdx,$S[3]
  1506. lea ($shift,$A0[0],2),$S[0] # t[2*i]<<1 | shift
  1507. mov $S[3],-8($tptr)
  1508. sbb $carry,$carry # mov cf,$carry
  1509. shr \$63,$A0[0]
  1510. lea ($j,$A0[1],2),$S[1] # t[2*i+1]<<1 |
  1511. shr \$63,$A0[1]
  1512. or $A0[0],$S[1] # | t[2*i]>>63
  1513. mov 16($tptr),$A0[0] # t[2*i+2] # prefetch
  1514. mov $A0[1],$shift # shift=t[2*i+1]>>63
  1515. mul %rax # a[i]*a[i]
  1516. neg $carry # mov $carry,cf
  1517. mov 24($tptr),$A0[1] # t[2*i+2+1] # prefetch
  1518. adc %rax,$S[0]
  1519. mov 8($aptr,$i),%rax # a[i+1] # prefetch
  1520. mov $S[0],0($tptr)
  1521. adc %rdx,$S[1]
  1522. lea ($shift,$A0[0],2),$S[2] # t[2*i]<<1 | shift
  1523. mov $S[1],8($tptr)
  1524. sbb $carry,$carry # mov cf,$carry
  1525. shr \$63,$A0[0]
  1526. lea ($j,$A0[1],2),$S[3] # t[2*i+1]<<1 |
  1527. shr \$63,$A0[1]
  1528. or $A0[0],$S[3] # | t[2*i]>>63
  1529. mov 32($tptr),$A0[0] # t[2*i+2] # prefetch
  1530. mov $A0[1],$shift # shift=t[2*i+1]>>63
  1531. mul %rax # a[i]*a[i]
  1532. neg $carry # mov $carry,cf
  1533. mov 40($tptr),$A0[1] # t[2*i+2+1] # prefetch
  1534. adc %rax,$S[2]
  1535. mov 16($aptr,$i),%rax # a[i+1] # prefetch
  1536. mov $S[2],16($tptr)
  1537. adc %rdx,$S[3]
  1538. mov $S[3],24($tptr)
  1539. sbb $carry,$carry # mov cf,$carry
  1540. lea 64($tptr),$tptr
  1541. add \$32,$i
  1542. jnz .Lsqr4x_shift_n_add
  1543. lea ($shift,$A0[0],2),$S[0] # t[2*i]<<1 | shift
  1544. .byte 0x67
  1545. shr \$63,$A0[0]
  1546. lea ($j,$A0[1],2),$S[1] # t[2*i+1]<<1 |
  1547. shr \$63,$A0[1]
  1548. or $A0[0],$S[1] # | t[2*i]>>63
  1549. mov -16($tptr),$A0[0] # t[2*i+2] # prefetch
  1550. mov $A0[1],$shift # shift=t[2*i+1]>>63
  1551. mul %rax # a[i]*a[i]
  1552. neg $carry # mov $carry,cf
  1553. mov -8($tptr),$A0[1] # t[2*i+2+1] # prefetch
  1554. adc %rax,$S[0]
  1555. mov -8($aptr),%rax # a[i+1] # prefetch
  1556. mov $S[0],-32($tptr)
  1557. adc %rdx,$S[1]
  1558. lea ($shift,$A0[0],2),$S[2] # t[2*i]<<1|shift
  1559. mov $S[1],-24($tptr)
  1560. sbb $carry,$carry # mov cf,$carry
  1561. shr \$63,$A0[0]
  1562. lea ($j,$A0[1],2),$S[3] # t[2*i+1]<<1 |
  1563. shr \$63,$A0[1]
  1564. or $A0[0],$S[3] # | t[2*i]>>63
  1565. mul %rax # a[i]*a[i]
  1566. neg $carry # mov $carry,cf
  1567. adc %rax,$S[2]
  1568. adc %rdx,$S[3]
  1569. mov $S[2],-16($tptr)
  1570. mov $S[3],-8($tptr)
  1571. ___
  1572. }
  1573. ######################################################################
  1574. # Montgomery reduction part, "word-by-word" algorithm.
  1575. #
  1576. # This new path is inspired by multiple submissions from Intel, by
  1577. # Shay Gueron, Vlad Krasnov, Erdinc Ozturk, James Guilford,
  1578. # Vinodh Gopal...
  1579. {
  1580. my ($nptr,$tptr,$carry,$m0)=("%rbp","%rdi","%rsi","%rbx");
  1581. $code.=<<___;
  1582. movq %xmm2,$nptr
  1583. __bn_sqr8x_reduction:
  1584. xor %rax,%rax
  1585. lea ($nptr,$num),%rcx # end of n[]
  1586. lea 48+8(%rsp,$num,2),%rdx # end of t[] buffer
  1587. mov %rcx,0+8(%rsp)
  1588. lea 48+8(%rsp,$num),$tptr # end of initial t[] window
  1589. mov %rdx,8+8(%rsp)
  1590. neg $num
  1591. jmp .L8x_reduction_loop
  1592. .align 32
  1593. .L8x_reduction_loop:
  1594. lea ($tptr,$num),$tptr # start of current t[] window
  1595. .byte 0x66
  1596. mov 8*0($tptr),$m0
  1597. mov 8*1($tptr),%r9
  1598. mov 8*2($tptr),%r10
  1599. mov 8*3($tptr),%r11
  1600. mov 8*4($tptr),%r12
  1601. mov 8*5($tptr),%r13
  1602. mov 8*6($tptr),%r14
  1603. mov 8*7($tptr),%r15
  1604. mov %rax,(%rdx) # store top-most carry bit
  1605. lea 8*8($tptr),$tptr
  1606. .byte 0x67
  1607. mov $m0,%r8
  1608. imulq 32+8(%rsp),$m0 # n0*a[0]
  1609. mov 8*0($nptr),%rax # n[0]
  1610. mov \$8,%ecx
  1611. jmp .L8x_reduce
  1612. .align 32
  1613. .L8x_reduce:
  1614. mulq $m0
  1615. mov 8*1($nptr),%rax # n[1]
  1616. neg %r8
  1617. mov %rdx,%r8
  1618. adc \$0,%r8
  1619. mulq $m0
  1620. add %rax,%r9
  1621. mov 8*2($nptr),%rax
  1622. adc \$0,%rdx
  1623. add %r9,%r8
  1624. mov $m0,48-8+8(%rsp,%rcx,8) # put aside n0*a[i]
  1625. mov %rdx,%r9
  1626. adc \$0,%r9
  1627. mulq $m0
  1628. add %rax,%r10
  1629. mov 8*3($nptr),%rax
  1630. adc \$0,%rdx
  1631. add %r10,%r9
  1632. mov 32+8(%rsp),$carry # pull n0, borrow $carry
  1633. mov %rdx,%r10
  1634. adc \$0,%r10
  1635. mulq $m0
  1636. add %rax,%r11
  1637. mov 8*4($nptr),%rax
  1638. adc \$0,%rdx
  1639. imulq %r8,$carry # modulo-scheduled
  1640. add %r11,%r10
  1641. mov %rdx,%r11
  1642. adc \$0,%r11
  1643. mulq $m0
  1644. add %rax,%r12
  1645. mov 8*5($nptr),%rax
  1646. adc \$0,%rdx
  1647. add %r12,%r11
  1648. mov %rdx,%r12
  1649. adc \$0,%r12
  1650. mulq $m0
  1651. add %rax,%r13
  1652. mov 8*6($nptr),%rax
  1653. adc \$0,%rdx
  1654. add %r13,%r12
  1655. mov %rdx,%r13
  1656. adc \$0,%r13
  1657. mulq $m0
  1658. add %rax,%r14
  1659. mov 8*7($nptr),%rax
  1660. adc \$0,%rdx
  1661. add %r14,%r13
  1662. mov %rdx,%r14
  1663. adc \$0,%r14
  1664. mulq $m0
  1665. mov $carry,$m0 # n0*a[i]
  1666. add %rax,%r15
  1667. mov 8*0($nptr),%rax # n[0]
  1668. adc \$0,%rdx
  1669. add %r15,%r14
  1670. mov %rdx,%r15
  1671. adc \$0,%r15
  1672. dec %ecx
  1673. jnz .L8x_reduce
  1674. lea 8*8($nptr),$nptr
  1675. xor %rax,%rax
  1676. mov 8+8(%rsp),%rdx # pull end of t[]
  1677. cmp 0+8(%rsp),$nptr # end of n[]?
  1678. jae .L8x_no_tail
  1679. .byte 0x66
  1680. add 8*0($tptr),%r8
  1681. adc 8*1($tptr),%r9
  1682. adc 8*2($tptr),%r10
  1683. adc 8*3($tptr),%r11
  1684. adc 8*4($tptr),%r12
  1685. adc 8*5($tptr),%r13
  1686. adc 8*6($tptr),%r14
  1687. adc 8*7($tptr),%r15
  1688. sbb $carry,$carry # top carry
  1689. mov 48+56+8(%rsp),$m0 # pull n0*a[0]
  1690. mov \$8,%ecx
  1691. mov 8*0($nptr),%rax
  1692. jmp .L8x_tail
  1693. .align 32
  1694. .L8x_tail:
  1695. mulq $m0
  1696. add %rax,%r8
  1697. mov 8*1($nptr),%rax
  1698. mov %r8,($tptr) # save result
  1699. mov %rdx,%r8
  1700. adc \$0,%r8
  1701. mulq $m0
  1702. add %rax,%r9
  1703. mov 8*2($nptr),%rax
  1704. adc \$0,%rdx
  1705. add %r9,%r8
  1706. lea 8($tptr),$tptr # $tptr++
  1707. mov %rdx,%r9
  1708. adc \$0,%r9
  1709. mulq $m0
  1710. add %rax,%r10
  1711. mov 8*3($nptr),%rax
  1712. adc \$0,%rdx
  1713. add %r10,%r9
  1714. mov %rdx,%r10
  1715. adc \$0,%r10
  1716. mulq $m0
  1717. add %rax,%r11
  1718. mov 8*4($nptr),%rax
  1719. adc \$0,%rdx
  1720. add %r11,%r10
  1721. mov %rdx,%r11
  1722. adc \$0,%r11
  1723. mulq $m0
  1724. add %rax,%r12
  1725. mov 8*5($nptr),%rax
  1726. adc \$0,%rdx
  1727. add %r12,%r11
  1728. mov %rdx,%r12
  1729. adc \$0,%r12
  1730. mulq $m0
  1731. add %rax,%r13
  1732. mov 8*6($nptr),%rax
  1733. adc \$0,%rdx
  1734. add %r13,%r12
  1735. mov %rdx,%r13
  1736. adc \$0,%r13
  1737. mulq $m0
  1738. add %rax,%r14
  1739. mov 8*7($nptr),%rax
  1740. adc \$0,%rdx
  1741. add %r14,%r13
  1742. mov %rdx,%r14
  1743. adc \$0,%r14
  1744. mulq $m0
  1745. mov 48-16+8(%rsp,%rcx,8),$m0# pull n0*a[i]
  1746. add %rax,%r15
  1747. adc \$0,%rdx
  1748. add %r15,%r14
  1749. mov 8*0($nptr),%rax # pull n[0]
  1750. mov %rdx,%r15
  1751. adc \$0,%r15
  1752. dec %ecx
  1753. jnz .L8x_tail
  1754. lea 8*8($nptr),$nptr
  1755. mov 8+8(%rsp),%rdx # pull end of t[]
  1756. cmp 0+8(%rsp),$nptr # end of n[]?
  1757. jae .L8x_tail_done # break out of loop
  1758. mov 48+56+8(%rsp),$m0 # pull n0*a[0]
  1759. neg $carry
  1760. mov 8*0($nptr),%rax # pull n[0]
  1761. adc 8*0($tptr),%r8
  1762. adc 8*1($tptr),%r9
  1763. adc 8*2($tptr),%r10
  1764. adc 8*3($tptr),%r11
  1765. adc 8*4($tptr),%r12
  1766. adc 8*5($tptr),%r13
  1767. adc 8*6($tptr),%r14
  1768. adc 8*7($tptr),%r15
  1769. sbb $carry,$carry # top carry
  1770. mov \$8,%ecx
  1771. jmp .L8x_tail
  1772. .align 32
  1773. .L8x_tail_done:
  1774. xor %rax,%rax
  1775. add (%rdx),%r8 # can this overflow?
  1776. adc \$0,%r9
  1777. adc \$0,%r10
  1778. adc \$0,%r11
  1779. adc \$0,%r12
  1780. adc \$0,%r13
  1781. adc \$0,%r14
  1782. adc \$0,%r15
  1783. adc \$0,%rax
  1784. neg $carry
  1785. .L8x_no_tail:
  1786. adc 8*0($tptr),%r8
  1787. adc 8*1($tptr),%r9
  1788. adc 8*2($tptr),%r10
  1789. adc 8*3($tptr),%r11
  1790. adc 8*4($tptr),%r12
  1791. adc 8*5($tptr),%r13
  1792. adc 8*6($tptr),%r14
  1793. adc 8*7($tptr),%r15
  1794. adc \$0,%rax # top-most carry
  1795. mov -8($nptr),%rcx # np[num-1]
  1796. xor $carry,$carry
  1797. movq %xmm2,$nptr # restore $nptr
  1798. mov %r8,8*0($tptr) # store top 512 bits
  1799. mov %r9,8*1($tptr)
  1800. movq %xmm3,$num # $num is %r9, can't be moved upwards
  1801. mov %r10,8*2($tptr)
  1802. mov %r11,8*3($tptr)
  1803. mov %r12,8*4($tptr)
  1804. mov %r13,8*5($tptr)
  1805. mov %r14,8*6($tptr)
  1806. mov %r15,8*7($tptr)
  1807. lea 8*8($tptr),$tptr
  1808. cmp %rdx,$tptr # end of t[]?
  1809. jb .L8x_reduction_loop
  1810. ret
  1811. .size bn_sqr8x_internal,.-bn_sqr8x_internal
  1812. ___
  1813. }
  1814. ##############################################################
  1815. # Post-condition, 4x unrolled
  1816. #
  1817. {
  1818. my ($tptr,$nptr)=("%rbx","%rbp");
  1819. $code.=<<___;
  1820. .type __bn_post4x_internal,\@abi-omnipotent
  1821. .align 32
  1822. __bn_post4x_internal:
  1823. mov 8*0($nptr),%r12
  1824. lea (%rdi,$num),$tptr # %rdi was $tptr above
  1825. mov $num,%rcx
  1826. movq %xmm1,$rptr # restore $rptr
  1827. neg %rax
  1828. movq %xmm1,$aptr # prepare for back-to-back call
  1829. sar \$3+2,%rcx
  1830. dec %r12 # so that after 'not' we get -n[0]
  1831. xor %r10,%r10
  1832. mov 8*1($nptr),%r13
  1833. mov 8*2($nptr),%r14
  1834. mov 8*3($nptr),%r15
  1835. jmp .Lsqr4x_sub_entry
  1836. .align 16
  1837. .Lsqr4x_sub:
  1838. mov 8*0($nptr),%r12
  1839. mov 8*1($nptr),%r13
  1840. mov 8*2($nptr),%r14
  1841. mov 8*3($nptr),%r15
  1842. .Lsqr4x_sub_entry:
  1843. lea 8*4($nptr),$nptr
  1844. not %r12
  1845. not %r13
  1846. not %r14
  1847. not %r15
  1848. and %rax,%r12
  1849. and %rax,%r13
  1850. and %rax,%r14
  1851. and %rax,%r15
  1852. neg %r10 # mov %r10,%cf
  1853. adc 8*0($tptr),%r12
  1854. adc 8*1($tptr),%r13
  1855. adc 8*2($tptr),%r14
  1856. adc 8*3($tptr),%r15
  1857. mov %r12,8*0($rptr)
  1858. lea 8*4($tptr),$tptr
  1859. mov %r13,8*1($rptr)
  1860. sbb %r10,%r10 # mov %cf,%r10
  1861. mov %r14,8*2($rptr)
  1862. mov %r15,8*3($rptr)
  1863. lea 8*4($rptr),$rptr
  1864. inc %rcx # pass %cf
  1865. jnz .Lsqr4x_sub
  1866. mov $num,%r10 # prepare for back-to-back call
  1867. neg $num # restore $num
  1868. ret
  1869. .size __bn_post4x_internal,.-__bn_post4x_internal
  1870. ___
  1871. }
  1872. {
  1873. $code.=<<___;
  1874. .globl bn_from_montgomery
  1875. .type bn_from_montgomery,\@abi-omnipotent
  1876. .align 32
  1877. bn_from_montgomery:
  1878. testl \$7,`($win64?"48(%rsp)":"%r9d")`
  1879. jz bn_from_mont8x
  1880. xor %eax,%eax
  1881. ret
  1882. .size bn_from_montgomery,.-bn_from_montgomery
  1883. .type bn_from_mont8x,\@function,6
  1884. .align 32
  1885. bn_from_mont8x:
  1886. .cfi_startproc
  1887. .byte 0x67
  1888. mov %rsp,%rax
  1889. .cfi_def_cfa_register %rax
  1890. push %rbx
  1891. .cfi_push %rbx
  1892. push %rbp
  1893. .cfi_push %rbp
  1894. push %r12
  1895. .cfi_push %r12
  1896. push %r13
  1897. .cfi_push %r13
  1898. push %r14
  1899. .cfi_push %r14
  1900. push %r15
  1901. .cfi_push %r15
  1902. .Lfrom_prologue:
  1903. shl \$3,${num}d # convert $num to bytes
  1904. lea ($num,$num,2),%r10 # 3*$num in bytes
  1905. neg $num
  1906. mov ($n0),$n0 # *n0
  1907. ##############################################################
  1908. # Ensure that stack frame doesn't alias with $rptr+3*$num
  1909. # modulo 4096, which covers ret[num], am[num] and n[num]
  1910. # (see bn_exp.c). The stack is allocated to aligned with
  1911. # bn_power5's frame, and as bn_from_montgomery happens to be
  1912. # last operation, we use the opportunity to cleanse it.
  1913. #
  1914. lea -320(%rsp,$num,2),%r11
  1915. mov %rsp,%rbp
  1916. sub $rptr,%r11
  1917. and \$4095,%r11
  1918. cmp %r11,%r10
  1919. jb .Lfrom_sp_alt
  1920. sub %r11,%rbp # align with $aptr
  1921. lea -320(%rbp,$num,2),%rbp # future alloca(frame+2*$num*8+256)
  1922. jmp .Lfrom_sp_done
  1923. .align 32
  1924. .Lfrom_sp_alt:
  1925. lea 4096-320(,$num,2),%r10
  1926. lea -320(%rbp,$num,2),%rbp # future alloca(frame+2*$num*8+256)
  1927. sub %r10,%r11
  1928. mov \$0,%r10
  1929. cmovc %r10,%r11
  1930. sub %r11,%rbp
  1931. .Lfrom_sp_done:
  1932. and \$-64,%rbp
  1933. mov %rsp,%r11
  1934. sub %rbp,%r11
  1935. and \$-4096,%r11
  1936. lea (%rbp,%r11),%rsp
  1937. mov (%rsp),%r10
  1938. cmp %rbp,%rsp
  1939. ja .Lfrom_page_walk
  1940. jmp .Lfrom_page_walk_done
  1941. .Lfrom_page_walk:
  1942. lea -4096(%rsp),%rsp
  1943. mov (%rsp),%r10
  1944. cmp %rbp,%rsp
  1945. ja .Lfrom_page_walk
  1946. .Lfrom_page_walk_done:
  1947. mov $num,%r10
  1948. neg $num
  1949. ##############################################################
  1950. # Stack layout
  1951. #
  1952. # +0 saved $num, used in reduction section
  1953. # +8 &t[2*$num], used in reduction section
  1954. # +32 saved *n0
  1955. # +40 saved %rsp
  1956. # +48 t[2*$num]
  1957. #
  1958. mov $n0, 32(%rsp)
  1959. mov %rax, 40(%rsp) # save original %rsp
  1960. .cfi_cfa_expression %rsp+40,deref,+8
  1961. .Lfrom_body:
  1962. mov $num,%r11
  1963. lea 48(%rsp),%rax
  1964. pxor %xmm0,%xmm0
  1965. jmp .Lmul_by_1
  1966. .align 32
  1967. .Lmul_by_1:
  1968. movdqu ($aptr),%xmm1
  1969. movdqu 16($aptr),%xmm2
  1970. movdqu 32($aptr),%xmm3
  1971. movdqa %xmm0,(%rax,$num)
  1972. movdqu 48($aptr),%xmm4
  1973. movdqa %xmm0,16(%rax,$num)
  1974. .byte 0x48,0x8d,0xb6,0x40,0x00,0x00,0x00 # lea 64($aptr),$aptr
  1975. movdqa %xmm1,(%rax)
  1976. movdqa %xmm0,32(%rax,$num)
  1977. movdqa %xmm2,16(%rax)
  1978. movdqa %xmm0,48(%rax,$num)
  1979. movdqa %xmm3,32(%rax)
  1980. movdqa %xmm4,48(%rax)
  1981. lea 64(%rax),%rax
  1982. sub \$64,%r11
  1983. jnz .Lmul_by_1
  1984. movq $rptr,%xmm1
  1985. movq $nptr,%xmm2
  1986. .byte 0x67
  1987. mov $nptr,%rbp
  1988. movq %r10, %xmm3 # -num
  1989. ___
  1990. $code.=<<___ if ($addx);
  1991. mov OPENSSL_ia32cap_P+8(%rip),%r11d
  1992. and \$0x80108,%r11d
  1993. cmp \$0x80108,%r11d # check for AD*X+BMI2+BMI1
  1994. jne .Lfrom_mont_nox
  1995. lea (%rax,$num),$rptr
  1996. call __bn_sqrx8x_reduction
  1997. call __bn_postx4x_internal
  1998. pxor %xmm0,%xmm0
  1999. lea 48(%rsp),%rax
  2000. jmp .Lfrom_mont_zero
  2001. .align 32
  2002. .Lfrom_mont_nox:
  2003. ___
  2004. $code.=<<___;
  2005. call __bn_sqr8x_reduction
  2006. call __bn_post4x_internal
  2007. pxor %xmm0,%xmm0
  2008. lea 48(%rsp),%rax
  2009. jmp .Lfrom_mont_zero
  2010. .align 32
  2011. .Lfrom_mont_zero:
  2012. mov 40(%rsp),%rsi # restore %rsp
  2013. .cfi_def_cfa %rsi,8
  2014. movdqa %xmm0,16*0(%rax)
  2015. movdqa %xmm0,16*1(%rax)
  2016. movdqa %xmm0,16*2(%rax)
  2017. movdqa %xmm0,16*3(%rax)
  2018. lea 16*4(%rax),%rax
  2019. sub \$32,$num
  2020. jnz .Lfrom_mont_zero
  2021. mov \$1,%rax
  2022. mov -48(%rsi),%r15
  2023. .cfi_restore %r15
  2024. mov -40(%rsi),%r14
  2025. .cfi_restore %r14
  2026. mov -32(%rsi),%r13
  2027. .cfi_restore %r13
  2028. mov -24(%rsi),%r12
  2029. .cfi_restore %r12
  2030. mov -16(%rsi),%rbp
  2031. .cfi_restore %rbp
  2032. mov -8(%rsi),%rbx
  2033. .cfi_restore %rbx
  2034. lea (%rsi),%rsp
  2035. .cfi_def_cfa_register %rsp
  2036. .Lfrom_epilogue:
  2037. ret
  2038. .cfi_endproc
  2039. .size bn_from_mont8x,.-bn_from_mont8x
  2040. ___
  2041. }
  2042. }}}
  2043. if ($addx) {{{
  2044. my $bp="%rdx"; # restore original value
  2045. $code.=<<___;
  2046. .type bn_mulx4x_mont_gather5,\@function,6
  2047. .align 32
  2048. bn_mulx4x_mont_gather5:
  2049. .cfi_startproc
  2050. mov %rsp,%rax
  2051. .cfi_def_cfa_register %rax
  2052. .Lmulx4x_enter:
  2053. push %rbx
  2054. .cfi_push %rbx
  2055. push %rbp
  2056. .cfi_push %rbp
  2057. push %r12
  2058. .cfi_push %r12
  2059. push %r13
  2060. .cfi_push %r13
  2061. push %r14
  2062. .cfi_push %r14
  2063. push %r15
  2064. .cfi_push %r15
  2065. .Lmulx4x_prologue:
  2066. shl \$3,${num}d # convert $num to bytes
  2067. lea ($num,$num,2),%r10 # 3*$num in bytes
  2068. neg $num # -$num
  2069. mov ($n0),$n0 # *n0
  2070. ##############################################################
  2071. # Ensure that stack frame doesn't alias with $rptr+3*$num
  2072. # modulo 4096, which covers ret[num], am[num] and n[num]
  2073. # (see bn_exp.c). This is done to allow memory disambiguation
  2074. # logic do its magic. [Extra [num] is allocated in order
  2075. # to align with bn_power5's frame, which is cleansed after
  2076. # completing exponentiation. Extra 256 bytes is for power mask
  2077. # calculated from 7th argument, the index.]
  2078. #
  2079. lea -320(%rsp,$num,2),%r11
  2080. mov %rsp,%rbp
  2081. sub $rp,%r11
  2082. and \$4095,%r11
  2083. cmp %r11,%r10
  2084. jb .Lmulx4xsp_alt
  2085. sub %r11,%rbp # align with $aptr
  2086. lea -320(%rbp,$num,2),%rbp # future alloca(frame+2*$num*8+256)
  2087. jmp .Lmulx4xsp_done
  2088. .Lmulx4xsp_alt:
  2089. lea 4096-320(,$num,2),%r10
  2090. lea -320(%rbp,$num,2),%rbp # future alloca(frame+2*$num*8+256)
  2091. sub %r10,%r11
  2092. mov \$0,%r10
  2093. cmovc %r10,%r11
  2094. sub %r11,%rbp
  2095. .Lmulx4xsp_done:
  2096. and \$-64,%rbp # ensure alignment
  2097. mov %rsp,%r11
  2098. sub %rbp,%r11
  2099. and \$-4096,%r11
  2100. lea (%rbp,%r11),%rsp
  2101. mov (%rsp),%r10
  2102. cmp %rbp,%rsp
  2103. ja .Lmulx4x_page_walk
  2104. jmp .Lmulx4x_page_walk_done
  2105. .Lmulx4x_page_walk:
  2106. lea -4096(%rsp),%rsp
  2107. mov (%rsp),%r10
  2108. cmp %rbp,%rsp
  2109. ja .Lmulx4x_page_walk
  2110. .Lmulx4x_page_walk_done:
  2111. ##############################################################
  2112. # Stack layout
  2113. # +0 -num
  2114. # +8 off-loaded &b[i]
  2115. # +16 end of b[num]
  2116. # +24 inner counter
  2117. # +32 saved n0
  2118. # +40 saved %rsp
  2119. # +48
  2120. # +56 saved rp
  2121. # +64 tmp[num+1]
  2122. #
  2123. mov $n0, 32(%rsp) # save *n0
  2124. mov %rax,40(%rsp) # save original %rsp
  2125. .cfi_cfa_expression %rsp+40,deref,+8
  2126. .Lmulx4x_body:
  2127. call mulx4x_internal
  2128. mov 40(%rsp),%rsi # restore %rsp
  2129. .cfi_def_cfa %rsi,8
  2130. mov \$1,%rax
  2131. mov -48(%rsi),%r15
  2132. .cfi_restore %r15
  2133. mov -40(%rsi),%r14
  2134. .cfi_restore %r14
  2135. mov -32(%rsi),%r13
  2136. .cfi_restore %r13
  2137. mov -24(%rsi),%r12
  2138. .cfi_restore %r12
  2139. mov -16(%rsi),%rbp
  2140. .cfi_restore %rbp
  2141. mov -8(%rsi),%rbx
  2142. .cfi_restore %rbx
  2143. lea (%rsi),%rsp
  2144. .cfi_def_cfa_register %rsp
  2145. .Lmulx4x_epilogue:
  2146. ret
  2147. .cfi_endproc
  2148. .size bn_mulx4x_mont_gather5,.-bn_mulx4x_mont_gather5
  2149. .type mulx4x_internal,\@abi-omnipotent
  2150. .align 32
  2151. mulx4x_internal:
  2152. mov $num,8(%rsp) # save -$num (it was in bytes)
  2153. mov $num,%r10
  2154. neg $num # restore $num
  2155. shl \$5,$num
  2156. neg %r10 # restore $num
  2157. lea 128($bp,$num),%r13 # end of powers table (+size optimization)
  2158. shr \$5+5,$num
  2159. movd `($win64?56:8)`(%rax),%xmm5 # load 7th argument
  2160. sub \$1,$num
  2161. lea .Linc(%rip),%rax
  2162. mov %r13,16+8(%rsp) # end of b[num]
  2163. mov $num,24+8(%rsp) # inner counter
  2164. mov $rp, 56+8(%rsp) # save $rp
  2165. ___
  2166. my ($aptr, $bptr, $nptr, $tptr, $mi, $bi, $zero, $num)=
  2167. ("%rsi","%rdi","%rcx","%rbx","%r8","%r9","%rbp","%rax");
  2168. my $rptr=$bptr;
  2169. my $STRIDE=2**5*8; # 5 is "window size"
  2170. my $N=$STRIDE/4; # should match cache line size
  2171. $code.=<<___;
  2172. movdqa 0(%rax),%xmm0 # 00000001000000010000000000000000
  2173. movdqa 16(%rax),%xmm1 # 00000002000000020000000200000002
  2174. lea 88-112(%rsp,%r10),%r10 # place the mask after tp[num+1] (+ICache optimization)
  2175. lea 128($bp),$bptr # size optimization
  2176. pshufd \$0,%xmm5,%xmm5 # broadcast index
  2177. movdqa %xmm1,%xmm4
  2178. .byte 0x67
  2179. movdqa %xmm1,%xmm2
  2180. ___
  2181. ########################################################################
  2182. # calculate mask by comparing 0..31 to index and save result to stack
  2183. #
  2184. $code.=<<___;
  2185. .byte 0x67
  2186. paddd %xmm0,%xmm1
  2187. pcmpeqd %xmm5,%xmm0 # compare to 1,0
  2188. movdqa %xmm4,%xmm3
  2189. ___
  2190. for($i=0;$i<$STRIDE/16-4;$i+=4) {
  2191. $code.=<<___;
  2192. paddd %xmm1,%xmm2
  2193. pcmpeqd %xmm5,%xmm1 # compare to 3,2
  2194. movdqa %xmm0,`16*($i+0)+112`(%r10)
  2195. movdqa %xmm4,%xmm0
  2196. paddd %xmm2,%xmm3
  2197. pcmpeqd %xmm5,%xmm2 # compare to 5,4
  2198. movdqa %xmm1,`16*($i+1)+112`(%r10)
  2199. movdqa %xmm4,%xmm1
  2200. paddd %xmm3,%xmm0
  2201. pcmpeqd %xmm5,%xmm3 # compare to 7,6
  2202. movdqa %xmm2,`16*($i+2)+112`(%r10)
  2203. movdqa %xmm4,%xmm2
  2204. paddd %xmm0,%xmm1
  2205. pcmpeqd %xmm5,%xmm0
  2206. movdqa %xmm3,`16*($i+3)+112`(%r10)
  2207. movdqa %xmm4,%xmm3
  2208. ___
  2209. }
  2210. $code.=<<___; # last iteration can be optimized
  2211. .byte 0x67
  2212. paddd %xmm1,%xmm2
  2213. pcmpeqd %xmm5,%xmm1
  2214. movdqa %xmm0,`16*($i+0)+112`(%r10)
  2215. paddd %xmm2,%xmm3
  2216. pcmpeqd %xmm5,%xmm2
  2217. movdqa %xmm1,`16*($i+1)+112`(%r10)
  2218. pcmpeqd %xmm5,%xmm3
  2219. movdqa %xmm2,`16*($i+2)+112`(%r10)
  2220. pand `16*($i+0)-128`($bptr),%xmm0 # while it's still in register
  2221. pand `16*($i+1)-128`($bptr),%xmm1
  2222. pand `16*($i+2)-128`($bptr),%xmm2
  2223. movdqa %xmm3,`16*($i+3)+112`(%r10)
  2224. pand `16*($i+3)-128`($bptr),%xmm3
  2225. por %xmm2,%xmm0
  2226. por %xmm3,%xmm1
  2227. ___
  2228. for($i=0;$i<$STRIDE/16-4;$i+=4) {
  2229. $code.=<<___;
  2230. movdqa `16*($i+0)-128`($bptr),%xmm4
  2231. movdqa `16*($i+1)-128`($bptr),%xmm5
  2232. movdqa `16*($i+2)-128`($bptr),%xmm2
  2233. pand `16*($i+0)+112`(%r10),%xmm4
  2234. movdqa `16*($i+3)-128`($bptr),%xmm3
  2235. pand `16*($i+1)+112`(%r10),%xmm5
  2236. por %xmm4,%xmm0
  2237. pand `16*($i+2)+112`(%r10),%xmm2
  2238. por %xmm5,%xmm1
  2239. pand `16*($i+3)+112`(%r10),%xmm3
  2240. por %xmm2,%xmm0
  2241. por %xmm3,%xmm1
  2242. ___
  2243. }
  2244. $code.=<<___;
  2245. pxor %xmm1,%xmm0
  2246. pshufd \$0x4e,%xmm0,%xmm1
  2247. por %xmm1,%xmm0
  2248. lea $STRIDE($bptr),$bptr
  2249. movq %xmm0,%rdx # bp[0]
  2250. lea 64+8*4+8(%rsp),$tptr
  2251. mov %rdx,$bi
  2252. mulx 0*8($aptr),$mi,%rax # a[0]*b[0]
  2253. mulx 1*8($aptr),%r11,%r12 # a[1]*b[0]
  2254. add %rax,%r11
  2255. mulx 2*8($aptr),%rax,%r13 # ...
  2256. adc %rax,%r12
  2257. adc \$0,%r13
  2258. mulx 3*8($aptr),%rax,%r14
  2259. mov $mi,%r15
  2260. imulq 32+8(%rsp),$mi # "t[0]"*n0
  2261. xor $zero,$zero # cf=0, of=0
  2262. mov $mi,%rdx
  2263. mov $bptr,8+8(%rsp) # off-load &b[i]
  2264. lea 4*8($aptr),$aptr
  2265. adcx %rax,%r13
  2266. adcx $zero,%r14 # cf=0
  2267. mulx 0*8($nptr),%rax,%r10
  2268. adcx %rax,%r15 # discarded
  2269. adox %r11,%r10
  2270. mulx 1*8($nptr),%rax,%r11
  2271. adcx %rax,%r10
  2272. adox %r12,%r11
  2273. mulx 2*8($nptr),%rax,%r12
  2274. mov 24+8(%rsp),$bptr # counter value
  2275. mov %r10,-8*4($tptr)
  2276. adcx %rax,%r11
  2277. adox %r13,%r12
  2278. mulx 3*8($nptr),%rax,%r15
  2279. mov $bi,%rdx
  2280. mov %r11,-8*3($tptr)
  2281. adcx %rax,%r12
  2282. adox $zero,%r15 # of=0
  2283. lea 4*8($nptr),$nptr
  2284. mov %r12,-8*2($tptr)
  2285. jmp .Lmulx4x_1st
  2286. .align 32
  2287. .Lmulx4x_1st:
  2288. adcx $zero,%r15 # cf=0, modulo-scheduled
  2289. mulx 0*8($aptr),%r10,%rax # a[4]*b[0]
  2290. adcx %r14,%r10
  2291. mulx 1*8($aptr),%r11,%r14 # a[5]*b[0]
  2292. adcx %rax,%r11
  2293. mulx 2*8($aptr),%r12,%rax # ...
  2294. adcx %r14,%r12
  2295. mulx 3*8($aptr),%r13,%r14
  2296. .byte 0x67,0x67
  2297. mov $mi,%rdx
  2298. adcx %rax,%r13
  2299. adcx $zero,%r14 # cf=0
  2300. lea 4*8($aptr),$aptr
  2301. lea 4*8($tptr),$tptr
  2302. adox %r15,%r10
  2303. mulx 0*8($nptr),%rax,%r15
  2304. adcx %rax,%r10
  2305. adox %r15,%r11
  2306. mulx 1*8($nptr),%rax,%r15
  2307. adcx %rax,%r11
  2308. adox %r15,%r12
  2309. mulx 2*8($nptr),%rax,%r15
  2310. mov %r10,-5*8($tptr)
  2311. adcx %rax,%r12
  2312. mov %r11,-4*8($tptr)
  2313. adox %r15,%r13
  2314. mulx 3*8($nptr),%rax,%r15
  2315. mov $bi,%rdx
  2316. mov %r12,-3*8($tptr)
  2317. adcx %rax,%r13
  2318. adox $zero,%r15
  2319. lea 4*8($nptr),$nptr
  2320. mov %r13,-2*8($tptr)
  2321. dec $bptr # of=0, pass cf
  2322. jnz .Lmulx4x_1st
  2323. mov 8(%rsp),$num # load -num
  2324. adc $zero,%r15 # modulo-scheduled
  2325. lea ($aptr,$num),$aptr # rewind $aptr
  2326. add %r15,%r14
  2327. mov 8+8(%rsp),$bptr # re-load &b[i]
  2328. adc $zero,$zero # top-most carry
  2329. mov %r14,-1*8($tptr)
  2330. jmp .Lmulx4x_outer
  2331. .align 32
  2332. .Lmulx4x_outer:
  2333. lea 16-256($tptr),%r10 # where 256-byte mask is (+density control)
  2334. pxor %xmm4,%xmm4
  2335. .byte 0x67,0x67
  2336. pxor %xmm5,%xmm5
  2337. ___
  2338. for($i=0;$i<$STRIDE/16;$i+=4) {
  2339. $code.=<<___;
  2340. movdqa `16*($i+0)-128`($bptr),%xmm0
  2341. movdqa `16*($i+1)-128`($bptr),%xmm1
  2342. movdqa `16*($i+2)-128`($bptr),%xmm2
  2343. pand `16*($i+0)+256`(%r10),%xmm0
  2344. movdqa `16*($i+3)-128`($bptr),%xmm3
  2345. pand `16*($i+1)+256`(%r10),%xmm1
  2346. por %xmm0,%xmm4
  2347. pand `16*($i+2)+256`(%r10),%xmm2
  2348. por %xmm1,%xmm5
  2349. pand `16*($i+3)+256`(%r10),%xmm3
  2350. por %xmm2,%xmm4
  2351. por %xmm3,%xmm5
  2352. ___
  2353. }
  2354. $code.=<<___;
  2355. por %xmm5,%xmm4
  2356. pshufd \$0x4e,%xmm4,%xmm0
  2357. por %xmm4,%xmm0
  2358. lea $STRIDE($bptr),$bptr
  2359. movq %xmm0,%rdx # m0=bp[i]
  2360. mov $zero,($tptr) # save top-most carry
  2361. lea 4*8($tptr,$num),$tptr # rewind $tptr
  2362. mulx 0*8($aptr),$mi,%r11 # a[0]*b[i]
  2363. xor $zero,$zero # cf=0, of=0
  2364. mov %rdx,$bi
  2365. mulx 1*8($aptr),%r14,%r12 # a[1]*b[i]
  2366. adox -4*8($tptr),$mi # +t[0]
  2367. adcx %r14,%r11
  2368. mulx 2*8($aptr),%r15,%r13 # ...
  2369. adox -3*8($tptr),%r11
  2370. adcx %r15,%r12
  2371. mulx 3*8($aptr),%rdx,%r14
  2372. adox -2*8($tptr),%r12
  2373. adcx %rdx,%r13
  2374. lea ($nptr,$num),$nptr # rewind $nptr
  2375. lea 4*8($aptr),$aptr
  2376. adox -1*8($tptr),%r13
  2377. adcx $zero,%r14
  2378. adox $zero,%r14
  2379. mov $mi,%r15
  2380. imulq 32+8(%rsp),$mi # "t[0]"*n0
  2381. mov $mi,%rdx
  2382. xor $zero,$zero # cf=0, of=0
  2383. mov $bptr,8+8(%rsp) # off-load &b[i]
  2384. mulx 0*8($nptr),%rax,%r10
  2385. adcx %rax,%r15 # discarded
  2386. adox %r11,%r10
  2387. mulx 1*8($nptr),%rax,%r11
  2388. adcx %rax,%r10
  2389. adox %r12,%r11
  2390. mulx 2*8($nptr),%rax,%r12
  2391. adcx %rax,%r11
  2392. adox %r13,%r12
  2393. mulx 3*8($nptr),%rax,%r15
  2394. mov $bi,%rdx
  2395. mov 24+8(%rsp),$bptr # counter value
  2396. mov %r10,-8*4($tptr)
  2397. adcx %rax,%r12
  2398. mov %r11,-8*3($tptr)
  2399. adox $zero,%r15 # of=0
  2400. mov %r12,-8*2($tptr)
  2401. lea 4*8($nptr),$nptr
  2402. jmp .Lmulx4x_inner
  2403. .align 32
  2404. .Lmulx4x_inner:
  2405. mulx 0*8($aptr),%r10,%rax # a[4]*b[i]
  2406. adcx $zero,%r15 # cf=0, modulo-scheduled
  2407. adox %r14,%r10
  2408. mulx 1*8($aptr),%r11,%r14 # a[5]*b[i]
  2409. adcx 0*8($tptr),%r10
  2410. adox %rax,%r11
  2411. mulx 2*8($aptr),%r12,%rax # ...
  2412. adcx 1*8($tptr),%r11
  2413. adox %r14,%r12
  2414. mulx 3*8($aptr),%r13,%r14
  2415. mov $mi,%rdx
  2416. adcx 2*8($tptr),%r12
  2417. adox %rax,%r13
  2418. adcx 3*8($tptr),%r13
  2419. adox $zero,%r14 # of=0
  2420. lea 4*8($aptr),$aptr
  2421. lea 4*8($tptr),$tptr
  2422. adcx $zero,%r14 # cf=0
  2423. adox %r15,%r10
  2424. mulx 0*8($nptr),%rax,%r15
  2425. adcx %rax,%r10
  2426. adox %r15,%r11
  2427. mulx 1*8($nptr),%rax,%r15
  2428. adcx %rax,%r11
  2429. adox %r15,%r12
  2430. mulx 2*8($nptr),%rax,%r15
  2431. mov %r10,-5*8($tptr)
  2432. adcx %rax,%r12
  2433. adox %r15,%r13
  2434. mov %r11,-4*8($tptr)
  2435. mulx 3*8($nptr),%rax,%r15
  2436. mov $bi,%rdx
  2437. lea 4*8($nptr),$nptr
  2438. mov %r12,-3*8($tptr)
  2439. adcx %rax,%r13
  2440. adox $zero,%r15
  2441. mov %r13,-2*8($tptr)
  2442. dec $bptr # of=0, pass cf
  2443. jnz .Lmulx4x_inner
  2444. mov 0+8(%rsp),$num # load -num
  2445. adc $zero,%r15 # modulo-scheduled
  2446. sub 0*8($tptr),$bptr # pull top-most carry to %cf
  2447. mov 8+8(%rsp),$bptr # re-load &b[i]
  2448. mov 16+8(%rsp),%r10
  2449. adc %r15,%r14
  2450. lea ($aptr,$num),$aptr # rewind $aptr
  2451. adc $zero,$zero # top-most carry
  2452. mov %r14,-1*8($tptr)
  2453. cmp %r10,$bptr
  2454. jb .Lmulx4x_outer
  2455. mov -8($nptr),%r10
  2456. mov $zero,%r8
  2457. mov ($nptr,$num),%r12
  2458. lea ($nptr,$num),%rbp # rewind $nptr
  2459. mov $num,%rcx
  2460. lea ($tptr,$num),%rdi # rewind $tptr
  2461. xor %eax,%eax
  2462. xor %r15,%r15
  2463. sub %r14,%r10 # compare top-most words
  2464. adc %r15,%r15
  2465. or %r15,%r8
  2466. sar \$3+2,%rcx
  2467. sub %r8,%rax # %rax=-%r8
  2468. mov 56+8(%rsp),%rdx # restore rp
  2469. dec %r12 # so that after 'not' we get -n[0]
  2470. mov 8*1(%rbp),%r13
  2471. xor %r8,%r8
  2472. mov 8*2(%rbp),%r14
  2473. mov 8*3(%rbp),%r15
  2474. jmp .Lsqrx4x_sub_entry # common post-condition
  2475. .size mulx4x_internal,.-mulx4x_internal
  2476. ___
  2477. } {
  2478. ######################################################################
  2479. # void bn_power5(
  2480. my $rptr="%rdi"; # BN_ULONG *rptr,
  2481. my $aptr="%rsi"; # const BN_ULONG *aptr,
  2482. my $bptr="%rdx"; # const void *table,
  2483. my $nptr="%rcx"; # const BN_ULONG *nptr,
  2484. my $n0 ="%r8"; # const BN_ULONG *n0);
  2485. my $num ="%r9"; # int num, has to be divisible by 8
  2486. # int pwr);
  2487. my ($i,$j,$tptr)=("%rbp","%rcx",$rptr);
  2488. my @A0=("%r10","%r11");
  2489. my @A1=("%r12","%r13");
  2490. my ($a0,$a1,$ai)=("%r14","%r15","%rbx");
  2491. $code.=<<___;
  2492. .type bn_powerx5,\@function,6
  2493. .align 32
  2494. bn_powerx5:
  2495. .cfi_startproc
  2496. mov %rsp,%rax
  2497. .cfi_def_cfa_register %rax
  2498. .Lpowerx5_enter:
  2499. push %rbx
  2500. .cfi_push %rbx
  2501. push %rbp
  2502. .cfi_push %rbp
  2503. push %r12
  2504. .cfi_push %r12
  2505. push %r13
  2506. .cfi_push %r13
  2507. push %r14
  2508. .cfi_push %r14
  2509. push %r15
  2510. .cfi_push %r15
  2511. .Lpowerx5_prologue:
  2512. shl \$3,${num}d # convert $num to bytes
  2513. lea ($num,$num,2),%r10 # 3*$num in bytes
  2514. neg $num
  2515. mov ($n0),$n0 # *n0
  2516. ##############################################################
  2517. # Ensure that stack frame doesn't alias with $rptr+3*$num
  2518. # modulo 4096, which covers ret[num], am[num] and n[num]
  2519. # (see bn_exp.c). This is done to allow memory disambiguation
  2520. # logic do its magic. [Extra 256 bytes is for power mask
  2521. # calculated from 7th argument, the index.]
  2522. #
  2523. lea -320(%rsp,$num,2),%r11
  2524. mov %rsp,%rbp
  2525. sub $rptr,%r11
  2526. and \$4095,%r11
  2527. cmp %r11,%r10
  2528. jb .Lpwrx_sp_alt
  2529. sub %r11,%rbp # align with $aptr
  2530. lea -320(%rbp,$num,2),%rbp # future alloca(frame+2*$num*8+256)
  2531. jmp .Lpwrx_sp_done
  2532. .align 32
  2533. .Lpwrx_sp_alt:
  2534. lea 4096-320(,$num,2),%r10
  2535. lea -320(%rbp,$num,2),%rbp # alloca(frame+2*$num*8+256)
  2536. sub %r10,%r11
  2537. mov \$0,%r10
  2538. cmovc %r10,%r11
  2539. sub %r11,%rbp
  2540. .Lpwrx_sp_done:
  2541. and \$-64,%rbp
  2542. mov %rsp,%r11
  2543. sub %rbp,%r11
  2544. and \$-4096,%r11
  2545. lea (%rbp,%r11),%rsp
  2546. mov (%rsp),%r10
  2547. cmp %rbp,%rsp
  2548. ja .Lpwrx_page_walk
  2549. jmp .Lpwrx_page_walk_done
  2550. .Lpwrx_page_walk:
  2551. lea -4096(%rsp),%rsp
  2552. mov (%rsp),%r10
  2553. cmp %rbp,%rsp
  2554. ja .Lpwrx_page_walk
  2555. .Lpwrx_page_walk_done:
  2556. mov $num,%r10
  2557. neg $num
  2558. ##############################################################
  2559. # Stack layout
  2560. #
  2561. # +0 saved $num, used in reduction section
  2562. # +8 &t[2*$num], used in reduction section
  2563. # +16 intermediate carry bit
  2564. # +24 top-most carry bit, used in reduction section
  2565. # +32 saved *n0
  2566. # +40 saved %rsp
  2567. # +48 t[2*$num]
  2568. #
  2569. pxor %xmm0,%xmm0
  2570. movq $rptr,%xmm1 # save $rptr
  2571. movq $nptr,%xmm2 # save $nptr
  2572. movq %r10, %xmm3 # -$num
  2573. movq $bptr,%xmm4
  2574. mov $n0, 32(%rsp)
  2575. mov %rax, 40(%rsp) # save original %rsp
  2576. .cfi_cfa_expression %rsp+40,deref,+8
  2577. .Lpowerx5_body:
  2578. call __bn_sqrx8x_internal
  2579. call __bn_postx4x_internal
  2580. call __bn_sqrx8x_internal
  2581. call __bn_postx4x_internal
  2582. call __bn_sqrx8x_internal
  2583. call __bn_postx4x_internal
  2584. call __bn_sqrx8x_internal
  2585. call __bn_postx4x_internal
  2586. call __bn_sqrx8x_internal
  2587. call __bn_postx4x_internal
  2588. mov %r10,$num # -num
  2589. mov $aptr,$rptr
  2590. movq %xmm2,$nptr
  2591. movq %xmm4,$bptr
  2592. mov 40(%rsp),%rax
  2593. call mulx4x_internal
  2594. mov 40(%rsp),%rsi # restore %rsp
  2595. .cfi_def_cfa %rsi,8
  2596. mov \$1,%rax
  2597. mov -48(%rsi),%r15
  2598. .cfi_restore %r15
  2599. mov -40(%rsi),%r14
  2600. .cfi_restore %r14
  2601. mov -32(%rsi),%r13
  2602. .cfi_restore %r13
  2603. mov -24(%rsi),%r12
  2604. .cfi_restore %r12
  2605. mov -16(%rsi),%rbp
  2606. .cfi_restore %rbp
  2607. mov -8(%rsi),%rbx
  2608. .cfi_restore %rbx
  2609. lea (%rsi),%rsp
  2610. .cfi_def_cfa_register %rsp
  2611. .Lpowerx5_epilogue:
  2612. ret
  2613. .cfi_endproc
  2614. .size bn_powerx5,.-bn_powerx5
  2615. .globl bn_sqrx8x_internal
  2616. .hidden bn_sqrx8x_internal
  2617. .type bn_sqrx8x_internal,\@abi-omnipotent
  2618. .align 32
  2619. bn_sqrx8x_internal:
  2620. __bn_sqrx8x_internal:
  2621. ##################################################################
  2622. # Squaring part:
  2623. #
  2624. # a) multiply-n-add everything but a[i]*a[i];
  2625. # b) shift result of a) by 1 to the left and accumulate
  2626. # a[i]*a[i] products;
  2627. #
  2628. ##################################################################
  2629. # a[7]a[7]a[6]a[6]a[5]a[5]a[4]a[4]a[3]a[3]a[2]a[2]a[1]a[1]a[0]a[0]
  2630. # a[1]a[0]
  2631. # a[2]a[0]
  2632. # a[3]a[0]
  2633. # a[2]a[1]
  2634. # a[3]a[1]
  2635. # a[3]a[2]
  2636. #
  2637. # a[4]a[0]
  2638. # a[5]a[0]
  2639. # a[6]a[0]
  2640. # a[7]a[0]
  2641. # a[4]a[1]
  2642. # a[5]a[1]
  2643. # a[6]a[1]
  2644. # a[7]a[1]
  2645. # a[4]a[2]
  2646. # a[5]a[2]
  2647. # a[6]a[2]
  2648. # a[7]a[2]
  2649. # a[4]a[3]
  2650. # a[5]a[3]
  2651. # a[6]a[3]
  2652. # a[7]a[3]
  2653. #
  2654. # a[5]a[4]
  2655. # a[6]a[4]
  2656. # a[7]a[4]
  2657. # a[6]a[5]
  2658. # a[7]a[5]
  2659. # a[7]a[6]
  2660. # a[7]a[7]a[6]a[6]a[5]a[5]a[4]a[4]a[3]a[3]a[2]a[2]a[1]a[1]a[0]a[0]
  2661. ___
  2662. {
  2663. my ($zero,$carry)=("%rbp","%rcx");
  2664. my $aaptr=$zero;
  2665. $code.=<<___;
  2666. lea 48+8(%rsp),$tptr
  2667. lea ($aptr,$num),$aaptr
  2668. mov $num,0+8(%rsp) # save $num
  2669. mov $aaptr,8+8(%rsp) # save end of $aptr
  2670. jmp .Lsqr8x_zero_start
  2671. .align 32
  2672. .byte 0x66,0x66,0x66,0x2e,0x0f,0x1f,0x84,0x00,0x00,0x00,0x00,0x00
  2673. .Lsqrx8x_zero:
  2674. .byte 0x3e
  2675. movdqa %xmm0,0*8($tptr)
  2676. movdqa %xmm0,2*8($tptr)
  2677. movdqa %xmm0,4*8($tptr)
  2678. movdqa %xmm0,6*8($tptr)
  2679. .Lsqr8x_zero_start: # aligned at 32
  2680. movdqa %xmm0,8*8($tptr)
  2681. movdqa %xmm0,10*8($tptr)
  2682. movdqa %xmm0,12*8($tptr)
  2683. movdqa %xmm0,14*8($tptr)
  2684. lea 16*8($tptr),$tptr
  2685. sub \$64,$num
  2686. jnz .Lsqrx8x_zero
  2687. mov 0*8($aptr),%rdx # a[0], modulo-scheduled
  2688. #xor %r9,%r9 # t[1], ex-$num, zero already
  2689. xor %r10,%r10
  2690. xor %r11,%r11
  2691. xor %r12,%r12
  2692. xor %r13,%r13
  2693. xor %r14,%r14
  2694. xor %r15,%r15
  2695. lea 48+8(%rsp),$tptr
  2696. xor $zero,$zero # cf=0, cf=0
  2697. jmp .Lsqrx8x_outer_loop
  2698. .align 32
  2699. .Lsqrx8x_outer_loop:
  2700. mulx 1*8($aptr),%r8,%rax # a[1]*a[0]
  2701. adcx %r9,%r8 # a[1]*a[0]+=t[1]
  2702. adox %rax,%r10
  2703. mulx 2*8($aptr),%r9,%rax # a[2]*a[0]
  2704. adcx %r10,%r9
  2705. adox %rax,%r11
  2706. .byte 0xc4,0xe2,0xab,0xf6,0x86,0x18,0x00,0x00,0x00 # mulx 3*8($aptr),%r10,%rax # ...
  2707. adcx %r11,%r10
  2708. adox %rax,%r12
  2709. .byte 0xc4,0xe2,0xa3,0xf6,0x86,0x20,0x00,0x00,0x00 # mulx 4*8($aptr),%r11,%rax
  2710. adcx %r12,%r11
  2711. adox %rax,%r13
  2712. mulx 5*8($aptr),%r12,%rax
  2713. adcx %r13,%r12
  2714. adox %rax,%r14
  2715. mulx 6*8($aptr),%r13,%rax
  2716. adcx %r14,%r13
  2717. adox %r15,%rax
  2718. mulx 7*8($aptr),%r14,%r15
  2719. mov 1*8($aptr),%rdx # a[1]
  2720. adcx %rax,%r14
  2721. adox $zero,%r15
  2722. adc 8*8($tptr),%r15
  2723. mov %r8,1*8($tptr) # t[1]
  2724. mov %r9,2*8($tptr) # t[2]
  2725. sbb $carry,$carry # mov %cf,$carry
  2726. xor $zero,$zero # cf=0, of=0
  2727. mulx 2*8($aptr),%r8,%rbx # a[2]*a[1]
  2728. mulx 3*8($aptr),%r9,%rax # a[3]*a[1]
  2729. adcx %r10,%r8
  2730. adox %rbx,%r9
  2731. mulx 4*8($aptr),%r10,%rbx # ...
  2732. adcx %r11,%r9
  2733. adox %rax,%r10
  2734. .byte 0xc4,0xe2,0xa3,0xf6,0x86,0x28,0x00,0x00,0x00 # mulx 5*8($aptr),%r11,%rax
  2735. adcx %r12,%r10
  2736. adox %rbx,%r11
  2737. .byte 0xc4,0xe2,0x9b,0xf6,0x9e,0x30,0x00,0x00,0x00 # mulx 6*8($aptr),%r12,%rbx
  2738. adcx %r13,%r11
  2739. adox %r14,%r12
  2740. .byte 0xc4,0x62,0x93,0xf6,0xb6,0x38,0x00,0x00,0x00 # mulx 7*8($aptr),%r13,%r14
  2741. mov 2*8($aptr),%rdx # a[2]
  2742. adcx %rax,%r12
  2743. adox %rbx,%r13
  2744. adcx %r15,%r13
  2745. adox $zero,%r14 # of=0
  2746. adcx $zero,%r14 # cf=0
  2747. mov %r8,3*8($tptr) # t[3]
  2748. mov %r9,4*8($tptr) # t[4]
  2749. mulx 3*8($aptr),%r8,%rbx # a[3]*a[2]
  2750. mulx 4*8($aptr),%r9,%rax # a[4]*a[2]
  2751. adcx %r10,%r8
  2752. adox %rbx,%r9
  2753. mulx 5*8($aptr),%r10,%rbx # ...
  2754. adcx %r11,%r9
  2755. adox %rax,%r10
  2756. .byte 0xc4,0xe2,0xa3,0xf6,0x86,0x30,0x00,0x00,0x00 # mulx 6*8($aptr),%r11,%rax
  2757. adcx %r12,%r10
  2758. adox %r13,%r11
  2759. .byte 0xc4,0x62,0x9b,0xf6,0xae,0x38,0x00,0x00,0x00 # mulx 7*8($aptr),%r12,%r13
  2760. .byte 0x3e
  2761. mov 3*8($aptr),%rdx # a[3]
  2762. adcx %rbx,%r11
  2763. adox %rax,%r12
  2764. adcx %r14,%r12
  2765. mov %r8,5*8($tptr) # t[5]
  2766. mov %r9,6*8($tptr) # t[6]
  2767. mulx 4*8($aptr),%r8,%rax # a[4]*a[3]
  2768. adox $zero,%r13 # of=0
  2769. adcx $zero,%r13 # cf=0
  2770. mulx 5*8($aptr),%r9,%rbx # a[5]*a[3]
  2771. adcx %r10,%r8
  2772. adox %rax,%r9
  2773. mulx 6*8($aptr),%r10,%rax # ...
  2774. adcx %r11,%r9
  2775. adox %r12,%r10
  2776. mulx 7*8($aptr),%r11,%r12
  2777. mov 4*8($aptr),%rdx # a[4]
  2778. mov 5*8($aptr),%r14 # a[5]
  2779. adcx %rbx,%r10
  2780. adox %rax,%r11
  2781. mov 6*8($aptr),%r15 # a[6]
  2782. adcx %r13,%r11
  2783. adox $zero,%r12 # of=0
  2784. adcx $zero,%r12 # cf=0
  2785. mov %r8,7*8($tptr) # t[7]
  2786. mov %r9,8*8($tptr) # t[8]
  2787. mulx %r14,%r9,%rax # a[5]*a[4]
  2788. mov 7*8($aptr),%r8 # a[7]
  2789. adcx %r10,%r9
  2790. mulx %r15,%r10,%rbx # a[6]*a[4]
  2791. adox %rax,%r10
  2792. adcx %r11,%r10
  2793. mulx %r8,%r11,%rax # a[7]*a[4]
  2794. mov %r14,%rdx # a[5]
  2795. adox %rbx,%r11
  2796. adcx %r12,%r11
  2797. #adox $zero,%rax # of=0
  2798. adcx $zero,%rax # cf=0
  2799. mulx %r15,%r14,%rbx # a[6]*a[5]
  2800. mulx %r8,%r12,%r13 # a[7]*a[5]
  2801. mov %r15,%rdx # a[6]
  2802. lea 8*8($aptr),$aptr
  2803. adcx %r14,%r11
  2804. adox %rbx,%r12
  2805. adcx %rax,%r12
  2806. adox $zero,%r13
  2807. .byte 0x67,0x67
  2808. mulx %r8,%r8,%r14 # a[7]*a[6]
  2809. adcx %r8,%r13
  2810. adcx $zero,%r14
  2811. cmp 8+8(%rsp),$aptr
  2812. je .Lsqrx8x_outer_break
  2813. neg $carry # mov $carry,%cf
  2814. mov \$-8,%rcx
  2815. mov $zero,%r15
  2816. mov 8*8($tptr),%r8
  2817. adcx 9*8($tptr),%r9 # +=t[9]
  2818. adcx 10*8($tptr),%r10 # ...
  2819. adcx 11*8($tptr),%r11
  2820. adc 12*8($tptr),%r12
  2821. adc 13*8($tptr),%r13
  2822. adc 14*8($tptr),%r14
  2823. adc 15*8($tptr),%r15
  2824. lea ($aptr),$aaptr
  2825. lea 2*64($tptr),$tptr
  2826. sbb %rax,%rax # mov %cf,$carry
  2827. mov -64($aptr),%rdx # a[0]
  2828. mov %rax,16+8(%rsp) # offload $carry
  2829. mov $tptr,24+8(%rsp)
  2830. #lea 8*8($tptr),$tptr # see 2*8*8($tptr) above
  2831. xor %eax,%eax # cf=0, of=0
  2832. jmp .Lsqrx8x_loop
  2833. .align 32
  2834. .Lsqrx8x_loop:
  2835. mov %r8,%rbx
  2836. mulx 0*8($aaptr),%rax,%r8 # a[8]*a[i]
  2837. adcx %rax,%rbx # +=t[8]
  2838. adox %r9,%r8
  2839. mulx 1*8($aaptr),%rax,%r9 # ...
  2840. adcx %rax,%r8
  2841. adox %r10,%r9
  2842. mulx 2*8($aaptr),%rax,%r10
  2843. adcx %rax,%r9
  2844. adox %r11,%r10
  2845. mulx 3*8($aaptr),%rax,%r11
  2846. adcx %rax,%r10
  2847. adox %r12,%r11
  2848. .byte 0xc4,0x62,0xfb,0xf6,0xa5,0x20,0x00,0x00,0x00 # mulx 4*8($aaptr),%rax,%r12
  2849. adcx %rax,%r11
  2850. adox %r13,%r12
  2851. mulx 5*8($aaptr),%rax,%r13
  2852. adcx %rax,%r12
  2853. adox %r14,%r13
  2854. mulx 6*8($aaptr),%rax,%r14
  2855. mov %rbx,($tptr,%rcx,8) # store t[8+i]
  2856. mov \$0,%ebx
  2857. adcx %rax,%r13
  2858. adox %r15,%r14
  2859. .byte 0xc4,0x62,0xfb,0xf6,0xbd,0x38,0x00,0x00,0x00 # mulx 7*8($aaptr),%rax,%r15
  2860. mov 8($aptr,%rcx,8),%rdx # a[i]
  2861. adcx %rax,%r14
  2862. adox %rbx,%r15 # %rbx is 0, of=0
  2863. adcx %rbx,%r15 # cf=0
  2864. .byte 0x67
  2865. inc %rcx # of=0
  2866. jnz .Lsqrx8x_loop
  2867. lea 8*8($aaptr),$aaptr
  2868. mov \$-8,%rcx
  2869. cmp 8+8(%rsp),$aaptr # done?
  2870. je .Lsqrx8x_break
  2871. sub 16+8(%rsp),%rbx # mov 16(%rsp),%cf
  2872. .byte 0x66
  2873. mov -64($aptr),%rdx
  2874. adcx 0*8($tptr),%r8
  2875. adcx 1*8($tptr),%r9
  2876. adc 2*8($tptr),%r10
  2877. adc 3*8($tptr),%r11
  2878. adc 4*8($tptr),%r12
  2879. adc 5*8($tptr),%r13
  2880. adc 6*8($tptr),%r14
  2881. adc 7*8($tptr),%r15
  2882. lea 8*8($tptr),$tptr
  2883. .byte 0x67
  2884. sbb %rax,%rax # mov %cf,%rax
  2885. xor %ebx,%ebx # cf=0, of=0
  2886. mov %rax,16+8(%rsp) # offload carry
  2887. jmp .Lsqrx8x_loop
  2888. .align 32
  2889. .Lsqrx8x_break:
  2890. xor $zero,$zero
  2891. sub 16+8(%rsp),%rbx # mov 16(%rsp),%cf
  2892. adcx $zero,%r8
  2893. mov 24+8(%rsp),$carry # initial $tptr, borrow $carry
  2894. adcx $zero,%r9
  2895. mov 0*8($aptr),%rdx # a[8], modulo-scheduled
  2896. adc \$0,%r10
  2897. mov %r8,0*8($tptr)
  2898. adc \$0,%r11
  2899. adc \$0,%r12
  2900. adc \$0,%r13
  2901. adc \$0,%r14
  2902. adc \$0,%r15
  2903. cmp $carry,$tptr # cf=0, of=0
  2904. je .Lsqrx8x_outer_loop
  2905. mov %r9,1*8($tptr)
  2906. mov 1*8($carry),%r9
  2907. mov %r10,2*8($tptr)
  2908. mov 2*8($carry),%r10
  2909. mov %r11,3*8($tptr)
  2910. mov 3*8($carry),%r11
  2911. mov %r12,4*8($tptr)
  2912. mov 4*8($carry),%r12
  2913. mov %r13,5*8($tptr)
  2914. mov 5*8($carry),%r13
  2915. mov %r14,6*8($tptr)
  2916. mov 6*8($carry),%r14
  2917. mov %r15,7*8($tptr)
  2918. mov 7*8($carry),%r15
  2919. mov $carry,$tptr
  2920. jmp .Lsqrx8x_outer_loop
  2921. .align 32
  2922. .Lsqrx8x_outer_break:
  2923. mov %r9,9*8($tptr) # t[9]
  2924. movq %xmm3,%rcx # -$num
  2925. mov %r10,10*8($tptr) # ...
  2926. mov %r11,11*8($tptr)
  2927. mov %r12,12*8($tptr)
  2928. mov %r13,13*8($tptr)
  2929. mov %r14,14*8($tptr)
  2930. ___
  2931. } {
  2932. my $i="%rcx";
  2933. $code.=<<___;
  2934. lea 48+8(%rsp),$tptr
  2935. mov ($aptr,$i),%rdx # a[0]
  2936. mov 8($tptr),$A0[1] # t[1]
  2937. xor $A0[0],$A0[0] # t[0], of=0, cf=0
  2938. mov 0+8(%rsp),$num # restore $num
  2939. adox $A0[1],$A0[1]
  2940. mov 16($tptr),$A1[0] # t[2] # prefetch
  2941. mov 24($tptr),$A1[1] # t[3] # prefetch
  2942. #jmp .Lsqrx4x_shift_n_add # happens to be aligned
  2943. .align 32
  2944. .Lsqrx4x_shift_n_add:
  2945. mulx %rdx,%rax,%rbx
  2946. adox $A1[0],$A1[0]
  2947. adcx $A0[0],%rax
  2948. .byte 0x48,0x8b,0x94,0x0e,0x08,0x00,0x00,0x00 # mov 8($aptr,$i),%rdx # a[i+1] # prefetch
  2949. .byte 0x4c,0x8b,0x97,0x20,0x00,0x00,0x00 # mov 32($tptr),$A0[0] # t[2*i+4] # prefetch
  2950. adox $A1[1],$A1[1]
  2951. adcx $A0[1],%rbx
  2952. mov 40($tptr),$A0[1] # t[2*i+4+1] # prefetch
  2953. mov %rax,0($tptr)
  2954. mov %rbx,8($tptr)
  2955. mulx %rdx,%rax,%rbx
  2956. adox $A0[0],$A0[0]
  2957. adcx $A1[0],%rax
  2958. mov 16($aptr,$i),%rdx # a[i+2] # prefetch
  2959. mov 48($tptr),$A1[0] # t[2*i+6] # prefetch
  2960. adox $A0[1],$A0[1]
  2961. adcx $A1[1],%rbx
  2962. mov 56($tptr),$A1[1] # t[2*i+6+1] # prefetch
  2963. mov %rax,16($tptr)
  2964. mov %rbx,24($tptr)
  2965. mulx %rdx,%rax,%rbx
  2966. adox $A1[0],$A1[0]
  2967. adcx $A0[0],%rax
  2968. mov 24($aptr,$i),%rdx # a[i+3] # prefetch
  2969. lea 32($i),$i
  2970. mov 64($tptr),$A0[0] # t[2*i+8] # prefetch
  2971. adox $A1[1],$A1[1]
  2972. adcx $A0[1],%rbx
  2973. mov 72($tptr),$A0[1] # t[2*i+8+1] # prefetch
  2974. mov %rax,32($tptr)
  2975. mov %rbx,40($tptr)
  2976. mulx %rdx,%rax,%rbx
  2977. adox $A0[0],$A0[0]
  2978. adcx $A1[0],%rax
  2979. jrcxz .Lsqrx4x_shift_n_add_break
  2980. .byte 0x48,0x8b,0x94,0x0e,0x00,0x00,0x00,0x00 # mov 0($aptr,$i),%rdx # a[i+4] # prefetch
  2981. adox $A0[1],$A0[1]
  2982. adcx $A1[1],%rbx
  2983. mov 80($tptr),$A1[0] # t[2*i+10] # prefetch
  2984. mov 88($tptr),$A1[1] # t[2*i+10+1] # prefetch
  2985. mov %rax,48($tptr)
  2986. mov %rbx,56($tptr)
  2987. lea 64($tptr),$tptr
  2988. nop
  2989. jmp .Lsqrx4x_shift_n_add
  2990. .align 32
  2991. .Lsqrx4x_shift_n_add_break:
  2992. adcx $A1[1],%rbx
  2993. mov %rax,48($tptr)
  2994. mov %rbx,56($tptr)
  2995. lea 64($tptr),$tptr # end of t[] buffer
  2996. ___
  2997. }
  2998. ######################################################################
  2999. # Montgomery reduction part, "word-by-word" algorithm.
  3000. #
  3001. # This new path is inspired by multiple submissions from Intel, by
  3002. # Shay Gueron, Vlad Krasnov, Erdinc Ozturk, James Guilford,
  3003. # Vinodh Gopal...
  3004. {
  3005. my ($nptr,$carry,$m0)=("%rbp","%rsi","%rdx");
  3006. $code.=<<___;
  3007. movq %xmm2,$nptr
  3008. __bn_sqrx8x_reduction:
  3009. xor %eax,%eax # initial top-most carry bit
  3010. mov 32+8(%rsp),%rbx # n0
  3011. mov 48+8(%rsp),%rdx # "%r8", 8*0($tptr)
  3012. lea -8*8($nptr,$num),%rcx # end of n[]
  3013. #lea 48+8(%rsp,$num,2),$tptr # end of t[] buffer
  3014. mov %rcx, 0+8(%rsp) # save end of n[]
  3015. mov $tptr,8+8(%rsp) # save end of t[]
  3016. lea 48+8(%rsp),$tptr # initial t[] window
  3017. jmp .Lsqrx8x_reduction_loop
  3018. .align 32
  3019. .Lsqrx8x_reduction_loop:
  3020. mov 8*1($tptr),%r9
  3021. mov 8*2($tptr),%r10
  3022. mov 8*3($tptr),%r11
  3023. mov 8*4($tptr),%r12
  3024. mov %rdx,%r8
  3025. imulq %rbx,%rdx # n0*a[i]
  3026. mov 8*5($tptr),%r13
  3027. mov 8*6($tptr),%r14
  3028. mov 8*7($tptr),%r15
  3029. mov %rax,24+8(%rsp) # store top-most carry bit
  3030. lea 8*8($tptr),$tptr
  3031. xor $carry,$carry # cf=0,of=0
  3032. mov \$-8,%rcx
  3033. jmp .Lsqrx8x_reduce
  3034. .align 32
  3035. .Lsqrx8x_reduce:
  3036. mov %r8, %rbx
  3037. mulx 8*0($nptr),%rax,%r8 # n[0]
  3038. adcx %rbx,%rax # discarded
  3039. adox %r9,%r8
  3040. mulx 8*1($nptr),%rbx,%r9 # n[1]
  3041. adcx %rbx,%r8
  3042. adox %r10,%r9
  3043. mulx 8*2($nptr),%rbx,%r10
  3044. adcx %rbx,%r9
  3045. adox %r11,%r10
  3046. mulx 8*3($nptr),%rbx,%r11
  3047. adcx %rbx,%r10
  3048. adox %r12,%r11
  3049. .byte 0xc4,0x62,0xe3,0xf6,0xa5,0x20,0x00,0x00,0x00 # mulx 8*4($nptr),%rbx,%r12
  3050. mov %rdx,%rax
  3051. mov %r8,%rdx
  3052. adcx %rbx,%r11
  3053. adox %r13,%r12
  3054. mulx 32+8(%rsp),%rbx,%rdx # %rdx discarded
  3055. mov %rax,%rdx
  3056. mov %rax,64+48+8(%rsp,%rcx,8) # put aside n0*a[i]
  3057. mulx 8*5($nptr),%rax,%r13
  3058. adcx %rax,%r12
  3059. adox %r14,%r13
  3060. mulx 8*6($nptr),%rax,%r14
  3061. adcx %rax,%r13
  3062. adox %r15,%r14
  3063. mulx 8*7($nptr),%rax,%r15
  3064. mov %rbx,%rdx
  3065. adcx %rax,%r14
  3066. adox $carry,%r15 # $carry is 0
  3067. adcx $carry,%r15 # cf=0
  3068. .byte 0x67,0x67,0x67
  3069. inc %rcx # of=0
  3070. jnz .Lsqrx8x_reduce
  3071. mov $carry,%rax # xor %rax,%rax
  3072. cmp 0+8(%rsp),$nptr # end of n[]?
  3073. jae .Lsqrx8x_no_tail
  3074. mov 48+8(%rsp),%rdx # pull n0*a[0]
  3075. add 8*0($tptr),%r8
  3076. lea 8*8($nptr),$nptr
  3077. mov \$-8,%rcx
  3078. adcx 8*1($tptr),%r9
  3079. adcx 8*2($tptr),%r10
  3080. adc 8*3($tptr),%r11
  3081. adc 8*4($tptr),%r12
  3082. adc 8*5($tptr),%r13
  3083. adc 8*6($tptr),%r14
  3084. adc 8*7($tptr),%r15
  3085. lea 8*8($tptr),$tptr
  3086. sbb %rax,%rax # top carry
  3087. xor $carry,$carry # of=0, cf=0
  3088. mov %rax,16+8(%rsp)
  3089. jmp .Lsqrx8x_tail
  3090. .align 32
  3091. .Lsqrx8x_tail:
  3092. mov %r8,%rbx
  3093. mulx 8*0($nptr),%rax,%r8
  3094. adcx %rax,%rbx
  3095. adox %r9,%r8
  3096. mulx 8*1($nptr),%rax,%r9
  3097. adcx %rax,%r8
  3098. adox %r10,%r9
  3099. mulx 8*2($nptr),%rax,%r10
  3100. adcx %rax,%r9
  3101. adox %r11,%r10
  3102. mulx 8*3($nptr),%rax,%r11
  3103. adcx %rax,%r10
  3104. adox %r12,%r11
  3105. .byte 0xc4,0x62,0xfb,0xf6,0xa5,0x20,0x00,0x00,0x00 # mulx 8*4($nptr),%rax,%r12
  3106. adcx %rax,%r11
  3107. adox %r13,%r12
  3108. mulx 8*5($nptr),%rax,%r13
  3109. adcx %rax,%r12
  3110. adox %r14,%r13
  3111. mulx 8*6($nptr),%rax,%r14
  3112. adcx %rax,%r13
  3113. adox %r15,%r14
  3114. mulx 8*7($nptr),%rax,%r15
  3115. mov 72+48+8(%rsp,%rcx,8),%rdx # pull n0*a[i]
  3116. adcx %rax,%r14
  3117. adox $carry,%r15
  3118. mov %rbx,($tptr,%rcx,8) # save result
  3119. mov %r8,%rbx
  3120. adcx $carry,%r15 # cf=0
  3121. inc %rcx # of=0
  3122. jnz .Lsqrx8x_tail
  3123. cmp 0+8(%rsp),$nptr # end of n[]?
  3124. jae .Lsqrx8x_tail_done # break out of loop
  3125. sub 16+8(%rsp),$carry # mov 16(%rsp),%cf
  3126. mov 48+8(%rsp),%rdx # pull n0*a[0]
  3127. lea 8*8($nptr),$nptr
  3128. adc 8*0($tptr),%r8
  3129. adc 8*1($tptr),%r9
  3130. adc 8*2($tptr),%r10
  3131. adc 8*3($tptr),%r11
  3132. adc 8*4($tptr),%r12
  3133. adc 8*5($tptr),%r13
  3134. adc 8*6($tptr),%r14
  3135. adc 8*7($tptr),%r15
  3136. lea 8*8($tptr),$tptr
  3137. sbb %rax,%rax
  3138. sub \$8,%rcx # mov \$-8,%rcx
  3139. xor $carry,$carry # of=0, cf=0
  3140. mov %rax,16+8(%rsp)
  3141. jmp .Lsqrx8x_tail
  3142. .align 32
  3143. .Lsqrx8x_tail_done:
  3144. xor %rax,%rax
  3145. add 24+8(%rsp),%r8 # can this overflow?
  3146. adc \$0,%r9
  3147. adc \$0,%r10
  3148. adc \$0,%r11
  3149. adc \$0,%r12
  3150. adc \$0,%r13
  3151. adc \$0,%r14
  3152. adc \$0,%r15
  3153. adc \$0,%rax
  3154. sub 16+8(%rsp),$carry # mov 16(%rsp),%cf
  3155. .Lsqrx8x_no_tail: # %cf is 0 if jumped here
  3156. adc 8*0($tptr),%r8
  3157. movq %xmm3,%rcx
  3158. adc 8*1($tptr),%r9
  3159. mov 8*7($nptr),$carry
  3160. movq %xmm2,$nptr # restore $nptr
  3161. adc 8*2($tptr),%r10
  3162. adc 8*3($tptr),%r11
  3163. adc 8*4($tptr),%r12
  3164. adc 8*5($tptr),%r13
  3165. adc 8*6($tptr),%r14
  3166. adc 8*7($tptr),%r15
  3167. adc \$0,%rax # top-most carry
  3168. mov 32+8(%rsp),%rbx # n0
  3169. mov 8*8($tptr,%rcx),%rdx # modulo-scheduled "%r8"
  3170. mov %r8,8*0($tptr) # store top 512 bits
  3171. lea 8*8($tptr),%r8 # borrow %r8
  3172. mov %r9,8*1($tptr)
  3173. mov %r10,8*2($tptr)
  3174. mov %r11,8*3($tptr)
  3175. mov %r12,8*4($tptr)
  3176. mov %r13,8*5($tptr)
  3177. mov %r14,8*6($tptr)
  3178. mov %r15,8*7($tptr)
  3179. lea 8*8($tptr,%rcx),$tptr # start of current t[] window
  3180. cmp 8+8(%rsp),%r8 # end of t[]?
  3181. jb .Lsqrx8x_reduction_loop
  3182. ret
  3183. .size bn_sqrx8x_internal,.-bn_sqrx8x_internal
  3184. ___
  3185. }
  3186. ##############################################################
  3187. # Post-condition, 4x unrolled
  3188. #
  3189. {
  3190. my ($rptr,$nptr)=("%rdx","%rbp");
  3191. $code.=<<___;
  3192. .align 32
  3193. __bn_postx4x_internal:
  3194. mov 8*0($nptr),%r12
  3195. mov %rcx,%r10 # -$num
  3196. mov %rcx,%r9 # -$num
  3197. neg %rax
  3198. sar \$3+2,%rcx
  3199. #lea 48+8(%rsp,%r9),$tptr
  3200. movq %xmm1,$rptr # restore $rptr
  3201. movq %xmm1,$aptr # prepare for back-to-back call
  3202. dec %r12 # so that after 'not' we get -n[0]
  3203. mov 8*1($nptr),%r13
  3204. xor %r8,%r8
  3205. mov 8*2($nptr),%r14
  3206. mov 8*3($nptr),%r15
  3207. jmp .Lsqrx4x_sub_entry
  3208. .align 16
  3209. .Lsqrx4x_sub:
  3210. mov 8*0($nptr),%r12
  3211. mov 8*1($nptr),%r13
  3212. mov 8*2($nptr),%r14
  3213. mov 8*3($nptr),%r15
  3214. .Lsqrx4x_sub_entry:
  3215. andn %rax,%r12,%r12
  3216. lea 8*4($nptr),$nptr
  3217. andn %rax,%r13,%r13
  3218. andn %rax,%r14,%r14
  3219. andn %rax,%r15,%r15
  3220. neg %r8 # mov %r8,%cf
  3221. adc 8*0($tptr),%r12
  3222. adc 8*1($tptr),%r13
  3223. adc 8*2($tptr),%r14
  3224. adc 8*3($tptr),%r15
  3225. mov %r12,8*0($rptr)
  3226. lea 8*4($tptr),$tptr
  3227. mov %r13,8*1($rptr)
  3228. sbb %r8,%r8 # mov %cf,%r8
  3229. mov %r14,8*2($rptr)
  3230. mov %r15,8*3($rptr)
  3231. lea 8*4($rptr),$rptr
  3232. inc %rcx
  3233. jnz .Lsqrx4x_sub
  3234. neg %r9 # restore $num
  3235. ret
  3236. .size __bn_postx4x_internal,.-__bn_postx4x_internal
  3237. ___
  3238. }
  3239. }}}
  3240. {
  3241. my ($inp,$num,$tbl,$idx)=$win64?("%rcx","%edx","%r8", "%r9d") : # Win64 order
  3242. ("%rdi","%esi","%rdx","%ecx"); # Unix order
  3243. my $out=$inp;
  3244. my $STRIDE=2**5*8;
  3245. my $N=$STRIDE/4;
  3246. $code.=<<___;
  3247. .globl bn_get_bits5
  3248. .type bn_get_bits5,\@abi-omnipotent
  3249. .align 16
  3250. bn_get_bits5:
  3251. lea 0($inp),%r10
  3252. lea 1($inp),%r11
  3253. mov $num,%ecx
  3254. shr \$4,$num
  3255. and \$15,%ecx
  3256. lea -8(%ecx),%eax
  3257. cmp \$11,%ecx
  3258. cmova %r11,%r10
  3259. cmova %eax,%ecx
  3260. movzw (%r10,$num,2),%eax
  3261. shrl %cl,%eax
  3262. and \$31,%eax
  3263. ret
  3264. .size bn_get_bits5,.-bn_get_bits5
  3265. .globl bn_scatter5
  3266. .type bn_scatter5,\@abi-omnipotent
  3267. .align 16
  3268. bn_scatter5:
  3269. cmp \$0, $num
  3270. jz .Lscatter_epilogue
  3271. lea ($tbl,$idx,8),$tbl
  3272. .Lscatter:
  3273. mov ($inp),%rax
  3274. lea 8($inp),$inp
  3275. mov %rax,($tbl)
  3276. lea 32*8($tbl),$tbl
  3277. sub \$1,$num
  3278. jnz .Lscatter
  3279. .Lscatter_epilogue:
  3280. ret
  3281. .size bn_scatter5,.-bn_scatter5
  3282. .globl bn_gather5
  3283. .type bn_gather5,\@abi-omnipotent
  3284. .align 32
  3285. bn_gather5:
  3286. .LSEH_begin_bn_gather5: # Win64 thing, but harmless in other cases
  3287. # I can't trust assembler to use specific encoding:-(
  3288. .byte 0x4c,0x8d,0x14,0x24 #lea (%rsp),%r10
  3289. .byte 0x48,0x81,0xec,0x08,0x01,0x00,0x00 #sub $0x108,%rsp
  3290. lea .Linc(%rip),%rax
  3291. and \$-16,%rsp # shouldn't be formally required
  3292. movd $idx,%xmm5
  3293. movdqa 0(%rax),%xmm0 # 00000001000000010000000000000000
  3294. movdqa 16(%rax),%xmm1 # 00000002000000020000000200000002
  3295. lea 128($tbl),%r11 # size optimization
  3296. lea 128(%rsp),%rax # size optimization
  3297. pshufd \$0,%xmm5,%xmm5 # broadcast $idx
  3298. movdqa %xmm1,%xmm4
  3299. movdqa %xmm1,%xmm2
  3300. ___
  3301. ########################################################################
  3302. # calculate mask by comparing 0..31 to $idx and save result to stack
  3303. #
  3304. for($i=0;$i<$STRIDE/16;$i+=4) {
  3305. $code.=<<___;
  3306. paddd %xmm0,%xmm1
  3307. pcmpeqd %xmm5,%xmm0 # compare to 1,0
  3308. ___
  3309. $code.=<<___ if ($i);
  3310. movdqa %xmm3,`16*($i-1)-128`(%rax)
  3311. ___
  3312. $code.=<<___;
  3313. movdqa %xmm4,%xmm3
  3314. paddd %xmm1,%xmm2
  3315. pcmpeqd %xmm5,%xmm1 # compare to 3,2
  3316. movdqa %xmm0,`16*($i+0)-128`(%rax)
  3317. movdqa %xmm4,%xmm0
  3318. paddd %xmm2,%xmm3
  3319. pcmpeqd %xmm5,%xmm2 # compare to 5,4
  3320. movdqa %xmm1,`16*($i+1)-128`(%rax)
  3321. movdqa %xmm4,%xmm1
  3322. paddd %xmm3,%xmm0
  3323. pcmpeqd %xmm5,%xmm3 # compare to 7,6
  3324. movdqa %xmm2,`16*($i+2)-128`(%rax)
  3325. movdqa %xmm4,%xmm2
  3326. ___
  3327. }
  3328. $code.=<<___;
  3329. movdqa %xmm3,`16*($i-1)-128`(%rax)
  3330. jmp .Lgather
  3331. .align 32
  3332. .Lgather:
  3333. pxor %xmm4,%xmm4
  3334. pxor %xmm5,%xmm5
  3335. ___
  3336. for($i=0;$i<$STRIDE/16;$i+=4) {
  3337. $code.=<<___;
  3338. movdqa `16*($i+0)-128`(%r11),%xmm0
  3339. movdqa `16*($i+1)-128`(%r11),%xmm1
  3340. movdqa `16*($i+2)-128`(%r11),%xmm2
  3341. pand `16*($i+0)-128`(%rax),%xmm0
  3342. movdqa `16*($i+3)-128`(%r11),%xmm3
  3343. pand `16*($i+1)-128`(%rax),%xmm1
  3344. por %xmm0,%xmm4
  3345. pand `16*($i+2)-128`(%rax),%xmm2
  3346. por %xmm1,%xmm5
  3347. pand `16*($i+3)-128`(%rax),%xmm3
  3348. por %xmm2,%xmm4
  3349. por %xmm3,%xmm5
  3350. ___
  3351. }
  3352. $code.=<<___;
  3353. por %xmm5,%xmm4
  3354. lea $STRIDE(%r11),%r11
  3355. pshufd \$0x4e,%xmm4,%xmm0
  3356. por %xmm4,%xmm0
  3357. movq %xmm0,($out) # m0=bp[0]
  3358. lea 8($out),$out
  3359. sub \$1,$num
  3360. jnz .Lgather
  3361. lea (%r10),%rsp
  3362. ret
  3363. .LSEH_end_bn_gather5:
  3364. .size bn_gather5,.-bn_gather5
  3365. ___
  3366. }
  3367. $code.=<<___;
  3368. .align 64
  3369. .Linc:
  3370. .long 0,0, 1,1
  3371. .long 2,2, 2,2
  3372. .asciz "Montgomery Multiplication with scatter/gather for x86_64, CRYPTOGAMS by <appro\@openssl.org>"
  3373. ___
  3374. # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame,
  3375. # CONTEXT *context,DISPATCHER_CONTEXT *disp)
  3376. if ($win64) {
  3377. $rec="%rcx";
  3378. $frame="%rdx";
  3379. $context="%r8";
  3380. $disp="%r9";
  3381. $code.=<<___;
  3382. .extern __imp_RtlVirtualUnwind
  3383. .type mul_handler,\@abi-omnipotent
  3384. .align 16
  3385. mul_handler:
  3386. push %rsi
  3387. push %rdi
  3388. push %rbx
  3389. push %rbp
  3390. push %r12
  3391. push %r13
  3392. push %r14
  3393. push %r15
  3394. pushfq
  3395. sub \$64,%rsp
  3396. mov 120($context),%rax # pull context->Rax
  3397. mov 248($context),%rbx # pull context->Rip
  3398. mov 8($disp),%rsi # disp->ImageBase
  3399. mov 56($disp),%r11 # disp->HandlerData
  3400. mov 0(%r11),%r10d # HandlerData[0]
  3401. lea (%rsi,%r10),%r10 # end of prologue label
  3402. cmp %r10,%rbx # context->Rip<end of prologue label
  3403. jb .Lcommon_seh_tail
  3404. mov 4(%r11),%r10d # HandlerData[1]
  3405. lea (%rsi,%r10),%r10 # beginning of body label
  3406. cmp %r10,%rbx # context->Rip<body label
  3407. jb .Lcommon_pop_regs
  3408. mov 152($context),%rax # pull context->Rsp
  3409. mov 8(%r11),%r10d # HandlerData[2]
  3410. lea (%rsi,%r10),%r10 # epilogue label
  3411. cmp %r10,%rbx # context->Rip>=epilogue label
  3412. jae .Lcommon_seh_tail
  3413. lea .Lmul_epilogue(%rip),%r10
  3414. cmp %r10,%rbx
  3415. ja .Lbody_40
  3416. mov 192($context),%r10 # pull $num
  3417. mov 8(%rax,%r10,8),%rax # pull saved stack pointer
  3418. jmp .Lcommon_pop_regs
  3419. .Lbody_40:
  3420. mov 40(%rax),%rax # pull saved stack pointer
  3421. .Lcommon_pop_regs:
  3422. mov -8(%rax),%rbx
  3423. mov -16(%rax),%rbp
  3424. mov -24(%rax),%r12
  3425. mov -32(%rax),%r13
  3426. mov -40(%rax),%r14
  3427. mov -48(%rax),%r15
  3428. mov %rbx,144($context) # restore context->Rbx
  3429. mov %rbp,160($context) # restore context->Rbp
  3430. mov %r12,216($context) # restore context->R12
  3431. mov %r13,224($context) # restore context->R13
  3432. mov %r14,232($context) # restore context->R14
  3433. mov %r15,240($context) # restore context->R15
  3434. .Lcommon_seh_tail:
  3435. mov 8(%rax),%rdi
  3436. mov 16(%rax),%rsi
  3437. mov %rax,152($context) # restore context->Rsp
  3438. mov %rsi,168($context) # restore context->Rsi
  3439. mov %rdi,176($context) # restore context->Rdi
  3440. mov 40($disp),%rdi # disp->ContextRecord
  3441. mov $context,%rsi # context
  3442. mov \$154,%ecx # sizeof(CONTEXT)
  3443. .long 0xa548f3fc # cld; rep movsq
  3444. mov $disp,%rsi
  3445. xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER
  3446. mov 8(%rsi),%rdx # arg2, disp->ImageBase
  3447. mov 0(%rsi),%r8 # arg3, disp->ControlPc
  3448. mov 16(%rsi),%r9 # arg4, disp->FunctionEntry
  3449. mov 40(%rsi),%r10 # disp->ContextRecord
  3450. lea 56(%rsi),%r11 # &disp->HandlerData
  3451. lea 24(%rsi),%r12 # &disp->EstablisherFrame
  3452. mov %r10,32(%rsp) # arg5
  3453. mov %r11,40(%rsp) # arg6
  3454. mov %r12,48(%rsp) # arg7
  3455. mov %rcx,56(%rsp) # arg8, (NULL)
  3456. call *__imp_RtlVirtualUnwind(%rip)
  3457. mov \$1,%eax # ExceptionContinueSearch
  3458. add \$64,%rsp
  3459. popfq
  3460. pop %r15
  3461. pop %r14
  3462. pop %r13
  3463. pop %r12
  3464. pop %rbp
  3465. pop %rbx
  3466. pop %rdi
  3467. pop %rsi
  3468. ret
  3469. .size mul_handler,.-mul_handler
  3470. .section .pdata
  3471. .align 4
  3472. .rva .LSEH_begin_bn_mul_mont_gather5
  3473. .rva .LSEH_end_bn_mul_mont_gather5
  3474. .rva .LSEH_info_bn_mul_mont_gather5
  3475. .rva .LSEH_begin_bn_mul4x_mont_gather5
  3476. .rva .LSEH_end_bn_mul4x_mont_gather5
  3477. .rva .LSEH_info_bn_mul4x_mont_gather5
  3478. .rva .LSEH_begin_bn_power5
  3479. .rva .LSEH_end_bn_power5
  3480. .rva .LSEH_info_bn_power5
  3481. .rva .LSEH_begin_bn_from_mont8x
  3482. .rva .LSEH_end_bn_from_mont8x
  3483. .rva .LSEH_info_bn_from_mont8x
  3484. ___
  3485. $code.=<<___ if ($addx);
  3486. .rva .LSEH_begin_bn_mulx4x_mont_gather5
  3487. .rva .LSEH_end_bn_mulx4x_mont_gather5
  3488. .rva .LSEH_info_bn_mulx4x_mont_gather5
  3489. .rva .LSEH_begin_bn_powerx5
  3490. .rva .LSEH_end_bn_powerx5
  3491. .rva .LSEH_info_bn_powerx5
  3492. ___
  3493. $code.=<<___;
  3494. .rva .LSEH_begin_bn_gather5
  3495. .rva .LSEH_end_bn_gather5
  3496. .rva .LSEH_info_bn_gather5
  3497. .section .xdata
  3498. .align 8
  3499. .LSEH_info_bn_mul_mont_gather5:
  3500. .byte 9,0,0,0
  3501. .rva mul_handler
  3502. .rva .Lmul_body,.Lmul_body,.Lmul_epilogue # HandlerData[]
  3503. .align 8
  3504. .LSEH_info_bn_mul4x_mont_gather5:
  3505. .byte 9,0,0,0
  3506. .rva mul_handler
  3507. .rva .Lmul4x_prologue,.Lmul4x_body,.Lmul4x_epilogue # HandlerData[]
  3508. .align 8
  3509. .LSEH_info_bn_power5:
  3510. .byte 9,0,0,0
  3511. .rva mul_handler
  3512. .rva .Lpower5_prologue,.Lpower5_body,.Lpower5_epilogue # HandlerData[]
  3513. .align 8
  3514. .LSEH_info_bn_from_mont8x:
  3515. .byte 9,0,0,0
  3516. .rva mul_handler
  3517. .rva .Lfrom_prologue,.Lfrom_body,.Lfrom_epilogue # HandlerData[]
  3518. ___
  3519. $code.=<<___ if ($addx);
  3520. .align 8
  3521. .LSEH_info_bn_mulx4x_mont_gather5:
  3522. .byte 9,0,0,0
  3523. .rva mul_handler
  3524. .rva .Lmulx4x_prologue,.Lmulx4x_body,.Lmulx4x_epilogue # HandlerData[]
  3525. .align 8
  3526. .LSEH_info_bn_powerx5:
  3527. .byte 9,0,0,0
  3528. .rva mul_handler
  3529. .rva .Lpowerx5_prologue,.Lpowerx5_body,.Lpowerx5_epilogue # HandlerData[]
  3530. ___
  3531. $code.=<<___;
  3532. .align 8
  3533. .LSEH_info_bn_gather5:
  3534. .byte 0x01,0x0b,0x03,0x0a
  3535. .byte 0x0b,0x01,0x21,0x00 # sub rsp,0x108
  3536. .byte 0x04,0xa3,0x00,0x00 # lea r10,(rsp)
  3537. .align 8
  3538. ___
  3539. }
  3540. $code =~ s/\`([^\`]*)\`/eval($1)/gem;
  3541. print $code;
  3542. close STDOUT;