aesni-x86.pl 100 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413
  1. #! /usr/bin/env perl
  2. # Copyright 2009-2020 The OpenSSL Project Authors. All Rights Reserved.
  3. #
  4. # Licensed under the Apache License 2.0 (the "License"). You may not use
  5. # this file except in compliance with the License. You can obtain a copy
  6. # in the file LICENSE in the source distribution or at
  7. # https://www.openssl.org/source/license.html
  8. # ====================================================================
  9. # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
  10. # project. The module is, however, dual licensed under OpenSSL and
  11. # CRYPTOGAMS licenses depending on where you obtain it. For further
  12. # details see http://www.openssl.org/~appro/cryptogams/.
  13. # ====================================================================
  14. #
  15. # This module implements support for Intel AES-NI extension. In
  16. # OpenSSL context it's used with Intel engine, but can also be used as
  17. # drop-in replacement for crypto/aes/asm/aes-586.pl [see below for
  18. # details].
  19. #
  20. # Performance.
  21. #
  22. # To start with see corresponding paragraph in aesni-x86_64.pl...
  23. # Instead of filling table similar to one found there I've chosen to
  24. # summarize *comparison* results for raw ECB, CTR and CBC benchmarks.
  25. # The simplified table below represents 32-bit performance relative
  26. # to 64-bit one in every given point. Ratios vary for different
  27. # encryption modes, therefore interval values.
  28. #
  29. # 16-byte 64-byte 256-byte 1-KB 8-KB
  30. # 53-67% 67-84% 91-94% 95-98% 97-99.5%
  31. #
  32. # Lower ratios for smaller block sizes are perfectly understandable,
  33. # because function call overhead is higher in 32-bit mode. Largest
  34. # 8-KB block performance is virtually same: 32-bit code is less than
  35. # 1% slower for ECB, CBC and CCM, and ~3% slower otherwise.
  36. # January 2011
  37. #
  38. # See aesni-x86_64.pl for details. Unlike x86_64 version this module
  39. # interleaves at most 6 aes[enc|dec] instructions, because there are
  40. # not enough registers for 8x interleave [which should be optimal for
  41. # Sandy Bridge]. Actually, performance results for 6x interleave
  42. # factor presented in aesni-x86_64.pl (except for CTR) are for this
  43. # module.
  44. # April 2011
  45. #
  46. # Add aesni_xts_[en|de]crypt. Westmere spends 1.50 cycles processing
  47. # one byte out of 8KB with 128-bit key, Sandy Bridge - 1.09.
  48. # November 2015
  49. #
  50. # Add aesni_ocb_[en|de]crypt.
  51. ######################################################################
  52. # Current large-block performance in cycles per byte processed with
  53. # 128-bit key (less is better).
  54. #
  55. # CBC en-/decrypt CTR XTS ECB OCB
  56. # Westmere 3.77/1.37 1.37 1.52 1.27
  57. # * Bridge 5.07/0.98 0.99 1.09 0.91 1.10
  58. # Haswell 4.44/0.80 0.97 1.03 0.72 0.76
  59. # Skylake 2.68/0.65 0.65 0.66 0.64 0.66
  60. # Silvermont 5.77/3.56 3.67 4.03 3.46 4.03
  61. # Goldmont 3.84/1.39 1.39 1.63 1.31 1.70
  62. # Bulldozer 5.80/0.98 1.05 1.24 0.93 1.23
  63. $PREFIX="aesni"; # if $PREFIX is set to "AES", the script
  64. # generates drop-in replacement for
  65. # crypto/aes/asm/aes-586.pl:-)
  66. $inline=1; # inline _aesni_[en|de]crypt
  67. $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
  68. push(@INC,"${dir}","${dir}../../perlasm");
  69. require "x86asm.pl";
  70. $output = pop and open STDOUT,">$output";
  71. &asm_init($ARGV[0]);
  72. &external_label("OPENSSL_ia32cap_P");
  73. &static_label("key_const");
  74. if ($PREFIX eq "aesni") { $movekey=\&movups; }
  75. else { $movekey=\&movups; }
  76. $len="eax";
  77. $rounds="ecx";
  78. $key="edx";
  79. $inp="esi";
  80. $out="edi";
  81. $rounds_="ebx"; # backup copy for $rounds
  82. $key_="ebp"; # backup copy for $key
  83. $rndkey0="xmm0";
  84. $rndkey1="xmm1";
  85. $inout0="xmm2";
  86. $inout1="xmm3";
  87. $inout2="xmm4";
  88. $inout3="xmm5"; $in1="xmm5";
  89. $inout4="xmm6"; $in0="xmm6";
  90. $inout5="xmm7"; $ivec="xmm7";
  91. # AESNI extension
  92. sub aeskeygenassist
  93. { my($dst,$src,$imm)=@_;
  94. if ("$dst:$src" =~ /xmm([0-7]):xmm([0-7])/)
  95. { &data_byte(0x66,0x0f,0x3a,0xdf,0xc0|($1<<3)|$2,$imm); }
  96. }
  97. sub aescommon
  98. { my($opcodelet,$dst,$src)=@_;
  99. if ("$dst:$src" =~ /xmm([0-7]):xmm([0-7])/)
  100. { &data_byte(0x66,0x0f,0x38,$opcodelet,0xc0|($1<<3)|$2);}
  101. }
  102. sub aesimc { aescommon(0xdb,@_); }
  103. sub aesenc { aescommon(0xdc,@_); }
  104. sub aesenclast { aescommon(0xdd,@_); }
  105. sub aesdec { aescommon(0xde,@_); }
  106. sub aesdeclast { aescommon(0xdf,@_); }
  107. # Inline version of internal aesni_[en|de]crypt1
  108. { my $sn;
  109. sub aesni_inline_generate1
  110. { my ($p,$inout,$ivec)=@_; $inout=$inout0 if (!defined($inout));
  111. $sn++;
  112. &$movekey ($rndkey0,&QWP(0,$key));
  113. &$movekey ($rndkey1,&QWP(16,$key));
  114. &xorps ($ivec,$rndkey0) if (defined($ivec));
  115. &lea ($key,&DWP(32,$key));
  116. &xorps ($inout,$ivec) if (defined($ivec));
  117. &xorps ($inout,$rndkey0) if (!defined($ivec));
  118. &set_label("${p}1_loop_$sn");
  119. eval"&aes${p} ($inout,$rndkey1)";
  120. &dec ($rounds);
  121. &$movekey ($rndkey1,&QWP(0,$key));
  122. &lea ($key,&DWP(16,$key));
  123. &jnz (&label("${p}1_loop_$sn"));
  124. eval"&aes${p}last ($inout,$rndkey1)";
  125. }}
  126. sub aesni_generate1 # fully unrolled loop
  127. { my ($p,$inout)=@_; $inout=$inout0 if (!defined($inout));
  128. &function_begin_B("_aesni_${p}rypt1");
  129. &movups ($rndkey0,&QWP(0,$key));
  130. &$movekey ($rndkey1,&QWP(0x10,$key));
  131. &xorps ($inout,$rndkey0);
  132. &$movekey ($rndkey0,&QWP(0x20,$key));
  133. &lea ($key,&DWP(0x30,$key));
  134. &cmp ($rounds,11);
  135. &jb (&label("${p}128"));
  136. &lea ($key,&DWP(0x20,$key));
  137. &je (&label("${p}192"));
  138. &lea ($key,&DWP(0x20,$key));
  139. eval"&aes${p} ($inout,$rndkey1)";
  140. &$movekey ($rndkey1,&QWP(-0x40,$key));
  141. eval"&aes${p} ($inout,$rndkey0)";
  142. &$movekey ($rndkey0,&QWP(-0x30,$key));
  143. &set_label("${p}192");
  144. eval"&aes${p} ($inout,$rndkey1)";
  145. &$movekey ($rndkey1,&QWP(-0x20,$key));
  146. eval"&aes${p} ($inout,$rndkey0)";
  147. &$movekey ($rndkey0,&QWP(-0x10,$key));
  148. &set_label("${p}128");
  149. eval"&aes${p} ($inout,$rndkey1)";
  150. &$movekey ($rndkey1,&QWP(0,$key));
  151. eval"&aes${p} ($inout,$rndkey0)";
  152. &$movekey ($rndkey0,&QWP(0x10,$key));
  153. eval"&aes${p} ($inout,$rndkey1)";
  154. &$movekey ($rndkey1,&QWP(0x20,$key));
  155. eval"&aes${p} ($inout,$rndkey0)";
  156. &$movekey ($rndkey0,&QWP(0x30,$key));
  157. eval"&aes${p} ($inout,$rndkey1)";
  158. &$movekey ($rndkey1,&QWP(0x40,$key));
  159. eval"&aes${p} ($inout,$rndkey0)";
  160. &$movekey ($rndkey0,&QWP(0x50,$key));
  161. eval"&aes${p} ($inout,$rndkey1)";
  162. &$movekey ($rndkey1,&QWP(0x60,$key));
  163. eval"&aes${p} ($inout,$rndkey0)";
  164. &$movekey ($rndkey0,&QWP(0x70,$key));
  165. eval"&aes${p} ($inout,$rndkey1)";
  166. eval"&aes${p}last ($inout,$rndkey0)";
  167. &ret();
  168. &function_end_B("_aesni_${p}rypt1");
  169. }
  170. # void $PREFIX_encrypt (const void *inp,void *out,const AES_KEY *key);
  171. &aesni_generate1("enc") if (!$inline);
  172. &function_begin_B("${PREFIX}_encrypt");
  173. &mov ("eax",&wparam(0));
  174. &mov ($key,&wparam(2));
  175. &movups ($inout0,&QWP(0,"eax"));
  176. &mov ($rounds,&DWP(240,$key));
  177. &mov ("eax",&wparam(1));
  178. if ($inline)
  179. { &aesni_inline_generate1("enc"); }
  180. else
  181. { &call ("_aesni_encrypt1"); }
  182. &pxor ($rndkey0,$rndkey0); # clear register bank
  183. &pxor ($rndkey1,$rndkey1);
  184. &movups (&QWP(0,"eax"),$inout0);
  185. &pxor ($inout0,$inout0);
  186. &ret ();
  187. &function_end_B("${PREFIX}_encrypt");
  188. # void $PREFIX_decrypt (const void *inp,void *out,const AES_KEY *key);
  189. &aesni_generate1("dec") if(!$inline);
  190. &function_begin_B("${PREFIX}_decrypt");
  191. &mov ("eax",&wparam(0));
  192. &mov ($key,&wparam(2));
  193. &movups ($inout0,&QWP(0,"eax"));
  194. &mov ($rounds,&DWP(240,$key));
  195. &mov ("eax",&wparam(1));
  196. if ($inline)
  197. { &aesni_inline_generate1("dec"); }
  198. else
  199. { &call ("_aesni_decrypt1"); }
  200. &pxor ($rndkey0,$rndkey0); # clear register bank
  201. &pxor ($rndkey1,$rndkey1);
  202. &movups (&QWP(0,"eax"),$inout0);
  203. &pxor ($inout0,$inout0);
  204. &ret ();
  205. &function_end_B("${PREFIX}_decrypt");
  206. # _aesni_[en|de]cryptN are private interfaces, N denotes interleave
  207. # factor. Why 3x subroutine were originally used in loops? Even though
  208. # aes[enc|dec] latency was originally 6, it could be scheduled only
  209. # every *2nd* cycle. Thus 3x interleave was the one providing optimal
  210. # utilization, i.e. when subroutine's throughput is virtually same as
  211. # of non-interleaved subroutine [for number of input blocks up to 3].
  212. # This is why it originally made no sense to implement 2x subroutine.
  213. # But times change and it became appropriate to spend extra 192 bytes
  214. # on 2x subroutine on Atom Silvermont account. For processors that
  215. # can schedule aes[enc|dec] every cycle optimal interleave factor
  216. # equals to corresponding instructions latency. 8x is optimal for
  217. # * Bridge, but it's unfeasible to accommodate such implementation
  218. # in XMM registers addressable in 32-bit mode and therefore maximum
  219. # of 6x is used instead...
  220. sub aesni_generate2
  221. { my $p=shift;
  222. &function_begin_B("_aesni_${p}rypt2");
  223. &$movekey ($rndkey0,&QWP(0,$key));
  224. &shl ($rounds,4);
  225. &$movekey ($rndkey1,&QWP(16,$key));
  226. &xorps ($inout0,$rndkey0);
  227. &pxor ($inout1,$rndkey0);
  228. &$movekey ($rndkey0,&QWP(32,$key));
  229. &lea ($key,&DWP(32,$key,$rounds));
  230. &neg ($rounds);
  231. &add ($rounds,16);
  232. &set_label("${p}2_loop");
  233. eval"&aes${p} ($inout0,$rndkey1)";
  234. eval"&aes${p} ($inout1,$rndkey1)";
  235. &$movekey ($rndkey1,&QWP(0,$key,$rounds));
  236. &add ($rounds,32);
  237. eval"&aes${p} ($inout0,$rndkey0)";
  238. eval"&aes${p} ($inout1,$rndkey0)";
  239. &$movekey ($rndkey0,&QWP(-16,$key,$rounds));
  240. &jnz (&label("${p}2_loop"));
  241. eval"&aes${p} ($inout0,$rndkey1)";
  242. eval"&aes${p} ($inout1,$rndkey1)";
  243. eval"&aes${p}last ($inout0,$rndkey0)";
  244. eval"&aes${p}last ($inout1,$rndkey0)";
  245. &ret();
  246. &function_end_B("_aesni_${p}rypt2");
  247. }
  248. sub aesni_generate3
  249. { my $p=shift;
  250. &function_begin_B("_aesni_${p}rypt3");
  251. &$movekey ($rndkey0,&QWP(0,$key));
  252. &shl ($rounds,4);
  253. &$movekey ($rndkey1,&QWP(16,$key));
  254. &xorps ($inout0,$rndkey0);
  255. &pxor ($inout1,$rndkey0);
  256. &pxor ($inout2,$rndkey0);
  257. &$movekey ($rndkey0,&QWP(32,$key));
  258. &lea ($key,&DWP(32,$key,$rounds));
  259. &neg ($rounds);
  260. &add ($rounds,16);
  261. &set_label("${p}3_loop");
  262. eval"&aes${p} ($inout0,$rndkey1)";
  263. eval"&aes${p} ($inout1,$rndkey1)";
  264. eval"&aes${p} ($inout2,$rndkey1)";
  265. &$movekey ($rndkey1,&QWP(0,$key,$rounds));
  266. &add ($rounds,32);
  267. eval"&aes${p} ($inout0,$rndkey0)";
  268. eval"&aes${p} ($inout1,$rndkey0)";
  269. eval"&aes${p} ($inout2,$rndkey0)";
  270. &$movekey ($rndkey0,&QWP(-16,$key,$rounds));
  271. &jnz (&label("${p}3_loop"));
  272. eval"&aes${p} ($inout0,$rndkey1)";
  273. eval"&aes${p} ($inout1,$rndkey1)";
  274. eval"&aes${p} ($inout2,$rndkey1)";
  275. eval"&aes${p}last ($inout0,$rndkey0)";
  276. eval"&aes${p}last ($inout1,$rndkey0)";
  277. eval"&aes${p}last ($inout2,$rndkey0)";
  278. &ret();
  279. &function_end_B("_aesni_${p}rypt3");
  280. }
  281. # 4x interleave is implemented to improve small block performance,
  282. # most notably [and naturally] 4 block by ~30%. One can argue that one
  283. # should have implemented 5x as well, but improvement would be <20%,
  284. # so it's not worth it...
  285. sub aesni_generate4
  286. { my $p=shift;
  287. &function_begin_B("_aesni_${p}rypt4");
  288. &$movekey ($rndkey0,&QWP(0,$key));
  289. &$movekey ($rndkey1,&QWP(16,$key));
  290. &shl ($rounds,4);
  291. &xorps ($inout0,$rndkey0);
  292. &pxor ($inout1,$rndkey0);
  293. &pxor ($inout2,$rndkey0);
  294. &pxor ($inout3,$rndkey0);
  295. &$movekey ($rndkey0,&QWP(32,$key));
  296. &lea ($key,&DWP(32,$key,$rounds));
  297. &neg ($rounds);
  298. &data_byte (0x0f,0x1f,0x40,0x00);
  299. &add ($rounds,16);
  300. &set_label("${p}4_loop");
  301. eval"&aes${p} ($inout0,$rndkey1)";
  302. eval"&aes${p} ($inout1,$rndkey1)";
  303. eval"&aes${p} ($inout2,$rndkey1)";
  304. eval"&aes${p} ($inout3,$rndkey1)";
  305. &$movekey ($rndkey1,&QWP(0,$key,$rounds));
  306. &add ($rounds,32);
  307. eval"&aes${p} ($inout0,$rndkey0)";
  308. eval"&aes${p} ($inout1,$rndkey0)";
  309. eval"&aes${p} ($inout2,$rndkey0)";
  310. eval"&aes${p} ($inout3,$rndkey0)";
  311. &$movekey ($rndkey0,&QWP(-16,$key,$rounds));
  312. &jnz (&label("${p}4_loop"));
  313. eval"&aes${p} ($inout0,$rndkey1)";
  314. eval"&aes${p} ($inout1,$rndkey1)";
  315. eval"&aes${p} ($inout2,$rndkey1)";
  316. eval"&aes${p} ($inout3,$rndkey1)";
  317. eval"&aes${p}last ($inout0,$rndkey0)";
  318. eval"&aes${p}last ($inout1,$rndkey0)";
  319. eval"&aes${p}last ($inout2,$rndkey0)";
  320. eval"&aes${p}last ($inout3,$rndkey0)";
  321. &ret();
  322. &function_end_B("_aesni_${p}rypt4");
  323. }
  324. sub aesni_generate6
  325. { my $p=shift;
  326. &function_begin_B("_aesni_${p}rypt6");
  327. &static_label("_aesni_${p}rypt6_enter");
  328. &$movekey ($rndkey0,&QWP(0,$key));
  329. &shl ($rounds,4);
  330. &$movekey ($rndkey1,&QWP(16,$key));
  331. &xorps ($inout0,$rndkey0);
  332. &pxor ($inout1,$rndkey0); # pxor does better here
  333. &pxor ($inout2,$rndkey0);
  334. eval"&aes${p} ($inout0,$rndkey1)";
  335. &pxor ($inout3,$rndkey0);
  336. &pxor ($inout4,$rndkey0);
  337. eval"&aes${p} ($inout1,$rndkey1)";
  338. &lea ($key,&DWP(32,$key,$rounds));
  339. &neg ($rounds);
  340. eval"&aes${p} ($inout2,$rndkey1)";
  341. &pxor ($inout5,$rndkey0);
  342. &$movekey ($rndkey0,&QWP(0,$key,$rounds));
  343. &add ($rounds,16);
  344. &jmp (&label("_aesni_${p}rypt6_inner"));
  345. &set_label("${p}6_loop",16);
  346. eval"&aes${p} ($inout0,$rndkey1)";
  347. eval"&aes${p} ($inout1,$rndkey1)";
  348. eval"&aes${p} ($inout2,$rndkey1)";
  349. &set_label("_aesni_${p}rypt6_inner");
  350. eval"&aes${p} ($inout3,$rndkey1)";
  351. eval"&aes${p} ($inout4,$rndkey1)";
  352. eval"&aes${p} ($inout5,$rndkey1)";
  353. &set_label("_aesni_${p}rypt6_enter");
  354. &$movekey ($rndkey1,&QWP(0,$key,$rounds));
  355. &add ($rounds,32);
  356. eval"&aes${p} ($inout0,$rndkey0)";
  357. eval"&aes${p} ($inout1,$rndkey0)";
  358. eval"&aes${p} ($inout2,$rndkey0)";
  359. eval"&aes${p} ($inout3,$rndkey0)";
  360. eval"&aes${p} ($inout4,$rndkey0)";
  361. eval"&aes${p} ($inout5,$rndkey0)";
  362. &$movekey ($rndkey0,&QWP(-16,$key,$rounds));
  363. &jnz (&label("${p}6_loop"));
  364. eval"&aes${p} ($inout0,$rndkey1)";
  365. eval"&aes${p} ($inout1,$rndkey1)";
  366. eval"&aes${p} ($inout2,$rndkey1)";
  367. eval"&aes${p} ($inout3,$rndkey1)";
  368. eval"&aes${p} ($inout4,$rndkey1)";
  369. eval"&aes${p} ($inout5,$rndkey1)";
  370. eval"&aes${p}last ($inout0,$rndkey0)";
  371. eval"&aes${p}last ($inout1,$rndkey0)";
  372. eval"&aes${p}last ($inout2,$rndkey0)";
  373. eval"&aes${p}last ($inout3,$rndkey0)";
  374. eval"&aes${p}last ($inout4,$rndkey0)";
  375. eval"&aes${p}last ($inout5,$rndkey0)";
  376. &ret();
  377. &function_end_B("_aesni_${p}rypt6");
  378. }
  379. &aesni_generate2("enc") if ($PREFIX eq "aesni");
  380. &aesni_generate2("dec");
  381. &aesni_generate3("enc") if ($PREFIX eq "aesni");
  382. &aesni_generate3("dec");
  383. &aesni_generate4("enc") if ($PREFIX eq "aesni");
  384. &aesni_generate4("dec");
  385. &aesni_generate6("enc") if ($PREFIX eq "aesni");
  386. &aesni_generate6("dec");
  387. if ($PREFIX eq "aesni") {
  388. ######################################################################
  389. # void aesni_ecb_encrypt (const void *in, void *out,
  390. # size_t length, const AES_KEY *key,
  391. # int enc);
  392. &function_begin("aesni_ecb_encrypt");
  393. &mov ($inp,&wparam(0));
  394. &mov ($out,&wparam(1));
  395. &mov ($len,&wparam(2));
  396. &mov ($key,&wparam(3));
  397. &mov ($rounds_,&wparam(4));
  398. &and ($len,-16);
  399. &jz (&label("ecb_ret"));
  400. &mov ($rounds,&DWP(240,$key));
  401. &test ($rounds_,$rounds_);
  402. &jz (&label("ecb_decrypt"));
  403. &mov ($key_,$key); # backup $key
  404. &mov ($rounds_,$rounds); # backup $rounds
  405. &cmp ($len,0x60);
  406. &jb (&label("ecb_enc_tail"));
  407. &movdqu ($inout0,&QWP(0,$inp));
  408. &movdqu ($inout1,&QWP(0x10,$inp));
  409. &movdqu ($inout2,&QWP(0x20,$inp));
  410. &movdqu ($inout3,&QWP(0x30,$inp));
  411. &movdqu ($inout4,&QWP(0x40,$inp));
  412. &movdqu ($inout5,&QWP(0x50,$inp));
  413. &lea ($inp,&DWP(0x60,$inp));
  414. &sub ($len,0x60);
  415. &jmp (&label("ecb_enc_loop6_enter"));
  416. &set_label("ecb_enc_loop6",16);
  417. &movups (&QWP(0,$out),$inout0);
  418. &movdqu ($inout0,&QWP(0,$inp));
  419. &movups (&QWP(0x10,$out),$inout1);
  420. &movdqu ($inout1,&QWP(0x10,$inp));
  421. &movups (&QWP(0x20,$out),$inout2);
  422. &movdqu ($inout2,&QWP(0x20,$inp));
  423. &movups (&QWP(0x30,$out),$inout3);
  424. &movdqu ($inout3,&QWP(0x30,$inp));
  425. &movups (&QWP(0x40,$out),$inout4);
  426. &movdqu ($inout4,&QWP(0x40,$inp));
  427. &movups (&QWP(0x50,$out),$inout5);
  428. &lea ($out,&DWP(0x60,$out));
  429. &movdqu ($inout5,&QWP(0x50,$inp));
  430. &lea ($inp,&DWP(0x60,$inp));
  431. &set_label("ecb_enc_loop6_enter");
  432. &call ("_aesni_encrypt6");
  433. &mov ($key,$key_); # restore $key
  434. &mov ($rounds,$rounds_); # restore $rounds
  435. &sub ($len,0x60);
  436. &jnc (&label("ecb_enc_loop6"));
  437. &movups (&QWP(0,$out),$inout0);
  438. &movups (&QWP(0x10,$out),$inout1);
  439. &movups (&QWP(0x20,$out),$inout2);
  440. &movups (&QWP(0x30,$out),$inout3);
  441. &movups (&QWP(0x40,$out),$inout4);
  442. &movups (&QWP(0x50,$out),$inout5);
  443. &lea ($out,&DWP(0x60,$out));
  444. &add ($len,0x60);
  445. &jz (&label("ecb_ret"));
  446. &set_label("ecb_enc_tail");
  447. &movups ($inout0,&QWP(0,$inp));
  448. &cmp ($len,0x20);
  449. &jb (&label("ecb_enc_one"));
  450. &movups ($inout1,&QWP(0x10,$inp));
  451. &je (&label("ecb_enc_two"));
  452. &movups ($inout2,&QWP(0x20,$inp));
  453. &cmp ($len,0x40);
  454. &jb (&label("ecb_enc_three"));
  455. &movups ($inout3,&QWP(0x30,$inp));
  456. &je (&label("ecb_enc_four"));
  457. &movups ($inout4,&QWP(0x40,$inp));
  458. &xorps ($inout5,$inout5);
  459. &call ("_aesni_encrypt6");
  460. &movups (&QWP(0,$out),$inout0);
  461. &movups (&QWP(0x10,$out),$inout1);
  462. &movups (&QWP(0x20,$out),$inout2);
  463. &movups (&QWP(0x30,$out),$inout3);
  464. &movups (&QWP(0x40,$out),$inout4);
  465. jmp (&label("ecb_ret"));
  466. &set_label("ecb_enc_one",16);
  467. if ($inline)
  468. { &aesni_inline_generate1("enc"); }
  469. else
  470. { &call ("_aesni_encrypt1"); }
  471. &movups (&QWP(0,$out),$inout0);
  472. &jmp (&label("ecb_ret"));
  473. &set_label("ecb_enc_two",16);
  474. &call ("_aesni_encrypt2");
  475. &movups (&QWP(0,$out),$inout0);
  476. &movups (&QWP(0x10,$out),$inout1);
  477. &jmp (&label("ecb_ret"));
  478. &set_label("ecb_enc_three",16);
  479. &call ("_aesni_encrypt3");
  480. &movups (&QWP(0,$out),$inout0);
  481. &movups (&QWP(0x10,$out),$inout1);
  482. &movups (&QWP(0x20,$out),$inout2);
  483. &jmp (&label("ecb_ret"));
  484. &set_label("ecb_enc_four",16);
  485. &call ("_aesni_encrypt4");
  486. &movups (&QWP(0,$out),$inout0);
  487. &movups (&QWP(0x10,$out),$inout1);
  488. &movups (&QWP(0x20,$out),$inout2);
  489. &movups (&QWP(0x30,$out),$inout3);
  490. &jmp (&label("ecb_ret"));
  491. ######################################################################
  492. &set_label("ecb_decrypt",16);
  493. &mov ($key_,$key); # backup $key
  494. &mov ($rounds_,$rounds); # backup $rounds
  495. &cmp ($len,0x60);
  496. &jb (&label("ecb_dec_tail"));
  497. &movdqu ($inout0,&QWP(0,$inp));
  498. &movdqu ($inout1,&QWP(0x10,$inp));
  499. &movdqu ($inout2,&QWP(0x20,$inp));
  500. &movdqu ($inout3,&QWP(0x30,$inp));
  501. &movdqu ($inout4,&QWP(0x40,$inp));
  502. &movdqu ($inout5,&QWP(0x50,$inp));
  503. &lea ($inp,&DWP(0x60,$inp));
  504. &sub ($len,0x60);
  505. &jmp (&label("ecb_dec_loop6_enter"));
  506. &set_label("ecb_dec_loop6",16);
  507. &movups (&QWP(0,$out),$inout0);
  508. &movdqu ($inout0,&QWP(0,$inp));
  509. &movups (&QWP(0x10,$out),$inout1);
  510. &movdqu ($inout1,&QWP(0x10,$inp));
  511. &movups (&QWP(0x20,$out),$inout2);
  512. &movdqu ($inout2,&QWP(0x20,$inp));
  513. &movups (&QWP(0x30,$out),$inout3);
  514. &movdqu ($inout3,&QWP(0x30,$inp));
  515. &movups (&QWP(0x40,$out),$inout4);
  516. &movdqu ($inout4,&QWP(0x40,$inp));
  517. &movups (&QWP(0x50,$out),$inout5);
  518. &lea ($out,&DWP(0x60,$out));
  519. &movdqu ($inout5,&QWP(0x50,$inp));
  520. &lea ($inp,&DWP(0x60,$inp));
  521. &set_label("ecb_dec_loop6_enter");
  522. &call ("_aesni_decrypt6");
  523. &mov ($key,$key_); # restore $key
  524. &mov ($rounds,$rounds_); # restore $rounds
  525. &sub ($len,0x60);
  526. &jnc (&label("ecb_dec_loop6"));
  527. &movups (&QWP(0,$out),$inout0);
  528. &movups (&QWP(0x10,$out),$inout1);
  529. &movups (&QWP(0x20,$out),$inout2);
  530. &movups (&QWP(0x30,$out),$inout3);
  531. &movups (&QWP(0x40,$out),$inout4);
  532. &movups (&QWP(0x50,$out),$inout5);
  533. &lea ($out,&DWP(0x60,$out));
  534. &add ($len,0x60);
  535. &jz (&label("ecb_ret"));
  536. &set_label("ecb_dec_tail");
  537. &movups ($inout0,&QWP(0,$inp));
  538. &cmp ($len,0x20);
  539. &jb (&label("ecb_dec_one"));
  540. &movups ($inout1,&QWP(0x10,$inp));
  541. &je (&label("ecb_dec_two"));
  542. &movups ($inout2,&QWP(0x20,$inp));
  543. &cmp ($len,0x40);
  544. &jb (&label("ecb_dec_three"));
  545. &movups ($inout3,&QWP(0x30,$inp));
  546. &je (&label("ecb_dec_four"));
  547. &movups ($inout4,&QWP(0x40,$inp));
  548. &xorps ($inout5,$inout5);
  549. &call ("_aesni_decrypt6");
  550. &movups (&QWP(0,$out),$inout0);
  551. &movups (&QWP(0x10,$out),$inout1);
  552. &movups (&QWP(0x20,$out),$inout2);
  553. &movups (&QWP(0x30,$out),$inout3);
  554. &movups (&QWP(0x40,$out),$inout4);
  555. &jmp (&label("ecb_ret"));
  556. &set_label("ecb_dec_one",16);
  557. if ($inline)
  558. { &aesni_inline_generate1("dec"); }
  559. else
  560. { &call ("_aesni_decrypt1"); }
  561. &movups (&QWP(0,$out),$inout0);
  562. &jmp (&label("ecb_ret"));
  563. &set_label("ecb_dec_two",16);
  564. &call ("_aesni_decrypt2");
  565. &movups (&QWP(0,$out),$inout0);
  566. &movups (&QWP(0x10,$out),$inout1);
  567. &jmp (&label("ecb_ret"));
  568. &set_label("ecb_dec_three",16);
  569. &call ("_aesni_decrypt3");
  570. &movups (&QWP(0,$out),$inout0);
  571. &movups (&QWP(0x10,$out),$inout1);
  572. &movups (&QWP(0x20,$out),$inout2);
  573. &jmp (&label("ecb_ret"));
  574. &set_label("ecb_dec_four",16);
  575. &call ("_aesni_decrypt4");
  576. &movups (&QWP(0,$out),$inout0);
  577. &movups (&QWP(0x10,$out),$inout1);
  578. &movups (&QWP(0x20,$out),$inout2);
  579. &movups (&QWP(0x30,$out),$inout3);
  580. &set_label("ecb_ret");
  581. &pxor ("xmm0","xmm0"); # clear register bank
  582. &pxor ("xmm1","xmm1");
  583. &pxor ("xmm2","xmm2");
  584. &pxor ("xmm3","xmm3");
  585. &pxor ("xmm4","xmm4");
  586. &pxor ("xmm5","xmm5");
  587. &pxor ("xmm6","xmm6");
  588. &pxor ("xmm7","xmm7");
  589. &function_end("aesni_ecb_encrypt");
  590. ######################################################################
  591. # void aesni_ccm64_[en|de]crypt_blocks (const void *in, void *out,
  592. # size_t blocks, const AES_KEY *key,
  593. # const char *ivec,char *cmac);
  594. #
  595. # Handles only complete blocks, operates on 64-bit counter and
  596. # does not update *ivec! Nor does it finalize CMAC value
  597. # (see engine/eng_aesni.c for details)
  598. #
  599. { my $cmac=$inout1;
  600. &function_begin("aesni_ccm64_encrypt_blocks");
  601. &mov ($inp,&wparam(0));
  602. &mov ($out,&wparam(1));
  603. &mov ($len,&wparam(2));
  604. &mov ($key,&wparam(3));
  605. &mov ($rounds_,&wparam(4));
  606. &mov ($rounds,&wparam(5));
  607. &mov ($key_,"esp");
  608. &sub ("esp",60);
  609. &and ("esp",-16); # align stack
  610. &mov (&DWP(48,"esp"),$key_);
  611. &movdqu ($ivec,&QWP(0,$rounds_)); # load ivec
  612. &movdqu ($cmac,&QWP(0,$rounds)); # load cmac
  613. &mov ($rounds,&DWP(240,$key));
  614. # compose byte-swap control mask for pshufb on stack
  615. &mov (&DWP(0,"esp"),0x0c0d0e0f);
  616. &mov (&DWP(4,"esp"),0x08090a0b);
  617. &mov (&DWP(8,"esp"),0x04050607);
  618. &mov (&DWP(12,"esp"),0x00010203);
  619. # compose counter increment vector on stack
  620. &mov ($rounds_,1);
  621. &xor ($key_,$key_);
  622. &mov (&DWP(16,"esp"),$rounds_);
  623. &mov (&DWP(20,"esp"),$key_);
  624. &mov (&DWP(24,"esp"),$key_);
  625. &mov (&DWP(28,"esp"),$key_);
  626. &shl ($rounds,4);
  627. &mov ($rounds_,16);
  628. &lea ($key_,&DWP(0,$key));
  629. &movdqa ($inout3,&QWP(0,"esp"));
  630. &movdqa ($inout0,$ivec);
  631. &lea ($key,&DWP(32,$key,$rounds));
  632. &sub ($rounds_,$rounds);
  633. &pshufb ($ivec,$inout3);
  634. &set_label("ccm64_enc_outer");
  635. &$movekey ($rndkey0,&QWP(0,$key_));
  636. &mov ($rounds,$rounds_);
  637. &movups ($in0,&QWP(0,$inp));
  638. &xorps ($inout0,$rndkey0);
  639. &$movekey ($rndkey1,&QWP(16,$key_));
  640. &xorps ($rndkey0,$in0);
  641. &xorps ($cmac,$rndkey0); # cmac^=inp
  642. &$movekey ($rndkey0,&QWP(32,$key_));
  643. &set_label("ccm64_enc2_loop");
  644. &aesenc ($inout0,$rndkey1);
  645. &aesenc ($cmac,$rndkey1);
  646. &$movekey ($rndkey1,&QWP(0,$key,$rounds));
  647. &add ($rounds,32);
  648. &aesenc ($inout0,$rndkey0);
  649. &aesenc ($cmac,$rndkey0);
  650. &$movekey ($rndkey0,&QWP(-16,$key,$rounds));
  651. &jnz (&label("ccm64_enc2_loop"));
  652. &aesenc ($inout0,$rndkey1);
  653. &aesenc ($cmac,$rndkey1);
  654. &paddq ($ivec,&QWP(16,"esp"));
  655. &dec ($len);
  656. &aesenclast ($inout0,$rndkey0);
  657. &aesenclast ($cmac,$rndkey0);
  658. &lea ($inp,&DWP(16,$inp));
  659. &xorps ($in0,$inout0); # inp^=E(ivec)
  660. &movdqa ($inout0,$ivec);
  661. &movups (&QWP(0,$out),$in0); # save output
  662. &pshufb ($inout0,$inout3);
  663. &lea ($out,&DWP(16,$out));
  664. &jnz (&label("ccm64_enc_outer"));
  665. &mov ("esp",&DWP(48,"esp"));
  666. &mov ($out,&wparam(5));
  667. &movups (&QWP(0,$out),$cmac);
  668. &pxor ("xmm0","xmm0"); # clear register bank
  669. &pxor ("xmm1","xmm1");
  670. &pxor ("xmm2","xmm2");
  671. &pxor ("xmm3","xmm3");
  672. &pxor ("xmm4","xmm4");
  673. &pxor ("xmm5","xmm5");
  674. &pxor ("xmm6","xmm6");
  675. &pxor ("xmm7","xmm7");
  676. &function_end("aesni_ccm64_encrypt_blocks");
  677. &function_begin("aesni_ccm64_decrypt_blocks");
  678. &mov ($inp,&wparam(0));
  679. &mov ($out,&wparam(1));
  680. &mov ($len,&wparam(2));
  681. &mov ($key,&wparam(3));
  682. &mov ($rounds_,&wparam(4));
  683. &mov ($rounds,&wparam(5));
  684. &mov ($key_,"esp");
  685. &sub ("esp",60);
  686. &and ("esp",-16); # align stack
  687. &mov (&DWP(48,"esp"),$key_);
  688. &movdqu ($ivec,&QWP(0,$rounds_)); # load ivec
  689. &movdqu ($cmac,&QWP(0,$rounds)); # load cmac
  690. &mov ($rounds,&DWP(240,$key));
  691. # compose byte-swap control mask for pshufb on stack
  692. &mov (&DWP(0,"esp"),0x0c0d0e0f);
  693. &mov (&DWP(4,"esp"),0x08090a0b);
  694. &mov (&DWP(8,"esp"),0x04050607);
  695. &mov (&DWP(12,"esp"),0x00010203);
  696. # compose counter increment vector on stack
  697. &mov ($rounds_,1);
  698. &xor ($key_,$key_);
  699. &mov (&DWP(16,"esp"),$rounds_);
  700. &mov (&DWP(20,"esp"),$key_);
  701. &mov (&DWP(24,"esp"),$key_);
  702. &mov (&DWP(28,"esp"),$key_);
  703. &movdqa ($inout3,&QWP(0,"esp")); # bswap mask
  704. &movdqa ($inout0,$ivec);
  705. &mov ($key_,$key);
  706. &mov ($rounds_,$rounds);
  707. &pshufb ($ivec,$inout3);
  708. if ($inline)
  709. { &aesni_inline_generate1("enc"); }
  710. else
  711. { &call ("_aesni_encrypt1"); }
  712. &shl ($rounds_,4);
  713. &mov ($rounds,16);
  714. &movups ($in0,&QWP(0,$inp)); # load inp
  715. &paddq ($ivec,&QWP(16,"esp"));
  716. &lea ($inp,&QWP(16,$inp));
  717. &sub ($rounds,$rounds_);
  718. &lea ($key,&DWP(32,$key_,$rounds_));
  719. &mov ($rounds_,$rounds);
  720. &jmp (&label("ccm64_dec_outer"));
  721. &set_label("ccm64_dec_outer",16);
  722. &xorps ($in0,$inout0); # inp ^= E(ivec)
  723. &movdqa ($inout0,$ivec);
  724. &movups (&QWP(0,$out),$in0); # save output
  725. &lea ($out,&DWP(16,$out));
  726. &pshufb ($inout0,$inout3);
  727. &sub ($len,1);
  728. &jz (&label("ccm64_dec_break"));
  729. &$movekey ($rndkey0,&QWP(0,$key_));
  730. &mov ($rounds,$rounds_);
  731. &$movekey ($rndkey1,&QWP(16,$key_));
  732. &xorps ($in0,$rndkey0);
  733. &xorps ($inout0,$rndkey0);
  734. &xorps ($cmac,$in0); # cmac^=out
  735. &$movekey ($rndkey0,&QWP(32,$key_));
  736. &set_label("ccm64_dec2_loop");
  737. &aesenc ($inout0,$rndkey1);
  738. &aesenc ($cmac,$rndkey1);
  739. &$movekey ($rndkey1,&QWP(0,$key,$rounds));
  740. &add ($rounds,32);
  741. &aesenc ($inout0,$rndkey0);
  742. &aesenc ($cmac,$rndkey0);
  743. &$movekey ($rndkey0,&QWP(-16,$key,$rounds));
  744. &jnz (&label("ccm64_dec2_loop"));
  745. &movups ($in0,&QWP(0,$inp)); # load inp
  746. &paddq ($ivec,&QWP(16,"esp"));
  747. &aesenc ($inout0,$rndkey1);
  748. &aesenc ($cmac,$rndkey1);
  749. &aesenclast ($inout0,$rndkey0);
  750. &aesenclast ($cmac,$rndkey0);
  751. &lea ($inp,&QWP(16,$inp));
  752. &jmp (&label("ccm64_dec_outer"));
  753. &set_label("ccm64_dec_break",16);
  754. &mov ($rounds,&DWP(240,$key_));
  755. &mov ($key,$key_);
  756. if ($inline)
  757. { &aesni_inline_generate1("enc",$cmac,$in0); }
  758. else
  759. { &call ("_aesni_encrypt1",$cmac); }
  760. &mov ("esp",&DWP(48,"esp"));
  761. &mov ($out,&wparam(5));
  762. &movups (&QWP(0,$out),$cmac);
  763. &pxor ("xmm0","xmm0"); # clear register bank
  764. &pxor ("xmm1","xmm1");
  765. &pxor ("xmm2","xmm2");
  766. &pxor ("xmm3","xmm3");
  767. &pxor ("xmm4","xmm4");
  768. &pxor ("xmm5","xmm5");
  769. &pxor ("xmm6","xmm6");
  770. &pxor ("xmm7","xmm7");
  771. &function_end("aesni_ccm64_decrypt_blocks");
  772. }
  773. ######################################################################
  774. # void aesni_ctr32_encrypt_blocks (const void *in, void *out,
  775. # size_t blocks, const AES_KEY *key,
  776. # const char *ivec);
  777. #
  778. # Handles only complete blocks, operates on 32-bit counter and
  779. # does not update *ivec! (see crypto/modes/ctr128.c for details)
  780. #
  781. # stack layout:
  782. # 0 pshufb mask
  783. # 16 vector addend: 0,6,6,6
  784. # 32 counter-less ivec
  785. # 48 1st triplet of counter vector
  786. # 64 2nd triplet of counter vector
  787. # 80 saved %esp
  788. &function_begin("aesni_ctr32_encrypt_blocks");
  789. &mov ($inp,&wparam(0));
  790. &mov ($out,&wparam(1));
  791. &mov ($len,&wparam(2));
  792. &mov ($key,&wparam(3));
  793. &mov ($rounds_,&wparam(4));
  794. &mov ($key_,"esp");
  795. &sub ("esp",88);
  796. &and ("esp",-16); # align stack
  797. &mov (&DWP(80,"esp"),$key_);
  798. &cmp ($len,1);
  799. &je (&label("ctr32_one_shortcut"));
  800. &movdqu ($inout5,&QWP(0,$rounds_)); # load ivec
  801. # compose byte-swap control mask for pshufb on stack
  802. &mov (&DWP(0,"esp"),0x0c0d0e0f);
  803. &mov (&DWP(4,"esp"),0x08090a0b);
  804. &mov (&DWP(8,"esp"),0x04050607);
  805. &mov (&DWP(12,"esp"),0x00010203);
  806. # compose counter increment vector on stack
  807. &mov ($rounds,6);
  808. &xor ($key_,$key_);
  809. &mov (&DWP(16,"esp"),$rounds);
  810. &mov (&DWP(20,"esp"),$rounds);
  811. &mov (&DWP(24,"esp"),$rounds);
  812. &mov (&DWP(28,"esp"),$key_);
  813. &pextrd ($rounds_,$inout5,3); # pull 32-bit counter
  814. &pinsrd ($inout5,$key_,3); # wipe 32-bit counter
  815. &mov ($rounds,&DWP(240,$key)); # key->rounds
  816. # compose 2 vectors of 3x32-bit counters
  817. &bswap ($rounds_);
  818. &pxor ($rndkey0,$rndkey0);
  819. &pxor ($rndkey1,$rndkey1);
  820. &movdqa ($inout0,&QWP(0,"esp")); # load byte-swap mask
  821. &pinsrd ($rndkey0,$rounds_,0);
  822. &lea ($key_,&DWP(3,$rounds_));
  823. &pinsrd ($rndkey1,$key_,0);
  824. &inc ($rounds_);
  825. &pinsrd ($rndkey0,$rounds_,1);
  826. &inc ($key_);
  827. &pinsrd ($rndkey1,$key_,1);
  828. &inc ($rounds_);
  829. &pinsrd ($rndkey0,$rounds_,2);
  830. &inc ($key_);
  831. &pinsrd ($rndkey1,$key_,2);
  832. &movdqa (&QWP(48,"esp"),$rndkey0); # save 1st triplet
  833. &pshufb ($rndkey0,$inout0); # byte swap
  834. &movdqu ($inout4,&QWP(0,$key)); # key[0]
  835. &movdqa (&QWP(64,"esp"),$rndkey1); # save 2nd triplet
  836. &pshufb ($rndkey1,$inout0); # byte swap
  837. &pshufd ($inout0,$rndkey0,3<<6); # place counter to upper dword
  838. &pshufd ($inout1,$rndkey0,2<<6);
  839. &cmp ($len,6);
  840. &jb (&label("ctr32_tail"));
  841. &pxor ($inout5,$inout4); # counter-less ivec^key[0]
  842. &shl ($rounds,4);
  843. &mov ($rounds_,16);
  844. &movdqa (&QWP(32,"esp"),$inout5); # save counter-less ivec^key[0]
  845. &mov ($key_,$key); # backup $key
  846. &sub ($rounds_,$rounds); # backup twisted $rounds
  847. &lea ($key,&DWP(32,$key,$rounds));
  848. &sub ($len,6);
  849. &jmp (&label("ctr32_loop6"));
  850. &set_label("ctr32_loop6",16);
  851. # inlining _aesni_encrypt6's prologue gives ~6% improvement...
  852. &pshufd ($inout2,$rndkey0,1<<6);
  853. &movdqa ($rndkey0,&QWP(32,"esp")); # pull counter-less ivec
  854. &pshufd ($inout3,$rndkey1,3<<6);
  855. &pxor ($inout0,$rndkey0); # merge counter-less ivec
  856. &pshufd ($inout4,$rndkey1,2<<6);
  857. &pxor ($inout1,$rndkey0);
  858. &pshufd ($inout5,$rndkey1,1<<6);
  859. &$movekey ($rndkey1,&QWP(16,$key_));
  860. &pxor ($inout2,$rndkey0);
  861. &pxor ($inout3,$rndkey0);
  862. &aesenc ($inout0,$rndkey1);
  863. &pxor ($inout4,$rndkey0);
  864. &pxor ($inout5,$rndkey0);
  865. &aesenc ($inout1,$rndkey1);
  866. &$movekey ($rndkey0,&QWP(32,$key_));
  867. &mov ($rounds,$rounds_);
  868. &aesenc ($inout2,$rndkey1);
  869. &aesenc ($inout3,$rndkey1);
  870. &aesenc ($inout4,$rndkey1);
  871. &aesenc ($inout5,$rndkey1);
  872. &call (&label("_aesni_encrypt6_enter"));
  873. &movups ($rndkey1,&QWP(0,$inp));
  874. &movups ($rndkey0,&QWP(0x10,$inp));
  875. &xorps ($inout0,$rndkey1);
  876. &movups ($rndkey1,&QWP(0x20,$inp));
  877. &xorps ($inout1,$rndkey0);
  878. &movups (&QWP(0,$out),$inout0);
  879. &movdqa ($rndkey0,&QWP(16,"esp")); # load increment
  880. &xorps ($inout2,$rndkey1);
  881. &movdqa ($rndkey1,&QWP(64,"esp")); # load 2nd triplet
  882. &movups (&QWP(0x10,$out),$inout1);
  883. &movups (&QWP(0x20,$out),$inout2);
  884. &paddd ($rndkey1,$rndkey0); # 2nd triplet increment
  885. &paddd ($rndkey0,&QWP(48,"esp")); # 1st triplet increment
  886. &movdqa ($inout0,&QWP(0,"esp")); # load byte swap mask
  887. &movups ($inout1,&QWP(0x30,$inp));
  888. &movups ($inout2,&QWP(0x40,$inp));
  889. &xorps ($inout3,$inout1);
  890. &movups ($inout1,&QWP(0x50,$inp));
  891. &lea ($inp,&DWP(0x60,$inp));
  892. &movdqa (&QWP(48,"esp"),$rndkey0); # save 1st triplet
  893. &pshufb ($rndkey0,$inout0); # byte swap
  894. &xorps ($inout4,$inout2);
  895. &movups (&QWP(0x30,$out),$inout3);
  896. &xorps ($inout5,$inout1);
  897. &movdqa (&QWP(64,"esp"),$rndkey1); # save 2nd triplet
  898. &pshufb ($rndkey1,$inout0); # byte swap
  899. &movups (&QWP(0x40,$out),$inout4);
  900. &pshufd ($inout0,$rndkey0,3<<6);
  901. &movups (&QWP(0x50,$out),$inout5);
  902. &lea ($out,&DWP(0x60,$out));
  903. &pshufd ($inout1,$rndkey0,2<<6);
  904. &sub ($len,6);
  905. &jnc (&label("ctr32_loop6"));
  906. &add ($len,6);
  907. &jz (&label("ctr32_ret"));
  908. &movdqu ($inout5,&QWP(0,$key_));
  909. &mov ($key,$key_);
  910. &pxor ($inout5,&QWP(32,"esp")); # restore count-less ivec
  911. &mov ($rounds,&DWP(240,$key_)); # restore $rounds
  912. &set_label("ctr32_tail");
  913. &por ($inout0,$inout5);
  914. &cmp ($len,2);
  915. &jb (&label("ctr32_one"));
  916. &pshufd ($inout2,$rndkey0,1<<6);
  917. &por ($inout1,$inout5);
  918. &je (&label("ctr32_two"));
  919. &pshufd ($inout3,$rndkey1,3<<6);
  920. &por ($inout2,$inout5);
  921. &cmp ($len,4);
  922. &jb (&label("ctr32_three"));
  923. &pshufd ($inout4,$rndkey1,2<<6);
  924. &por ($inout3,$inout5);
  925. &je (&label("ctr32_four"));
  926. &por ($inout4,$inout5);
  927. &call ("_aesni_encrypt6");
  928. &movups ($rndkey1,&QWP(0,$inp));
  929. &movups ($rndkey0,&QWP(0x10,$inp));
  930. &xorps ($inout0,$rndkey1);
  931. &movups ($rndkey1,&QWP(0x20,$inp));
  932. &xorps ($inout1,$rndkey0);
  933. &movups ($rndkey0,&QWP(0x30,$inp));
  934. &xorps ($inout2,$rndkey1);
  935. &movups ($rndkey1,&QWP(0x40,$inp));
  936. &xorps ($inout3,$rndkey0);
  937. &movups (&QWP(0,$out),$inout0);
  938. &xorps ($inout4,$rndkey1);
  939. &movups (&QWP(0x10,$out),$inout1);
  940. &movups (&QWP(0x20,$out),$inout2);
  941. &movups (&QWP(0x30,$out),$inout3);
  942. &movups (&QWP(0x40,$out),$inout4);
  943. &jmp (&label("ctr32_ret"));
  944. &set_label("ctr32_one_shortcut",16);
  945. &movups ($inout0,&QWP(0,$rounds_)); # load ivec
  946. &mov ($rounds,&DWP(240,$key));
  947. &set_label("ctr32_one");
  948. if ($inline)
  949. { &aesni_inline_generate1("enc"); }
  950. else
  951. { &call ("_aesni_encrypt1"); }
  952. &movups ($in0,&QWP(0,$inp));
  953. &xorps ($in0,$inout0);
  954. &movups (&QWP(0,$out),$in0);
  955. &jmp (&label("ctr32_ret"));
  956. &set_label("ctr32_two",16);
  957. &call ("_aesni_encrypt2");
  958. &movups ($inout3,&QWP(0,$inp));
  959. &movups ($inout4,&QWP(0x10,$inp));
  960. &xorps ($inout0,$inout3);
  961. &xorps ($inout1,$inout4);
  962. &movups (&QWP(0,$out),$inout0);
  963. &movups (&QWP(0x10,$out),$inout1);
  964. &jmp (&label("ctr32_ret"));
  965. &set_label("ctr32_three",16);
  966. &call ("_aesni_encrypt3");
  967. &movups ($inout3,&QWP(0,$inp));
  968. &movups ($inout4,&QWP(0x10,$inp));
  969. &xorps ($inout0,$inout3);
  970. &movups ($inout5,&QWP(0x20,$inp));
  971. &xorps ($inout1,$inout4);
  972. &movups (&QWP(0,$out),$inout0);
  973. &xorps ($inout2,$inout5);
  974. &movups (&QWP(0x10,$out),$inout1);
  975. &movups (&QWP(0x20,$out),$inout2);
  976. &jmp (&label("ctr32_ret"));
  977. &set_label("ctr32_four",16);
  978. &call ("_aesni_encrypt4");
  979. &movups ($inout4,&QWP(0,$inp));
  980. &movups ($inout5,&QWP(0x10,$inp));
  981. &movups ($rndkey1,&QWP(0x20,$inp));
  982. &xorps ($inout0,$inout4);
  983. &movups ($rndkey0,&QWP(0x30,$inp));
  984. &xorps ($inout1,$inout5);
  985. &movups (&QWP(0,$out),$inout0);
  986. &xorps ($inout2,$rndkey1);
  987. &movups (&QWP(0x10,$out),$inout1);
  988. &xorps ($inout3,$rndkey0);
  989. &movups (&QWP(0x20,$out),$inout2);
  990. &movups (&QWP(0x30,$out),$inout3);
  991. &set_label("ctr32_ret");
  992. &pxor ("xmm0","xmm0"); # clear register bank
  993. &pxor ("xmm1","xmm1");
  994. &pxor ("xmm2","xmm2");
  995. &pxor ("xmm3","xmm3");
  996. &pxor ("xmm4","xmm4");
  997. &movdqa (&QWP(32,"esp"),"xmm0"); # clear stack
  998. &pxor ("xmm5","xmm5");
  999. &movdqa (&QWP(48,"esp"),"xmm0");
  1000. &pxor ("xmm6","xmm6");
  1001. &movdqa (&QWP(64,"esp"),"xmm0");
  1002. &pxor ("xmm7","xmm7");
  1003. &mov ("esp",&DWP(80,"esp"));
  1004. &function_end("aesni_ctr32_encrypt_blocks");
  1005. ######################################################################
  1006. # void aesni_xts_[en|de]crypt(const char *inp,char *out,size_t len,
  1007. # const AES_KEY *key1, const AES_KEY *key2
  1008. # const unsigned char iv[16]);
  1009. #
  1010. { my ($tweak,$twtmp,$twres,$twmask)=($rndkey1,$rndkey0,$inout0,$inout1);
  1011. &function_begin("aesni_xts_encrypt");
  1012. &mov ($key,&wparam(4)); # key2
  1013. &mov ($inp,&wparam(5)); # clear-text tweak
  1014. &mov ($rounds,&DWP(240,$key)); # key2->rounds
  1015. &movups ($inout0,&QWP(0,$inp));
  1016. if ($inline)
  1017. { &aesni_inline_generate1("enc"); }
  1018. else
  1019. { &call ("_aesni_encrypt1"); }
  1020. &mov ($inp,&wparam(0));
  1021. &mov ($out,&wparam(1));
  1022. &mov ($len,&wparam(2));
  1023. &mov ($key,&wparam(3)); # key1
  1024. &mov ($key_,"esp");
  1025. &sub ("esp",16*7+8);
  1026. &mov ($rounds,&DWP(240,$key)); # key1->rounds
  1027. &and ("esp",-16); # align stack
  1028. &mov (&DWP(16*6+0,"esp"),0x87); # compose the magic constant
  1029. &mov (&DWP(16*6+4,"esp"),0);
  1030. &mov (&DWP(16*6+8,"esp"),1);
  1031. &mov (&DWP(16*6+12,"esp"),0);
  1032. &mov (&DWP(16*7+0,"esp"),$len); # save original $len
  1033. &mov (&DWP(16*7+4,"esp"),$key_); # save original %esp
  1034. &movdqa ($tweak,$inout0);
  1035. &pxor ($twtmp,$twtmp);
  1036. &movdqa ($twmask,&QWP(6*16,"esp")); # 0x0...010...87
  1037. &pcmpgtd($twtmp,$tweak); # broadcast upper bits
  1038. &and ($len,-16);
  1039. &mov ($key_,$key); # backup $key
  1040. &mov ($rounds_,$rounds); # backup $rounds
  1041. &sub ($len,16*6);
  1042. &jc (&label("xts_enc_short"));
  1043. &shl ($rounds,4);
  1044. &mov ($rounds_,16);
  1045. &sub ($rounds_,$rounds);
  1046. &lea ($key,&DWP(32,$key,$rounds));
  1047. &jmp (&label("xts_enc_loop6"));
  1048. &set_label("xts_enc_loop6",16);
  1049. for ($i=0;$i<4;$i++) {
  1050. &pshufd ($twres,$twtmp,0x13);
  1051. &pxor ($twtmp,$twtmp);
  1052. &movdqa (&QWP(16*$i,"esp"),$tweak);
  1053. &paddq ($tweak,$tweak); # &psllq($tweak,1);
  1054. &pand ($twres,$twmask); # isolate carry and residue
  1055. &pcmpgtd ($twtmp,$tweak); # broadcast upper bits
  1056. &pxor ($tweak,$twres);
  1057. }
  1058. &pshufd ($inout5,$twtmp,0x13);
  1059. &movdqa (&QWP(16*$i++,"esp"),$tweak);
  1060. &paddq ($tweak,$tweak); # &psllq($tweak,1);
  1061. &$movekey ($rndkey0,&QWP(0,$key_));
  1062. &pand ($inout5,$twmask); # isolate carry and residue
  1063. &movups ($inout0,&QWP(0,$inp)); # load input
  1064. &pxor ($inout5,$tweak);
  1065. # inline _aesni_encrypt6 prologue and flip xor with tweak and key[0]
  1066. &mov ($rounds,$rounds_); # restore $rounds
  1067. &movdqu ($inout1,&QWP(16*1,$inp));
  1068. &xorps ($inout0,$rndkey0); # input^=rndkey[0]
  1069. &movdqu ($inout2,&QWP(16*2,$inp));
  1070. &pxor ($inout1,$rndkey0);
  1071. &movdqu ($inout3,&QWP(16*3,$inp));
  1072. &pxor ($inout2,$rndkey0);
  1073. &movdqu ($inout4,&QWP(16*4,$inp));
  1074. &pxor ($inout3,$rndkey0);
  1075. &movdqu ($rndkey1,&QWP(16*5,$inp));
  1076. &pxor ($inout4,$rndkey0);
  1077. &lea ($inp,&DWP(16*6,$inp));
  1078. &pxor ($inout0,&QWP(16*0,"esp")); # input^=tweak
  1079. &movdqa (&QWP(16*$i,"esp"),$inout5); # save last tweak
  1080. &pxor ($inout5,$rndkey1);
  1081. &$movekey ($rndkey1,&QWP(16,$key_));
  1082. &pxor ($inout1,&QWP(16*1,"esp"));
  1083. &pxor ($inout2,&QWP(16*2,"esp"));
  1084. &aesenc ($inout0,$rndkey1);
  1085. &pxor ($inout3,&QWP(16*3,"esp"));
  1086. &pxor ($inout4,&QWP(16*4,"esp"));
  1087. &aesenc ($inout1,$rndkey1);
  1088. &pxor ($inout5,$rndkey0);
  1089. &$movekey ($rndkey0,&QWP(32,$key_));
  1090. &aesenc ($inout2,$rndkey1);
  1091. &aesenc ($inout3,$rndkey1);
  1092. &aesenc ($inout4,$rndkey1);
  1093. &aesenc ($inout5,$rndkey1);
  1094. &call (&label("_aesni_encrypt6_enter"));
  1095. &movdqa ($tweak,&QWP(16*5,"esp")); # last tweak
  1096. &pxor ($twtmp,$twtmp);
  1097. &xorps ($inout0,&QWP(16*0,"esp")); # output^=tweak
  1098. &pcmpgtd ($twtmp,$tweak); # broadcast upper bits
  1099. &xorps ($inout1,&QWP(16*1,"esp"));
  1100. &movups (&QWP(16*0,$out),$inout0); # write output
  1101. &xorps ($inout2,&QWP(16*2,"esp"));
  1102. &movups (&QWP(16*1,$out),$inout1);
  1103. &xorps ($inout3,&QWP(16*3,"esp"));
  1104. &movups (&QWP(16*2,$out),$inout2);
  1105. &xorps ($inout4,&QWP(16*4,"esp"));
  1106. &movups (&QWP(16*3,$out),$inout3);
  1107. &xorps ($inout5,$tweak);
  1108. &movups (&QWP(16*4,$out),$inout4);
  1109. &pshufd ($twres,$twtmp,0x13);
  1110. &movups (&QWP(16*5,$out),$inout5);
  1111. &lea ($out,&DWP(16*6,$out));
  1112. &movdqa ($twmask,&QWP(16*6,"esp")); # 0x0...010...87
  1113. &pxor ($twtmp,$twtmp);
  1114. &paddq ($tweak,$tweak); # &psllq($tweak,1);
  1115. &pand ($twres,$twmask); # isolate carry and residue
  1116. &pcmpgtd($twtmp,$tweak); # broadcast upper bits
  1117. &pxor ($tweak,$twres);
  1118. &sub ($len,16*6);
  1119. &jnc (&label("xts_enc_loop6"));
  1120. &mov ($rounds,&DWP(240,$key_)); # restore $rounds
  1121. &mov ($key,$key_); # restore $key
  1122. &mov ($rounds_,$rounds);
  1123. &set_label("xts_enc_short");
  1124. &add ($len,16*6);
  1125. &jz (&label("xts_enc_done6x"));
  1126. &movdqa ($inout3,$tweak); # put aside previous tweak
  1127. &cmp ($len,0x20);
  1128. &jb (&label("xts_enc_one"));
  1129. &pshufd ($twres,$twtmp,0x13);
  1130. &pxor ($twtmp,$twtmp);
  1131. &paddq ($tweak,$tweak); # &psllq($tweak,1);
  1132. &pand ($twres,$twmask); # isolate carry and residue
  1133. &pcmpgtd($twtmp,$tweak); # broadcast upper bits
  1134. &pxor ($tweak,$twres);
  1135. &je (&label("xts_enc_two"));
  1136. &pshufd ($twres,$twtmp,0x13);
  1137. &pxor ($twtmp,$twtmp);
  1138. &movdqa ($inout4,$tweak); # put aside previous tweak
  1139. &paddq ($tweak,$tweak); # &psllq($tweak,1);
  1140. &pand ($twres,$twmask); # isolate carry and residue
  1141. &pcmpgtd($twtmp,$tweak); # broadcast upper bits
  1142. &pxor ($tweak,$twres);
  1143. &cmp ($len,0x40);
  1144. &jb (&label("xts_enc_three"));
  1145. &pshufd ($twres,$twtmp,0x13);
  1146. &pxor ($twtmp,$twtmp);
  1147. &movdqa ($inout5,$tweak); # put aside previous tweak
  1148. &paddq ($tweak,$tweak); # &psllq($tweak,1);
  1149. &pand ($twres,$twmask); # isolate carry and residue
  1150. &pcmpgtd($twtmp,$tweak); # broadcast upper bits
  1151. &pxor ($tweak,$twres);
  1152. &movdqa (&QWP(16*0,"esp"),$inout3);
  1153. &movdqa (&QWP(16*1,"esp"),$inout4);
  1154. &je (&label("xts_enc_four"));
  1155. &movdqa (&QWP(16*2,"esp"),$inout5);
  1156. &pshufd ($inout5,$twtmp,0x13);
  1157. &movdqa (&QWP(16*3,"esp"),$tweak);
  1158. &paddq ($tweak,$tweak); # &psllq($inout0,1);
  1159. &pand ($inout5,$twmask); # isolate carry and residue
  1160. &pxor ($inout5,$tweak);
  1161. &movdqu ($inout0,&QWP(16*0,$inp)); # load input
  1162. &movdqu ($inout1,&QWP(16*1,$inp));
  1163. &movdqu ($inout2,&QWP(16*2,$inp));
  1164. &pxor ($inout0,&QWP(16*0,"esp")); # input^=tweak
  1165. &movdqu ($inout3,&QWP(16*3,$inp));
  1166. &pxor ($inout1,&QWP(16*1,"esp"));
  1167. &movdqu ($inout4,&QWP(16*4,$inp));
  1168. &pxor ($inout2,&QWP(16*2,"esp"));
  1169. &lea ($inp,&DWP(16*5,$inp));
  1170. &pxor ($inout3,&QWP(16*3,"esp"));
  1171. &movdqa (&QWP(16*4,"esp"),$inout5); # save last tweak
  1172. &pxor ($inout4,$inout5);
  1173. &call ("_aesni_encrypt6");
  1174. &movaps ($tweak,&QWP(16*4,"esp")); # last tweak
  1175. &xorps ($inout0,&QWP(16*0,"esp")); # output^=tweak
  1176. &xorps ($inout1,&QWP(16*1,"esp"));
  1177. &xorps ($inout2,&QWP(16*2,"esp"));
  1178. &movups (&QWP(16*0,$out),$inout0); # write output
  1179. &xorps ($inout3,&QWP(16*3,"esp"));
  1180. &movups (&QWP(16*1,$out),$inout1);
  1181. &xorps ($inout4,$tweak);
  1182. &movups (&QWP(16*2,$out),$inout2);
  1183. &movups (&QWP(16*3,$out),$inout3);
  1184. &movups (&QWP(16*4,$out),$inout4);
  1185. &lea ($out,&DWP(16*5,$out));
  1186. &jmp (&label("xts_enc_done"));
  1187. &set_label("xts_enc_one",16);
  1188. &movups ($inout0,&QWP(16*0,$inp)); # load input
  1189. &lea ($inp,&DWP(16*1,$inp));
  1190. &xorps ($inout0,$inout3); # input^=tweak
  1191. if ($inline)
  1192. { &aesni_inline_generate1("enc"); }
  1193. else
  1194. { &call ("_aesni_encrypt1"); }
  1195. &xorps ($inout0,$inout3); # output^=tweak
  1196. &movups (&QWP(16*0,$out),$inout0); # write output
  1197. &lea ($out,&DWP(16*1,$out));
  1198. &movdqa ($tweak,$inout3); # last tweak
  1199. &jmp (&label("xts_enc_done"));
  1200. &set_label("xts_enc_two",16);
  1201. &movaps ($inout4,$tweak); # put aside last tweak
  1202. &movups ($inout0,&QWP(16*0,$inp)); # load input
  1203. &movups ($inout1,&QWP(16*1,$inp));
  1204. &lea ($inp,&DWP(16*2,$inp));
  1205. &xorps ($inout0,$inout3); # input^=tweak
  1206. &xorps ($inout1,$inout4);
  1207. &call ("_aesni_encrypt2");
  1208. &xorps ($inout0,$inout3); # output^=tweak
  1209. &xorps ($inout1,$inout4);
  1210. &movups (&QWP(16*0,$out),$inout0); # write output
  1211. &movups (&QWP(16*1,$out),$inout1);
  1212. &lea ($out,&DWP(16*2,$out));
  1213. &movdqa ($tweak,$inout4); # last tweak
  1214. &jmp (&label("xts_enc_done"));
  1215. &set_label("xts_enc_three",16);
  1216. &movaps ($inout5,$tweak); # put aside last tweak
  1217. &movups ($inout0,&QWP(16*0,$inp)); # load input
  1218. &movups ($inout1,&QWP(16*1,$inp));
  1219. &movups ($inout2,&QWP(16*2,$inp));
  1220. &lea ($inp,&DWP(16*3,$inp));
  1221. &xorps ($inout0,$inout3); # input^=tweak
  1222. &xorps ($inout1,$inout4);
  1223. &xorps ($inout2,$inout5);
  1224. &call ("_aesni_encrypt3");
  1225. &xorps ($inout0,$inout3); # output^=tweak
  1226. &xorps ($inout1,$inout4);
  1227. &xorps ($inout2,$inout5);
  1228. &movups (&QWP(16*0,$out),$inout0); # write output
  1229. &movups (&QWP(16*1,$out),$inout1);
  1230. &movups (&QWP(16*2,$out),$inout2);
  1231. &lea ($out,&DWP(16*3,$out));
  1232. &movdqa ($tweak,$inout5); # last tweak
  1233. &jmp (&label("xts_enc_done"));
  1234. &set_label("xts_enc_four",16);
  1235. &movaps ($inout4,$tweak); # put aside last tweak
  1236. &movups ($inout0,&QWP(16*0,$inp)); # load input
  1237. &movups ($inout1,&QWP(16*1,$inp));
  1238. &movups ($inout2,&QWP(16*2,$inp));
  1239. &xorps ($inout0,&QWP(16*0,"esp")); # input^=tweak
  1240. &movups ($inout3,&QWP(16*3,$inp));
  1241. &lea ($inp,&DWP(16*4,$inp));
  1242. &xorps ($inout1,&QWP(16*1,"esp"));
  1243. &xorps ($inout2,$inout5);
  1244. &xorps ($inout3,$inout4);
  1245. &call ("_aesni_encrypt4");
  1246. &xorps ($inout0,&QWP(16*0,"esp")); # output^=tweak
  1247. &xorps ($inout1,&QWP(16*1,"esp"));
  1248. &xorps ($inout2,$inout5);
  1249. &movups (&QWP(16*0,$out),$inout0); # write output
  1250. &xorps ($inout3,$inout4);
  1251. &movups (&QWP(16*1,$out),$inout1);
  1252. &movups (&QWP(16*2,$out),$inout2);
  1253. &movups (&QWP(16*3,$out),$inout3);
  1254. &lea ($out,&DWP(16*4,$out));
  1255. &movdqa ($tweak,$inout4); # last tweak
  1256. &jmp (&label("xts_enc_done"));
  1257. &set_label("xts_enc_done6x",16); # $tweak is pre-calculated
  1258. &mov ($len,&DWP(16*7+0,"esp")); # restore original $len
  1259. &and ($len,15);
  1260. &jz (&label("xts_enc_ret"));
  1261. &movdqa ($inout3,$tweak);
  1262. &mov (&DWP(16*7+0,"esp"),$len); # save $len%16
  1263. &jmp (&label("xts_enc_steal"));
  1264. &set_label("xts_enc_done",16);
  1265. &mov ($len,&DWP(16*7+0,"esp")); # restore original $len
  1266. &pxor ($twtmp,$twtmp);
  1267. &and ($len,15);
  1268. &jz (&label("xts_enc_ret"));
  1269. &pcmpgtd($twtmp,$tweak); # broadcast upper bits
  1270. &mov (&DWP(16*7+0,"esp"),$len); # save $len%16
  1271. &pshufd ($inout3,$twtmp,0x13);
  1272. &paddq ($tweak,$tweak); # &psllq($tweak,1);
  1273. &pand ($inout3,&QWP(16*6,"esp")); # isolate carry and residue
  1274. &pxor ($inout3,$tweak);
  1275. &set_label("xts_enc_steal");
  1276. &movz ($rounds,&BP(0,$inp));
  1277. &movz ($key,&BP(-16,$out));
  1278. &lea ($inp,&DWP(1,$inp));
  1279. &mov (&BP(-16,$out),&LB($rounds));
  1280. &mov (&BP(0,$out),&LB($key));
  1281. &lea ($out,&DWP(1,$out));
  1282. &sub ($len,1);
  1283. &jnz (&label("xts_enc_steal"));
  1284. &sub ($out,&DWP(16*7+0,"esp")); # rewind $out
  1285. &mov ($key,$key_); # restore $key
  1286. &mov ($rounds,$rounds_); # restore $rounds
  1287. &movups ($inout0,&QWP(-16,$out)); # load input
  1288. &xorps ($inout0,$inout3); # input^=tweak
  1289. if ($inline)
  1290. { &aesni_inline_generate1("enc"); }
  1291. else
  1292. { &call ("_aesni_encrypt1"); }
  1293. &xorps ($inout0,$inout3); # output^=tweak
  1294. &movups (&QWP(-16,$out),$inout0); # write output
  1295. &set_label("xts_enc_ret");
  1296. &pxor ("xmm0","xmm0"); # clear register bank
  1297. &pxor ("xmm1","xmm1");
  1298. &pxor ("xmm2","xmm2");
  1299. &movdqa (&QWP(16*0,"esp"),"xmm0"); # clear stack
  1300. &pxor ("xmm3","xmm3");
  1301. &movdqa (&QWP(16*1,"esp"),"xmm0");
  1302. &pxor ("xmm4","xmm4");
  1303. &movdqa (&QWP(16*2,"esp"),"xmm0");
  1304. &pxor ("xmm5","xmm5");
  1305. &movdqa (&QWP(16*3,"esp"),"xmm0");
  1306. &pxor ("xmm6","xmm6");
  1307. &movdqa (&QWP(16*4,"esp"),"xmm0");
  1308. &pxor ("xmm7","xmm7");
  1309. &movdqa (&QWP(16*5,"esp"),"xmm0");
  1310. &mov ("esp",&DWP(16*7+4,"esp")); # restore %esp
  1311. &function_end("aesni_xts_encrypt");
  1312. &function_begin("aesni_xts_decrypt");
  1313. &mov ($key,&wparam(4)); # key2
  1314. &mov ($inp,&wparam(5)); # clear-text tweak
  1315. &mov ($rounds,&DWP(240,$key)); # key2->rounds
  1316. &movups ($inout0,&QWP(0,$inp));
  1317. if ($inline)
  1318. { &aesni_inline_generate1("enc"); }
  1319. else
  1320. { &call ("_aesni_encrypt1"); }
  1321. &mov ($inp,&wparam(0));
  1322. &mov ($out,&wparam(1));
  1323. &mov ($len,&wparam(2));
  1324. &mov ($key,&wparam(3)); # key1
  1325. &mov ($key_,"esp");
  1326. &sub ("esp",16*7+8);
  1327. &and ("esp",-16); # align stack
  1328. &xor ($rounds_,$rounds_); # if(len%16) len-=16;
  1329. &test ($len,15);
  1330. &setnz (&LB($rounds_));
  1331. &shl ($rounds_,4);
  1332. &sub ($len,$rounds_);
  1333. &mov (&DWP(16*6+0,"esp"),0x87); # compose the magic constant
  1334. &mov (&DWP(16*6+4,"esp"),0);
  1335. &mov (&DWP(16*6+8,"esp"),1);
  1336. &mov (&DWP(16*6+12,"esp"),0);
  1337. &mov (&DWP(16*7+0,"esp"),$len); # save original $len
  1338. &mov (&DWP(16*7+4,"esp"),$key_); # save original %esp
  1339. &mov ($rounds,&DWP(240,$key)); # key1->rounds
  1340. &mov ($key_,$key); # backup $key
  1341. &mov ($rounds_,$rounds); # backup $rounds
  1342. &movdqa ($tweak,$inout0);
  1343. &pxor ($twtmp,$twtmp);
  1344. &movdqa ($twmask,&QWP(6*16,"esp")); # 0x0...010...87
  1345. &pcmpgtd($twtmp,$tweak); # broadcast upper bits
  1346. &and ($len,-16);
  1347. &sub ($len,16*6);
  1348. &jc (&label("xts_dec_short"));
  1349. &shl ($rounds,4);
  1350. &mov ($rounds_,16);
  1351. &sub ($rounds_,$rounds);
  1352. &lea ($key,&DWP(32,$key,$rounds));
  1353. &jmp (&label("xts_dec_loop6"));
  1354. &set_label("xts_dec_loop6",16);
  1355. for ($i=0;$i<4;$i++) {
  1356. &pshufd ($twres,$twtmp,0x13);
  1357. &pxor ($twtmp,$twtmp);
  1358. &movdqa (&QWP(16*$i,"esp"),$tweak);
  1359. &paddq ($tweak,$tweak); # &psllq($tweak,1);
  1360. &pand ($twres,$twmask); # isolate carry and residue
  1361. &pcmpgtd ($twtmp,$tweak); # broadcast upper bits
  1362. &pxor ($tweak,$twres);
  1363. }
  1364. &pshufd ($inout5,$twtmp,0x13);
  1365. &movdqa (&QWP(16*$i++,"esp"),$tweak);
  1366. &paddq ($tweak,$tweak); # &psllq($tweak,1);
  1367. &$movekey ($rndkey0,&QWP(0,$key_));
  1368. &pand ($inout5,$twmask); # isolate carry and residue
  1369. &movups ($inout0,&QWP(0,$inp)); # load input
  1370. &pxor ($inout5,$tweak);
  1371. # inline _aesni_encrypt6 prologue and flip xor with tweak and key[0]
  1372. &mov ($rounds,$rounds_);
  1373. &movdqu ($inout1,&QWP(16*1,$inp));
  1374. &xorps ($inout0,$rndkey0); # input^=rndkey[0]
  1375. &movdqu ($inout2,&QWP(16*2,$inp));
  1376. &pxor ($inout1,$rndkey0);
  1377. &movdqu ($inout3,&QWP(16*3,$inp));
  1378. &pxor ($inout2,$rndkey0);
  1379. &movdqu ($inout4,&QWP(16*4,$inp));
  1380. &pxor ($inout3,$rndkey0);
  1381. &movdqu ($rndkey1,&QWP(16*5,$inp));
  1382. &pxor ($inout4,$rndkey0);
  1383. &lea ($inp,&DWP(16*6,$inp));
  1384. &pxor ($inout0,&QWP(16*0,"esp")); # input^=tweak
  1385. &movdqa (&QWP(16*$i,"esp"),$inout5); # save last tweak
  1386. &pxor ($inout5,$rndkey1);
  1387. &$movekey ($rndkey1,&QWP(16,$key_));
  1388. &pxor ($inout1,&QWP(16*1,"esp"));
  1389. &pxor ($inout2,&QWP(16*2,"esp"));
  1390. &aesdec ($inout0,$rndkey1);
  1391. &pxor ($inout3,&QWP(16*3,"esp"));
  1392. &pxor ($inout4,&QWP(16*4,"esp"));
  1393. &aesdec ($inout1,$rndkey1);
  1394. &pxor ($inout5,$rndkey0);
  1395. &$movekey ($rndkey0,&QWP(32,$key_));
  1396. &aesdec ($inout2,$rndkey1);
  1397. &aesdec ($inout3,$rndkey1);
  1398. &aesdec ($inout4,$rndkey1);
  1399. &aesdec ($inout5,$rndkey1);
  1400. &call (&label("_aesni_decrypt6_enter"));
  1401. &movdqa ($tweak,&QWP(16*5,"esp")); # last tweak
  1402. &pxor ($twtmp,$twtmp);
  1403. &xorps ($inout0,&QWP(16*0,"esp")); # output^=tweak
  1404. &pcmpgtd ($twtmp,$tweak); # broadcast upper bits
  1405. &xorps ($inout1,&QWP(16*1,"esp"));
  1406. &movups (&QWP(16*0,$out),$inout0); # write output
  1407. &xorps ($inout2,&QWP(16*2,"esp"));
  1408. &movups (&QWP(16*1,$out),$inout1);
  1409. &xorps ($inout3,&QWP(16*3,"esp"));
  1410. &movups (&QWP(16*2,$out),$inout2);
  1411. &xorps ($inout4,&QWP(16*4,"esp"));
  1412. &movups (&QWP(16*3,$out),$inout3);
  1413. &xorps ($inout5,$tweak);
  1414. &movups (&QWP(16*4,$out),$inout4);
  1415. &pshufd ($twres,$twtmp,0x13);
  1416. &movups (&QWP(16*5,$out),$inout5);
  1417. &lea ($out,&DWP(16*6,$out));
  1418. &movdqa ($twmask,&QWP(16*6,"esp")); # 0x0...010...87
  1419. &pxor ($twtmp,$twtmp);
  1420. &paddq ($tweak,$tweak); # &psllq($tweak,1);
  1421. &pand ($twres,$twmask); # isolate carry and residue
  1422. &pcmpgtd($twtmp,$tweak); # broadcast upper bits
  1423. &pxor ($tweak,$twres);
  1424. &sub ($len,16*6);
  1425. &jnc (&label("xts_dec_loop6"));
  1426. &mov ($rounds,&DWP(240,$key_)); # restore $rounds
  1427. &mov ($key,$key_); # restore $key
  1428. &mov ($rounds_,$rounds);
  1429. &set_label("xts_dec_short");
  1430. &add ($len,16*6);
  1431. &jz (&label("xts_dec_done6x"));
  1432. &movdqa ($inout3,$tweak); # put aside previous tweak
  1433. &cmp ($len,0x20);
  1434. &jb (&label("xts_dec_one"));
  1435. &pshufd ($twres,$twtmp,0x13);
  1436. &pxor ($twtmp,$twtmp);
  1437. &paddq ($tweak,$tweak); # &psllq($tweak,1);
  1438. &pand ($twres,$twmask); # isolate carry and residue
  1439. &pcmpgtd($twtmp,$tweak); # broadcast upper bits
  1440. &pxor ($tweak,$twres);
  1441. &je (&label("xts_dec_two"));
  1442. &pshufd ($twres,$twtmp,0x13);
  1443. &pxor ($twtmp,$twtmp);
  1444. &movdqa ($inout4,$tweak); # put aside previous tweak
  1445. &paddq ($tweak,$tweak); # &psllq($tweak,1);
  1446. &pand ($twres,$twmask); # isolate carry and residue
  1447. &pcmpgtd($twtmp,$tweak); # broadcast upper bits
  1448. &pxor ($tweak,$twres);
  1449. &cmp ($len,0x40);
  1450. &jb (&label("xts_dec_three"));
  1451. &pshufd ($twres,$twtmp,0x13);
  1452. &pxor ($twtmp,$twtmp);
  1453. &movdqa ($inout5,$tweak); # put aside previous tweak
  1454. &paddq ($tweak,$tweak); # &psllq($tweak,1);
  1455. &pand ($twres,$twmask); # isolate carry and residue
  1456. &pcmpgtd($twtmp,$tweak); # broadcast upper bits
  1457. &pxor ($tweak,$twres);
  1458. &movdqa (&QWP(16*0,"esp"),$inout3);
  1459. &movdqa (&QWP(16*1,"esp"),$inout4);
  1460. &je (&label("xts_dec_four"));
  1461. &movdqa (&QWP(16*2,"esp"),$inout5);
  1462. &pshufd ($inout5,$twtmp,0x13);
  1463. &movdqa (&QWP(16*3,"esp"),$tweak);
  1464. &paddq ($tweak,$tweak); # &psllq($inout0,1);
  1465. &pand ($inout5,$twmask); # isolate carry and residue
  1466. &pxor ($inout5,$tweak);
  1467. &movdqu ($inout0,&QWP(16*0,$inp)); # load input
  1468. &movdqu ($inout1,&QWP(16*1,$inp));
  1469. &movdqu ($inout2,&QWP(16*2,$inp));
  1470. &pxor ($inout0,&QWP(16*0,"esp")); # input^=tweak
  1471. &movdqu ($inout3,&QWP(16*3,$inp));
  1472. &pxor ($inout1,&QWP(16*1,"esp"));
  1473. &movdqu ($inout4,&QWP(16*4,$inp));
  1474. &pxor ($inout2,&QWP(16*2,"esp"));
  1475. &lea ($inp,&DWP(16*5,$inp));
  1476. &pxor ($inout3,&QWP(16*3,"esp"));
  1477. &movdqa (&QWP(16*4,"esp"),$inout5); # save last tweak
  1478. &pxor ($inout4,$inout5);
  1479. &call ("_aesni_decrypt6");
  1480. &movaps ($tweak,&QWP(16*4,"esp")); # last tweak
  1481. &xorps ($inout0,&QWP(16*0,"esp")); # output^=tweak
  1482. &xorps ($inout1,&QWP(16*1,"esp"));
  1483. &xorps ($inout2,&QWP(16*2,"esp"));
  1484. &movups (&QWP(16*0,$out),$inout0); # write output
  1485. &xorps ($inout3,&QWP(16*3,"esp"));
  1486. &movups (&QWP(16*1,$out),$inout1);
  1487. &xorps ($inout4,$tweak);
  1488. &movups (&QWP(16*2,$out),$inout2);
  1489. &movups (&QWP(16*3,$out),$inout3);
  1490. &movups (&QWP(16*4,$out),$inout4);
  1491. &lea ($out,&DWP(16*5,$out));
  1492. &jmp (&label("xts_dec_done"));
  1493. &set_label("xts_dec_one",16);
  1494. &movups ($inout0,&QWP(16*0,$inp)); # load input
  1495. &lea ($inp,&DWP(16*1,$inp));
  1496. &xorps ($inout0,$inout3); # input^=tweak
  1497. if ($inline)
  1498. { &aesni_inline_generate1("dec"); }
  1499. else
  1500. { &call ("_aesni_decrypt1"); }
  1501. &xorps ($inout0,$inout3); # output^=tweak
  1502. &movups (&QWP(16*0,$out),$inout0); # write output
  1503. &lea ($out,&DWP(16*1,$out));
  1504. &movdqa ($tweak,$inout3); # last tweak
  1505. &jmp (&label("xts_dec_done"));
  1506. &set_label("xts_dec_two",16);
  1507. &movaps ($inout4,$tweak); # put aside last tweak
  1508. &movups ($inout0,&QWP(16*0,$inp)); # load input
  1509. &movups ($inout1,&QWP(16*1,$inp));
  1510. &lea ($inp,&DWP(16*2,$inp));
  1511. &xorps ($inout0,$inout3); # input^=tweak
  1512. &xorps ($inout1,$inout4);
  1513. &call ("_aesni_decrypt2");
  1514. &xorps ($inout0,$inout3); # output^=tweak
  1515. &xorps ($inout1,$inout4);
  1516. &movups (&QWP(16*0,$out),$inout0); # write output
  1517. &movups (&QWP(16*1,$out),$inout1);
  1518. &lea ($out,&DWP(16*2,$out));
  1519. &movdqa ($tweak,$inout4); # last tweak
  1520. &jmp (&label("xts_dec_done"));
  1521. &set_label("xts_dec_three",16);
  1522. &movaps ($inout5,$tweak); # put aside last tweak
  1523. &movups ($inout0,&QWP(16*0,$inp)); # load input
  1524. &movups ($inout1,&QWP(16*1,$inp));
  1525. &movups ($inout2,&QWP(16*2,$inp));
  1526. &lea ($inp,&DWP(16*3,$inp));
  1527. &xorps ($inout0,$inout3); # input^=tweak
  1528. &xorps ($inout1,$inout4);
  1529. &xorps ($inout2,$inout5);
  1530. &call ("_aesni_decrypt3");
  1531. &xorps ($inout0,$inout3); # output^=tweak
  1532. &xorps ($inout1,$inout4);
  1533. &xorps ($inout2,$inout5);
  1534. &movups (&QWP(16*0,$out),$inout0); # write output
  1535. &movups (&QWP(16*1,$out),$inout1);
  1536. &movups (&QWP(16*2,$out),$inout2);
  1537. &lea ($out,&DWP(16*3,$out));
  1538. &movdqa ($tweak,$inout5); # last tweak
  1539. &jmp (&label("xts_dec_done"));
  1540. &set_label("xts_dec_four",16);
  1541. &movaps ($inout4,$tweak); # put aside last tweak
  1542. &movups ($inout0,&QWP(16*0,$inp)); # load input
  1543. &movups ($inout1,&QWP(16*1,$inp));
  1544. &movups ($inout2,&QWP(16*2,$inp));
  1545. &xorps ($inout0,&QWP(16*0,"esp")); # input^=tweak
  1546. &movups ($inout3,&QWP(16*3,$inp));
  1547. &lea ($inp,&DWP(16*4,$inp));
  1548. &xorps ($inout1,&QWP(16*1,"esp"));
  1549. &xorps ($inout2,$inout5);
  1550. &xorps ($inout3,$inout4);
  1551. &call ("_aesni_decrypt4");
  1552. &xorps ($inout0,&QWP(16*0,"esp")); # output^=tweak
  1553. &xorps ($inout1,&QWP(16*1,"esp"));
  1554. &xorps ($inout2,$inout5);
  1555. &movups (&QWP(16*0,$out),$inout0); # write output
  1556. &xorps ($inout3,$inout4);
  1557. &movups (&QWP(16*1,$out),$inout1);
  1558. &movups (&QWP(16*2,$out),$inout2);
  1559. &movups (&QWP(16*3,$out),$inout3);
  1560. &lea ($out,&DWP(16*4,$out));
  1561. &movdqa ($tweak,$inout4); # last tweak
  1562. &jmp (&label("xts_dec_done"));
  1563. &set_label("xts_dec_done6x",16); # $tweak is pre-calculated
  1564. &mov ($len,&DWP(16*7+0,"esp")); # restore original $len
  1565. &and ($len,15);
  1566. &jz (&label("xts_dec_ret"));
  1567. &mov (&DWP(16*7+0,"esp"),$len); # save $len%16
  1568. &jmp (&label("xts_dec_only_one_more"));
  1569. &set_label("xts_dec_done",16);
  1570. &mov ($len,&DWP(16*7+0,"esp")); # restore original $len
  1571. &pxor ($twtmp,$twtmp);
  1572. &and ($len,15);
  1573. &jz (&label("xts_dec_ret"));
  1574. &pcmpgtd($twtmp,$tweak); # broadcast upper bits
  1575. &mov (&DWP(16*7+0,"esp"),$len); # save $len%16
  1576. &pshufd ($twres,$twtmp,0x13);
  1577. &pxor ($twtmp,$twtmp);
  1578. &movdqa ($twmask,&QWP(16*6,"esp"));
  1579. &paddq ($tweak,$tweak); # &psllq($tweak,1);
  1580. &pand ($twres,$twmask); # isolate carry and residue
  1581. &pcmpgtd($twtmp,$tweak); # broadcast upper bits
  1582. &pxor ($tweak,$twres);
  1583. &set_label("xts_dec_only_one_more");
  1584. &pshufd ($inout3,$twtmp,0x13);
  1585. &movdqa ($inout4,$tweak); # put aside previous tweak
  1586. &paddq ($tweak,$tweak); # &psllq($tweak,1);
  1587. &pand ($inout3,$twmask); # isolate carry and residue
  1588. &pxor ($inout3,$tweak);
  1589. &mov ($key,$key_); # restore $key
  1590. &mov ($rounds,$rounds_); # restore $rounds
  1591. &movups ($inout0,&QWP(0,$inp)); # load input
  1592. &xorps ($inout0,$inout3); # input^=tweak
  1593. if ($inline)
  1594. { &aesni_inline_generate1("dec"); }
  1595. else
  1596. { &call ("_aesni_decrypt1"); }
  1597. &xorps ($inout0,$inout3); # output^=tweak
  1598. &movups (&QWP(0,$out),$inout0); # write output
  1599. &set_label("xts_dec_steal");
  1600. &movz ($rounds,&BP(16,$inp));
  1601. &movz ($key,&BP(0,$out));
  1602. &lea ($inp,&DWP(1,$inp));
  1603. &mov (&BP(0,$out),&LB($rounds));
  1604. &mov (&BP(16,$out),&LB($key));
  1605. &lea ($out,&DWP(1,$out));
  1606. &sub ($len,1);
  1607. &jnz (&label("xts_dec_steal"));
  1608. &sub ($out,&DWP(16*7+0,"esp")); # rewind $out
  1609. &mov ($key,$key_); # restore $key
  1610. &mov ($rounds,$rounds_); # restore $rounds
  1611. &movups ($inout0,&QWP(0,$out)); # load input
  1612. &xorps ($inout0,$inout4); # input^=tweak
  1613. if ($inline)
  1614. { &aesni_inline_generate1("dec"); }
  1615. else
  1616. { &call ("_aesni_decrypt1"); }
  1617. &xorps ($inout0,$inout4); # output^=tweak
  1618. &movups (&QWP(0,$out),$inout0); # write output
  1619. &set_label("xts_dec_ret");
  1620. &pxor ("xmm0","xmm0"); # clear register bank
  1621. &pxor ("xmm1","xmm1");
  1622. &pxor ("xmm2","xmm2");
  1623. &movdqa (&QWP(16*0,"esp"),"xmm0"); # clear stack
  1624. &pxor ("xmm3","xmm3");
  1625. &movdqa (&QWP(16*1,"esp"),"xmm0");
  1626. &pxor ("xmm4","xmm4");
  1627. &movdqa (&QWP(16*2,"esp"),"xmm0");
  1628. &pxor ("xmm5","xmm5");
  1629. &movdqa (&QWP(16*3,"esp"),"xmm0");
  1630. &pxor ("xmm6","xmm6");
  1631. &movdqa (&QWP(16*4,"esp"),"xmm0");
  1632. &pxor ("xmm7","xmm7");
  1633. &movdqa (&QWP(16*5,"esp"),"xmm0");
  1634. &mov ("esp",&DWP(16*7+4,"esp")); # restore %esp
  1635. &function_end("aesni_xts_decrypt");
  1636. }
  1637. ######################################################################
  1638. # void aesni_ocb_[en|de]crypt(const char *inp, char *out, size_t blocks,
  1639. # const AES_KEY *key, unsigned int start_block_num,
  1640. # unsigned char offset_i[16], const unsigned char L_[][16],
  1641. # unsigned char checksum[16]);
  1642. #
  1643. {
  1644. # offsets within stack frame
  1645. my $checksum = 16*6;
  1646. my ($key_off,$rounds_off,$out_off,$end_off,$esp_off)=map(16*7+4*$_,(0..4));
  1647. # reassigned registers
  1648. my ($l_,$block,$i1,$i3,$i5) = ($rounds_,$key_,$rounds,$len,$out);
  1649. # $l_, $blocks, $inp, $key are permanently allocated in registers;
  1650. # remaining non-volatile ones are offloaded to stack, which even
  1651. # stay invariant after written to stack.
  1652. &function_begin("aesni_ocb_encrypt");
  1653. &mov ($rounds,&wparam(5)); # &offset_i
  1654. &mov ($rounds_,&wparam(7)); # &checksum
  1655. &mov ($inp,&wparam(0));
  1656. &mov ($out,&wparam(1));
  1657. &mov ($len,&wparam(2));
  1658. &mov ($key,&wparam(3));
  1659. &movdqu ($rndkey0,&QWP(0,$rounds)); # load offset_i
  1660. &mov ($block,&wparam(4)); # start_block_num
  1661. &movdqu ($rndkey1,&QWP(0,$rounds_)); # load checksum
  1662. &mov ($l_,&wparam(6)); # L_
  1663. &mov ($rounds,"esp");
  1664. &sub ("esp",$esp_off+4); # alloca
  1665. &and ("esp",-16); # align stack
  1666. &sub ($out,$inp);
  1667. &shl ($len,4);
  1668. &lea ($len,&DWP(-16*6,$inp,$len)); # end of input - 16*6
  1669. &mov (&DWP($out_off,"esp"),$out);
  1670. &mov (&DWP($end_off,"esp"),$len);
  1671. &mov (&DWP($esp_off,"esp"),$rounds);
  1672. &mov ($rounds,&DWP(240,$key));
  1673. &test ($block,1);
  1674. &jnz (&label("odd"));
  1675. &bsf ($i3,$block);
  1676. &add ($block,1);
  1677. &shl ($i3,4);
  1678. &movdqu ($inout5,&QWP(0,$l_,$i3));
  1679. &mov ($i3,$key); # put aside key
  1680. &movdqu ($inout0,&QWP(16*0,$inp)); # load input
  1681. &lea ($inp,&DWP(16,$inp));
  1682. &pxor ($inout5,$rndkey0); # ^ last offset_i
  1683. &pxor ($rndkey1,$inout0); # checksum
  1684. &pxor ($inout0,$inout5); # ^ offset_i
  1685. &movdqa ($inout4,$rndkey1);
  1686. if ($inline)
  1687. { &aesni_inline_generate1("enc"); }
  1688. else
  1689. { &call ("_aesni_encrypt1"); }
  1690. &xorps ($inout0,$inout5); # ^ offset_i
  1691. &movdqa ($rndkey0,$inout5); # pass last offset_i
  1692. &movdqa ($rndkey1,$inout4); # pass the checksum
  1693. &movups (&QWP(-16,$out,$inp),$inout0); # store output
  1694. &mov ($rounds,&DWP(240,$i3));
  1695. &mov ($key,$i3); # restore key
  1696. &mov ($len,&DWP($end_off,"esp"));
  1697. &set_label("odd");
  1698. &shl ($rounds,4);
  1699. &mov ($out,16);
  1700. &sub ($out,$rounds); # twisted rounds
  1701. &mov (&DWP($key_off,"esp"),$key);
  1702. &lea ($key,&DWP(32,$key,$rounds)); # end of key schedule
  1703. &mov (&DWP($rounds_off,"esp"),$out);
  1704. &cmp ($inp,$len);
  1705. &ja (&label("short"));
  1706. &jmp (&label("grandloop"));
  1707. &set_label("grandloop",32);
  1708. &lea ($i1,&DWP(1,$block));
  1709. &lea ($i3,&DWP(3,$block));
  1710. &lea ($i5,&DWP(5,$block));
  1711. &add ($block,6);
  1712. &bsf ($i1,$i1);
  1713. &bsf ($i3,$i3);
  1714. &bsf ($i5,$i5);
  1715. &shl ($i1,4);
  1716. &shl ($i3,4);
  1717. &shl ($i5,4);
  1718. &movdqu ($inout0,&QWP(0,$l_));
  1719. &movdqu ($inout1,&QWP(0,$l_,$i1));
  1720. &mov ($rounds,&DWP($rounds_off,"esp"));
  1721. &movdqa ($inout2,$inout0);
  1722. &movdqu ($inout3,&QWP(0,$l_,$i3));
  1723. &movdqa ($inout4,$inout0);
  1724. &movdqu ($inout5,&QWP(0,$l_,$i5));
  1725. &pxor ($inout0,$rndkey0); # ^ last offset_i
  1726. &pxor ($inout1,$inout0);
  1727. &movdqa (&QWP(16*0,"esp"),$inout0);
  1728. &pxor ($inout2,$inout1);
  1729. &movdqa (&QWP(16*1,"esp"),$inout1);
  1730. &pxor ($inout3,$inout2);
  1731. &movdqa (&QWP(16*2,"esp"),$inout2);
  1732. &pxor ($inout4,$inout3);
  1733. &movdqa (&QWP(16*3,"esp"),$inout3);
  1734. &pxor ($inout5,$inout4);
  1735. &movdqa (&QWP(16*4,"esp"),$inout4);
  1736. &movdqa (&QWP(16*5,"esp"),$inout5);
  1737. &$movekey ($rndkey0,&QWP(-48,$key,$rounds));
  1738. &movdqu ($inout0,&QWP(16*0,$inp)); # load input
  1739. &movdqu ($inout1,&QWP(16*1,$inp));
  1740. &movdqu ($inout2,&QWP(16*2,$inp));
  1741. &movdqu ($inout3,&QWP(16*3,$inp));
  1742. &movdqu ($inout4,&QWP(16*4,$inp));
  1743. &movdqu ($inout5,&QWP(16*5,$inp));
  1744. &lea ($inp,&DWP(16*6,$inp));
  1745. &pxor ($rndkey1,$inout0); # checksum
  1746. &pxor ($inout0,$rndkey0); # ^ roundkey[0]
  1747. &pxor ($rndkey1,$inout1);
  1748. &pxor ($inout1,$rndkey0);
  1749. &pxor ($rndkey1,$inout2);
  1750. &pxor ($inout2,$rndkey0);
  1751. &pxor ($rndkey1,$inout3);
  1752. &pxor ($inout3,$rndkey0);
  1753. &pxor ($rndkey1,$inout4);
  1754. &pxor ($inout4,$rndkey0);
  1755. &pxor ($rndkey1,$inout5);
  1756. &pxor ($inout5,$rndkey0);
  1757. &movdqa (&QWP($checksum,"esp"),$rndkey1);
  1758. &$movekey ($rndkey1,&QWP(-32,$key,$rounds));
  1759. &pxor ($inout0,&QWP(16*0,"esp")); # ^ offset_i
  1760. &pxor ($inout1,&QWP(16*1,"esp"));
  1761. &pxor ($inout2,&QWP(16*2,"esp"));
  1762. &pxor ($inout3,&QWP(16*3,"esp"));
  1763. &pxor ($inout4,&QWP(16*4,"esp"));
  1764. &pxor ($inout5,&QWP(16*5,"esp"));
  1765. &$movekey ($rndkey0,&QWP(-16,$key,$rounds));
  1766. &aesenc ($inout0,$rndkey1);
  1767. &aesenc ($inout1,$rndkey1);
  1768. &aesenc ($inout2,$rndkey1);
  1769. &aesenc ($inout3,$rndkey1);
  1770. &aesenc ($inout4,$rndkey1);
  1771. &aesenc ($inout5,$rndkey1);
  1772. &mov ($out,&DWP($out_off,"esp"));
  1773. &mov ($len,&DWP($end_off,"esp"));
  1774. &call ("_aesni_encrypt6_enter");
  1775. &movdqa ($rndkey0,&QWP(16*5,"esp")); # pass last offset_i
  1776. &pxor ($inout0,&QWP(16*0,"esp")); # ^ offset_i
  1777. &pxor ($inout1,&QWP(16*1,"esp"));
  1778. &pxor ($inout2,&QWP(16*2,"esp"));
  1779. &pxor ($inout3,&QWP(16*3,"esp"));
  1780. &pxor ($inout4,&QWP(16*4,"esp"));
  1781. &pxor ($inout5,$rndkey0);
  1782. &movdqa ($rndkey1,&QWP($checksum,"esp"));# pass the checksum
  1783. &movdqu (&QWP(-16*6,$out,$inp),$inout0);# store output
  1784. &movdqu (&QWP(-16*5,$out,$inp),$inout1);
  1785. &movdqu (&QWP(-16*4,$out,$inp),$inout2);
  1786. &movdqu (&QWP(-16*3,$out,$inp),$inout3);
  1787. &movdqu (&QWP(-16*2,$out,$inp),$inout4);
  1788. &movdqu (&QWP(-16*1,$out,$inp),$inout5);
  1789. &cmp ($inp,$len); # done yet?
  1790. &jbe (&label("grandloop"));
  1791. &set_label("short");
  1792. &add ($len,16*6);
  1793. &sub ($len,$inp);
  1794. &jz (&label("done"));
  1795. &cmp ($len,16*2);
  1796. &jb (&label("one"));
  1797. &je (&label("two"));
  1798. &cmp ($len,16*4);
  1799. &jb (&label("three"));
  1800. &je (&label("four"));
  1801. &lea ($i1,&DWP(1,$block));
  1802. &lea ($i3,&DWP(3,$block));
  1803. &bsf ($i1,$i1);
  1804. &bsf ($i3,$i3);
  1805. &shl ($i1,4);
  1806. &shl ($i3,4);
  1807. &movdqu ($inout0,&QWP(0,$l_));
  1808. &movdqu ($inout1,&QWP(0,$l_,$i1));
  1809. &mov ($rounds,&DWP($rounds_off,"esp"));
  1810. &movdqa ($inout2,$inout0);
  1811. &movdqu ($inout3,&QWP(0,$l_,$i3));
  1812. &movdqa ($inout4,$inout0);
  1813. &pxor ($inout0,$rndkey0); # ^ last offset_i
  1814. &pxor ($inout1,$inout0);
  1815. &movdqa (&QWP(16*0,"esp"),$inout0);
  1816. &pxor ($inout2,$inout1);
  1817. &movdqa (&QWP(16*1,"esp"),$inout1);
  1818. &pxor ($inout3,$inout2);
  1819. &movdqa (&QWP(16*2,"esp"),$inout2);
  1820. &pxor ($inout4,$inout3);
  1821. &movdqa (&QWP(16*3,"esp"),$inout3);
  1822. &pxor ($inout5,$inout4);
  1823. &movdqa (&QWP(16*4,"esp"),$inout4);
  1824. &$movekey ($rndkey0,&QWP(-48,$key,$rounds));
  1825. &movdqu ($inout0,&QWP(16*0,$inp)); # load input
  1826. &movdqu ($inout1,&QWP(16*1,$inp));
  1827. &movdqu ($inout2,&QWP(16*2,$inp));
  1828. &movdqu ($inout3,&QWP(16*3,$inp));
  1829. &movdqu ($inout4,&QWP(16*4,$inp));
  1830. &pxor ($inout5,$inout5);
  1831. &pxor ($rndkey1,$inout0); # checksum
  1832. &pxor ($inout0,$rndkey0); # ^ roundkey[0]
  1833. &pxor ($rndkey1,$inout1);
  1834. &pxor ($inout1,$rndkey0);
  1835. &pxor ($rndkey1,$inout2);
  1836. &pxor ($inout2,$rndkey0);
  1837. &pxor ($rndkey1,$inout3);
  1838. &pxor ($inout3,$rndkey0);
  1839. &pxor ($rndkey1,$inout4);
  1840. &pxor ($inout4,$rndkey0);
  1841. &movdqa (&QWP($checksum,"esp"),$rndkey1);
  1842. &$movekey ($rndkey1,&QWP(-32,$key,$rounds));
  1843. &pxor ($inout0,&QWP(16*0,"esp")); # ^ offset_i
  1844. &pxor ($inout1,&QWP(16*1,"esp"));
  1845. &pxor ($inout2,&QWP(16*2,"esp"));
  1846. &pxor ($inout3,&QWP(16*3,"esp"));
  1847. &pxor ($inout4,&QWP(16*4,"esp"));
  1848. &$movekey ($rndkey0,&QWP(-16,$key,$rounds));
  1849. &aesenc ($inout0,$rndkey1);
  1850. &aesenc ($inout1,$rndkey1);
  1851. &aesenc ($inout2,$rndkey1);
  1852. &aesenc ($inout3,$rndkey1);
  1853. &aesenc ($inout4,$rndkey1);
  1854. &aesenc ($inout5,$rndkey1);
  1855. &mov ($out,&DWP($out_off,"esp"));
  1856. &call ("_aesni_encrypt6_enter");
  1857. &movdqa ($rndkey0,&QWP(16*4,"esp")); # pass last offset_i
  1858. &pxor ($inout0,&QWP(16*0,"esp")); # ^ offset_i
  1859. &pxor ($inout1,&QWP(16*1,"esp"));
  1860. &pxor ($inout2,&QWP(16*2,"esp"));
  1861. &pxor ($inout3,&QWP(16*3,"esp"));
  1862. &pxor ($inout4,$rndkey0);
  1863. &movdqa ($rndkey1,&QWP($checksum,"esp"));# pass the checksum
  1864. &movdqu (&QWP(16*0,$out,$inp),$inout0); # store output
  1865. &movdqu (&QWP(16*1,$out,$inp),$inout1);
  1866. &movdqu (&QWP(16*2,$out,$inp),$inout2);
  1867. &movdqu (&QWP(16*3,$out,$inp),$inout3);
  1868. &movdqu (&QWP(16*4,$out,$inp),$inout4);
  1869. &jmp (&label("done"));
  1870. &set_label("one",16);
  1871. &movdqu ($inout5,&QWP(0,$l_));
  1872. &mov ($key,&DWP($key_off,"esp")); # restore key
  1873. &movdqu ($inout0,&QWP(16*0,$inp)); # load input
  1874. &mov ($rounds,&DWP(240,$key));
  1875. &pxor ($inout5,$rndkey0); # ^ last offset_i
  1876. &pxor ($rndkey1,$inout0); # checksum
  1877. &pxor ($inout0,$inout5); # ^ offset_i
  1878. &movdqa ($inout4,$rndkey1);
  1879. &mov ($out,&DWP($out_off,"esp"));
  1880. if ($inline)
  1881. { &aesni_inline_generate1("enc"); }
  1882. else
  1883. { &call ("_aesni_encrypt1"); }
  1884. &xorps ($inout0,$inout5); # ^ offset_i
  1885. &movdqa ($rndkey0,$inout5); # pass last offset_i
  1886. &movdqa ($rndkey1,$inout4); # pass the checksum
  1887. &movups (&QWP(0,$out,$inp),$inout0);
  1888. &jmp (&label("done"));
  1889. &set_label("two",16);
  1890. &lea ($i1,&DWP(1,$block));
  1891. &mov ($key,&DWP($key_off,"esp")); # restore key
  1892. &bsf ($i1,$i1);
  1893. &shl ($i1,4);
  1894. &movdqu ($inout4,&QWP(0,$l_));
  1895. &movdqu ($inout5,&QWP(0,$l_,$i1));
  1896. &movdqu ($inout0,&QWP(16*0,$inp)); # load input
  1897. &movdqu ($inout1,&QWP(16*1,$inp));
  1898. &mov ($rounds,&DWP(240,$key));
  1899. &pxor ($inout4,$rndkey0); # ^ last offset_i
  1900. &pxor ($inout5,$inout4);
  1901. &pxor ($rndkey1,$inout0); # checksum
  1902. &pxor ($inout0,$inout4); # ^ offset_i
  1903. &pxor ($rndkey1,$inout1);
  1904. &pxor ($inout1,$inout5);
  1905. &movdqa ($inout3,$rndkey1)
  1906. &mov ($out,&DWP($out_off,"esp"));
  1907. &call ("_aesni_encrypt2");
  1908. &xorps ($inout0,$inout4); # ^ offset_i
  1909. &xorps ($inout1,$inout5);
  1910. &movdqa ($rndkey0,$inout5); # pass last offset_i
  1911. &movdqa ($rndkey1,$inout3); # pass the checksum
  1912. &movups (&QWP(16*0,$out,$inp),$inout0); # store output
  1913. &movups (&QWP(16*1,$out,$inp),$inout1);
  1914. &jmp (&label("done"));
  1915. &set_label("three",16);
  1916. &lea ($i1,&DWP(1,$block));
  1917. &mov ($key,&DWP($key_off,"esp")); # restore key
  1918. &bsf ($i1,$i1);
  1919. &shl ($i1,4);
  1920. &movdqu ($inout3,&QWP(0,$l_));
  1921. &movdqu ($inout4,&QWP(0,$l_,$i1));
  1922. &movdqa ($inout5,$inout3);
  1923. &movdqu ($inout0,&QWP(16*0,$inp)); # load input
  1924. &movdqu ($inout1,&QWP(16*1,$inp));
  1925. &movdqu ($inout2,&QWP(16*2,$inp));
  1926. &mov ($rounds,&DWP(240,$key));
  1927. &pxor ($inout3,$rndkey0); # ^ last offset_i
  1928. &pxor ($inout4,$inout3);
  1929. &pxor ($inout5,$inout4);
  1930. &pxor ($rndkey1,$inout0); # checksum
  1931. &pxor ($inout0,$inout3); # ^ offset_i
  1932. &pxor ($rndkey1,$inout1);
  1933. &pxor ($inout1,$inout4);
  1934. &pxor ($rndkey1,$inout2);
  1935. &pxor ($inout2,$inout5);
  1936. &movdqa (&QWP($checksum,"esp"),$rndkey1);
  1937. &mov ($out,&DWP($out_off,"esp"));
  1938. &call ("_aesni_encrypt3");
  1939. &xorps ($inout0,$inout3); # ^ offset_i
  1940. &xorps ($inout1,$inout4);
  1941. &xorps ($inout2,$inout5);
  1942. &movdqa ($rndkey0,$inout5); # pass last offset_i
  1943. &movdqa ($rndkey1,&QWP($checksum,"esp"));# pass the checksum
  1944. &movups (&QWP(16*0,$out,$inp),$inout0); # store output
  1945. &movups (&QWP(16*1,$out,$inp),$inout1);
  1946. &movups (&QWP(16*2,$out,$inp),$inout2);
  1947. &jmp (&label("done"));
  1948. &set_label("four",16);
  1949. &lea ($i1,&DWP(1,$block));
  1950. &lea ($i3,&DWP(3,$block));
  1951. &bsf ($i1,$i1);
  1952. &bsf ($i3,$i3);
  1953. &mov ($key,&DWP($key_off,"esp")); # restore key
  1954. &shl ($i1,4);
  1955. &shl ($i3,4);
  1956. &movdqu ($inout2,&QWP(0,$l_));
  1957. &movdqu ($inout3,&QWP(0,$l_,$i1));
  1958. &movdqa ($inout4,$inout2);
  1959. &movdqu ($inout5,&QWP(0,$l_,$i3));
  1960. &pxor ($inout2,$rndkey0); # ^ last offset_i
  1961. &movdqu ($inout0,&QWP(16*0,$inp)); # load input
  1962. &pxor ($inout3,$inout2);
  1963. &movdqu ($inout1,&QWP(16*1,$inp));
  1964. &pxor ($inout4,$inout3);
  1965. &movdqa (&QWP(16*0,"esp"),$inout2);
  1966. &pxor ($inout5,$inout4);
  1967. &movdqa (&QWP(16*1,"esp"),$inout3);
  1968. &movdqu ($inout2,&QWP(16*2,$inp));
  1969. &movdqu ($inout3,&QWP(16*3,$inp));
  1970. &mov ($rounds,&DWP(240,$key));
  1971. &pxor ($rndkey1,$inout0); # checksum
  1972. &pxor ($inout0,&QWP(16*0,"esp")); # ^ offset_i
  1973. &pxor ($rndkey1,$inout1);
  1974. &pxor ($inout1,&QWP(16*1,"esp"));
  1975. &pxor ($rndkey1,$inout2);
  1976. &pxor ($inout2,$inout4);
  1977. &pxor ($rndkey1,$inout3);
  1978. &pxor ($inout3,$inout5);
  1979. &movdqa (&QWP($checksum,"esp"),$rndkey1)
  1980. &mov ($out,&DWP($out_off,"esp"));
  1981. &call ("_aesni_encrypt4");
  1982. &xorps ($inout0,&QWP(16*0,"esp")); # ^ offset_i
  1983. &xorps ($inout1,&QWP(16*1,"esp"));
  1984. &xorps ($inout2,$inout4);
  1985. &movups (&QWP(16*0,$out,$inp),$inout0); # store output
  1986. &xorps ($inout3,$inout5);
  1987. &movups (&QWP(16*1,$out,$inp),$inout1);
  1988. &movdqa ($rndkey0,$inout5); # pass last offset_i
  1989. &movups (&QWP(16*2,$out,$inp),$inout2);
  1990. &movdqa ($rndkey1,&QWP($checksum,"esp"));# pass the checksum
  1991. &movups (&QWP(16*3,$out,$inp),$inout3);
  1992. &set_label("done");
  1993. &mov ($key,&DWP($esp_off,"esp"));
  1994. &pxor ($inout0,$inout0); # clear register bank
  1995. &pxor ($inout1,$inout1);
  1996. &movdqa (&QWP(16*0,"esp"),$inout0); # clear stack
  1997. &pxor ($inout2,$inout2);
  1998. &movdqa (&QWP(16*1,"esp"),$inout0);
  1999. &pxor ($inout3,$inout3);
  2000. &movdqa (&QWP(16*2,"esp"),$inout0);
  2001. &pxor ($inout4,$inout4);
  2002. &movdqa (&QWP(16*3,"esp"),$inout0);
  2003. &pxor ($inout5,$inout5);
  2004. &movdqa (&QWP(16*4,"esp"),$inout0);
  2005. &movdqa (&QWP(16*5,"esp"),$inout0);
  2006. &movdqa (&QWP(16*6,"esp"),$inout0);
  2007. &lea ("esp",&DWP(0,$key));
  2008. &mov ($rounds,&wparam(5)); # &offset_i
  2009. &mov ($rounds_,&wparam(7)); # &checksum
  2010. &movdqu (&QWP(0,$rounds),$rndkey0);
  2011. &pxor ($rndkey0,$rndkey0);
  2012. &movdqu (&QWP(0,$rounds_),$rndkey1);
  2013. &pxor ($rndkey1,$rndkey1);
  2014. &function_end("aesni_ocb_encrypt");
  2015. &function_begin("aesni_ocb_decrypt");
  2016. &mov ($rounds,&wparam(5)); # &offset_i
  2017. &mov ($rounds_,&wparam(7)); # &checksum
  2018. &mov ($inp,&wparam(0));
  2019. &mov ($out,&wparam(1));
  2020. &mov ($len,&wparam(2));
  2021. &mov ($key,&wparam(3));
  2022. &movdqu ($rndkey0,&QWP(0,$rounds)); # load offset_i
  2023. &mov ($block,&wparam(4)); # start_block_num
  2024. &movdqu ($rndkey1,&QWP(0,$rounds_)); # load checksum
  2025. &mov ($l_,&wparam(6)); # L_
  2026. &mov ($rounds,"esp");
  2027. &sub ("esp",$esp_off+4); # alloca
  2028. &and ("esp",-16); # align stack
  2029. &sub ($out,$inp);
  2030. &shl ($len,4);
  2031. &lea ($len,&DWP(-16*6,$inp,$len)); # end of input - 16*6
  2032. &mov (&DWP($out_off,"esp"),$out);
  2033. &mov (&DWP($end_off,"esp"),$len);
  2034. &mov (&DWP($esp_off,"esp"),$rounds);
  2035. &mov ($rounds,&DWP(240,$key));
  2036. &test ($block,1);
  2037. &jnz (&label("odd"));
  2038. &bsf ($i3,$block);
  2039. &add ($block,1);
  2040. &shl ($i3,4);
  2041. &movdqu ($inout5,&QWP(0,$l_,$i3));
  2042. &mov ($i3,$key); # put aside key
  2043. &movdqu ($inout0,&QWP(16*0,$inp)); # load input
  2044. &lea ($inp,&DWP(16,$inp));
  2045. &pxor ($inout5,$rndkey0); # ^ last offset_i
  2046. &pxor ($inout0,$inout5); # ^ offset_i
  2047. &movdqa ($inout4,$rndkey1);
  2048. if ($inline)
  2049. { &aesni_inline_generate1("dec"); }
  2050. else
  2051. { &call ("_aesni_decrypt1"); }
  2052. &xorps ($inout0,$inout5); # ^ offset_i
  2053. &movaps ($rndkey1,$inout4); # pass the checksum
  2054. &movdqa ($rndkey0,$inout5); # pass last offset_i
  2055. &xorps ($rndkey1,$inout0); # checksum
  2056. &movups (&QWP(-16,$out,$inp),$inout0); # store output
  2057. &mov ($rounds,&DWP(240,$i3));
  2058. &mov ($key,$i3); # restore key
  2059. &mov ($len,&DWP($end_off,"esp"));
  2060. &set_label("odd");
  2061. &shl ($rounds,4);
  2062. &mov ($out,16);
  2063. &sub ($out,$rounds); # twisted rounds
  2064. &mov (&DWP($key_off,"esp"),$key);
  2065. &lea ($key,&DWP(32,$key,$rounds)); # end of key schedule
  2066. &mov (&DWP($rounds_off,"esp"),$out);
  2067. &cmp ($inp,$len);
  2068. &ja (&label("short"));
  2069. &jmp (&label("grandloop"));
  2070. &set_label("grandloop",32);
  2071. &lea ($i1,&DWP(1,$block));
  2072. &lea ($i3,&DWP(3,$block));
  2073. &lea ($i5,&DWP(5,$block));
  2074. &add ($block,6);
  2075. &bsf ($i1,$i1);
  2076. &bsf ($i3,$i3);
  2077. &bsf ($i5,$i5);
  2078. &shl ($i1,4);
  2079. &shl ($i3,4);
  2080. &shl ($i5,4);
  2081. &movdqu ($inout0,&QWP(0,$l_));
  2082. &movdqu ($inout1,&QWP(0,$l_,$i1));
  2083. &mov ($rounds,&DWP($rounds_off,"esp"));
  2084. &movdqa ($inout2,$inout0);
  2085. &movdqu ($inout3,&QWP(0,$l_,$i3));
  2086. &movdqa ($inout4,$inout0);
  2087. &movdqu ($inout5,&QWP(0,$l_,$i5));
  2088. &pxor ($inout0,$rndkey0); # ^ last offset_i
  2089. &pxor ($inout1,$inout0);
  2090. &movdqa (&QWP(16*0,"esp"),$inout0);
  2091. &pxor ($inout2,$inout1);
  2092. &movdqa (&QWP(16*1,"esp"),$inout1);
  2093. &pxor ($inout3,$inout2);
  2094. &movdqa (&QWP(16*2,"esp"),$inout2);
  2095. &pxor ($inout4,$inout3);
  2096. &movdqa (&QWP(16*3,"esp"),$inout3);
  2097. &pxor ($inout5,$inout4);
  2098. &movdqa (&QWP(16*4,"esp"),$inout4);
  2099. &movdqa (&QWP(16*5,"esp"),$inout5);
  2100. &$movekey ($rndkey0,&QWP(-48,$key,$rounds));
  2101. &movdqu ($inout0,&QWP(16*0,$inp)); # load input
  2102. &movdqu ($inout1,&QWP(16*1,$inp));
  2103. &movdqu ($inout2,&QWP(16*2,$inp));
  2104. &movdqu ($inout3,&QWP(16*3,$inp));
  2105. &movdqu ($inout4,&QWP(16*4,$inp));
  2106. &movdqu ($inout5,&QWP(16*5,$inp));
  2107. &lea ($inp,&DWP(16*6,$inp));
  2108. &movdqa (&QWP($checksum,"esp"),$rndkey1);
  2109. &pxor ($inout0,$rndkey0); # ^ roundkey[0]
  2110. &pxor ($inout1,$rndkey0);
  2111. &pxor ($inout2,$rndkey0);
  2112. &pxor ($inout3,$rndkey0);
  2113. &pxor ($inout4,$rndkey0);
  2114. &pxor ($inout5,$rndkey0);
  2115. &$movekey ($rndkey1,&QWP(-32,$key,$rounds));
  2116. &pxor ($inout0,&QWP(16*0,"esp")); # ^ offset_i
  2117. &pxor ($inout1,&QWP(16*1,"esp"));
  2118. &pxor ($inout2,&QWP(16*2,"esp"));
  2119. &pxor ($inout3,&QWP(16*3,"esp"));
  2120. &pxor ($inout4,&QWP(16*4,"esp"));
  2121. &pxor ($inout5,&QWP(16*5,"esp"));
  2122. &$movekey ($rndkey0,&QWP(-16,$key,$rounds));
  2123. &aesdec ($inout0,$rndkey1);
  2124. &aesdec ($inout1,$rndkey1);
  2125. &aesdec ($inout2,$rndkey1);
  2126. &aesdec ($inout3,$rndkey1);
  2127. &aesdec ($inout4,$rndkey1);
  2128. &aesdec ($inout5,$rndkey1);
  2129. &mov ($out,&DWP($out_off,"esp"));
  2130. &mov ($len,&DWP($end_off,"esp"));
  2131. &call ("_aesni_decrypt6_enter");
  2132. &movdqa ($rndkey0,&QWP(16*5,"esp")); # pass last offset_i
  2133. &pxor ($inout0,&QWP(16*0,"esp")); # ^ offset_i
  2134. &movdqa ($rndkey1,&QWP($checksum,"esp"));
  2135. &pxor ($inout1,&QWP(16*1,"esp"));
  2136. &pxor ($inout2,&QWP(16*2,"esp"));
  2137. &pxor ($inout3,&QWP(16*3,"esp"));
  2138. &pxor ($inout4,&QWP(16*4,"esp"));
  2139. &pxor ($inout5,$rndkey0);
  2140. &pxor ($rndkey1,$inout0); # checksum
  2141. &movdqu (&QWP(-16*6,$out,$inp),$inout0);# store output
  2142. &pxor ($rndkey1,$inout1);
  2143. &movdqu (&QWP(-16*5,$out,$inp),$inout1);
  2144. &pxor ($rndkey1,$inout2);
  2145. &movdqu (&QWP(-16*4,$out,$inp),$inout2);
  2146. &pxor ($rndkey1,$inout3);
  2147. &movdqu (&QWP(-16*3,$out,$inp),$inout3);
  2148. &pxor ($rndkey1,$inout4);
  2149. &movdqu (&QWP(-16*2,$out,$inp),$inout4);
  2150. &pxor ($rndkey1,$inout5);
  2151. &movdqu (&QWP(-16*1,$out,$inp),$inout5);
  2152. &cmp ($inp,$len); # done yet?
  2153. &jbe (&label("grandloop"));
  2154. &set_label("short");
  2155. &add ($len,16*6);
  2156. &sub ($len,$inp);
  2157. &jz (&label("done"));
  2158. &cmp ($len,16*2);
  2159. &jb (&label("one"));
  2160. &je (&label("two"));
  2161. &cmp ($len,16*4);
  2162. &jb (&label("three"));
  2163. &je (&label("four"));
  2164. &lea ($i1,&DWP(1,$block));
  2165. &lea ($i3,&DWP(3,$block));
  2166. &bsf ($i1,$i1);
  2167. &bsf ($i3,$i3);
  2168. &shl ($i1,4);
  2169. &shl ($i3,4);
  2170. &movdqu ($inout0,&QWP(0,$l_));
  2171. &movdqu ($inout1,&QWP(0,$l_,$i1));
  2172. &mov ($rounds,&DWP($rounds_off,"esp"));
  2173. &movdqa ($inout2,$inout0);
  2174. &movdqu ($inout3,&QWP(0,$l_,$i3));
  2175. &movdqa ($inout4,$inout0);
  2176. &pxor ($inout0,$rndkey0); # ^ last offset_i
  2177. &pxor ($inout1,$inout0);
  2178. &movdqa (&QWP(16*0,"esp"),$inout0);
  2179. &pxor ($inout2,$inout1);
  2180. &movdqa (&QWP(16*1,"esp"),$inout1);
  2181. &pxor ($inout3,$inout2);
  2182. &movdqa (&QWP(16*2,"esp"),$inout2);
  2183. &pxor ($inout4,$inout3);
  2184. &movdqa (&QWP(16*3,"esp"),$inout3);
  2185. &pxor ($inout5,$inout4);
  2186. &movdqa (&QWP(16*4,"esp"),$inout4);
  2187. &$movekey ($rndkey0,&QWP(-48,$key,$rounds));
  2188. &movdqu ($inout0,&QWP(16*0,$inp)); # load input
  2189. &movdqu ($inout1,&QWP(16*1,$inp));
  2190. &movdqu ($inout2,&QWP(16*2,$inp));
  2191. &movdqu ($inout3,&QWP(16*3,$inp));
  2192. &movdqu ($inout4,&QWP(16*4,$inp));
  2193. &pxor ($inout5,$inout5);
  2194. &movdqa (&QWP($checksum,"esp"),$rndkey1);
  2195. &pxor ($inout0,$rndkey0); # ^ roundkey[0]
  2196. &pxor ($inout1,$rndkey0);
  2197. &pxor ($inout2,$rndkey0);
  2198. &pxor ($inout3,$rndkey0);
  2199. &pxor ($inout4,$rndkey0);
  2200. &$movekey ($rndkey1,&QWP(-32,$key,$rounds));
  2201. &pxor ($inout0,&QWP(16*0,"esp")); # ^ offset_i
  2202. &pxor ($inout1,&QWP(16*1,"esp"));
  2203. &pxor ($inout2,&QWP(16*2,"esp"));
  2204. &pxor ($inout3,&QWP(16*3,"esp"));
  2205. &pxor ($inout4,&QWP(16*4,"esp"));
  2206. &$movekey ($rndkey0,&QWP(-16,$key,$rounds));
  2207. &aesdec ($inout0,$rndkey1);
  2208. &aesdec ($inout1,$rndkey1);
  2209. &aesdec ($inout2,$rndkey1);
  2210. &aesdec ($inout3,$rndkey1);
  2211. &aesdec ($inout4,$rndkey1);
  2212. &aesdec ($inout5,$rndkey1);
  2213. &mov ($out,&DWP($out_off,"esp"));
  2214. &call ("_aesni_decrypt6_enter");
  2215. &movdqa ($rndkey0,&QWP(16*4,"esp")); # pass last offset_i
  2216. &pxor ($inout0,&QWP(16*0,"esp")); # ^ offset_i
  2217. &movdqa ($rndkey1,&QWP($checksum,"esp"));
  2218. &pxor ($inout1,&QWP(16*1,"esp"));
  2219. &pxor ($inout2,&QWP(16*2,"esp"));
  2220. &pxor ($inout3,&QWP(16*3,"esp"));
  2221. &pxor ($inout4,$rndkey0);
  2222. &pxor ($rndkey1,$inout0); # checksum
  2223. &movdqu (&QWP(16*0,$out,$inp),$inout0); # store output
  2224. &pxor ($rndkey1,$inout1);
  2225. &movdqu (&QWP(16*1,$out,$inp),$inout1);
  2226. &pxor ($rndkey1,$inout2);
  2227. &movdqu (&QWP(16*2,$out,$inp),$inout2);
  2228. &pxor ($rndkey1,$inout3);
  2229. &movdqu (&QWP(16*3,$out,$inp),$inout3);
  2230. &pxor ($rndkey1,$inout4);
  2231. &movdqu (&QWP(16*4,$out,$inp),$inout4);
  2232. &jmp (&label("done"));
  2233. &set_label("one",16);
  2234. &movdqu ($inout5,&QWP(0,$l_));
  2235. &mov ($key,&DWP($key_off,"esp")); # restore key
  2236. &movdqu ($inout0,&QWP(16*0,$inp)); # load input
  2237. &mov ($rounds,&DWP(240,$key));
  2238. &pxor ($inout5,$rndkey0); # ^ last offset_i
  2239. &pxor ($inout0,$inout5); # ^ offset_i
  2240. &movdqa ($inout4,$rndkey1);
  2241. &mov ($out,&DWP($out_off,"esp"));
  2242. if ($inline)
  2243. { &aesni_inline_generate1("dec"); }
  2244. else
  2245. { &call ("_aesni_decrypt1"); }
  2246. &xorps ($inout0,$inout5); # ^ offset_i
  2247. &movaps ($rndkey1,$inout4); # pass the checksum
  2248. &movdqa ($rndkey0,$inout5); # pass last offset_i
  2249. &xorps ($rndkey1,$inout0); # checksum
  2250. &movups (&QWP(0,$out,$inp),$inout0);
  2251. &jmp (&label("done"));
  2252. &set_label("two",16);
  2253. &lea ($i1,&DWP(1,$block));
  2254. &mov ($key,&DWP($key_off,"esp")); # restore key
  2255. &bsf ($i1,$i1);
  2256. &shl ($i1,4);
  2257. &movdqu ($inout4,&QWP(0,$l_));
  2258. &movdqu ($inout5,&QWP(0,$l_,$i1));
  2259. &movdqu ($inout0,&QWP(16*0,$inp)); # load input
  2260. &movdqu ($inout1,&QWP(16*1,$inp));
  2261. &mov ($rounds,&DWP(240,$key));
  2262. &movdqa ($inout3,$rndkey1);
  2263. &pxor ($inout4,$rndkey0); # ^ last offset_i
  2264. &pxor ($inout5,$inout4);
  2265. &pxor ($inout0,$inout4); # ^ offset_i
  2266. &pxor ($inout1,$inout5);
  2267. &mov ($out,&DWP($out_off,"esp"));
  2268. &call ("_aesni_decrypt2");
  2269. &xorps ($inout0,$inout4); # ^ offset_i
  2270. &xorps ($inout1,$inout5);
  2271. &movdqa ($rndkey0,$inout5); # pass last offset_i
  2272. &xorps ($inout3,$inout0); # checksum
  2273. &movups (&QWP(16*0,$out,$inp),$inout0); # store output
  2274. &xorps ($inout3,$inout1);
  2275. &movups (&QWP(16*1,$out,$inp),$inout1);
  2276. &movaps ($rndkey1,$inout3); # pass the checksum
  2277. &jmp (&label("done"));
  2278. &set_label("three",16);
  2279. &lea ($i1,&DWP(1,$block));
  2280. &mov ($key,&DWP($key_off,"esp")); # restore key
  2281. &bsf ($i1,$i1);
  2282. &shl ($i1,4);
  2283. &movdqu ($inout3,&QWP(0,$l_));
  2284. &movdqu ($inout4,&QWP(0,$l_,$i1));
  2285. &movdqa ($inout5,$inout3);
  2286. &movdqu ($inout0,&QWP(16*0,$inp)); # load input
  2287. &movdqu ($inout1,&QWP(16*1,$inp));
  2288. &movdqu ($inout2,&QWP(16*2,$inp));
  2289. &mov ($rounds,&DWP(240,$key));
  2290. &movdqa (&QWP($checksum,"esp"),$rndkey1);
  2291. &pxor ($inout3,$rndkey0); # ^ last offset_i
  2292. &pxor ($inout4,$inout3);
  2293. &pxor ($inout5,$inout4);
  2294. &pxor ($inout0,$inout3); # ^ offset_i
  2295. &pxor ($inout1,$inout4);
  2296. &pxor ($inout2,$inout5);
  2297. &mov ($out,&DWP($out_off,"esp"));
  2298. &call ("_aesni_decrypt3");
  2299. &movdqa ($rndkey1,&QWP($checksum,"esp"));# pass the checksum
  2300. &xorps ($inout0,$inout3); # ^ offset_i
  2301. &xorps ($inout1,$inout4);
  2302. &xorps ($inout2,$inout5);
  2303. &movups (&QWP(16*0,$out,$inp),$inout0); # store output
  2304. &pxor ($rndkey1,$inout0); # checksum
  2305. &movdqa ($rndkey0,$inout5); # pass last offset_i
  2306. &movups (&QWP(16*1,$out,$inp),$inout1);
  2307. &pxor ($rndkey1,$inout1);
  2308. &movups (&QWP(16*2,$out,$inp),$inout2);
  2309. &pxor ($rndkey1,$inout2);
  2310. &jmp (&label("done"));
  2311. &set_label("four",16);
  2312. &lea ($i1,&DWP(1,$block));
  2313. &lea ($i3,&DWP(3,$block));
  2314. &bsf ($i1,$i1);
  2315. &bsf ($i3,$i3);
  2316. &mov ($key,&DWP($key_off,"esp")); # restore key
  2317. &shl ($i1,4);
  2318. &shl ($i3,4);
  2319. &movdqu ($inout2,&QWP(0,$l_));
  2320. &movdqu ($inout3,&QWP(0,$l_,$i1));
  2321. &movdqa ($inout4,$inout2);
  2322. &movdqu ($inout5,&QWP(0,$l_,$i3));
  2323. &pxor ($inout2,$rndkey0); # ^ last offset_i
  2324. &movdqu ($inout0,&QWP(16*0,$inp)); # load input
  2325. &pxor ($inout3,$inout2);
  2326. &movdqu ($inout1,&QWP(16*1,$inp));
  2327. &pxor ($inout4,$inout3);
  2328. &movdqa (&QWP(16*0,"esp"),$inout2);
  2329. &pxor ($inout5,$inout4);
  2330. &movdqa (&QWP(16*1,"esp"),$inout3);
  2331. &movdqu ($inout2,&QWP(16*2,$inp));
  2332. &movdqu ($inout3,&QWP(16*3,$inp));
  2333. &mov ($rounds,&DWP(240,$key));
  2334. &movdqa (&QWP($checksum,"esp"),$rndkey1);
  2335. &pxor ($inout0,&QWP(16*0,"esp")); # ^ offset_i
  2336. &pxor ($inout1,&QWP(16*1,"esp"));
  2337. &pxor ($inout2,$inout4);
  2338. &pxor ($inout3,$inout5);
  2339. &mov ($out,&DWP($out_off,"esp"));
  2340. &call ("_aesni_decrypt4");
  2341. &movdqa ($rndkey1,&QWP($checksum,"esp"));# pass the checksum
  2342. &xorps ($inout0,&QWP(16*0,"esp")); # ^ offset_i
  2343. &xorps ($inout1,&QWP(16*1,"esp"));
  2344. &xorps ($inout2,$inout4);
  2345. &movups (&QWP(16*0,$out,$inp),$inout0); # store output
  2346. &pxor ($rndkey1,$inout0); # checksum
  2347. &xorps ($inout3,$inout5);
  2348. &movups (&QWP(16*1,$out,$inp),$inout1);
  2349. &pxor ($rndkey1,$inout1);
  2350. &movdqa ($rndkey0,$inout5); # pass last offset_i
  2351. &movups (&QWP(16*2,$out,$inp),$inout2);
  2352. &pxor ($rndkey1,$inout2);
  2353. &movups (&QWP(16*3,$out,$inp),$inout3);
  2354. &pxor ($rndkey1,$inout3);
  2355. &set_label("done");
  2356. &mov ($key,&DWP($esp_off,"esp"));
  2357. &pxor ($inout0,$inout0); # clear register bank
  2358. &pxor ($inout1,$inout1);
  2359. &movdqa (&QWP(16*0,"esp"),$inout0); # clear stack
  2360. &pxor ($inout2,$inout2);
  2361. &movdqa (&QWP(16*1,"esp"),$inout0);
  2362. &pxor ($inout3,$inout3);
  2363. &movdqa (&QWP(16*2,"esp"),$inout0);
  2364. &pxor ($inout4,$inout4);
  2365. &movdqa (&QWP(16*3,"esp"),$inout0);
  2366. &pxor ($inout5,$inout5);
  2367. &movdqa (&QWP(16*4,"esp"),$inout0);
  2368. &movdqa (&QWP(16*5,"esp"),$inout0);
  2369. &movdqa (&QWP(16*6,"esp"),$inout0);
  2370. &lea ("esp",&DWP(0,$key));
  2371. &mov ($rounds,&wparam(5)); # &offset_i
  2372. &mov ($rounds_,&wparam(7)); # &checksum
  2373. &movdqu (&QWP(0,$rounds),$rndkey0);
  2374. &pxor ($rndkey0,$rndkey0);
  2375. &movdqu (&QWP(0,$rounds_),$rndkey1);
  2376. &pxor ($rndkey1,$rndkey1);
  2377. &function_end("aesni_ocb_decrypt");
  2378. }
  2379. }
  2380. ######################################################################
  2381. # void $PREFIX_cbc_encrypt (const void *inp, void *out,
  2382. # size_t length, const AES_KEY *key,
  2383. # unsigned char *ivp,const int enc);
  2384. &function_begin("${PREFIX}_cbc_encrypt");
  2385. &mov ($inp,&wparam(0));
  2386. &mov ($rounds_,"esp");
  2387. &mov ($out,&wparam(1));
  2388. &sub ($rounds_,24);
  2389. &mov ($len,&wparam(2));
  2390. &and ($rounds_,-16);
  2391. &mov ($key,&wparam(3));
  2392. &mov ($key_,&wparam(4));
  2393. &test ($len,$len);
  2394. &jz (&label("cbc_abort"));
  2395. &cmp (&wparam(5),0);
  2396. &xchg ($rounds_,"esp"); # alloca
  2397. &movups ($ivec,&QWP(0,$key_)); # load IV
  2398. &mov ($rounds,&DWP(240,$key));
  2399. &mov ($key_,$key); # backup $key
  2400. &mov (&DWP(16,"esp"),$rounds_); # save original %esp
  2401. &mov ($rounds_,$rounds); # backup $rounds
  2402. &je (&label("cbc_decrypt"));
  2403. &movaps ($inout0,$ivec);
  2404. &cmp ($len,16);
  2405. &jb (&label("cbc_enc_tail"));
  2406. &sub ($len,16);
  2407. &jmp (&label("cbc_enc_loop"));
  2408. &set_label("cbc_enc_loop",16);
  2409. &movups ($ivec,&QWP(0,$inp)); # input actually
  2410. &lea ($inp,&DWP(16,$inp));
  2411. if ($inline)
  2412. { &aesni_inline_generate1("enc",$inout0,$ivec); }
  2413. else
  2414. { &xorps($inout0,$ivec); &call("_aesni_encrypt1"); }
  2415. &mov ($rounds,$rounds_); # restore $rounds
  2416. &mov ($key,$key_); # restore $key
  2417. &movups (&QWP(0,$out),$inout0); # store output
  2418. &lea ($out,&DWP(16,$out));
  2419. &sub ($len,16);
  2420. &jnc (&label("cbc_enc_loop"));
  2421. &add ($len,16);
  2422. &jnz (&label("cbc_enc_tail"));
  2423. &movaps ($ivec,$inout0);
  2424. &pxor ($inout0,$inout0);
  2425. &jmp (&label("cbc_ret"));
  2426. &set_label("cbc_enc_tail");
  2427. &mov ("ecx",$len); # zaps $rounds
  2428. &data_word(0xA4F3F689); # rep movsb
  2429. &mov ("ecx",16); # zero tail
  2430. &sub ("ecx",$len);
  2431. &xor ("eax","eax"); # zaps $len
  2432. &data_word(0xAAF3F689); # rep stosb
  2433. &lea ($out,&DWP(-16,$out)); # rewind $out by 1 block
  2434. &mov ($rounds,$rounds_); # restore $rounds
  2435. &mov ($inp,$out); # $inp and $out are the same
  2436. &mov ($key,$key_); # restore $key
  2437. &jmp (&label("cbc_enc_loop"));
  2438. ######################################################################
  2439. &set_label("cbc_decrypt",16);
  2440. &cmp ($len,0x50);
  2441. &jbe (&label("cbc_dec_tail"));
  2442. &movaps (&QWP(0,"esp"),$ivec); # save IV
  2443. &sub ($len,0x50);
  2444. &jmp (&label("cbc_dec_loop6_enter"));
  2445. &set_label("cbc_dec_loop6",16);
  2446. &movaps (&QWP(0,"esp"),$rndkey0); # save IV
  2447. &movups (&QWP(0,$out),$inout5);
  2448. &lea ($out,&DWP(0x10,$out));
  2449. &set_label("cbc_dec_loop6_enter");
  2450. &movdqu ($inout0,&QWP(0,$inp));
  2451. &movdqu ($inout1,&QWP(0x10,$inp));
  2452. &movdqu ($inout2,&QWP(0x20,$inp));
  2453. &movdqu ($inout3,&QWP(0x30,$inp));
  2454. &movdqu ($inout4,&QWP(0x40,$inp));
  2455. &movdqu ($inout5,&QWP(0x50,$inp));
  2456. &call ("_aesni_decrypt6");
  2457. &movups ($rndkey1,&QWP(0,$inp));
  2458. &movups ($rndkey0,&QWP(0x10,$inp));
  2459. &xorps ($inout0,&QWP(0,"esp")); # ^=IV
  2460. &xorps ($inout1,$rndkey1);
  2461. &movups ($rndkey1,&QWP(0x20,$inp));
  2462. &xorps ($inout2,$rndkey0);
  2463. &movups ($rndkey0,&QWP(0x30,$inp));
  2464. &xorps ($inout3,$rndkey1);
  2465. &movups ($rndkey1,&QWP(0x40,$inp));
  2466. &xorps ($inout4,$rndkey0);
  2467. &movups ($rndkey0,&QWP(0x50,$inp)); # IV
  2468. &xorps ($inout5,$rndkey1);
  2469. &movups (&QWP(0,$out),$inout0);
  2470. &movups (&QWP(0x10,$out),$inout1);
  2471. &lea ($inp,&DWP(0x60,$inp));
  2472. &movups (&QWP(0x20,$out),$inout2);
  2473. &mov ($rounds,$rounds_); # restore $rounds
  2474. &movups (&QWP(0x30,$out),$inout3);
  2475. &mov ($key,$key_); # restore $key
  2476. &movups (&QWP(0x40,$out),$inout4);
  2477. &lea ($out,&DWP(0x50,$out));
  2478. &sub ($len,0x60);
  2479. &ja (&label("cbc_dec_loop6"));
  2480. &movaps ($inout0,$inout5);
  2481. &movaps ($ivec,$rndkey0);
  2482. &add ($len,0x50);
  2483. &jle (&label("cbc_dec_clear_tail_collected"));
  2484. &movups (&QWP(0,$out),$inout0);
  2485. &lea ($out,&DWP(0x10,$out));
  2486. &set_label("cbc_dec_tail");
  2487. &movups ($inout0,&QWP(0,$inp));
  2488. &movaps ($in0,$inout0);
  2489. &cmp ($len,0x10);
  2490. &jbe (&label("cbc_dec_one"));
  2491. &movups ($inout1,&QWP(0x10,$inp));
  2492. &movaps ($in1,$inout1);
  2493. &cmp ($len,0x20);
  2494. &jbe (&label("cbc_dec_two"));
  2495. &movups ($inout2,&QWP(0x20,$inp));
  2496. &cmp ($len,0x30);
  2497. &jbe (&label("cbc_dec_three"));
  2498. &movups ($inout3,&QWP(0x30,$inp));
  2499. &cmp ($len,0x40);
  2500. &jbe (&label("cbc_dec_four"));
  2501. &movups ($inout4,&QWP(0x40,$inp));
  2502. &movaps (&QWP(0,"esp"),$ivec); # save IV
  2503. &movups ($inout0,&QWP(0,$inp));
  2504. &xorps ($inout5,$inout5);
  2505. &call ("_aesni_decrypt6");
  2506. &movups ($rndkey1,&QWP(0,$inp));
  2507. &movups ($rndkey0,&QWP(0x10,$inp));
  2508. &xorps ($inout0,&QWP(0,"esp")); # ^= IV
  2509. &xorps ($inout1,$rndkey1);
  2510. &movups ($rndkey1,&QWP(0x20,$inp));
  2511. &xorps ($inout2,$rndkey0);
  2512. &movups ($rndkey0,&QWP(0x30,$inp));
  2513. &xorps ($inout3,$rndkey1);
  2514. &movups ($ivec,&QWP(0x40,$inp)); # IV
  2515. &xorps ($inout4,$rndkey0);
  2516. &movups (&QWP(0,$out),$inout0);
  2517. &movups (&QWP(0x10,$out),$inout1);
  2518. &pxor ($inout1,$inout1);
  2519. &movups (&QWP(0x20,$out),$inout2);
  2520. &pxor ($inout2,$inout2);
  2521. &movups (&QWP(0x30,$out),$inout3);
  2522. &pxor ($inout3,$inout3);
  2523. &lea ($out,&DWP(0x40,$out));
  2524. &movaps ($inout0,$inout4);
  2525. &pxor ($inout4,$inout4);
  2526. &sub ($len,0x50);
  2527. &jmp (&label("cbc_dec_tail_collected"));
  2528. &set_label("cbc_dec_one",16);
  2529. if ($inline)
  2530. { &aesni_inline_generate1("dec"); }
  2531. else
  2532. { &call ("_aesni_decrypt1"); }
  2533. &xorps ($inout0,$ivec);
  2534. &movaps ($ivec,$in0);
  2535. &sub ($len,0x10);
  2536. &jmp (&label("cbc_dec_tail_collected"));
  2537. &set_label("cbc_dec_two",16);
  2538. &call ("_aesni_decrypt2");
  2539. &xorps ($inout0,$ivec);
  2540. &xorps ($inout1,$in0);
  2541. &movups (&QWP(0,$out),$inout0);
  2542. &movaps ($inout0,$inout1);
  2543. &pxor ($inout1,$inout1);
  2544. &lea ($out,&DWP(0x10,$out));
  2545. &movaps ($ivec,$in1);
  2546. &sub ($len,0x20);
  2547. &jmp (&label("cbc_dec_tail_collected"));
  2548. &set_label("cbc_dec_three",16);
  2549. &call ("_aesni_decrypt3");
  2550. &xorps ($inout0,$ivec);
  2551. &xorps ($inout1,$in0);
  2552. &xorps ($inout2,$in1);
  2553. &movups (&QWP(0,$out),$inout0);
  2554. &movaps ($inout0,$inout2);
  2555. &pxor ($inout2,$inout2);
  2556. &movups (&QWP(0x10,$out),$inout1);
  2557. &pxor ($inout1,$inout1);
  2558. &lea ($out,&DWP(0x20,$out));
  2559. &movups ($ivec,&QWP(0x20,$inp));
  2560. &sub ($len,0x30);
  2561. &jmp (&label("cbc_dec_tail_collected"));
  2562. &set_label("cbc_dec_four",16);
  2563. &call ("_aesni_decrypt4");
  2564. &movups ($rndkey1,&QWP(0x10,$inp));
  2565. &movups ($rndkey0,&QWP(0x20,$inp));
  2566. &xorps ($inout0,$ivec);
  2567. &movups ($ivec,&QWP(0x30,$inp));
  2568. &xorps ($inout1,$in0);
  2569. &movups (&QWP(0,$out),$inout0);
  2570. &xorps ($inout2,$rndkey1);
  2571. &movups (&QWP(0x10,$out),$inout1);
  2572. &pxor ($inout1,$inout1);
  2573. &xorps ($inout3,$rndkey0);
  2574. &movups (&QWP(0x20,$out),$inout2);
  2575. &pxor ($inout2,$inout2);
  2576. &lea ($out,&DWP(0x30,$out));
  2577. &movaps ($inout0,$inout3);
  2578. &pxor ($inout3,$inout3);
  2579. &sub ($len,0x40);
  2580. &jmp (&label("cbc_dec_tail_collected"));
  2581. &set_label("cbc_dec_clear_tail_collected",16);
  2582. &pxor ($inout1,$inout1);
  2583. &pxor ($inout2,$inout2);
  2584. &pxor ($inout3,$inout3);
  2585. &pxor ($inout4,$inout4);
  2586. &set_label("cbc_dec_tail_collected");
  2587. &and ($len,15);
  2588. &jnz (&label("cbc_dec_tail_partial"));
  2589. &movups (&QWP(0,$out),$inout0);
  2590. &pxor ($rndkey0,$rndkey0);
  2591. &jmp (&label("cbc_ret"));
  2592. &set_label("cbc_dec_tail_partial",16);
  2593. &movaps (&QWP(0,"esp"),$inout0);
  2594. &pxor ($rndkey0,$rndkey0);
  2595. &mov ("ecx",16);
  2596. &mov ($inp,"esp");
  2597. &sub ("ecx",$len);
  2598. &data_word(0xA4F3F689); # rep movsb
  2599. &movdqa (&QWP(0,"esp"),$inout0);
  2600. &set_label("cbc_ret");
  2601. &mov ("esp",&DWP(16,"esp")); # pull original %esp
  2602. &mov ($key_,&wparam(4));
  2603. &pxor ($inout0,$inout0);
  2604. &pxor ($rndkey1,$rndkey1);
  2605. &movups (&QWP(0,$key_),$ivec); # output IV
  2606. &pxor ($ivec,$ivec);
  2607. &set_label("cbc_abort");
  2608. &function_end("${PREFIX}_cbc_encrypt");
  2609. ######################################################################
  2610. # Mechanical port from aesni-x86_64.pl.
  2611. #
  2612. # _aesni_set_encrypt_key is private interface,
  2613. # input:
  2614. # "eax" const unsigned char *userKey
  2615. # $rounds int bits
  2616. # $key AES_KEY *key
  2617. # output:
  2618. # "eax" return code
  2619. # $round rounds
  2620. &function_begin_B("_aesni_set_encrypt_key");
  2621. &push ("ebp");
  2622. &push ("ebx");
  2623. &test ("eax","eax");
  2624. &jz (&label("bad_pointer"));
  2625. &test ($key,$key);
  2626. &jz (&label("bad_pointer"));
  2627. &call (&label("pic"));
  2628. &set_label("pic");
  2629. &blindpop("ebx");
  2630. &lea ("ebx",&DWP(&label("key_const")."-".&label("pic"),"ebx"));
  2631. &picmeup("ebp","OPENSSL_ia32cap_P","ebx",&label("key_const"));
  2632. &movups ("xmm0",&QWP(0,"eax")); # pull first 128 bits of *userKey
  2633. &xorps ("xmm4","xmm4"); # low dword of xmm4 is assumed 0
  2634. &mov ("ebp",&DWP(4,"ebp"));
  2635. &lea ($key,&DWP(16,$key));
  2636. &and ("ebp",1<<28|1<<11); # AVX and XOP bits
  2637. &cmp ($rounds,256);
  2638. &je (&label("14rounds"));
  2639. &cmp ($rounds,192);
  2640. &je (&label("12rounds"));
  2641. &cmp ($rounds,128);
  2642. &jne (&label("bad_keybits"));
  2643. &set_label("10rounds",16);
  2644. &cmp ("ebp",1<<28);
  2645. &je (&label("10rounds_alt"));
  2646. &mov ($rounds,9);
  2647. &$movekey (&QWP(-16,$key),"xmm0"); # round 0
  2648. &aeskeygenassist("xmm1","xmm0",0x01); # round 1
  2649. &call (&label("key_128_cold"));
  2650. &aeskeygenassist("xmm1","xmm0",0x2); # round 2
  2651. &call (&label("key_128"));
  2652. &aeskeygenassist("xmm1","xmm0",0x04); # round 3
  2653. &call (&label("key_128"));
  2654. &aeskeygenassist("xmm1","xmm0",0x08); # round 4
  2655. &call (&label("key_128"));
  2656. &aeskeygenassist("xmm1","xmm0",0x10); # round 5
  2657. &call (&label("key_128"));
  2658. &aeskeygenassist("xmm1","xmm0",0x20); # round 6
  2659. &call (&label("key_128"));
  2660. &aeskeygenassist("xmm1","xmm0",0x40); # round 7
  2661. &call (&label("key_128"));
  2662. &aeskeygenassist("xmm1","xmm0",0x80); # round 8
  2663. &call (&label("key_128"));
  2664. &aeskeygenassist("xmm1","xmm0",0x1b); # round 9
  2665. &call (&label("key_128"));
  2666. &aeskeygenassist("xmm1","xmm0",0x36); # round 10
  2667. &call (&label("key_128"));
  2668. &$movekey (&QWP(0,$key),"xmm0");
  2669. &mov (&DWP(80,$key),$rounds);
  2670. &jmp (&label("good_key"));
  2671. &set_label("key_128",16);
  2672. &$movekey (&QWP(0,$key),"xmm0");
  2673. &lea ($key,&DWP(16,$key));
  2674. &set_label("key_128_cold");
  2675. &shufps ("xmm4","xmm0",0b00010000);
  2676. &xorps ("xmm0","xmm4");
  2677. &shufps ("xmm4","xmm0",0b10001100);
  2678. &xorps ("xmm0","xmm4");
  2679. &shufps ("xmm1","xmm1",0b11111111); # critical path
  2680. &xorps ("xmm0","xmm1");
  2681. &ret();
  2682. &set_label("10rounds_alt",16);
  2683. &movdqa ("xmm5",&QWP(0x00,"ebx"));
  2684. &mov ($rounds,8);
  2685. &movdqa ("xmm4",&QWP(0x20,"ebx"));
  2686. &movdqa ("xmm2","xmm0");
  2687. &movdqu (&QWP(-16,$key),"xmm0");
  2688. &set_label("loop_key128");
  2689. &pshufb ("xmm0","xmm5");
  2690. &aesenclast ("xmm0","xmm4");
  2691. &pslld ("xmm4",1);
  2692. &lea ($key,&DWP(16,$key));
  2693. &movdqa ("xmm3","xmm2");
  2694. &pslldq ("xmm2",4);
  2695. &pxor ("xmm3","xmm2");
  2696. &pslldq ("xmm2",4);
  2697. &pxor ("xmm3","xmm2");
  2698. &pslldq ("xmm2",4);
  2699. &pxor ("xmm2","xmm3");
  2700. &pxor ("xmm0","xmm2");
  2701. &movdqu (&QWP(-16,$key),"xmm0");
  2702. &movdqa ("xmm2","xmm0");
  2703. &dec ($rounds);
  2704. &jnz (&label("loop_key128"));
  2705. &movdqa ("xmm4",&QWP(0x30,"ebx"));
  2706. &pshufb ("xmm0","xmm5");
  2707. &aesenclast ("xmm0","xmm4");
  2708. &pslld ("xmm4",1);
  2709. &movdqa ("xmm3","xmm2");
  2710. &pslldq ("xmm2",4);
  2711. &pxor ("xmm3","xmm2");
  2712. &pslldq ("xmm2",4);
  2713. &pxor ("xmm3","xmm2");
  2714. &pslldq ("xmm2",4);
  2715. &pxor ("xmm2","xmm3");
  2716. &pxor ("xmm0","xmm2");
  2717. &movdqu (&QWP(0,$key),"xmm0");
  2718. &movdqa ("xmm2","xmm0");
  2719. &pshufb ("xmm0","xmm5");
  2720. &aesenclast ("xmm0","xmm4");
  2721. &movdqa ("xmm3","xmm2");
  2722. &pslldq ("xmm2",4);
  2723. &pxor ("xmm3","xmm2");
  2724. &pslldq ("xmm2",4);
  2725. &pxor ("xmm3","xmm2");
  2726. &pslldq ("xmm2",4);
  2727. &pxor ("xmm2","xmm3");
  2728. &pxor ("xmm0","xmm2");
  2729. &movdqu (&QWP(16,$key),"xmm0");
  2730. &mov ($rounds,9);
  2731. &mov (&DWP(96,$key),$rounds);
  2732. &jmp (&label("good_key"));
  2733. &set_label("12rounds",16);
  2734. &movq ("xmm2",&QWP(16,"eax")); # remaining 1/3 of *userKey
  2735. &cmp ("ebp",1<<28);
  2736. &je (&label("12rounds_alt"));
  2737. &mov ($rounds,11);
  2738. &$movekey (&QWP(-16,$key),"xmm0"); # round 0
  2739. &aeskeygenassist("xmm1","xmm2",0x01); # round 1,2
  2740. &call (&label("key_192a_cold"));
  2741. &aeskeygenassist("xmm1","xmm2",0x02); # round 2,3
  2742. &call (&label("key_192b"));
  2743. &aeskeygenassist("xmm1","xmm2",0x04); # round 4,5
  2744. &call (&label("key_192a"));
  2745. &aeskeygenassist("xmm1","xmm2",0x08); # round 5,6
  2746. &call (&label("key_192b"));
  2747. &aeskeygenassist("xmm1","xmm2",0x10); # round 7,8
  2748. &call (&label("key_192a"));
  2749. &aeskeygenassist("xmm1","xmm2",0x20); # round 8,9
  2750. &call (&label("key_192b"));
  2751. &aeskeygenassist("xmm1","xmm2",0x40); # round 10,11
  2752. &call (&label("key_192a"));
  2753. &aeskeygenassist("xmm1","xmm2",0x80); # round 11,12
  2754. &call (&label("key_192b"));
  2755. &$movekey (&QWP(0,$key),"xmm0");
  2756. &mov (&DWP(48,$key),$rounds);
  2757. &jmp (&label("good_key"));
  2758. &set_label("key_192a",16);
  2759. &$movekey (&QWP(0,$key),"xmm0");
  2760. &lea ($key,&DWP(16,$key));
  2761. &set_label("key_192a_cold",16);
  2762. &movaps ("xmm5","xmm2");
  2763. &set_label("key_192b_warm");
  2764. &shufps ("xmm4","xmm0",0b00010000);
  2765. &movdqa ("xmm3","xmm2");
  2766. &xorps ("xmm0","xmm4");
  2767. &shufps ("xmm4","xmm0",0b10001100);
  2768. &pslldq ("xmm3",4);
  2769. &xorps ("xmm0","xmm4");
  2770. &pshufd ("xmm1","xmm1",0b01010101); # critical path
  2771. &pxor ("xmm2","xmm3");
  2772. &pxor ("xmm0","xmm1");
  2773. &pshufd ("xmm3","xmm0",0b11111111);
  2774. &pxor ("xmm2","xmm3");
  2775. &ret();
  2776. &set_label("key_192b",16);
  2777. &movaps ("xmm3","xmm0");
  2778. &shufps ("xmm5","xmm0",0b01000100);
  2779. &$movekey (&QWP(0,$key),"xmm5");
  2780. &shufps ("xmm3","xmm2",0b01001110);
  2781. &$movekey (&QWP(16,$key),"xmm3");
  2782. &lea ($key,&DWP(32,$key));
  2783. &jmp (&label("key_192b_warm"));
  2784. &set_label("12rounds_alt",16);
  2785. &movdqa ("xmm5",&QWP(0x10,"ebx"));
  2786. &movdqa ("xmm4",&QWP(0x20,"ebx"));
  2787. &mov ($rounds,8);
  2788. &movdqu (&QWP(-16,$key),"xmm0");
  2789. &set_label("loop_key192");
  2790. &movq (&QWP(0,$key),"xmm2");
  2791. &movdqa ("xmm1","xmm2");
  2792. &pshufb ("xmm2","xmm5");
  2793. &aesenclast ("xmm2","xmm4");
  2794. &pslld ("xmm4",1);
  2795. &lea ($key,&DWP(24,$key));
  2796. &movdqa ("xmm3","xmm0");
  2797. &pslldq ("xmm0",4);
  2798. &pxor ("xmm3","xmm0");
  2799. &pslldq ("xmm0",4);
  2800. &pxor ("xmm3","xmm0");
  2801. &pslldq ("xmm0",4);
  2802. &pxor ("xmm0","xmm3");
  2803. &pshufd ("xmm3","xmm0",0xff);
  2804. &pxor ("xmm3","xmm1");
  2805. &pslldq ("xmm1",4);
  2806. &pxor ("xmm3","xmm1");
  2807. &pxor ("xmm0","xmm2");
  2808. &pxor ("xmm2","xmm3");
  2809. &movdqu (&QWP(-16,$key),"xmm0");
  2810. &dec ($rounds);
  2811. &jnz (&label("loop_key192"));
  2812. &mov ($rounds,11);
  2813. &mov (&DWP(32,$key),$rounds);
  2814. &jmp (&label("good_key"));
  2815. &set_label("14rounds",16);
  2816. &movups ("xmm2",&QWP(16,"eax")); # remaining half of *userKey
  2817. &lea ($key,&DWP(16,$key));
  2818. &cmp ("ebp",1<<28);
  2819. &je (&label("14rounds_alt"));
  2820. &mov ($rounds,13);
  2821. &$movekey (&QWP(-32,$key),"xmm0"); # round 0
  2822. &$movekey (&QWP(-16,$key),"xmm2"); # round 1
  2823. &aeskeygenassist("xmm1","xmm2",0x01); # round 2
  2824. &call (&label("key_256a_cold"));
  2825. &aeskeygenassist("xmm1","xmm0",0x01); # round 3
  2826. &call (&label("key_256b"));
  2827. &aeskeygenassist("xmm1","xmm2",0x02); # round 4
  2828. &call (&label("key_256a"));
  2829. &aeskeygenassist("xmm1","xmm0",0x02); # round 5
  2830. &call (&label("key_256b"));
  2831. &aeskeygenassist("xmm1","xmm2",0x04); # round 6
  2832. &call (&label("key_256a"));
  2833. &aeskeygenassist("xmm1","xmm0",0x04); # round 7
  2834. &call (&label("key_256b"));
  2835. &aeskeygenassist("xmm1","xmm2",0x08); # round 8
  2836. &call (&label("key_256a"));
  2837. &aeskeygenassist("xmm1","xmm0",0x08); # round 9
  2838. &call (&label("key_256b"));
  2839. &aeskeygenassist("xmm1","xmm2",0x10); # round 10
  2840. &call (&label("key_256a"));
  2841. &aeskeygenassist("xmm1","xmm0",0x10); # round 11
  2842. &call (&label("key_256b"));
  2843. &aeskeygenassist("xmm1","xmm2",0x20); # round 12
  2844. &call (&label("key_256a"));
  2845. &aeskeygenassist("xmm1","xmm0",0x20); # round 13
  2846. &call (&label("key_256b"));
  2847. &aeskeygenassist("xmm1","xmm2",0x40); # round 14
  2848. &call (&label("key_256a"));
  2849. &$movekey (&QWP(0,$key),"xmm0");
  2850. &mov (&DWP(16,$key),$rounds);
  2851. &xor ("eax","eax");
  2852. &jmp (&label("good_key"));
  2853. &set_label("key_256a",16);
  2854. &$movekey (&QWP(0,$key),"xmm2");
  2855. &lea ($key,&DWP(16,$key));
  2856. &set_label("key_256a_cold");
  2857. &shufps ("xmm4","xmm0",0b00010000);
  2858. &xorps ("xmm0","xmm4");
  2859. &shufps ("xmm4","xmm0",0b10001100);
  2860. &xorps ("xmm0","xmm4");
  2861. &shufps ("xmm1","xmm1",0b11111111); # critical path
  2862. &xorps ("xmm0","xmm1");
  2863. &ret();
  2864. &set_label("key_256b",16);
  2865. &$movekey (&QWP(0,$key),"xmm0");
  2866. &lea ($key,&DWP(16,$key));
  2867. &shufps ("xmm4","xmm2",0b00010000);
  2868. &xorps ("xmm2","xmm4");
  2869. &shufps ("xmm4","xmm2",0b10001100);
  2870. &xorps ("xmm2","xmm4");
  2871. &shufps ("xmm1","xmm1",0b10101010); # critical path
  2872. &xorps ("xmm2","xmm1");
  2873. &ret();
  2874. &set_label("14rounds_alt",16);
  2875. &movdqa ("xmm5",&QWP(0x00,"ebx"));
  2876. &movdqa ("xmm4",&QWP(0x20,"ebx"));
  2877. &mov ($rounds,7);
  2878. &movdqu (&QWP(-32,$key),"xmm0");
  2879. &movdqa ("xmm1","xmm2");
  2880. &movdqu (&QWP(-16,$key),"xmm2");
  2881. &set_label("loop_key256");
  2882. &pshufb ("xmm2","xmm5");
  2883. &aesenclast ("xmm2","xmm4");
  2884. &movdqa ("xmm3","xmm0");
  2885. &pslldq ("xmm0",4);
  2886. &pxor ("xmm3","xmm0");
  2887. &pslldq ("xmm0",4);
  2888. &pxor ("xmm3","xmm0");
  2889. &pslldq ("xmm0",4);
  2890. &pxor ("xmm0","xmm3");
  2891. &pslld ("xmm4",1);
  2892. &pxor ("xmm0","xmm2");
  2893. &movdqu (&QWP(0,$key),"xmm0");
  2894. &dec ($rounds);
  2895. &jz (&label("done_key256"));
  2896. &pshufd ("xmm2","xmm0",0xff);
  2897. &pxor ("xmm3","xmm3");
  2898. &aesenclast ("xmm2","xmm3");
  2899. &movdqa ("xmm3","xmm1");
  2900. &pslldq ("xmm1",4);
  2901. &pxor ("xmm3","xmm1");
  2902. &pslldq ("xmm1",4);
  2903. &pxor ("xmm3","xmm1");
  2904. &pslldq ("xmm1",4);
  2905. &pxor ("xmm1","xmm3");
  2906. &pxor ("xmm2","xmm1");
  2907. &movdqu (&QWP(16,$key),"xmm2");
  2908. &lea ($key,&DWP(32,$key));
  2909. &movdqa ("xmm1","xmm2");
  2910. &jmp (&label("loop_key256"));
  2911. &set_label("done_key256");
  2912. &mov ($rounds,13);
  2913. &mov (&DWP(16,$key),$rounds);
  2914. &set_label("good_key");
  2915. &pxor ("xmm0","xmm0");
  2916. &pxor ("xmm1","xmm1");
  2917. &pxor ("xmm2","xmm2");
  2918. &pxor ("xmm3","xmm3");
  2919. &pxor ("xmm4","xmm4");
  2920. &pxor ("xmm5","xmm5");
  2921. &xor ("eax","eax");
  2922. &pop ("ebx");
  2923. &pop ("ebp");
  2924. &ret ();
  2925. &set_label("bad_pointer",4);
  2926. &mov ("eax",-1);
  2927. &pop ("ebx");
  2928. &pop ("ebp");
  2929. &ret ();
  2930. &set_label("bad_keybits",4);
  2931. &pxor ("xmm0","xmm0");
  2932. &mov ("eax",-2);
  2933. &pop ("ebx");
  2934. &pop ("ebp");
  2935. &ret ();
  2936. &function_end_B("_aesni_set_encrypt_key");
  2937. # int $PREFIX_set_encrypt_key (const unsigned char *userKey, int bits,
  2938. # AES_KEY *key)
  2939. &function_begin_B("${PREFIX}_set_encrypt_key");
  2940. &mov ("eax",&wparam(0));
  2941. &mov ($rounds,&wparam(1));
  2942. &mov ($key,&wparam(2));
  2943. &call ("_aesni_set_encrypt_key");
  2944. &ret ();
  2945. &function_end_B("${PREFIX}_set_encrypt_key");
  2946. # int $PREFIX_set_decrypt_key (const unsigned char *userKey, int bits,
  2947. # AES_KEY *key)
  2948. &function_begin_B("${PREFIX}_set_decrypt_key");
  2949. &mov ("eax",&wparam(0));
  2950. &mov ($rounds,&wparam(1));
  2951. &mov ($key,&wparam(2));
  2952. &call ("_aesni_set_encrypt_key");
  2953. &mov ($key,&wparam(2));
  2954. &shl ($rounds,4); # rounds-1 after _aesni_set_encrypt_key
  2955. &test ("eax","eax");
  2956. &jnz (&label("dec_key_ret"));
  2957. &lea ("eax",&DWP(16,$key,$rounds)); # end of key schedule
  2958. &$movekey ("xmm0",&QWP(0,$key)); # just swap
  2959. &$movekey ("xmm1",&QWP(0,"eax"));
  2960. &$movekey (&QWP(0,"eax"),"xmm0");
  2961. &$movekey (&QWP(0,$key),"xmm1");
  2962. &lea ($key,&DWP(16,$key));
  2963. &lea ("eax",&DWP(-16,"eax"));
  2964. &set_label("dec_key_inverse");
  2965. &$movekey ("xmm0",&QWP(0,$key)); # swap and inverse
  2966. &$movekey ("xmm1",&QWP(0,"eax"));
  2967. &aesimc ("xmm0","xmm0");
  2968. &aesimc ("xmm1","xmm1");
  2969. &lea ($key,&DWP(16,$key));
  2970. &lea ("eax",&DWP(-16,"eax"));
  2971. &$movekey (&QWP(16,"eax"),"xmm0");
  2972. &$movekey (&QWP(-16,$key),"xmm1");
  2973. &cmp ("eax",$key);
  2974. &ja (&label("dec_key_inverse"));
  2975. &$movekey ("xmm0",&QWP(0,$key)); # inverse middle
  2976. &aesimc ("xmm0","xmm0");
  2977. &$movekey (&QWP(0,$key),"xmm0");
  2978. &pxor ("xmm0","xmm0");
  2979. &pxor ("xmm1","xmm1");
  2980. &xor ("eax","eax"); # return success
  2981. &set_label("dec_key_ret");
  2982. &ret ();
  2983. &function_end_B("${PREFIX}_set_decrypt_key");
  2984. &set_label("key_const",64);
  2985. &data_word(0x0c0f0e0d,0x0c0f0e0d,0x0c0f0e0d,0x0c0f0e0d);
  2986. &data_word(0x04070605,0x04070605,0x04070605,0x04070605);
  2987. &data_word(1,1,1,1);
  2988. &data_word(0x1b,0x1b,0x1b,0x1b);
  2989. &asciz("AES for Intel AES-NI, CRYPTOGAMS by <appro\@openssl.org>");
  2990. &asm_finish();
  2991. close STDOUT or die "error closing STDOUT: $!";