aesni-sha1-x86_64.pl 54 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148
  1. #! /usr/bin/env perl
  2. # Copyright 2011-2020 The OpenSSL Project Authors. All Rights Reserved.
  3. #
  4. # Licensed under the Apache License 2.0 (the "License"). You may not use
  5. # this file except in compliance with the License. You can obtain a copy
  6. # in the file LICENSE in the source distribution or at
  7. # https://www.openssl.org/source/license.html
  8. #
  9. # ====================================================================
  10. # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
  11. # project. The module is, however, dual licensed under OpenSSL and
  12. # CRYPTOGAMS licenses depending on where you obtain it. For further
  13. # details see http://www.openssl.org/~appro/cryptogams/.
  14. # ====================================================================
  15. #
  16. # June 2011
  17. #
  18. # This is AESNI-CBC+SHA1 "stitch" implementation. The idea, as spelled
  19. # in http://download.intel.com/design/intarch/papers/323686.pdf, is
  20. # that since AESNI-CBC encrypt exhibit *very* low instruction-level
  21. # parallelism, interleaving it with another algorithm would allow to
  22. # utilize processor resources better and achieve better performance.
  23. # SHA1 instruction sequences(*) are taken from sha1-x86_64.pl and
  24. # AESNI code is weaved into it. Below are performance numbers in
  25. # cycles per processed byte, less is better, for standalone AESNI-CBC
  26. # encrypt, sum of the latter and standalone SHA1, and "stitched"
  27. # subroutine:
  28. #
  29. # AES-128-CBC +SHA1 stitch gain
  30. # Westmere 3.77[+5.3] 9.07 6.55 +38%
  31. # Sandy Bridge 5.05[+5.0(6.1)] 10.06(11.15) 5.98(7.05) +68%(+58%)
  32. # Ivy Bridge 5.05[+4.6] 9.65 5.54 +74%
  33. # Haswell 4.43[+3.6(4.2)] 8.00(8.58) 4.55(5.21) +75%(+65%)
  34. # Skylake 2.63[+3.5(4.1)] 6.17(6.69) 4.23(4.44) +46%(+51%)
  35. # Bulldozer 5.77[+6.0] 11.72 6.37 +84%
  36. # Ryzen(**) 2.71[+1.93] 4.64 2.74 +69%
  37. # Goldmont(**) 3.82[+1.70] 5.52 4.20 +31%
  38. #
  39. # AES-192-CBC
  40. # Westmere 4.51 9.81 6.80 +44%
  41. # Sandy Bridge 6.05 11.06(12.15) 6.11(7.19) +81%(+69%)
  42. # Ivy Bridge 6.05 10.65 6.07 +75%
  43. # Haswell 5.29 8.86(9.44) 5.32(5.32) +67%(+77%)
  44. # Bulldozer 6.89 12.84 6.96 +84%
  45. #
  46. # AES-256-CBC
  47. # Westmere 5.25 10.55 7.21 +46%
  48. # Sandy Bridge 7.05 12.06(13.15) 7.12(7.72) +69%(+70%)
  49. # Ivy Bridge 7.05 11.65 7.12 +64%
  50. # Haswell 6.19 9.76(10.34) 6.21(6.25) +57%(+65%)
  51. # Skylake 3.62 7.16(7.68) 4.56(4.76) +57%(+61%)
  52. # Bulldozer 8.00 13.95 8.25 +69%
  53. # Ryzen(**) 3.71 5.64 3.72 +52%
  54. # Goldmont(**) 5.35 7.05 5.76 +22%
  55. #
  56. # (*) There are two code paths: SSSE3 and AVX. See sha1-568.pl for
  57. # background information. Above numbers in parentheses are SSSE3
  58. # results collected on AVX-capable CPU, i.e. apply on OSes that
  59. # don't support AVX.
  60. # (**) SHAEXT results.
  61. #
  62. # Needless to mention that it makes no sense to implement "stitched"
  63. # *decrypt* subroutine. Because *both* AESNI-CBC decrypt and SHA1
  64. # fully utilize parallelism, so stitching would not give any gain
  65. # anyway. Well, there might be some, e.g. because of better cache
  66. # locality... For reference, here are performance results for
  67. # standalone AESNI-CBC decrypt:
  68. #
  69. # AES-128-CBC AES-192-CBC AES-256-CBC
  70. # Westmere 1.25 1.50 1.75
  71. # Sandy Bridge 0.74 0.91 1.09
  72. # Ivy Bridge 0.74 0.90 1.11
  73. # Haswell 0.63 0.76 0.88
  74. # Bulldozer 0.70 0.85 0.99
  75. # And indeed:
  76. #
  77. # AES-256-CBC +SHA1 stitch gain
  78. # Westmere 1.75 7.20 6.68 +7.8%
  79. # Sandy Bridge 1.09 6.09(7.22) 5.82(6.95) +4.6%(+3.9%)
  80. # Ivy Bridge 1.11 5.70 5.45 +4.6%
  81. # Haswell 0.88 4.45(5.00) 4.39(4.69) +1.4%(*)(+6.6%)
  82. # Bulldozer 0.99 6.95 5.95 +17%(**)
  83. #
  84. # (*) Tiny improvement coefficient on Haswell is because we compare
  85. # AVX1 stitch to sum with AVX2 SHA1.
  86. # (**) Execution is fully dominated by integer code sequence and
  87. # SIMD still hardly shows [in single-process benchmark;-]
  88. # $output is the last argument if it looks like a file (it has an extension)
  89. # $flavour is the first argument if it doesn't look like a file
  90. $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
  91. $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
  92. $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
  93. $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
  94. ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
  95. ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
  96. die "can't locate x86_64-xlate.pl";
  97. $avx=1 if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
  98. =~ /GNU assembler version ([2-9]\.[0-9]+)/ &&
  99. $1>=2.19);
  100. $avx=1 if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) &&
  101. `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/ &&
  102. $1>=2.09);
  103. $avx=1 if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) &&
  104. `ml64 2>&1` =~ /Version ([0-9]+)\./ &&
  105. $1>=10);
  106. $avx=1 if (!$avx && `$ENV{CC} -v 2>&1` =~ /((?:clang|LLVM) version|.*based on LLVM) ([0-9]+\.[0-9]+)/ && $2>=3.0);
  107. $shaext=1; ### set to zero if compiling for 1.0.1
  108. $stitched_decrypt=0;
  109. open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\""
  110. or die "can't call $xlate: $!";
  111. *STDOUT=*OUT;
  112. # void aesni_cbc_sha1_enc(const void *inp,
  113. # void *out,
  114. # size_t length,
  115. # const AES_KEY *key,
  116. # unsigned char *iv,
  117. # SHA_CTX *ctx,
  118. # const void *in0);
  119. $code.=<<___;
  120. .text
  121. .extern OPENSSL_ia32cap_P
  122. .globl aesni_cbc_sha1_enc
  123. .type aesni_cbc_sha1_enc,\@abi-omnipotent
  124. .align 32
  125. aesni_cbc_sha1_enc:
  126. .cfi_startproc
  127. # caller should check for SSSE3 and AES-NI bits
  128. mov OPENSSL_ia32cap_P+0(%rip),%r10d
  129. mov OPENSSL_ia32cap_P+4(%rip),%r11
  130. ___
  131. $code.=<<___ if ($shaext);
  132. bt \$61,%r11 # check SHA bit
  133. jc aesni_cbc_sha1_enc_shaext
  134. ___
  135. $code.=<<___ if ($avx);
  136. and \$`1<<28`,%r11d # mask AVX bit
  137. and \$`1<<30`,%r10d # mask "Intel CPU" bit
  138. or %r11d,%r10d
  139. cmp \$`1<<28|1<<30`,%r10d
  140. je aesni_cbc_sha1_enc_avx
  141. ___
  142. $code.=<<___;
  143. jmp aesni_cbc_sha1_enc_ssse3
  144. ret
  145. .cfi_endproc
  146. .size aesni_cbc_sha1_enc,.-aesni_cbc_sha1_enc
  147. ___
  148. my ($in0,$out,$len,$key,$ivp,$ctx,$inp)=("%rdi","%rsi","%rdx","%rcx","%r8","%r9","%r10");
  149. my $Xi=4;
  150. my @X=map("%xmm$_",(4..7,0..3));
  151. my @Tx=map("%xmm$_",(8..10));
  152. my @V=($A,$B,$C,$D,$E)=("%eax","%ebx","%ecx","%edx","%ebp"); # size optimization
  153. my @T=("%esi","%edi");
  154. my $j=0; my $jj=0; my $r=0; my $sn=0; my $rx=0;
  155. my $K_XX_XX="%r11";
  156. my ($rndkey0,$iv,$in)=map("%xmm$_",(11..13)); # for enc
  157. my @rndkey=("%xmm14","%xmm15"); # for enc
  158. my ($inout0,$inout1,$inout2,$inout3)=map("%xmm$_",(12..15)); # for dec
  159. if (1) { # reassign for Atom Silvermont
  160. # The goal is to minimize amount of instructions with more than
  161. # 3 prefix bytes. Or in more practical terms to keep AES-NI *and*
  162. # SSSE3 instructions to upper half of the register bank.
  163. @X=map("%xmm$_",(8..11,4..7));
  164. @Tx=map("%xmm$_",(12,13,3));
  165. ($iv,$in,$rndkey0)=map("%xmm$_",(2,14,15));
  166. @rndkey=("%xmm0","%xmm1");
  167. }
  168. sub AUTOLOAD() # thunk [simplified] 32-bit style perlasm
  169. { my $opcode = $AUTOLOAD; $opcode =~ s/.*:://;
  170. my $arg = pop;
  171. $arg = "\$$arg" if ($arg*1 eq $arg);
  172. $code .= "\t$opcode\t".join(',',$arg,reverse @_)."\n";
  173. }
  174. my $_rol=sub { &rol(@_) };
  175. my $_ror=sub { &ror(@_) };
  176. $code.=<<___;
  177. .type aesni_cbc_sha1_enc_ssse3,\@function,6
  178. .align 32
  179. aesni_cbc_sha1_enc_ssse3:
  180. .cfi_startproc
  181. mov `($win64?56:8)`(%rsp),$inp # load 7th argument
  182. #shr \$6,$len # debugging artefact
  183. #jz .Lepilogue_ssse3 # debugging artefact
  184. push %rbx
  185. .cfi_push %rbx
  186. push %rbp
  187. .cfi_push %rbp
  188. push %r12
  189. .cfi_push %r12
  190. push %r13
  191. .cfi_push %r13
  192. push %r14
  193. .cfi_push %r14
  194. push %r15
  195. .cfi_push %r15
  196. lea `-104-($win64?10*16:0)`(%rsp),%rsp
  197. .cfi_adjust_cfa_offset `104+($win64?10*16:0)`
  198. #mov $in0,$inp # debugging artefact
  199. #lea 64(%rsp),$ctx # debugging artefact
  200. ___
  201. $code.=<<___ if ($win64);
  202. movaps %xmm6,96+0(%rsp)
  203. movaps %xmm7,96+16(%rsp)
  204. movaps %xmm8,96+32(%rsp)
  205. movaps %xmm9,96+48(%rsp)
  206. movaps %xmm10,96+64(%rsp)
  207. movaps %xmm11,96+80(%rsp)
  208. movaps %xmm12,96+96(%rsp)
  209. movaps %xmm13,96+112(%rsp)
  210. movaps %xmm14,96+128(%rsp)
  211. movaps %xmm15,96+144(%rsp)
  212. .Lprologue_ssse3:
  213. ___
  214. $code.=<<___;
  215. mov $in0,%r12 # reassign arguments
  216. mov $out,%r13
  217. mov $len,%r14
  218. lea 112($key),%r15 # size optimization
  219. movdqu ($ivp),$iv # load IV
  220. mov $ivp,88(%rsp) # save $ivp
  221. ___
  222. ($in0,$out,$len,$key)=map("%r$_",(12..15)); # reassign arguments
  223. my $rounds="${ivp}d";
  224. $code.=<<___;
  225. shl \$6,$len
  226. sub $in0,$out
  227. mov 240-112($key),$rounds
  228. add $inp,$len # end of input
  229. lea K_XX_XX(%rip),$K_XX_XX
  230. mov 0($ctx),$A # load context
  231. mov 4($ctx),$B
  232. mov 8($ctx),$C
  233. mov 12($ctx),$D
  234. mov $B,@T[0] # magic seed
  235. mov 16($ctx),$E
  236. mov $C,@T[1]
  237. xor $D,@T[1]
  238. and @T[1],@T[0]
  239. movdqa 64($K_XX_XX),@Tx[2] # pbswap mask
  240. movdqa 0($K_XX_XX),@Tx[1] # K_00_19
  241. movdqu 0($inp),@X[-4&7] # load input to %xmm[0-3]
  242. movdqu 16($inp),@X[-3&7]
  243. movdqu 32($inp),@X[-2&7]
  244. movdqu 48($inp),@X[-1&7]
  245. pshufb @Tx[2],@X[-4&7] # byte swap
  246. pshufb @Tx[2],@X[-3&7]
  247. pshufb @Tx[2],@X[-2&7]
  248. add \$64,$inp
  249. paddd @Tx[1],@X[-4&7] # add K_00_19
  250. pshufb @Tx[2],@X[-1&7]
  251. paddd @Tx[1],@X[-3&7]
  252. paddd @Tx[1],@X[-2&7]
  253. movdqa @X[-4&7],0(%rsp) # X[]+K xfer to IALU
  254. psubd @Tx[1],@X[-4&7] # restore X[]
  255. movdqa @X[-3&7],16(%rsp)
  256. psubd @Tx[1],@X[-3&7]
  257. movdqa @X[-2&7],32(%rsp)
  258. psubd @Tx[1],@X[-2&7]
  259. movups -112($key),$rndkey0 # $key[0]
  260. movups 16-112($key),$rndkey[0] # forward reference
  261. jmp .Loop_ssse3
  262. ___
  263. my $aesenc=sub {
  264. use integer;
  265. my ($n,$k)=($r/10,$r%10);
  266. if ($k==0) {
  267. $code.=<<___;
  268. movups `16*$n`($in0),$in # load input
  269. xorps $rndkey0,$in
  270. ___
  271. $code.=<<___ if ($n);
  272. movups $iv,`16*($n-1)`($out,$in0) # write output
  273. ___
  274. $code.=<<___;
  275. xorps $in,$iv
  276. movups `32+16*$k-112`($key),$rndkey[1]
  277. aesenc $rndkey[0],$iv
  278. ___
  279. } elsif ($k==9) {
  280. $sn++;
  281. $code.=<<___;
  282. cmp \$11,$rounds
  283. jb .Laesenclast$sn
  284. movups `32+16*($k+0)-112`($key),$rndkey[1]
  285. aesenc $rndkey[0],$iv
  286. movups `32+16*($k+1)-112`($key),$rndkey[0]
  287. aesenc $rndkey[1],$iv
  288. je .Laesenclast$sn
  289. movups `32+16*($k+2)-112`($key),$rndkey[1]
  290. aesenc $rndkey[0],$iv
  291. movups `32+16*($k+3)-112`($key),$rndkey[0]
  292. aesenc $rndkey[1],$iv
  293. .Laesenclast$sn:
  294. aesenclast $rndkey[0],$iv
  295. movups 16-112($key),$rndkey[1] # forward reference
  296. ___
  297. } else {
  298. $code.=<<___;
  299. movups `32+16*$k-112`($key),$rndkey[1]
  300. aesenc $rndkey[0],$iv
  301. ___
  302. }
  303. $r++; unshift(@rndkey,pop(@rndkey));
  304. };
  305. sub Xupdate_ssse3_16_31() # recall that $Xi starts with 4
  306. { use integer;
  307. my $body = shift;
  308. my @insns = (&$body,&$body,&$body,&$body); # 40 instructions
  309. my ($a,$b,$c,$d,$e);
  310. eval(shift(@insns)); # ror
  311. &pshufd (@X[0],@X[-4&7],0xee); # was &movdqa (@X[0],@X[-3&7]);
  312. eval(shift(@insns));
  313. &movdqa (@Tx[0],@X[-1&7]);
  314. &paddd (@Tx[1],@X[-1&7]);
  315. eval(shift(@insns));
  316. eval(shift(@insns));
  317. &punpcklqdq(@X[0],@X[-3&7]); # compose "X[-14]" in "X[0]", was &palignr(@X[0],@X[-4&7],8);
  318. eval(shift(@insns));
  319. eval(shift(@insns)); # rol
  320. eval(shift(@insns));
  321. &psrldq (@Tx[0],4); # "X[-3]", 3 dwords
  322. eval(shift(@insns));
  323. eval(shift(@insns));
  324. &pxor (@X[0],@X[-4&7]); # "X[0]"^="X[-16]"
  325. eval(shift(@insns));
  326. eval(shift(@insns)); # ror
  327. &pxor (@Tx[0],@X[-2&7]); # "X[-3]"^"X[-8]"
  328. eval(shift(@insns));
  329. eval(shift(@insns));
  330. eval(shift(@insns));
  331. &pxor (@X[0],@Tx[0]); # "X[0]"^="X[-3]"^"X[-8]"
  332. eval(shift(@insns));
  333. eval(shift(@insns)); # rol
  334. &movdqa (eval(16*(($Xi-1)&3))."(%rsp)",@Tx[1]); # X[]+K xfer to IALU
  335. eval(shift(@insns));
  336. eval(shift(@insns));
  337. &movdqa (@Tx[2],@X[0]);
  338. eval(shift(@insns));
  339. eval(shift(@insns));
  340. eval(shift(@insns)); # ror
  341. &movdqa (@Tx[0],@X[0]);
  342. eval(shift(@insns));
  343. &pslldq (@Tx[2],12); # "X[0]"<<96, extract one dword
  344. &paddd (@X[0],@X[0]);
  345. eval(shift(@insns));
  346. eval(shift(@insns));
  347. &psrld (@Tx[0],31);
  348. eval(shift(@insns));
  349. eval(shift(@insns)); # rol
  350. eval(shift(@insns));
  351. &movdqa (@Tx[1],@Tx[2]);
  352. eval(shift(@insns));
  353. eval(shift(@insns));
  354. &psrld (@Tx[2],30);
  355. eval(shift(@insns));
  356. eval(shift(@insns)); # ror
  357. &por (@X[0],@Tx[0]); # "X[0]"<<<=1
  358. eval(shift(@insns));
  359. eval(shift(@insns));
  360. eval(shift(@insns));
  361. &pslld (@Tx[1],2);
  362. &pxor (@X[0],@Tx[2]);
  363. eval(shift(@insns));
  364. &movdqa (@Tx[2],eval(16*(($Xi)/5))."($K_XX_XX)"); # K_XX_XX
  365. eval(shift(@insns)); # rol
  366. eval(shift(@insns));
  367. eval(shift(@insns));
  368. &pxor (@X[0],@Tx[1]); # "X[0]"^=("X[0]">>96)<<<2
  369. &pshufd (@Tx[1],@X[-1&7],0xee) if ($Xi==7); # was &movdqa (@Tx[0],@X[-1&7]) in Xupdate_ssse3_32_79
  370. foreach (@insns) { eval; } # remaining instructions [if any]
  371. $Xi++; push(@X,shift(@X)); # "rotate" X[]
  372. push(@Tx,shift(@Tx));
  373. }
  374. sub Xupdate_ssse3_32_79()
  375. { use integer;
  376. my $body = shift;
  377. my @insns = (&$body,&$body,&$body,&$body); # 32 to 44 instructions
  378. my ($a,$b,$c,$d,$e);
  379. eval(shift(@insns)) if ($Xi==8);
  380. &pxor (@X[0],@X[-4&7]); # "X[0]"="X[-32]"^"X[-16]"
  381. eval(shift(@insns)) if ($Xi==8);
  382. eval(shift(@insns)); # body_20_39
  383. eval(shift(@insns));
  384. eval(shift(@insns)) if (@insns[1] =~ /_ror/);
  385. eval(shift(@insns)) if (@insns[0] =~ /_ror/);
  386. &punpcklqdq(@Tx[0],@X[-1&7]); # compose "X[-6]", was &palignr(@Tx[0],@X[-2&7],8);
  387. eval(shift(@insns));
  388. eval(shift(@insns)); # rol
  389. &pxor (@X[0],@X[-7&7]); # "X[0]"^="X[-28]"
  390. eval(shift(@insns));
  391. eval(shift(@insns));
  392. if ($Xi%5) {
  393. &movdqa (@Tx[2],@Tx[1]);# "perpetuate" K_XX_XX...
  394. } else { # ... or load next one
  395. &movdqa (@Tx[2],eval(16*($Xi/5))."($K_XX_XX)");
  396. }
  397. eval(shift(@insns)); # ror
  398. &paddd (@Tx[1],@X[-1&7]);
  399. eval(shift(@insns));
  400. &pxor (@X[0],@Tx[0]); # "X[0]"^="X[-6]"
  401. eval(shift(@insns)); # body_20_39
  402. eval(shift(@insns));
  403. eval(shift(@insns));
  404. eval(shift(@insns)); # rol
  405. eval(shift(@insns)) if (@insns[0] =~ /_ror/);
  406. &movdqa (@Tx[0],@X[0]);
  407. eval(shift(@insns));
  408. eval(shift(@insns));
  409. &movdqa (eval(16*(($Xi-1)&3))."(%rsp)",@Tx[1]); # X[]+K xfer to IALU
  410. eval(shift(@insns)); # ror
  411. eval(shift(@insns));
  412. eval(shift(@insns)); # body_20_39
  413. &pslld (@X[0],2);
  414. eval(shift(@insns));
  415. eval(shift(@insns));
  416. &psrld (@Tx[0],30);
  417. eval(shift(@insns)) if (@insns[0] =~ /_rol/);# rol
  418. eval(shift(@insns));
  419. eval(shift(@insns));
  420. eval(shift(@insns)); # ror
  421. &por (@X[0],@Tx[0]); # "X[0]"<<<=2
  422. eval(shift(@insns));
  423. eval(shift(@insns)); # body_20_39
  424. eval(shift(@insns)) if (@insns[1] =~ /_rol/);
  425. eval(shift(@insns)) if (@insns[0] =~ /_rol/);
  426. &pshufd(@Tx[1],@X[-1&7],0xee) if ($Xi<19); # was &movdqa (@Tx[1],@X[0])
  427. eval(shift(@insns));
  428. eval(shift(@insns)); # rol
  429. eval(shift(@insns));
  430. eval(shift(@insns));
  431. eval(shift(@insns)); # rol
  432. eval(shift(@insns));
  433. foreach (@insns) { eval; } # remaining instructions
  434. $Xi++; push(@X,shift(@X)); # "rotate" X[]
  435. push(@Tx,shift(@Tx));
  436. }
  437. sub Xuplast_ssse3_80()
  438. { use integer;
  439. my $body = shift;
  440. my @insns = (&$body,&$body,&$body,&$body); # 32 instructions
  441. my ($a,$b,$c,$d,$e);
  442. eval(shift(@insns));
  443. eval(shift(@insns));
  444. eval(shift(@insns));
  445. eval(shift(@insns));
  446. &paddd (@Tx[1],@X[-1&7]);
  447. eval(shift(@insns));
  448. eval(shift(@insns));
  449. &movdqa (eval(16*(($Xi-1)&3))."(%rsp)",@Tx[1]); # X[]+K xfer IALU
  450. foreach (@insns) { eval; } # remaining instructions
  451. &cmp ($inp,$len);
  452. &je (shift);
  453. unshift(@Tx,pop(@Tx));
  454. &movdqa (@Tx[2],"64($K_XX_XX)"); # pbswap mask
  455. &movdqa (@Tx[1],"0($K_XX_XX)"); # K_00_19
  456. &movdqu (@X[-4&7],"0($inp)"); # load input
  457. &movdqu (@X[-3&7],"16($inp)");
  458. &movdqu (@X[-2&7],"32($inp)");
  459. &movdqu (@X[-1&7],"48($inp)");
  460. &pshufb (@X[-4&7],@Tx[2]); # byte swap
  461. &add ($inp,64);
  462. $Xi=0;
  463. }
  464. sub Xloop_ssse3()
  465. { use integer;
  466. my $body = shift;
  467. my @insns = (&$body,&$body,&$body,&$body); # 32 instructions
  468. my ($a,$b,$c,$d,$e);
  469. eval(shift(@insns));
  470. eval(shift(@insns));
  471. eval(shift(@insns));
  472. &pshufb (@X[($Xi-3)&7],@Tx[2]);
  473. eval(shift(@insns));
  474. eval(shift(@insns));
  475. eval(shift(@insns));
  476. eval(shift(@insns));
  477. &paddd (@X[($Xi-4)&7],@Tx[1]);
  478. eval(shift(@insns));
  479. eval(shift(@insns));
  480. eval(shift(@insns));
  481. eval(shift(@insns));
  482. &movdqa (eval(16*$Xi)."(%rsp)",@X[($Xi-4)&7]); # X[]+K xfer to IALU
  483. eval(shift(@insns));
  484. eval(shift(@insns));
  485. eval(shift(@insns));
  486. eval(shift(@insns));
  487. &psubd (@X[($Xi-4)&7],@Tx[1]);
  488. foreach (@insns) { eval; }
  489. $Xi++;
  490. }
  491. sub Xtail_ssse3()
  492. { use integer;
  493. my $body = shift;
  494. my @insns = (&$body,&$body,&$body,&$body); # 32 instructions
  495. my ($a,$b,$c,$d,$e);
  496. foreach (@insns) { eval; }
  497. }
  498. my @body_00_19 = (
  499. '($a,$b,$c,$d,$e)=@V;'.
  500. '&$_ror ($b,$j?7:2);', # $b>>>2
  501. '&xor (@T[0],$d);',
  502. '&mov (@T[1],$a);', # $b for next round
  503. '&add ($e,eval(4*($j&15))."(%rsp)");',# X[]+K xfer
  504. '&xor ($b,$c);', # $c^$d for next round
  505. '&$_rol ($a,5);',
  506. '&add ($e,@T[0]);',
  507. '&and (@T[1],$b);', # ($b&($c^$d)) for next round
  508. '&xor ($b,$c);', # restore $b
  509. '&add ($e,$a);' .'$j++; unshift(@V,pop(@V)); unshift(@T,pop(@T));'
  510. );
  511. sub body_00_19 () { # ((c^d)&b)^d
  512. # on start @T[0]=(c^d)&b
  513. return &body_20_39() if ($rx==19); $rx++;
  514. use integer;
  515. my ($k,$n);
  516. my @r=@body_00_19;
  517. $n = scalar(@r);
  518. $k = (($jj+1)*12/20)*20*$n/12; # 12 aesencs per these 20 rounds
  519. @r[$k%$n].='&$aesenc();' if ($jj==$k/$n);
  520. $jj++;
  521. return @r;
  522. }
  523. my @body_20_39 = (
  524. '($a,$b,$c,$d,$e)=@V;'.
  525. '&add ($e,eval(4*($j&15))."(%rsp)");',# X[]+K xfer
  526. '&xor (@T[0],$d) if($j==19);'.
  527. '&xor (@T[0],$c) if($j> 19);', # ($b^$d^$c)
  528. '&mov (@T[1],$a);', # $b for next round
  529. '&$_rol ($a,5);',
  530. '&add ($e,@T[0]);',
  531. '&xor (@T[1],$c) if ($j< 79);', # $b^$d for next round
  532. '&$_ror ($b,7);', # $b>>>2
  533. '&add ($e,$a);' .'$j++; unshift(@V,pop(@V)); unshift(@T,pop(@T));'
  534. );
  535. sub body_20_39 () { # b^d^c
  536. # on entry @T[0]=b^d
  537. return &body_40_59() if ($rx==39); $rx++;
  538. use integer;
  539. my ($k,$n);
  540. my @r=@body_20_39;
  541. $n = scalar(@r);
  542. $k = (($jj+1)*8/20)*20*$n/8; # 8 aesencs per these 20 rounds
  543. @r[$k%$n].='&$aesenc();' if ($jj==$k/$n && $rx!=20);
  544. $jj++;
  545. return @r;
  546. }
  547. my @body_40_59 = (
  548. '($a,$b,$c,$d,$e)=@V;'.
  549. '&add ($e,eval(4*($j&15))."(%rsp)");',# X[]+K xfer
  550. '&and (@T[0],$c) if ($j>=40);', # (b^c)&(c^d)
  551. '&xor ($c,$d) if ($j>=40);', # restore $c
  552. '&$_ror ($b,7);', # $b>>>2
  553. '&mov (@T[1],$a);', # $b for next round
  554. '&xor (@T[0],$c);',
  555. '&$_rol ($a,5);',
  556. '&add ($e,@T[0]);',
  557. '&xor (@T[1],$c) if ($j==59);'.
  558. '&xor (@T[1],$b) if ($j< 59);', # b^c for next round
  559. '&xor ($b,$c) if ($j< 59);', # c^d for next round
  560. '&add ($e,$a);' .'$j++; unshift(@V,pop(@V)); unshift(@T,pop(@T));'
  561. );
  562. sub body_40_59 () { # ((b^c)&(c^d))^c
  563. # on entry @T[0]=(b^c), (c^=d)
  564. $rx++;
  565. use integer;
  566. my ($k,$n);
  567. my @r=@body_40_59;
  568. $n = scalar(@r);
  569. $k=(($jj+1)*12/20)*20*$n/12; # 12 aesencs per these 20 rounds
  570. @r[$k%$n].='&$aesenc();' if ($jj==$k/$n && $rx!=40);
  571. $jj++;
  572. return @r;
  573. }
  574. $code.=<<___;
  575. .align 32
  576. .Loop_ssse3:
  577. ___
  578. &Xupdate_ssse3_16_31(\&body_00_19);
  579. &Xupdate_ssse3_16_31(\&body_00_19);
  580. &Xupdate_ssse3_16_31(\&body_00_19);
  581. &Xupdate_ssse3_16_31(\&body_00_19);
  582. &Xupdate_ssse3_32_79(\&body_00_19);
  583. &Xupdate_ssse3_32_79(\&body_20_39);
  584. &Xupdate_ssse3_32_79(\&body_20_39);
  585. &Xupdate_ssse3_32_79(\&body_20_39);
  586. &Xupdate_ssse3_32_79(\&body_20_39);
  587. &Xupdate_ssse3_32_79(\&body_20_39);
  588. &Xupdate_ssse3_32_79(\&body_40_59);
  589. &Xupdate_ssse3_32_79(\&body_40_59);
  590. &Xupdate_ssse3_32_79(\&body_40_59);
  591. &Xupdate_ssse3_32_79(\&body_40_59);
  592. &Xupdate_ssse3_32_79(\&body_40_59);
  593. &Xupdate_ssse3_32_79(\&body_20_39);
  594. &Xuplast_ssse3_80(\&body_20_39,".Ldone_ssse3"); # can jump to "done"
  595. $saved_j=$j; @saved_V=@V;
  596. $saved_r=$r; @saved_rndkey=@rndkey;
  597. &Xloop_ssse3(\&body_20_39);
  598. &Xloop_ssse3(\&body_20_39);
  599. &Xloop_ssse3(\&body_20_39);
  600. $code.=<<___;
  601. movups $iv,48($out,$in0) # write output
  602. lea 64($in0),$in0
  603. add 0($ctx),$A # update context
  604. add 4($ctx),@T[0]
  605. add 8($ctx),$C
  606. add 12($ctx),$D
  607. mov $A,0($ctx)
  608. add 16($ctx),$E
  609. mov @T[0],4($ctx)
  610. mov @T[0],$B # magic seed
  611. mov $C,8($ctx)
  612. mov $C,@T[1]
  613. mov $D,12($ctx)
  614. xor $D,@T[1]
  615. mov $E,16($ctx)
  616. and @T[1],@T[0]
  617. jmp .Loop_ssse3
  618. .Ldone_ssse3:
  619. ___
  620. $jj=$j=$saved_j; @V=@saved_V;
  621. $r=$saved_r; @rndkey=@saved_rndkey;
  622. &Xtail_ssse3(\&body_20_39);
  623. &Xtail_ssse3(\&body_20_39);
  624. &Xtail_ssse3(\&body_20_39);
  625. $code.=<<___;
  626. movups $iv,48($out,$in0) # write output
  627. mov 88(%rsp),$ivp # restore $ivp
  628. add 0($ctx),$A # update context
  629. add 4($ctx),@T[0]
  630. add 8($ctx),$C
  631. mov $A,0($ctx)
  632. add 12($ctx),$D
  633. mov @T[0],4($ctx)
  634. add 16($ctx),$E
  635. mov $C,8($ctx)
  636. mov $D,12($ctx)
  637. mov $E,16($ctx)
  638. movups $iv,($ivp) # write IV
  639. ___
  640. $code.=<<___ if ($win64);
  641. movaps 96+0(%rsp),%xmm6
  642. movaps 96+16(%rsp),%xmm7
  643. movaps 96+32(%rsp),%xmm8
  644. movaps 96+48(%rsp),%xmm9
  645. movaps 96+64(%rsp),%xmm10
  646. movaps 96+80(%rsp),%xmm11
  647. movaps 96+96(%rsp),%xmm12
  648. movaps 96+112(%rsp),%xmm13
  649. movaps 96+128(%rsp),%xmm14
  650. movaps 96+144(%rsp),%xmm15
  651. ___
  652. $code.=<<___;
  653. lea `104+($win64?10*16:0)`(%rsp),%rsi
  654. .cfi_def_cfa %rsi,56
  655. mov 0(%rsi),%r15
  656. .cfi_restore %r15
  657. mov 8(%rsi),%r14
  658. .cfi_restore %r14
  659. mov 16(%rsi),%r13
  660. .cfi_restore %r13
  661. mov 24(%rsi),%r12
  662. .cfi_restore %r12
  663. mov 32(%rsi),%rbp
  664. .cfi_restore %rbp
  665. mov 40(%rsi),%rbx
  666. .cfi_restore %rbx
  667. lea 48(%rsi),%rsp
  668. .cfi_def_cfa %rsp,8
  669. .Lepilogue_ssse3:
  670. ret
  671. .cfi_endproc
  672. .size aesni_cbc_sha1_enc_ssse3,.-aesni_cbc_sha1_enc_ssse3
  673. ___
  674. if ($stitched_decrypt) {{{
  675. # reset
  676. ($in0,$out,$len,$key,$ivp,$ctx,$inp)=("%rdi","%rsi","%rdx","%rcx","%r8","%r9","%r10");
  677. $j=$jj=$r=$rx=0;
  678. $Xi=4;
  679. # reassign for Atom Silvermont (see above)
  680. ($inout0,$inout1,$inout2,$inout3,$rndkey0)=map("%xmm$_",(0..4));
  681. @X=map("%xmm$_",(8..13,6,7));
  682. @Tx=map("%xmm$_",(14,15,5));
  683. my @aes256_dec = (
  684. '&movdqu($inout0,"0x00($in0)");',
  685. '&movdqu($inout1,"0x10($in0)"); &pxor ($inout0,$rndkey0);',
  686. '&movdqu($inout2,"0x20($in0)"); &pxor ($inout1,$rndkey0);',
  687. '&movdqu($inout3,"0x30($in0)"); &pxor ($inout2,$rndkey0);',
  688. '&pxor ($inout3,$rndkey0); &movups ($rndkey0,"16-112($key)");',
  689. '&movaps("64(%rsp)",@X[2]);', # save IV, originally @X[3]
  690. undef,undef
  691. );
  692. for ($i=0;$i<13;$i++) {
  693. push (@aes256_dec,(
  694. '&aesdec ($inout0,$rndkey0);',
  695. '&aesdec ($inout1,$rndkey0);',
  696. '&aesdec ($inout2,$rndkey0);',
  697. '&aesdec ($inout3,$rndkey0); &movups($rndkey0,"'.(16*($i+2)-112).'($key)");'
  698. ));
  699. push (@aes256_dec,(undef,undef)) if (($i>=3 && $i<=5) || $i>=11);
  700. push (@aes256_dec,(undef,undef)) if ($i==5);
  701. }
  702. push(@aes256_dec,(
  703. '&aesdeclast ($inout0,$rndkey0); &movups (@X[0],"0x00($in0)");',
  704. '&aesdeclast ($inout1,$rndkey0); &movups (@X[1],"0x10($in0)");',
  705. '&aesdeclast ($inout2,$rndkey0); &movups (@X[2],"0x20($in0)");',
  706. '&aesdeclast ($inout3,$rndkey0); &movups (@X[3],"0x30($in0)");',
  707. '&xorps ($inout0,"64(%rsp)"); &movdqu ($rndkey0,"-112($key)");',
  708. '&xorps ($inout1,@X[0]); &movups ("0x00($out,$in0)",$inout0);',
  709. '&xorps ($inout2,@X[1]); &movups ("0x10($out,$in0)",$inout1);',
  710. '&xorps ($inout3,@X[2]); &movups ("0x20($out,$in0)",$inout2);',
  711. '&movups ("0x30($out,$in0)",$inout3);'
  712. ));
  713. sub body_00_19_dec () { # ((c^d)&b)^d
  714. # on start @T[0]=(c^d)&b
  715. return &body_20_39_dec() if ($rx==19);
  716. my @r=@body_00_19;
  717. unshift (@r,@aes256_dec[$rx]) if (@aes256_dec[$rx]);
  718. $rx++;
  719. return @r;
  720. }
  721. sub body_20_39_dec () { # b^d^c
  722. # on entry @T[0]=b^d
  723. return &body_40_59_dec() if ($rx==39);
  724. my @r=@body_20_39;
  725. unshift (@r,@aes256_dec[$rx]) if (@aes256_dec[$rx]);
  726. $rx++;
  727. return @r;
  728. }
  729. sub body_40_59_dec () { # ((b^c)&(c^d))^c
  730. # on entry @T[0]=(b^c), (c^=d)
  731. my @r=@body_40_59;
  732. unshift (@r,@aes256_dec[$rx]) if (@aes256_dec[$rx]);
  733. $rx++;
  734. return @r;
  735. }
  736. $code.=<<___;
  737. .globl aesni256_cbc_sha1_dec
  738. .type aesni256_cbc_sha1_dec,\@abi-omnipotent
  739. .align 32
  740. aesni256_cbc_sha1_dec:
  741. .cfi_startproc
  742. # caller should check for SSSE3 and AES-NI bits
  743. mov OPENSSL_ia32cap_P+0(%rip),%r10d
  744. mov OPENSSL_ia32cap_P+4(%rip),%r11d
  745. ___
  746. $code.=<<___ if ($avx);
  747. and \$`1<<28`,%r11d # mask AVX bit
  748. and \$`1<<30`,%r10d # mask "Intel CPU" bit
  749. or %r11d,%r10d
  750. cmp \$`1<<28|1<<30`,%r10d
  751. je aesni256_cbc_sha1_dec_avx
  752. ___
  753. $code.=<<___;
  754. jmp aesni256_cbc_sha1_dec_ssse3
  755. ret
  756. .cfi_endproc
  757. .size aesni256_cbc_sha1_dec,.-aesni256_cbc_sha1_dec
  758. .type aesni256_cbc_sha1_dec_ssse3,\@function,6
  759. .align 32
  760. aesni256_cbc_sha1_dec_ssse3:
  761. .cfi_startproc
  762. mov `($win64?56:8)`(%rsp),$inp # load 7th argument
  763. push %rbx
  764. .cfi_push %rbx
  765. push %rbp
  766. .cfi_push %rbp
  767. push %r12
  768. .cfi_push %r12
  769. push %r13
  770. .cfi_push %r13
  771. push %r14
  772. .cfi_push %r14
  773. push %r15
  774. .cfi_push %r15
  775. lea `-104-($win64?10*16:0)`(%rsp),%rsp
  776. .cfi_adjust_cfa_offset `104+($win64?10*16:0)`
  777. ___
  778. $code.=<<___ if ($win64);
  779. movaps %xmm6,96+0(%rsp)
  780. movaps %xmm7,96+16(%rsp)
  781. movaps %xmm8,96+32(%rsp)
  782. movaps %xmm9,96+48(%rsp)
  783. movaps %xmm10,96+64(%rsp)
  784. movaps %xmm11,96+80(%rsp)
  785. movaps %xmm12,96+96(%rsp)
  786. movaps %xmm13,96+112(%rsp)
  787. movaps %xmm14,96+128(%rsp)
  788. movaps %xmm15,96+144(%rsp)
  789. .Lprologue_dec_ssse3:
  790. ___
  791. $code.=<<___;
  792. mov $in0,%r12 # reassign arguments
  793. mov $out,%r13
  794. mov $len,%r14
  795. lea 112($key),%r15 # size optimization
  796. movdqu ($ivp),@X[3] # load IV
  797. #mov $ivp,88(%rsp) # save $ivp
  798. ___
  799. ($in0,$out,$len,$key)=map("%r$_",(12..15)); # reassign arguments
  800. $code.=<<___;
  801. shl \$6,$len
  802. sub $in0,$out
  803. add $inp,$len # end of input
  804. lea K_XX_XX(%rip),$K_XX_XX
  805. mov 0($ctx),$A # load context
  806. mov 4($ctx),$B
  807. mov 8($ctx),$C
  808. mov 12($ctx),$D
  809. mov $B,@T[0] # magic seed
  810. mov 16($ctx),$E
  811. mov $C,@T[1]
  812. xor $D,@T[1]
  813. and @T[1],@T[0]
  814. movdqa 64($K_XX_XX),@Tx[2] # pbswap mask
  815. movdqa 0($K_XX_XX),@Tx[1] # K_00_19
  816. movdqu 0($inp),@X[-4&7] # load input to %xmm[0-3]
  817. movdqu 16($inp),@X[-3&7]
  818. movdqu 32($inp),@X[-2&7]
  819. movdqu 48($inp),@X[-1&7]
  820. pshufb @Tx[2],@X[-4&7] # byte swap
  821. add \$64,$inp
  822. pshufb @Tx[2],@X[-3&7]
  823. pshufb @Tx[2],@X[-2&7]
  824. pshufb @Tx[2],@X[-1&7]
  825. paddd @Tx[1],@X[-4&7] # add K_00_19
  826. paddd @Tx[1],@X[-3&7]
  827. paddd @Tx[1],@X[-2&7]
  828. movdqa @X[-4&7],0(%rsp) # X[]+K xfer to IALU
  829. psubd @Tx[1],@X[-4&7] # restore X[]
  830. movdqa @X[-3&7],16(%rsp)
  831. psubd @Tx[1],@X[-3&7]
  832. movdqa @X[-2&7],32(%rsp)
  833. psubd @Tx[1],@X[-2&7]
  834. movdqu -112($key),$rndkey0 # $key[0]
  835. jmp .Loop_dec_ssse3
  836. .align 32
  837. .Loop_dec_ssse3:
  838. ___
  839. &Xupdate_ssse3_16_31(\&body_00_19_dec);
  840. &Xupdate_ssse3_16_31(\&body_00_19_dec);
  841. &Xupdate_ssse3_16_31(\&body_00_19_dec);
  842. &Xupdate_ssse3_16_31(\&body_00_19_dec);
  843. &Xupdate_ssse3_32_79(\&body_00_19_dec);
  844. &Xupdate_ssse3_32_79(\&body_20_39_dec);
  845. &Xupdate_ssse3_32_79(\&body_20_39_dec);
  846. &Xupdate_ssse3_32_79(\&body_20_39_dec);
  847. &Xupdate_ssse3_32_79(\&body_20_39_dec);
  848. &Xupdate_ssse3_32_79(\&body_20_39_dec);
  849. &Xupdate_ssse3_32_79(\&body_40_59_dec);
  850. &Xupdate_ssse3_32_79(\&body_40_59_dec);
  851. &Xupdate_ssse3_32_79(\&body_40_59_dec);
  852. &Xupdate_ssse3_32_79(\&body_40_59_dec);
  853. &Xupdate_ssse3_32_79(\&body_40_59_dec);
  854. &Xupdate_ssse3_32_79(\&body_20_39_dec);
  855. &Xuplast_ssse3_80(\&body_20_39_dec,".Ldone_dec_ssse3"); # can jump to "done"
  856. $saved_j=$j; @saved_V=@V;
  857. $saved_rx=$rx;
  858. &Xloop_ssse3(\&body_20_39_dec);
  859. &Xloop_ssse3(\&body_20_39_dec);
  860. &Xloop_ssse3(\&body_20_39_dec);
  861. eval(@aes256_dec[-1]); # last store
  862. $code.=<<___;
  863. lea 64($in0),$in0
  864. add 0($ctx),$A # update context
  865. add 4($ctx),@T[0]
  866. add 8($ctx),$C
  867. add 12($ctx),$D
  868. mov $A,0($ctx)
  869. add 16($ctx),$E
  870. mov @T[0],4($ctx)
  871. mov @T[0],$B # magic seed
  872. mov $C,8($ctx)
  873. mov $C,@T[1]
  874. mov $D,12($ctx)
  875. xor $D,@T[1]
  876. mov $E,16($ctx)
  877. and @T[1],@T[0]
  878. jmp .Loop_dec_ssse3
  879. .Ldone_dec_ssse3:
  880. ___
  881. $jj=$j=$saved_j; @V=@saved_V;
  882. $rx=$saved_rx;
  883. &Xtail_ssse3(\&body_20_39_dec);
  884. &Xtail_ssse3(\&body_20_39_dec);
  885. &Xtail_ssse3(\&body_20_39_dec);
  886. eval(@aes256_dec[-1]); # last store
  887. $code.=<<___;
  888. add 0($ctx),$A # update context
  889. add 4($ctx),@T[0]
  890. add 8($ctx),$C
  891. mov $A,0($ctx)
  892. add 12($ctx),$D
  893. mov @T[0],4($ctx)
  894. add 16($ctx),$E
  895. mov $C,8($ctx)
  896. mov $D,12($ctx)
  897. mov $E,16($ctx)
  898. movups @X[3],($ivp) # write IV
  899. ___
  900. $code.=<<___ if ($win64);
  901. movaps 96+0(%rsp),%xmm6
  902. movaps 96+16(%rsp),%xmm7
  903. movaps 96+32(%rsp),%xmm8
  904. movaps 96+48(%rsp),%xmm9
  905. movaps 96+64(%rsp),%xmm10
  906. movaps 96+80(%rsp),%xmm11
  907. movaps 96+96(%rsp),%xmm12
  908. movaps 96+112(%rsp),%xmm13
  909. movaps 96+128(%rsp),%xmm14
  910. movaps 96+144(%rsp),%xmm15
  911. ___
  912. $code.=<<___;
  913. lea `104+($win64?10*16:0)`(%rsp),%rsi
  914. .cfi_cfa_def %rsi,56
  915. mov 0(%rsi),%r15
  916. .cfi_restore %r15
  917. mov 8(%rsi),%r14
  918. .cfi_restore %r14
  919. mov 16(%rsi),%r13
  920. .cfi_restore %r13
  921. mov 24(%rsi),%r12
  922. .cfi_restore %r12
  923. mov 32(%rsi),%rbp
  924. .cfi_restore %rbp
  925. mov 40(%rsi),%rbx
  926. .cfi_restore %rbx
  927. lea 48(%rsi),%rsp
  928. .cfi_cfa_def %rsp,8
  929. .Lepilogue_dec_ssse3:
  930. ret
  931. .cfi_endproc
  932. .size aesni256_cbc_sha1_dec_ssse3,.-aesni256_cbc_sha1_dec_ssse3
  933. ___
  934. }}}
  935. $j=$jj=$r=$rx=0;
  936. if ($avx) {
  937. my ($in0,$out,$len,$key,$ivp,$ctx,$inp)=("%rdi","%rsi","%rdx","%rcx","%r8","%r9","%r10");
  938. my $Xi=4;
  939. my @X=map("%xmm$_",(4..7,0..3));
  940. my @Tx=map("%xmm$_",(8..10));
  941. my @V=($A,$B,$C,$D,$E)=("%eax","%ebx","%ecx","%edx","%ebp"); # size optimization
  942. my @T=("%esi","%edi");
  943. my ($rndkey0,$iv,$in)=map("%xmm$_",(11..13));
  944. my @rndkey=("%xmm14","%xmm15");
  945. my ($inout0,$inout1,$inout2,$inout3)=map("%xmm$_",(12..15)); # for dec
  946. my $Kx=@Tx[2];
  947. my $_rol=sub { &shld(@_[0],@_) };
  948. my $_ror=sub { &shrd(@_[0],@_) };
  949. $code.=<<___;
  950. .type aesni_cbc_sha1_enc_avx,\@function,6
  951. .align 32
  952. aesni_cbc_sha1_enc_avx:
  953. .cfi_startproc
  954. mov `($win64?56:8)`(%rsp),$inp # load 7th argument
  955. #shr \$6,$len # debugging artefact
  956. #jz .Lepilogue_avx # debugging artefact
  957. push %rbx
  958. .cfi_push %rbx
  959. push %rbp
  960. .cfi_push %rbp
  961. push %r12
  962. .cfi_push %r12
  963. push %r13
  964. .cfi_push %r13
  965. push %r14
  966. .cfi_push %r14
  967. push %r15
  968. .cfi_push %r15
  969. lea `-104-($win64?10*16:0)`(%rsp),%rsp
  970. .cfi_adjust_cfa_offset `104+($win64?10*16:0)`
  971. #mov $in0,$inp # debugging artefact
  972. #lea 64(%rsp),$ctx # debugging artefact
  973. ___
  974. $code.=<<___ if ($win64);
  975. movaps %xmm6,96+0(%rsp)
  976. movaps %xmm7,96+16(%rsp)
  977. movaps %xmm8,96+32(%rsp)
  978. movaps %xmm9,96+48(%rsp)
  979. movaps %xmm10,96+64(%rsp)
  980. movaps %xmm11,96+80(%rsp)
  981. movaps %xmm12,96+96(%rsp)
  982. movaps %xmm13,96+112(%rsp)
  983. movaps %xmm14,96+128(%rsp)
  984. movaps %xmm15,96+144(%rsp)
  985. .Lprologue_avx:
  986. ___
  987. $code.=<<___;
  988. vzeroall
  989. mov $in0,%r12 # reassign arguments
  990. mov $out,%r13
  991. mov $len,%r14
  992. lea 112($key),%r15 # size optimization
  993. vmovdqu ($ivp),$iv # load IV
  994. mov $ivp,88(%rsp) # save $ivp
  995. ___
  996. ($in0,$out,$len,$key)=map("%r$_",(12..15)); # reassign arguments
  997. my $rounds="${ivp}d";
  998. $code.=<<___;
  999. shl \$6,$len
  1000. sub $in0,$out
  1001. mov 240-112($key),$rounds
  1002. add $inp,$len # end of input
  1003. lea K_XX_XX(%rip),$K_XX_XX
  1004. mov 0($ctx),$A # load context
  1005. mov 4($ctx),$B
  1006. mov 8($ctx),$C
  1007. mov 12($ctx),$D
  1008. mov $B,@T[0] # magic seed
  1009. mov 16($ctx),$E
  1010. mov $C,@T[1]
  1011. xor $D,@T[1]
  1012. and @T[1],@T[0]
  1013. vmovdqa 64($K_XX_XX),@X[2] # pbswap mask
  1014. vmovdqa 0($K_XX_XX),$Kx # K_00_19
  1015. vmovdqu 0($inp),@X[-4&7] # load input to %xmm[0-3]
  1016. vmovdqu 16($inp),@X[-3&7]
  1017. vmovdqu 32($inp),@X[-2&7]
  1018. vmovdqu 48($inp),@X[-1&7]
  1019. vpshufb @X[2],@X[-4&7],@X[-4&7] # byte swap
  1020. add \$64,$inp
  1021. vpshufb @X[2],@X[-3&7],@X[-3&7]
  1022. vpshufb @X[2],@X[-2&7],@X[-2&7]
  1023. vpshufb @X[2],@X[-1&7],@X[-1&7]
  1024. vpaddd $Kx,@X[-4&7],@X[0] # add K_00_19
  1025. vpaddd $Kx,@X[-3&7],@X[1]
  1026. vpaddd $Kx,@X[-2&7],@X[2]
  1027. vmovdqa @X[0],0(%rsp) # X[]+K xfer to IALU
  1028. vmovdqa @X[1],16(%rsp)
  1029. vmovdqa @X[2],32(%rsp)
  1030. vmovups -112($key),$rndkey[1] # $key[0]
  1031. vmovups 16-112($key),$rndkey[0] # forward reference
  1032. jmp .Loop_avx
  1033. ___
  1034. my $aesenc=sub {
  1035. use integer;
  1036. my ($n,$k)=($r/10,$r%10);
  1037. if ($k==0) {
  1038. $code.=<<___;
  1039. vmovdqu `16*$n`($in0),$in # load input
  1040. vpxor $rndkey[1],$in,$in
  1041. ___
  1042. $code.=<<___ if ($n);
  1043. vmovups $iv,`16*($n-1)`($out,$in0) # write output
  1044. ___
  1045. $code.=<<___;
  1046. vpxor $in,$iv,$iv
  1047. vaesenc $rndkey[0],$iv,$iv
  1048. vmovups `32+16*$k-112`($key),$rndkey[1]
  1049. ___
  1050. } elsif ($k==9) {
  1051. $sn++;
  1052. $code.=<<___;
  1053. cmp \$11,$rounds
  1054. jb .Lvaesenclast$sn
  1055. vaesenc $rndkey[0],$iv,$iv
  1056. vmovups `32+16*($k+0)-112`($key),$rndkey[1]
  1057. vaesenc $rndkey[1],$iv,$iv
  1058. vmovups `32+16*($k+1)-112`($key),$rndkey[0]
  1059. je .Lvaesenclast$sn
  1060. vaesenc $rndkey[0],$iv,$iv
  1061. vmovups `32+16*($k+2)-112`($key),$rndkey[1]
  1062. vaesenc $rndkey[1],$iv,$iv
  1063. vmovups `32+16*($k+3)-112`($key),$rndkey[0]
  1064. .Lvaesenclast$sn:
  1065. vaesenclast $rndkey[0],$iv,$iv
  1066. vmovups -112($key),$rndkey[0]
  1067. vmovups 16-112($key),$rndkey[1] # forward reference
  1068. ___
  1069. } else {
  1070. $code.=<<___;
  1071. vaesenc $rndkey[0],$iv,$iv
  1072. vmovups `32+16*$k-112`($key),$rndkey[1]
  1073. ___
  1074. }
  1075. $r++; unshift(@rndkey,pop(@rndkey));
  1076. };
  1077. sub Xupdate_avx_16_31() # recall that $Xi starts with 4
  1078. { use integer;
  1079. my $body = shift;
  1080. my @insns = (&$body,&$body,&$body,&$body); # 40 instructions
  1081. my ($a,$b,$c,$d,$e);
  1082. eval(shift(@insns));
  1083. eval(shift(@insns));
  1084. &vpalignr(@X[0],@X[-3&7],@X[-4&7],8); # compose "X[-14]" in "X[0]"
  1085. eval(shift(@insns));
  1086. eval(shift(@insns));
  1087. &vpaddd (@Tx[1],$Kx,@X[-1&7]);
  1088. eval(shift(@insns));
  1089. eval(shift(@insns));
  1090. &vpsrldq(@Tx[0],@X[-1&7],4); # "X[-3]", 3 dwords
  1091. eval(shift(@insns));
  1092. eval(shift(@insns));
  1093. &vpxor (@X[0],@X[0],@X[-4&7]); # "X[0]"^="X[-16]"
  1094. eval(shift(@insns));
  1095. eval(shift(@insns));
  1096. &vpxor (@Tx[0],@Tx[0],@X[-2&7]); # "X[-3]"^"X[-8]"
  1097. eval(shift(@insns));
  1098. eval(shift(@insns));
  1099. eval(shift(@insns));
  1100. eval(shift(@insns));
  1101. &vpxor (@X[0],@X[0],@Tx[0]); # "X[0]"^="X[-3]"^"X[-8]"
  1102. eval(shift(@insns));
  1103. eval(shift(@insns));
  1104. &vmovdqa (eval(16*(($Xi-1)&3))."(%rsp)",@Tx[1]); # X[]+K xfer to IALU
  1105. eval(shift(@insns));
  1106. eval(shift(@insns));
  1107. &vpsrld (@Tx[0],@X[0],31);
  1108. eval(shift(@insns));
  1109. eval(shift(@insns));
  1110. eval(shift(@insns));
  1111. eval(shift(@insns));
  1112. &vpslldq(@Tx[1],@X[0],12); # "X[0]"<<96, extract one dword
  1113. &vpaddd (@X[0],@X[0],@X[0]);
  1114. eval(shift(@insns));
  1115. eval(shift(@insns));
  1116. eval(shift(@insns));
  1117. eval(shift(@insns));
  1118. &vpor (@X[0],@X[0],@Tx[0]); # "X[0]"<<<=1
  1119. &vpsrld (@Tx[0],@Tx[1],30);
  1120. eval(shift(@insns));
  1121. eval(shift(@insns));
  1122. eval(shift(@insns));
  1123. eval(shift(@insns));
  1124. &vpslld (@Tx[1],@Tx[1],2);
  1125. &vpxor (@X[0],@X[0],@Tx[0]);
  1126. eval(shift(@insns));
  1127. eval(shift(@insns));
  1128. eval(shift(@insns));
  1129. eval(shift(@insns));
  1130. &vpxor (@X[0],@X[0],@Tx[1]); # "X[0]"^=("X[0]">>96)<<<2
  1131. eval(shift(@insns));
  1132. eval(shift(@insns));
  1133. &vmovdqa ($Kx,eval(16*(($Xi)/5))."($K_XX_XX)") if ($Xi%5==0); # K_XX_XX
  1134. eval(shift(@insns));
  1135. eval(shift(@insns));
  1136. foreach (@insns) { eval; } # remaining instructions [if any]
  1137. $Xi++; push(@X,shift(@X)); # "rotate" X[]
  1138. }
  1139. sub Xupdate_avx_32_79()
  1140. { use integer;
  1141. my $body = shift;
  1142. my @insns = (&$body,&$body,&$body,&$body); # 32 to 48 instructions
  1143. my ($a,$b,$c,$d,$e);
  1144. &vpalignr(@Tx[0],@X[-1&7],@X[-2&7],8); # compose "X[-6]"
  1145. &vpxor (@X[0],@X[0],@X[-4&7]); # "X[0]"="X[-32]"^"X[-16]"
  1146. eval(shift(@insns)); # body_20_39
  1147. eval(shift(@insns));
  1148. eval(shift(@insns));
  1149. eval(shift(@insns)); # rol
  1150. &vpxor (@X[0],@X[0],@X[-7&7]); # "X[0]"^="X[-28]"
  1151. eval(shift(@insns));
  1152. eval(shift(@insns)) if (@insns[0] !~ /&ro[rl]/);
  1153. &vpaddd (@Tx[1],$Kx,@X[-1&7]);
  1154. &vmovdqa ($Kx,eval(16*($Xi/5))."($K_XX_XX)") if ($Xi%5==0);
  1155. eval(shift(@insns)); # ror
  1156. eval(shift(@insns));
  1157. &vpxor (@X[0],@X[0],@Tx[0]); # "X[0]"^="X[-6]"
  1158. eval(shift(@insns)); # body_20_39
  1159. eval(shift(@insns));
  1160. eval(shift(@insns));
  1161. eval(shift(@insns)); # rol
  1162. &vpsrld (@Tx[0],@X[0],30);
  1163. &vmovdqa (eval(16*(($Xi-1)&3))."(%rsp)",@Tx[1]); # X[]+K xfer to IALU
  1164. eval(shift(@insns));
  1165. eval(shift(@insns));
  1166. eval(shift(@insns)); # ror
  1167. eval(shift(@insns));
  1168. &vpslld (@X[0],@X[0],2);
  1169. eval(shift(@insns)); # body_20_39
  1170. eval(shift(@insns));
  1171. eval(shift(@insns));
  1172. eval(shift(@insns)); # rol
  1173. eval(shift(@insns));
  1174. eval(shift(@insns));
  1175. eval(shift(@insns)); # ror
  1176. eval(shift(@insns));
  1177. &vpor (@X[0],@X[0],@Tx[0]); # "X[0]"<<<=2
  1178. eval(shift(@insns)); # body_20_39
  1179. eval(shift(@insns));
  1180. eval(shift(@insns));
  1181. eval(shift(@insns)); # rol
  1182. eval(shift(@insns));
  1183. eval(shift(@insns));
  1184. eval(shift(@insns)); # rol
  1185. eval(shift(@insns));
  1186. foreach (@insns) { eval; } # remaining instructions
  1187. $Xi++; push(@X,shift(@X)); # "rotate" X[]
  1188. }
  1189. sub Xuplast_avx_80()
  1190. { use integer;
  1191. my $body = shift;
  1192. my @insns = (&$body,&$body,&$body,&$body); # 32 instructions
  1193. my ($a,$b,$c,$d,$e);
  1194. eval(shift(@insns));
  1195. &vpaddd (@Tx[1],$Kx,@X[-1&7]);
  1196. eval(shift(@insns));
  1197. eval(shift(@insns));
  1198. eval(shift(@insns));
  1199. eval(shift(@insns));
  1200. &vmovdqa (eval(16*(($Xi-1)&3))."(%rsp)",@Tx[1]); # X[]+K xfer IALU
  1201. foreach (@insns) { eval; } # remaining instructions
  1202. &cmp ($inp,$len);
  1203. &je (shift);
  1204. &vmovdqa(@Tx[1],"64($K_XX_XX)"); # pbswap mask
  1205. &vmovdqa($Kx,"0($K_XX_XX)"); # K_00_19
  1206. &vmovdqu(@X[-4&7],"0($inp)"); # load input
  1207. &vmovdqu(@X[-3&7],"16($inp)");
  1208. &vmovdqu(@X[-2&7],"32($inp)");
  1209. &vmovdqu(@X[-1&7],"48($inp)");
  1210. &vpshufb(@X[-4&7],@X[-4&7],@Tx[1]); # byte swap
  1211. &add ($inp,64);
  1212. $Xi=0;
  1213. }
  1214. sub Xloop_avx()
  1215. { use integer;
  1216. my $body = shift;
  1217. my @insns = (&$body,&$body,&$body,&$body); # 32 instructions
  1218. my ($a,$b,$c,$d,$e);
  1219. eval(shift(@insns));
  1220. eval(shift(@insns));
  1221. &vpshufb(@X[($Xi-3)&7],@X[($Xi-3)&7],@Tx[1]);
  1222. eval(shift(@insns));
  1223. eval(shift(@insns));
  1224. &vpaddd (@Tx[0],@X[($Xi-4)&7],$Kx);
  1225. eval(shift(@insns));
  1226. eval(shift(@insns));
  1227. eval(shift(@insns));
  1228. eval(shift(@insns));
  1229. &vmovdqa(eval(16*$Xi)."(%rsp)",@Tx[0]); # X[]+K xfer to IALU
  1230. eval(shift(@insns));
  1231. eval(shift(@insns));
  1232. foreach (@insns) { eval; }
  1233. $Xi++;
  1234. }
  1235. sub Xtail_avx()
  1236. { use integer;
  1237. my $body = shift;
  1238. my @insns = (&$body,&$body,&$body,&$body); # 32 instructions
  1239. my ($a,$b,$c,$d,$e);
  1240. foreach (@insns) { eval; }
  1241. }
  1242. $code.=<<___;
  1243. .align 32
  1244. .Loop_avx:
  1245. ___
  1246. &Xupdate_avx_16_31(\&body_00_19);
  1247. &Xupdate_avx_16_31(\&body_00_19);
  1248. &Xupdate_avx_16_31(\&body_00_19);
  1249. &Xupdate_avx_16_31(\&body_00_19);
  1250. &Xupdate_avx_32_79(\&body_00_19);
  1251. &Xupdate_avx_32_79(\&body_20_39);
  1252. &Xupdate_avx_32_79(\&body_20_39);
  1253. &Xupdate_avx_32_79(\&body_20_39);
  1254. &Xupdate_avx_32_79(\&body_20_39);
  1255. &Xupdate_avx_32_79(\&body_20_39);
  1256. &Xupdate_avx_32_79(\&body_40_59);
  1257. &Xupdate_avx_32_79(\&body_40_59);
  1258. &Xupdate_avx_32_79(\&body_40_59);
  1259. &Xupdate_avx_32_79(\&body_40_59);
  1260. &Xupdate_avx_32_79(\&body_40_59);
  1261. &Xupdate_avx_32_79(\&body_20_39);
  1262. &Xuplast_avx_80(\&body_20_39,".Ldone_avx"); # can jump to "done"
  1263. $saved_j=$j; @saved_V=@V;
  1264. $saved_r=$r; @saved_rndkey=@rndkey;
  1265. &Xloop_avx(\&body_20_39);
  1266. &Xloop_avx(\&body_20_39);
  1267. &Xloop_avx(\&body_20_39);
  1268. $code.=<<___;
  1269. vmovups $iv,48($out,$in0) # write output
  1270. lea 64($in0),$in0
  1271. add 0($ctx),$A # update context
  1272. add 4($ctx),@T[0]
  1273. add 8($ctx),$C
  1274. add 12($ctx),$D
  1275. mov $A,0($ctx)
  1276. add 16($ctx),$E
  1277. mov @T[0],4($ctx)
  1278. mov @T[0],$B # magic seed
  1279. mov $C,8($ctx)
  1280. mov $C,@T[1]
  1281. mov $D,12($ctx)
  1282. xor $D,@T[1]
  1283. mov $E,16($ctx)
  1284. and @T[1],@T[0]
  1285. jmp .Loop_avx
  1286. .Ldone_avx:
  1287. ___
  1288. $jj=$j=$saved_j; @V=@saved_V;
  1289. $r=$saved_r; @rndkey=@saved_rndkey;
  1290. &Xtail_avx(\&body_20_39);
  1291. &Xtail_avx(\&body_20_39);
  1292. &Xtail_avx(\&body_20_39);
  1293. $code.=<<___;
  1294. vmovups $iv,48($out,$in0) # write output
  1295. mov 88(%rsp),$ivp # restore $ivp
  1296. add 0($ctx),$A # update context
  1297. add 4($ctx),@T[0]
  1298. add 8($ctx),$C
  1299. mov $A,0($ctx)
  1300. add 12($ctx),$D
  1301. mov @T[0],4($ctx)
  1302. add 16($ctx),$E
  1303. mov $C,8($ctx)
  1304. mov $D,12($ctx)
  1305. mov $E,16($ctx)
  1306. vmovups $iv,($ivp) # write IV
  1307. vzeroall
  1308. ___
  1309. $code.=<<___ if ($win64);
  1310. movaps 96+0(%rsp),%xmm6
  1311. movaps 96+16(%rsp),%xmm7
  1312. movaps 96+32(%rsp),%xmm8
  1313. movaps 96+48(%rsp),%xmm9
  1314. movaps 96+64(%rsp),%xmm10
  1315. movaps 96+80(%rsp),%xmm11
  1316. movaps 96+96(%rsp),%xmm12
  1317. movaps 96+112(%rsp),%xmm13
  1318. movaps 96+128(%rsp),%xmm14
  1319. movaps 96+144(%rsp),%xmm15
  1320. ___
  1321. $code.=<<___;
  1322. lea `104+($win64?10*16:0)`(%rsp),%rsi
  1323. .cfi_def_cfa %rsi,56
  1324. mov 0(%rsi),%r15
  1325. .cfi_restore %r15
  1326. mov 8(%rsi),%r14
  1327. .cfi_restore %r14
  1328. mov 16(%rsi),%r13
  1329. .cfi_restore %r13
  1330. mov 24(%rsi),%r12
  1331. .cfi_restore %r12
  1332. mov 32(%rsi),%rbp
  1333. .cfi_restore %rbp
  1334. mov 40(%rsi),%rbx
  1335. .cfi_restore %rbx
  1336. lea 48(%rsi),%rsp
  1337. .cfi_def_cfa %rsp,8
  1338. .Lepilogue_avx:
  1339. ret
  1340. .cfi_endproc
  1341. .size aesni_cbc_sha1_enc_avx,.-aesni_cbc_sha1_enc_avx
  1342. ___
  1343. if ($stitched_decrypt) {{{
  1344. # reset
  1345. ($in0,$out,$len,$key,$ivp,$ctx,$inp)=("%rdi","%rsi","%rdx","%rcx","%r8","%r9","%r10");
  1346. $j=$jj=$r=$rx=0;
  1347. $Xi=4;
  1348. @aes256_dec = (
  1349. '&vpxor ($inout0,$rndkey0,"0x00($in0)");',
  1350. '&vpxor ($inout1,$rndkey0,"0x10($in0)");',
  1351. '&vpxor ($inout2,$rndkey0,"0x20($in0)");',
  1352. '&vpxor ($inout3,$rndkey0,"0x30($in0)");',
  1353. '&vmovups($rndkey0,"16-112($key)");',
  1354. '&vmovups("64(%rsp)",@X[2]);', # save IV, originally @X[3]
  1355. undef,undef
  1356. );
  1357. for ($i=0;$i<13;$i++) {
  1358. push (@aes256_dec,(
  1359. '&vaesdec ($inout0,$inout0,$rndkey0);',
  1360. '&vaesdec ($inout1,$inout1,$rndkey0);',
  1361. '&vaesdec ($inout2,$inout2,$rndkey0);',
  1362. '&vaesdec ($inout3,$inout3,$rndkey0); &vmovups($rndkey0,"'.(16*($i+2)-112).'($key)");'
  1363. ));
  1364. push (@aes256_dec,(undef,undef)) if (($i>=3 && $i<=5) || $i>=11);
  1365. push (@aes256_dec,(undef,undef)) if ($i==5);
  1366. }
  1367. push(@aes256_dec,(
  1368. '&vaesdeclast ($inout0,$inout0,$rndkey0); &vmovups(@X[0],"0x00($in0)");',
  1369. '&vaesdeclast ($inout1,$inout1,$rndkey0); &vmovups(@X[1],"0x10($in0)");',
  1370. '&vaesdeclast ($inout2,$inout2,$rndkey0); &vmovups(@X[2],"0x20($in0)");',
  1371. '&vaesdeclast ($inout3,$inout3,$rndkey0); &vmovups(@X[3],"0x30($in0)");',
  1372. '&vxorps ($inout0,$inout0,"64(%rsp)"); &vmovdqu($rndkey0,"-112($key)");',
  1373. '&vxorps ($inout1,$inout1,@X[0]); &vmovups("0x00($out,$in0)",$inout0);',
  1374. '&vxorps ($inout2,$inout2,@X[1]); &vmovups("0x10($out,$in0)",$inout1);',
  1375. '&vxorps ($inout3,$inout3,@X[2]); &vmovups("0x20($out,$in0)",$inout2);',
  1376. '&vmovups ("0x30($out,$in0)",$inout3);'
  1377. ));
  1378. $code.=<<___;
  1379. .type aesni256_cbc_sha1_dec_avx,\@function,6
  1380. .align 32
  1381. aesni256_cbc_sha1_dec_avx:
  1382. .cfi_startproc
  1383. mov `($win64?56:8)`(%rsp),$inp # load 7th argument
  1384. push %rbx
  1385. .cfi_push %rbx
  1386. push %rbp
  1387. .cfi_push %rbp
  1388. push %r12
  1389. .cfi_push %r12
  1390. push %r13
  1391. .cfi_push %r13
  1392. push %r14
  1393. .cfi_push %r14
  1394. push %r15
  1395. .cfi_push %r15
  1396. lea `-104-($win64?10*16:0)`(%rsp),%rsp
  1397. .cfi_adjust_cfa_offset `104+($win64?10*16:0)`
  1398. ___
  1399. $code.=<<___ if ($win64);
  1400. movaps %xmm6,96+0(%rsp)
  1401. movaps %xmm7,96+16(%rsp)
  1402. movaps %xmm8,96+32(%rsp)
  1403. movaps %xmm9,96+48(%rsp)
  1404. movaps %xmm10,96+64(%rsp)
  1405. movaps %xmm11,96+80(%rsp)
  1406. movaps %xmm12,96+96(%rsp)
  1407. movaps %xmm13,96+112(%rsp)
  1408. movaps %xmm14,96+128(%rsp)
  1409. movaps %xmm15,96+144(%rsp)
  1410. .Lprologue_dec_avx:
  1411. ___
  1412. $code.=<<___;
  1413. vzeroall
  1414. mov $in0,%r12 # reassign arguments
  1415. mov $out,%r13
  1416. mov $len,%r14
  1417. lea 112($key),%r15 # size optimization
  1418. vmovdqu ($ivp),@X[3] # load IV
  1419. ___
  1420. ($in0,$out,$len,$key)=map("%r$_",(12..15)); # reassign arguments
  1421. $code.=<<___;
  1422. shl \$6,$len
  1423. sub $in0,$out
  1424. add $inp,$len # end of input
  1425. lea K_XX_XX(%rip),$K_XX_XX
  1426. mov 0($ctx),$A # load context
  1427. mov 4($ctx),$B
  1428. mov 8($ctx),$C
  1429. mov 12($ctx),$D
  1430. mov $B,@T[0] # magic seed
  1431. mov 16($ctx),$E
  1432. mov $C,@T[1]
  1433. xor $D,@T[1]
  1434. and @T[1],@T[0]
  1435. vmovdqa 64($K_XX_XX),@X[2] # pbswap mask
  1436. vmovdqa 0($K_XX_XX),$Kx # K_00_19
  1437. vmovdqu 0($inp),@X[-4&7] # load input to %xmm[0-3]
  1438. vmovdqu 16($inp),@X[-3&7]
  1439. vmovdqu 32($inp),@X[-2&7]
  1440. vmovdqu 48($inp),@X[-1&7]
  1441. vpshufb @X[2],@X[-4&7],@X[-4&7] # byte swap
  1442. add \$64,$inp
  1443. vpshufb @X[2],@X[-3&7],@X[-3&7]
  1444. vpshufb @X[2],@X[-2&7],@X[-2&7]
  1445. vpshufb @X[2],@X[-1&7],@X[-1&7]
  1446. vpaddd $Kx,@X[-4&7],@X[0] # add K_00_19
  1447. vpaddd $Kx,@X[-3&7],@X[1]
  1448. vpaddd $Kx,@X[-2&7],@X[2]
  1449. vmovdqa @X[0],0(%rsp) # X[]+K xfer to IALU
  1450. vmovdqa @X[1],16(%rsp)
  1451. vmovdqa @X[2],32(%rsp)
  1452. vmovups -112($key),$rndkey0 # $key[0]
  1453. jmp .Loop_dec_avx
  1454. .align 32
  1455. .Loop_dec_avx:
  1456. ___
  1457. &Xupdate_avx_16_31(\&body_00_19_dec);
  1458. &Xupdate_avx_16_31(\&body_00_19_dec);
  1459. &Xupdate_avx_16_31(\&body_00_19_dec);
  1460. &Xupdate_avx_16_31(\&body_00_19_dec);
  1461. &Xupdate_avx_32_79(\&body_00_19_dec);
  1462. &Xupdate_avx_32_79(\&body_20_39_dec);
  1463. &Xupdate_avx_32_79(\&body_20_39_dec);
  1464. &Xupdate_avx_32_79(\&body_20_39_dec);
  1465. &Xupdate_avx_32_79(\&body_20_39_dec);
  1466. &Xupdate_avx_32_79(\&body_20_39_dec);
  1467. &Xupdate_avx_32_79(\&body_40_59_dec);
  1468. &Xupdate_avx_32_79(\&body_40_59_dec);
  1469. &Xupdate_avx_32_79(\&body_40_59_dec);
  1470. &Xupdate_avx_32_79(\&body_40_59_dec);
  1471. &Xupdate_avx_32_79(\&body_40_59_dec);
  1472. &Xupdate_avx_32_79(\&body_20_39_dec);
  1473. &Xuplast_avx_80(\&body_20_39_dec,".Ldone_dec_avx"); # can jump to "done"
  1474. $saved_j=$j; @saved_V=@V;
  1475. $saved_rx=$rx;
  1476. &Xloop_avx(\&body_20_39_dec);
  1477. &Xloop_avx(\&body_20_39_dec);
  1478. &Xloop_avx(\&body_20_39_dec);
  1479. eval(@aes256_dec[-1]); # last store
  1480. $code.=<<___;
  1481. lea 64($in0),$in0
  1482. add 0($ctx),$A # update context
  1483. add 4($ctx),@T[0]
  1484. add 8($ctx),$C
  1485. add 12($ctx),$D
  1486. mov $A,0($ctx)
  1487. add 16($ctx),$E
  1488. mov @T[0],4($ctx)
  1489. mov @T[0],$B # magic seed
  1490. mov $C,8($ctx)
  1491. mov $C,@T[1]
  1492. mov $D,12($ctx)
  1493. xor $D,@T[1]
  1494. mov $E,16($ctx)
  1495. and @T[1],@T[0]
  1496. jmp .Loop_dec_avx
  1497. .Ldone_dec_avx:
  1498. ___
  1499. $jj=$j=$saved_j; @V=@saved_V;
  1500. $rx=$saved_rx;
  1501. &Xtail_avx(\&body_20_39_dec);
  1502. &Xtail_avx(\&body_20_39_dec);
  1503. &Xtail_avx(\&body_20_39_dec);
  1504. eval(@aes256_dec[-1]); # last store
  1505. $code.=<<___;
  1506. add 0($ctx),$A # update context
  1507. add 4($ctx),@T[0]
  1508. add 8($ctx),$C
  1509. mov $A,0($ctx)
  1510. add 12($ctx),$D
  1511. mov @T[0],4($ctx)
  1512. add 16($ctx),$E
  1513. mov $C,8($ctx)
  1514. mov $D,12($ctx)
  1515. mov $E,16($ctx)
  1516. vmovups @X[3],($ivp) # write IV
  1517. vzeroall
  1518. ___
  1519. $code.=<<___ if ($win64);
  1520. movaps 96+0(%rsp),%xmm6
  1521. movaps 96+16(%rsp),%xmm7
  1522. movaps 96+32(%rsp),%xmm8
  1523. movaps 96+48(%rsp),%xmm9
  1524. movaps 96+64(%rsp),%xmm10
  1525. movaps 96+80(%rsp),%xmm11
  1526. movaps 96+96(%rsp),%xmm12
  1527. movaps 96+112(%rsp),%xmm13
  1528. movaps 96+128(%rsp),%xmm14
  1529. movaps 96+144(%rsp),%xmm15
  1530. ___
  1531. $code.=<<___;
  1532. lea `104+($win64?10*16:0)`(%rsp),%rsi
  1533. .cfi_def_cfa %rsi,56
  1534. mov 0(%rsi),%r15
  1535. .cfi_restore %r15
  1536. mov 8(%rsi),%r14
  1537. .cfi_restore %r14
  1538. mov 16(%rsi),%r13
  1539. .cfi_restore %r13
  1540. mov 24(%rsi),%r12
  1541. .cfi_restore %r12
  1542. mov 32(%rsi),%rbp
  1543. .cfi_restore %rbp
  1544. mov 40(%rsi),%rbx
  1545. .cfi_restore %rbx
  1546. lea 48(%rsi),%rsp
  1547. .cfi_def_cfa %rsp,8
  1548. .Lepilogue_dec_avx:
  1549. ret
  1550. .cfi_endproc
  1551. .size aesni256_cbc_sha1_dec_avx,.-aesni256_cbc_sha1_dec_avx
  1552. ___
  1553. }}}
  1554. }
  1555. $code.=<<___;
  1556. .align 64
  1557. K_XX_XX:
  1558. .long 0x5a827999,0x5a827999,0x5a827999,0x5a827999 # K_00_19
  1559. .long 0x6ed9eba1,0x6ed9eba1,0x6ed9eba1,0x6ed9eba1 # K_20_39
  1560. .long 0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc # K_40_59
  1561. .long 0xca62c1d6,0xca62c1d6,0xca62c1d6,0xca62c1d6 # K_60_79
  1562. .long 0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f # pbswap mask
  1563. .byte 0xf,0xe,0xd,0xc,0xb,0xa,0x9,0x8,0x7,0x6,0x5,0x4,0x3,0x2,0x1,0x0
  1564. .asciz "AESNI-CBC+SHA1 stitch for x86_64, CRYPTOGAMS by <appro\@openssl.org>"
  1565. .align 64
  1566. ___
  1567. if ($shaext) {{{
  1568. ($in0,$out,$len,$key,$ivp,$ctx,$inp)=("%rdi","%rsi","%rdx","%rcx","%r8","%r9","%r10");
  1569. $rounds="%r11d";
  1570. ($iv,$in,$rndkey0)=map("%xmm$_",(2,14,15));
  1571. @rndkey=("%xmm0","%xmm1");
  1572. $r=0;
  1573. my ($BSWAP,$ABCD,$E,$E_,$ABCD_SAVE,$E_SAVE)=map("%xmm$_",(7..12));
  1574. my @MSG=map("%xmm$_",(3..6));
  1575. $code.=<<___;
  1576. .type aesni_cbc_sha1_enc_shaext,\@function,6
  1577. .align 32
  1578. aesni_cbc_sha1_enc_shaext:
  1579. .cfi_startproc
  1580. mov `($win64?56:8)`(%rsp),$inp # load 7th argument
  1581. ___
  1582. $code.=<<___ if ($win64);
  1583. lea `-8-10*16`(%rsp),%rsp
  1584. movaps %xmm6,-8-10*16(%rax)
  1585. movaps %xmm7,-8-9*16(%rax)
  1586. movaps %xmm8,-8-8*16(%rax)
  1587. movaps %xmm9,-8-7*16(%rax)
  1588. movaps %xmm10,-8-6*16(%rax)
  1589. movaps %xmm11,-8-5*16(%rax)
  1590. movaps %xmm12,-8-4*16(%rax)
  1591. movaps %xmm13,-8-3*16(%rax)
  1592. movaps %xmm14,-8-2*16(%rax)
  1593. movaps %xmm15,-8-1*16(%rax)
  1594. .Lprologue_shaext:
  1595. ___
  1596. $code.=<<___;
  1597. movdqu ($ctx),$ABCD
  1598. movd 16($ctx),$E
  1599. movdqa K_XX_XX+0x50(%rip),$BSWAP # byte-n-word swap
  1600. mov 240($key),$rounds
  1601. sub $in0,$out
  1602. movups ($key),$rndkey0 # $key[0]
  1603. movups ($ivp),$iv # load IV
  1604. movups 16($key),$rndkey[0] # forward reference
  1605. lea 112($key),$key # size optimization
  1606. pshufd \$0b00011011,$ABCD,$ABCD # flip word order
  1607. pshufd \$0b00011011,$E,$E # flip word order
  1608. jmp .Loop_shaext
  1609. .align 16
  1610. .Loop_shaext:
  1611. ___
  1612. &$aesenc();
  1613. $code.=<<___;
  1614. movdqu ($inp),@MSG[0]
  1615. movdqa $E,$E_SAVE # offload $E
  1616. pshufb $BSWAP,@MSG[0]
  1617. movdqu 0x10($inp),@MSG[1]
  1618. movdqa $ABCD,$ABCD_SAVE # offload $ABCD
  1619. ___
  1620. &$aesenc();
  1621. $code.=<<___;
  1622. pshufb $BSWAP,@MSG[1]
  1623. paddd @MSG[0],$E
  1624. movdqu 0x20($inp),@MSG[2]
  1625. lea 0x40($inp),$inp
  1626. pxor $E_SAVE,@MSG[0] # black magic
  1627. ___
  1628. &$aesenc();
  1629. $code.=<<___;
  1630. pxor $E_SAVE,@MSG[0] # black magic
  1631. movdqa $ABCD,$E_
  1632. pshufb $BSWAP,@MSG[2]
  1633. sha1rnds4 \$0,$E,$ABCD # 0-3
  1634. sha1nexte @MSG[1],$E_
  1635. ___
  1636. &$aesenc();
  1637. $code.=<<___;
  1638. sha1msg1 @MSG[1],@MSG[0]
  1639. movdqu -0x10($inp),@MSG[3]
  1640. movdqa $ABCD,$E
  1641. pshufb $BSWAP,@MSG[3]
  1642. ___
  1643. &$aesenc();
  1644. $code.=<<___;
  1645. sha1rnds4 \$0,$E_,$ABCD # 4-7
  1646. sha1nexte @MSG[2],$E
  1647. pxor @MSG[2],@MSG[0]
  1648. sha1msg1 @MSG[2],@MSG[1]
  1649. ___
  1650. &$aesenc();
  1651. for($i=2;$i<20-4;$i++) {
  1652. $code.=<<___;
  1653. movdqa $ABCD,$E_
  1654. sha1rnds4 \$`int($i/5)`,$E,$ABCD # 8-11
  1655. sha1nexte @MSG[3],$E_
  1656. ___
  1657. &$aesenc();
  1658. $code.=<<___;
  1659. sha1msg2 @MSG[3],@MSG[0]
  1660. pxor @MSG[3],@MSG[1]
  1661. sha1msg1 @MSG[3],@MSG[2]
  1662. ___
  1663. ($E,$E_)=($E_,$E);
  1664. push(@MSG,shift(@MSG));
  1665. &$aesenc();
  1666. }
  1667. $code.=<<___;
  1668. movdqa $ABCD,$E_
  1669. sha1rnds4 \$3,$E,$ABCD # 64-67
  1670. sha1nexte @MSG[3],$E_
  1671. sha1msg2 @MSG[3],@MSG[0]
  1672. pxor @MSG[3],@MSG[1]
  1673. ___
  1674. &$aesenc();
  1675. $code.=<<___;
  1676. movdqa $ABCD,$E
  1677. sha1rnds4 \$3,$E_,$ABCD # 68-71
  1678. sha1nexte @MSG[0],$E
  1679. sha1msg2 @MSG[0],@MSG[1]
  1680. ___
  1681. &$aesenc();
  1682. $code.=<<___;
  1683. movdqa $E_SAVE,@MSG[0]
  1684. movdqa $ABCD,$E_
  1685. sha1rnds4 \$3,$E,$ABCD # 72-75
  1686. sha1nexte @MSG[1],$E_
  1687. ___
  1688. &$aesenc();
  1689. $code.=<<___;
  1690. movdqa $ABCD,$E
  1691. sha1rnds4 \$3,$E_,$ABCD # 76-79
  1692. sha1nexte $MSG[0],$E
  1693. ___
  1694. while($r<40) { &$aesenc(); } # remaining aesenc's
  1695. $code.=<<___;
  1696. dec $len
  1697. paddd $ABCD_SAVE,$ABCD
  1698. movups $iv,48($out,$in0) # write output
  1699. lea 64($in0),$in0
  1700. jnz .Loop_shaext
  1701. pshufd \$0b00011011,$ABCD,$ABCD
  1702. pshufd \$0b00011011,$E,$E
  1703. movups $iv,($ivp) # write IV
  1704. movdqu $ABCD,($ctx)
  1705. movd $E,16($ctx)
  1706. ___
  1707. $code.=<<___ if ($win64);
  1708. movaps -8-10*16(%rax),%xmm6
  1709. movaps -8-9*16(%rax),%xmm7
  1710. movaps -8-8*16(%rax),%xmm8
  1711. movaps -8-7*16(%rax),%xmm9
  1712. movaps -8-6*16(%rax),%xmm10
  1713. movaps -8-5*16(%rax),%xmm11
  1714. movaps -8-4*16(%rax),%xmm12
  1715. movaps -8-3*16(%rax),%xmm13
  1716. movaps -8-2*16(%rax),%xmm14
  1717. movaps -8-1*16(%rax),%xmm15
  1718. mov %rax,%rsp
  1719. .Lepilogue_shaext:
  1720. ___
  1721. $code.=<<___;
  1722. ret
  1723. .cfi_endproc
  1724. .size aesni_cbc_sha1_enc_shaext,.-aesni_cbc_sha1_enc_shaext
  1725. ___
  1726. }}}
  1727. # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame,
  1728. # CONTEXT *context,DISPATCHER_CONTEXT *disp)
  1729. if ($win64) {
  1730. $rec="%rcx";
  1731. $frame="%rdx";
  1732. $context="%r8";
  1733. $disp="%r9";
  1734. $code.=<<___;
  1735. .extern __imp_RtlVirtualUnwind
  1736. .type ssse3_handler,\@abi-omnipotent
  1737. .align 16
  1738. ssse3_handler:
  1739. push %rsi
  1740. push %rdi
  1741. push %rbx
  1742. push %rbp
  1743. push %r12
  1744. push %r13
  1745. push %r14
  1746. push %r15
  1747. pushfq
  1748. sub \$64,%rsp
  1749. mov 120($context),%rax # pull context->Rax
  1750. mov 248($context),%rbx # pull context->Rip
  1751. mov 8($disp),%rsi # disp->ImageBase
  1752. mov 56($disp),%r11 # disp->HandlerData
  1753. mov 0(%r11),%r10d # HandlerData[0]
  1754. lea (%rsi,%r10),%r10 # prologue label
  1755. cmp %r10,%rbx # context->Rip<prologue label
  1756. jb .Lcommon_seh_tail
  1757. mov 152($context),%rax # pull context->Rsp
  1758. mov 4(%r11),%r10d # HandlerData[1]
  1759. lea (%rsi,%r10),%r10 # epilogue label
  1760. cmp %r10,%rbx # context->Rip>=epilogue label
  1761. jae .Lcommon_seh_tail
  1762. ___
  1763. $code.=<<___ if ($shaext);
  1764. lea aesni_cbc_sha1_enc_shaext(%rip),%r10
  1765. cmp %r10,%rbx
  1766. jb .Lseh_no_shaext
  1767. lea (%rax),%rsi
  1768. lea 512($context),%rdi # &context.Xmm6
  1769. mov \$20,%ecx
  1770. .long 0xa548f3fc # cld; rep movsq
  1771. lea 168(%rax),%rax # adjust stack pointer
  1772. jmp .Lcommon_seh_tail
  1773. .Lseh_no_shaext:
  1774. ___
  1775. $code.=<<___;
  1776. lea 96(%rax),%rsi
  1777. lea 512($context),%rdi # &context.Xmm6
  1778. mov \$20,%ecx
  1779. .long 0xa548f3fc # cld; rep movsq
  1780. lea `104+10*16`(%rax),%rax # adjust stack pointer
  1781. mov 0(%rax),%r15
  1782. mov 8(%rax),%r14
  1783. mov 16(%rax),%r13
  1784. mov 24(%rax),%r12
  1785. mov 32(%rax),%rbp
  1786. mov 40(%rax),%rbx
  1787. lea 48(%rax),%rax
  1788. mov %rbx,144($context) # restore context->Rbx
  1789. mov %rbp,160($context) # restore context->Rbp
  1790. mov %r12,216($context) # restore context->R12
  1791. mov %r13,224($context) # restore context->R13
  1792. mov %r14,232($context) # restore context->R14
  1793. mov %r15,240($context) # restore context->R15
  1794. .Lcommon_seh_tail:
  1795. mov 8(%rax),%rdi
  1796. mov 16(%rax),%rsi
  1797. mov %rax,152($context) # restore context->Rsp
  1798. mov %rsi,168($context) # restore context->Rsi
  1799. mov %rdi,176($context) # restore context->Rdi
  1800. mov 40($disp),%rdi # disp->ContextRecord
  1801. mov $context,%rsi # context
  1802. mov \$154,%ecx # sizeof(CONTEXT)
  1803. .long 0xa548f3fc # cld; rep movsq
  1804. mov $disp,%rsi
  1805. xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER
  1806. mov 8(%rsi),%rdx # arg2, disp->ImageBase
  1807. mov 0(%rsi),%r8 # arg3, disp->ControlPc
  1808. mov 16(%rsi),%r9 # arg4, disp->FunctionEntry
  1809. mov 40(%rsi),%r10 # disp->ContextRecord
  1810. lea 56(%rsi),%r11 # &disp->HandlerData
  1811. lea 24(%rsi),%r12 # &disp->EstablisherFrame
  1812. mov %r10,32(%rsp) # arg5
  1813. mov %r11,40(%rsp) # arg6
  1814. mov %r12,48(%rsp) # arg7
  1815. mov %rcx,56(%rsp) # arg8, (NULL)
  1816. call *__imp_RtlVirtualUnwind(%rip)
  1817. mov \$1,%eax # ExceptionContinueSearch
  1818. add \$64,%rsp
  1819. popfq
  1820. pop %r15
  1821. pop %r14
  1822. pop %r13
  1823. pop %r12
  1824. pop %rbp
  1825. pop %rbx
  1826. pop %rdi
  1827. pop %rsi
  1828. ret
  1829. .size ssse3_handler,.-ssse3_handler
  1830. .section .pdata
  1831. .align 4
  1832. .rva .LSEH_begin_aesni_cbc_sha1_enc_ssse3
  1833. .rva .LSEH_end_aesni_cbc_sha1_enc_ssse3
  1834. .rva .LSEH_info_aesni_cbc_sha1_enc_ssse3
  1835. ___
  1836. $code.=<<___ if ($avx);
  1837. .rva .LSEH_begin_aesni_cbc_sha1_enc_avx
  1838. .rva .LSEH_end_aesni_cbc_sha1_enc_avx
  1839. .rva .LSEH_info_aesni_cbc_sha1_enc_avx
  1840. ___
  1841. $code.=<<___ if ($shaext);
  1842. .rva .LSEH_begin_aesni_cbc_sha1_enc_shaext
  1843. .rva .LSEH_end_aesni_cbc_sha1_enc_shaext
  1844. .rva .LSEH_info_aesni_cbc_sha1_enc_shaext
  1845. ___
  1846. $code.=<<___;
  1847. .section .xdata
  1848. .align 8
  1849. .LSEH_info_aesni_cbc_sha1_enc_ssse3:
  1850. .byte 9,0,0,0
  1851. .rva ssse3_handler
  1852. .rva .Lprologue_ssse3,.Lepilogue_ssse3 # HandlerData[]
  1853. ___
  1854. $code.=<<___ if ($avx);
  1855. .LSEH_info_aesni_cbc_sha1_enc_avx:
  1856. .byte 9,0,0,0
  1857. .rva ssse3_handler
  1858. .rva .Lprologue_avx,.Lepilogue_avx # HandlerData[]
  1859. ___
  1860. $code.=<<___ if ($shaext);
  1861. .LSEH_info_aesni_cbc_sha1_enc_shaext:
  1862. .byte 9,0,0,0
  1863. .rva ssse3_handler
  1864. .rva .Lprologue_shaext,.Lepilogue_shaext # HandlerData[]
  1865. ___
  1866. }
  1867. ####################################################################
  1868. sub rex {
  1869. local *opcode=shift;
  1870. my ($dst,$src)=@_;
  1871. my $rex=0;
  1872. $rex|=0x04 if($dst>=8);
  1873. $rex|=0x01 if($src>=8);
  1874. unshift @opcode,$rex|0x40 if($rex);
  1875. }
  1876. sub sha1rnds4 {
  1877. if (@_[0] =~ /\$([x0-9a-f]+),\s*%xmm([0-9]+),\s*%xmm([0-9]+)/) {
  1878. my @opcode=(0x0f,0x3a,0xcc);
  1879. rex(\@opcode,$3,$2);
  1880. push @opcode,0xc0|($2&7)|(($3&7)<<3); # ModR/M
  1881. my $c=$1;
  1882. push @opcode,$c=~/^0/?oct($c):$c;
  1883. return ".byte\t".join(',',@opcode);
  1884. } else {
  1885. return "sha1rnds4\t".@_[0];
  1886. }
  1887. }
  1888. sub sha1op38 {
  1889. my $instr = shift;
  1890. my %opcodelet = (
  1891. "sha1nexte" => 0xc8,
  1892. "sha1msg1" => 0xc9,
  1893. "sha1msg2" => 0xca );
  1894. if (defined($opcodelet{$instr}) && @_[0] =~ /%xmm([0-9]+),\s*%xmm([0-9]+)/) {
  1895. my @opcode=(0x0f,0x38);
  1896. rex(\@opcode,$2,$1);
  1897. push @opcode,$opcodelet{$instr};
  1898. push @opcode,0xc0|($1&7)|(($2&7)<<3); # ModR/M
  1899. return ".byte\t".join(',',@opcode);
  1900. } else {
  1901. return $instr."\t".@_[0];
  1902. }
  1903. }
  1904. sub aesni {
  1905. my $line=shift;
  1906. my @opcode=(0x0f,0x38);
  1907. if ($line=~/(aes[a-z]+)\s+%xmm([0-9]+),\s*%xmm([0-9]+)/) {
  1908. my %opcodelet = (
  1909. "aesenc" => 0xdc, "aesenclast" => 0xdd,
  1910. "aesdec" => 0xde, "aesdeclast" => 0xdf
  1911. );
  1912. return undef if (!defined($opcodelet{$1}));
  1913. rex(\@opcode,$3,$2);
  1914. push @opcode,$opcodelet{$1},0xc0|($2&7)|(($3&7)<<3); # ModR/M
  1915. unshift @opcode,0x66;
  1916. return ".byte\t".join(',',@opcode);
  1917. }
  1918. return $line;
  1919. }
  1920. foreach (split("\n",$code)) {
  1921. s/\`([^\`]*)\`/eval $1/geo;
  1922. s/\b(sha1rnds4)\s+(.*)/sha1rnds4($2)/geo or
  1923. s/\b(sha1[^\s]*)\s+(.*)/sha1op38($1,$2)/geo or
  1924. s/\b(aes.*%xmm[0-9]+).*$/aesni($1)/geo;
  1925. print $_,"\n";
  1926. }
  1927. close STDOUT or die "error closing STDOUT: $!";