sha1-x86_64.pl 51 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132
  1. #! /usr/bin/env perl
  2. # Copyright 2006-2016 The OpenSSL Project Authors. All Rights Reserved.
  3. #
  4. # Licensed under the OpenSSL license (the "License"). You may not use
  5. # this file except in compliance with the License. You can obtain a copy
  6. # in the file LICENSE in the source distribution or at
  7. # https://www.openssl.org/source/license.html
  8. #
  9. # ====================================================================
  10. # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
  11. # project. The module is, however, dual licensed under OpenSSL and
  12. # CRYPTOGAMS licenses depending on where you obtain it. For further
  13. # details see http://www.openssl.org/~appro/cryptogams/.
  14. # ====================================================================
  15. #
  16. # sha1_block procedure for x86_64.
  17. #
  18. # It was brought to my attention that on EM64T compiler-generated code
  19. # was far behind 32-bit assembler implementation. This is unlike on
  20. # Opteron where compiler-generated code was only 15% behind 32-bit
  21. # assembler, which originally made it hard to motivate the effort.
  22. # There was suggestion to mechanically translate 32-bit code, but I
  23. # dismissed it, reasoning that x86_64 offers enough register bank
  24. # capacity to fully utilize SHA-1 parallelism. Therefore this fresh
  25. # implementation:-) However! While 64-bit code does perform better
  26. # on Opteron, I failed to beat 32-bit assembler on EM64T core. Well,
  27. # x86_64 does offer larger *addressable* bank, but out-of-order core
  28. # reaches for even more registers through dynamic aliasing, and EM64T
  29. # core must have managed to run-time optimize even 32-bit code just as
  30. # good as 64-bit one. Performance improvement is summarized in the
  31. # following table:
  32. #
  33. # gcc 3.4 32-bit asm cycles/byte
  34. # Opteron +45% +20% 6.8
  35. # Xeon P4 +65% +0% 9.9
  36. # Core2 +60% +10% 7.0
  37. # August 2009.
  38. #
  39. # The code was revised to minimize code size and to maximize
  40. # "distance" between instructions producing input to 'lea'
  41. # instruction and the 'lea' instruction itself, which is essential
  42. # for Intel Atom core.
  43. # October 2010.
  44. #
  45. # Add SSSE3, Supplemental[!] SSE3, implementation. The idea behind it
  46. # is to offload message schedule denoted by Wt in NIST specification,
  47. # or Xupdate in OpenSSL source, to SIMD unit. See sha1-586.pl module
  48. # for background and implementation details. The only difference from
  49. # 32-bit code is that 64-bit code doesn't have to spill @X[] elements
  50. # to free temporary registers.
  51. # April 2011.
  52. #
  53. # Add AVX code path. See sha1-586.pl for further information.
  54. # May 2013.
  55. #
  56. # Add AVX2+BMI code path. Initial attempt (utilizing BMI instructions
  57. # and loading pair of consecutive blocks to 256-bit %ymm registers)
  58. # did not provide impressive performance improvement till a crucial
  59. # hint regarding the number of Xupdate iterations to pre-compute in
  60. # advance was provided by Ilya Albrekht of Intel Corp.
  61. # March 2014.
  62. #
  63. # Add support for Intel SHA Extensions.
  64. ######################################################################
  65. # Current performance is summarized in following table. Numbers are
  66. # CPU clock cycles spent to process single byte (less is better).
  67. #
  68. # x86_64 SSSE3 AVX[2]
  69. # P4 9.05 -
  70. # Opteron 6.26 -
  71. # Core2 6.55 6.05/+8% -
  72. # Westmere 6.73 5.30/+27% -
  73. # Sandy Bridge 7.70 6.10/+26% 4.99/+54%
  74. # Ivy Bridge 6.06 4.67/+30% 4.60/+32%
  75. # Haswell 5.45 4.15/+31% 3.57/+53%
  76. # Skylake 5.18 4.06/+28% 3.54/+46%
  77. # Bulldozer 9.11 5.95/+53%
  78. # Ryzen 4.75 3.80/+24% 1.93/+150%(**)
  79. # VIA Nano 9.32 7.15/+30%
  80. # Atom 10.3 9.17/+12%
  81. # Silvermont 13.1(*) 9.37/+40%
  82. # Knights L 13.2(*) 9.68/+36% 8.30/+59%
  83. # Goldmont 8.13 6.42/+27% 1.70/+380%(**)
  84. #
  85. # (*) obviously suboptimal result, nothing was done about it,
  86. # because SSSE3 code is compiled unconditionally;
  87. # (**) SHAEXT result
  88. $flavour = shift;
  89. $output = shift;
  90. if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
  91. $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
  92. $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
  93. ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
  94. ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
  95. die "can't locate x86_64-xlate.pl";
  96. if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
  97. =~ /GNU assembler version ([2-9]\.[0-9]+)/) {
  98. $avx = ($1>=2.19) + ($1>=2.22);
  99. }
  100. if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) &&
  101. `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) {
  102. $avx = ($1>=2.09) + ($1>=2.10);
  103. }
  104. if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) &&
  105. `ml64 2>&1` =~ /Version ([0-9]+)\./) {
  106. $avx = ($1>=10) + ($1>=11);
  107. }
  108. if (!$avx && `$ENV{CC} -v 2>&1` =~ /((?:^clang|LLVM) version|.*based on LLVM) ([2-9]\.[0-9]+)/) {
  109. $avx = ($2>=3.0) + ($2>3.0);
  110. }
  111. $shaext=1; ### set to zero if compiling for 1.0.1
  112. $avx=1 if (!$shaext && $avx);
  113. open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\"";
  114. *STDOUT=*OUT;
  115. $ctx="%rdi"; # 1st arg
  116. $inp="%rsi"; # 2nd arg
  117. $num="%rdx"; # 3rd arg
  118. # reassign arguments in order to produce more compact code
  119. $ctx="%r8";
  120. $inp="%r9";
  121. $num="%r10";
  122. $t0="%eax";
  123. $t1="%ebx";
  124. $t2="%ecx";
  125. @xi=("%edx","%ebp","%r14d");
  126. $A="%esi";
  127. $B="%edi";
  128. $C="%r11d";
  129. $D="%r12d";
  130. $E="%r13d";
  131. @V=($A,$B,$C,$D,$E);
  132. sub BODY_00_19 {
  133. my ($i,$a,$b,$c,$d,$e)=@_;
  134. my $j=$i+1;
  135. $code.=<<___ if ($i==0);
  136. mov `4*$i`($inp),$xi[0]
  137. bswap $xi[0]
  138. ___
  139. $code.=<<___ if ($i<15);
  140. mov `4*$j`($inp),$xi[1]
  141. mov $d,$t0
  142. mov $xi[0],`4*$i`(%rsp)
  143. mov $a,$t2
  144. bswap $xi[1]
  145. xor $c,$t0
  146. rol \$5,$t2
  147. and $b,$t0
  148. lea 0x5a827999($xi[0],$e),$e
  149. add $t2,$e
  150. xor $d,$t0
  151. rol \$30,$b
  152. add $t0,$e
  153. ___
  154. $code.=<<___ if ($i>=15);
  155. xor `4*($j%16)`(%rsp),$xi[1]
  156. mov $d,$t0
  157. mov $xi[0],`4*($i%16)`(%rsp)
  158. mov $a,$t2
  159. xor `4*(($j+2)%16)`(%rsp),$xi[1]
  160. xor $c,$t0
  161. rol \$5,$t2
  162. xor `4*(($j+8)%16)`(%rsp),$xi[1]
  163. and $b,$t0
  164. lea 0x5a827999($xi[0],$e),$e
  165. rol \$30,$b
  166. xor $d,$t0
  167. add $t2,$e
  168. rol \$1,$xi[1]
  169. add $t0,$e
  170. ___
  171. push(@xi,shift(@xi));
  172. }
  173. sub BODY_20_39 {
  174. my ($i,$a,$b,$c,$d,$e)=@_;
  175. my $j=$i+1;
  176. my $K=($i<40)?0x6ed9eba1:0xca62c1d6;
  177. $code.=<<___ if ($i<79);
  178. xor `4*($j%16)`(%rsp),$xi[1]
  179. mov $b,$t0
  180. `"mov $xi[0],".4*($i%16)."(%rsp)" if ($i<72)`
  181. mov $a,$t2
  182. xor `4*(($j+2)%16)`(%rsp),$xi[1]
  183. xor $d,$t0
  184. rol \$5,$t2
  185. xor `4*(($j+8)%16)`(%rsp),$xi[1]
  186. lea $K($xi[0],$e),$e
  187. xor $c,$t0
  188. add $t2,$e
  189. rol \$30,$b
  190. add $t0,$e
  191. rol \$1,$xi[1]
  192. ___
  193. $code.=<<___ if ($i==79);
  194. mov $b,$t0
  195. mov $a,$t2
  196. xor $d,$t0
  197. lea $K($xi[0],$e),$e
  198. rol \$5,$t2
  199. xor $c,$t0
  200. add $t2,$e
  201. rol \$30,$b
  202. add $t0,$e
  203. ___
  204. push(@xi,shift(@xi));
  205. }
  206. sub BODY_40_59 {
  207. my ($i,$a,$b,$c,$d,$e)=@_;
  208. my $j=$i+1;
  209. $code.=<<___;
  210. xor `4*($j%16)`(%rsp),$xi[1]
  211. mov $d,$t0
  212. mov $xi[0],`4*($i%16)`(%rsp)
  213. mov $d,$t1
  214. xor `4*(($j+2)%16)`(%rsp),$xi[1]
  215. and $c,$t0
  216. mov $a,$t2
  217. xor `4*(($j+8)%16)`(%rsp),$xi[1]
  218. lea 0x8f1bbcdc($xi[0],$e),$e
  219. xor $c,$t1
  220. rol \$5,$t2
  221. add $t0,$e
  222. rol \$1,$xi[1]
  223. and $b,$t1
  224. add $t2,$e
  225. rol \$30,$b
  226. add $t1,$e
  227. ___
  228. push(@xi,shift(@xi));
  229. }
  230. $code.=<<___;
  231. .text
  232. .extern OPENSSL_ia32cap_P
  233. .globl sha1_block_data_order
  234. .type sha1_block_data_order,\@function,3
  235. .align 16
  236. sha1_block_data_order:
  237. .cfi_startproc
  238. mov OPENSSL_ia32cap_P+0(%rip),%r9d
  239. mov OPENSSL_ia32cap_P+4(%rip),%r8d
  240. mov OPENSSL_ia32cap_P+8(%rip),%r10d
  241. test \$`1<<9`,%r8d # check SSSE3 bit
  242. jz .Lialu
  243. ___
  244. $code.=<<___ if ($shaext);
  245. test \$`1<<29`,%r10d # check SHA bit
  246. jnz _shaext_shortcut
  247. ___
  248. $code.=<<___ if ($avx>1);
  249. and \$`1<<3|1<<5|1<<8`,%r10d # check AVX2+BMI1+BMI2
  250. cmp \$`1<<3|1<<5|1<<8`,%r10d
  251. je _avx2_shortcut
  252. ___
  253. $code.=<<___ if ($avx);
  254. and \$`1<<28`,%r8d # mask AVX bit
  255. and \$`1<<30`,%r9d # mask "Intel CPU" bit
  256. or %r9d,%r8d
  257. cmp \$`1<<28|1<<30`,%r8d
  258. je _avx_shortcut
  259. ___
  260. $code.=<<___;
  261. jmp _ssse3_shortcut
  262. .align 16
  263. .Lialu:
  264. mov %rsp,%rax
  265. .cfi_def_cfa_register %rax
  266. push %rbx
  267. .cfi_push %rbx
  268. push %rbp
  269. .cfi_push %rbp
  270. push %r12
  271. .cfi_push %r12
  272. push %r13
  273. .cfi_push %r13
  274. push %r14
  275. .cfi_push %r14
  276. mov %rdi,$ctx # reassigned argument
  277. sub \$`8+16*4`,%rsp
  278. mov %rsi,$inp # reassigned argument
  279. and \$-64,%rsp
  280. mov %rdx,$num # reassigned argument
  281. mov %rax,`16*4`(%rsp)
  282. .cfi_cfa_expression %rsp+64,deref,+8
  283. .Lprologue:
  284. mov 0($ctx),$A
  285. mov 4($ctx),$B
  286. mov 8($ctx),$C
  287. mov 12($ctx),$D
  288. mov 16($ctx),$E
  289. jmp .Lloop
  290. .align 16
  291. .Lloop:
  292. ___
  293. for($i=0;$i<20;$i++) { &BODY_00_19($i,@V); unshift(@V,pop(@V)); }
  294. for(;$i<40;$i++) { &BODY_20_39($i,@V); unshift(@V,pop(@V)); }
  295. for(;$i<60;$i++) { &BODY_40_59($i,@V); unshift(@V,pop(@V)); }
  296. for(;$i<80;$i++) { &BODY_20_39($i,@V); unshift(@V,pop(@V)); }
  297. $code.=<<___;
  298. add 0($ctx),$A
  299. add 4($ctx),$B
  300. add 8($ctx),$C
  301. add 12($ctx),$D
  302. add 16($ctx),$E
  303. mov $A,0($ctx)
  304. mov $B,4($ctx)
  305. mov $C,8($ctx)
  306. mov $D,12($ctx)
  307. mov $E,16($ctx)
  308. sub \$1,$num
  309. lea `16*4`($inp),$inp
  310. jnz .Lloop
  311. mov `16*4`(%rsp),%rsi
  312. .cfi_def_cfa %rsi,8
  313. mov -40(%rsi),%r14
  314. .cfi_restore %r14
  315. mov -32(%rsi),%r13
  316. .cfi_restore %r13
  317. mov -24(%rsi),%r12
  318. .cfi_restore %r12
  319. mov -16(%rsi),%rbp
  320. .cfi_restore %rbp
  321. mov -8(%rsi),%rbx
  322. .cfi_restore %rbx
  323. lea (%rsi),%rsp
  324. .cfi_def_cfa_register %rsp
  325. .Lepilogue:
  326. ret
  327. .cfi_endproc
  328. .size sha1_block_data_order,.-sha1_block_data_order
  329. ___
  330. if ($shaext) {{{
  331. ######################################################################
  332. # Intel SHA Extensions implementation of SHA1 update function.
  333. #
  334. my ($ctx,$inp,$num)=("%rdi","%rsi","%rdx");
  335. my ($ABCD,$E,$E_,$BSWAP,$ABCD_SAVE,$E_SAVE)=map("%xmm$_",(0..3,8,9));
  336. my @MSG=map("%xmm$_",(4..7));
  337. $code.=<<___;
  338. .type sha1_block_data_order_shaext,\@function,3
  339. .align 32
  340. sha1_block_data_order_shaext:
  341. _shaext_shortcut:
  342. .cfi_startproc
  343. ___
  344. $code.=<<___ if ($win64);
  345. lea `-8-4*16`(%rsp),%rsp
  346. movaps %xmm6,-8-4*16(%rax)
  347. movaps %xmm7,-8-3*16(%rax)
  348. movaps %xmm8,-8-2*16(%rax)
  349. movaps %xmm9,-8-1*16(%rax)
  350. .Lprologue_shaext:
  351. ___
  352. $code.=<<___;
  353. movdqu ($ctx),$ABCD
  354. movd 16($ctx),$E
  355. movdqa K_XX_XX+0xa0(%rip),$BSWAP # byte-n-word swap
  356. movdqu ($inp),@MSG[0]
  357. pshufd \$0b00011011,$ABCD,$ABCD # flip word order
  358. movdqu 0x10($inp),@MSG[1]
  359. pshufd \$0b00011011,$E,$E # flip word order
  360. movdqu 0x20($inp),@MSG[2]
  361. pshufb $BSWAP,@MSG[0]
  362. movdqu 0x30($inp),@MSG[3]
  363. pshufb $BSWAP,@MSG[1]
  364. pshufb $BSWAP,@MSG[2]
  365. movdqa $E,$E_SAVE # offload $E
  366. pshufb $BSWAP,@MSG[3]
  367. jmp .Loop_shaext
  368. .align 16
  369. .Loop_shaext:
  370. dec $num
  371. lea 0x40($inp),%r8 # next input block
  372. paddd @MSG[0],$E
  373. cmovne %r8,$inp
  374. movdqa $ABCD,$ABCD_SAVE # offload $ABCD
  375. ___
  376. for($i=0;$i<20-4;$i+=2) {
  377. $code.=<<___;
  378. sha1msg1 @MSG[1],@MSG[0]
  379. movdqa $ABCD,$E_
  380. sha1rnds4 \$`int($i/5)`,$E,$ABCD # 0-3...
  381. sha1nexte @MSG[1],$E_
  382. pxor @MSG[2],@MSG[0]
  383. sha1msg1 @MSG[2],@MSG[1]
  384. sha1msg2 @MSG[3],@MSG[0]
  385. movdqa $ABCD,$E
  386. sha1rnds4 \$`int(($i+1)/5)`,$E_,$ABCD
  387. sha1nexte @MSG[2],$E
  388. pxor @MSG[3],@MSG[1]
  389. sha1msg2 @MSG[0],@MSG[1]
  390. ___
  391. push(@MSG,shift(@MSG)); push(@MSG,shift(@MSG));
  392. }
  393. $code.=<<___;
  394. movdqu ($inp),@MSG[0]
  395. movdqa $ABCD,$E_
  396. sha1rnds4 \$3,$E,$ABCD # 64-67
  397. sha1nexte @MSG[1],$E_
  398. movdqu 0x10($inp),@MSG[1]
  399. pshufb $BSWAP,@MSG[0]
  400. movdqa $ABCD,$E
  401. sha1rnds4 \$3,$E_,$ABCD # 68-71
  402. sha1nexte @MSG[2],$E
  403. movdqu 0x20($inp),@MSG[2]
  404. pshufb $BSWAP,@MSG[1]
  405. movdqa $ABCD,$E_
  406. sha1rnds4 \$3,$E,$ABCD # 72-75
  407. sha1nexte @MSG[3],$E_
  408. movdqu 0x30($inp),@MSG[3]
  409. pshufb $BSWAP,@MSG[2]
  410. movdqa $ABCD,$E
  411. sha1rnds4 \$3,$E_,$ABCD # 76-79
  412. sha1nexte $E_SAVE,$E
  413. pshufb $BSWAP,@MSG[3]
  414. paddd $ABCD_SAVE,$ABCD
  415. movdqa $E,$E_SAVE # offload $E
  416. jnz .Loop_shaext
  417. pshufd \$0b00011011,$ABCD,$ABCD
  418. pshufd \$0b00011011,$E,$E
  419. movdqu $ABCD,($ctx)
  420. movd $E,16($ctx)
  421. ___
  422. $code.=<<___ if ($win64);
  423. movaps -8-4*16(%rax),%xmm6
  424. movaps -8-3*16(%rax),%xmm7
  425. movaps -8-2*16(%rax),%xmm8
  426. movaps -8-1*16(%rax),%xmm9
  427. mov %rax,%rsp
  428. .Lepilogue_shaext:
  429. ___
  430. $code.=<<___;
  431. .cfi_endproc
  432. ret
  433. .size sha1_block_data_order_shaext,.-sha1_block_data_order_shaext
  434. ___
  435. }}}
  436. {{{
  437. my $Xi=4;
  438. my @X=map("%xmm$_",(4..7,0..3));
  439. my @Tx=map("%xmm$_",(8..10));
  440. my $Kx="%xmm11";
  441. my @V=($A,$B,$C,$D,$E)=("%eax","%ebx","%ecx","%edx","%ebp"); # size optimization
  442. my @T=("%esi","%edi");
  443. my $j=0;
  444. my $rx=0;
  445. my $K_XX_XX="%r14";
  446. my $fp="%r11";
  447. my $_rol=sub { &rol(@_) };
  448. my $_ror=sub { &ror(@_) };
  449. { my $sn;
  450. sub align32() {
  451. ++$sn;
  452. $code.=<<___;
  453. jmp .Lalign32_$sn # see "Decoded ICache" in manual
  454. .align 32
  455. .Lalign32_$sn:
  456. ___
  457. }
  458. }
  459. $code.=<<___;
  460. .type sha1_block_data_order_ssse3,\@function,3
  461. .align 16
  462. sha1_block_data_order_ssse3:
  463. _ssse3_shortcut:
  464. .cfi_startproc
  465. mov %rsp,$fp # frame pointer
  466. .cfi_def_cfa_register $fp
  467. push %rbx
  468. .cfi_push %rbx
  469. push %rbp
  470. .cfi_push %rbp
  471. push %r12
  472. .cfi_push %r12
  473. push %r13 # redundant, done to share Win64 SE handler
  474. .cfi_push %r13
  475. push %r14
  476. .cfi_push %r14
  477. lea `-64-($win64?6*16:0)`(%rsp),%rsp
  478. ___
  479. $code.=<<___ if ($win64);
  480. movaps %xmm6,-40-6*16($fp)
  481. movaps %xmm7,-40-5*16($fp)
  482. movaps %xmm8,-40-4*16($fp)
  483. movaps %xmm9,-40-3*16($fp)
  484. movaps %xmm10,-40-2*16($fp)
  485. movaps %xmm11,-40-1*16($fp)
  486. .Lprologue_ssse3:
  487. ___
  488. $code.=<<___;
  489. and \$-64,%rsp
  490. mov %rdi,$ctx # reassigned argument
  491. mov %rsi,$inp # reassigned argument
  492. mov %rdx,$num # reassigned argument
  493. shl \$6,$num
  494. add $inp,$num
  495. lea K_XX_XX+64(%rip),$K_XX_XX
  496. mov 0($ctx),$A # load context
  497. mov 4($ctx),$B
  498. mov 8($ctx),$C
  499. mov 12($ctx),$D
  500. mov $B,@T[0] # magic seed
  501. mov 16($ctx),$E
  502. mov $C,@T[1]
  503. xor $D,@T[1]
  504. and @T[1],@T[0]
  505. movdqa 64($K_XX_XX),@X[2] # pbswap mask
  506. movdqa -64($K_XX_XX),@Tx[1] # K_00_19
  507. movdqu 0($inp),@X[-4&7] # load input to %xmm[0-3]
  508. movdqu 16($inp),@X[-3&7]
  509. movdqu 32($inp),@X[-2&7]
  510. movdqu 48($inp),@X[-1&7]
  511. pshufb @X[2],@X[-4&7] # byte swap
  512. pshufb @X[2],@X[-3&7]
  513. pshufb @X[2],@X[-2&7]
  514. add \$64,$inp
  515. paddd @Tx[1],@X[-4&7] # add K_00_19
  516. pshufb @X[2],@X[-1&7]
  517. paddd @Tx[1],@X[-3&7]
  518. paddd @Tx[1],@X[-2&7]
  519. movdqa @X[-4&7],0(%rsp) # X[]+K xfer to IALU
  520. psubd @Tx[1],@X[-4&7] # restore X[]
  521. movdqa @X[-3&7],16(%rsp)
  522. psubd @Tx[1],@X[-3&7]
  523. movdqa @X[-2&7],32(%rsp)
  524. psubd @Tx[1],@X[-2&7]
  525. jmp .Loop_ssse3
  526. ___
  527. sub AUTOLOAD() # thunk [simplified] 32-bit style perlasm
  528. { my $opcode = $AUTOLOAD; $opcode =~ s/.*:://;
  529. my $arg = pop;
  530. $arg = "\$$arg" if ($arg*1 eq $arg);
  531. $code .= "\t$opcode\t".join(',',$arg,reverse @_)."\n";
  532. }
  533. sub Xupdate_ssse3_16_31() # recall that $Xi starts with 4
  534. { use integer;
  535. my $body = shift;
  536. my @insns = (&$body,&$body,&$body,&$body); # 40 instructions
  537. my ($a,$b,$c,$d,$e);
  538. eval(shift(@insns)); # ror
  539. &pshufd (@X[0],@X[-4&7],0xee); # was &movdqa (@X[0],@X[-3&7]);
  540. eval(shift(@insns));
  541. &movdqa (@Tx[0],@X[-1&7]);
  542. &paddd (@Tx[1],@X[-1&7]);
  543. eval(shift(@insns));
  544. eval(shift(@insns));
  545. &punpcklqdq(@X[0],@X[-3&7]); # compose "X[-14]" in "X[0]", was &palignr(@X[0],@X[-4&7],8);
  546. eval(shift(@insns));
  547. eval(shift(@insns)); # rol
  548. eval(shift(@insns));
  549. &psrldq (@Tx[0],4); # "X[-3]", 3 dwords
  550. eval(shift(@insns));
  551. eval(shift(@insns));
  552. &pxor (@X[0],@X[-4&7]); # "X[0]"^="X[-16]"
  553. eval(shift(@insns));
  554. eval(shift(@insns)); # ror
  555. &pxor (@Tx[0],@X[-2&7]); # "X[-3]"^"X[-8]"
  556. eval(shift(@insns));
  557. eval(shift(@insns));
  558. eval(shift(@insns));
  559. &pxor (@X[0],@Tx[0]); # "X[0]"^="X[-3]"^"X[-8]"
  560. eval(shift(@insns));
  561. eval(shift(@insns)); # rol
  562. &movdqa (eval(16*(($Xi-1)&3))."(%rsp)",@Tx[1]); # X[]+K xfer to IALU
  563. eval(shift(@insns));
  564. eval(shift(@insns));
  565. &movdqa (@Tx[2],@X[0]);
  566. eval(shift(@insns));
  567. eval(shift(@insns));
  568. eval(shift(@insns)); # ror
  569. &movdqa (@Tx[0],@X[0]);
  570. eval(shift(@insns));
  571. &pslldq (@Tx[2],12); # "X[0]"<<96, extract one dword
  572. &paddd (@X[0],@X[0]);
  573. eval(shift(@insns));
  574. eval(shift(@insns));
  575. &psrld (@Tx[0],31);
  576. eval(shift(@insns));
  577. eval(shift(@insns)); # rol
  578. eval(shift(@insns));
  579. &movdqa (@Tx[1],@Tx[2]);
  580. eval(shift(@insns));
  581. eval(shift(@insns));
  582. &psrld (@Tx[2],30);
  583. eval(shift(@insns));
  584. eval(shift(@insns)); # ror
  585. &por (@X[0],@Tx[0]); # "X[0]"<<<=1
  586. eval(shift(@insns));
  587. eval(shift(@insns));
  588. eval(shift(@insns));
  589. &pslld (@Tx[1],2);
  590. &pxor (@X[0],@Tx[2]);
  591. eval(shift(@insns));
  592. &movdqa (@Tx[2],eval(2*16*(($Xi)/5)-64)."($K_XX_XX)"); # K_XX_XX
  593. eval(shift(@insns)); # rol
  594. eval(shift(@insns));
  595. eval(shift(@insns));
  596. &pxor (@X[0],@Tx[1]); # "X[0]"^=("X[0]">>96)<<<2
  597. &pshufd (@Tx[1],@X[-1&7],0xee) if ($Xi==7); # was &movdqa (@Tx[0],@X[-1&7]) in Xupdate_ssse3_32_79
  598. foreach (@insns) { eval; } # remaining instructions [if any]
  599. $Xi++; push(@X,shift(@X)); # "rotate" X[]
  600. push(@Tx,shift(@Tx));
  601. }
  602. sub Xupdate_ssse3_32_79()
  603. { use integer;
  604. my $body = shift;
  605. my @insns = (&$body,&$body,&$body,&$body); # 32 to 44 instructions
  606. my ($a,$b,$c,$d,$e);
  607. eval(shift(@insns)) if ($Xi==8);
  608. &pxor (@X[0],@X[-4&7]); # "X[0]"="X[-32]"^"X[-16]"
  609. eval(shift(@insns)) if ($Xi==8);
  610. eval(shift(@insns)); # body_20_39
  611. eval(shift(@insns));
  612. eval(shift(@insns)) if (@insns[1] =~ /_ror/);
  613. eval(shift(@insns)) if (@insns[0] =~ /_ror/);
  614. &punpcklqdq(@Tx[0],@X[-1&7]); # compose "X[-6]", was &palignr(@Tx[0],@X[-2&7],8);
  615. eval(shift(@insns));
  616. eval(shift(@insns)); # rol
  617. &pxor (@X[0],@X[-7&7]); # "X[0]"^="X[-28]"
  618. eval(shift(@insns));
  619. eval(shift(@insns));
  620. if ($Xi%5) {
  621. &movdqa (@Tx[2],@Tx[1]);# "perpetuate" K_XX_XX...
  622. } else { # ... or load next one
  623. &movdqa (@Tx[2],eval(2*16*($Xi/5)-64)."($K_XX_XX)");
  624. }
  625. eval(shift(@insns)); # ror
  626. &paddd (@Tx[1],@X[-1&7]);
  627. eval(shift(@insns));
  628. &pxor (@X[0],@Tx[0]); # "X[0]"^="X[-6]"
  629. eval(shift(@insns)); # body_20_39
  630. eval(shift(@insns));
  631. eval(shift(@insns));
  632. eval(shift(@insns)); # rol
  633. eval(shift(@insns)) if (@insns[0] =~ /_ror/);
  634. &movdqa (@Tx[0],@X[0]);
  635. eval(shift(@insns));
  636. eval(shift(@insns));
  637. &movdqa (eval(16*(($Xi-1)&3))."(%rsp)",@Tx[1]); # X[]+K xfer to IALU
  638. eval(shift(@insns)); # ror
  639. eval(shift(@insns));
  640. eval(shift(@insns)); # body_20_39
  641. &pslld (@X[0],2);
  642. eval(shift(@insns));
  643. eval(shift(@insns));
  644. &psrld (@Tx[0],30);
  645. eval(shift(@insns)) if (@insns[0] =~ /_rol/);# rol
  646. eval(shift(@insns));
  647. eval(shift(@insns));
  648. eval(shift(@insns)); # ror
  649. &por (@X[0],@Tx[0]); # "X[0]"<<<=2
  650. eval(shift(@insns));
  651. eval(shift(@insns)); # body_20_39
  652. eval(shift(@insns)) if (@insns[1] =~ /_rol/);
  653. eval(shift(@insns)) if (@insns[0] =~ /_rol/);
  654. &pshufd(@Tx[1],@X[-1&7],0xee) if ($Xi<19); # was &movdqa (@Tx[1],@X[0])
  655. eval(shift(@insns));
  656. eval(shift(@insns)); # rol
  657. eval(shift(@insns));
  658. eval(shift(@insns));
  659. eval(shift(@insns)); # rol
  660. eval(shift(@insns));
  661. foreach (@insns) { eval; } # remaining instructions
  662. $Xi++; push(@X,shift(@X)); # "rotate" X[]
  663. push(@Tx,shift(@Tx));
  664. }
  665. sub Xuplast_ssse3_80()
  666. { use integer;
  667. my $body = shift;
  668. my @insns = (&$body,&$body,&$body,&$body); # 32 instructions
  669. my ($a,$b,$c,$d,$e);
  670. eval(shift(@insns));
  671. eval(shift(@insns));
  672. eval(shift(@insns));
  673. eval(shift(@insns));
  674. &paddd (@Tx[1],@X[-1&7]);
  675. eval(shift(@insns));
  676. eval(shift(@insns));
  677. &movdqa (eval(16*(($Xi-1)&3))."(%rsp)",@Tx[1]); # X[]+K xfer IALU
  678. foreach (@insns) { eval; } # remaining instructions
  679. &cmp ($inp,$num);
  680. &je (".Ldone_ssse3");
  681. unshift(@Tx,pop(@Tx));
  682. &movdqa (@X[2],"64($K_XX_XX)"); # pbswap mask
  683. &movdqa (@Tx[1],"-64($K_XX_XX)"); # K_00_19
  684. &movdqu (@X[-4&7],"0($inp)"); # load input
  685. &movdqu (@X[-3&7],"16($inp)");
  686. &movdqu (@X[-2&7],"32($inp)");
  687. &movdqu (@X[-1&7],"48($inp)");
  688. &pshufb (@X[-4&7],@X[2]); # byte swap
  689. &add ($inp,64);
  690. $Xi=0;
  691. }
  692. sub Xloop_ssse3()
  693. { use integer;
  694. my $body = shift;
  695. my @insns = (&$body,&$body,&$body,&$body); # 32 instructions
  696. my ($a,$b,$c,$d,$e);
  697. eval(shift(@insns));
  698. eval(shift(@insns));
  699. eval(shift(@insns));
  700. &pshufb (@X[($Xi-3)&7],@X[2]);
  701. eval(shift(@insns));
  702. eval(shift(@insns));
  703. eval(shift(@insns));
  704. eval(shift(@insns));
  705. &paddd (@X[($Xi-4)&7],@Tx[1]);
  706. eval(shift(@insns));
  707. eval(shift(@insns));
  708. eval(shift(@insns));
  709. eval(shift(@insns));
  710. &movdqa (eval(16*$Xi)."(%rsp)",@X[($Xi-4)&7]); # X[]+K xfer to IALU
  711. eval(shift(@insns));
  712. eval(shift(@insns));
  713. eval(shift(@insns));
  714. eval(shift(@insns));
  715. &psubd (@X[($Xi-4)&7],@Tx[1]);
  716. foreach (@insns) { eval; }
  717. $Xi++;
  718. }
  719. sub Xtail_ssse3()
  720. { use integer;
  721. my $body = shift;
  722. my @insns = (&$body,&$body,&$body,&$body); # 32 instructions
  723. my ($a,$b,$c,$d,$e);
  724. foreach (@insns) { eval; }
  725. }
  726. sub body_00_19 () { # ((c^d)&b)^d
  727. # on start @T[0]=(c^d)&b
  728. return &body_20_39() if ($rx==19); $rx++;
  729. (
  730. '($a,$b,$c,$d,$e)=@V;'.
  731. '&$_ror ($b,$j?7:2)', # $b>>>2
  732. '&xor (@T[0],$d)',
  733. '&mov (@T[1],$a)', # $b for next round
  734. '&add ($e,eval(4*($j&15))."(%rsp)")', # X[]+K xfer
  735. '&xor ($b,$c)', # $c^$d for next round
  736. '&$_rol ($a,5)',
  737. '&add ($e,@T[0])',
  738. '&and (@T[1],$b)', # ($b&($c^$d)) for next round
  739. '&xor ($b,$c)', # restore $b
  740. '&add ($e,$a);' .'$j++; unshift(@V,pop(@V)); unshift(@T,pop(@T));'
  741. );
  742. }
  743. sub body_20_39 () { # b^d^c
  744. # on entry @T[0]=b^d
  745. return &body_40_59() if ($rx==39); $rx++;
  746. (
  747. '($a,$b,$c,$d,$e)=@V;'.
  748. '&add ($e,eval(4*($j&15))."(%rsp)")', # X[]+K xfer
  749. '&xor (@T[0],$d) if($j==19);'.
  750. '&xor (@T[0],$c) if($j> 19)', # ($b^$d^$c)
  751. '&mov (@T[1],$a)', # $b for next round
  752. '&$_rol ($a,5)',
  753. '&add ($e,@T[0])',
  754. '&xor (@T[1],$c) if ($j< 79)', # $b^$d for next round
  755. '&$_ror ($b,7)', # $b>>>2
  756. '&add ($e,$a);' .'$j++; unshift(@V,pop(@V)); unshift(@T,pop(@T));'
  757. );
  758. }
  759. sub body_40_59 () { # ((b^c)&(c^d))^c
  760. # on entry @T[0]=(b^c), (c^=d)
  761. $rx++;
  762. (
  763. '($a,$b,$c,$d,$e)=@V;'.
  764. '&add ($e,eval(4*($j&15))."(%rsp)")', # X[]+K xfer
  765. '&and (@T[0],$c) if ($j>=40)', # (b^c)&(c^d)
  766. '&xor ($c,$d) if ($j>=40)', # restore $c
  767. '&$_ror ($b,7)', # $b>>>2
  768. '&mov (@T[1],$a)', # $b for next round
  769. '&xor (@T[0],$c)',
  770. '&$_rol ($a,5)',
  771. '&add ($e,@T[0])',
  772. '&xor (@T[1],$c) if ($j==59);'.
  773. '&xor (@T[1],$b) if ($j< 59)', # b^c for next round
  774. '&xor ($b,$c) if ($j< 59)', # c^d for next round
  775. '&add ($e,$a);' .'$j++; unshift(@V,pop(@V)); unshift(@T,pop(@T));'
  776. );
  777. }
  778. $code.=<<___;
  779. .align 16
  780. .Loop_ssse3:
  781. ___
  782. &Xupdate_ssse3_16_31(\&body_00_19);
  783. &Xupdate_ssse3_16_31(\&body_00_19);
  784. &Xupdate_ssse3_16_31(\&body_00_19);
  785. &Xupdate_ssse3_16_31(\&body_00_19);
  786. &Xupdate_ssse3_32_79(\&body_00_19);
  787. &Xupdate_ssse3_32_79(\&body_20_39);
  788. &Xupdate_ssse3_32_79(\&body_20_39);
  789. &Xupdate_ssse3_32_79(\&body_20_39);
  790. &Xupdate_ssse3_32_79(\&body_20_39);
  791. &Xupdate_ssse3_32_79(\&body_20_39);
  792. &Xupdate_ssse3_32_79(\&body_40_59);
  793. &Xupdate_ssse3_32_79(\&body_40_59);
  794. &Xupdate_ssse3_32_79(\&body_40_59);
  795. &Xupdate_ssse3_32_79(\&body_40_59);
  796. &Xupdate_ssse3_32_79(\&body_40_59);
  797. &Xupdate_ssse3_32_79(\&body_20_39);
  798. &Xuplast_ssse3_80(\&body_20_39); # can jump to "done"
  799. $saved_j=$j; @saved_V=@V;
  800. &Xloop_ssse3(\&body_20_39);
  801. &Xloop_ssse3(\&body_20_39);
  802. &Xloop_ssse3(\&body_20_39);
  803. $code.=<<___;
  804. add 0($ctx),$A # update context
  805. add 4($ctx),@T[0]
  806. add 8($ctx),$C
  807. add 12($ctx),$D
  808. mov $A,0($ctx)
  809. add 16($ctx),$E
  810. mov @T[0],4($ctx)
  811. mov @T[0],$B # magic seed
  812. mov $C,8($ctx)
  813. mov $C,@T[1]
  814. mov $D,12($ctx)
  815. xor $D,@T[1]
  816. mov $E,16($ctx)
  817. and @T[1],@T[0]
  818. jmp .Loop_ssse3
  819. .align 16
  820. .Ldone_ssse3:
  821. ___
  822. $j=$saved_j; @V=@saved_V;
  823. &Xtail_ssse3(\&body_20_39);
  824. &Xtail_ssse3(\&body_20_39);
  825. &Xtail_ssse3(\&body_20_39);
  826. $code.=<<___;
  827. add 0($ctx),$A # update context
  828. add 4($ctx),@T[0]
  829. add 8($ctx),$C
  830. mov $A,0($ctx)
  831. add 12($ctx),$D
  832. mov @T[0],4($ctx)
  833. add 16($ctx),$E
  834. mov $C,8($ctx)
  835. mov $D,12($ctx)
  836. mov $E,16($ctx)
  837. ___
  838. $code.=<<___ if ($win64);
  839. movaps -40-6*16($fp),%xmm6
  840. movaps -40-5*16($fp),%xmm7
  841. movaps -40-4*16($fp),%xmm8
  842. movaps -40-3*16($fp),%xmm9
  843. movaps -40-2*16($fp),%xmm10
  844. movaps -40-1*16($fp),%xmm11
  845. ___
  846. $code.=<<___;
  847. mov -40($fp),%r14
  848. .cfi_restore %r14
  849. mov -32($fp),%r13
  850. .cfi_restore %r13
  851. mov -24($fp),%r12
  852. .cfi_restore %r12
  853. mov -16($fp),%rbp
  854. .cfi_restore %rbp
  855. mov -8($fp),%rbx
  856. .cfi_restore %rbx
  857. lea ($fp),%rsp
  858. .cfi_def_cfa_register %rsp
  859. .Lepilogue_ssse3:
  860. ret
  861. .cfi_endproc
  862. .size sha1_block_data_order_ssse3,.-sha1_block_data_order_ssse3
  863. ___
  864. if ($avx) {
  865. $Xi=4; # reset variables
  866. @X=map("%xmm$_",(4..7,0..3));
  867. @Tx=map("%xmm$_",(8..10));
  868. $j=0;
  869. $rx=0;
  870. my $done_avx_label=".Ldone_avx";
  871. my $_rol=sub { &shld(@_[0],@_) };
  872. my $_ror=sub { &shrd(@_[0],@_) };
  873. $code.=<<___;
  874. .type sha1_block_data_order_avx,\@function,3
  875. .align 16
  876. sha1_block_data_order_avx:
  877. _avx_shortcut:
  878. .cfi_startproc
  879. mov %rsp,$fp
  880. .cfi_def_cfa_register $fp
  881. push %rbx
  882. .cfi_push %rbx
  883. push %rbp
  884. .cfi_push %rbp
  885. push %r12
  886. .cfi_push %r12
  887. push %r13 # redundant, done to share Win64 SE handler
  888. .cfi_push %r13
  889. push %r14
  890. .cfi_push %r14
  891. lea `-64-($win64?6*16:0)`(%rsp),%rsp
  892. vzeroupper
  893. ___
  894. $code.=<<___ if ($win64);
  895. vmovaps %xmm6,-40-6*16($fp)
  896. vmovaps %xmm7,-40-5*16($fp)
  897. vmovaps %xmm8,-40-4*16($fp)
  898. vmovaps %xmm9,-40-3*16($fp)
  899. vmovaps %xmm10,-40-2*16($fp)
  900. vmovaps %xmm11,-40-1*16($fp)
  901. .Lprologue_avx:
  902. ___
  903. $code.=<<___;
  904. and \$-64,%rsp
  905. mov %rdi,$ctx # reassigned argument
  906. mov %rsi,$inp # reassigned argument
  907. mov %rdx,$num # reassigned argument
  908. shl \$6,$num
  909. add $inp,$num
  910. lea K_XX_XX+64(%rip),$K_XX_XX
  911. mov 0($ctx),$A # load context
  912. mov 4($ctx),$B
  913. mov 8($ctx),$C
  914. mov 12($ctx),$D
  915. mov $B,@T[0] # magic seed
  916. mov 16($ctx),$E
  917. mov $C,@T[1]
  918. xor $D,@T[1]
  919. and @T[1],@T[0]
  920. vmovdqa 64($K_XX_XX),@X[2] # pbswap mask
  921. vmovdqa -64($K_XX_XX),$Kx # K_00_19
  922. vmovdqu 0($inp),@X[-4&7] # load input to %xmm[0-3]
  923. vmovdqu 16($inp),@X[-3&7]
  924. vmovdqu 32($inp),@X[-2&7]
  925. vmovdqu 48($inp),@X[-1&7]
  926. vpshufb @X[2],@X[-4&7],@X[-4&7] # byte swap
  927. add \$64,$inp
  928. vpshufb @X[2],@X[-3&7],@X[-3&7]
  929. vpshufb @X[2],@X[-2&7],@X[-2&7]
  930. vpshufb @X[2],@X[-1&7],@X[-1&7]
  931. vpaddd $Kx,@X[-4&7],@X[0] # add K_00_19
  932. vpaddd $Kx,@X[-3&7],@X[1]
  933. vpaddd $Kx,@X[-2&7],@X[2]
  934. vmovdqa @X[0],0(%rsp) # X[]+K xfer to IALU
  935. vmovdqa @X[1],16(%rsp)
  936. vmovdqa @X[2],32(%rsp)
  937. jmp .Loop_avx
  938. ___
  939. sub Xupdate_avx_16_31() # recall that $Xi starts with 4
  940. { use integer;
  941. my $body = shift;
  942. my @insns = (&$body,&$body,&$body,&$body); # 40 instructions
  943. my ($a,$b,$c,$d,$e);
  944. eval(shift(@insns));
  945. eval(shift(@insns));
  946. &vpalignr(@X[0],@X[-3&7],@X[-4&7],8); # compose "X[-14]" in "X[0]"
  947. eval(shift(@insns));
  948. eval(shift(@insns));
  949. &vpaddd (@Tx[1],$Kx,@X[-1&7]);
  950. eval(shift(@insns));
  951. eval(shift(@insns));
  952. &vpsrldq(@Tx[0],@X[-1&7],4); # "X[-3]", 3 dwords
  953. eval(shift(@insns));
  954. eval(shift(@insns));
  955. &vpxor (@X[0],@X[0],@X[-4&7]); # "X[0]"^="X[-16]"
  956. eval(shift(@insns));
  957. eval(shift(@insns));
  958. &vpxor (@Tx[0],@Tx[0],@X[-2&7]); # "X[-3]"^"X[-8]"
  959. eval(shift(@insns));
  960. eval(shift(@insns));
  961. eval(shift(@insns));
  962. eval(shift(@insns));
  963. &vpxor (@X[0],@X[0],@Tx[0]); # "X[0]"^="X[-3]"^"X[-8]"
  964. eval(shift(@insns));
  965. eval(shift(@insns));
  966. &vmovdqa (eval(16*(($Xi-1)&3))."(%rsp)",@Tx[1]); # X[]+K xfer to IALU
  967. eval(shift(@insns));
  968. eval(shift(@insns));
  969. &vpsrld (@Tx[0],@X[0],31);
  970. eval(shift(@insns));
  971. eval(shift(@insns));
  972. eval(shift(@insns));
  973. eval(shift(@insns));
  974. &vpslldq(@Tx[2],@X[0],12); # "X[0]"<<96, extract one dword
  975. &vpaddd (@X[0],@X[0],@X[0]);
  976. eval(shift(@insns));
  977. eval(shift(@insns));
  978. eval(shift(@insns));
  979. eval(shift(@insns));
  980. &vpsrld (@Tx[1],@Tx[2],30);
  981. &vpor (@X[0],@X[0],@Tx[0]); # "X[0]"<<<=1
  982. eval(shift(@insns));
  983. eval(shift(@insns));
  984. eval(shift(@insns));
  985. eval(shift(@insns));
  986. &vpslld (@Tx[2],@Tx[2],2);
  987. &vpxor (@X[0],@X[0],@Tx[1]);
  988. eval(shift(@insns));
  989. eval(shift(@insns));
  990. eval(shift(@insns));
  991. eval(shift(@insns));
  992. &vpxor (@X[0],@X[0],@Tx[2]); # "X[0]"^=("X[0]">>96)<<<2
  993. eval(shift(@insns));
  994. eval(shift(@insns));
  995. &vmovdqa ($Kx,eval(2*16*(($Xi)/5)-64)."($K_XX_XX)") if ($Xi%5==0); # K_XX_XX
  996. eval(shift(@insns));
  997. eval(shift(@insns));
  998. foreach (@insns) { eval; } # remaining instructions [if any]
  999. $Xi++; push(@X,shift(@X)); # "rotate" X[]
  1000. }
  1001. sub Xupdate_avx_32_79()
  1002. { use integer;
  1003. my $body = shift;
  1004. my @insns = (&$body,&$body,&$body,&$body); # 32 to 44 instructions
  1005. my ($a,$b,$c,$d,$e);
  1006. &vpalignr(@Tx[0],@X[-1&7],@X[-2&7],8); # compose "X[-6]"
  1007. &vpxor (@X[0],@X[0],@X[-4&7]); # "X[0]"="X[-32]"^"X[-16]"
  1008. eval(shift(@insns)); # body_20_39
  1009. eval(shift(@insns));
  1010. eval(shift(@insns));
  1011. eval(shift(@insns)); # rol
  1012. &vpxor (@X[0],@X[0],@X[-7&7]); # "X[0]"^="X[-28]"
  1013. eval(shift(@insns));
  1014. eval(shift(@insns)) if (@insns[0] !~ /&ro[rl]/);
  1015. &vpaddd (@Tx[1],$Kx,@X[-1&7]);
  1016. &vmovdqa ($Kx,eval(2*16*($Xi/5)-64)."($K_XX_XX)") if ($Xi%5==0);
  1017. eval(shift(@insns)); # ror
  1018. eval(shift(@insns));
  1019. &vpxor (@X[0],@X[0],@Tx[0]); # "X[0]"^="X[-6]"
  1020. eval(shift(@insns)); # body_20_39
  1021. eval(shift(@insns));
  1022. eval(shift(@insns));
  1023. eval(shift(@insns)); # rol
  1024. &vpsrld (@Tx[0],@X[0],30);
  1025. &vmovdqa (eval(16*(($Xi-1)&3))."(%rsp)",@Tx[1]); # X[]+K xfer to IALU
  1026. eval(shift(@insns));
  1027. eval(shift(@insns));
  1028. eval(shift(@insns)); # ror
  1029. eval(shift(@insns));
  1030. &vpslld (@X[0],@X[0],2);
  1031. eval(shift(@insns)); # body_20_39
  1032. eval(shift(@insns));
  1033. eval(shift(@insns));
  1034. eval(shift(@insns)); # rol
  1035. eval(shift(@insns));
  1036. eval(shift(@insns));
  1037. eval(shift(@insns)); # ror
  1038. eval(shift(@insns));
  1039. &vpor (@X[0],@X[0],@Tx[0]); # "X[0]"<<<=2
  1040. eval(shift(@insns)); # body_20_39
  1041. eval(shift(@insns));
  1042. eval(shift(@insns));
  1043. eval(shift(@insns)); # rol
  1044. eval(shift(@insns));
  1045. eval(shift(@insns));
  1046. eval(shift(@insns)); # rol
  1047. eval(shift(@insns));
  1048. foreach (@insns) { eval; } # remaining instructions
  1049. $Xi++; push(@X,shift(@X)); # "rotate" X[]
  1050. }
  1051. sub Xuplast_avx_80()
  1052. { use integer;
  1053. my $body = shift;
  1054. my @insns = (&$body,&$body,&$body,&$body); # 32 instructions
  1055. my ($a,$b,$c,$d,$e);
  1056. eval(shift(@insns));
  1057. &vpaddd (@Tx[1],$Kx,@X[-1&7]);
  1058. eval(shift(@insns));
  1059. eval(shift(@insns));
  1060. eval(shift(@insns));
  1061. eval(shift(@insns));
  1062. &vmovdqa (eval(16*(($Xi-1)&3))."(%rsp)",@Tx[1]); # X[]+K xfer IALU
  1063. foreach (@insns) { eval; } # remaining instructions
  1064. &cmp ($inp,$num);
  1065. &je ($done_avx_label);
  1066. &vmovdqa(@X[2],"64($K_XX_XX)"); # pbswap mask
  1067. &vmovdqa($Kx,"-64($K_XX_XX)"); # K_00_19
  1068. &vmovdqu(@X[-4&7],"0($inp)"); # load input
  1069. &vmovdqu(@X[-3&7],"16($inp)");
  1070. &vmovdqu(@X[-2&7],"32($inp)");
  1071. &vmovdqu(@X[-1&7],"48($inp)");
  1072. &vpshufb(@X[-4&7],@X[-4&7],@X[2]); # byte swap
  1073. &add ($inp,64);
  1074. $Xi=0;
  1075. }
  1076. sub Xloop_avx()
  1077. { use integer;
  1078. my $body = shift;
  1079. my @insns = (&$body,&$body,&$body,&$body); # 32 instructions
  1080. my ($a,$b,$c,$d,$e);
  1081. eval(shift(@insns));
  1082. eval(shift(@insns));
  1083. &vpshufb(@X[($Xi-3)&7],@X[($Xi-3)&7],@X[2]);
  1084. eval(shift(@insns));
  1085. eval(shift(@insns));
  1086. &vpaddd (@X[$Xi&7],@X[($Xi-4)&7],$Kx);
  1087. eval(shift(@insns));
  1088. eval(shift(@insns));
  1089. eval(shift(@insns));
  1090. eval(shift(@insns));
  1091. &vmovdqa(eval(16*$Xi)."(%rsp)",@X[$Xi&7]); # X[]+K xfer to IALU
  1092. eval(shift(@insns));
  1093. eval(shift(@insns));
  1094. foreach (@insns) { eval; }
  1095. $Xi++;
  1096. }
  1097. sub Xtail_avx()
  1098. { use integer;
  1099. my $body = shift;
  1100. my @insns = (&$body,&$body,&$body,&$body); # 32 instructions
  1101. my ($a,$b,$c,$d,$e);
  1102. foreach (@insns) { eval; }
  1103. }
  1104. $code.=<<___;
  1105. .align 16
  1106. .Loop_avx:
  1107. ___
  1108. &Xupdate_avx_16_31(\&body_00_19);
  1109. &Xupdate_avx_16_31(\&body_00_19);
  1110. &Xupdate_avx_16_31(\&body_00_19);
  1111. &Xupdate_avx_16_31(\&body_00_19);
  1112. &Xupdate_avx_32_79(\&body_00_19);
  1113. &Xupdate_avx_32_79(\&body_20_39);
  1114. &Xupdate_avx_32_79(\&body_20_39);
  1115. &Xupdate_avx_32_79(\&body_20_39);
  1116. &Xupdate_avx_32_79(\&body_20_39);
  1117. &Xupdate_avx_32_79(\&body_20_39);
  1118. &Xupdate_avx_32_79(\&body_40_59);
  1119. &Xupdate_avx_32_79(\&body_40_59);
  1120. &Xupdate_avx_32_79(\&body_40_59);
  1121. &Xupdate_avx_32_79(\&body_40_59);
  1122. &Xupdate_avx_32_79(\&body_40_59);
  1123. &Xupdate_avx_32_79(\&body_20_39);
  1124. &Xuplast_avx_80(\&body_20_39); # can jump to "done"
  1125. $saved_j=$j; @saved_V=@V;
  1126. &Xloop_avx(\&body_20_39);
  1127. &Xloop_avx(\&body_20_39);
  1128. &Xloop_avx(\&body_20_39);
  1129. $code.=<<___;
  1130. add 0($ctx),$A # update context
  1131. add 4($ctx),@T[0]
  1132. add 8($ctx),$C
  1133. add 12($ctx),$D
  1134. mov $A,0($ctx)
  1135. add 16($ctx),$E
  1136. mov @T[0],4($ctx)
  1137. mov @T[0],$B # magic seed
  1138. mov $C,8($ctx)
  1139. mov $C,@T[1]
  1140. mov $D,12($ctx)
  1141. xor $D,@T[1]
  1142. mov $E,16($ctx)
  1143. and @T[1],@T[0]
  1144. jmp .Loop_avx
  1145. .align 16
  1146. $done_avx_label:
  1147. ___
  1148. $j=$saved_j; @V=@saved_V;
  1149. &Xtail_avx(\&body_20_39);
  1150. &Xtail_avx(\&body_20_39);
  1151. &Xtail_avx(\&body_20_39);
  1152. $code.=<<___;
  1153. vzeroupper
  1154. add 0($ctx),$A # update context
  1155. add 4($ctx),@T[0]
  1156. add 8($ctx),$C
  1157. mov $A,0($ctx)
  1158. add 12($ctx),$D
  1159. mov @T[0],4($ctx)
  1160. add 16($ctx),$E
  1161. mov $C,8($ctx)
  1162. mov $D,12($ctx)
  1163. mov $E,16($ctx)
  1164. ___
  1165. $code.=<<___ if ($win64);
  1166. movaps -40-6*16($fp),%xmm6
  1167. movaps -40-5*16($fp),%xmm7
  1168. movaps -40-4*16($fp),%xmm8
  1169. movaps -40-3*16($fp),%xmm9
  1170. movaps -40-2*16($fp),%xmm10
  1171. movaps -40-1*16($fp),%xmm11
  1172. ___
  1173. $code.=<<___;
  1174. mov -40($fp),%r14
  1175. .cfi_restore %r14
  1176. mov -32($fp),%r13
  1177. .cfi_restore %r13
  1178. mov -24($fp),%r12
  1179. .cfi_restore %r12
  1180. mov -16($fp),%rbp
  1181. .cfi_restore %rbp
  1182. mov -8($fp),%rbx
  1183. .cfi_restore %rbx
  1184. lea ($fp),%rsp
  1185. .cfi_def_cfa_register %rsp
  1186. .Lepilogue_avx:
  1187. ret
  1188. .cfi_endproc
  1189. .size sha1_block_data_order_avx,.-sha1_block_data_order_avx
  1190. ___
  1191. if ($avx>1) {
  1192. use integer;
  1193. $Xi=4; # reset variables
  1194. @X=map("%ymm$_",(4..7,0..3));
  1195. @Tx=map("%ymm$_",(8..10));
  1196. $Kx="%ymm11";
  1197. $j=0;
  1198. my @ROTX=("%eax","%ebp","%ebx","%ecx","%edx","%esi");
  1199. my ($a5,$t0)=("%r12d","%edi");
  1200. my ($A,$F,$B,$C,$D,$E)=@ROTX;
  1201. my $rx=0;
  1202. my $frame="%r13";
  1203. $code.=<<___;
  1204. .type sha1_block_data_order_avx2,\@function,3
  1205. .align 16
  1206. sha1_block_data_order_avx2:
  1207. _avx2_shortcut:
  1208. .cfi_startproc
  1209. mov %rsp,$fp
  1210. .cfi_def_cfa_register $fp
  1211. push %rbx
  1212. .cfi_push %rbx
  1213. push %rbp
  1214. .cfi_push %rbp
  1215. push %r12
  1216. .cfi_push %r12
  1217. push %r13
  1218. .cfi_push %r13
  1219. push %r14
  1220. .cfi_push %r14
  1221. vzeroupper
  1222. ___
  1223. $code.=<<___ if ($win64);
  1224. lea -6*16(%rsp),%rsp
  1225. vmovaps %xmm6,-40-6*16($fp)
  1226. vmovaps %xmm7,-40-5*16($fp)
  1227. vmovaps %xmm8,-40-4*16($fp)
  1228. vmovaps %xmm9,-40-3*16($fp)
  1229. vmovaps %xmm10,-40-2*16($fp)
  1230. vmovaps %xmm11,-40-1*16($fp)
  1231. .Lprologue_avx2:
  1232. ___
  1233. $code.=<<___;
  1234. mov %rdi,$ctx # reassigned argument
  1235. mov %rsi,$inp # reassigned argument
  1236. mov %rdx,$num # reassigned argument
  1237. lea -640(%rsp),%rsp
  1238. shl \$6,$num
  1239. lea 64($inp),$frame
  1240. and \$-128,%rsp
  1241. add $inp,$num
  1242. lea K_XX_XX+64(%rip),$K_XX_XX
  1243. mov 0($ctx),$A # load context
  1244. cmp $num,$frame
  1245. cmovae $inp,$frame # next or same block
  1246. mov 4($ctx),$F
  1247. mov 8($ctx),$C
  1248. mov 12($ctx),$D
  1249. mov 16($ctx),$E
  1250. vmovdqu 64($K_XX_XX),@X[2] # pbswap mask
  1251. vmovdqu ($inp),%xmm0
  1252. vmovdqu 16($inp),%xmm1
  1253. vmovdqu 32($inp),%xmm2
  1254. vmovdqu 48($inp),%xmm3
  1255. lea 64($inp),$inp
  1256. vinserti128 \$1,($frame),@X[-4&7],@X[-4&7]
  1257. vinserti128 \$1,16($frame),@X[-3&7],@X[-3&7]
  1258. vpshufb @X[2],@X[-4&7],@X[-4&7]
  1259. vinserti128 \$1,32($frame),@X[-2&7],@X[-2&7]
  1260. vpshufb @X[2],@X[-3&7],@X[-3&7]
  1261. vinserti128 \$1,48($frame),@X[-1&7],@X[-1&7]
  1262. vpshufb @X[2],@X[-2&7],@X[-2&7]
  1263. vmovdqu -64($K_XX_XX),$Kx # K_00_19
  1264. vpshufb @X[2],@X[-1&7],@X[-1&7]
  1265. vpaddd $Kx,@X[-4&7],@X[0] # add K_00_19
  1266. vpaddd $Kx,@X[-3&7],@X[1]
  1267. vmovdqu @X[0],0(%rsp) # X[]+K xfer to IALU
  1268. vpaddd $Kx,@X[-2&7],@X[2]
  1269. vmovdqu @X[1],32(%rsp)
  1270. vpaddd $Kx,@X[-1&7],@X[3]
  1271. vmovdqu @X[2],64(%rsp)
  1272. vmovdqu @X[3],96(%rsp)
  1273. ___
  1274. for (;$Xi<8;$Xi++) { # Xupdate_avx2_16_31
  1275. use integer;
  1276. &vpalignr(@X[0],@X[-3&7],@X[-4&7],8); # compose "X[-14]" in "X[0]"
  1277. &vpsrldq(@Tx[0],@X[-1&7],4); # "X[-3]", 3 dwords
  1278. &vpxor (@X[0],@X[0],@X[-4&7]); # "X[0]"^="X[-16]"
  1279. &vpxor (@Tx[0],@Tx[0],@X[-2&7]); # "X[-3]"^"X[-8]"
  1280. &vpxor (@X[0],@X[0],@Tx[0]); # "X[0]"^="X[-3]"^"X[-8]"
  1281. &vpsrld (@Tx[0],@X[0],31);
  1282. &vmovdqu($Kx,eval(2*16*(($Xi)/5)-64)."($K_XX_XX)") if ($Xi%5==0); # K_XX_XX
  1283. &vpslldq(@Tx[2],@X[0],12); # "X[0]"<<96, extract one dword
  1284. &vpaddd (@X[0],@X[0],@X[0]);
  1285. &vpsrld (@Tx[1],@Tx[2],30);
  1286. &vpor (@X[0],@X[0],@Tx[0]); # "X[0]"<<<=1
  1287. &vpslld (@Tx[2],@Tx[2],2);
  1288. &vpxor (@X[0],@X[0],@Tx[1]);
  1289. &vpxor (@X[0],@X[0],@Tx[2]); # "X[0]"^=("X[0]">>96)<<<2
  1290. &vpaddd (@Tx[1],@X[0],$Kx);
  1291. &vmovdqu("32*$Xi(%rsp)",@Tx[1]); # X[]+K xfer to IALU
  1292. push(@X,shift(@X)); # "rotate" X[]
  1293. }
  1294. $code.=<<___;
  1295. lea 128(%rsp),$frame
  1296. jmp .Loop_avx2
  1297. .align 32
  1298. .Loop_avx2:
  1299. rorx \$2,$F,$B
  1300. andn $D,$F,$t0
  1301. and $C,$F
  1302. xor $t0,$F
  1303. ___
  1304. sub bodyx_00_19 () { # 8 instructions, 3 cycles critical path
  1305. # at start $f=(b&c)^(~b&d), $b>>>=2
  1306. return &bodyx_20_39() if ($rx==19); $rx++;
  1307. (
  1308. '($a,$f,$b,$c,$d,$e)=@ROTX;'.
  1309. '&add ($e,((32*($j/4)+4*($j%4))%256-128)."($frame)");'. # e+=X[i]+K
  1310. '&lea ($frame,"256($frame)") if ($j%32==31);',
  1311. '&andn ($t0,$a,$c)', # ~b&d for next round
  1312. '&add ($e,$f)', # e+=(b&c)^(~b&d)
  1313. '&rorx ($a5,$a,27)', # a<<<5
  1314. '&rorx ($f,$a,2)', # b>>>2 for next round
  1315. '&and ($a,$b)', # b&c for next round
  1316. '&add ($e,$a5)', # e+=a<<<5
  1317. '&xor ($a,$t0);'. # f=(b&c)^(~b&d) for next round
  1318. 'unshift(@ROTX,pop(@ROTX)); $j++;'
  1319. )
  1320. }
  1321. sub bodyx_20_39 () { # 7 instructions, 2 cycles critical path
  1322. # on entry $f=b^c^d, $b>>>=2
  1323. return &bodyx_40_59() if ($rx==39); $rx++;
  1324. (
  1325. '($a,$f,$b,$c,$d,$e)=@ROTX;'.
  1326. '&add ($e,((32*($j/4)+4*($j%4))%256-128)."($frame)");'. # e+=X[i]+K
  1327. '&lea ($frame,"256($frame)") if ($j%32==31);',
  1328. '&lea ($e,"($e,$f)")', # e+=b^c^d
  1329. '&rorx ($a5,$a,27)', # a<<<5
  1330. '&rorx ($f,$a,2) if ($j<79)', # b>>>2 in next round
  1331. '&xor ($a,$b) if ($j<79)', # b^c for next round
  1332. '&add ($e,$a5)', # e+=a<<<5
  1333. '&xor ($a,$c) if ($j<79);'. # f=b^c^d for next round
  1334. 'unshift(@ROTX,pop(@ROTX)); $j++;'
  1335. )
  1336. }
  1337. sub bodyx_40_59 () { # 10 instructions, 3 cycles critical path
  1338. # on entry $f=((b^c)&(c^d)), $b>>>=2
  1339. $rx++;
  1340. (
  1341. '($a,$f,$b,$c,$d,$e)=@ROTX;'.
  1342. '&add ($e,((32*($j/4)+4*($j%4))%256-128)."($frame)");'. # e+=X[i]+K
  1343. '&lea ($frame,"256($frame)") if ($j%32==31);',
  1344. '&xor ($f,$c) if ($j>39)', # (b^c)&(c^d)^c
  1345. '&mov ($t0,$b) if ($j<59)', # count on zero latency
  1346. '&xor ($t0,$c) if ($j<59)', # c^d for next round
  1347. '&lea ($e,"($e,$f)")', # e+=(b^c)&(c^d)^c
  1348. '&rorx ($a5,$a,27)', # a<<<5
  1349. '&rorx ($f,$a,2)', # b>>>2 in next round
  1350. '&xor ($a,$b)', # b^c for next round
  1351. '&add ($e,$a5)', # e+=a<<<5
  1352. '&and ($a,$t0) if ($j< 59);'. # f=(b^c)&(c^d) for next round
  1353. '&xor ($a,$c) if ($j==59);'. # f=b^c^d for next round
  1354. 'unshift(@ROTX,pop(@ROTX)); $j++;'
  1355. )
  1356. }
  1357. sub Xupdate_avx2_16_31() # recall that $Xi starts with 4
  1358. { use integer;
  1359. my $body = shift;
  1360. my @insns = (&$body,&$body,&$body,&$body,&$body); # 35 instructions
  1361. my ($a,$b,$c,$d,$e);
  1362. &vpalignr(@X[0],@X[-3&7],@X[-4&7],8); # compose "X[-14]" in "X[0]"
  1363. eval(shift(@insns));
  1364. eval(shift(@insns));
  1365. eval(shift(@insns));
  1366. eval(shift(@insns));
  1367. &vpsrldq(@Tx[0],@X[-1&7],4); # "X[-3]", 3 dwords
  1368. eval(shift(@insns));
  1369. eval(shift(@insns));
  1370. eval(shift(@insns));
  1371. &vpxor (@X[0],@X[0],@X[-4&7]); # "X[0]"^="X[-16]"
  1372. &vpxor (@Tx[0],@Tx[0],@X[-2&7]); # "X[-3]"^"X[-8]"
  1373. eval(shift(@insns));
  1374. eval(shift(@insns));
  1375. &vpxor (@X[0],@X[0],@Tx[0]); # "X[0]"^="X[-3]"^"X[-8]"
  1376. eval(shift(@insns));
  1377. eval(shift(@insns));
  1378. eval(shift(@insns));
  1379. eval(shift(@insns));
  1380. &vpsrld (@Tx[0],@X[0],31);
  1381. &vmovdqu($Kx,eval(2*16*(($Xi)/5)-64)."($K_XX_XX)") if ($Xi%5==0); # K_XX_XX
  1382. eval(shift(@insns));
  1383. eval(shift(@insns));
  1384. eval(shift(@insns));
  1385. &vpslldq(@Tx[2],@X[0],12); # "X[0]"<<96, extract one dword
  1386. &vpaddd (@X[0],@X[0],@X[0]);
  1387. eval(shift(@insns));
  1388. eval(shift(@insns));
  1389. &vpsrld (@Tx[1],@Tx[2],30);
  1390. &vpor (@X[0],@X[0],@Tx[0]); # "X[0]"<<<=1
  1391. eval(shift(@insns));
  1392. eval(shift(@insns));
  1393. &vpslld (@Tx[2],@Tx[2],2);
  1394. &vpxor (@X[0],@X[0],@Tx[1]);
  1395. eval(shift(@insns));
  1396. eval(shift(@insns));
  1397. &vpxor (@X[0],@X[0],@Tx[2]); # "X[0]"^=("X[0]">>96)<<<2
  1398. eval(shift(@insns));
  1399. eval(shift(@insns));
  1400. eval(shift(@insns));
  1401. &vpaddd (@Tx[1],@X[0],$Kx);
  1402. eval(shift(@insns));
  1403. eval(shift(@insns));
  1404. eval(shift(@insns));
  1405. &vmovdqu(eval(32*($Xi))."(%rsp)",@Tx[1]); # X[]+K xfer to IALU
  1406. foreach (@insns) { eval; } # remaining instructions [if any]
  1407. $Xi++;
  1408. push(@X,shift(@X)); # "rotate" X[]
  1409. }
  1410. sub Xupdate_avx2_32_79()
  1411. { use integer;
  1412. my $body = shift;
  1413. my @insns = (&$body,&$body,&$body,&$body,&$body); # 35 to 50 instructions
  1414. my ($a,$b,$c,$d,$e);
  1415. &vpalignr(@Tx[0],@X[-1&7],@X[-2&7],8); # compose "X[-6]"
  1416. &vpxor (@X[0],@X[0],@X[-4&7]); # "X[0]"="X[-32]"^"X[-16]"
  1417. eval(shift(@insns));
  1418. eval(shift(@insns));
  1419. &vpxor (@X[0],@X[0],@X[-7&7]); # "X[0]"^="X[-28]"
  1420. &vmovdqu($Kx,eval(2*16*($Xi/5)-64)."($K_XX_XX)") if ($Xi%5==0);
  1421. eval(shift(@insns));
  1422. eval(shift(@insns));
  1423. eval(shift(@insns));
  1424. &vpxor (@X[0],@X[0],@Tx[0]); # "X[0]"^="X[-6]"
  1425. eval(shift(@insns));
  1426. eval(shift(@insns));
  1427. eval(shift(@insns));
  1428. &vpsrld (@Tx[0],@X[0],30);
  1429. &vpslld (@X[0],@X[0],2);
  1430. eval(shift(@insns));
  1431. eval(shift(@insns));
  1432. eval(shift(@insns));
  1433. #&vpslld (@X[0],@X[0],2);
  1434. eval(shift(@insns));
  1435. eval(shift(@insns));
  1436. eval(shift(@insns));
  1437. &vpor (@X[0],@X[0],@Tx[0]); # "X[0]"<<<=2
  1438. eval(shift(@insns));
  1439. eval(shift(@insns));
  1440. eval(shift(@insns));
  1441. eval(shift(@insns));
  1442. &vpaddd (@Tx[1],@X[0],$Kx);
  1443. eval(shift(@insns));
  1444. eval(shift(@insns));
  1445. eval(shift(@insns));
  1446. eval(shift(@insns));
  1447. &vmovdqu("32*$Xi(%rsp)",@Tx[1]); # X[]+K xfer to IALU
  1448. foreach (@insns) { eval; } # remaining instructions
  1449. $Xi++;
  1450. push(@X,shift(@X)); # "rotate" X[]
  1451. }
  1452. sub Xloop_avx2()
  1453. { use integer;
  1454. my $body = shift;
  1455. my @insns = (&$body,&$body,&$body,&$body,&$body); # 32 instructions
  1456. my ($a,$b,$c,$d,$e);
  1457. foreach (@insns) { eval; }
  1458. }
  1459. &align32();
  1460. &Xupdate_avx2_32_79(\&bodyx_00_19);
  1461. &Xupdate_avx2_32_79(\&bodyx_00_19);
  1462. &Xupdate_avx2_32_79(\&bodyx_00_19);
  1463. &Xupdate_avx2_32_79(\&bodyx_00_19);
  1464. &Xupdate_avx2_32_79(\&bodyx_20_39);
  1465. &Xupdate_avx2_32_79(\&bodyx_20_39);
  1466. &Xupdate_avx2_32_79(\&bodyx_20_39);
  1467. &Xupdate_avx2_32_79(\&bodyx_20_39);
  1468. &align32();
  1469. &Xupdate_avx2_32_79(\&bodyx_40_59);
  1470. &Xupdate_avx2_32_79(\&bodyx_40_59);
  1471. &Xupdate_avx2_32_79(\&bodyx_40_59);
  1472. &Xupdate_avx2_32_79(\&bodyx_40_59);
  1473. &Xloop_avx2(\&bodyx_20_39);
  1474. &Xloop_avx2(\&bodyx_20_39);
  1475. &Xloop_avx2(\&bodyx_20_39);
  1476. &Xloop_avx2(\&bodyx_20_39);
  1477. $code.=<<___;
  1478. lea 128($inp),$frame
  1479. lea 128($inp),%rdi # borrow $t0
  1480. cmp $num,$frame
  1481. cmovae $inp,$frame # next or previous block
  1482. # output is d-e-[a]-f-b-c => A=d,F=e,C=f,D=b,E=c
  1483. add 0($ctx),@ROTX[0] # update context
  1484. add 4($ctx),@ROTX[1]
  1485. add 8($ctx),@ROTX[3]
  1486. mov @ROTX[0],0($ctx)
  1487. add 12($ctx),@ROTX[4]
  1488. mov @ROTX[1],4($ctx)
  1489. mov @ROTX[0],$A # A=d
  1490. add 16($ctx),@ROTX[5]
  1491. mov @ROTX[3],$a5
  1492. mov @ROTX[3],8($ctx)
  1493. mov @ROTX[4],$D # D=b
  1494. #xchg @ROTX[5],$F # F=c, C=f
  1495. mov @ROTX[4],12($ctx)
  1496. mov @ROTX[1],$F # F=e
  1497. mov @ROTX[5],16($ctx)
  1498. #mov $F,16($ctx)
  1499. mov @ROTX[5],$E # E=c
  1500. mov $a5,$C # C=f
  1501. #xchg $F,$E # E=c, F=e
  1502. cmp $num,$inp
  1503. je .Ldone_avx2
  1504. ___
  1505. $Xi=4; # reset variables
  1506. @X=map("%ymm$_",(4..7,0..3));
  1507. $code.=<<___;
  1508. vmovdqu 64($K_XX_XX),@X[2] # pbswap mask
  1509. cmp $num,%rdi # borrowed $t0
  1510. ja .Last_avx2
  1511. vmovdqu -64(%rdi),%xmm0 # low part of @X[-4&7]
  1512. vmovdqu -48(%rdi),%xmm1
  1513. vmovdqu -32(%rdi),%xmm2
  1514. vmovdqu -16(%rdi),%xmm3
  1515. vinserti128 \$1,0($frame),@X[-4&7],@X[-4&7]
  1516. vinserti128 \$1,16($frame),@X[-3&7],@X[-3&7]
  1517. vinserti128 \$1,32($frame),@X[-2&7],@X[-2&7]
  1518. vinserti128 \$1,48($frame),@X[-1&7],@X[-1&7]
  1519. jmp .Last_avx2
  1520. .align 32
  1521. .Last_avx2:
  1522. lea 128+16(%rsp),$frame
  1523. rorx \$2,$F,$B
  1524. andn $D,$F,$t0
  1525. and $C,$F
  1526. xor $t0,$F
  1527. sub \$-128,$inp
  1528. ___
  1529. $rx=$j=0; @ROTX=($A,$F,$B,$C,$D,$E);
  1530. &Xloop_avx2 (\&bodyx_00_19);
  1531. &Xloop_avx2 (\&bodyx_00_19);
  1532. &Xloop_avx2 (\&bodyx_00_19);
  1533. &Xloop_avx2 (\&bodyx_00_19);
  1534. &Xloop_avx2 (\&bodyx_20_39);
  1535. &vmovdqu ($Kx,"-64($K_XX_XX)"); # K_00_19
  1536. &vpshufb (@X[-4&7],@X[-4&7],@X[2]); # byte swap
  1537. &Xloop_avx2 (\&bodyx_20_39);
  1538. &vpshufb (@X[-3&7],@X[-3&7],@X[2]);
  1539. &vpaddd (@Tx[0],@X[-4&7],$Kx); # add K_00_19
  1540. &Xloop_avx2 (\&bodyx_20_39);
  1541. &vmovdqu ("0(%rsp)",@Tx[0]);
  1542. &vpshufb (@X[-2&7],@X[-2&7],@X[2]);
  1543. &vpaddd (@Tx[1],@X[-3&7],$Kx);
  1544. &Xloop_avx2 (\&bodyx_20_39);
  1545. &vmovdqu ("32(%rsp)",@Tx[1]);
  1546. &vpshufb (@X[-1&7],@X[-1&7],@X[2]);
  1547. &vpaddd (@X[2],@X[-2&7],$Kx);
  1548. &Xloop_avx2 (\&bodyx_40_59);
  1549. &align32 ();
  1550. &vmovdqu ("64(%rsp)",@X[2]);
  1551. &vpaddd (@X[3],@X[-1&7],$Kx);
  1552. &Xloop_avx2 (\&bodyx_40_59);
  1553. &vmovdqu ("96(%rsp)",@X[3]);
  1554. &Xloop_avx2 (\&bodyx_40_59);
  1555. &Xupdate_avx2_16_31(\&bodyx_40_59);
  1556. &Xupdate_avx2_16_31(\&bodyx_20_39);
  1557. &Xupdate_avx2_16_31(\&bodyx_20_39);
  1558. &Xupdate_avx2_16_31(\&bodyx_20_39);
  1559. &Xloop_avx2 (\&bodyx_20_39);
  1560. $code.=<<___;
  1561. lea 128(%rsp),$frame
  1562. # output is d-e-[a]-f-b-c => A=d,F=e,C=f,D=b,E=c
  1563. add 0($ctx),@ROTX[0] # update context
  1564. add 4($ctx),@ROTX[1]
  1565. add 8($ctx),@ROTX[3]
  1566. mov @ROTX[0],0($ctx)
  1567. add 12($ctx),@ROTX[4]
  1568. mov @ROTX[1],4($ctx)
  1569. mov @ROTX[0],$A # A=d
  1570. add 16($ctx),@ROTX[5]
  1571. mov @ROTX[3],$a5
  1572. mov @ROTX[3],8($ctx)
  1573. mov @ROTX[4],$D # D=b
  1574. #xchg @ROTX[5],$F # F=c, C=f
  1575. mov @ROTX[4],12($ctx)
  1576. mov @ROTX[1],$F # F=e
  1577. mov @ROTX[5],16($ctx)
  1578. #mov $F,16($ctx)
  1579. mov @ROTX[5],$E # E=c
  1580. mov $a5,$C # C=f
  1581. #xchg $F,$E # E=c, F=e
  1582. cmp $num,$inp
  1583. jbe .Loop_avx2
  1584. .Ldone_avx2:
  1585. vzeroupper
  1586. ___
  1587. $code.=<<___ if ($win64);
  1588. movaps -40-6*16($fp),%xmm6
  1589. movaps -40-5*16($fp),%xmm7
  1590. movaps -40-4*16($fp),%xmm8
  1591. movaps -40-3*16($fp),%xmm9
  1592. movaps -40-2*16($fp),%xmm10
  1593. movaps -40-1*16($fp),%xmm11
  1594. ___
  1595. $code.=<<___;
  1596. mov -40($fp),%r14
  1597. .cfi_restore %r14
  1598. mov -32($fp),%r13
  1599. .cfi_restore %r13
  1600. mov -24($fp),%r12
  1601. .cfi_restore %r12
  1602. mov -16($fp),%rbp
  1603. .cfi_restore %rbp
  1604. mov -8($fp),%rbx
  1605. .cfi_restore %rbx
  1606. lea ($fp),%rsp
  1607. .cfi_def_cfa_register %rsp
  1608. .Lepilogue_avx2:
  1609. ret
  1610. .cfi_endproc
  1611. .size sha1_block_data_order_avx2,.-sha1_block_data_order_avx2
  1612. ___
  1613. }
  1614. }
  1615. $code.=<<___;
  1616. .align 64
  1617. K_XX_XX:
  1618. .long 0x5a827999,0x5a827999,0x5a827999,0x5a827999 # K_00_19
  1619. .long 0x5a827999,0x5a827999,0x5a827999,0x5a827999 # K_00_19
  1620. .long 0x6ed9eba1,0x6ed9eba1,0x6ed9eba1,0x6ed9eba1 # K_20_39
  1621. .long 0x6ed9eba1,0x6ed9eba1,0x6ed9eba1,0x6ed9eba1 # K_20_39
  1622. .long 0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc # K_40_59
  1623. .long 0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc # K_40_59
  1624. .long 0xca62c1d6,0xca62c1d6,0xca62c1d6,0xca62c1d6 # K_60_79
  1625. .long 0xca62c1d6,0xca62c1d6,0xca62c1d6,0xca62c1d6 # K_60_79
  1626. .long 0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f # pbswap mask
  1627. .long 0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f # pbswap mask
  1628. .byte 0xf,0xe,0xd,0xc,0xb,0xa,0x9,0x8,0x7,0x6,0x5,0x4,0x3,0x2,0x1,0x0
  1629. ___
  1630. }}}
  1631. $code.=<<___;
  1632. .asciz "SHA1 block transform for x86_64, CRYPTOGAMS by <appro\@openssl.org>"
  1633. .align 64
  1634. ___
  1635. # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame,
  1636. # CONTEXT *context,DISPATCHER_CONTEXT *disp)
  1637. if ($win64) {
  1638. $rec="%rcx";
  1639. $frame="%rdx";
  1640. $context="%r8";
  1641. $disp="%r9";
  1642. $code.=<<___;
  1643. .extern __imp_RtlVirtualUnwind
  1644. .type se_handler,\@abi-omnipotent
  1645. .align 16
  1646. se_handler:
  1647. push %rsi
  1648. push %rdi
  1649. push %rbx
  1650. push %rbp
  1651. push %r12
  1652. push %r13
  1653. push %r14
  1654. push %r15
  1655. pushfq
  1656. sub \$64,%rsp
  1657. mov 120($context),%rax # pull context->Rax
  1658. mov 248($context),%rbx # pull context->Rip
  1659. lea .Lprologue(%rip),%r10
  1660. cmp %r10,%rbx # context->Rip<.Lprologue
  1661. jb .Lcommon_seh_tail
  1662. mov 152($context),%rax # pull context->Rsp
  1663. lea .Lepilogue(%rip),%r10
  1664. cmp %r10,%rbx # context->Rip>=.Lepilogue
  1665. jae .Lcommon_seh_tail
  1666. mov `16*4`(%rax),%rax # pull saved stack pointer
  1667. mov -8(%rax),%rbx
  1668. mov -16(%rax),%rbp
  1669. mov -24(%rax),%r12
  1670. mov -32(%rax),%r13
  1671. mov -40(%rax),%r14
  1672. mov %rbx,144($context) # restore context->Rbx
  1673. mov %rbp,160($context) # restore context->Rbp
  1674. mov %r12,216($context) # restore context->R12
  1675. mov %r13,224($context) # restore context->R13
  1676. mov %r14,232($context) # restore context->R14
  1677. jmp .Lcommon_seh_tail
  1678. .size se_handler,.-se_handler
  1679. ___
  1680. $code.=<<___ if ($shaext);
  1681. .type shaext_handler,\@abi-omnipotent
  1682. .align 16
  1683. shaext_handler:
  1684. push %rsi
  1685. push %rdi
  1686. push %rbx
  1687. push %rbp
  1688. push %r12
  1689. push %r13
  1690. push %r14
  1691. push %r15
  1692. pushfq
  1693. sub \$64,%rsp
  1694. mov 120($context),%rax # pull context->Rax
  1695. mov 248($context),%rbx # pull context->Rip
  1696. lea .Lprologue_shaext(%rip),%r10
  1697. cmp %r10,%rbx # context->Rip<.Lprologue
  1698. jb .Lcommon_seh_tail
  1699. lea .Lepilogue_shaext(%rip),%r10
  1700. cmp %r10,%rbx # context->Rip>=.Lepilogue
  1701. jae .Lcommon_seh_tail
  1702. lea -8-4*16(%rax),%rsi
  1703. lea 512($context),%rdi # &context.Xmm6
  1704. mov \$8,%ecx
  1705. .long 0xa548f3fc # cld; rep movsq
  1706. jmp .Lcommon_seh_tail
  1707. .size shaext_handler,.-shaext_handler
  1708. ___
  1709. $code.=<<___;
  1710. .type ssse3_handler,\@abi-omnipotent
  1711. .align 16
  1712. ssse3_handler:
  1713. push %rsi
  1714. push %rdi
  1715. push %rbx
  1716. push %rbp
  1717. push %r12
  1718. push %r13
  1719. push %r14
  1720. push %r15
  1721. pushfq
  1722. sub \$64,%rsp
  1723. mov 120($context),%rax # pull context->Rax
  1724. mov 248($context),%rbx # pull context->Rip
  1725. mov 8($disp),%rsi # disp->ImageBase
  1726. mov 56($disp),%r11 # disp->HandlerData
  1727. mov 0(%r11),%r10d # HandlerData[0]
  1728. lea (%rsi,%r10),%r10 # prologue label
  1729. cmp %r10,%rbx # context->Rip<prologue label
  1730. jb .Lcommon_seh_tail
  1731. mov 208($context),%rax # pull context->R11
  1732. mov 4(%r11),%r10d # HandlerData[1]
  1733. lea (%rsi,%r10),%r10 # epilogue label
  1734. cmp %r10,%rbx # context->Rip>=epilogue label
  1735. jae .Lcommon_seh_tail
  1736. lea -40-6*16(%rax),%rsi
  1737. lea 512($context),%rdi # &context.Xmm6
  1738. mov \$12,%ecx
  1739. .long 0xa548f3fc # cld; rep movsq
  1740. mov -8(%rax),%rbx
  1741. mov -16(%rax),%rbp
  1742. mov -24(%rax),%r12
  1743. mov -32(%rax),%r13
  1744. mov -40(%rax),%r14
  1745. mov %rbx,144($context) # restore context->Rbx
  1746. mov %rbp,160($context) # restore context->Rbp
  1747. mov %r12,216($context) # restore context->R12
  1748. mov %r13,224($context) # restore context->R13
  1749. mov %r14,232($context) # restore context->R14
  1750. .Lcommon_seh_tail:
  1751. mov 8(%rax),%rdi
  1752. mov 16(%rax),%rsi
  1753. mov %rax,152($context) # restore context->Rsp
  1754. mov %rsi,168($context) # restore context->Rsi
  1755. mov %rdi,176($context) # restore context->Rdi
  1756. mov 40($disp),%rdi # disp->ContextRecord
  1757. mov $context,%rsi # context
  1758. mov \$154,%ecx # sizeof(CONTEXT)
  1759. .long 0xa548f3fc # cld; rep movsq
  1760. mov $disp,%rsi
  1761. xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER
  1762. mov 8(%rsi),%rdx # arg2, disp->ImageBase
  1763. mov 0(%rsi),%r8 # arg3, disp->ControlPc
  1764. mov 16(%rsi),%r9 # arg4, disp->FunctionEntry
  1765. mov 40(%rsi),%r10 # disp->ContextRecord
  1766. lea 56(%rsi),%r11 # &disp->HandlerData
  1767. lea 24(%rsi),%r12 # &disp->EstablisherFrame
  1768. mov %r10,32(%rsp) # arg5
  1769. mov %r11,40(%rsp) # arg6
  1770. mov %r12,48(%rsp) # arg7
  1771. mov %rcx,56(%rsp) # arg8, (NULL)
  1772. call *__imp_RtlVirtualUnwind(%rip)
  1773. mov \$1,%eax # ExceptionContinueSearch
  1774. add \$64,%rsp
  1775. popfq
  1776. pop %r15
  1777. pop %r14
  1778. pop %r13
  1779. pop %r12
  1780. pop %rbp
  1781. pop %rbx
  1782. pop %rdi
  1783. pop %rsi
  1784. ret
  1785. .size ssse3_handler,.-ssse3_handler
  1786. .section .pdata
  1787. .align 4
  1788. .rva .LSEH_begin_sha1_block_data_order
  1789. .rva .LSEH_end_sha1_block_data_order
  1790. .rva .LSEH_info_sha1_block_data_order
  1791. ___
  1792. $code.=<<___ if ($shaext);
  1793. .rva .LSEH_begin_sha1_block_data_order_shaext
  1794. .rva .LSEH_end_sha1_block_data_order_shaext
  1795. .rva .LSEH_info_sha1_block_data_order_shaext
  1796. ___
  1797. $code.=<<___;
  1798. .rva .LSEH_begin_sha1_block_data_order_ssse3
  1799. .rva .LSEH_end_sha1_block_data_order_ssse3
  1800. .rva .LSEH_info_sha1_block_data_order_ssse3
  1801. ___
  1802. $code.=<<___ if ($avx);
  1803. .rva .LSEH_begin_sha1_block_data_order_avx
  1804. .rva .LSEH_end_sha1_block_data_order_avx
  1805. .rva .LSEH_info_sha1_block_data_order_avx
  1806. ___
  1807. $code.=<<___ if ($avx>1);
  1808. .rva .LSEH_begin_sha1_block_data_order_avx2
  1809. .rva .LSEH_end_sha1_block_data_order_avx2
  1810. .rva .LSEH_info_sha1_block_data_order_avx2
  1811. ___
  1812. $code.=<<___;
  1813. .section .xdata
  1814. .align 8
  1815. .LSEH_info_sha1_block_data_order:
  1816. .byte 9,0,0,0
  1817. .rva se_handler
  1818. ___
  1819. $code.=<<___ if ($shaext);
  1820. .LSEH_info_sha1_block_data_order_shaext:
  1821. .byte 9,0,0,0
  1822. .rva shaext_handler
  1823. ___
  1824. $code.=<<___;
  1825. .LSEH_info_sha1_block_data_order_ssse3:
  1826. .byte 9,0,0,0
  1827. .rva ssse3_handler
  1828. .rva .Lprologue_ssse3,.Lepilogue_ssse3 # HandlerData[]
  1829. ___
  1830. $code.=<<___ if ($avx);
  1831. .LSEH_info_sha1_block_data_order_avx:
  1832. .byte 9,0,0,0
  1833. .rva ssse3_handler
  1834. .rva .Lprologue_avx,.Lepilogue_avx # HandlerData[]
  1835. ___
  1836. $code.=<<___ if ($avx>1);
  1837. .LSEH_info_sha1_block_data_order_avx2:
  1838. .byte 9,0,0,0
  1839. .rva ssse3_handler
  1840. .rva .Lprologue_avx2,.Lepilogue_avx2 # HandlerData[]
  1841. ___
  1842. }
  1843. ####################################################################
  1844. sub sha1rnds4 {
  1845. if (@_[0] =~ /\$([x0-9a-f]+),\s*%xmm([0-7]),\s*%xmm([0-7])/) {
  1846. my @opcode=(0x0f,0x3a,0xcc);
  1847. push @opcode,0xc0|($2&7)|(($3&7)<<3); # ModR/M
  1848. my $c=$1;
  1849. push @opcode,$c=~/^0/?oct($c):$c;
  1850. return ".byte\t".join(',',@opcode);
  1851. } else {
  1852. return "sha1rnds4\t".@_[0];
  1853. }
  1854. }
  1855. sub sha1op38 {
  1856. my $instr = shift;
  1857. my %opcodelet = (
  1858. "sha1nexte" => 0xc8,
  1859. "sha1msg1" => 0xc9,
  1860. "sha1msg2" => 0xca );
  1861. if (defined($opcodelet{$instr}) && @_[0] =~ /%xmm([0-9]+),\s*%xmm([0-9]+)/) {
  1862. my @opcode=(0x0f,0x38);
  1863. my $rex=0;
  1864. $rex|=0x04 if ($2>=8);
  1865. $rex|=0x01 if ($1>=8);
  1866. unshift @opcode,0x40|$rex if ($rex);
  1867. push @opcode,$opcodelet{$instr};
  1868. push @opcode,0xc0|($1&7)|(($2&7)<<3); # ModR/M
  1869. return ".byte\t".join(',',@opcode);
  1870. } else {
  1871. return $instr."\t".@_[0];
  1872. }
  1873. }
  1874. foreach (split("\n",$code)) {
  1875. s/\`([^\`]*)\`/eval $1/geo;
  1876. s/\b(sha1rnds4)\s+(.*)/sha1rnds4($2)/geo or
  1877. s/\b(sha1[^\s]*)\s+(.*)/sha1op38($1,$2)/geo;
  1878. print $_,"\n";
  1879. }
  1880. close STDOUT;