hash_sha1_x86-64.S.sh 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478
  1. #!/bin/sh
  2. # We don't regenerate it on every "make" invocation - only by hand.
  3. # The reason is that the changes to generated code are difficult
  4. # to visualize by looking only at this script, it helps when the commit
  5. # also contains the diff of the generated file.
  6. exec >hash_sha1_x86-64.S
  7. # Based on http://arctic.org/~dean/crypto/sha1.html.
  8. # ("This SHA1 implementation is public domain.")
  9. #
  10. # x86-64 has at least SSE2 vector insns always available.
  11. # We can use them without any CPUID checks (and without a need
  12. # for a fallback code if needed insns are not available).
  13. # This code uses them to calculate W[] ahead of time.
  14. #
  15. # Unfortunately, results are passed from vector unit to
  16. # integer ALUs on the stack. MOVD/Q insns to move them directly
  17. # from vector to integer registers are slower than store-to-load
  18. # forwarding in LSU (on Skylake at least).
  19. #
  20. # The win against a purely integer code is small on Skylake,
  21. # only about 7-8%. We offload about 1/3 of our operations to the vector unit.
  22. # It can do 4 ops at once in one 128-bit register,
  23. # but we have to use x2 of them because of W[0] complication,
  24. # SSE2 has no "rotate each word by N bits" insns,
  25. # moving data to/from vector unit is clunky, and Skylake
  26. # has four integer ALUs unified with three vector ALUs,
  27. # which makes pure integer code rather fast, and makes
  28. # vector ops compete with integer ones.
  29. #
  30. # Zen3, with its separate vector ALUs, wins more, about 12%.
  31. xmmT1="%xmm4"
  32. xmmT2="%xmm5"
  33. xmmRCONST="%xmm6"
  34. xmmALLRCONST="%xmm7"
  35. T=`printf '\t'`
  36. # SSE instructions are longer than 4 bytes on average.
  37. # Intel CPUs (up to Tiger Lake at least) can't decode
  38. # more than 16 bytes of code in one cycle.
  39. # By interleaving SSE code and integer code
  40. # we mostly achieve a situation where 16-byte decode fetch window
  41. # contains 4 (or more) insns.
  42. #
  43. # However. On Skylake, there was no observed difference,
  44. # but on Zen3, non-interleaved code is ~3% faster
  45. # (822 Mb/s versus 795 Mb/s hashing speed).
  46. # Off for now:
  47. interleave=false
  48. INTERLEAVE() {
  49. $interleave || \
  50. {
  51. # Generate non-interleaved code
  52. # (it should work correctly too)
  53. echo "$1"
  54. echo "$2"
  55. return
  56. }
  57. (
  58. echo "$1" | grep -v '^$' >"$0.temp1"
  59. echo "$2" | grep -v '^$' >"$0.temp2"
  60. exec 3<"$0.temp1"
  61. exec 4<"$0.temp2"
  62. IFS=''
  63. while :; do
  64. line1=''
  65. line2=''
  66. while :; do
  67. read -r line1 <&3
  68. if test "${line1:0:1}" != "#" && test "${line1:0:2}" != "$T#"; then
  69. break
  70. fi
  71. echo "$line1"
  72. done
  73. while :; do
  74. read -r line2 <&4
  75. if test "${line2:0:4}" = "${T}lea"; then
  76. # We use 7-8 byte long forms of LEA.
  77. # Do not interleave them with SSE insns
  78. # which are also long.
  79. echo "$line2"
  80. read -r line2 <&4
  81. echo "$line2"
  82. continue
  83. fi
  84. if test "${line2:0:1}" != "#" && test "${line2:0:2}" != "$T#"; then
  85. break
  86. fi
  87. echo "$line2"
  88. done
  89. test "$line1$line2" || break
  90. echo "$line1"
  91. echo "$line2"
  92. done
  93. rm "$0.temp1" "$0.temp2"
  94. )
  95. }
  96. # movaps bswap32_mask(%rip), $xmmT1
  97. # Load W[] to xmm0..3, byteswapping on the fly.
  98. # For iterations 0..15, we pass RCONST+W[] in rsi,r8..r14
  99. # for use in RD1As instead of spilling them to stack.
  100. # (We use rsi instead of rN because this makes two
  101. # ADDs in two first RD1As shorter by one byte).
  102. # movups 16*0(%rdi), %xmm0
  103. # pshufb $xmmT1, %xmm0 #SSSE3 insn
  104. # movaps %xmm0, $xmmT2
  105. # paddd $xmmRCONST, $xmmT2
  106. # movq $xmmT2, %rsi
  107. # #pextrq \$1, $xmmT2, %r8 #SSE4.1 insn
  108. # #movhpd $xmmT2, %r8 #can only move to mem, not to reg
  109. # shufps \$0x0e, $xmmT2, $xmmT2 # have to use two-insn sequence
  110. # movq $xmmT2, %r8 # instead
  111. # ...
  112. # <repeat for xmm1,2,3>
  113. # ...
  114. #- leal $RCONST(%r$e,%rsi), %e$e # e += RCONST + W[n]
  115. #+ addl %esi, %e$e # e += RCONST + W[n]
  116. # ^^^^^^^^^^^^^^^^^^^^^^^^
  117. # The above is -97 bytes of code...
  118. # ...but pshufb is a SSSE3 insn. Can't use it.
  119. echo \
  120. "### Generated by hash_sha1_x86-64.S.sh ###
  121. #if CONFIG_SHA1_SMALL == 0 && defined(__GNUC__) && defined(__x86_64__)
  122. #ifdef __linux__
  123. .section .note.GNU-stack, \"\", @progbits
  124. #endif
  125. .section .text.sha1_process_block64, \"ax\", @progbits
  126. .globl sha1_process_block64
  127. .hidden sha1_process_block64
  128. .type sha1_process_block64, @function
  129. .balign 8 # allow decoders to fetch at least 5 first insns
  130. sha1_process_block64:
  131. pushq %rbp # 1 byte insn
  132. pushq %rbx # 1 byte insn
  133. # pushq %r15 # 2 byte insn
  134. pushq %r14 # 2 byte insn
  135. pushq %r13 # 2 byte insn
  136. pushq %r12 # 2 byte insn
  137. pushq %rdi # we need ctx at the end
  138. #Register and stack use:
  139. # eax..edx: a..d
  140. # ebp: e
  141. # esi,edi,r8..r14: temps
  142. # r15: unused
  143. # xmm0..xmm3: W[]
  144. # xmm4,xmm5: temps
  145. # xmm6: current round constant
  146. # xmm7: all round constants
  147. # -64(%rsp): area for passing RCONST + W[] from vector to integer units
  148. movl 80(%rdi), %eax # a = ctx->hash[0]
  149. movl 84(%rdi), %ebx # b = ctx->hash[1]
  150. movl 88(%rdi), %ecx # c = ctx->hash[2]
  151. movl 92(%rdi), %edx # d = ctx->hash[3]
  152. movl 96(%rdi), %ebp # e = ctx->hash[4]
  153. movaps sha1const(%rip), $xmmALLRCONST
  154. pshufd \$0x00, $xmmALLRCONST, $xmmRCONST
  155. # Load W[] to xmm0..3, byteswapping on the fly.
  156. #
  157. # For iterations 0..15, we pass W[] in rsi,r8..r14
  158. # for use in RD1As instead of spilling them to stack.
  159. # We lose parallelized addition of RCONST, but LEA
  160. # can do two additions at once, so it is probably a wash.
  161. # (We use rsi instead of rN because this makes two
  162. # LEAs in two first RD1As shorter by one byte).
  163. movq 4*0(%rdi), %rsi
  164. movq 4*2(%rdi), %r8
  165. bswapq %rsi
  166. bswapq %r8
  167. rolq \$32, %rsi # rsi = W[1]:W[0]
  168. rolq \$32, %r8 # r8 = W[3]:W[2]
  169. movq %rsi, %xmm0
  170. movq %r8, $xmmT1
  171. punpcklqdq $xmmT1, %xmm0 # xmm0 = r8:rsi = (W[0],W[1],W[2],W[3])
  172. # movaps %xmm0, $xmmT1 # add RCONST, spill to stack
  173. # paddd $xmmRCONST, $xmmT1
  174. # movups $xmmT1, -64+16*0(%rsp)
  175. movq 4*4(%rdi), %r9
  176. movq 4*6(%rdi), %r10
  177. bswapq %r9
  178. bswapq %r10
  179. rolq \$32, %r9 # r9 = W[5]:W[4]
  180. rolq \$32, %r10 # r10 = W[7]:W[6]
  181. movq %r9, %xmm1
  182. movq %r10, $xmmT1
  183. punpcklqdq $xmmT1, %xmm1 # xmm1 = r10:r9 = (W[4],W[5],W[6],W[7])
  184. movq 4*8(%rdi), %r11
  185. movq 4*10(%rdi), %r12
  186. bswapq %r11
  187. bswapq %r12
  188. rolq \$32, %r11 # r11 = W[9]:W[8]
  189. rolq \$32, %r12 # r12 = W[11]:W[10]
  190. movq %r11, %xmm2
  191. movq %r12, $xmmT1
  192. punpcklqdq $xmmT1, %xmm2 # xmm2 = r12:r11 = (W[8],W[9],W[10],W[11])
  193. movq 4*12(%rdi), %r13
  194. movq 4*14(%rdi), %r14
  195. bswapq %r13
  196. bswapq %r14
  197. rolq \$32, %r13 # r13 = W[13]:W[12]
  198. rolq \$32, %r14 # r14 = W[15]:W[14]
  199. movq %r13, %xmm3
  200. movq %r14, $xmmT1
  201. punpcklqdq $xmmT1, %xmm3 # xmm3 = r14:r13 = (W[12],W[13],W[14],W[15])
  202. "
  203. PREP() {
  204. local xmmW0=$1
  205. local xmmW4=$2
  206. local xmmW8=$3
  207. local xmmW12=$4
  208. # the above must be %xmm0..3 in some permutation
  209. local dstmem=$5
  210. #W[0] = rol(W[13] ^ W[8] ^ W[2] ^ W[0], 1);
  211. #W[1] = rol(W[14] ^ W[9] ^ W[3] ^ W[1], 1);
  212. #W[2] = rol(W[15] ^ W[10] ^ W[4] ^ W[2], 1);
  213. #W[3] = rol( 0 ^ W[11] ^ W[5] ^ W[3], 1);
  214. #W[3] ^= rol(W[0], 1);
  215. echo "# PREP $@
  216. movaps $xmmW12, $xmmT1
  217. psrldq \$4, $xmmT1 # rshift by 4 bytes: T1 = ([13],[14],[15],0)
  218. # pshufd \$0x4e, $xmmW0, $xmmT2 # 01001110=2,3,0,1 shuffle, ([2],[3],x,x)
  219. # punpcklqdq $xmmW4, $xmmT2 # T2 = W4[0..63]:T2[0..63] = ([2],[3],[4],[5])
  220. # same result as above, but shorter and faster:
  221. # pshufd/shufps are subtly different: pshufd takes all dwords from source operand,
  222. # shufps takes dwords 0,1 from *2nd* operand, and dwords 2,3 from 1st one!
  223. movaps $xmmW0, $xmmT2
  224. shufps \$0x4e, $xmmW4, $xmmT2 # 01001110=(T2.dw[2], T2.dw[3], W4.dw[0], W4.dw[1]) = ([2],[3],[4],[5])
  225. xorps $xmmW8, $xmmW0 # ([8],[9],[10],[11]) ^ ([0],[1],[2],[3])
  226. xorps $xmmT1, $xmmT2 # ([13],[14],[15],0) ^ ([2],[3],[4],[5])
  227. xorps $xmmT2, $xmmW0 # ^
  228. # W0 = unrotated (W[0]..W[3]), still needs W[3] fixup
  229. movaps $xmmW0, $xmmT2
  230. xorps $xmmT1, $xmmT1 # rol(W0,1):
  231. pcmpgtd $xmmW0, $xmmT1 # ffffffff for elements <0 (ones with msb bit 1)
  232. paddd $xmmW0, $xmmW0 # shift left by 1
  233. psubd $xmmT1, $xmmW0 # add 1 to those who had msb bit 1
  234. # W0 = rotated (W[0]..W[3]), still needs W[3] fixup
  235. pslldq \$12, $xmmT2 # lshift by 12 bytes: T2 = (0,0,0,unrotW[0])
  236. movaps $xmmT2, $xmmT1
  237. pslld \$2, $xmmT2
  238. psrld \$30, $xmmT1
  239. # xorps $xmmT1, $xmmT2 # rol((0,0,0,unrotW[0]),2)
  240. xorps $xmmT1, $xmmW0 # same result, but does not depend on/does not modify T2
  241. xorps $xmmT2, $xmmW0 # W0 = rol(W[0]..W[3],1) ^ (0,0,0,rol(unrotW[0],2))
  242. "
  243. # movq $xmmW0, %r8 # high latency (~6 cycles)
  244. # movaps $xmmW0, $xmmT1
  245. # psrldq \$8, $xmmT1 # rshift by 8 bytes: move upper 64 bits to lower
  246. # movq $xmmT1, %r10 # high latency
  247. # movq %r8, %r9
  248. # movq %r10, %r11
  249. # shrq \$32, %r9
  250. # shrq \$32, %r11
  251. # ^^^ slower than passing the results on stack (!!!)
  252. echo "
  253. movaps $xmmW0, $xmmT2
  254. paddd $xmmRCONST, $xmmT2
  255. movups $xmmT2, $dstmem
  256. "
  257. }
  258. # It's possible to interleave integer insns in rounds to mostly eliminate
  259. # dependency chains, but this likely to only help old Pentium-based
  260. # CPUs (ones without OOO, which can only simultaneously execute a pair
  261. # of _adjacent_ insns).
  262. # Testing on old-ish Silvermont CPU (which has OOO window of only
  263. # about ~8 insns) shows very small (~1%) speedup.
  264. RD1A() {
  265. local a=$1;local b=$2;local c=$3;local d=$4;local e=$5
  266. local n=$(($6))
  267. local n0=$(((n+0) & 15))
  268. local rN=$((7+n0/2))
  269. echo "
  270. # $n
  271. ";test $n0 = 0 && echo "
  272. leal $RCONST(%r$e,%rsi), %e$e # e += RCONST + W[n]
  273. shrq \$32, %rsi
  274. ";test $n0 = 1 && echo "
  275. leal $RCONST(%r$e,%rsi), %e$e # e += RCONST + W[n]
  276. ";test $n0 -ge 2 && test $((n0 & 1)) = 0 && echo "
  277. leal $RCONST(%r$e,%r$rN), %e$e # e += RCONST + W[n]
  278. shrq \$32, %r$rN
  279. ";test $n0 -ge 2 && test $((n0 & 1)) = 1 && echo "
  280. leal $RCONST(%r$e,%r$rN), %e$e # e += RCONST + W[n]
  281. ";echo "
  282. movl %e$c, %edi # c
  283. xorl %e$d, %edi # ^d
  284. andl %e$b, %edi # &b
  285. xorl %e$d, %edi # (((c ^ d) & b) ^ d)
  286. addl %edi, %e$e # e += (((c ^ d) & b) ^ d)
  287. movl %e$a, %edi #
  288. roll \$5, %edi # rotl32(a,5)
  289. addl %edi, %e$e # e += rotl32(a,5)
  290. rorl \$2, %e$b # b = rotl32(b,30)
  291. "
  292. }
  293. RD1B() {
  294. local a=$1;local b=$2;local c=$3;local d=$4;local e=$5
  295. local n=$(($6))
  296. local n13=$(((n+13) & 15))
  297. local n8=$(((n+8) & 15))
  298. local n2=$(((n+2) & 15))
  299. local n0=$(((n+0) & 15))
  300. echo "
  301. # $n
  302. movl %e$c, %edi # c
  303. xorl %e$d, %edi # ^d
  304. andl %e$b, %edi # &b
  305. xorl %e$d, %edi # (((c ^ d) & b) ^ d)
  306. addl -64+4*$n0(%rsp), %e$e # e += RCONST + W[n & 15]
  307. addl %edi, %e$e # e += (((c ^ d) & b) ^ d)
  308. movl %e$a, %esi #
  309. roll \$5, %esi # rotl32(a,5)
  310. addl %esi, %e$e # e += rotl32(a,5)
  311. rorl \$2, %e$b # b = rotl32(b,30)
  312. "
  313. }
  314. RD2() {
  315. local a=$1;local b=$2;local c=$3;local d=$4;local e=$5
  316. local n=$(($6))
  317. local n13=$(((n+13) & 15))
  318. local n8=$(((n+8) & 15))
  319. local n2=$(((n+2) & 15))
  320. local n0=$(((n+0) & 15))
  321. echo "
  322. # $n
  323. movl %e$c, %edi # c
  324. xorl %e$d, %edi # ^d
  325. xorl %e$b, %edi # ^b
  326. addl -64+4*$n0(%rsp), %e$e # e += RCONST + W[n & 15]
  327. addl %edi, %e$e # e += (c ^ d ^ b)
  328. movl %e$a, %esi #
  329. roll \$5, %esi # rotl32(a,5)
  330. addl %esi, %e$e # e += rotl32(a,5)
  331. rorl \$2, %e$b # b = rotl32(b,30)
  332. "
  333. }
  334. RD3() {
  335. local a=$1;local b=$2;local c=$3;local d=$4;local e=$5
  336. local n=$(($6))
  337. local n13=$(((n+13) & 15))
  338. local n8=$(((n+8) & 15))
  339. local n2=$(((n+2) & 15))
  340. local n0=$(((n+0) & 15))
  341. echo "
  342. # $n
  343. movl %e$b, %edi # di: b
  344. movl %e$b, %esi # si: b
  345. orl %e$c, %edi # di: b | c
  346. andl %e$c, %esi # si: b & c
  347. andl %e$d, %edi # di: (b | c) & d
  348. orl %esi, %edi # ((b | c) & d) | (b & c)
  349. addl %edi, %e$e # += ((b | c) & d) | (b & c)
  350. addl -64+4*$n0(%rsp), %e$e # e += RCONST + W[n & 15]
  351. movl %e$a, %esi #
  352. roll \$5, %esi # rotl32(a,5)
  353. addl %esi, %e$e # e += rotl32(a,5)
  354. rorl \$2, %e$b # b = rotl32(b,30)
  355. "
  356. }
  357. {
  358. # Round 1
  359. RCONST=0x5A827999
  360. RD1A ax bx cx dx bp 0; RD1A bp ax bx cx dx 1; RD1A dx bp ax bx cx 2; RD1A cx dx bp ax bx 3;
  361. RD1A bx cx dx bp ax 4; RD1A ax bx cx dx bp 5; RD1A bp ax bx cx dx 6; RD1A dx bp ax bx cx 7;
  362. a=`PREP %xmm0 %xmm1 %xmm2 %xmm3 "-64+16*0(%rsp)"`
  363. b=`RD1A cx dx bp ax bx 8; RD1A bx cx dx bp ax 9; RD1A ax bx cx dx bp 10; RD1A bp ax bx cx dx 11;`
  364. INTERLEAVE "$a" "$b"
  365. a=`echo " pshufd \\$0x55, $xmmALLRCONST, $xmmRCONST"
  366. PREP %xmm1 %xmm2 %xmm3 %xmm0 "-64+16*1(%rsp)"`
  367. b=`RD1A dx bp ax bx cx 12; RD1A cx dx bp ax bx 13; RD1A bx cx dx bp ax 14; RD1A ax bx cx dx bp 15;`
  368. INTERLEAVE "$a" "$b"
  369. a=`PREP %xmm2 %xmm3 %xmm0 %xmm1 "-64+16*2(%rsp)"`
  370. b=`RD1B bp ax bx cx dx 16; RD1B dx bp ax bx cx 17; RD1B cx dx bp ax bx 18; RD1B bx cx dx bp ax 19;`
  371. INTERLEAVE "$a" "$b"
  372. # Round 2
  373. RCONST=0x6ED9EBA1
  374. a=`PREP %xmm3 %xmm0 %xmm1 %xmm2 "-64+16*3(%rsp)"`
  375. b=`RD2 ax bx cx dx bp 20; RD2 bp ax bx cx dx 21; RD2 dx bp ax bx cx 22; RD2 cx dx bp ax bx 23;`
  376. INTERLEAVE "$a" "$b"
  377. a=`PREP %xmm0 %xmm1 %xmm2 %xmm3 "-64+16*0(%rsp)"`
  378. b=`RD2 bx cx dx bp ax 24; RD2 ax bx cx dx bp 25; RD2 bp ax bx cx dx 26; RD2 dx bp ax bx cx 27;`
  379. INTERLEAVE "$a" "$b"
  380. a=`PREP %xmm1 %xmm2 %xmm3 %xmm0 "-64+16*1(%rsp)"`
  381. b=`RD2 cx dx bp ax bx 28; RD2 bx cx dx bp ax 29; RD2 ax bx cx dx bp 30; RD2 bp ax bx cx dx 31;`
  382. INTERLEAVE "$a" "$b"
  383. a=`echo " pshufd \\$0xaa, $xmmALLRCONST, $xmmRCONST"
  384. PREP %xmm2 %xmm3 %xmm0 %xmm1 "-64+16*2(%rsp)"`
  385. b=`RD2 dx bp ax bx cx 32; RD2 cx dx bp ax bx 33; RD2 bx cx dx bp ax 34; RD2 ax bx cx dx bp 35;`
  386. INTERLEAVE "$a" "$b"
  387. a=`PREP %xmm3 %xmm0 %xmm1 %xmm2 "-64+16*3(%rsp)"`
  388. b=`RD2 bp ax bx cx dx 36; RD2 dx bp ax bx cx 37; RD2 cx dx bp ax bx 38; RD2 bx cx dx bp ax 39;`
  389. INTERLEAVE "$a" "$b"
  390. # Round 3
  391. RCONST=0x8F1BBCDC
  392. a=`PREP %xmm0 %xmm1 %xmm2 %xmm3 "-64+16*0(%rsp)"`
  393. b=`RD3 ax bx cx dx bp 40; RD3 bp ax bx cx dx 41; RD3 dx bp ax bx cx 42; RD3 cx dx bp ax bx 43;`
  394. INTERLEAVE "$a" "$b"
  395. a=`PREP %xmm1 %xmm2 %xmm3 %xmm0 "-64+16*1(%rsp)"`
  396. b=`RD3 bx cx dx bp ax 44; RD3 ax bx cx dx bp 45; RD3 bp ax bx cx dx 46; RD3 dx bp ax bx cx 47;`
  397. INTERLEAVE "$a" "$b"
  398. a=`PREP %xmm2 %xmm3 %xmm0 %xmm1 "-64+16*2(%rsp)"`
  399. b=`RD3 cx dx bp ax bx 48; RD3 bx cx dx bp ax 49; RD3 ax bx cx dx bp 50; RD3 bp ax bx cx dx 51;`
  400. INTERLEAVE "$a" "$b"
  401. a=`echo " pshufd \\$0xff, $xmmALLRCONST, $xmmRCONST"
  402. PREP %xmm3 %xmm0 %xmm1 %xmm2 "-64+16*3(%rsp)"`
  403. b=`RD3 dx bp ax bx cx 52; RD3 cx dx bp ax bx 53; RD3 bx cx dx bp ax 54; RD3 ax bx cx dx bp 55;`
  404. INTERLEAVE "$a" "$b"
  405. a=`PREP %xmm0 %xmm1 %xmm2 %xmm3 "-64+16*0(%rsp)"`
  406. b=`RD3 bp ax bx cx dx 56; RD3 dx bp ax bx cx 57; RD3 cx dx bp ax bx 58; RD3 bx cx dx bp ax 59;`
  407. INTERLEAVE "$a" "$b"
  408. # Round 4 has the same logic as round 2, only n and RCONST are different
  409. RCONST=0xCA62C1D6
  410. a=`PREP %xmm1 %xmm2 %xmm3 %xmm0 "-64+16*1(%rsp)"`
  411. b=`RD2 ax bx cx dx bp 60; RD2 bp ax bx cx dx 61; RD2 dx bp ax bx cx 62; RD2 cx dx bp ax bx 63;`
  412. INTERLEAVE "$a" "$b"
  413. a=`PREP %xmm2 %xmm3 %xmm0 %xmm1 "-64+16*2(%rsp)"`
  414. b=`RD2 bx cx dx bp ax 64; RD2 ax bx cx dx bp 65; RD2 bp ax bx cx dx 66; RD2 dx bp ax bx cx 67;`
  415. INTERLEAVE "$a" "$b"
  416. a=`PREP %xmm3 %xmm0 %xmm1 %xmm2 "-64+16*3(%rsp)"`
  417. b=`RD2 cx dx bp ax bx 68; RD2 bx cx dx bp ax 69; RD2 ax bx cx dx bp 70; RD2 bp ax bx cx dx 71;`
  418. INTERLEAVE "$a" "$b"
  419. RD2 dx bp ax bx cx 72; RD2 cx dx bp ax bx 73; RD2 bx cx dx bp ax 74; RD2 ax bx cx dx bp 75;
  420. RD2 bp ax bx cx dx 76; RD2 dx bp ax bx cx 77; RD2 cx dx bp ax bx 78; RD2 bx cx dx bp ax 79;
  421. } | grep -v '^$'
  422. echo "
  423. popq %rdi #
  424. popq %r12 #
  425. addl %eax, 80(%rdi) # ctx->hash[0] += a
  426. popq %r13 #
  427. addl %ebx, 84(%rdi) # ctx->hash[1] += b
  428. popq %r14 #
  429. addl %ecx, 88(%rdi) # ctx->hash[2] += c
  430. # popq %r15 #
  431. addl %edx, 92(%rdi) # ctx->hash[3] += d
  432. popq %rbx #
  433. addl %ebp, 96(%rdi) # ctx->hash[4] += e
  434. popq %rbp #
  435. ret
  436. .size sha1_process_block64, .-sha1_process_block64
  437. .section .rodata.cst16.sha1const, \"aM\", @progbits, 16
  438. .balign 16
  439. sha1const:
  440. .long 0x5A827999
  441. .long 0x6ED9EBA1
  442. .long 0x8F1BBCDC
  443. .long 0xCA62C1D6
  444. #endif"