keccak1600-mmx.pl 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439
  1. #!/usr/bin/env perl
  2. # Copyright 2017-2020 The OpenSSL Project Authors. All Rights Reserved.
  3. #
  4. # Licensed under the Apache License 2.0 (the "License"). You may not use
  5. # this file except in compliance with the License. You can obtain a copy
  6. # in the file LICENSE in the source distribution or at
  7. # https://www.openssl.org/source/license.html
  8. #
  9. # ====================================================================
  10. # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
  11. # project. The module is, however, dual licensed under OpenSSL and
  12. # CRYPTOGAMS licenses depending on where you obtain it. For further
  13. # details see http://www.openssl.org/~appro/cryptogams/.
  14. # ====================================================================
  15. #
  16. # Keccak-1600 for x86 MMX.
  17. #
  18. # June 2017.
  19. #
  20. # Below code is KECCAK_2X implementation (see sha/keccak1600.c) with
  21. # C[5] held in register bank and D[5] offloaded to memory. Though
  22. # instead of actually unrolling the loop pair-wise I simply flip
  23. # pointers to T[][] and A[][] and the end of round. Since number of
  24. # rounds is even, last round writes to A[][] and everything works out.
  25. # It's argued that MMX is the only code path meaningful to implement
  26. # for x86. This is because non-MMX-capable processors is an extinct
  27. # breed, and they as well can lurk executing compiler-generated code.
  28. # For reference gcc-5.x-generated KECCAK_2X code takes 89 cycles per
  29. # processed byte on Pentium. Which is fair result. But older compilers
  30. # produce worse code. On the other hand one can wonder why not 128-bit
  31. # SSE2? Well, SSE2 won't provide double improvement, rather far from
  32. # that, if any at all on some processors, because it will take extra
  33. # permutations and inter-bank data transfers. Besides, contemporary
  34. # CPUs are better off executing 64-bit code, and it makes lesser sense
  35. # to invest into fancy 32-bit code. And the decision doesn't seem to
  36. # be inadequate, if one compares below results to "64-bit platforms in
  37. # 32-bit mode" SIMD data points available at
  38. # http://keccak.noekeon.org/sw_performance.html.
  39. #
  40. ########################################################################
  41. # Numbers are cycles per processed byte out of large message.
  42. #
  43. # r=1088(i)
  44. #
  45. # PIII 30/+150%
  46. # Pentium M 27/+150%
  47. # P4 40/+85%
  48. # Core 2 19/+170%
  49. # Sandy Bridge(ii) 18/+140%
  50. # Atom 33/+180%
  51. # Silvermont(ii) 30/+180%
  52. # VIA Nano(ii) 43/+60%
  53. # Sledgehammer(ii)(iii) 24/+130%
  54. #
  55. # (i) Corresponds to SHA3-256. Numbers after slash are improvement
  56. # coefficients over KECCAK_2X [with bit interleave and lane
  57. # complementing] position-independent *scalar* code generated
  58. # by gcc-5.x. It's not exactly fair comparison, but it's a
  59. # datapoint...
  60. # (ii) 64-bit processor executing 32-bit code.
  61. # (iii) Result is considered to be representative even for older AMD
  62. # processors.
  63. $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
  64. push(@INC,"${dir}","${dir}../../perlasm");
  65. require "x86asm.pl";
  66. $output=pop and open STDOUT,">$output";
  67. &asm_init($ARGV[0],$ARGV[$#ARGV] eq "386");
  68. my @C = map("mm$_",(0..4));
  69. my @T = map("mm$_",(5..7));
  70. my @A = map([ 8*$_-100, 8*($_+1)-100, 8*($_+2)-100,
  71. 8*($_+3)-100, 8*($_+4)-100 ], (0,5,10,15,20));
  72. my @D = map(8*$_+4, (0..4));
  73. my @rhotates = ([ 0, 1, 62, 28, 27 ],
  74. [ 36, 44, 6, 55, 20 ],
  75. [ 3, 10, 43, 25, 39 ],
  76. [ 41, 45, 15, 21, 8 ],
  77. [ 18, 2, 61, 56, 14 ]);
  78. &static_label("iotas");
  79. &function_begin_B("_KeccakF1600");
  80. &movq (@C[0],&QWP($A[4][0],"esi"));
  81. &movq (@C[1],&QWP($A[4][1],"esi"));
  82. &movq (@C[2],&QWP($A[4][2],"esi"));
  83. &movq (@C[3],&QWP($A[4][3],"esi"));
  84. &movq (@C[4],&QWP($A[4][4],"esi"));
  85. &mov ("ecx",24); # loop counter
  86. &jmp (&label("loop"));
  87. &set_label("loop",16);
  88. ######################################### Theta
  89. &pxor (@C[0],&QWP($A[0][0],"esi"));
  90. &pxor (@C[1],&QWP($A[0][1],"esi"));
  91. &pxor (@C[2],&QWP($A[0][2],"esi"));
  92. &pxor (@C[3],&QWP($A[0][3],"esi"));
  93. &pxor (@C[4],&QWP($A[0][4],"esi"));
  94. &pxor (@C[0],&QWP($A[1][0],"esi"));
  95. &pxor (@C[1],&QWP($A[1][1],"esi"));
  96. &pxor (@C[2],&QWP($A[1][2],"esi"));
  97. &pxor (@C[3],&QWP($A[1][3],"esi"));
  98. &pxor (@C[4],&QWP($A[1][4],"esi"));
  99. &pxor (@C[0],&QWP($A[2][0],"esi"));
  100. &pxor (@C[1],&QWP($A[2][1],"esi"));
  101. &pxor (@C[2],&QWP($A[2][2],"esi"));
  102. &pxor (@C[3],&QWP($A[2][3],"esi"));
  103. &pxor (@C[4],&QWP($A[2][4],"esi"));
  104. &pxor (@C[2],&QWP($A[3][2],"esi"));
  105. &pxor (@C[0],&QWP($A[3][0],"esi"));
  106. &pxor (@C[1],&QWP($A[3][1],"esi"));
  107. &pxor (@C[3],&QWP($A[3][3],"esi"));
  108. &movq (@T[0],@C[2]);
  109. &pxor (@C[4],&QWP($A[3][4],"esi"));
  110. &movq (@T[2],@C[2]);
  111. &psrlq (@T[0],63);
  112. &movq (@T[1],@C[0]);
  113. &psllq (@T[2],1);
  114. &pxor (@T[0],@C[0]);
  115. &psrlq (@C[0],63);
  116. &pxor (@T[0],@T[2]);
  117. &psllq (@T[1],1);
  118. &movq (@T[2],@C[1]);
  119. &movq (&QWP(@D[1],"esp"),@T[0]); # D[1] = E[0] = ROL64(C[2], 1) ^ C[0];
  120. &pxor (@T[1],@C[0]);
  121. &psrlq (@T[2],63);
  122. &pxor (@T[1],@C[3]);
  123. &movq (@C[0],@C[1]);
  124. &movq (&QWP(@D[4],"esp"),@T[1]); # D[4] = E[1] = ROL64(C[0], 1) ^ C[3];
  125. &psllq (@C[0],1);
  126. &pxor (@T[2],@C[4]);
  127. &pxor (@C[0],@T[2]);
  128. &movq (@T[2],@C[3]);
  129. &psrlq (@C[3],63);
  130. &movq (&QWP(@D[0],"esp"),@C[0]); # D[0] = C[0] = ROL64(C[1], 1) ^ C[4];
  131. &psllq (@T[2],1);
  132. &movq (@T[0],@C[4]);
  133. &psrlq (@C[4],63);
  134. &pxor (@C[1],@C[3]);
  135. &psllq (@T[0],1);
  136. &pxor (@C[1],@T[2]);
  137. &pxor (@C[2],@C[4]);
  138. &movq (&QWP(@D[2],"esp"),@C[1]); # D[2] = C[1] = ROL64(C[3], 1) ^ C[1];
  139. &pxor (@C[2],@T[0]);
  140. ######################################### first Rho(0) is special
  141. &movq (@C[3],&QWP($A[3][3],"esi"));
  142. &movq (&QWP(@D[3],"esp"),@C[2]); # D[3] = C[2] = ROL64(C[4], 1) ^ C[2];
  143. &pxor (@C[3],@C[2]);
  144. &movq (@C[4],&QWP($A[4][4],"esi"));
  145. &movq (@T[2],@C[3]);
  146. &psrlq (@C[3],64-$rhotates[3][3]);
  147. &pxor (@C[4],@T[1]);
  148. &psllq (@T[2],$rhotates[3][3]);
  149. &movq (@T[1],@C[4]);
  150. &psrlq (@C[4],64-$rhotates[4][4]);
  151. &por (@C[3],@T[2]); # C[3] = ROL64(A[3][3] ^ C[2], rhotates[3][3]); /* D[3] */
  152. &psllq (@T[1],$rhotates[4][4]);
  153. &movq (@C[2],&QWP($A[2][2],"esi"));
  154. &por (@C[4],@T[1]); # C[4] = ROL64(A[4][4] ^ E[1], rhotates[4][4]); /* D[4] */
  155. &pxor (@C[2],@C[1]);
  156. &movq (@C[1],&QWP($A[1][1],"esi"));
  157. &movq (@T[1],@C[2]);
  158. &psrlq (@C[2],64-$rhotates[2][2]);
  159. &pxor (@C[1],&QWP(@D[1],"esp"));
  160. &psllq (@T[1],$rhotates[2][2]);
  161. &movq (@T[2],@C[1]);
  162. &psrlq (@C[1],64-$rhotates[1][1]);
  163. &por (@C[2],@T[1]); # C[2] = ROL64(A[2][2] ^ C[1], rhotates[2][2]); /* D[2] */
  164. &psllq (@T[2],$rhotates[1][1]);
  165. &pxor (@C[0],&QWP($A[0][0],"esi")); # /* rotate by 0 */ /* D[0] */
  166. &por (@C[1],@T[2]); # C[1] = ROL64(A[1][1] ^ D[1], rhotates[1][1]);
  167. sub Chi() { ######### regular Chi step
  168. my ($y,$xrho) = @_;
  169. &movq (@T[0],@C[1]);
  170. &movq (@T[1],@C[2]);
  171. &pandn (@T[0],@C[2]);
  172. &pandn (@C[2],@C[3]);
  173. &pxor (@T[0],@C[0]);
  174. &pxor (@C[2],@C[1]);
  175. &pxor (@T[0],&QWP(0,"ebx")) if ($y == 0);
  176. &lea ("ebx",&DWP(8,"ebx")) if ($y == 0);
  177. &movq (@T[2],@C[3]);
  178. &movq (&QWP($A[$y][0],"edi"),@T[0]); # R[0][0] = C[0] ^ (~C[1] & C[2]) ^ iotas[i];
  179. &movq (@T[0],@C[4]);
  180. &pandn (@C[3],@C[4]);
  181. &pandn (@C[4],@C[0]);
  182. &pxor (@C[3],@T[1]);
  183. &movq (&QWP($A[$y][1],"edi"),@C[2]); # R[0][1] = C[1] ^ (~C[2] & C[3]);
  184. &pxor (@C[4],@T[2]);
  185. &movq (@T[2],&QWP($A[0][$xrho],"esi")) if (defined($xrho));
  186. &movq (&QWP($A[$y][2],"edi"),@C[3]); # R[0][2] = C[2] ^ (~C[3] & C[4]);
  187. &pandn (@C[0],@C[1]);
  188. &movq (&QWP($A[$y][3],"edi"),@C[4]); # R[0][3] = C[3] ^ (~C[4] & C[0]);
  189. &pxor (@C[0],@T[0]);
  190. &pxor (@T[2],&QWP(@D[$xrho],"esp")) if (defined($xrho));
  191. &movq (&QWP($A[$y][4],"edi"),@C[0]); # R[0][4] = C[4] ^ (~C[0] & C[1]);
  192. }
  193. &Chi (0, 3);
  194. sub Rho() { ######### regular Rho step
  195. my $x = shift;
  196. #&movq (@T[2],&QWP($A[0][$x],"esi")); # moved to Chi
  197. #&pxor (@T[2],&QWP(@D[$x],"esp")); # moved to Chi
  198. &movq (@C[0],@T[2]);
  199. &psrlq (@T[2],64-$rhotates[0][$x]);
  200. &movq (@C[1],&QWP($A[1][($x+1)%5],"esi"));
  201. &psllq (@C[0],$rhotates[0][$x]);
  202. &pxor (@C[1],&QWP(@D[($x+1)%5],"esp"));
  203. &por (@C[0],@T[2]); # C[0] = ROL64(A[0][3] ^ D[3], rhotates[0][3]);
  204. &movq (@T[1],@C[1]);
  205. &psrlq (@C[1],64-$rhotates[1][($x+1)%5]);
  206. &movq (@C[2],&QWP($A[2][($x+2)%5],"esi"));
  207. &psllq (@T[1],$rhotates[1][($x+1)%5]);
  208. &pxor (@C[2],&QWP(@D[($x+2)%5],"esp"));
  209. &por (@C[1],@T[1]); # C[1] = ROL64(A[1][4] ^ D[4], rhotates[1][4]);
  210. &movq (@T[2],@C[2]);
  211. &psrlq (@C[2],64-$rhotates[2][($x+2)%5]);
  212. &movq (@C[3],&QWP($A[3][($x+3)%5],"esi"));
  213. &psllq (@T[2],$rhotates[2][($x+2)%5]);
  214. &pxor (@C[3],&QWP(@D[($x+3)%5],"esp"));
  215. &por (@C[2],@T[2]); # C[2] = ROL64(A[2][0] ^ D[0], rhotates[2][0]);
  216. &movq (@T[0],@C[3]);
  217. &psrlq (@C[3],64-$rhotates[3][($x+3)%5]);
  218. &movq (@C[4],&QWP($A[4][($x+4)%5],"esi"));
  219. &psllq (@T[0],$rhotates[3][($x+3)%5]);
  220. &pxor (@C[4],&QWP(@D[($x+4)%5],"esp"));
  221. &por (@C[3],@T[0]); # C[3] = ROL64(A[3][1] ^ D[1], rhotates[3][1]);
  222. &movq (@T[1],@C[4]);
  223. &psrlq (@C[4],64-$rhotates[4][($x+4)%5]);
  224. &psllq (@T[1],$rhotates[4][($x+4)%5]);
  225. &por (@C[4],@T[1]); # C[4] = ROL64(A[4][2] ^ D[2], rhotates[4][2]);
  226. }
  227. &Rho (3); &Chi (1, 1);
  228. &Rho (1); &Chi (2, 4);
  229. &Rho (4); &Chi (3, 2);
  230. &Rho (2); ###&Chi (4);
  231. &movq (@T[0],@C[0]); ######### last Chi(4) is special
  232. &xor ("edi","esi"); # &xchg ("esi","edi");
  233. &movq (&QWP(@D[1],"esp"),@C[1]);
  234. &xor ("esi","edi");
  235. &xor ("edi","esi");
  236. &movq (@T[1],@C[1]);
  237. &movq (@T[2],@C[2]);
  238. &pandn (@T[1],@C[2]);
  239. &pandn (@T[2],@C[3]);
  240. &pxor (@C[0],@T[1]);
  241. &pxor (@C[1],@T[2]);
  242. &movq (@T[1],@C[3]);
  243. &movq (&QWP($A[4][0],"esi"),@C[0]); # R[4][0] = C[0] ^= (~C[1] & C[2]);
  244. &pandn (@T[1],@C[4]);
  245. &movq (&QWP($A[4][1],"esi"),@C[1]); # R[4][1] = C[1] ^= (~C[2] & C[3]);
  246. &pxor (@C[2],@T[1]);
  247. &movq (@T[2],@C[4]);
  248. &movq (&QWP($A[4][2],"esi"),@C[2]); # R[4][2] = C[2] ^= (~C[3] & C[4]);
  249. &pandn (@T[2],@T[0]);
  250. &pandn (@T[0],&QWP(@D[1],"esp"));
  251. &pxor (@C[3],@T[2]);
  252. &pxor (@C[4],@T[0]);
  253. &movq (&QWP($A[4][3],"esi"),@C[3]); # R[4][3] = C[3] ^= (~C[4] & D[0]);
  254. &sub ("ecx",1);
  255. &movq (&QWP($A[4][4],"esi"),@C[4]); # R[4][4] = C[4] ^= (~D[0] & D[1]);
  256. &jnz (&label("loop"));
  257. &lea ("ebx",&DWP(-192,"ebx")); # rewind iotas
  258. &ret ();
  259. &function_end_B("_KeccakF1600");
  260. &function_begin("KeccakF1600");
  261. &mov ("esi",&wparam(0));
  262. &mov ("ebp","esp");
  263. &sub ("esp",240);
  264. &call (&label("pic_point"));
  265. &set_label("pic_point");
  266. &blindpop("ebx");
  267. &lea ("ebx",&DWP(&label("iotas")."-".&label("pic_point"),"ebx"));
  268. &and ("esp",-8);
  269. &lea ("esi",&DWP(100,"esi")); # size optimization
  270. &lea ("edi",&DWP(8*5+100,"esp")); # size optimization
  271. &call ("_KeccakF1600");
  272. &mov ("esp","ebp");
  273. &emms ();
  274. &function_end("KeccakF1600");
  275. &function_begin("SHA3_absorb");
  276. &mov ("esi",&wparam(0)); # A[][]
  277. &mov ("eax",&wparam(1)); # inp
  278. &mov ("ecx",&wparam(2)); # len
  279. &mov ("edx",&wparam(3)); # bsz
  280. &mov ("ebp","esp");
  281. &sub ("esp",240+8);
  282. &call (&label("pic_point"));
  283. &set_label("pic_point");
  284. &blindpop("ebx");
  285. &lea ("ebx",&DWP(&label("iotas")."-".&label("pic_point"),"ebx"));
  286. &and ("esp",-8);
  287. &mov ("edi","esi");
  288. &lea ("esi",&DWP(100,"esi")); # size optimization
  289. &mov (&DWP(-4,"ebp"),"edx"); # save bsz
  290. &jmp (&label("loop"));
  291. &set_label("loop",16);
  292. &cmp ("ecx","edx"); # len < bsz?
  293. &jc (&label("absorbed"));
  294. &shr ("edx",3); # bsz /= 8
  295. &set_label("block");
  296. &movq ("mm0",&QWP(0,"eax"));
  297. &lea ("eax",&DWP(8,"eax"));
  298. &pxor ("mm0",&QWP(0,"edi"));
  299. &lea ("edi",&DWP(8,"edi"));
  300. &sub ("ecx",8); # len -= 8
  301. &movq (&QWP(-8,"edi"),"mm0");
  302. &dec ("edx"); # bsz--
  303. &jnz (&label("block"));
  304. &lea ("edi",&DWP(8*5+100,"esp")); # size optimization
  305. &mov (&DWP(-8,"ebp"),"ecx"); # save len
  306. &call ("_KeccakF1600");
  307. &mov ("ecx",&DWP(-8,"ebp")); # pull len
  308. &mov ("edx",&DWP(-4,"ebp")); # pull bsz
  309. &lea ("edi",&DWP(-100,"esi"));
  310. &jmp (&label("loop"));
  311. &set_label("absorbed",16);
  312. &mov ("eax","ecx"); # return value
  313. &mov ("esp","ebp");
  314. &emms ();
  315. &function_end("SHA3_absorb");
  316. &function_begin("SHA3_squeeze");
  317. &mov ("esi",&wparam(0)); # A[][]
  318. &mov ("eax",&wparam(1)); # out
  319. &mov ("ecx",&wparam(2)); # len
  320. &mov ("edx",&wparam(3)); # bsz
  321. &mov ("ebp","esp");
  322. &sub ("esp",240+8);
  323. &call (&label("pic_point"));
  324. &set_label("pic_point");
  325. &blindpop("ebx");
  326. &lea ("ebx",&DWP(&label("iotas")."-".&label("pic_point"),"ebx"));
  327. &and ("esp",-8);
  328. &shr ("edx",3); # bsz /= 8
  329. &mov ("edi","esi");
  330. &lea ("esi",&DWP(100,"esi")); # size optimization
  331. &mov (&DWP(-4,"ebp"),"edx"); # save bsz
  332. &jmp (&label("loop"));
  333. &set_label("loop",16);
  334. &cmp ("ecx",8); # len < 8?
  335. &jc (&label("tail"));
  336. &movq ("mm0",&QWP(0,"edi"));
  337. &lea ("edi",&DWP(8,"edi"));
  338. &movq (&QWP(0,"eax"),"mm0");
  339. &lea ("eax",&DWP(8,"eax"));
  340. &sub ("ecx",8); # len -= 8
  341. &jz (&label("done"));
  342. &dec ("edx"); # bsz--
  343. &jnz (&label("loop"));
  344. &lea ("edi",&DWP(8*5+100,"esp")); # size optimization
  345. &mov (&DWP(-8,"ebp"),"ecx"); # save len
  346. &call ("_KeccakF1600");
  347. &mov ("ecx",&DWP(-8,"ebp")); # pull len
  348. &mov ("edx",&DWP(-4,"ebp")); # pull bsz
  349. &lea ("edi",&DWP(-100,"esi"));
  350. &jmp (&label("loop"));
  351. &set_label("tail",16);
  352. &mov ("esi","edi");
  353. &mov ("edi","eax");
  354. &data_word("0xA4F39066"); # rep movsb
  355. &set_label("done");
  356. &mov ("esp","ebp");
  357. &emms ();
  358. &function_end("SHA3_squeeze");
  359. &set_label("iotas",32);
  360. &data_word(0x00000001,0x00000000);
  361. &data_word(0x00008082,0x00000000);
  362. &data_word(0x0000808a,0x80000000);
  363. &data_word(0x80008000,0x80000000);
  364. &data_word(0x0000808b,0x00000000);
  365. &data_word(0x80000001,0x00000000);
  366. &data_word(0x80008081,0x80000000);
  367. &data_word(0x00008009,0x80000000);
  368. &data_word(0x0000008a,0x00000000);
  369. &data_word(0x00000088,0x00000000);
  370. &data_word(0x80008009,0x00000000);
  371. &data_word(0x8000000a,0x00000000);
  372. &data_word(0x8000808b,0x00000000);
  373. &data_word(0x0000008b,0x80000000);
  374. &data_word(0x00008089,0x80000000);
  375. &data_word(0x00008003,0x80000000);
  376. &data_word(0x00008002,0x80000000);
  377. &data_word(0x00000080,0x80000000);
  378. &data_word(0x0000800a,0x00000000);
  379. &data_word(0x8000000a,0x80000000);
  380. &data_word(0x80008081,0x80000000);
  381. &data_word(0x00008080,0x80000000);
  382. &data_word(0x80000001,0x00000000);
  383. &data_word(0x80008008,0x80000000);
  384. &asciz("Keccak-1600 absorb and squeeze for MMX, CRYPTOGAMS by <appro\@openssl.org>");
  385. &asm_finish();
  386. close STDOUT or die "error closing STDOUT: $!";