ecp_nistz256-x86.pl 56 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861
  1. #! /usr/bin/env perl
  2. # Copyright 2015-2020 The OpenSSL Project Authors. All Rights Reserved.
  3. #
  4. # Licensed under the Apache License 2.0 (the "License"). You may not use
  5. # this file except in compliance with the License. You can obtain a copy
  6. # in the file LICENSE in the source distribution or at
  7. # https://www.openssl.org/source/license.html
  8. # ====================================================================
  9. # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
  10. # project. The module is, however, dual licensed under OpenSSL and
  11. # CRYPTOGAMS licenses depending on where you obtain it. For further
  12. # details see http://www.openssl.org/~appro/cryptogams/.
  13. # ====================================================================
  14. #
  15. # ECP_NISTZ256 module for x86/SSE2.
  16. #
  17. # October 2014.
  18. #
  19. # Original ECP_NISTZ256 submission targeting x86_64 is detailed in
  20. # http://eprint.iacr.org/2013/816. In the process of adaptation
  21. # original .c module was made 32-bit savvy in order to make this
  22. # implementation possible.
  23. #
  24. # with/without -DECP_NISTZ256_ASM
  25. # Pentium +66-163%
  26. # PIII +72-172%
  27. # P4 +65-132%
  28. # Core2 +90-215%
  29. # Sandy Bridge +105-265% (contemporary i[57]-* are all close to this)
  30. # Atom +65-155%
  31. # Opteron +54-110%
  32. # Bulldozer +99-240%
  33. # VIA Nano +93-290%
  34. #
  35. # Ranges denote minimum and maximum improvement coefficients depending
  36. # on benchmark. Lower coefficients are for ECDSA sign, server-side
  37. # operation. Keep in mind that +200% means 3x improvement.
  38. $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
  39. push(@INC,"${dir}","${dir}../../perlasm");
  40. require "x86asm.pl";
  41. $output=pop and open STDOUT,">$output";
  42. &asm_init($ARGV[0],$ARGV[$#ARGV] eq "386");
  43. $sse2=0;
  44. for (@ARGV) { $sse2=1 if (/-DOPENSSL_IA32_SSE2/); }
  45. &external_label("OPENSSL_ia32cap_P") if ($sse2);
  46. ########################################################################
  47. # Convert ecp_nistz256_table.c to layout expected by ecp_nistz_gather_w7
  48. #
  49. open TABLE,"<ecp_nistz256_table.c" or
  50. open TABLE,"<${dir}../ecp_nistz256_table.c" or
  51. die "failed to open ecp_nistz256_table.c:",$!;
  52. use integer;
  53. foreach(<TABLE>) {
  54. s/TOBN\(\s*(0x[0-9a-f]+),\s*(0x[0-9a-f]+)\s*\)/push @arr,hex($2),hex($1)/geo;
  55. }
  56. close TABLE;
  57. # See ecp_nistz256_table.c for explanation for why it's 64*16*37.
  58. # 64*16*37-1 is because $#arr returns last valid index or @arr, not
  59. # amount of elements.
  60. die "insane number of elements" if ($#arr != 64*16*37-1);
  61. &public_label("ecp_nistz256_precomputed");
  62. &align(4096);
  63. &set_label("ecp_nistz256_precomputed");
  64. ########################################################################
  65. # this conversion smashes P256_POINT_AFFINE by individual bytes with
  66. # 64 byte interval, similar to
  67. # 1111222233334444
  68. # 1234123412341234
  69. for(1..37) {
  70. @tbl = splice(@arr,0,64*16);
  71. for($i=0;$i<64;$i++) {
  72. undef @line;
  73. for($j=0;$j<64;$j++) {
  74. push @line,(@tbl[$j*16+$i/4]>>(($i%4)*8))&0xff;
  75. }
  76. &data_byte(join(',',map { sprintf "0x%02x",$_} @line));
  77. }
  78. }
  79. ########################################################################
  80. # Keep in mind that constants are stored least to most significant word
  81. &static_label("RR");
  82. &set_label("RR",64);
  83. &data_word(3,0,-1,-5,-2,-1,-3,4); # 2^512 mod P-256
  84. &static_label("ONE_mont");
  85. &set_label("ONE_mont");
  86. &data_word(1,0,0,-1,-1,-1,-2,0);
  87. &static_label("ONE");
  88. &set_label("ONE");
  89. &data_word(1,0,0,0,0,0,0,0);
  90. &asciz("ECP_NISZ256 for x86/SSE2, CRYPTOGAMS by <appro\@openssl.org>");
  91. &align(64);
  92. ########################################################################
  93. # void ecp_nistz256_mul_by_2(BN_ULONG edi[8],const BN_ULONG esi[8]);
  94. &function_begin("ecp_nistz256_mul_by_2");
  95. &mov ("esi",&wparam(1));
  96. &mov ("edi",&wparam(0));
  97. &mov ("ebp","esi");
  98. ########################################################################
  99. # common pattern for internal functions is that %edi is result pointer,
  100. # %esi and %ebp are input ones, %ebp being optional. %edi is preserved.
  101. &call ("_ecp_nistz256_add");
  102. &function_end("ecp_nistz256_mul_by_2");
  103. ########################################################################
  104. # void ecp_nistz256_mul_by_3(BN_ULONG edi[8],const BN_ULONG esi[8]);
  105. &function_begin("ecp_nistz256_mul_by_3");
  106. &mov ("esi",&wparam(1));
  107. # multiplication by 3 is performed
  108. # as 2*n+n, but we can't use output
  109. # to store 2*n, because if output
  110. # pointer equals to input, then
  111. # we'll get 2*n+2*n.
  112. &stack_push(8); # therefore we need to allocate
  113. # 256-bit intermediate buffer.
  114. &mov ("edi","esp");
  115. &mov ("ebp","esi");
  116. &call ("_ecp_nistz256_add");
  117. &lea ("esi",&DWP(0,"edi"));
  118. &mov ("ebp",&wparam(1));
  119. &mov ("edi",&wparam(0));
  120. &call ("_ecp_nistz256_add");
  121. &stack_pop(8);
  122. &function_end("ecp_nistz256_mul_by_3");
  123. ########################################################################
  124. # void ecp_nistz256_div_by_2(BN_ULONG edi[8],const BN_ULONG esi[8]);
  125. &function_begin("ecp_nistz256_div_by_2");
  126. &mov ("esi",&wparam(1));
  127. &mov ("edi",&wparam(0));
  128. &call ("_ecp_nistz256_div_by_2");
  129. &function_end("ecp_nistz256_div_by_2");
  130. &function_begin_B("_ecp_nistz256_div_by_2");
  131. # tmp = a is odd ? a+mod : a
  132. #
  133. # note that because mod has special form, i.e. consists of
  134. # 0xffffffff, 1 and 0s, we can conditionally synthesize it by
  135. # assigning least significant bit of input to one register,
  136. # %ebp, and its negative to another, %edx.
  137. &mov ("ebp",&DWP(0,"esi"));
  138. &xor ("edx","edx");
  139. &mov ("ebx",&DWP(4,"esi"));
  140. &mov ("eax","ebp");
  141. &and ("ebp",1);
  142. &mov ("ecx",&DWP(8,"esi"));
  143. &sub ("edx","ebp");
  144. &add ("eax","edx");
  145. &adc ("ebx","edx");
  146. &mov (&DWP(0,"edi"),"eax");
  147. &adc ("ecx","edx");
  148. &mov (&DWP(4,"edi"),"ebx");
  149. &mov (&DWP(8,"edi"),"ecx");
  150. &mov ("eax",&DWP(12,"esi"));
  151. &mov ("ebx",&DWP(16,"esi"));
  152. &adc ("eax",0);
  153. &mov ("ecx",&DWP(20,"esi"));
  154. &adc ("ebx",0);
  155. &mov (&DWP(12,"edi"),"eax");
  156. &adc ("ecx",0);
  157. &mov (&DWP(16,"edi"),"ebx");
  158. &mov (&DWP(20,"edi"),"ecx");
  159. &mov ("eax",&DWP(24,"esi"));
  160. &mov ("ebx",&DWP(28,"esi"));
  161. &adc ("eax","ebp");
  162. &adc ("ebx","edx");
  163. &mov (&DWP(24,"edi"),"eax");
  164. &sbb ("esi","esi"); # broadcast carry bit
  165. &mov (&DWP(28,"edi"),"ebx");
  166. # ret = tmp >> 1
  167. &mov ("eax",&DWP(0,"edi"));
  168. &mov ("ebx",&DWP(4,"edi"));
  169. &mov ("ecx",&DWP(8,"edi"));
  170. &mov ("edx",&DWP(12,"edi"));
  171. &shr ("eax",1);
  172. &mov ("ebp","ebx");
  173. &shl ("ebx",31);
  174. &or ("eax","ebx");
  175. &shr ("ebp",1);
  176. &mov ("ebx","ecx");
  177. &shl ("ecx",31);
  178. &mov (&DWP(0,"edi"),"eax");
  179. &or ("ebp","ecx");
  180. &mov ("eax",&DWP(16,"edi"));
  181. &shr ("ebx",1);
  182. &mov ("ecx","edx");
  183. &shl ("edx",31);
  184. &mov (&DWP(4,"edi"),"ebp");
  185. &or ("ebx","edx");
  186. &mov ("ebp",&DWP(20,"edi"));
  187. &shr ("ecx",1);
  188. &mov ("edx","eax");
  189. &shl ("eax",31);
  190. &mov (&DWP(8,"edi"),"ebx");
  191. &or ("ecx","eax");
  192. &mov ("ebx",&DWP(24,"edi"));
  193. &shr ("edx",1);
  194. &mov ("eax","ebp");
  195. &shl ("ebp",31);
  196. &mov (&DWP(12,"edi"),"ecx");
  197. &or ("edx","ebp");
  198. &mov ("ecx",&DWP(28,"edi"));
  199. &shr ("eax",1);
  200. &mov ("ebp","ebx");
  201. &shl ("ebx",31);
  202. &mov (&DWP(16,"edi"),"edx");
  203. &or ("eax","ebx");
  204. &shr ("ebp",1);
  205. &mov ("ebx","ecx");
  206. &shl ("ecx",31);
  207. &mov (&DWP(20,"edi"),"eax");
  208. &or ("ebp","ecx");
  209. &shr ("ebx",1);
  210. &shl ("esi",31);
  211. &mov (&DWP(24,"edi"),"ebp");
  212. &or ("ebx","esi"); # handle top-most carry bit
  213. &mov (&DWP(28,"edi"),"ebx");
  214. &ret ();
  215. &function_end_B("_ecp_nistz256_div_by_2");
  216. ########################################################################
  217. # void ecp_nistz256_add(BN_ULONG edi[8],const BN_ULONG esi[8],
  218. # const BN_ULONG ebp[8]);
  219. &function_begin("ecp_nistz256_add");
  220. &mov ("esi",&wparam(1));
  221. &mov ("ebp",&wparam(2));
  222. &mov ("edi",&wparam(0));
  223. &call ("_ecp_nistz256_add");
  224. &function_end("ecp_nistz256_add");
  225. &function_begin_B("_ecp_nistz256_add");
  226. &mov ("eax",&DWP(0,"esi"));
  227. &mov ("ebx",&DWP(4,"esi"));
  228. &mov ("ecx",&DWP(8,"esi"));
  229. &add ("eax",&DWP(0,"ebp"));
  230. &mov ("edx",&DWP(12,"esi"));
  231. &adc ("ebx",&DWP(4,"ebp"));
  232. &mov (&DWP(0,"edi"),"eax");
  233. &adc ("ecx",&DWP(8,"ebp"));
  234. &mov (&DWP(4,"edi"),"ebx");
  235. &adc ("edx",&DWP(12,"ebp"));
  236. &mov (&DWP(8,"edi"),"ecx");
  237. &mov (&DWP(12,"edi"),"edx");
  238. &mov ("eax",&DWP(16,"esi"));
  239. &mov ("ebx",&DWP(20,"esi"));
  240. &mov ("ecx",&DWP(24,"esi"));
  241. &adc ("eax",&DWP(16,"ebp"));
  242. &mov ("edx",&DWP(28,"esi"));
  243. &adc ("ebx",&DWP(20,"ebp"));
  244. &mov (&DWP(16,"edi"),"eax");
  245. &adc ("ecx",&DWP(24,"ebp"));
  246. &mov (&DWP(20,"edi"),"ebx");
  247. &mov ("esi",0);
  248. &adc ("edx",&DWP(28,"ebp"));
  249. &mov (&DWP(24,"edi"),"ecx");
  250. &adc ("esi",0);
  251. &mov (&DWP(28,"edi"),"edx");
  252. # if a+b >= modulus, subtract modulus.
  253. #
  254. # But since comparison implies subtraction, we subtract modulus
  255. # to see if it borrows, and then subtract it for real if
  256. # subtraction didn't borrow.
  257. &mov ("eax",&DWP(0,"edi"));
  258. &mov ("ebx",&DWP(4,"edi"));
  259. &mov ("ecx",&DWP(8,"edi"));
  260. &sub ("eax",-1);
  261. &mov ("edx",&DWP(12,"edi"));
  262. &sbb ("ebx",-1);
  263. &mov ("eax",&DWP(16,"edi"));
  264. &sbb ("ecx",-1);
  265. &mov ("ebx",&DWP(20,"edi"));
  266. &sbb ("edx",0);
  267. &mov ("ecx",&DWP(24,"edi"));
  268. &sbb ("eax",0);
  269. &mov ("edx",&DWP(28,"edi"));
  270. &sbb ("ebx",0);
  271. &sbb ("ecx",1);
  272. &sbb ("edx",-1);
  273. &sbb ("esi",0);
  274. # Note that because mod has special form, i.e. consists of
  275. # 0xffffffff, 1 and 0s, we can conditionally synthesize it by
  276. # by using borrow.
  277. &not ("esi");
  278. &mov ("eax",&DWP(0,"edi"));
  279. &mov ("ebp","esi");
  280. &mov ("ebx",&DWP(4,"edi"));
  281. &shr ("ebp",31);
  282. &mov ("ecx",&DWP(8,"edi"));
  283. &sub ("eax","esi");
  284. &mov ("edx",&DWP(12,"edi"));
  285. &sbb ("ebx","esi");
  286. &mov (&DWP(0,"edi"),"eax");
  287. &sbb ("ecx","esi");
  288. &mov (&DWP(4,"edi"),"ebx");
  289. &sbb ("edx",0);
  290. &mov (&DWP(8,"edi"),"ecx");
  291. &mov (&DWP(12,"edi"),"edx");
  292. &mov ("eax",&DWP(16,"edi"));
  293. &mov ("ebx",&DWP(20,"edi"));
  294. &mov ("ecx",&DWP(24,"edi"));
  295. &sbb ("eax",0);
  296. &mov ("edx",&DWP(28,"edi"));
  297. &sbb ("ebx",0);
  298. &mov (&DWP(16,"edi"),"eax");
  299. &sbb ("ecx","ebp");
  300. &mov (&DWP(20,"edi"),"ebx");
  301. &sbb ("edx","esi");
  302. &mov (&DWP(24,"edi"),"ecx");
  303. &mov (&DWP(28,"edi"),"edx");
  304. &ret ();
  305. &function_end_B("_ecp_nistz256_add");
  306. ########################################################################
  307. # void ecp_nistz256_sub(BN_ULONG edi[8],const BN_ULONG esi[8],
  308. # const BN_ULONG ebp[8]);
  309. &function_begin("ecp_nistz256_sub");
  310. &mov ("esi",&wparam(1));
  311. &mov ("ebp",&wparam(2));
  312. &mov ("edi",&wparam(0));
  313. &call ("_ecp_nistz256_sub");
  314. &function_end("ecp_nistz256_sub");
  315. &function_begin_B("_ecp_nistz256_sub");
  316. &mov ("eax",&DWP(0,"esi"));
  317. &mov ("ebx",&DWP(4,"esi"));
  318. &mov ("ecx",&DWP(8,"esi"));
  319. &sub ("eax",&DWP(0,"ebp"));
  320. &mov ("edx",&DWP(12,"esi"));
  321. &sbb ("ebx",&DWP(4,"ebp"));
  322. &mov (&DWP(0,"edi"),"eax");
  323. &sbb ("ecx",&DWP(8,"ebp"));
  324. &mov (&DWP(4,"edi"),"ebx");
  325. &sbb ("edx",&DWP(12,"ebp"));
  326. &mov (&DWP(8,"edi"),"ecx");
  327. &mov (&DWP(12,"edi"),"edx");
  328. &mov ("eax",&DWP(16,"esi"));
  329. &mov ("ebx",&DWP(20,"esi"));
  330. &mov ("ecx",&DWP(24,"esi"));
  331. &sbb ("eax",&DWP(16,"ebp"));
  332. &mov ("edx",&DWP(28,"esi"));
  333. &sbb ("ebx",&DWP(20,"ebp"));
  334. &sbb ("ecx",&DWP(24,"ebp"));
  335. &mov (&DWP(16,"edi"),"eax");
  336. &sbb ("edx",&DWP(28,"ebp"));
  337. &mov (&DWP(20,"edi"),"ebx");
  338. &sbb ("esi","esi"); # broadcast borrow bit
  339. &mov (&DWP(24,"edi"),"ecx");
  340. &mov (&DWP(28,"edi"),"edx");
  341. # if a-b borrows, add modulus.
  342. #
  343. # Note that because mod has special form, i.e. consists of
  344. # 0xffffffff, 1 and 0s, we can conditionally synthesize it by
  345. # assigning borrow bit to one register, %ebp, and its negative
  346. # to another, %esi. But we started by calculating %esi...
  347. &mov ("eax",&DWP(0,"edi"));
  348. &mov ("ebp","esi");
  349. &mov ("ebx",&DWP(4,"edi"));
  350. &shr ("ebp",31);
  351. &mov ("ecx",&DWP(8,"edi"));
  352. &add ("eax","esi");
  353. &mov ("edx",&DWP(12,"edi"));
  354. &adc ("ebx","esi");
  355. &mov (&DWP(0,"edi"),"eax");
  356. &adc ("ecx","esi");
  357. &mov (&DWP(4,"edi"),"ebx");
  358. &adc ("edx",0);
  359. &mov (&DWP(8,"edi"),"ecx");
  360. &mov (&DWP(12,"edi"),"edx");
  361. &mov ("eax",&DWP(16,"edi"));
  362. &mov ("ebx",&DWP(20,"edi"));
  363. &mov ("ecx",&DWP(24,"edi"));
  364. &adc ("eax",0);
  365. &mov ("edx",&DWP(28,"edi"));
  366. &adc ("ebx",0);
  367. &mov (&DWP(16,"edi"),"eax");
  368. &adc ("ecx","ebp");
  369. &mov (&DWP(20,"edi"),"ebx");
  370. &adc ("edx","esi");
  371. &mov (&DWP(24,"edi"),"ecx");
  372. &mov (&DWP(28,"edi"),"edx");
  373. &ret ();
  374. &function_end_B("_ecp_nistz256_sub");
  375. ########################################################################
  376. # void ecp_nistz256_neg(BN_ULONG edi[8],const BN_ULONG esi[8]);
  377. &function_begin("ecp_nistz256_neg");
  378. &mov ("ebp",&wparam(1));
  379. &mov ("edi",&wparam(0));
  380. &xor ("eax","eax");
  381. &stack_push(8);
  382. &mov (&DWP(0,"esp"),"eax");
  383. &mov ("esi","esp");
  384. &mov (&DWP(4,"esp"),"eax");
  385. &mov (&DWP(8,"esp"),"eax");
  386. &mov (&DWP(12,"esp"),"eax");
  387. &mov (&DWP(16,"esp"),"eax");
  388. &mov (&DWP(20,"esp"),"eax");
  389. &mov (&DWP(24,"esp"),"eax");
  390. &mov (&DWP(28,"esp"),"eax");
  391. &call ("_ecp_nistz256_sub");
  392. &stack_pop(8);
  393. &function_end("ecp_nistz256_neg");
  394. &function_begin_B("_picup_eax");
  395. &mov ("eax",&DWP(0,"esp"));
  396. &ret ();
  397. &function_end_B("_picup_eax");
  398. ########################################################################
  399. # void ecp_nistz256_to_mont(BN_ULONG edi[8],const BN_ULONG esi[8]);
  400. &function_begin("ecp_nistz256_to_mont");
  401. &mov ("esi",&wparam(1));
  402. &call ("_picup_eax");
  403. &set_label("pic");
  404. &lea ("ebp",&DWP(&label("RR")."-".&label("pic"),"eax"));
  405. if ($sse2) {
  406. &picmeup("eax","OPENSSL_ia32cap_P","eax",&label("pic"));
  407. &mov ("eax",&DWP(0,"eax")); }
  408. &mov ("edi",&wparam(0));
  409. &call ("_ecp_nistz256_mul_mont");
  410. &function_end("ecp_nistz256_to_mont");
  411. ########################################################################
  412. # void ecp_nistz256_from_mont(BN_ULONG edi[8],const BN_ULONG esi[8]);
  413. &function_begin("ecp_nistz256_from_mont");
  414. &mov ("esi",&wparam(1));
  415. &call ("_picup_eax");
  416. &set_label("pic");
  417. &lea ("ebp",&DWP(&label("ONE")."-".&label("pic"),"eax"));
  418. if ($sse2) {
  419. &picmeup("eax","OPENSSL_ia32cap_P","eax",&label("pic"));
  420. &mov ("eax",&DWP(0,"eax")); }
  421. &mov ("edi",&wparam(0));
  422. &call ("_ecp_nistz256_mul_mont");
  423. &function_end("ecp_nistz256_from_mont");
  424. ########################################################################
  425. # void ecp_nistz256_mul_mont(BN_ULONG edi[8],const BN_ULONG esi[8],
  426. # const BN_ULONG ebp[8]);
  427. &function_begin("ecp_nistz256_mul_mont");
  428. &mov ("esi",&wparam(1));
  429. &mov ("ebp",&wparam(2));
  430. if ($sse2) {
  431. &call ("_picup_eax");
  432. &set_label("pic");
  433. &picmeup("eax","OPENSSL_ia32cap_P","eax",&label("pic"));
  434. &mov ("eax",&DWP(0,"eax")); }
  435. &mov ("edi",&wparam(0));
  436. &call ("_ecp_nistz256_mul_mont");
  437. &function_end("ecp_nistz256_mul_mont");
  438. ########################################################################
  439. # void ecp_nistz256_sqr_mont(BN_ULONG edi[8],const BN_ULONG esi[8]);
  440. &function_begin("ecp_nistz256_sqr_mont");
  441. &mov ("esi",&wparam(1));
  442. if ($sse2) {
  443. &call ("_picup_eax");
  444. &set_label("pic");
  445. &picmeup("eax","OPENSSL_ia32cap_P","eax",&label("pic"));
  446. &mov ("eax",&DWP(0,"eax")); }
  447. &mov ("edi",&wparam(0));
  448. &mov ("ebp","esi");
  449. &call ("_ecp_nistz256_mul_mont");
  450. &function_end("ecp_nistz256_sqr_mont");
  451. &function_begin_B("_ecp_nistz256_mul_mont");
  452. if ($sse2) {
  453. &and ("eax",1<<24|1<<26);
  454. &cmp ("eax",1<<24|1<<26); # see if XMM+SSE2 is on
  455. &jne (&label("mul_mont_ialu"));
  456. ########################################
  457. # SSE2 code path featuring 32x16-bit
  458. # multiplications is ~2x faster than
  459. # IALU counterpart (except on Atom)...
  460. ########################################
  461. # stack layout:
  462. # +------------------------------------+< %esp
  463. # | 7 16-byte temporary XMM words, |
  464. # | "sliding" toward lower address |
  465. # . .
  466. # +------------------------------------+
  467. # | unused XMM word |
  468. # +------------------------------------+< +128,%ebx
  469. # | 8 16-byte XMM words holding copies |
  470. # | of a[i]<<64|a[i] |
  471. # . .
  472. # . .
  473. # +------------------------------------+< +256
  474. &mov ("edx","esp");
  475. &sub ("esp",0x100);
  476. &movd ("xmm7",&DWP(0,"ebp")); # b[0] -> 0000.00xy
  477. &lea ("ebp",&DWP(4,"ebp"));
  478. &pcmpeqd("xmm6","xmm6");
  479. &psrlq ("xmm6",48); # compose 0xffff<<64|0xffff
  480. &pshuflw("xmm7","xmm7",0b11011100); # 0000.00xy -> 0000.0x0y
  481. &and ("esp",-64);
  482. &pshufd ("xmm7","xmm7",0b11011100); # 0000.0x0y -> 000x.000y
  483. &lea ("ebx",&DWP(0x80,"esp"));
  484. &movd ("xmm0",&DWP(4*0,"esi")); # a[0] -> 0000.00xy
  485. &pshufd ("xmm0","xmm0",0b11001100); # 0000.00xy -> 00xy.00xy
  486. &movd ("xmm1",&DWP(4*1,"esi")); # a[1] -> ...
  487. &movdqa (&QWP(0x00,"ebx"),"xmm0"); # offload converted a[0]
  488. &pmuludq("xmm0","xmm7"); # a[0]*b[0]
  489. &movd ("xmm2",&DWP(4*2,"esi"));
  490. &pshufd ("xmm1","xmm1",0b11001100);
  491. &movdqa (&QWP(0x10,"ebx"),"xmm1");
  492. &pmuludq("xmm1","xmm7"); # a[1]*b[0]
  493. &movq ("xmm4","xmm0"); # clear upper 64 bits
  494. &pslldq("xmm4",6);
  495. &paddq ("xmm4","xmm0");
  496. &movdqa("xmm5","xmm4");
  497. &psrldq("xmm4",10); # upper 32 bits of a[0]*b[0]
  498. &pand ("xmm5","xmm6"); # lower 32 bits of a[0]*b[0]
  499. # Upper half of a[0]*b[i] is carried into next multiplication
  500. # iteration, while lower one "participates" in actual reduction.
  501. # Normally latter is done by accumulating result of multiplication
  502. # of modulus by "magic" digit, but thanks to special form of modulus
  503. # and "magic" digit it can be performed only with additions and
  504. # subtractions (see note in IALU section below). Note that we are
  505. # not bothered with carry bits, they are accumulated in "flatten"
  506. # phase after all multiplications and reductions.
  507. &movd ("xmm3",&DWP(4*3,"esi"));
  508. &pshufd ("xmm2","xmm2",0b11001100);
  509. &movdqa (&QWP(0x20,"ebx"),"xmm2");
  510. &pmuludq("xmm2","xmm7"); # a[2]*b[0]
  511. &paddq ("xmm1","xmm4"); # a[1]*b[0]+hw(a[0]*b[0]), carry
  512. &movdqa (&QWP(0x00,"esp"),"xmm1"); # t[0]
  513. &movd ("xmm0",&DWP(4*4,"esi"));
  514. &pshufd ("xmm3","xmm3",0b11001100);
  515. &movdqa (&QWP(0x30,"ebx"),"xmm3");
  516. &pmuludq("xmm3","xmm7"); # a[3]*b[0]
  517. &movdqa (&QWP(0x10,"esp"),"xmm2");
  518. &movd ("xmm1",&DWP(4*5,"esi"));
  519. &pshufd ("xmm0","xmm0",0b11001100);
  520. &movdqa (&QWP(0x40,"ebx"),"xmm0");
  521. &pmuludq("xmm0","xmm7"); # a[4]*b[0]
  522. &paddq ("xmm3","xmm5"); # a[3]*b[0]+lw(a[0]*b[0]), reduction step
  523. &movdqa (&QWP(0x20,"esp"),"xmm3");
  524. &movd ("xmm2",&DWP(4*6,"esi"));
  525. &pshufd ("xmm1","xmm1",0b11001100);
  526. &movdqa (&QWP(0x50,"ebx"),"xmm1");
  527. &pmuludq("xmm1","xmm7"); # a[5]*b[0]
  528. &movdqa (&QWP(0x30,"esp"),"xmm0");
  529. &pshufd("xmm4","xmm5",0b10110001); # xmm4 = xmm5<<32, reduction step
  530. &movd ("xmm3",&DWP(4*7,"esi"));
  531. &pshufd ("xmm2","xmm2",0b11001100);
  532. &movdqa (&QWP(0x60,"ebx"),"xmm2");
  533. &pmuludq("xmm2","xmm7"); # a[6]*b[0]
  534. &movdqa (&QWP(0x40,"esp"),"xmm1");
  535. &psubq ("xmm4","xmm5"); # xmm4 = xmm5*0xffffffff, reduction step
  536. &movd ("xmm0",&DWP(0,"ebp")); # b[1] -> 0000.00xy
  537. &pshufd ("xmm3","xmm3",0b11001100);
  538. &movdqa (&QWP(0x70,"ebx"),"xmm3");
  539. &pmuludq("xmm3","xmm7"); # a[7]*b[0]
  540. &pshuflw("xmm7","xmm0",0b11011100); # 0000.00xy -> 0000.0x0y
  541. &movdqa ("xmm0",&QWP(0x00,"ebx")); # pre-load converted a[0]
  542. &pshufd ("xmm7","xmm7",0b11011100); # 0000.0x0y -> 000x.000y
  543. &mov ("ecx",6);
  544. &lea ("ebp",&DWP(4,"ebp"));
  545. &jmp (&label("madd_sse2"));
  546. &set_label("madd_sse2",16);
  547. &paddq ("xmm2","xmm5"); # a[6]*b[i-1]+lw(a[0]*b[i-1]), reduction step [modulo-scheduled]
  548. &paddq ("xmm3","xmm4"); # a[7]*b[i-1]+lw(a[0]*b[i-1])*0xffffffff, reduction step [modulo-scheduled]
  549. &movdqa ("xmm1",&QWP(0x10,"ebx"));
  550. &pmuludq("xmm0","xmm7"); # a[0]*b[i]
  551. &movdqa(&QWP(0x50,"esp"),"xmm2");
  552. &movdqa ("xmm2",&QWP(0x20,"ebx"));
  553. &pmuludq("xmm1","xmm7"); # a[1]*b[i]
  554. &movdqa(&QWP(0x60,"esp"),"xmm3");
  555. &paddq ("xmm0",&QWP(0x00,"esp"));
  556. &movdqa ("xmm3",&QWP(0x30,"ebx"));
  557. &pmuludq("xmm2","xmm7"); # a[2]*b[i]
  558. &movq ("xmm4","xmm0"); # clear upper 64 bits
  559. &pslldq("xmm4",6);
  560. &paddq ("xmm1",&QWP(0x10,"esp"));
  561. &paddq ("xmm4","xmm0");
  562. &movdqa("xmm5","xmm4");
  563. &psrldq("xmm4",10); # upper 33 bits of a[0]*b[i]+t[0]
  564. &movdqa ("xmm0",&QWP(0x40,"ebx"));
  565. &pmuludq("xmm3","xmm7"); # a[3]*b[i]
  566. &paddq ("xmm1","xmm4"); # a[1]*b[i]+hw(a[0]*b[i]), carry
  567. &paddq ("xmm2",&QWP(0x20,"esp"));
  568. &movdqa (&QWP(0x00,"esp"),"xmm1");
  569. &movdqa ("xmm1",&QWP(0x50,"ebx"));
  570. &pmuludq("xmm0","xmm7"); # a[4]*b[i]
  571. &paddq ("xmm3",&QWP(0x30,"esp"));
  572. &movdqa (&QWP(0x10,"esp"),"xmm2");
  573. &pand ("xmm5","xmm6"); # lower 32 bits of a[0]*b[i]
  574. &movdqa ("xmm2",&QWP(0x60,"ebx"));
  575. &pmuludq("xmm1","xmm7"); # a[5]*b[i]
  576. &paddq ("xmm3","xmm5"); # a[3]*b[i]+lw(a[0]*b[i]), reduction step
  577. &paddq ("xmm0",&QWP(0x40,"esp"));
  578. &movdqa (&QWP(0x20,"esp"),"xmm3");
  579. &pshufd("xmm4","xmm5",0b10110001); # xmm4 = xmm5<<32, reduction step
  580. &movdqa ("xmm3","xmm7");
  581. &pmuludq("xmm2","xmm7"); # a[6]*b[i]
  582. &movd ("xmm7",&DWP(0,"ebp")); # b[i++] -> 0000.00xy
  583. &lea ("ebp",&DWP(4,"ebp"));
  584. &paddq ("xmm1",&QWP(0x50,"esp"));
  585. &psubq ("xmm4","xmm5"); # xmm4 = xmm5*0xffffffff, reduction step
  586. &movdqa (&QWP(0x30,"esp"),"xmm0");
  587. &pshuflw("xmm7","xmm7",0b11011100); # 0000.00xy -> 0000.0x0y
  588. &pmuludq("xmm3",&QWP(0x70,"ebx")); # a[7]*b[i]
  589. &pshufd("xmm7","xmm7",0b11011100); # 0000.0x0y -> 000x.000y
  590. &movdqa("xmm0",&QWP(0x00,"ebx")); # pre-load converted a[0]
  591. &movdqa (&QWP(0x40,"esp"),"xmm1");
  592. &paddq ("xmm2",&QWP(0x60,"esp"));
  593. &dec ("ecx");
  594. &jnz (&label("madd_sse2"));
  595. &paddq ("xmm2","xmm5"); # a[6]*b[6]+lw(a[0]*b[6]), reduction step [modulo-scheduled]
  596. &paddq ("xmm3","xmm4"); # a[7]*b[6]+lw(a[0]*b[6])*0xffffffff, reduction step [modulo-scheduled]
  597. &movdqa ("xmm1",&QWP(0x10,"ebx"));
  598. &pmuludq("xmm0","xmm7"); # a[0]*b[7]
  599. &movdqa(&QWP(0x50,"esp"),"xmm2");
  600. &movdqa ("xmm2",&QWP(0x20,"ebx"));
  601. &pmuludq("xmm1","xmm7"); # a[1]*b[7]
  602. &movdqa(&QWP(0x60,"esp"),"xmm3");
  603. &paddq ("xmm0",&QWP(0x00,"esp"));
  604. &movdqa ("xmm3",&QWP(0x30,"ebx"));
  605. &pmuludq("xmm2","xmm7"); # a[2]*b[7]
  606. &movq ("xmm4","xmm0"); # clear upper 64 bits
  607. &pslldq("xmm4",6);
  608. &paddq ("xmm1",&QWP(0x10,"esp"));
  609. &paddq ("xmm4","xmm0");
  610. &movdqa("xmm5","xmm4");
  611. &psrldq("xmm4",10); # upper 33 bits of a[0]*b[i]+t[0]
  612. &movdqa ("xmm0",&QWP(0x40,"ebx"));
  613. &pmuludq("xmm3","xmm7"); # a[3]*b[7]
  614. &paddq ("xmm1","xmm4"); # a[1]*b[7]+hw(a[0]*b[7]), carry
  615. &paddq ("xmm2",&QWP(0x20,"esp"));
  616. &movdqa (&QWP(0x00,"esp"),"xmm1");
  617. &movdqa ("xmm1",&QWP(0x50,"ebx"));
  618. &pmuludq("xmm0","xmm7"); # a[4]*b[7]
  619. &paddq ("xmm3",&QWP(0x30,"esp"));
  620. &movdqa (&QWP(0x10,"esp"),"xmm2");
  621. &pand ("xmm5","xmm6"); # lower 32 bits of a[0]*b[i]
  622. &movdqa ("xmm2",&QWP(0x60,"ebx"));
  623. &pmuludq("xmm1","xmm7"); # a[5]*b[7]
  624. &paddq ("xmm3","xmm5"); # reduction step
  625. &paddq ("xmm0",&QWP(0x40,"esp"));
  626. &movdqa (&QWP(0x20,"esp"),"xmm3");
  627. &pshufd("xmm4","xmm5",0b10110001); # xmm4 = xmm5<<32, reduction step
  628. &movdqa ("xmm3",&QWP(0x70,"ebx"));
  629. &pmuludq("xmm2","xmm7"); # a[6]*b[7]
  630. &paddq ("xmm1",&QWP(0x50,"esp"));
  631. &psubq ("xmm4","xmm5"); # xmm4 = xmm5*0xffffffff, reduction step
  632. &movdqa (&QWP(0x30,"esp"),"xmm0");
  633. &pmuludq("xmm3","xmm7"); # a[7]*b[7]
  634. &pcmpeqd("xmm7","xmm7");
  635. &movdqa ("xmm0",&QWP(0x00,"esp"));
  636. &pslldq ("xmm7",8);
  637. &movdqa (&QWP(0x40,"esp"),"xmm1");
  638. &paddq ("xmm2",&QWP(0x60,"esp"));
  639. &paddq ("xmm2","xmm5"); # a[6]*b[7]+lw(a[0]*b[7]), reduction step
  640. &paddq ("xmm3","xmm4"); # a[6]*b[7]+lw(a[0]*b[7])*0xffffffff, reduction step
  641. &movdqa(&QWP(0x50,"esp"),"xmm2");
  642. &movdqa(&QWP(0x60,"esp"),"xmm3");
  643. &movdqa ("xmm1",&QWP(0x10,"esp"));
  644. &movdqa ("xmm2",&QWP(0x20,"esp"));
  645. &movdqa ("xmm3",&QWP(0x30,"esp"));
  646. &movq ("xmm4","xmm0"); # "flatten"
  647. &pand ("xmm0","xmm7");
  648. &xor ("ebp","ebp");
  649. &pslldq ("xmm4",6);
  650. &movq ("xmm5","xmm1");
  651. &paddq ("xmm0","xmm4");
  652. &pand ("xmm1","xmm7");
  653. &psrldq ("xmm0",6);
  654. &movd ("eax","xmm0");
  655. &psrldq ("xmm0",4);
  656. &paddq ("xmm5","xmm0");
  657. &movdqa ("xmm0",&QWP(0x40,"esp"));
  658. &sub ("eax",-1); # start subtracting modulus,
  659. # this is used to determine
  660. # if result is larger/smaller
  661. # than modulus (see below)
  662. &pslldq ("xmm5",6);
  663. &movq ("xmm4","xmm2");
  664. &paddq ("xmm1","xmm5");
  665. &pand ("xmm2","xmm7");
  666. &psrldq ("xmm1",6);
  667. &mov (&DWP(4*0,"edi"),"eax");
  668. &movd ("eax","xmm1");
  669. &psrldq ("xmm1",4);
  670. &paddq ("xmm4","xmm1");
  671. &movdqa ("xmm1",&QWP(0x50,"esp"));
  672. &sbb ("eax",-1);
  673. &pslldq ("xmm4",6);
  674. &movq ("xmm5","xmm3");
  675. &paddq ("xmm2","xmm4");
  676. &pand ("xmm3","xmm7");
  677. &psrldq ("xmm2",6);
  678. &mov (&DWP(4*1,"edi"),"eax");
  679. &movd ("eax","xmm2");
  680. &psrldq ("xmm2",4);
  681. &paddq ("xmm5","xmm2");
  682. &movdqa ("xmm2",&QWP(0x60,"esp"));
  683. &sbb ("eax",-1);
  684. &pslldq ("xmm5",6);
  685. &movq ("xmm4","xmm0");
  686. &paddq ("xmm3","xmm5");
  687. &pand ("xmm0","xmm7");
  688. &psrldq ("xmm3",6);
  689. &mov (&DWP(4*2,"edi"),"eax");
  690. &movd ("eax","xmm3");
  691. &psrldq ("xmm3",4);
  692. &paddq ("xmm4","xmm3");
  693. &sbb ("eax",0);
  694. &pslldq ("xmm4",6);
  695. &movq ("xmm5","xmm1");
  696. &paddq ("xmm0","xmm4");
  697. &pand ("xmm1","xmm7");
  698. &psrldq ("xmm0",6);
  699. &mov (&DWP(4*3,"edi"),"eax");
  700. &movd ("eax","xmm0");
  701. &psrldq ("xmm0",4);
  702. &paddq ("xmm5","xmm0");
  703. &sbb ("eax",0);
  704. &pslldq ("xmm5",6);
  705. &movq ("xmm4","xmm2");
  706. &paddq ("xmm1","xmm5");
  707. &pand ("xmm2","xmm7");
  708. &psrldq ("xmm1",6);
  709. &movd ("ebx","xmm1");
  710. &psrldq ("xmm1",4);
  711. &mov ("esp","edx");
  712. &paddq ("xmm4","xmm1");
  713. &pslldq ("xmm4",6);
  714. &paddq ("xmm2","xmm4");
  715. &psrldq ("xmm2",6);
  716. &movd ("ecx","xmm2");
  717. &psrldq ("xmm2",4);
  718. &sbb ("ebx",0);
  719. &movd ("edx","xmm2");
  720. &pextrw ("esi","xmm2",2); # top-most overflow bit
  721. &sbb ("ecx",1);
  722. &sbb ("edx",-1);
  723. &sbb ("esi",0); # borrow from subtraction
  724. # Final step is "if result > mod, subtract mod", and at this point
  725. # we have result - mod written to output buffer, as well as borrow
  726. # bit from this subtraction, and if borrow bit is set, we add
  727. # modulus back.
  728. #
  729. # Note that because mod has special form, i.e. consists of
  730. # 0xffffffff, 1 and 0s, we can conditionally synthesize it by
  731. # assigning borrow bit to one register, %ebp, and its negative
  732. # to another, %esi. But we started by calculating %esi...
  733. &sub ("ebp","esi");
  734. &add (&DWP(4*0,"edi"),"esi"); # add modulus or zero
  735. &adc (&DWP(4*1,"edi"),"esi");
  736. &adc (&DWP(4*2,"edi"),"esi");
  737. &adc (&DWP(4*3,"edi"),0);
  738. &adc ("eax",0);
  739. &adc ("ebx",0);
  740. &mov (&DWP(4*4,"edi"),"eax");
  741. &adc ("ecx","ebp");
  742. &mov (&DWP(4*5,"edi"),"ebx");
  743. &adc ("edx","esi");
  744. &mov (&DWP(4*6,"edi"),"ecx");
  745. &mov (&DWP(4*7,"edi"),"edx");
  746. &ret ();
  747. &set_label("mul_mont_ialu",16); }
  748. ########################################
  749. # IALU code path suitable for all CPUs.
  750. ########################################
  751. # stack layout:
  752. # +------------------------------------+< %esp
  753. # | 8 32-bit temporary words, accessed |
  754. # | as circular buffer |
  755. # . .
  756. # . .
  757. # +------------------------------------+< +32
  758. # | offloaded destination pointer |
  759. # +------------------------------------+
  760. # | unused |
  761. # +------------------------------------+< +40
  762. &sub ("esp",10*4);
  763. &mov ("eax",&DWP(0*4,"esi")); # a[0]
  764. &mov ("ebx",&DWP(0*4,"ebp")); # b[0]
  765. &mov (&DWP(8*4,"esp"),"edi"); # off-load dst ptr
  766. &mul ("ebx"); # a[0]*b[0]
  767. &mov (&DWP(0*4,"esp"),"eax"); # t[0]
  768. &mov ("eax",&DWP(1*4,"esi"));
  769. &mov ("ecx","edx")
  770. &mul ("ebx"); # a[1]*b[0]
  771. &add ("ecx","eax");
  772. &mov ("eax",&DWP(2*4,"esi"));
  773. &adc ("edx",0);
  774. &mov (&DWP(1*4,"esp"),"ecx"); # t[1]
  775. &mov ("ecx","edx");
  776. &mul ("ebx"); # a[2]*b[0]
  777. &add ("ecx","eax");
  778. &mov ("eax",&DWP(3*4,"esi"));
  779. &adc ("edx",0);
  780. &mov (&DWP(2*4,"esp"),"ecx"); # t[2]
  781. &mov ("ecx","edx");
  782. &mul ("ebx"); # a[3]*b[0]
  783. &add ("ecx","eax");
  784. &mov ("eax",&DWP(4*4,"esi"));
  785. &adc ("edx",0);
  786. &mov (&DWP(3*4,"esp"),"ecx"); # t[3]
  787. &mov ("ecx","edx");
  788. &mul ("ebx"); # a[4]*b[0]
  789. &add ("ecx","eax");
  790. &mov ("eax",&DWP(5*4,"esi"));
  791. &adc ("edx",0);
  792. &mov (&DWP(4*4,"esp"),"ecx"); # t[4]
  793. &mov ("ecx","edx");
  794. &mul ("ebx"); # a[5]*b[0]
  795. &add ("ecx","eax");
  796. &mov ("eax",&DWP(6*4,"esi"));
  797. &adc ("edx",0);
  798. &mov (&DWP(5*4,"esp"),"ecx"); # t[5]
  799. &mov ("ecx","edx");
  800. &mul ("ebx"); # a[6]*b[0]
  801. &add ("ecx","eax");
  802. &mov ("eax",&DWP(7*4,"esi"));
  803. &adc ("edx",0);
  804. &mov (&DWP(6*4,"esp"),"ecx"); # t[6]
  805. &mov ("ecx","edx");
  806. &xor ("edi","edi"); # initial top-most carry
  807. &mul ("ebx"); # a[7]*b[0]
  808. &add ("ecx","eax"); # t[7]
  809. &mov ("eax",&DWP(0*4,"esp")); # t[0]
  810. &adc ("edx",0); # t[8]
  811. for ($i=0;$i<7;$i++) {
  812. my $j=$i+1;
  813. # Reduction iteration is normally performed by accumulating
  814. # result of multiplication of modulus by "magic" digit [and
  815. # omitting least significant word, which is guaranteed to
  816. # be 0], but thanks to special form of modulus and "magic"
  817. # digit being equal to least significant word, it can be
  818. # performed with additions and subtractions alone. Indeed:
  819. #
  820. # ffff.0001.0000.0000.0000.ffff.ffff.ffff
  821. # * abcd
  822. # + xxxx.xxxx.xxxx.xxxx.xxxx.xxxx.xxxx.xxxx.abcd
  823. #
  824. # Now observing that ff..ff*x = (2^n-1)*x = 2^n*x-x, we
  825. # rewrite above as:
  826. #
  827. # xxxx.xxxx.xxxx.xxxx.xxxx.xxxx.xxxx.xxxx.abcd
  828. # + abcd.0000.abcd.0000.0000.abcd.0000.0000.0000
  829. # - abcd.0000.0000.0000.0000.0000.0000.abcd
  830. #
  831. # or marking redundant operations:
  832. #
  833. # xxxx.xxxx.xxxx.xxxx.xxxx.xxxx.xxxx.xxxx.----
  834. # + abcd.0000.abcd.0000.0000.abcd.----.----.----
  835. # - abcd.----.----.----.----.----.----.----
  836. &add (&DWP((($i+3)%8)*4,"esp"),"eax"); # t[3]+=t[0]
  837. &adc (&DWP((($i+4)%8)*4,"esp"),0); # t[4]+=0
  838. &adc (&DWP((($i+5)%8)*4,"esp"),0); # t[5]+=0
  839. &adc (&DWP((($i+6)%8)*4,"esp"),"eax"); # t[6]+=t[0]
  840. &adc ("ecx",0); # t[7]+=0
  841. &adc ("edx","eax"); # t[8]+=t[0]
  842. &adc ("edi",0); # top-most carry
  843. &mov ("ebx",&DWP($j*4,"ebp")); # b[i]
  844. &sub ("ecx","eax"); # t[7]-=t[0]
  845. &mov ("eax",&DWP(0*4,"esi")); # a[0]
  846. &sbb ("edx",0); # t[8]-=0
  847. &mov (&DWP((($i+7)%8)*4,"esp"),"ecx");
  848. &sbb ("edi",0); # top-most carry,
  849. # keep in mind that
  850. # netto result is
  851. # *addition* of value
  852. # with (abcd<<32)-abcd
  853. # on top, so that
  854. # underflow is
  855. # impossible, because
  856. # (abcd<<32)-abcd
  857. # doesn't underflow
  858. &mov (&DWP((($i+8)%8)*4,"esp"),"edx");
  859. &mul ("ebx"); # a[0]*b[i]
  860. &add ("eax",&DWP((($j+0)%8)*4,"esp"));
  861. &adc ("edx",0);
  862. &mov (&DWP((($j+0)%8)*4,"esp"),"eax");
  863. &mov ("eax",&DWP(1*4,"esi"));
  864. &mov ("ecx","edx")
  865. &mul ("ebx"); # a[1]*b[i]
  866. &add ("ecx",&DWP((($j+1)%8)*4,"esp"));
  867. &adc ("edx",0);
  868. &add ("ecx","eax");
  869. &adc ("edx",0);
  870. &mov ("eax",&DWP(2*4,"esi"));
  871. &mov (&DWP((($j+1)%8)*4,"esp"),"ecx");
  872. &mov ("ecx","edx");
  873. &mul ("ebx"); # a[2]*b[i]
  874. &add ("ecx",&DWP((($j+2)%8)*4,"esp"));
  875. &adc ("edx",0);
  876. &add ("ecx","eax");
  877. &adc ("edx",0);
  878. &mov ("eax",&DWP(3*4,"esi"));
  879. &mov (&DWP((($j+2)%8)*4,"esp"),"ecx");
  880. &mov ("ecx","edx");
  881. &mul ("ebx"); # a[3]*b[i]
  882. &add ("ecx",&DWP((($j+3)%8)*4,"esp"));
  883. &adc ("edx",0);
  884. &add ("ecx","eax");
  885. &adc ("edx",0);
  886. &mov ("eax",&DWP(4*4,"esi"));
  887. &mov (&DWP((($j+3)%8)*4,"esp"),"ecx");
  888. &mov ("ecx","edx");
  889. &mul ("ebx"); # a[4]*b[i]
  890. &add ("ecx",&DWP((($j+4)%8)*4,"esp"));
  891. &adc ("edx",0);
  892. &add ("ecx","eax");
  893. &adc ("edx",0);
  894. &mov ("eax",&DWP(5*4,"esi"));
  895. &mov (&DWP((($j+4)%8)*4,"esp"),"ecx");
  896. &mov ("ecx","edx");
  897. &mul ("ebx"); # a[5]*b[i]
  898. &add ("ecx",&DWP((($j+5)%8)*4,"esp"));
  899. &adc ("edx",0);
  900. &add ("ecx","eax");
  901. &adc ("edx",0);
  902. &mov ("eax",&DWP(6*4,"esi"));
  903. &mov (&DWP((($j+5)%8)*4,"esp"),"ecx");
  904. &mov ("ecx","edx");
  905. &mul ("ebx"); # a[6]*b[i]
  906. &add ("ecx",&DWP((($j+6)%8)*4,"esp"));
  907. &adc ("edx",0);
  908. &add ("ecx","eax");
  909. &adc ("edx",0);
  910. &mov ("eax",&DWP(7*4,"esi"));
  911. &mov (&DWP((($j+6)%8)*4,"esp"),"ecx");
  912. &mov ("ecx","edx");
  913. &mul ("ebx"); # a[7]*b[i]
  914. &add ("ecx",&DWP((($j+7)%8)*4,"esp"));
  915. &adc ("edx",0);
  916. &add ("ecx","eax"); # t[7]
  917. &mov ("eax",&DWP((($j+0)%8)*4,"esp")); # t[0]
  918. &adc ("edx","edi"); # t[8]
  919. &mov ("edi",0);
  920. &adc ("edi",0); # top-most carry
  921. }
  922. &mov ("ebp",&DWP(8*4,"esp")); # restore dst ptr
  923. &xor ("esi","esi");
  924. my $j=$i+1;
  925. # last multiplication-less reduction
  926. &add (&DWP((($i+3)%8)*4,"esp"),"eax"); # t[3]+=t[0]
  927. &adc (&DWP((($i+4)%8)*4,"esp"),0); # t[4]+=0
  928. &adc (&DWP((($i+5)%8)*4,"esp"),0); # t[5]+=0
  929. &adc (&DWP((($i+6)%8)*4,"esp"),"eax"); # t[6]+=t[0]
  930. &adc ("ecx",0); # t[7]+=0
  931. &adc ("edx","eax"); # t[8]+=t[0]
  932. &adc ("edi",0); # top-most carry
  933. &mov ("ebx",&DWP((($j+1)%8)*4,"esp"));
  934. &sub ("ecx","eax"); # t[7]-=t[0]
  935. &mov ("eax",&DWP((($j+0)%8)*4,"esp"));
  936. &sbb ("edx",0); # t[8]-=0
  937. &mov (&DWP((($i+7)%8)*4,"esp"),"ecx");
  938. &sbb ("edi",0); # top-most carry
  939. &mov (&DWP((($i+8)%8)*4,"esp"),"edx");
  940. # Final step is "if result > mod, subtract mod", but we do it
  941. # "other way around", namely write result - mod to output buffer
  942. # and if subtraction borrowed, add modulus back.
  943. &mov ("ecx",&DWP((($j+2)%8)*4,"esp"));
  944. &sub ("eax",-1);
  945. &mov ("edx",&DWP((($j+3)%8)*4,"esp"));
  946. &sbb ("ebx",-1);
  947. &mov (&DWP(0*4,"ebp"),"eax");
  948. &sbb ("ecx",-1);
  949. &mov (&DWP(1*4,"ebp"),"ebx");
  950. &sbb ("edx",0);
  951. &mov (&DWP(2*4,"ebp"),"ecx");
  952. &mov (&DWP(3*4,"ebp"),"edx");
  953. &mov ("eax",&DWP((($j+4)%8)*4,"esp"));
  954. &mov ("ebx",&DWP((($j+5)%8)*4,"esp"));
  955. &mov ("ecx",&DWP((($j+6)%8)*4,"esp"));
  956. &sbb ("eax",0);
  957. &mov ("edx",&DWP((($j+7)%8)*4,"esp"));
  958. &sbb ("ebx",0);
  959. &sbb ("ecx",1);
  960. &sbb ("edx",-1);
  961. &sbb ("edi",0);
  962. # Note that because mod has special form, i.e. consists of
  963. # 0xffffffff, 1 and 0s, we can conditionally synthesize it by
  964. # assigning borrow bit to one register, %ebp, and its negative
  965. # to another, %esi. But we started by calculating %esi...
  966. &sub ("esi","edi");
  967. &add (&DWP(0*4,"ebp"),"edi"); # add modulus or zero
  968. &adc (&DWP(1*4,"ebp"),"edi");
  969. &adc (&DWP(2*4,"ebp"),"edi");
  970. &adc (&DWP(3*4,"ebp"),0);
  971. &adc ("eax",0);
  972. &adc ("ebx",0);
  973. &mov (&DWP(4*4,"ebp"),"eax");
  974. &adc ("ecx","esi");
  975. &mov (&DWP(5*4,"ebp"),"ebx");
  976. &adc ("edx","edi");
  977. &mov (&DWP(6*4,"ebp"),"ecx");
  978. &mov ("edi","ebp"); # fulfill contract
  979. &mov (&DWP(7*4,"ebp"),"edx");
  980. &add ("esp",10*4);
  981. &ret ();
  982. &function_end_B("_ecp_nistz256_mul_mont");
  983. ########################################################################
  984. # void ecp_nistz256_scatter_w5(void *edi,const P256_POINT *esi,
  985. # int ebp);
  986. &function_begin("ecp_nistz256_scatter_w5");
  987. &mov ("edi",&wparam(0));
  988. &mov ("esi",&wparam(1));
  989. &mov ("ebp",&wparam(2));
  990. &lea ("edi",&DWP(128-4,"edi","ebp",4));
  991. &mov ("ebp",96/16);
  992. &set_label("scatter_w5_loop");
  993. &mov ("eax",&DWP(0,"esi"));
  994. &mov ("ebx",&DWP(4,"esi"));
  995. &mov ("ecx",&DWP(8,"esi"));
  996. &mov ("edx",&DWP(12,"esi"));
  997. &lea ("esi",&DWP(16,"esi"));
  998. &mov (&DWP(64*0-128,"edi"),"eax");
  999. &mov (&DWP(64*1-128,"edi"),"ebx");
  1000. &mov (&DWP(64*2-128,"edi"),"ecx");
  1001. &mov (&DWP(64*3-128,"edi"),"edx");
  1002. &lea ("edi",&DWP(64*4,"edi"));
  1003. &dec ("ebp");
  1004. &jnz (&label("scatter_w5_loop"));
  1005. &function_end("ecp_nistz256_scatter_w5");
  1006. ########################################################################
  1007. # void ecp_nistz256_gather_w5(P256_POINT *edi,const void *esi,
  1008. # int ebp);
  1009. &function_begin("ecp_nistz256_gather_w5");
  1010. &mov ("esi",&wparam(1));
  1011. &mov ("ebp",&wparam(2));
  1012. &lea ("esi",&DWP(0,"esi","ebp",4));
  1013. &neg ("ebp");
  1014. &sar ("ebp",31);
  1015. &mov ("edi",&wparam(0));
  1016. &lea ("esi",&DWP(0,"esi","ebp",4));
  1017. for($i=0;$i<24;$i+=4) {
  1018. &mov ("eax",&DWP(64*($i+0),"esi"));
  1019. &mov ("ebx",&DWP(64*($i+1),"esi"));
  1020. &mov ("ecx",&DWP(64*($i+2),"esi"));
  1021. &mov ("edx",&DWP(64*($i+3),"esi"));
  1022. &and ("eax","ebp");
  1023. &and ("ebx","ebp");
  1024. &and ("ecx","ebp");
  1025. &and ("edx","ebp");
  1026. &mov (&DWP(4*($i+0),"edi"),"eax");
  1027. &mov (&DWP(4*($i+1),"edi"),"ebx");
  1028. &mov (&DWP(4*($i+2),"edi"),"ecx");
  1029. &mov (&DWP(4*($i+3),"edi"),"edx");
  1030. }
  1031. &function_end("ecp_nistz256_gather_w5");
  1032. ########################################################################
  1033. # void ecp_nistz256_scatter_w7(void *edi,const P256_POINT_AFFINE *esi,
  1034. # int ebp);
  1035. &function_begin("ecp_nistz256_scatter_w7");
  1036. &mov ("edi",&wparam(0));
  1037. &mov ("esi",&wparam(1));
  1038. &mov ("ebp",&wparam(2));
  1039. &lea ("edi",&DWP(0,"edi","ebp"));
  1040. &mov ("ebp",64/4);
  1041. &set_label("scatter_w7_loop");
  1042. &mov ("eax",&DWP(0,"esi"));
  1043. &lea ("esi",&DWP(4,"esi"));
  1044. &mov (&BP(64*0,"edi"),"al");
  1045. &mov (&BP(64*1,"edi"),"ah");
  1046. &shr ("eax",16);
  1047. &mov (&BP(64*2,"edi"),"al");
  1048. &mov (&BP(64*3,"edi"),"ah");
  1049. &lea ("edi",&DWP(64*4,"edi"));
  1050. &dec ("ebp");
  1051. &jnz (&label("scatter_w7_loop"));
  1052. &function_end("ecp_nistz256_scatter_w7");
  1053. ########################################################################
  1054. # void ecp_nistz256_gather_w7(P256_POINT_AFFINE *edi,const void *esi,
  1055. # int ebp);
  1056. &function_begin("ecp_nistz256_gather_w7");
  1057. &mov ("esi",&wparam(1));
  1058. &mov ("ebp",&wparam(2));
  1059. &add ("esi","ebp");
  1060. &neg ("ebp"),
  1061. &sar ("ebp",31);
  1062. &mov ("edi",&wparam(0));
  1063. &lea ("esi",&DWP(0,"esi","ebp"));
  1064. for($i=0;$i<64;$i+=4) {
  1065. &movz ("eax",&BP(64*($i+0),"esi"));
  1066. &movz ("ebx",&BP(64*($i+1),"esi"));
  1067. &movz ("ecx",&BP(64*($i+2),"esi"));
  1068. &and ("eax","ebp");
  1069. &movz ("edx",&BP(64*($i+3),"esi"));
  1070. &and ("ebx","ebp");
  1071. &mov (&BP($i+0,"edi"),"al");
  1072. &and ("ecx","ebp");
  1073. &mov (&BP($i+1,"edi"),"bl");
  1074. &and ("edx","ebp");
  1075. &mov (&BP($i+2,"edi"),"cl");
  1076. &mov (&BP($i+3,"edi"),"dl");
  1077. }
  1078. &function_end("ecp_nistz256_gather_w7");
  1079. ########################################################################
  1080. # following subroutines are "literal" implementation of those found in
  1081. # ecp_nistz256.c
  1082. #
  1083. ########################################################################
  1084. # void ecp_nistz256_point_double(P256_POINT *out,const P256_POINT *inp);
  1085. #
  1086. &static_label("point_double_shortcut");
  1087. &function_begin("ecp_nistz256_point_double");
  1088. { my ($S,$M,$Zsqr,$in_x,$tmp0)=map(32*$_,(0..4));
  1089. &mov ("esi",&wparam(1));
  1090. # above map() describes stack layout with 5 temporary
  1091. # 256-bit vectors on top, then we take extra word for
  1092. # OPENSSL_ia32cap_P copy.
  1093. &stack_push(8*5+1);
  1094. if ($sse2) {
  1095. &call ("_picup_eax");
  1096. &set_label("pic");
  1097. &picmeup("edx","OPENSSL_ia32cap_P","eax",&label("pic"));
  1098. &mov ("ebp",&DWP(0,"edx")); }
  1099. &set_label("point_double_shortcut");
  1100. &mov ("eax",&DWP(0,"esi")); # copy in_x
  1101. &mov ("ebx",&DWP(4,"esi"));
  1102. &mov ("ecx",&DWP(8,"esi"));
  1103. &mov ("edx",&DWP(12,"esi"));
  1104. &mov (&DWP($in_x+0,"esp"),"eax");
  1105. &mov (&DWP($in_x+4,"esp"),"ebx");
  1106. &mov (&DWP($in_x+8,"esp"),"ecx");
  1107. &mov (&DWP($in_x+12,"esp"),"edx");
  1108. &mov ("eax",&DWP(16,"esi"));
  1109. &mov ("ebx",&DWP(20,"esi"));
  1110. &mov ("ecx",&DWP(24,"esi"));
  1111. &mov ("edx",&DWP(28,"esi"));
  1112. &mov (&DWP($in_x+16,"esp"),"eax");
  1113. &mov (&DWP($in_x+20,"esp"),"ebx");
  1114. &mov (&DWP($in_x+24,"esp"),"ecx");
  1115. &mov (&DWP($in_x+28,"esp"),"edx");
  1116. &mov (&DWP(32*5,"esp"),"ebp"); # OPENSSL_ia32cap_P copy
  1117. &lea ("ebp",&DWP(32,"esi"));
  1118. &lea ("esi",&DWP(32,"esi"));
  1119. &lea ("edi",&DWP($S,"esp"));
  1120. &call ("_ecp_nistz256_add"); # p256_mul_by_2(S, in_y);
  1121. &mov ("eax",&DWP(32*5,"esp")); # OPENSSL_ia32cap_P copy
  1122. &mov ("esi",64);
  1123. &add ("esi",&wparam(1));
  1124. &lea ("edi",&DWP($Zsqr,"esp"));
  1125. &mov ("ebp","esi");
  1126. &call ("_ecp_nistz256_mul_mont"); # p256_sqr_mont(Zsqr, in_z);
  1127. &mov ("eax",&DWP(32*5,"esp")); # OPENSSL_ia32cap_P copy
  1128. &lea ("esi",&DWP($S,"esp"));
  1129. &lea ("ebp",&DWP($S,"esp"));
  1130. &lea ("edi",&DWP($S,"esp"));
  1131. &call ("_ecp_nistz256_mul_mont"); # p256_sqr_mont(S, S);
  1132. &mov ("eax",&DWP(32*5,"esp")); # OPENSSL_ia32cap_P copy
  1133. &mov ("ebp",&wparam(1));
  1134. &lea ("esi",&DWP(32,"ebp"));
  1135. &lea ("ebp",&DWP(64,"ebp"));
  1136. &lea ("edi",&DWP($tmp0,"esp"));
  1137. &call ("_ecp_nistz256_mul_mont"); # p256_mul_mont(tmp0, in_z, in_y);
  1138. &lea ("esi",&DWP($in_x,"esp"));
  1139. &lea ("ebp",&DWP($Zsqr,"esp"));
  1140. &lea ("edi",&DWP($M,"esp"));
  1141. &call ("_ecp_nistz256_add"); # p256_add(M, in_x, Zsqr);
  1142. &mov ("edi",64);
  1143. &lea ("esi",&DWP($tmp0,"esp"));
  1144. &lea ("ebp",&DWP($tmp0,"esp"));
  1145. &add ("edi",&wparam(0));
  1146. &call ("_ecp_nistz256_add"); # p256_mul_by_2(res_z, tmp0);
  1147. &lea ("esi",&DWP($in_x,"esp"));
  1148. &lea ("ebp",&DWP($Zsqr,"esp"));
  1149. &lea ("edi",&DWP($Zsqr,"esp"));
  1150. &call ("_ecp_nistz256_sub"); # p256_sub(Zsqr, in_x, Zsqr);
  1151. &mov ("eax",&DWP(32*5,"esp")); # OPENSSL_ia32cap_P copy
  1152. &lea ("esi",&DWP($S,"esp"));
  1153. &lea ("ebp",&DWP($S,"esp"));
  1154. &lea ("edi",&DWP($tmp0,"esp"));
  1155. &call ("_ecp_nistz256_mul_mont"); # p256_sqr_mont(tmp0, S);
  1156. &mov ("eax",&DWP(32*5,"esp")); # OPENSSL_ia32cap_P copy
  1157. &lea ("esi",&DWP($M,"esp"));
  1158. &lea ("ebp",&DWP($Zsqr,"esp"));
  1159. &lea ("edi",&DWP($M,"esp"));
  1160. &call ("_ecp_nistz256_mul_mont"); # p256_mul_mont(M, M, Zsqr);
  1161. &mov ("edi",32);
  1162. &lea ("esi",&DWP($tmp0,"esp"));
  1163. &add ("edi",&wparam(0));
  1164. &call ("_ecp_nistz256_div_by_2"); # p256_div_by_2(res_y, tmp0);
  1165. &lea ("esi",&DWP($M,"esp"));
  1166. &lea ("ebp",&DWP($M,"esp"));
  1167. &lea ("edi",&DWP($tmp0,"esp"));
  1168. &call ("_ecp_nistz256_add"); # 1/2 p256_mul_by_3(M, M);
  1169. &mov ("eax",&DWP(32*5,"esp")); # OPENSSL_ia32cap_P copy
  1170. &lea ("esi",&DWP($in_x,"esp"));
  1171. &lea ("ebp",&DWP($S,"esp"));
  1172. &lea ("edi",&DWP($S,"esp"));
  1173. &call ("_ecp_nistz256_mul_mont"); # p256_mul_mont(S, S, in_x);
  1174. &lea ("esi",&DWP($tmp0,"esp"));
  1175. &lea ("ebp",&DWP($M,"esp"));
  1176. &lea ("edi",&DWP($M,"esp"));
  1177. &call ("_ecp_nistz256_add"); # 2/2 p256_mul_by_3(M, M);
  1178. &lea ("esi",&DWP($S,"esp"));
  1179. &lea ("ebp",&DWP($S,"esp"));
  1180. &lea ("edi",&DWP($tmp0,"esp"));
  1181. &call ("_ecp_nistz256_add"); # p256_mul_by_2(tmp0, S);
  1182. &mov ("eax",&DWP(32*5,"esp")); # OPENSSL_ia32cap_P copy
  1183. &lea ("esi",&DWP($M,"esp"));
  1184. &lea ("ebp",&DWP($M,"esp"));
  1185. &mov ("edi",&wparam(0));
  1186. &call ("_ecp_nistz256_mul_mont"); # p256_sqr_mont(res_x, M);
  1187. &mov ("esi","edi"); # %edi is still res_x here
  1188. &lea ("ebp",&DWP($tmp0,"esp"));
  1189. &call ("_ecp_nistz256_sub"); # p256_sub(res_x, res_x, tmp0);
  1190. &lea ("esi",&DWP($S,"esp"));
  1191. &mov ("ebp","edi"); # %edi is still res_x
  1192. &lea ("edi",&DWP($S,"esp"));
  1193. &call ("_ecp_nistz256_sub"); # p256_sub(S, S, res_x);
  1194. &mov ("eax",&DWP(32*5,"esp")); # OPENSSL_ia32cap_P copy
  1195. &mov ("esi","edi"); # %edi is still &S
  1196. &lea ("ebp",&DWP($M,"esp"));
  1197. &call ("_ecp_nistz256_mul_mont"); # p256_mul_mont(S, S, M);
  1198. &mov ("ebp",32);
  1199. &lea ("esi",&DWP($S,"esp"));
  1200. &add ("ebp",&wparam(0));
  1201. &mov ("edi","ebp");
  1202. &call ("_ecp_nistz256_sub"); # p256_sub(res_y, S, res_y);
  1203. &stack_pop(8*5+1);
  1204. } &function_end("ecp_nistz256_point_double");
  1205. ########################################################################
  1206. # void ecp_nistz256_point_add(P256_POINT *out,const P256_POINT *in1,
  1207. # const P256_POINT *in2);
  1208. &function_begin("ecp_nistz256_point_add");
  1209. { my ($res_x,$res_y,$res_z,
  1210. $in1_x,$in1_y,$in1_z,
  1211. $in2_x,$in2_y,$in2_z,
  1212. $H,$Hsqr,$R,$Rsqr,$Hcub,
  1213. $U1,$U2,$S1,$S2)=map(32*$_,(0..17));
  1214. my ($Z1sqr, $Z2sqr) = ($Hsqr, $Rsqr);
  1215. &mov ("esi",&wparam(2));
  1216. # above map() describes stack layout with 18 temporary
  1217. # 256-bit vectors on top, then we take extra words for
  1218. # ~in1infty, ~in2infty, result of check for zero and
  1219. # OPENSSL_ia32cap_P copy. [one unused word for padding]
  1220. &stack_push(8*18+5);
  1221. if ($sse2) {
  1222. &call ("_picup_eax");
  1223. &set_label("pic");
  1224. &picmeup("edx","OPENSSL_ia32cap_P","eax",&label("pic"));
  1225. &mov ("ebp",&DWP(0,"edx")); }
  1226. &lea ("edi",&DWP($in2_x,"esp"));
  1227. for($i=0;$i<96;$i+=16) {
  1228. &mov ("eax",&DWP($i+0,"esi")); # copy in2
  1229. &mov ("ebx",&DWP($i+4,"esi"));
  1230. &mov ("ecx",&DWP($i+8,"esi"));
  1231. &mov ("edx",&DWP($i+12,"esi"));
  1232. &mov (&DWP($i+0,"edi"),"eax");
  1233. &mov (&DWP(32*18+12,"esp"),"ebp") if ($i==0);
  1234. &mov ("ebp","eax") if ($i==64);
  1235. &or ("ebp","eax") if ($i>64);
  1236. &mov (&DWP($i+4,"edi"),"ebx");
  1237. &or ("ebp","ebx") if ($i>=64);
  1238. &mov (&DWP($i+8,"edi"),"ecx");
  1239. &or ("ebp","ecx") if ($i>=64);
  1240. &mov (&DWP($i+12,"edi"),"edx");
  1241. &or ("ebp","edx") if ($i>=64);
  1242. }
  1243. &xor ("eax","eax");
  1244. &mov ("esi",&wparam(1));
  1245. &sub ("eax","ebp");
  1246. &or ("ebp","eax");
  1247. &sar ("ebp",31);
  1248. &mov (&DWP(32*18+4,"esp"),"ebp"); # ~in2infty
  1249. &lea ("edi",&DWP($in1_x,"esp"));
  1250. for($i=0;$i<96;$i+=16) {
  1251. &mov ("eax",&DWP($i+0,"esi")); # copy in1
  1252. &mov ("ebx",&DWP($i+4,"esi"));
  1253. &mov ("ecx",&DWP($i+8,"esi"));
  1254. &mov ("edx",&DWP($i+12,"esi"));
  1255. &mov (&DWP($i+0,"edi"),"eax");
  1256. &mov ("ebp","eax") if ($i==64);
  1257. &or ("ebp","eax") if ($i>64);
  1258. &mov (&DWP($i+4,"edi"),"ebx");
  1259. &or ("ebp","ebx") if ($i>=64);
  1260. &mov (&DWP($i+8,"edi"),"ecx");
  1261. &or ("ebp","ecx") if ($i>=64);
  1262. &mov (&DWP($i+12,"edi"),"edx");
  1263. &or ("ebp","edx") if ($i>=64);
  1264. }
  1265. &xor ("eax","eax");
  1266. &sub ("eax","ebp");
  1267. &or ("ebp","eax");
  1268. &sar ("ebp",31);
  1269. &mov (&DWP(32*18+0,"esp"),"ebp"); # ~in1infty
  1270. &mov ("eax",&DWP(32*18+12,"esp")); # OPENSSL_ia32cap_P copy
  1271. &lea ("esi",&DWP($in2_z,"esp"));
  1272. &lea ("ebp",&DWP($in2_z,"esp"));
  1273. &lea ("edi",&DWP($Z2sqr,"esp"));
  1274. &call ("_ecp_nistz256_mul_mont"); # p256_sqr_mont(Z2sqr, in2_z);
  1275. &mov ("eax",&DWP(32*18+12,"esp")); # OPENSSL_ia32cap_P copy
  1276. &lea ("esi",&DWP($in1_z,"esp"));
  1277. &lea ("ebp",&DWP($in1_z,"esp"));
  1278. &lea ("edi",&DWP($Z1sqr,"esp"));
  1279. &call ("_ecp_nistz256_mul_mont"); # p256_sqr_mont(Z1sqr, in1_z);
  1280. &mov ("eax",&DWP(32*18+12,"esp")); # OPENSSL_ia32cap_P copy
  1281. &lea ("esi",&DWP($Z2sqr,"esp"));
  1282. &lea ("ebp",&DWP($in2_z,"esp"));
  1283. &lea ("edi",&DWP($S1,"esp"));
  1284. &call ("_ecp_nistz256_mul_mont"); # p256_mul_mont(S1, Z2sqr, in2_z);
  1285. &mov ("eax",&DWP(32*18+12,"esp")); # OPENSSL_ia32cap_P copy
  1286. &lea ("esi",&DWP($Z1sqr,"esp"));
  1287. &lea ("ebp",&DWP($in1_z,"esp"));
  1288. &lea ("edi",&DWP($S2,"esp"));
  1289. &call ("_ecp_nistz256_mul_mont"); # p256_mul_mont(S2, Z1sqr, in1_z);
  1290. &mov ("eax",&DWP(32*18+12,"esp")); # OPENSSL_ia32cap_P copy
  1291. &lea ("esi",&DWP($in1_y,"esp"));
  1292. &lea ("ebp",&DWP($S1,"esp"));
  1293. &lea ("edi",&DWP($S1,"esp"));
  1294. &call ("_ecp_nistz256_mul_mont"); # p256_mul_mont(S1, S1, in1_y);
  1295. &mov ("eax",&DWP(32*18+12,"esp")); # OPENSSL_ia32cap_P copy
  1296. &lea ("esi",&DWP($in2_y,"esp"));
  1297. &lea ("ebp",&DWP($S2,"esp"));
  1298. &lea ("edi",&DWP($S2,"esp"));
  1299. &call ("_ecp_nistz256_mul_mont"); # p256_mul_mont(S2, S2, in2_y);
  1300. &lea ("esi",&DWP($S2,"esp"));
  1301. &lea ("ebp",&DWP($S1,"esp"));
  1302. &lea ("edi",&DWP($R,"esp"));
  1303. &call ("_ecp_nistz256_sub"); # p256_sub(R, S2, S1);
  1304. &or ("ebx","eax"); # see if result is zero
  1305. &mov ("eax",&DWP(32*18+12,"esp")); # OPENSSL_ia32cap_P copy
  1306. &or ("ebx","ecx");
  1307. &or ("ebx","edx");
  1308. &or ("ebx",&DWP(0,"edi"));
  1309. &or ("ebx",&DWP(4,"edi"));
  1310. &lea ("esi",&DWP($in1_x,"esp"));
  1311. &or ("ebx",&DWP(8,"edi"));
  1312. &lea ("ebp",&DWP($Z2sqr,"esp"));
  1313. &or ("ebx",&DWP(12,"edi"));
  1314. &lea ("edi",&DWP($U1,"esp"));
  1315. &mov (&DWP(32*18+8,"esp"),"ebx");
  1316. &call ("_ecp_nistz256_mul_mont"); # p256_mul_mont(U1, in1_x, Z2sqr);
  1317. &mov ("eax",&DWP(32*18+12,"esp")); # OPENSSL_ia32cap_P copy
  1318. &lea ("esi",&DWP($in2_x,"esp"));
  1319. &lea ("ebp",&DWP($Z1sqr,"esp"));
  1320. &lea ("edi",&DWP($U2,"esp"));
  1321. &call ("_ecp_nistz256_mul_mont"); # p256_mul_mont(U2, in2_x, Z1sqr);
  1322. &lea ("esi",&DWP($U2,"esp"));
  1323. &lea ("ebp",&DWP($U1,"esp"));
  1324. &lea ("edi",&DWP($H,"esp"));
  1325. &call ("_ecp_nistz256_sub"); # p256_sub(H, U2, U1);
  1326. &or ("eax","ebx"); # see if result is zero
  1327. &or ("eax","ecx");
  1328. &or ("eax","edx");
  1329. &or ("eax",&DWP(0,"edi"));
  1330. &or ("eax",&DWP(4,"edi"));
  1331. &or ("eax",&DWP(8,"edi"));
  1332. &or ("eax",&DWP(12,"edi")); # ~is_equal(U1,U2)
  1333. &mov ("ebx",&DWP(32*18+0,"esp")); # ~in1infty
  1334. &not ("ebx"); # -1/0 -> 0/-1
  1335. &or ("eax","ebx");
  1336. &mov ("ebx",&DWP(32*18+4,"esp")); # ~in2infty
  1337. &not ("ebx"); # -1/0 -> 0/-1
  1338. &or ("eax","ebx");
  1339. &or ("eax",&DWP(32*18+8,"esp")); # ~is_equal(S1,S2)
  1340. # if (~is_equal(U1,U2) | in1infty | in2infty | ~is_equal(S1,S2))
  1341. &data_byte(0x3e); # predict taken
  1342. &jnz (&label("add_proceed"));
  1343. &set_label("add_double",16);
  1344. &mov ("esi",&wparam(1));
  1345. &mov ("ebp",&DWP(32*18+12,"esp")); # OPENSSL_ia32cap_P copy
  1346. &add ("esp",4*((8*18+5)-(8*5+1))); # difference in frame sizes
  1347. &jmp (&label("point_double_shortcut"));
  1348. &set_label("add_proceed",16);
  1349. &mov ("eax",&DWP(32*18+12,"esp")); # OPENSSL_ia32cap_P copy
  1350. &lea ("esi",&DWP($R,"esp"));
  1351. &lea ("ebp",&DWP($R,"esp"));
  1352. &lea ("edi",&DWP($Rsqr,"esp"));
  1353. &call ("_ecp_nistz256_mul_mont"); # p256_sqr_mont(Rsqr, R);
  1354. &mov ("eax",&DWP(32*18+12,"esp")); # OPENSSL_ia32cap_P copy
  1355. &lea ("esi",&DWP($H,"esp"));
  1356. &lea ("ebp",&DWP($in1_z,"esp"));
  1357. &lea ("edi",&DWP($res_z,"esp"));
  1358. &call ("_ecp_nistz256_mul_mont"); # p256_mul_mont(res_z, H, in1_z);
  1359. &mov ("eax",&DWP(32*18+12,"esp")); # OPENSSL_ia32cap_P copy
  1360. &lea ("esi",&DWP($H,"esp"));
  1361. &lea ("ebp",&DWP($H,"esp"));
  1362. &lea ("edi",&DWP($Hsqr,"esp"));
  1363. &call ("_ecp_nistz256_mul_mont"); # p256_sqr_mont(Hsqr, H);
  1364. &mov ("eax",&DWP(32*18+12,"esp")); # OPENSSL_ia32cap_P copy
  1365. &lea ("esi",&DWP($in2_z,"esp"));
  1366. &lea ("ebp",&DWP($res_z,"esp"));
  1367. &lea ("edi",&DWP($res_z,"esp"));
  1368. &call ("_ecp_nistz256_mul_mont"); # p256_mul_mont(res_z, res_z, in2_z);
  1369. &mov ("eax",&DWP(32*18+12,"esp")); # OPENSSL_ia32cap_P copy
  1370. &lea ("esi",&DWP($Hsqr,"esp"));
  1371. &lea ("ebp",&DWP($U1,"esp"));
  1372. &lea ("edi",&DWP($U2,"esp"));
  1373. &call ("_ecp_nistz256_mul_mont"); # p256_mul_mont(U2, U1, Hsqr);
  1374. &mov ("eax",&DWP(32*18+12,"esp")); # OPENSSL_ia32cap_P copy
  1375. &lea ("esi",&DWP($H,"esp"));
  1376. &lea ("ebp",&DWP($Hsqr,"esp"));
  1377. &lea ("edi",&DWP($Hcub,"esp"));
  1378. &call ("_ecp_nistz256_mul_mont"); # p256_mul_mont(Hcub, Hsqr, H);
  1379. &lea ("esi",&DWP($U2,"esp"));
  1380. &lea ("ebp",&DWP($U2,"esp"));
  1381. &lea ("edi",&DWP($Hsqr,"esp"));
  1382. &call ("_ecp_nistz256_add"); # p256_mul_by_2(Hsqr, U2);
  1383. &lea ("esi",&DWP($Rsqr,"esp"));
  1384. &lea ("ebp",&DWP($Hsqr,"esp"));
  1385. &lea ("edi",&DWP($res_x,"esp"));
  1386. &call ("_ecp_nistz256_sub"); # p256_sub(res_x, Rsqr, Hsqr);
  1387. &lea ("esi",&DWP($res_x,"esp"));
  1388. &lea ("ebp",&DWP($Hcub,"esp"));
  1389. &lea ("edi",&DWP($res_x,"esp"));
  1390. &call ("_ecp_nistz256_sub"); # p256_sub(res_x, res_x, Hcub);
  1391. &lea ("esi",&DWP($U2,"esp"));
  1392. &lea ("ebp",&DWP($res_x,"esp"));
  1393. &lea ("edi",&DWP($res_y,"esp"));
  1394. &call ("_ecp_nistz256_sub"); # p256_sub(res_y, U2, res_x);
  1395. &mov ("eax",&DWP(32*18+12,"esp")); # OPENSSL_ia32cap_P copy
  1396. &lea ("esi",&DWP($Hcub,"esp"));
  1397. &lea ("ebp",&DWP($S1,"esp"));
  1398. &lea ("edi",&DWP($S2,"esp"));
  1399. &call ("_ecp_nistz256_mul_mont"); # p256_mul_mont(S2, S1, Hcub);
  1400. &mov ("eax",&DWP(32*18+12,"esp")); # OPENSSL_ia32cap_P copy
  1401. &lea ("esi",&DWP($R,"esp"));
  1402. &lea ("ebp",&DWP($res_y,"esp"));
  1403. &lea ("edi",&DWP($res_y,"esp"));
  1404. &call ("_ecp_nistz256_mul_mont"); # p256_mul_mont(res_y, R, res_y);
  1405. &lea ("esi",&DWP($res_y,"esp"));
  1406. &lea ("ebp",&DWP($S2,"esp"));
  1407. &lea ("edi",&DWP($res_y,"esp"));
  1408. &call ("_ecp_nistz256_sub"); # p256_sub(res_y, res_y, S2);
  1409. &mov ("ebp",&DWP(32*18+0,"esp")); # ~in1infty
  1410. &mov ("esi",&DWP(32*18+4,"esp")); # ~in2infty
  1411. &mov ("edi",&wparam(0));
  1412. &mov ("edx","ebp");
  1413. &not ("ebp");
  1414. &and ("edx","esi"); # ~in1infty & ~in2infty
  1415. &and ("ebp","esi"); # in1infty & ~in2infty
  1416. &not ("esi"); # in2infty
  1417. ########################################
  1418. # conditional moves
  1419. for($i=64;$i<96;$i+=4) {
  1420. &mov ("eax","edx"); # ~in1infty & ~in2infty
  1421. &and ("eax",&DWP($res_x+$i,"esp"));
  1422. &mov ("ebx","ebp"); # in1infty & ~in2infty
  1423. &and ("ebx",&DWP($in2_x+$i,"esp"));
  1424. &mov ("ecx","esi"); # in2infty
  1425. &and ("ecx",&DWP($in1_x+$i,"esp"));
  1426. &or ("eax","ebx");
  1427. &or ("eax","ecx");
  1428. &mov (&DWP($i,"edi"),"eax");
  1429. }
  1430. for($i=0;$i<64;$i+=4) {
  1431. &mov ("eax","edx"); # ~in1infty & ~in2infty
  1432. &and ("eax",&DWP($res_x+$i,"esp"));
  1433. &mov ("ebx","ebp"); # in1infty & ~in2infty
  1434. &and ("ebx",&DWP($in2_x+$i,"esp"));
  1435. &mov ("ecx","esi"); # in2infty
  1436. &and ("ecx",&DWP($in1_x+$i,"esp"));
  1437. &or ("eax","ebx");
  1438. &or ("eax","ecx");
  1439. &mov (&DWP($i,"edi"),"eax");
  1440. }
  1441. &set_label("add_done");
  1442. &stack_pop(8*18+5);
  1443. } &function_end("ecp_nistz256_point_add");
  1444. ########################################################################
  1445. # void ecp_nistz256_point_add_affine(P256_POINT *out,
  1446. # const P256_POINT *in1,
  1447. # const P256_POINT_AFFINE *in2);
  1448. &function_begin("ecp_nistz256_point_add_affine");
  1449. {
  1450. my ($res_x,$res_y,$res_z,
  1451. $in1_x,$in1_y,$in1_z,
  1452. $in2_x,$in2_y,
  1453. $U2,$S2,$H,$R,$Hsqr,$Hcub,$Rsqr)=map(32*$_,(0..14));
  1454. my $Z1sqr = $S2;
  1455. my @ONE_mont=(1,0,0,-1,-1,-1,-2,0);
  1456. &mov ("esi",&wparam(1));
  1457. # above map() describes stack layout with 15 temporary
  1458. # 256-bit vectors on top, then we take extra words for
  1459. # ~in1infty, ~in2infty, and OPENSSL_ia32cap_P copy.
  1460. &stack_push(8*15+3);
  1461. if ($sse2) {
  1462. &call ("_picup_eax");
  1463. &set_label("pic");
  1464. &picmeup("edx","OPENSSL_ia32cap_P","eax",&label("pic"));
  1465. &mov ("ebp",&DWP(0,"edx")); }
  1466. &lea ("edi",&DWP($in1_x,"esp"));
  1467. for($i=0;$i<96;$i+=16) {
  1468. &mov ("eax",&DWP($i+0,"esi")); # copy in1
  1469. &mov ("ebx",&DWP($i+4,"esi"));
  1470. &mov ("ecx",&DWP($i+8,"esi"));
  1471. &mov ("edx",&DWP($i+12,"esi"));
  1472. &mov (&DWP($i+0,"edi"),"eax");
  1473. &mov (&DWP(32*15+8,"esp"),"ebp") if ($i==0);
  1474. &mov ("ebp","eax") if ($i==64);
  1475. &or ("ebp","eax") if ($i>64);
  1476. &mov (&DWP($i+4,"edi"),"ebx");
  1477. &or ("ebp","ebx") if ($i>=64);
  1478. &mov (&DWP($i+8,"edi"),"ecx");
  1479. &or ("ebp","ecx") if ($i>=64);
  1480. &mov (&DWP($i+12,"edi"),"edx");
  1481. &or ("ebp","edx") if ($i>=64);
  1482. }
  1483. &xor ("eax","eax");
  1484. &mov ("esi",&wparam(2));
  1485. &sub ("eax","ebp");
  1486. &or ("ebp","eax");
  1487. &sar ("ebp",31);
  1488. &mov (&DWP(32*15+0,"esp"),"ebp"); # ~in1infty
  1489. &lea ("edi",&DWP($in2_x,"esp"));
  1490. for($i=0;$i<64;$i+=16) {
  1491. &mov ("eax",&DWP($i+0,"esi")); # copy in2
  1492. &mov ("ebx",&DWP($i+4,"esi"));
  1493. &mov ("ecx",&DWP($i+8,"esi"));
  1494. &mov ("edx",&DWP($i+12,"esi"));
  1495. &mov (&DWP($i+0,"edi"),"eax");
  1496. &mov ("ebp","eax") if ($i==0);
  1497. &or ("ebp","eax") if ($i!=0);
  1498. &mov (&DWP($i+4,"edi"),"ebx");
  1499. &or ("ebp","ebx");
  1500. &mov (&DWP($i+8,"edi"),"ecx");
  1501. &or ("ebp","ecx");
  1502. &mov (&DWP($i+12,"edi"),"edx");
  1503. &or ("ebp","edx");
  1504. }
  1505. &xor ("ebx","ebx");
  1506. &mov ("eax",&DWP(32*15+8,"esp")); # OPENSSL_ia32cap_P copy
  1507. &sub ("ebx","ebp");
  1508. &lea ("esi",&DWP($in1_z,"esp"));
  1509. &or ("ebx","ebp");
  1510. &lea ("ebp",&DWP($in1_z,"esp"));
  1511. &sar ("ebx",31);
  1512. &lea ("edi",&DWP($Z1sqr,"esp"));
  1513. &mov (&DWP(32*15+4,"esp"),"ebx"); # ~in2infty
  1514. &call ("_ecp_nistz256_mul_mont"); # p256_sqr_mont(Z1sqr, in1_z);
  1515. &mov ("eax",&DWP(32*15+8,"esp")); # OPENSSL_ia32cap_P copy
  1516. &lea ("esi",&DWP($in2_x,"esp"));
  1517. &mov ("ebp","edi"); # %esi is stull &Z1sqr
  1518. &lea ("edi",&DWP($U2,"esp"));
  1519. &call ("_ecp_nistz256_mul_mont"); # p256_mul_mont(U2, Z1sqr, in2_x);
  1520. &mov ("eax",&DWP(32*15+8,"esp")); # OPENSSL_ia32cap_P copy
  1521. &lea ("esi",&DWP($in1_z,"esp"));
  1522. &lea ("ebp",&DWP($Z1sqr,"esp"));
  1523. &lea ("edi",&DWP($S2,"esp"));
  1524. &call ("_ecp_nistz256_mul_mont"); # p256_mul_mont(S2, Z1sqr, in1_z);
  1525. &lea ("esi",&DWP($U2,"esp"));
  1526. &lea ("ebp",&DWP($in1_x,"esp"));
  1527. &lea ("edi",&DWP($H,"esp"));
  1528. &call ("_ecp_nistz256_sub"); # p256_sub(H, U2, in1_x);
  1529. &mov ("eax",&DWP(32*15+8,"esp")); # OPENSSL_ia32cap_P copy
  1530. &lea ("esi",&DWP($in2_y,"esp"));
  1531. &lea ("ebp",&DWP($S2,"esp"));
  1532. &lea ("edi",&DWP($S2,"esp"));
  1533. &call ("_ecp_nistz256_mul_mont"); # p256_mul_mont(S2, S2, in2_y);
  1534. &mov ("eax",&DWP(32*15+8,"esp")); # OPENSSL_ia32cap_P copy
  1535. &lea ("esi",&DWP($in1_z,"esp"));
  1536. &lea ("ebp",&DWP($H,"esp"));
  1537. &lea ("edi",&DWP($res_z,"esp"));
  1538. &call ("_ecp_nistz256_mul_mont"); # p256_mul_mont(res_z, H, in1_z);
  1539. &lea ("esi",&DWP($S2,"esp"));
  1540. &lea ("ebp",&DWP($in1_y,"esp"));
  1541. &lea ("edi",&DWP($R,"esp"));
  1542. &call ("_ecp_nistz256_sub"); # p256_sub(R, S2, in1_y);
  1543. &mov ("eax",&DWP(32*15+8,"esp")); # OPENSSL_ia32cap_P copy
  1544. &lea ("esi",&DWP($H,"esp"));
  1545. &lea ("ebp",&DWP($H,"esp"));
  1546. &lea ("edi",&DWP($Hsqr,"esp"));
  1547. &call ("_ecp_nistz256_mul_mont"); # p256_sqr_mont(Hsqr, H);
  1548. &mov ("eax",&DWP(32*15+8,"esp")); # OPENSSL_ia32cap_P copy
  1549. &lea ("esi",&DWP($R,"esp"));
  1550. &lea ("ebp",&DWP($R,"esp"));
  1551. &lea ("edi",&DWP($Rsqr,"esp"));
  1552. &call ("_ecp_nistz256_mul_mont"); # p256_sqr_mont(Rsqr, R);
  1553. &mov ("eax",&DWP(32*15+8,"esp")); # OPENSSL_ia32cap_P copy
  1554. &lea ("esi",&DWP($in1_x,"esp"));
  1555. &lea ("ebp",&DWP($Hsqr,"esp"));
  1556. &lea ("edi",&DWP($U2,"esp"));
  1557. &call ("_ecp_nistz256_mul_mont"); # p256_mul_mont(U2, in1_x, Hsqr);
  1558. &mov ("eax",&DWP(32*15+8,"esp")); # OPENSSL_ia32cap_P copy
  1559. &lea ("esi",&DWP($H,"esp"));
  1560. &lea ("ebp",&DWP($Hsqr,"esp"));
  1561. &lea ("edi",&DWP($Hcub,"esp"));
  1562. &call ("_ecp_nistz256_mul_mont"); # p256_mul_mont(Hcub, Hsqr, H);
  1563. &lea ("esi",&DWP($U2,"esp"));
  1564. &lea ("ebp",&DWP($U2,"esp"));
  1565. &lea ("edi",&DWP($Hsqr,"esp"));
  1566. &call ("_ecp_nistz256_add"); # p256_mul_by_2(Hsqr, U2);
  1567. &lea ("esi",&DWP($Rsqr,"esp"));
  1568. &lea ("ebp",&DWP($Hsqr,"esp"));
  1569. &lea ("edi",&DWP($res_x,"esp"));
  1570. &call ("_ecp_nistz256_sub"); # p256_sub(res_x, Rsqr, Hsqr);
  1571. &lea ("esi",&DWP($res_x,"esp"));
  1572. &lea ("ebp",&DWP($Hcub,"esp"));
  1573. &lea ("edi",&DWP($res_x,"esp"));
  1574. &call ("_ecp_nistz256_sub"); # p256_sub(res_x, res_x, Hcub);
  1575. &lea ("esi",&DWP($U2,"esp"));
  1576. &lea ("ebp",&DWP($res_x,"esp"));
  1577. &lea ("edi",&DWP($res_y,"esp"));
  1578. &call ("_ecp_nistz256_sub"); # p256_sub(res_y, U2, res_x);
  1579. &mov ("eax",&DWP(32*15+8,"esp")); # OPENSSL_ia32cap_P copy
  1580. &lea ("esi",&DWP($Hcub,"esp"));
  1581. &lea ("ebp",&DWP($in1_y,"esp"));
  1582. &lea ("edi",&DWP($S2,"esp"));
  1583. &call ("_ecp_nistz256_mul_mont"); # p256_mul_mont(S2, Hcub, in1_y);
  1584. &mov ("eax",&DWP(32*15+8,"esp")); # OPENSSL_ia32cap_P copy
  1585. &lea ("esi",&DWP($R,"esp"));
  1586. &lea ("ebp",&DWP($res_y,"esp"));
  1587. &lea ("edi",&DWP($res_y,"esp"));
  1588. &call ("_ecp_nistz256_mul_mont"); # p256_mul_mont(res_y, res_y, R);
  1589. &lea ("esi",&DWP($res_y,"esp"));
  1590. &lea ("ebp",&DWP($S2,"esp"));
  1591. &lea ("edi",&DWP($res_y,"esp"));
  1592. &call ("_ecp_nistz256_sub"); # p256_sub(res_y, res_y, S2);
  1593. &mov ("ebp",&DWP(32*15+0,"esp")); # ~in1infty
  1594. &mov ("esi",&DWP(32*15+4,"esp")); # ~in2infty
  1595. &mov ("edi",&wparam(0));
  1596. &mov ("edx","ebp");
  1597. &not ("ebp");
  1598. &and ("edx","esi"); # ~in1infty & ~in2infty
  1599. &and ("ebp","esi"); # in1infty & ~in2infty
  1600. &not ("esi"); # in2infty
  1601. ########################################
  1602. # conditional moves
  1603. for($i=64;$i<96;$i+=4) {
  1604. my $one=@ONE_mont[($i-64)/4];
  1605. &mov ("eax","edx");
  1606. &and ("eax",&DWP($res_x+$i,"esp"));
  1607. &mov ("ebx","ebp") if ($one && $one!=-1);
  1608. &and ("ebx",$one) if ($one && $one!=-1);
  1609. &mov ("ecx","esi");
  1610. &and ("ecx",&DWP($in1_x+$i,"esp"));
  1611. &or ("eax",$one==-1?"ebp":"ebx") if ($one);
  1612. &or ("eax","ecx");
  1613. &mov (&DWP($i,"edi"),"eax");
  1614. }
  1615. for($i=0;$i<64;$i+=4) {
  1616. &mov ("eax","edx"); # ~in1infty & ~in2infty
  1617. &and ("eax",&DWP($res_x+$i,"esp"));
  1618. &mov ("ebx","ebp"); # in1infty & ~in2infty
  1619. &and ("ebx",&DWP($in2_x+$i,"esp"));
  1620. &mov ("ecx","esi"); # in2infty
  1621. &and ("ecx",&DWP($in1_x+$i,"esp"));
  1622. &or ("eax","ebx");
  1623. &or ("eax","ecx");
  1624. &mov (&DWP($i,"edi"),"eax");
  1625. }
  1626. &stack_pop(8*15+3);
  1627. } &function_end("ecp_nistz256_point_add_affine");
  1628. &asm_finish();
  1629. close STDOUT or die "error closing STDOUT: $!";