ecp_nistz256-armv8.pl 46 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877
  1. #! /usr/bin/env perl
  2. # Copyright 2015-2018 The OpenSSL Project Authors. All Rights Reserved.
  3. #
  4. # Licensed under the Apache License 2.0 (the "License"). You may not use
  5. # this file except in compliance with the License. You can obtain a copy
  6. # in the file LICENSE in the source distribution or at
  7. # https://www.openssl.org/source/license.html
  8. # ====================================================================
  9. # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
  10. # project. The module is, however, dual licensed under OpenSSL and
  11. # CRYPTOGAMS licenses depending on where you obtain it. For further
  12. # details see http://www.openssl.org/~appro/cryptogams/.
  13. # ====================================================================
  14. #
  15. # ECP_NISTZ256 module for ARMv8.
  16. #
  17. # February 2015.
  18. #
  19. # Original ECP_NISTZ256 submission targeting x86_64 is detailed in
  20. # http://eprint.iacr.org/2013/816.
  21. #
  22. # with/without -DECP_NISTZ256_ASM
  23. # Apple A7 +190-360%
  24. # Cortex-A53 +190-400%
  25. # Cortex-A57 +190-350%
  26. # Denver +230-400%
  27. #
  28. # Ranges denote minimum and maximum improvement coefficients depending
  29. # on benchmark. Lower coefficients are for ECDSA sign, server-side
  30. # operation. Keep in mind that +400% means 5x improvement.
  31. # $output is the last argument if it looks like a file (it has an extension)
  32. # $flavour is the first argument if it doesn't look like a file
  33. $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
  34. $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
  35. $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
  36. ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
  37. ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
  38. die "can't locate arm-xlate.pl";
  39. open OUT,"| \"$^X\" $xlate $flavour \"$output\""
  40. or die "can't call $xlate: $!";
  41. *STDOUT=*OUT;
  42. {
  43. my ($rp,$ap,$bp,$bi,$a0,$a1,$a2,$a3,$t0,$t1,$t2,$t3,$poly1,$poly3,
  44. $acc0,$acc1,$acc2,$acc3,$acc4,$acc5) =
  45. map("x$_",(0..17,19,20));
  46. my ($acc6,$acc7)=($ap,$bp); # used in __ecp_nistz256_sqr_mont
  47. $code.=<<___;
  48. #include "arm_arch.h"
  49. .text
  50. ___
  51. ########################################################################
  52. # Convert ecp_nistz256_table.c to layout expected by ecp_nistz_gather_w7
  53. #
  54. $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
  55. open TABLE,"<ecp_nistz256_table.c" or
  56. open TABLE,"<${dir}../ecp_nistz256_table.c" or
  57. die "failed to open ecp_nistz256_table.c:",$!;
  58. use integer;
  59. foreach(<TABLE>) {
  60. s/TOBN\(\s*(0x[0-9a-f]+),\s*(0x[0-9a-f]+)\s*\)/push @arr,hex($2),hex($1)/geo;
  61. }
  62. close TABLE;
  63. # See ecp_nistz256_table.c for explanation for why it's 64*16*37.
  64. # 64*16*37-1 is because $#arr returns last valid index or @arr, not
  65. # amount of elements.
  66. die "insane number of elements" if ($#arr != 64*16*37-1);
  67. $code.=<<___;
  68. .globl ecp_nistz256_precomputed
  69. .type ecp_nistz256_precomputed,%object
  70. .align 12
  71. ecp_nistz256_precomputed:
  72. ___
  73. ########################################################################
  74. # this conversion smashes P256_POINT_AFFINE by individual bytes with
  75. # 64 byte interval, similar to
  76. # 1111222233334444
  77. # 1234123412341234
  78. for(1..37) {
  79. @tbl = splice(@arr,0,64*16);
  80. for($i=0;$i<64;$i++) {
  81. undef @line;
  82. for($j=0;$j<64;$j++) {
  83. push @line,(@tbl[$j*16+$i/4]>>(($i%4)*8))&0xff;
  84. }
  85. $code.=".byte\t";
  86. $code.=join(',',map { sprintf "0x%02x",$_} @line);
  87. $code.="\n";
  88. }
  89. }
  90. $code.=<<___;
  91. .size ecp_nistz256_precomputed,.-ecp_nistz256_precomputed
  92. .align 5
  93. .Lpoly:
  94. .quad 0xffffffffffffffff,0x00000000ffffffff,0x0000000000000000,0xffffffff00000001
  95. .LRR: // 2^512 mod P precomputed for NIST P256 polynomial
  96. .quad 0x0000000000000003,0xfffffffbffffffff,0xfffffffffffffffe,0x00000004fffffffd
  97. .Lone_mont:
  98. .quad 0x0000000000000001,0xffffffff00000000,0xffffffffffffffff,0x00000000fffffffe
  99. .Lone:
  100. .quad 1,0,0,0
  101. .Lord:
  102. .quad 0xf3b9cac2fc632551,0xbce6faada7179e84,0xffffffffffffffff,0xffffffff00000000
  103. .LordK:
  104. .quad 0xccd1c8aaee00bc4f
  105. .asciz "ECP_NISTZ256 for ARMv8, CRYPTOGAMS by <appro\@openssl.org>"
  106. // void ecp_nistz256_to_mont(BN_ULONG x0[4],const BN_ULONG x1[4]);
  107. .globl ecp_nistz256_to_mont
  108. .type ecp_nistz256_to_mont,%function
  109. .align 6
  110. ecp_nistz256_to_mont:
  111. .inst 0xd503233f // paciasp
  112. stp x29,x30,[sp,#-32]!
  113. add x29,sp,#0
  114. stp x19,x20,[sp,#16]
  115. ldr $bi,.LRR // bp[0]
  116. ldp $a0,$a1,[$ap]
  117. ldp $a2,$a3,[$ap,#16]
  118. ldr $poly1,.Lpoly+8
  119. ldr $poly3,.Lpoly+24
  120. adr $bp,.LRR // &bp[0]
  121. bl __ecp_nistz256_mul_mont
  122. ldp x19,x20,[sp,#16]
  123. ldp x29,x30,[sp],#32
  124. .inst 0xd50323bf // autiasp
  125. ret
  126. .size ecp_nistz256_to_mont,.-ecp_nistz256_to_mont
  127. // void ecp_nistz256_from_mont(BN_ULONG x0[4],const BN_ULONG x1[4]);
  128. .globl ecp_nistz256_from_mont
  129. .type ecp_nistz256_from_mont,%function
  130. .align 4
  131. ecp_nistz256_from_mont:
  132. .inst 0xd503233f // paciasp
  133. stp x29,x30,[sp,#-32]!
  134. add x29,sp,#0
  135. stp x19,x20,[sp,#16]
  136. mov $bi,#1 // bp[0]
  137. ldp $a0,$a1,[$ap]
  138. ldp $a2,$a3,[$ap,#16]
  139. ldr $poly1,.Lpoly+8
  140. ldr $poly3,.Lpoly+24
  141. adr $bp,.Lone // &bp[0]
  142. bl __ecp_nistz256_mul_mont
  143. ldp x19,x20,[sp,#16]
  144. ldp x29,x30,[sp],#32
  145. .inst 0xd50323bf // autiasp
  146. ret
  147. .size ecp_nistz256_from_mont,.-ecp_nistz256_from_mont
  148. // void ecp_nistz256_mul_mont(BN_ULONG x0[4],const BN_ULONG x1[4],
  149. // const BN_ULONG x2[4]);
  150. .globl ecp_nistz256_mul_mont
  151. .type ecp_nistz256_mul_mont,%function
  152. .align 4
  153. ecp_nistz256_mul_mont:
  154. .inst 0xd503233f // paciasp
  155. stp x29,x30,[sp,#-32]!
  156. add x29,sp,#0
  157. stp x19,x20,[sp,#16]
  158. ldr $bi,[$bp] // bp[0]
  159. ldp $a0,$a1,[$ap]
  160. ldp $a2,$a3,[$ap,#16]
  161. ldr $poly1,.Lpoly+8
  162. ldr $poly3,.Lpoly+24
  163. bl __ecp_nistz256_mul_mont
  164. ldp x19,x20,[sp,#16]
  165. ldp x29,x30,[sp],#32
  166. .inst 0xd50323bf // autiasp
  167. ret
  168. .size ecp_nistz256_mul_mont,.-ecp_nistz256_mul_mont
  169. // void ecp_nistz256_sqr_mont(BN_ULONG x0[4],const BN_ULONG x1[4]);
  170. .globl ecp_nistz256_sqr_mont
  171. .type ecp_nistz256_sqr_mont,%function
  172. .align 4
  173. ecp_nistz256_sqr_mont:
  174. .inst 0xd503233f // paciasp
  175. stp x29,x30,[sp,#-32]!
  176. add x29,sp,#0
  177. stp x19,x20,[sp,#16]
  178. ldp $a0,$a1,[$ap]
  179. ldp $a2,$a3,[$ap,#16]
  180. ldr $poly1,.Lpoly+8
  181. ldr $poly3,.Lpoly+24
  182. bl __ecp_nistz256_sqr_mont
  183. ldp x19,x20,[sp,#16]
  184. ldp x29,x30,[sp],#32
  185. .inst 0xd50323bf // autiasp
  186. ret
  187. .size ecp_nistz256_sqr_mont,.-ecp_nistz256_sqr_mont
  188. // void ecp_nistz256_add(BN_ULONG x0[4],const BN_ULONG x1[4],
  189. // const BN_ULONG x2[4]);
  190. .globl ecp_nistz256_add
  191. .type ecp_nistz256_add,%function
  192. .align 4
  193. ecp_nistz256_add:
  194. .inst 0xd503233f // paciasp
  195. stp x29,x30,[sp,#-16]!
  196. add x29,sp,#0
  197. ldp $acc0,$acc1,[$ap]
  198. ldp $t0,$t1,[$bp]
  199. ldp $acc2,$acc3,[$ap,#16]
  200. ldp $t2,$t3,[$bp,#16]
  201. ldr $poly1,.Lpoly+8
  202. ldr $poly3,.Lpoly+24
  203. bl __ecp_nistz256_add
  204. ldp x29,x30,[sp],#16
  205. .inst 0xd50323bf // autiasp
  206. ret
  207. .size ecp_nistz256_add,.-ecp_nistz256_add
  208. // void ecp_nistz256_div_by_2(BN_ULONG x0[4],const BN_ULONG x1[4]);
  209. .globl ecp_nistz256_div_by_2
  210. .type ecp_nistz256_div_by_2,%function
  211. .align 4
  212. ecp_nistz256_div_by_2:
  213. .inst 0xd503233f // paciasp
  214. stp x29,x30,[sp,#-16]!
  215. add x29,sp,#0
  216. ldp $acc0,$acc1,[$ap]
  217. ldp $acc2,$acc3,[$ap,#16]
  218. ldr $poly1,.Lpoly+8
  219. ldr $poly3,.Lpoly+24
  220. bl __ecp_nistz256_div_by_2
  221. ldp x29,x30,[sp],#16
  222. .inst 0xd50323bf // autiasp
  223. ret
  224. .size ecp_nistz256_div_by_2,.-ecp_nistz256_div_by_2
  225. // void ecp_nistz256_mul_by_2(BN_ULONG x0[4],const BN_ULONG x1[4]);
  226. .globl ecp_nistz256_mul_by_2
  227. .type ecp_nistz256_mul_by_2,%function
  228. .align 4
  229. ecp_nistz256_mul_by_2:
  230. .inst 0xd503233f // paciasp
  231. stp x29,x30,[sp,#-16]!
  232. add x29,sp,#0
  233. ldp $acc0,$acc1,[$ap]
  234. ldp $acc2,$acc3,[$ap,#16]
  235. ldr $poly1,.Lpoly+8
  236. ldr $poly3,.Lpoly+24
  237. mov $t0,$acc0
  238. mov $t1,$acc1
  239. mov $t2,$acc2
  240. mov $t3,$acc3
  241. bl __ecp_nistz256_add // ret = a+a // 2*a
  242. ldp x29,x30,[sp],#16
  243. .inst 0xd50323bf // autiasp
  244. ret
  245. .size ecp_nistz256_mul_by_2,.-ecp_nistz256_mul_by_2
  246. // void ecp_nistz256_mul_by_3(BN_ULONG x0[4],const BN_ULONG x1[4]);
  247. .globl ecp_nistz256_mul_by_3
  248. .type ecp_nistz256_mul_by_3,%function
  249. .align 4
  250. ecp_nistz256_mul_by_3:
  251. .inst 0xd503233f // paciasp
  252. stp x29,x30,[sp,#-16]!
  253. add x29,sp,#0
  254. ldp $acc0,$acc1,[$ap]
  255. ldp $acc2,$acc3,[$ap,#16]
  256. ldr $poly1,.Lpoly+8
  257. ldr $poly3,.Lpoly+24
  258. mov $t0,$acc0
  259. mov $t1,$acc1
  260. mov $t2,$acc2
  261. mov $t3,$acc3
  262. mov $a0,$acc0
  263. mov $a1,$acc1
  264. mov $a2,$acc2
  265. mov $a3,$acc3
  266. bl __ecp_nistz256_add // ret = a+a // 2*a
  267. mov $t0,$a0
  268. mov $t1,$a1
  269. mov $t2,$a2
  270. mov $t3,$a3
  271. bl __ecp_nistz256_add // ret += a // 2*a+a=3*a
  272. ldp x29,x30,[sp],#16
  273. .inst 0xd50323bf // autiasp
  274. ret
  275. .size ecp_nistz256_mul_by_3,.-ecp_nistz256_mul_by_3
  276. // void ecp_nistz256_sub(BN_ULONG x0[4],const BN_ULONG x1[4],
  277. // const BN_ULONG x2[4]);
  278. .globl ecp_nistz256_sub
  279. .type ecp_nistz256_sub,%function
  280. .align 4
  281. ecp_nistz256_sub:
  282. .inst 0xd503233f // paciasp
  283. stp x29,x30,[sp,#-16]!
  284. add x29,sp,#0
  285. ldp $acc0,$acc1,[$ap]
  286. ldp $acc2,$acc3,[$ap,#16]
  287. ldr $poly1,.Lpoly+8
  288. ldr $poly3,.Lpoly+24
  289. bl __ecp_nistz256_sub_from
  290. ldp x29,x30,[sp],#16
  291. .inst 0xd50323bf // autiasp
  292. ret
  293. .size ecp_nistz256_sub,.-ecp_nistz256_sub
  294. // void ecp_nistz256_neg(BN_ULONG x0[4],const BN_ULONG x1[4]);
  295. .globl ecp_nistz256_neg
  296. .type ecp_nistz256_neg,%function
  297. .align 4
  298. ecp_nistz256_neg:
  299. .inst 0xd503233f // paciasp
  300. stp x29,x30,[sp,#-16]!
  301. add x29,sp,#0
  302. mov $bp,$ap
  303. mov $acc0,xzr // a = 0
  304. mov $acc1,xzr
  305. mov $acc2,xzr
  306. mov $acc3,xzr
  307. ldr $poly1,.Lpoly+8
  308. ldr $poly3,.Lpoly+24
  309. bl __ecp_nistz256_sub_from
  310. ldp x29,x30,[sp],#16
  311. .inst 0xd50323bf // autiasp
  312. ret
  313. .size ecp_nistz256_neg,.-ecp_nistz256_neg
  314. // note that __ecp_nistz256_mul_mont expects a[0-3] input pre-loaded
  315. // to $a0-$a3 and b[0] - to $bi
  316. .type __ecp_nistz256_mul_mont,%function
  317. .align 4
  318. __ecp_nistz256_mul_mont:
  319. mul $acc0,$a0,$bi // a[0]*b[0]
  320. umulh $t0,$a0,$bi
  321. mul $acc1,$a1,$bi // a[1]*b[0]
  322. umulh $t1,$a1,$bi
  323. mul $acc2,$a2,$bi // a[2]*b[0]
  324. umulh $t2,$a2,$bi
  325. mul $acc3,$a3,$bi // a[3]*b[0]
  326. umulh $t3,$a3,$bi
  327. ldr $bi,[$bp,#8] // b[1]
  328. adds $acc1,$acc1,$t0 // accumulate high parts of multiplication
  329. lsl $t0,$acc0,#32
  330. adcs $acc2,$acc2,$t1
  331. lsr $t1,$acc0,#32
  332. adcs $acc3,$acc3,$t2
  333. adc $acc4,xzr,$t3
  334. mov $acc5,xzr
  335. ___
  336. for($i=1;$i<4;$i++) {
  337. # Reduction iteration is normally performed by accumulating
  338. # result of multiplication of modulus by "magic" digit [and
  339. # omitting least significant word, which is guaranteed to
  340. # be 0], but thanks to special form of modulus and "magic"
  341. # digit being equal to least significant word, it can be
  342. # performed with additions and subtractions alone. Indeed:
  343. #
  344. # ffff0001.00000000.0000ffff.ffffffff
  345. # * abcdefgh
  346. # + xxxxxxxx.xxxxxxxx.xxxxxxxx.xxxxxxxx.abcdefgh
  347. #
  348. # Now observing that ff..ff*x = (2^n-1)*x = 2^n*x-x, we
  349. # rewrite above as:
  350. #
  351. # xxxxxxxx.xxxxxxxx.xxxxxxxx.xxxxxxxx.abcdefgh
  352. # + abcdefgh.abcdefgh.0000abcd.efgh0000.00000000
  353. # - 0000abcd.efgh0000.00000000.00000000.abcdefgh
  354. #
  355. # or marking redundant operations:
  356. #
  357. # xxxxxxxx.xxxxxxxx.xxxxxxxx.xxxxxxxx.--------
  358. # + abcdefgh.abcdefgh.0000abcd.efgh0000.--------
  359. # - 0000abcd.efgh0000.--------.--------.--------
  360. $code.=<<___;
  361. subs $t2,$acc0,$t0 // "*0xffff0001"
  362. sbc $t3,$acc0,$t1
  363. adds $acc0,$acc1,$t0 // +=acc[0]<<96 and omit acc[0]
  364. mul $t0,$a0,$bi // lo(a[0]*b[i])
  365. adcs $acc1,$acc2,$t1
  366. mul $t1,$a1,$bi // lo(a[1]*b[i])
  367. adcs $acc2,$acc3,$t2 // +=acc[0]*0xffff0001
  368. mul $t2,$a2,$bi // lo(a[2]*b[i])
  369. adcs $acc3,$acc4,$t3
  370. mul $t3,$a3,$bi // lo(a[3]*b[i])
  371. adc $acc4,$acc5,xzr
  372. adds $acc0,$acc0,$t0 // accumulate low parts of multiplication
  373. umulh $t0,$a0,$bi // hi(a[0]*b[i])
  374. adcs $acc1,$acc1,$t1
  375. umulh $t1,$a1,$bi // hi(a[1]*b[i])
  376. adcs $acc2,$acc2,$t2
  377. umulh $t2,$a2,$bi // hi(a[2]*b[i])
  378. adcs $acc3,$acc3,$t3
  379. umulh $t3,$a3,$bi // hi(a[3]*b[i])
  380. adc $acc4,$acc4,xzr
  381. ___
  382. $code.=<<___ if ($i<3);
  383. ldr $bi,[$bp,#8*($i+1)] // b[$i+1]
  384. ___
  385. $code.=<<___;
  386. adds $acc1,$acc1,$t0 // accumulate high parts of multiplication
  387. lsl $t0,$acc0,#32
  388. adcs $acc2,$acc2,$t1
  389. lsr $t1,$acc0,#32
  390. adcs $acc3,$acc3,$t2
  391. adcs $acc4,$acc4,$t3
  392. adc $acc5,xzr,xzr
  393. ___
  394. }
  395. $code.=<<___;
  396. // last reduction
  397. subs $t2,$acc0,$t0 // "*0xffff0001"
  398. sbc $t3,$acc0,$t1
  399. adds $acc0,$acc1,$t0 // +=acc[0]<<96 and omit acc[0]
  400. adcs $acc1,$acc2,$t1
  401. adcs $acc2,$acc3,$t2 // +=acc[0]*0xffff0001
  402. adcs $acc3,$acc4,$t3
  403. adc $acc4,$acc5,xzr
  404. adds $t0,$acc0,#1 // subs $t0,$acc0,#-1 // tmp = ret-modulus
  405. sbcs $t1,$acc1,$poly1
  406. sbcs $t2,$acc2,xzr
  407. sbcs $t3,$acc3,$poly3
  408. sbcs xzr,$acc4,xzr // did it borrow?
  409. csel $acc0,$acc0,$t0,lo // ret = borrow ? ret : ret-modulus
  410. csel $acc1,$acc1,$t1,lo
  411. csel $acc2,$acc2,$t2,lo
  412. stp $acc0,$acc1,[$rp]
  413. csel $acc3,$acc3,$t3,lo
  414. stp $acc2,$acc3,[$rp,#16]
  415. ret
  416. .size __ecp_nistz256_mul_mont,.-__ecp_nistz256_mul_mont
  417. // note that __ecp_nistz256_sqr_mont expects a[0-3] input pre-loaded
  418. // to $a0-$a3
  419. .type __ecp_nistz256_sqr_mont,%function
  420. .align 4
  421. __ecp_nistz256_sqr_mont:
  422. // | | | | | |a1*a0| |
  423. // | | | | |a2*a0| | |
  424. // | |a3*a2|a3*a0| | | |
  425. // | | | |a2*a1| | | |
  426. // | | |a3*a1| | | | |
  427. // *| | | | | | | | 2|
  428. // +|a3*a3|a2*a2|a1*a1|a0*a0|
  429. // |--+--+--+--+--+--+--+--|
  430. // |A7|A6|A5|A4|A3|A2|A1|A0|, where Ax is $accx, i.e. follow $accx
  431. //
  432. // "can't overflow" below mark carrying into high part of
  433. // multiplication result, which can't overflow, because it
  434. // can never be all ones.
  435. mul $acc1,$a1,$a0 // a[1]*a[0]
  436. umulh $t1,$a1,$a0
  437. mul $acc2,$a2,$a0 // a[2]*a[0]
  438. umulh $t2,$a2,$a0
  439. mul $acc3,$a3,$a0 // a[3]*a[0]
  440. umulh $acc4,$a3,$a0
  441. adds $acc2,$acc2,$t1 // accumulate high parts of multiplication
  442. mul $t0,$a2,$a1 // a[2]*a[1]
  443. umulh $t1,$a2,$a1
  444. adcs $acc3,$acc3,$t2
  445. mul $t2,$a3,$a1 // a[3]*a[1]
  446. umulh $t3,$a3,$a1
  447. adc $acc4,$acc4,xzr // can't overflow
  448. mul $acc5,$a3,$a2 // a[3]*a[2]
  449. umulh $acc6,$a3,$a2
  450. adds $t1,$t1,$t2 // accumulate high parts of multiplication
  451. mul $acc0,$a0,$a0 // a[0]*a[0]
  452. adc $t2,$t3,xzr // can't overflow
  453. adds $acc3,$acc3,$t0 // accumulate low parts of multiplication
  454. umulh $a0,$a0,$a0
  455. adcs $acc4,$acc4,$t1
  456. mul $t1,$a1,$a1 // a[1]*a[1]
  457. adcs $acc5,$acc5,$t2
  458. umulh $a1,$a1,$a1
  459. adc $acc6,$acc6,xzr // can't overflow
  460. adds $acc1,$acc1,$acc1 // acc[1-6]*=2
  461. mul $t2,$a2,$a2 // a[2]*a[2]
  462. adcs $acc2,$acc2,$acc2
  463. umulh $a2,$a2,$a2
  464. adcs $acc3,$acc3,$acc3
  465. mul $t3,$a3,$a3 // a[3]*a[3]
  466. adcs $acc4,$acc4,$acc4
  467. umulh $a3,$a3,$a3
  468. adcs $acc5,$acc5,$acc5
  469. adcs $acc6,$acc6,$acc6
  470. adc $acc7,xzr,xzr
  471. adds $acc1,$acc1,$a0 // +a[i]*a[i]
  472. adcs $acc2,$acc2,$t1
  473. adcs $acc3,$acc3,$a1
  474. adcs $acc4,$acc4,$t2
  475. adcs $acc5,$acc5,$a2
  476. lsl $t0,$acc0,#32
  477. adcs $acc6,$acc6,$t3
  478. lsr $t1,$acc0,#32
  479. adc $acc7,$acc7,$a3
  480. ___
  481. for($i=0;$i<3;$i++) { # reductions, see commentary in
  482. # multiplication for details
  483. $code.=<<___;
  484. subs $t2,$acc0,$t0 // "*0xffff0001"
  485. sbc $t3,$acc0,$t1
  486. adds $acc0,$acc1,$t0 // +=acc[0]<<96 and omit acc[0]
  487. adcs $acc1,$acc2,$t1
  488. lsl $t0,$acc0,#32
  489. adcs $acc2,$acc3,$t2 // +=acc[0]*0xffff0001
  490. lsr $t1,$acc0,#32
  491. adc $acc3,$t3,xzr // can't overflow
  492. ___
  493. }
  494. $code.=<<___;
  495. subs $t2,$acc0,$t0 // "*0xffff0001"
  496. sbc $t3,$acc0,$t1
  497. adds $acc0,$acc1,$t0 // +=acc[0]<<96 and omit acc[0]
  498. adcs $acc1,$acc2,$t1
  499. adcs $acc2,$acc3,$t2 // +=acc[0]*0xffff0001
  500. adc $acc3,$t3,xzr // can't overflow
  501. adds $acc0,$acc0,$acc4 // accumulate upper half
  502. adcs $acc1,$acc1,$acc5
  503. adcs $acc2,$acc2,$acc6
  504. adcs $acc3,$acc3,$acc7
  505. adc $acc4,xzr,xzr
  506. adds $t0,$acc0,#1 // subs $t0,$acc0,#-1 // tmp = ret-modulus
  507. sbcs $t1,$acc1,$poly1
  508. sbcs $t2,$acc2,xzr
  509. sbcs $t3,$acc3,$poly3
  510. sbcs xzr,$acc4,xzr // did it borrow?
  511. csel $acc0,$acc0,$t0,lo // ret = borrow ? ret : ret-modulus
  512. csel $acc1,$acc1,$t1,lo
  513. csel $acc2,$acc2,$t2,lo
  514. stp $acc0,$acc1,[$rp]
  515. csel $acc3,$acc3,$t3,lo
  516. stp $acc2,$acc3,[$rp,#16]
  517. ret
  518. .size __ecp_nistz256_sqr_mont,.-__ecp_nistz256_sqr_mont
  519. // Note that __ecp_nistz256_add expects both input vectors pre-loaded to
  520. // $a0-$a3 and $t0-$t3. This is done because it's used in multiple
  521. // contexts, e.g. in multiplication by 2 and 3...
  522. .type __ecp_nistz256_add,%function
  523. .align 4
  524. __ecp_nistz256_add:
  525. adds $acc0,$acc0,$t0 // ret = a+b
  526. adcs $acc1,$acc1,$t1
  527. adcs $acc2,$acc2,$t2
  528. adcs $acc3,$acc3,$t3
  529. adc $ap,xzr,xzr // zap $ap
  530. adds $t0,$acc0,#1 // subs $t0,$a0,#-1 // tmp = ret-modulus
  531. sbcs $t1,$acc1,$poly1
  532. sbcs $t2,$acc2,xzr
  533. sbcs $t3,$acc3,$poly3
  534. sbcs xzr,$ap,xzr // did subtraction borrow?
  535. csel $acc0,$acc0,$t0,lo // ret = borrow ? ret : ret-modulus
  536. csel $acc1,$acc1,$t1,lo
  537. csel $acc2,$acc2,$t2,lo
  538. stp $acc0,$acc1,[$rp]
  539. csel $acc3,$acc3,$t3,lo
  540. stp $acc2,$acc3,[$rp,#16]
  541. ret
  542. .size __ecp_nistz256_add,.-__ecp_nistz256_add
  543. .type __ecp_nistz256_sub_from,%function
  544. .align 4
  545. __ecp_nistz256_sub_from:
  546. ldp $t0,$t1,[$bp]
  547. ldp $t2,$t3,[$bp,#16]
  548. subs $acc0,$acc0,$t0 // ret = a-b
  549. sbcs $acc1,$acc1,$t1
  550. sbcs $acc2,$acc2,$t2
  551. sbcs $acc3,$acc3,$t3
  552. sbc $ap,xzr,xzr // zap $ap
  553. subs $t0,$acc0,#1 // adds $t0,$a0,#-1 // tmp = ret+modulus
  554. adcs $t1,$acc1,$poly1
  555. adcs $t2,$acc2,xzr
  556. adc $t3,$acc3,$poly3
  557. cmp $ap,xzr // did subtraction borrow?
  558. csel $acc0,$acc0,$t0,eq // ret = borrow ? ret+modulus : ret
  559. csel $acc1,$acc1,$t1,eq
  560. csel $acc2,$acc2,$t2,eq
  561. stp $acc0,$acc1,[$rp]
  562. csel $acc3,$acc3,$t3,eq
  563. stp $acc2,$acc3,[$rp,#16]
  564. ret
  565. .size __ecp_nistz256_sub_from,.-__ecp_nistz256_sub_from
  566. .type __ecp_nistz256_sub_morf,%function
  567. .align 4
  568. __ecp_nistz256_sub_morf:
  569. ldp $t0,$t1,[$bp]
  570. ldp $t2,$t3,[$bp,#16]
  571. subs $acc0,$t0,$acc0 // ret = b-a
  572. sbcs $acc1,$t1,$acc1
  573. sbcs $acc2,$t2,$acc2
  574. sbcs $acc3,$t3,$acc3
  575. sbc $ap,xzr,xzr // zap $ap
  576. subs $t0,$acc0,#1 // adds $t0,$a0,#-1 // tmp = ret+modulus
  577. adcs $t1,$acc1,$poly1
  578. adcs $t2,$acc2,xzr
  579. adc $t3,$acc3,$poly3
  580. cmp $ap,xzr // did subtraction borrow?
  581. csel $acc0,$acc0,$t0,eq // ret = borrow ? ret+modulus : ret
  582. csel $acc1,$acc1,$t1,eq
  583. csel $acc2,$acc2,$t2,eq
  584. stp $acc0,$acc1,[$rp]
  585. csel $acc3,$acc3,$t3,eq
  586. stp $acc2,$acc3,[$rp,#16]
  587. ret
  588. .size __ecp_nistz256_sub_morf,.-__ecp_nistz256_sub_morf
  589. .type __ecp_nistz256_div_by_2,%function
  590. .align 4
  591. __ecp_nistz256_div_by_2:
  592. subs $t0,$acc0,#1 // adds $t0,$a0,#-1 // tmp = a+modulus
  593. adcs $t1,$acc1,$poly1
  594. adcs $t2,$acc2,xzr
  595. adcs $t3,$acc3,$poly3
  596. adc $ap,xzr,xzr // zap $ap
  597. tst $acc0,#1 // is a even?
  598. csel $acc0,$acc0,$t0,eq // ret = even ? a : a+modulus
  599. csel $acc1,$acc1,$t1,eq
  600. csel $acc2,$acc2,$t2,eq
  601. csel $acc3,$acc3,$t3,eq
  602. csel $ap,xzr,$ap,eq
  603. lsr $acc0,$acc0,#1 // ret >>= 1
  604. orr $acc0,$acc0,$acc1,lsl#63
  605. lsr $acc1,$acc1,#1
  606. orr $acc1,$acc1,$acc2,lsl#63
  607. lsr $acc2,$acc2,#1
  608. orr $acc2,$acc2,$acc3,lsl#63
  609. lsr $acc3,$acc3,#1
  610. stp $acc0,$acc1,[$rp]
  611. orr $acc3,$acc3,$ap,lsl#63
  612. stp $acc2,$acc3,[$rp,#16]
  613. ret
  614. .size __ecp_nistz256_div_by_2,.-__ecp_nistz256_div_by_2
  615. ___
  616. ########################################################################
  617. # following subroutines are "literal" implementation of those found in
  618. # ecp_nistz256.c
  619. #
  620. ########################################################################
  621. # void ecp_nistz256_point_double(P256_POINT *out,const P256_POINT *inp);
  622. #
  623. {
  624. my ($S,$M,$Zsqr,$tmp0)=map(32*$_,(0..3));
  625. # above map() describes stack layout with 4 temporary
  626. # 256-bit vectors on top.
  627. my ($rp_real,$ap_real) = map("x$_",(21,22));
  628. $code.=<<___;
  629. .globl ecp_nistz256_point_double
  630. .type ecp_nistz256_point_double,%function
  631. .align 5
  632. ecp_nistz256_point_double:
  633. .inst 0xd503233f // paciasp
  634. stp x29,x30,[sp,#-96]!
  635. add x29,sp,#0
  636. stp x19,x20,[sp,#16]
  637. stp x21,x22,[sp,#32]
  638. sub sp,sp,#32*4
  639. .Ldouble_shortcut:
  640. ldp $acc0,$acc1,[$ap,#32]
  641. mov $rp_real,$rp
  642. ldp $acc2,$acc3,[$ap,#48]
  643. mov $ap_real,$ap
  644. ldr $poly1,.Lpoly+8
  645. mov $t0,$acc0
  646. ldr $poly3,.Lpoly+24
  647. mov $t1,$acc1
  648. ldp $a0,$a1,[$ap_real,#64] // forward load for p256_sqr_mont
  649. mov $t2,$acc2
  650. mov $t3,$acc3
  651. ldp $a2,$a3,[$ap_real,#64+16]
  652. add $rp,sp,#$S
  653. bl __ecp_nistz256_add // p256_mul_by_2(S, in_y);
  654. add $rp,sp,#$Zsqr
  655. bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Zsqr, in_z);
  656. ldp $t0,$t1,[$ap_real]
  657. ldp $t2,$t3,[$ap_real,#16]
  658. mov $a0,$acc0 // put Zsqr aside for p256_sub
  659. mov $a1,$acc1
  660. mov $a2,$acc2
  661. mov $a3,$acc3
  662. add $rp,sp,#$M
  663. bl __ecp_nistz256_add // p256_add(M, Zsqr, in_x);
  664. add $bp,$ap_real,#0
  665. mov $acc0,$a0 // restore Zsqr
  666. mov $acc1,$a1
  667. ldp $a0,$a1,[sp,#$S] // forward load for p256_sqr_mont
  668. mov $acc2,$a2
  669. mov $acc3,$a3
  670. ldp $a2,$a3,[sp,#$S+16]
  671. add $rp,sp,#$Zsqr
  672. bl __ecp_nistz256_sub_morf // p256_sub(Zsqr, in_x, Zsqr);
  673. add $rp,sp,#$S
  674. bl __ecp_nistz256_sqr_mont // p256_sqr_mont(S, S);
  675. ldr $bi,[$ap_real,#32]
  676. ldp $a0,$a1,[$ap_real,#64]
  677. ldp $a2,$a3,[$ap_real,#64+16]
  678. add $bp,$ap_real,#32
  679. add $rp,sp,#$tmp0
  680. bl __ecp_nistz256_mul_mont // p256_mul_mont(tmp0, in_z, in_y);
  681. mov $t0,$acc0
  682. mov $t1,$acc1
  683. ldp $a0,$a1,[sp,#$S] // forward load for p256_sqr_mont
  684. mov $t2,$acc2
  685. mov $t3,$acc3
  686. ldp $a2,$a3,[sp,#$S+16]
  687. add $rp,$rp_real,#64
  688. bl __ecp_nistz256_add // p256_mul_by_2(res_z, tmp0);
  689. add $rp,sp,#$tmp0
  690. bl __ecp_nistz256_sqr_mont // p256_sqr_mont(tmp0, S);
  691. ldr $bi,[sp,#$Zsqr] // forward load for p256_mul_mont
  692. ldp $a0,$a1,[sp,#$M]
  693. ldp $a2,$a3,[sp,#$M+16]
  694. add $rp,$rp_real,#32
  695. bl __ecp_nistz256_div_by_2 // p256_div_by_2(res_y, tmp0);
  696. add $bp,sp,#$Zsqr
  697. add $rp,sp,#$M
  698. bl __ecp_nistz256_mul_mont // p256_mul_mont(M, M, Zsqr);
  699. mov $t0,$acc0 // duplicate M
  700. mov $t1,$acc1
  701. mov $t2,$acc2
  702. mov $t3,$acc3
  703. mov $a0,$acc0 // put M aside
  704. mov $a1,$acc1
  705. mov $a2,$acc2
  706. mov $a3,$acc3
  707. add $rp,sp,#$M
  708. bl __ecp_nistz256_add
  709. mov $t0,$a0 // restore M
  710. mov $t1,$a1
  711. ldr $bi,[$ap_real] // forward load for p256_mul_mont
  712. mov $t2,$a2
  713. ldp $a0,$a1,[sp,#$S]
  714. mov $t3,$a3
  715. ldp $a2,$a3,[sp,#$S+16]
  716. bl __ecp_nistz256_add // p256_mul_by_3(M, M);
  717. add $bp,$ap_real,#0
  718. add $rp,sp,#$S
  719. bl __ecp_nistz256_mul_mont // p256_mul_mont(S, S, in_x);
  720. mov $t0,$acc0
  721. mov $t1,$acc1
  722. ldp $a0,$a1,[sp,#$M] // forward load for p256_sqr_mont
  723. mov $t2,$acc2
  724. mov $t3,$acc3
  725. ldp $a2,$a3,[sp,#$M+16]
  726. add $rp,sp,#$tmp0
  727. bl __ecp_nistz256_add // p256_mul_by_2(tmp0, S);
  728. add $rp,$rp_real,#0
  729. bl __ecp_nistz256_sqr_mont // p256_sqr_mont(res_x, M);
  730. add $bp,sp,#$tmp0
  731. bl __ecp_nistz256_sub_from // p256_sub(res_x, res_x, tmp0);
  732. add $bp,sp,#$S
  733. add $rp,sp,#$S
  734. bl __ecp_nistz256_sub_morf // p256_sub(S, S, res_x);
  735. ldr $bi,[sp,#$M]
  736. mov $a0,$acc0 // copy S
  737. mov $a1,$acc1
  738. mov $a2,$acc2
  739. mov $a3,$acc3
  740. add $bp,sp,#$M
  741. bl __ecp_nistz256_mul_mont // p256_mul_mont(S, S, M);
  742. add $bp,$rp_real,#32
  743. add $rp,$rp_real,#32
  744. bl __ecp_nistz256_sub_from // p256_sub(res_y, S, res_y);
  745. add sp,x29,#0 // destroy frame
  746. ldp x19,x20,[x29,#16]
  747. ldp x21,x22,[x29,#32]
  748. ldp x29,x30,[sp],#96
  749. .inst 0xd50323bf // autiasp
  750. ret
  751. .size ecp_nistz256_point_double,.-ecp_nistz256_point_double
  752. ___
  753. }
  754. ########################################################################
  755. # void ecp_nistz256_point_add(P256_POINT *out,const P256_POINT *in1,
  756. # const P256_POINT *in2);
  757. {
  758. my ($res_x,$res_y,$res_z,
  759. $H,$Hsqr,$R,$Rsqr,$Hcub,
  760. $U1,$U2,$S1,$S2)=map(32*$_,(0..11));
  761. my ($Z1sqr, $Z2sqr) = ($Hsqr, $Rsqr);
  762. # above map() describes stack layout with 12 temporary
  763. # 256-bit vectors on top.
  764. my ($rp_real,$ap_real,$bp_real,$in1infty,$in2infty,$temp0,$temp1,$temp2)=map("x$_",(21..28));
  765. $code.=<<___;
  766. .globl ecp_nistz256_point_add
  767. .type ecp_nistz256_point_add,%function
  768. .align 5
  769. ecp_nistz256_point_add:
  770. .inst 0xd503233f // paciasp
  771. stp x29,x30,[sp,#-96]!
  772. add x29,sp,#0
  773. stp x19,x20,[sp,#16]
  774. stp x21,x22,[sp,#32]
  775. stp x23,x24,[sp,#48]
  776. stp x25,x26,[sp,#64]
  777. stp x27,x28,[sp,#80]
  778. sub sp,sp,#32*12
  779. ldp $a0,$a1,[$bp,#64] // in2_z
  780. ldp $a2,$a3,[$bp,#64+16]
  781. mov $rp_real,$rp
  782. mov $ap_real,$ap
  783. mov $bp_real,$bp
  784. ldr $poly1,.Lpoly+8
  785. ldr $poly3,.Lpoly+24
  786. orr $t0,$a0,$a1
  787. orr $t2,$a2,$a3
  788. orr $in2infty,$t0,$t2
  789. cmp $in2infty,#0
  790. csetm $in2infty,ne // ~in2infty
  791. add $rp,sp,#$Z2sqr
  792. bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Z2sqr, in2_z);
  793. ldp $a0,$a1,[$ap_real,#64] // in1_z
  794. ldp $a2,$a3,[$ap_real,#64+16]
  795. orr $t0,$a0,$a1
  796. orr $t2,$a2,$a3
  797. orr $in1infty,$t0,$t2
  798. cmp $in1infty,#0
  799. csetm $in1infty,ne // ~in1infty
  800. add $rp,sp,#$Z1sqr
  801. bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Z1sqr, in1_z);
  802. ldr $bi,[$bp_real,#64]
  803. ldp $a0,$a1,[sp,#$Z2sqr]
  804. ldp $a2,$a3,[sp,#$Z2sqr+16]
  805. add $bp,$bp_real,#64
  806. add $rp,sp,#$S1
  807. bl __ecp_nistz256_mul_mont // p256_mul_mont(S1, Z2sqr, in2_z);
  808. ldr $bi,[$ap_real,#64]
  809. ldp $a0,$a1,[sp,#$Z1sqr]
  810. ldp $a2,$a3,[sp,#$Z1sqr+16]
  811. add $bp,$ap_real,#64
  812. add $rp,sp,#$S2
  813. bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, Z1sqr, in1_z);
  814. ldr $bi,[$ap_real,#32]
  815. ldp $a0,$a1,[sp,#$S1]
  816. ldp $a2,$a3,[sp,#$S1+16]
  817. add $bp,$ap_real,#32
  818. add $rp,sp,#$S1
  819. bl __ecp_nistz256_mul_mont // p256_mul_mont(S1, S1, in1_y);
  820. ldr $bi,[$bp_real,#32]
  821. ldp $a0,$a1,[sp,#$S2]
  822. ldp $a2,$a3,[sp,#$S2+16]
  823. add $bp,$bp_real,#32
  824. add $rp,sp,#$S2
  825. bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, S2, in2_y);
  826. add $bp,sp,#$S1
  827. ldr $bi,[sp,#$Z2sqr] // forward load for p256_mul_mont
  828. ldp $a0,$a1,[$ap_real]
  829. ldp $a2,$a3,[$ap_real,#16]
  830. add $rp,sp,#$R
  831. bl __ecp_nistz256_sub_from // p256_sub(R, S2, S1);
  832. orr $acc0,$acc0,$acc1 // see if result is zero
  833. orr $acc2,$acc2,$acc3
  834. orr $temp0,$acc0,$acc2 // ~is_equal(S1,S2)
  835. add $bp,sp,#$Z2sqr
  836. add $rp,sp,#$U1
  837. bl __ecp_nistz256_mul_mont // p256_mul_mont(U1, in1_x, Z2sqr);
  838. ldr $bi,[sp,#$Z1sqr]
  839. ldp $a0,$a1,[$bp_real]
  840. ldp $a2,$a3,[$bp_real,#16]
  841. add $bp,sp,#$Z1sqr
  842. add $rp,sp,#$U2
  843. bl __ecp_nistz256_mul_mont // p256_mul_mont(U2, in2_x, Z1sqr);
  844. add $bp,sp,#$U1
  845. ldp $a0,$a1,[sp,#$R] // forward load for p256_sqr_mont
  846. ldp $a2,$a3,[sp,#$R+16]
  847. add $rp,sp,#$H
  848. bl __ecp_nistz256_sub_from // p256_sub(H, U2, U1);
  849. orr $acc0,$acc0,$acc1 // see if result is zero
  850. orr $acc2,$acc2,$acc3
  851. orr $acc0,$acc0,$acc2 // ~is_equal(U1,U2)
  852. mvn $temp1,$in1infty // -1/0 -> 0/-1
  853. mvn $temp2,$in2infty // -1/0 -> 0/-1
  854. orr $acc0,$acc0,$temp1
  855. orr $acc0,$acc0,$temp2
  856. orr $acc0,$acc0,$temp0
  857. cbnz $acc0,.Ladd_proceed // if(~is_equal(U1,U2) | in1infty | in2infty | ~is_equal(S1,S2))
  858. .Ladd_double:
  859. mov $ap,$ap_real
  860. mov $rp,$rp_real
  861. ldp x23,x24,[x29,#48]
  862. ldp x25,x26,[x29,#64]
  863. ldp x27,x28,[x29,#80]
  864. add sp,sp,#32*(12-4) // difference in stack frames
  865. b .Ldouble_shortcut
  866. .align 4
  867. .Ladd_proceed:
  868. add $rp,sp,#$Rsqr
  869. bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Rsqr, R);
  870. ldr $bi,[$ap_real,#64]
  871. ldp $a0,$a1,[sp,#$H]
  872. ldp $a2,$a3,[sp,#$H+16]
  873. add $bp,$ap_real,#64
  874. add $rp,sp,#$res_z
  875. bl __ecp_nistz256_mul_mont // p256_mul_mont(res_z, H, in1_z);
  876. ldp $a0,$a1,[sp,#$H]
  877. ldp $a2,$a3,[sp,#$H+16]
  878. add $rp,sp,#$Hsqr
  879. bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Hsqr, H);
  880. ldr $bi,[$bp_real,#64]
  881. ldp $a0,$a1,[sp,#$res_z]
  882. ldp $a2,$a3,[sp,#$res_z+16]
  883. add $bp,$bp_real,#64
  884. add $rp,sp,#$res_z
  885. bl __ecp_nistz256_mul_mont // p256_mul_mont(res_z, res_z, in2_z);
  886. ldr $bi,[sp,#$H]
  887. ldp $a0,$a1,[sp,#$Hsqr]
  888. ldp $a2,$a3,[sp,#$Hsqr+16]
  889. add $bp,sp,#$H
  890. add $rp,sp,#$Hcub
  891. bl __ecp_nistz256_mul_mont // p256_mul_mont(Hcub, Hsqr, H);
  892. ldr $bi,[sp,#$Hsqr]
  893. ldp $a0,$a1,[sp,#$U1]
  894. ldp $a2,$a3,[sp,#$U1+16]
  895. add $bp,sp,#$Hsqr
  896. add $rp,sp,#$U2
  897. bl __ecp_nistz256_mul_mont // p256_mul_mont(U2, U1, Hsqr);
  898. mov $t0,$acc0
  899. mov $t1,$acc1
  900. mov $t2,$acc2
  901. mov $t3,$acc3
  902. add $rp,sp,#$Hsqr
  903. bl __ecp_nistz256_add // p256_mul_by_2(Hsqr, U2);
  904. add $bp,sp,#$Rsqr
  905. add $rp,sp,#$res_x
  906. bl __ecp_nistz256_sub_morf // p256_sub(res_x, Rsqr, Hsqr);
  907. add $bp,sp,#$Hcub
  908. bl __ecp_nistz256_sub_from // p256_sub(res_x, res_x, Hcub);
  909. add $bp,sp,#$U2
  910. ldr $bi,[sp,#$Hcub] // forward load for p256_mul_mont
  911. ldp $a0,$a1,[sp,#$S1]
  912. ldp $a2,$a3,[sp,#$S1+16]
  913. add $rp,sp,#$res_y
  914. bl __ecp_nistz256_sub_morf // p256_sub(res_y, U2, res_x);
  915. add $bp,sp,#$Hcub
  916. add $rp,sp,#$S2
  917. bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, S1, Hcub);
  918. ldr $bi,[sp,#$R]
  919. ldp $a0,$a1,[sp,#$res_y]
  920. ldp $a2,$a3,[sp,#$res_y+16]
  921. add $bp,sp,#$R
  922. add $rp,sp,#$res_y
  923. bl __ecp_nistz256_mul_mont // p256_mul_mont(res_y, res_y, R);
  924. add $bp,sp,#$S2
  925. bl __ecp_nistz256_sub_from // p256_sub(res_y, res_y, S2);
  926. ldp $a0,$a1,[sp,#$res_x] // res
  927. ldp $a2,$a3,[sp,#$res_x+16]
  928. ldp $t0,$t1,[$bp_real] // in2
  929. ldp $t2,$t3,[$bp_real,#16]
  930. ___
  931. for($i=0;$i<64;$i+=32) { # conditional moves
  932. $code.=<<___;
  933. ldp $acc0,$acc1,[$ap_real,#$i] // in1
  934. cmp $in1infty,#0 // ~$in1intfy, remember?
  935. ldp $acc2,$acc3,[$ap_real,#$i+16]
  936. csel $t0,$a0,$t0,ne
  937. csel $t1,$a1,$t1,ne
  938. ldp $a0,$a1,[sp,#$res_x+$i+32] // res
  939. csel $t2,$a2,$t2,ne
  940. csel $t3,$a3,$t3,ne
  941. cmp $in2infty,#0 // ~$in2intfy, remember?
  942. ldp $a2,$a3,[sp,#$res_x+$i+48]
  943. csel $acc0,$t0,$acc0,ne
  944. csel $acc1,$t1,$acc1,ne
  945. ldp $t0,$t1,[$bp_real,#$i+32] // in2
  946. csel $acc2,$t2,$acc2,ne
  947. csel $acc3,$t3,$acc3,ne
  948. ldp $t2,$t3,[$bp_real,#$i+48]
  949. stp $acc0,$acc1,[$rp_real,#$i]
  950. stp $acc2,$acc3,[$rp_real,#$i+16]
  951. ___
  952. }
  953. $code.=<<___;
  954. ldp $acc0,$acc1,[$ap_real,#$i] // in1
  955. cmp $in1infty,#0 // ~$in1intfy, remember?
  956. ldp $acc2,$acc3,[$ap_real,#$i+16]
  957. csel $t0,$a0,$t0,ne
  958. csel $t1,$a1,$t1,ne
  959. csel $t2,$a2,$t2,ne
  960. csel $t3,$a3,$t3,ne
  961. cmp $in2infty,#0 // ~$in2intfy, remember?
  962. csel $acc0,$t0,$acc0,ne
  963. csel $acc1,$t1,$acc1,ne
  964. csel $acc2,$t2,$acc2,ne
  965. csel $acc3,$t3,$acc3,ne
  966. stp $acc0,$acc1,[$rp_real,#$i]
  967. stp $acc2,$acc3,[$rp_real,#$i+16]
  968. .Ladd_done:
  969. add sp,x29,#0 // destroy frame
  970. ldp x19,x20,[x29,#16]
  971. ldp x21,x22,[x29,#32]
  972. ldp x23,x24,[x29,#48]
  973. ldp x25,x26,[x29,#64]
  974. ldp x27,x28,[x29,#80]
  975. ldp x29,x30,[sp],#96
  976. .inst 0xd50323bf // autiasp
  977. ret
  978. .size ecp_nistz256_point_add,.-ecp_nistz256_point_add
  979. ___
  980. }
  981. ########################################################################
  982. # void ecp_nistz256_point_add_affine(P256_POINT *out,const P256_POINT *in1,
  983. # const P256_POINT_AFFINE *in2);
  984. {
  985. my ($res_x,$res_y,$res_z,
  986. $U2,$S2,$H,$R,$Hsqr,$Hcub,$Rsqr)=map(32*$_,(0..9));
  987. my $Z1sqr = $S2;
  988. # above map() describes stack layout with 10 temporary
  989. # 256-bit vectors on top.
  990. my ($rp_real,$ap_real,$bp_real,$in1infty,$in2infty,$temp)=map("x$_",(21..26));
  991. $code.=<<___;
  992. .globl ecp_nistz256_point_add_affine
  993. .type ecp_nistz256_point_add_affine,%function
  994. .align 5
  995. ecp_nistz256_point_add_affine:
  996. .inst 0xd503233f // paciasp
  997. stp x29,x30,[sp,#-80]!
  998. add x29,sp,#0
  999. stp x19,x20,[sp,#16]
  1000. stp x21,x22,[sp,#32]
  1001. stp x23,x24,[sp,#48]
  1002. stp x25,x26,[sp,#64]
  1003. sub sp,sp,#32*10
  1004. mov $rp_real,$rp
  1005. mov $ap_real,$ap
  1006. mov $bp_real,$bp
  1007. ldr $poly1,.Lpoly+8
  1008. ldr $poly3,.Lpoly+24
  1009. ldp $a0,$a1,[$ap,#64] // in1_z
  1010. ldp $a2,$a3,[$ap,#64+16]
  1011. orr $t0,$a0,$a1
  1012. orr $t2,$a2,$a3
  1013. orr $in1infty,$t0,$t2
  1014. cmp $in1infty,#0
  1015. csetm $in1infty,ne // ~in1infty
  1016. ldp $acc0,$acc1,[$bp] // in2_x
  1017. ldp $acc2,$acc3,[$bp,#16]
  1018. ldp $t0,$t1,[$bp,#32] // in2_y
  1019. ldp $t2,$t3,[$bp,#48]
  1020. orr $acc0,$acc0,$acc1
  1021. orr $acc2,$acc2,$acc3
  1022. orr $t0,$t0,$t1
  1023. orr $t2,$t2,$t3
  1024. orr $acc0,$acc0,$acc2
  1025. orr $t0,$t0,$t2
  1026. orr $in2infty,$acc0,$t0
  1027. cmp $in2infty,#0
  1028. csetm $in2infty,ne // ~in2infty
  1029. add $rp,sp,#$Z1sqr
  1030. bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Z1sqr, in1_z);
  1031. mov $a0,$acc0
  1032. mov $a1,$acc1
  1033. mov $a2,$acc2
  1034. mov $a3,$acc3
  1035. ldr $bi,[$bp_real]
  1036. add $bp,$bp_real,#0
  1037. add $rp,sp,#$U2
  1038. bl __ecp_nistz256_mul_mont // p256_mul_mont(U2, Z1sqr, in2_x);
  1039. add $bp,$ap_real,#0
  1040. ldr $bi,[$ap_real,#64] // forward load for p256_mul_mont
  1041. ldp $a0,$a1,[sp,#$Z1sqr]
  1042. ldp $a2,$a3,[sp,#$Z1sqr+16]
  1043. add $rp,sp,#$H
  1044. bl __ecp_nistz256_sub_from // p256_sub(H, U2, in1_x);
  1045. add $bp,$ap_real,#64
  1046. add $rp,sp,#$S2
  1047. bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, Z1sqr, in1_z);
  1048. ldr $bi,[$ap_real,#64]
  1049. ldp $a0,$a1,[sp,#$H]
  1050. ldp $a2,$a3,[sp,#$H+16]
  1051. add $bp,$ap_real,#64
  1052. add $rp,sp,#$res_z
  1053. bl __ecp_nistz256_mul_mont // p256_mul_mont(res_z, H, in1_z);
  1054. ldr $bi,[$bp_real,#32]
  1055. ldp $a0,$a1,[sp,#$S2]
  1056. ldp $a2,$a3,[sp,#$S2+16]
  1057. add $bp,$bp_real,#32
  1058. add $rp,sp,#$S2
  1059. bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, S2, in2_y);
  1060. add $bp,$ap_real,#32
  1061. ldp $a0,$a1,[sp,#$H] // forward load for p256_sqr_mont
  1062. ldp $a2,$a3,[sp,#$H+16]
  1063. add $rp,sp,#$R
  1064. bl __ecp_nistz256_sub_from // p256_sub(R, S2, in1_y);
  1065. add $rp,sp,#$Hsqr
  1066. bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Hsqr, H);
  1067. ldp $a0,$a1,[sp,#$R]
  1068. ldp $a2,$a3,[sp,#$R+16]
  1069. add $rp,sp,#$Rsqr
  1070. bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Rsqr, R);
  1071. ldr $bi,[sp,#$H]
  1072. ldp $a0,$a1,[sp,#$Hsqr]
  1073. ldp $a2,$a3,[sp,#$Hsqr+16]
  1074. add $bp,sp,#$H
  1075. add $rp,sp,#$Hcub
  1076. bl __ecp_nistz256_mul_mont // p256_mul_mont(Hcub, Hsqr, H);
  1077. ldr $bi,[$ap_real]
  1078. ldp $a0,$a1,[sp,#$Hsqr]
  1079. ldp $a2,$a3,[sp,#$Hsqr+16]
  1080. add $bp,$ap_real,#0
  1081. add $rp,sp,#$U2
  1082. bl __ecp_nistz256_mul_mont // p256_mul_mont(U2, in1_x, Hsqr);
  1083. mov $t0,$acc0
  1084. mov $t1,$acc1
  1085. mov $t2,$acc2
  1086. mov $t3,$acc3
  1087. add $rp,sp,#$Hsqr
  1088. bl __ecp_nistz256_add // p256_mul_by_2(Hsqr, U2);
  1089. add $bp,sp,#$Rsqr
  1090. add $rp,sp,#$res_x
  1091. bl __ecp_nistz256_sub_morf // p256_sub(res_x, Rsqr, Hsqr);
  1092. add $bp,sp,#$Hcub
  1093. bl __ecp_nistz256_sub_from // p256_sub(res_x, res_x, Hcub);
  1094. add $bp,sp,#$U2
  1095. ldr $bi,[$ap_real,#32] // forward load for p256_mul_mont
  1096. ldp $a0,$a1,[sp,#$Hcub]
  1097. ldp $a2,$a3,[sp,#$Hcub+16]
  1098. add $rp,sp,#$res_y
  1099. bl __ecp_nistz256_sub_morf // p256_sub(res_y, U2, res_x);
  1100. add $bp,$ap_real,#32
  1101. add $rp,sp,#$S2
  1102. bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, in1_y, Hcub);
  1103. ldr $bi,[sp,#$R]
  1104. ldp $a0,$a1,[sp,#$res_y]
  1105. ldp $a2,$a3,[sp,#$res_y+16]
  1106. add $bp,sp,#$R
  1107. add $rp,sp,#$res_y
  1108. bl __ecp_nistz256_mul_mont // p256_mul_mont(res_y, res_y, R);
  1109. add $bp,sp,#$S2
  1110. bl __ecp_nistz256_sub_from // p256_sub(res_y, res_y, S2);
  1111. ldp $a0,$a1,[sp,#$res_x] // res
  1112. ldp $a2,$a3,[sp,#$res_x+16]
  1113. ldp $t0,$t1,[$bp_real] // in2
  1114. ldp $t2,$t3,[$bp_real,#16]
  1115. ___
  1116. for($i=0;$i<64;$i+=32) { # conditional moves
  1117. $code.=<<___;
  1118. ldp $acc0,$acc1,[$ap_real,#$i] // in1
  1119. cmp $in1infty,#0 // ~$in1intfy, remember?
  1120. ldp $acc2,$acc3,[$ap_real,#$i+16]
  1121. csel $t0,$a0,$t0,ne
  1122. csel $t1,$a1,$t1,ne
  1123. ldp $a0,$a1,[sp,#$res_x+$i+32] // res
  1124. csel $t2,$a2,$t2,ne
  1125. csel $t3,$a3,$t3,ne
  1126. cmp $in2infty,#0 // ~$in2intfy, remember?
  1127. ldp $a2,$a3,[sp,#$res_x+$i+48]
  1128. csel $acc0,$t0,$acc0,ne
  1129. csel $acc1,$t1,$acc1,ne
  1130. ldp $t0,$t1,[$bp_real,#$i+32] // in2
  1131. csel $acc2,$t2,$acc2,ne
  1132. csel $acc3,$t3,$acc3,ne
  1133. ldp $t2,$t3,[$bp_real,#$i+48]
  1134. stp $acc0,$acc1,[$rp_real,#$i]
  1135. stp $acc2,$acc3,[$rp_real,#$i+16]
  1136. ___
  1137. $code.=<<___ if ($i == 0);
  1138. adr $bp_real,.Lone_mont-64
  1139. ___
  1140. }
  1141. $code.=<<___;
  1142. ldp $acc0,$acc1,[$ap_real,#$i] // in1
  1143. cmp $in1infty,#0 // ~$in1intfy, remember?
  1144. ldp $acc2,$acc3,[$ap_real,#$i+16]
  1145. csel $t0,$a0,$t0,ne
  1146. csel $t1,$a1,$t1,ne
  1147. csel $t2,$a2,$t2,ne
  1148. csel $t3,$a3,$t3,ne
  1149. cmp $in2infty,#0 // ~$in2intfy, remember?
  1150. csel $acc0,$t0,$acc0,ne
  1151. csel $acc1,$t1,$acc1,ne
  1152. csel $acc2,$t2,$acc2,ne
  1153. csel $acc3,$t3,$acc3,ne
  1154. stp $acc0,$acc1,[$rp_real,#$i]
  1155. stp $acc2,$acc3,[$rp_real,#$i+16]
  1156. add sp,x29,#0 // destroy frame
  1157. ldp x19,x20,[x29,#16]
  1158. ldp x21,x22,[x29,#32]
  1159. ldp x23,x24,[x29,#48]
  1160. ldp x25,x26,[x29,#64]
  1161. ldp x29,x30,[sp],#80
  1162. .inst 0xd50323bf // autiasp
  1163. ret
  1164. .size ecp_nistz256_point_add_affine,.-ecp_nistz256_point_add_affine
  1165. ___
  1166. }
  1167. if (1) {
  1168. my ($ord0,$ord1) = ($poly1,$poly3);
  1169. my ($ord2,$ord3,$ordk,$t4) = map("x$_",(21..24));
  1170. my $acc7 = $bi;
  1171. $code.=<<___;
  1172. ////////////////////////////////////////////////////////////////////////
  1173. // void ecp_nistz256_ord_mul_mont(uint64_t res[4], uint64_t a[4],
  1174. // uint64_t b[4]);
  1175. .globl ecp_nistz256_ord_mul_mont
  1176. .type ecp_nistz256_ord_mul_mont,%function
  1177. .align 4
  1178. ecp_nistz256_ord_mul_mont:
  1179. stp x29,x30,[sp,#-64]!
  1180. add x29,sp,#0
  1181. stp x19,x20,[sp,#16]
  1182. stp x21,x22,[sp,#32]
  1183. stp x23,x24,[sp,#48]
  1184. adr $ordk,.Lord
  1185. ldr $bi,[$bp] // bp[0]
  1186. ldp $a0,$a1,[$ap]
  1187. ldp $a2,$a3,[$ap,#16]
  1188. ldp $ord0,$ord1,[$ordk,#0]
  1189. ldp $ord2,$ord3,[$ordk,#16]
  1190. ldr $ordk,[$ordk,#32]
  1191. mul $acc0,$a0,$bi // a[0]*b[0]
  1192. umulh $t0,$a0,$bi
  1193. mul $acc1,$a1,$bi // a[1]*b[0]
  1194. umulh $t1,$a1,$bi
  1195. mul $acc2,$a2,$bi // a[2]*b[0]
  1196. umulh $t2,$a2,$bi
  1197. mul $acc3,$a3,$bi // a[3]*b[0]
  1198. umulh $acc4,$a3,$bi
  1199. mul $t4,$acc0,$ordk
  1200. adds $acc1,$acc1,$t0 // accumulate high parts of multiplication
  1201. adcs $acc2,$acc2,$t1
  1202. adcs $acc3,$acc3,$t2
  1203. adc $acc4,$acc4,xzr
  1204. mov $acc5,xzr
  1205. ___
  1206. for ($i=1;$i<4;$i++) {
  1207. ################################################################
  1208. # ffff0000.ffffffff.yyyyyyyy.zzzzzzzz
  1209. # * abcdefgh
  1210. # + xxxxxxxx.xxxxxxxx.xxxxxxxx.xxxxxxxx.xxxxxxxx
  1211. #
  1212. # Now observing that ff..ff*x = (2^n-1)*x = 2^n*x-x, we
  1213. # rewrite above as:
  1214. #
  1215. # xxxxxxxx.xxxxxxxx.xxxxxxxx.xxxxxxxx.xxxxxxxx
  1216. # - 0000abcd.efgh0000.abcdefgh.00000000.00000000
  1217. # + abcdefgh.abcdefgh.yzayzbyz.cyzdyzey.zfyzgyzh
  1218. $code.=<<___;
  1219. ldr $bi,[$bp,#8*$i] // b[i]
  1220. lsl $t0,$t4,#32
  1221. subs $acc2,$acc2,$t4
  1222. lsr $t1,$t4,#32
  1223. sbcs $acc3,$acc3,$t0
  1224. sbcs $acc4,$acc4,$t1
  1225. sbc $acc5,$acc5,xzr
  1226. subs xzr,$acc0,#1
  1227. umulh $t1,$ord0,$t4
  1228. mul $t2,$ord1,$t4
  1229. umulh $t3,$ord1,$t4
  1230. adcs $t2,$t2,$t1
  1231. mul $t0,$a0,$bi
  1232. adc $t3,$t3,xzr
  1233. mul $t1,$a1,$bi
  1234. adds $acc0,$acc1,$t2
  1235. mul $t2,$a2,$bi
  1236. adcs $acc1,$acc2,$t3
  1237. mul $t3,$a3,$bi
  1238. adcs $acc2,$acc3,$t4
  1239. adcs $acc3,$acc4,$t4
  1240. adc $acc4,$acc5,xzr
  1241. adds $acc0,$acc0,$t0 // accumulate low parts
  1242. umulh $t0,$a0,$bi
  1243. adcs $acc1,$acc1,$t1
  1244. umulh $t1,$a1,$bi
  1245. adcs $acc2,$acc2,$t2
  1246. umulh $t2,$a2,$bi
  1247. adcs $acc3,$acc3,$t3
  1248. umulh $t3,$a3,$bi
  1249. adc $acc4,$acc4,xzr
  1250. mul $t4,$acc0,$ordk
  1251. adds $acc1,$acc1,$t0 // accumulate high parts
  1252. adcs $acc2,$acc2,$t1
  1253. adcs $acc3,$acc3,$t2
  1254. adcs $acc4,$acc4,$t3
  1255. adc $acc5,xzr,xzr
  1256. ___
  1257. }
  1258. $code.=<<___;
  1259. lsl $t0,$t4,#32 // last reduction
  1260. subs $acc2,$acc2,$t4
  1261. lsr $t1,$t4,#32
  1262. sbcs $acc3,$acc3,$t0
  1263. sbcs $acc4,$acc4,$t1
  1264. sbc $acc5,$acc5,xzr
  1265. subs xzr,$acc0,#1
  1266. umulh $t1,$ord0,$t4
  1267. mul $t2,$ord1,$t4
  1268. umulh $t3,$ord1,$t4
  1269. adcs $t2,$t2,$t1
  1270. adc $t3,$t3,xzr
  1271. adds $acc0,$acc1,$t2
  1272. adcs $acc1,$acc2,$t3
  1273. adcs $acc2,$acc3,$t4
  1274. adcs $acc3,$acc4,$t4
  1275. adc $acc4,$acc5,xzr
  1276. subs $t0,$acc0,$ord0 // ret -= modulus
  1277. sbcs $t1,$acc1,$ord1
  1278. sbcs $t2,$acc2,$ord2
  1279. sbcs $t3,$acc3,$ord3
  1280. sbcs xzr,$acc4,xzr
  1281. csel $acc0,$acc0,$t0,lo // ret = borrow ? ret : ret-modulus
  1282. csel $acc1,$acc1,$t1,lo
  1283. csel $acc2,$acc2,$t2,lo
  1284. stp $acc0,$acc1,[$rp]
  1285. csel $acc3,$acc3,$t3,lo
  1286. stp $acc2,$acc3,[$rp,#16]
  1287. ldp x19,x20,[sp,#16]
  1288. ldp x21,x22,[sp,#32]
  1289. ldp x23,x24,[sp,#48]
  1290. ldr x29,[sp],#64
  1291. ret
  1292. .size ecp_nistz256_ord_mul_mont,.-ecp_nistz256_ord_mul_mont
  1293. ////////////////////////////////////////////////////////////////////////
  1294. // void ecp_nistz256_ord_sqr_mont(uint64_t res[4], uint64_t a[4],
  1295. // uint64_t rep);
  1296. .globl ecp_nistz256_ord_sqr_mont
  1297. .type ecp_nistz256_ord_sqr_mont,%function
  1298. .align 4
  1299. ecp_nistz256_ord_sqr_mont:
  1300. stp x29,x30,[sp,#-64]!
  1301. add x29,sp,#0
  1302. stp x19,x20,[sp,#16]
  1303. stp x21,x22,[sp,#32]
  1304. stp x23,x24,[sp,#48]
  1305. adr $ordk,.Lord
  1306. ldp $a0,$a1,[$ap]
  1307. ldp $a2,$a3,[$ap,#16]
  1308. ldp $ord0,$ord1,[$ordk,#0]
  1309. ldp $ord2,$ord3,[$ordk,#16]
  1310. ldr $ordk,[$ordk,#32]
  1311. b .Loop_ord_sqr
  1312. .align 4
  1313. .Loop_ord_sqr:
  1314. sub $bp,$bp,#1
  1315. ////////////////////////////////////////////////////////////////
  1316. // | | | | | |a1*a0| |
  1317. // | | | | |a2*a0| | |
  1318. // | |a3*a2|a3*a0| | | |
  1319. // | | | |a2*a1| | | |
  1320. // | | |a3*a1| | | | |
  1321. // *| | | | | | | | 2|
  1322. // +|a3*a3|a2*a2|a1*a1|a0*a0|
  1323. // |--+--+--+--+--+--+--+--|
  1324. // |A7|A6|A5|A4|A3|A2|A1|A0|, where Ax is $accx, i.e. follow $accx
  1325. //
  1326. // "can't overflow" below mark carrying into high part of
  1327. // multiplication result, which can't overflow, because it
  1328. // can never be all ones.
  1329. mul $acc1,$a1,$a0 // a[1]*a[0]
  1330. umulh $t1,$a1,$a0
  1331. mul $acc2,$a2,$a0 // a[2]*a[0]
  1332. umulh $t2,$a2,$a0
  1333. mul $acc3,$a3,$a0 // a[3]*a[0]
  1334. umulh $acc4,$a3,$a0
  1335. adds $acc2,$acc2,$t1 // accumulate high parts of multiplication
  1336. mul $t0,$a2,$a1 // a[2]*a[1]
  1337. umulh $t1,$a2,$a1
  1338. adcs $acc3,$acc3,$t2
  1339. mul $t2,$a3,$a1 // a[3]*a[1]
  1340. umulh $t3,$a3,$a1
  1341. adc $acc4,$acc4,xzr // can't overflow
  1342. mul $acc5,$a3,$a2 // a[3]*a[2]
  1343. umulh $acc6,$a3,$a2
  1344. adds $t1,$t1,$t2 // accumulate high parts of multiplication
  1345. mul $acc0,$a0,$a0 // a[0]*a[0]
  1346. adc $t2,$t3,xzr // can't overflow
  1347. adds $acc3,$acc3,$t0 // accumulate low parts of multiplication
  1348. umulh $a0,$a0,$a0
  1349. adcs $acc4,$acc4,$t1
  1350. mul $t1,$a1,$a1 // a[1]*a[1]
  1351. adcs $acc5,$acc5,$t2
  1352. umulh $a1,$a1,$a1
  1353. adc $acc6,$acc6,xzr // can't overflow
  1354. adds $acc1,$acc1,$acc1 // acc[1-6]*=2
  1355. mul $t2,$a2,$a2 // a[2]*a[2]
  1356. adcs $acc2,$acc2,$acc2
  1357. umulh $a2,$a2,$a2
  1358. adcs $acc3,$acc3,$acc3
  1359. mul $t3,$a3,$a3 // a[3]*a[3]
  1360. adcs $acc4,$acc4,$acc4
  1361. umulh $a3,$a3,$a3
  1362. adcs $acc5,$acc5,$acc5
  1363. adcs $acc6,$acc6,$acc6
  1364. adc $acc7,xzr,xzr
  1365. adds $acc1,$acc1,$a0 // +a[i]*a[i]
  1366. mul $t4,$acc0,$ordk
  1367. adcs $acc2,$acc2,$t1
  1368. adcs $acc3,$acc3,$a1
  1369. adcs $acc4,$acc4,$t2
  1370. adcs $acc5,$acc5,$a2
  1371. adcs $acc6,$acc6,$t3
  1372. adc $acc7,$acc7,$a3
  1373. ___
  1374. for($i=0; $i<4; $i++) { # reductions
  1375. $code.=<<___;
  1376. subs xzr,$acc0,#1
  1377. umulh $t1,$ord0,$t4
  1378. mul $t2,$ord1,$t4
  1379. umulh $t3,$ord1,$t4
  1380. adcs $t2,$t2,$t1
  1381. adc $t3,$t3,xzr
  1382. adds $acc0,$acc1,$t2
  1383. adcs $acc1,$acc2,$t3
  1384. adcs $acc2,$acc3,$t4
  1385. adc $acc3,xzr,$t4 // can't overflow
  1386. ___
  1387. $code.=<<___ if ($i<3);
  1388. mul $t3,$acc0,$ordk
  1389. ___
  1390. $code.=<<___;
  1391. lsl $t0,$t4,#32
  1392. subs $acc1,$acc1,$t4
  1393. lsr $t1,$t4,#32
  1394. sbcs $acc2,$acc2,$t0
  1395. sbc $acc3,$acc3,$t1 // can't borrow
  1396. ___
  1397. ($t3,$t4) = ($t4,$t3);
  1398. }
  1399. $code.=<<___;
  1400. adds $acc0,$acc0,$acc4 // accumulate upper half
  1401. adcs $acc1,$acc1,$acc5
  1402. adcs $acc2,$acc2,$acc6
  1403. adcs $acc3,$acc3,$acc7
  1404. adc $acc4,xzr,xzr
  1405. subs $t0,$acc0,$ord0 // ret -= modulus
  1406. sbcs $t1,$acc1,$ord1
  1407. sbcs $t2,$acc2,$ord2
  1408. sbcs $t3,$acc3,$ord3
  1409. sbcs xzr,$acc4,xzr
  1410. csel $a0,$acc0,$t0,lo // ret = borrow ? ret : ret-modulus
  1411. csel $a1,$acc1,$t1,lo
  1412. csel $a2,$acc2,$t2,lo
  1413. csel $a3,$acc3,$t3,lo
  1414. cbnz $bp,.Loop_ord_sqr
  1415. stp $a0,$a1,[$rp]
  1416. stp $a2,$a3,[$rp,#16]
  1417. ldp x19,x20,[sp,#16]
  1418. ldp x21,x22,[sp,#32]
  1419. ldp x23,x24,[sp,#48]
  1420. ldr x29,[sp],#64
  1421. ret
  1422. .size ecp_nistz256_ord_sqr_mont,.-ecp_nistz256_ord_sqr_mont
  1423. ___
  1424. } }
  1425. ########################################################################
  1426. # scatter-gather subroutines
  1427. {
  1428. my ($out,$inp,$index,$mask)=map("x$_",(0..3));
  1429. $code.=<<___;
  1430. // void ecp_nistz256_scatter_w5(void *x0,const P256_POINT *x1,
  1431. // int x2);
  1432. .globl ecp_nistz256_scatter_w5
  1433. .type ecp_nistz256_scatter_w5,%function
  1434. .align 4
  1435. ecp_nistz256_scatter_w5:
  1436. stp x29,x30,[sp,#-16]!
  1437. add x29,sp,#0
  1438. add $out,$out,$index,lsl#2
  1439. ldp x4,x5,[$inp] // X
  1440. ldp x6,x7,[$inp,#16]
  1441. stur w4,[$out,#64*0-4]
  1442. lsr x4,x4,#32
  1443. str w5,[$out,#64*1-4]
  1444. lsr x5,x5,#32
  1445. str w6,[$out,#64*2-4]
  1446. lsr x6,x6,#32
  1447. str w7,[$out,#64*3-4]
  1448. lsr x7,x7,#32
  1449. str w4,[$out,#64*4-4]
  1450. str w5,[$out,#64*5-4]
  1451. str w6,[$out,#64*6-4]
  1452. str w7,[$out,#64*7-4]
  1453. add $out,$out,#64*8
  1454. ldp x4,x5,[$inp,#32] // Y
  1455. ldp x6,x7,[$inp,#48]
  1456. stur w4,[$out,#64*0-4]
  1457. lsr x4,x4,#32
  1458. str w5,[$out,#64*1-4]
  1459. lsr x5,x5,#32
  1460. str w6,[$out,#64*2-4]
  1461. lsr x6,x6,#32
  1462. str w7,[$out,#64*3-4]
  1463. lsr x7,x7,#32
  1464. str w4,[$out,#64*4-4]
  1465. str w5,[$out,#64*5-4]
  1466. str w6,[$out,#64*6-4]
  1467. str w7,[$out,#64*7-4]
  1468. add $out,$out,#64*8
  1469. ldp x4,x5,[$inp,#64] // Z
  1470. ldp x6,x7,[$inp,#80]
  1471. stur w4,[$out,#64*0-4]
  1472. lsr x4,x4,#32
  1473. str w5,[$out,#64*1-4]
  1474. lsr x5,x5,#32
  1475. str w6,[$out,#64*2-4]
  1476. lsr x6,x6,#32
  1477. str w7,[$out,#64*3-4]
  1478. lsr x7,x7,#32
  1479. str w4,[$out,#64*4-4]
  1480. str w5,[$out,#64*5-4]
  1481. str w6,[$out,#64*6-4]
  1482. str w7,[$out,#64*7-4]
  1483. ldr x29,[sp],#16
  1484. ret
  1485. .size ecp_nistz256_scatter_w5,.-ecp_nistz256_scatter_w5
  1486. // void ecp_nistz256_gather_w5(P256_POINT *x0,const void *x1,
  1487. // int x2);
  1488. .globl ecp_nistz256_gather_w5
  1489. .type ecp_nistz256_gather_w5,%function
  1490. .align 4
  1491. ecp_nistz256_gather_w5:
  1492. stp x29,x30,[sp,#-16]!
  1493. add x29,sp,#0
  1494. cmp $index,xzr
  1495. csetm x3,ne
  1496. add $index,$index,x3
  1497. add $inp,$inp,$index,lsl#2
  1498. ldr w4,[$inp,#64*0]
  1499. ldr w5,[$inp,#64*1]
  1500. ldr w6,[$inp,#64*2]
  1501. ldr w7,[$inp,#64*3]
  1502. ldr w8,[$inp,#64*4]
  1503. ldr w9,[$inp,#64*5]
  1504. ldr w10,[$inp,#64*6]
  1505. ldr w11,[$inp,#64*7]
  1506. add $inp,$inp,#64*8
  1507. orr x4,x4,x8,lsl#32
  1508. orr x5,x5,x9,lsl#32
  1509. orr x6,x6,x10,lsl#32
  1510. orr x7,x7,x11,lsl#32
  1511. csel x4,x4,xzr,ne
  1512. csel x5,x5,xzr,ne
  1513. csel x6,x6,xzr,ne
  1514. csel x7,x7,xzr,ne
  1515. stp x4,x5,[$out] // X
  1516. stp x6,x7,[$out,#16]
  1517. ldr w4,[$inp,#64*0]
  1518. ldr w5,[$inp,#64*1]
  1519. ldr w6,[$inp,#64*2]
  1520. ldr w7,[$inp,#64*3]
  1521. ldr w8,[$inp,#64*4]
  1522. ldr w9,[$inp,#64*5]
  1523. ldr w10,[$inp,#64*6]
  1524. ldr w11,[$inp,#64*7]
  1525. add $inp,$inp,#64*8
  1526. orr x4,x4,x8,lsl#32
  1527. orr x5,x5,x9,lsl#32
  1528. orr x6,x6,x10,lsl#32
  1529. orr x7,x7,x11,lsl#32
  1530. csel x4,x4,xzr,ne
  1531. csel x5,x5,xzr,ne
  1532. csel x6,x6,xzr,ne
  1533. csel x7,x7,xzr,ne
  1534. stp x4,x5,[$out,#32] // Y
  1535. stp x6,x7,[$out,#48]
  1536. ldr w4,[$inp,#64*0]
  1537. ldr w5,[$inp,#64*1]
  1538. ldr w6,[$inp,#64*2]
  1539. ldr w7,[$inp,#64*3]
  1540. ldr w8,[$inp,#64*4]
  1541. ldr w9,[$inp,#64*5]
  1542. ldr w10,[$inp,#64*6]
  1543. ldr w11,[$inp,#64*7]
  1544. orr x4,x4,x8,lsl#32
  1545. orr x5,x5,x9,lsl#32
  1546. orr x6,x6,x10,lsl#32
  1547. orr x7,x7,x11,lsl#32
  1548. csel x4,x4,xzr,ne
  1549. csel x5,x5,xzr,ne
  1550. csel x6,x6,xzr,ne
  1551. csel x7,x7,xzr,ne
  1552. stp x4,x5,[$out,#64] // Z
  1553. stp x6,x7,[$out,#80]
  1554. ldr x29,[sp],#16
  1555. ret
  1556. .size ecp_nistz256_gather_w5,.-ecp_nistz256_gather_w5
  1557. // void ecp_nistz256_scatter_w7(void *x0,const P256_POINT_AFFINE *x1,
  1558. // int x2);
  1559. .globl ecp_nistz256_scatter_w7
  1560. .type ecp_nistz256_scatter_w7,%function
  1561. .align 4
  1562. ecp_nistz256_scatter_w7:
  1563. stp x29,x30,[sp,#-16]!
  1564. add x29,sp,#0
  1565. add $out,$out,$index
  1566. mov $index,#64/8
  1567. .Loop_scatter_w7:
  1568. ldr x3,[$inp],#8
  1569. subs $index,$index,#1
  1570. prfm pstl1strm,[$out,#4096+64*0]
  1571. prfm pstl1strm,[$out,#4096+64*1]
  1572. prfm pstl1strm,[$out,#4096+64*2]
  1573. prfm pstl1strm,[$out,#4096+64*3]
  1574. prfm pstl1strm,[$out,#4096+64*4]
  1575. prfm pstl1strm,[$out,#4096+64*5]
  1576. prfm pstl1strm,[$out,#4096+64*6]
  1577. prfm pstl1strm,[$out,#4096+64*7]
  1578. strb w3,[$out,#64*0]
  1579. lsr x3,x3,#8
  1580. strb w3,[$out,#64*1]
  1581. lsr x3,x3,#8
  1582. strb w3,[$out,#64*2]
  1583. lsr x3,x3,#8
  1584. strb w3,[$out,#64*3]
  1585. lsr x3,x3,#8
  1586. strb w3,[$out,#64*4]
  1587. lsr x3,x3,#8
  1588. strb w3,[$out,#64*5]
  1589. lsr x3,x3,#8
  1590. strb w3,[$out,#64*6]
  1591. lsr x3,x3,#8
  1592. strb w3,[$out,#64*7]
  1593. add $out,$out,#64*8
  1594. b.ne .Loop_scatter_w7
  1595. ldr x29,[sp],#16
  1596. ret
  1597. .size ecp_nistz256_scatter_w7,.-ecp_nistz256_scatter_w7
  1598. // void ecp_nistz256_gather_w7(P256_POINT_AFFINE *x0,const void *x1,
  1599. // int x2);
  1600. .globl ecp_nistz256_gather_w7
  1601. .type ecp_nistz256_gather_w7,%function
  1602. .align 4
  1603. ecp_nistz256_gather_w7:
  1604. stp x29,x30,[sp,#-16]!
  1605. add x29,sp,#0
  1606. cmp $index,xzr
  1607. csetm x3,ne
  1608. add $index,$index,x3
  1609. add $inp,$inp,$index
  1610. mov $index,#64/8
  1611. nop
  1612. .Loop_gather_w7:
  1613. ldrb w4,[$inp,#64*0]
  1614. prfm pldl1strm,[$inp,#4096+64*0]
  1615. subs $index,$index,#1
  1616. ldrb w5,[$inp,#64*1]
  1617. prfm pldl1strm,[$inp,#4096+64*1]
  1618. ldrb w6,[$inp,#64*2]
  1619. prfm pldl1strm,[$inp,#4096+64*2]
  1620. ldrb w7,[$inp,#64*3]
  1621. prfm pldl1strm,[$inp,#4096+64*3]
  1622. ldrb w8,[$inp,#64*4]
  1623. prfm pldl1strm,[$inp,#4096+64*4]
  1624. ldrb w9,[$inp,#64*5]
  1625. prfm pldl1strm,[$inp,#4096+64*5]
  1626. ldrb w10,[$inp,#64*6]
  1627. prfm pldl1strm,[$inp,#4096+64*6]
  1628. ldrb w11,[$inp,#64*7]
  1629. prfm pldl1strm,[$inp,#4096+64*7]
  1630. add $inp,$inp,#64*8
  1631. orr x4,x4,x5,lsl#8
  1632. orr x6,x6,x7,lsl#8
  1633. orr x8,x8,x9,lsl#8
  1634. orr x4,x4,x6,lsl#16
  1635. orr x10,x10,x11,lsl#8
  1636. orr x4,x4,x8,lsl#32
  1637. orr x4,x4,x10,lsl#48
  1638. and x4,x4,x3
  1639. str x4,[$out],#8
  1640. b.ne .Loop_gather_w7
  1641. ldr x29,[sp],#16
  1642. ret
  1643. .size ecp_nistz256_gather_w7,.-ecp_nistz256_gather_w7
  1644. ___
  1645. }
  1646. foreach (split("\n",$code)) {
  1647. s/\`([^\`]*)\`/eval $1/ge;
  1648. print $_,"\n";
  1649. }
  1650. close STDOUT or die "error closing STDOUT"; # enforce flush