2
0

ecp_nistz256-armv4.pl 45 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856
  1. #! /usr/bin/env perl
  2. # Copyright 2015-2020 The OpenSSL Project Authors. All Rights Reserved.
  3. #
  4. # Licensed under the Apache License 2.0 (the "License"). You may not use
  5. # this file except in compliance with the License. You can obtain a copy
  6. # in the file LICENSE in the source distribution or at
  7. # https://www.openssl.org/source/license.html
  8. # ====================================================================
  9. # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
  10. # project. The module is, however, dual licensed under OpenSSL and
  11. # CRYPTOGAMS licenses depending on where you obtain it. For further
  12. # details see http://www.openssl.org/~appro/cryptogams/.
  13. # ====================================================================
  14. #
  15. # ECP_NISTZ256 module for ARMv4.
  16. #
  17. # October 2014.
  18. #
  19. # Original ECP_NISTZ256 submission targeting x86_64 is detailed in
  20. # http://eprint.iacr.org/2013/816. In the process of adaptation
  21. # original .c module was made 32-bit savvy in order to make this
  22. # implementation possible.
  23. #
  24. # with/without -DECP_NISTZ256_ASM
  25. # Cortex-A8 +53-170%
  26. # Cortex-A9 +76-205%
  27. # Cortex-A15 +100-316%
  28. # Snapdragon S4 +66-187%
  29. #
  30. # Ranges denote minimum and maximum improvement coefficients depending
  31. # on benchmark. Lower coefficients are for ECDSA sign, server-side
  32. # operation. Keep in mind that +200% means 3x improvement.
  33. # $output is the last argument if it looks like a file (it has an extension)
  34. # $flavour is the first argument if it doesn't look like a file
  35. $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
  36. $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
  37. if ($flavour && $flavour ne "void") {
  38. $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
  39. ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
  40. ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
  41. die "can't locate arm-xlate.pl";
  42. open STDOUT,"| \"$^X\" $xlate $flavour \"$output\""
  43. or die "can't call $xlate: $!";
  44. } else {
  45. $output and open STDOUT,">$output";
  46. }
  47. $code.=<<___;
  48. #include "arm_arch.h"
  49. #if defined(__thumb2__)
  50. .syntax unified
  51. .thumb
  52. #else
  53. .code 32
  54. #endif
  55. ___
  56. ########################################################################
  57. # Convert ecp_nistz256_table.c to layout expected by ecp_nistz_gather_w7
  58. #
  59. $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
  60. open TABLE,"<ecp_nistz256_table.c" or
  61. open TABLE,"<${dir}../ecp_nistz256_table.c" or
  62. die "failed to open ecp_nistz256_table.c:",$!;
  63. use integer;
  64. foreach(<TABLE>) {
  65. s/TOBN\(\s*(0x[0-9a-f]+),\s*(0x[0-9a-f]+)\s*\)/push @arr,hex($2),hex($1)/geo;
  66. }
  67. close TABLE;
  68. # See ecp_nistz256_table.c for explanation for why it's 64*16*37.
  69. # 64*16*37-1 is because $#arr returns last valid index or @arr, not
  70. # amount of elements.
  71. die "insane number of elements" if ($#arr != 64*16*37-1);
  72. $code.=<<___;
  73. .rodata
  74. .globl ecp_nistz256_precomputed
  75. .type ecp_nistz256_precomputed,%object
  76. .align 12
  77. ecp_nistz256_precomputed:
  78. ___
  79. ########################################################################
  80. # this conversion smashes P256_POINT_AFFINE by individual bytes with
  81. # 64 byte interval, similar to
  82. # 1111222233334444
  83. # 1234123412341234
  84. for(1..37) {
  85. @tbl = splice(@arr,0,64*16);
  86. for($i=0;$i<64;$i++) {
  87. undef @line;
  88. for($j=0;$j<64;$j++) {
  89. push @line,(@tbl[$j*16+$i/4]>>(($i%4)*8))&0xff;
  90. }
  91. $code.=".byte\t";
  92. $code.=join(',',map { sprintf "0x%02x",$_} @line);
  93. $code.="\n";
  94. }
  95. }
  96. $code.=<<___;
  97. .size ecp_nistz256_precomputed,.-ecp_nistz256_precomputed
  98. .text
  99. .align 5
  100. .LRR: @ 2^512 mod P precomputed for NIST P256 polynomial
  101. .long 0x00000003, 0x00000000, 0xffffffff, 0xfffffffb
  102. .long 0xfffffffe, 0xffffffff, 0xfffffffd, 0x00000004
  103. .Lone:
  104. .long 1,0,0,0,0,0,0,0
  105. .asciz "ECP_NISTZ256 for ARMv4, CRYPTOGAMS by <appro\@openssl.org>"
  106. .align 6
  107. ___
  108. ########################################################################
  109. # common register layout, note that $t2 is link register, so that if
  110. # internal subroutine uses $t2, then it has to offload lr...
  111. ($r_ptr,$a_ptr,$b_ptr,$ff,$a0,$a1,$a2,$a3,$a4,$a5,$a6,$a7,$t1,$t2)=
  112. map("r$_",(0..12,14));
  113. ($t0,$t3)=($ff,$a_ptr);
  114. $code.=<<___;
  115. @ void ecp_nistz256_to_mont(BN_ULONG r0[8],const BN_ULONG r1[8]);
  116. .globl ecp_nistz256_to_mont
  117. .type ecp_nistz256_to_mont,%function
  118. ecp_nistz256_to_mont:
  119. adr $b_ptr,.LRR
  120. b .Lecp_nistz256_mul_mont
  121. .size ecp_nistz256_to_mont,.-ecp_nistz256_to_mont
  122. @ void ecp_nistz256_from_mont(BN_ULONG r0[8],const BN_ULONG r1[8]);
  123. .globl ecp_nistz256_from_mont
  124. .type ecp_nistz256_from_mont,%function
  125. ecp_nistz256_from_mont:
  126. adr $b_ptr,.Lone
  127. b .Lecp_nistz256_mul_mont
  128. .size ecp_nistz256_from_mont,.-ecp_nistz256_from_mont
  129. @ void ecp_nistz256_mul_by_2(BN_ULONG r0[8],const BN_ULONG r1[8]);
  130. .globl ecp_nistz256_mul_by_2
  131. .type ecp_nistz256_mul_by_2,%function
  132. .align 4
  133. ecp_nistz256_mul_by_2:
  134. stmdb sp!,{r4-r12,lr}
  135. bl __ecp_nistz256_mul_by_2
  136. #if __ARM_ARCH__>=5 || !defined(__thumb__)
  137. ldmia sp!,{r4-r12,pc}
  138. #else
  139. ldmia sp!,{r4-r12,lr}
  140. bx lr @ interoperable with Thumb ISA:-)
  141. #endif
  142. .size ecp_nistz256_mul_by_2,.-ecp_nistz256_mul_by_2
  143. .type __ecp_nistz256_mul_by_2,%function
  144. .align 4
  145. __ecp_nistz256_mul_by_2:
  146. ldr $a0,[$a_ptr,#0]
  147. ldr $a1,[$a_ptr,#4]
  148. ldr $a2,[$a_ptr,#8]
  149. adds $a0,$a0,$a0 @ a[0:7]+=a[0:7], i.e. add with itself
  150. ldr $a3,[$a_ptr,#12]
  151. adcs $a1,$a1,$a1
  152. ldr $a4,[$a_ptr,#16]
  153. adcs $a2,$a2,$a2
  154. ldr $a5,[$a_ptr,#20]
  155. adcs $a3,$a3,$a3
  156. ldr $a6,[$a_ptr,#24]
  157. adcs $a4,$a4,$a4
  158. ldr $a7,[$a_ptr,#28]
  159. adcs $a5,$a5,$a5
  160. adcs $a6,$a6,$a6
  161. mov $ff,#0
  162. adcs $a7,$a7,$a7
  163. adc $ff,$ff,#0
  164. b .Lreduce_by_sub
  165. .size __ecp_nistz256_mul_by_2,.-__ecp_nistz256_mul_by_2
  166. @ void ecp_nistz256_add(BN_ULONG r0[8],const BN_ULONG r1[8],
  167. @ const BN_ULONG r2[8]);
  168. .globl ecp_nistz256_add
  169. .type ecp_nistz256_add,%function
  170. .align 4
  171. ecp_nistz256_add:
  172. stmdb sp!,{r4-r12,lr}
  173. bl __ecp_nistz256_add
  174. #if __ARM_ARCH__>=5 || !defined(__thumb__)
  175. ldmia sp!,{r4-r12,pc}
  176. #else
  177. ldmia sp!,{r4-r12,lr}
  178. bx lr @ interoperable with Thumb ISA:-)
  179. #endif
  180. .size ecp_nistz256_add,.-ecp_nistz256_add
  181. .type __ecp_nistz256_add,%function
  182. .align 4
  183. __ecp_nistz256_add:
  184. str lr,[sp,#-4]! @ push lr
  185. ldr $a0,[$a_ptr,#0]
  186. ldr $a1,[$a_ptr,#4]
  187. ldr $a2,[$a_ptr,#8]
  188. ldr $a3,[$a_ptr,#12]
  189. ldr $a4,[$a_ptr,#16]
  190. ldr $t0,[$b_ptr,#0]
  191. ldr $a5,[$a_ptr,#20]
  192. ldr $t1,[$b_ptr,#4]
  193. ldr $a6,[$a_ptr,#24]
  194. ldr $t2,[$b_ptr,#8]
  195. ldr $a7,[$a_ptr,#28]
  196. ldr $t3,[$b_ptr,#12]
  197. adds $a0,$a0,$t0
  198. ldr $t0,[$b_ptr,#16]
  199. adcs $a1,$a1,$t1
  200. ldr $t1,[$b_ptr,#20]
  201. adcs $a2,$a2,$t2
  202. ldr $t2,[$b_ptr,#24]
  203. adcs $a3,$a3,$t3
  204. ldr $t3,[$b_ptr,#28]
  205. adcs $a4,$a4,$t0
  206. adcs $a5,$a5,$t1
  207. adcs $a6,$a6,$t2
  208. mov $ff,#0
  209. adcs $a7,$a7,$t3
  210. adc $ff,$ff,#0
  211. ldr lr,[sp],#4 @ pop lr
  212. .Lreduce_by_sub:
  213. @ if a+b >= modulus, subtract modulus.
  214. @
  215. @ But since comparison implies subtraction, we subtract
  216. @ modulus and then add it back if subtraction borrowed.
  217. subs $a0,$a0,#-1
  218. sbcs $a1,$a1,#-1
  219. sbcs $a2,$a2,#-1
  220. sbcs $a3,$a3,#0
  221. sbcs $a4,$a4,#0
  222. sbcs $a5,$a5,#0
  223. sbcs $a6,$a6,#1
  224. sbcs $a7,$a7,#-1
  225. sbc $ff,$ff,#0
  226. @ Note that because mod has special form, i.e. consists of
  227. @ 0xffffffff, 1 and 0s, we can conditionally synthesize it by
  228. @ using value of borrow as a whole or extracting single bit.
  229. @ Follow $ff register...
  230. adds $a0,$a0,$ff @ add synthesized modulus
  231. adcs $a1,$a1,$ff
  232. str $a0,[$r_ptr,#0]
  233. adcs $a2,$a2,$ff
  234. str $a1,[$r_ptr,#4]
  235. adcs $a3,$a3,#0
  236. str $a2,[$r_ptr,#8]
  237. adcs $a4,$a4,#0
  238. str $a3,[$r_ptr,#12]
  239. adcs $a5,$a5,#0
  240. str $a4,[$r_ptr,#16]
  241. adcs $a6,$a6,$ff,lsr#31
  242. str $a5,[$r_ptr,#20]
  243. adcs $a7,$a7,$ff
  244. str $a6,[$r_ptr,#24]
  245. str $a7,[$r_ptr,#28]
  246. mov pc,lr
  247. .size __ecp_nistz256_add,.-__ecp_nistz256_add
  248. @ void ecp_nistz256_mul_by_3(BN_ULONG r0[8],const BN_ULONG r1[8]);
  249. .globl ecp_nistz256_mul_by_3
  250. .type ecp_nistz256_mul_by_3,%function
  251. .align 4
  252. ecp_nistz256_mul_by_3:
  253. stmdb sp!,{r4-r12,lr}
  254. bl __ecp_nistz256_mul_by_3
  255. #if __ARM_ARCH__>=5 || !defined(__thumb__)
  256. ldmia sp!,{r4-r12,pc}
  257. #else
  258. ldmia sp!,{r4-r12,lr}
  259. bx lr @ interoperable with Thumb ISA:-)
  260. #endif
  261. .size ecp_nistz256_mul_by_3,.-ecp_nistz256_mul_by_3
  262. .type __ecp_nistz256_mul_by_3,%function
  263. .align 4
  264. __ecp_nistz256_mul_by_3:
  265. str lr,[sp,#-4]! @ push lr
  266. @ As multiplication by 3 is performed as 2*n+n, below are inline
  267. @ copies of __ecp_nistz256_mul_by_2 and __ecp_nistz256_add, see
  268. @ corresponding subroutines for details.
  269. ldr $a0,[$a_ptr,#0]
  270. ldr $a1,[$a_ptr,#4]
  271. ldr $a2,[$a_ptr,#8]
  272. adds $a0,$a0,$a0 @ a[0:7]+=a[0:7]
  273. ldr $a3,[$a_ptr,#12]
  274. adcs $a1,$a1,$a1
  275. ldr $a4,[$a_ptr,#16]
  276. adcs $a2,$a2,$a2
  277. ldr $a5,[$a_ptr,#20]
  278. adcs $a3,$a3,$a3
  279. ldr $a6,[$a_ptr,#24]
  280. adcs $a4,$a4,$a4
  281. ldr $a7,[$a_ptr,#28]
  282. adcs $a5,$a5,$a5
  283. adcs $a6,$a6,$a6
  284. mov $ff,#0
  285. adcs $a7,$a7,$a7
  286. adc $ff,$ff,#0
  287. subs $a0,$a0,#-1 @ .Lreduce_by_sub but without stores
  288. sbcs $a1,$a1,#-1
  289. sbcs $a2,$a2,#-1
  290. sbcs $a3,$a3,#0
  291. sbcs $a4,$a4,#0
  292. sbcs $a5,$a5,#0
  293. sbcs $a6,$a6,#1
  294. sbcs $a7,$a7,#-1
  295. sbc $ff,$ff,#0
  296. adds $a0,$a0,$ff @ add synthesized modulus
  297. adcs $a1,$a1,$ff
  298. adcs $a2,$a2,$ff
  299. adcs $a3,$a3,#0
  300. adcs $a4,$a4,#0
  301. ldr $b_ptr,[$a_ptr,#0]
  302. adcs $a5,$a5,#0
  303. ldr $t1,[$a_ptr,#4]
  304. adcs $a6,$a6,$ff,lsr#31
  305. ldr $t2,[$a_ptr,#8]
  306. adc $a7,$a7,$ff
  307. ldr $t0,[$a_ptr,#12]
  308. adds $a0,$a0,$b_ptr @ 2*a[0:7]+=a[0:7]
  309. ldr $b_ptr,[$a_ptr,#16]
  310. adcs $a1,$a1,$t1
  311. ldr $t1,[$a_ptr,#20]
  312. adcs $a2,$a2,$t2
  313. ldr $t2,[$a_ptr,#24]
  314. adcs $a3,$a3,$t0
  315. ldr $t3,[$a_ptr,#28]
  316. adcs $a4,$a4,$b_ptr
  317. adcs $a5,$a5,$t1
  318. adcs $a6,$a6,$t2
  319. mov $ff,#0
  320. adcs $a7,$a7,$t3
  321. adc $ff,$ff,#0
  322. ldr lr,[sp],#4 @ pop lr
  323. b .Lreduce_by_sub
  324. .size ecp_nistz256_mul_by_3,.-ecp_nistz256_mul_by_3
  325. @ void ecp_nistz256_div_by_2(BN_ULONG r0[8],const BN_ULONG r1[8]);
  326. .globl ecp_nistz256_div_by_2
  327. .type ecp_nistz256_div_by_2,%function
  328. .align 4
  329. ecp_nistz256_div_by_2:
  330. stmdb sp!,{r4-r12,lr}
  331. bl __ecp_nistz256_div_by_2
  332. #if __ARM_ARCH__>=5 || !defined(__thumb__)
  333. ldmia sp!,{r4-r12,pc}
  334. #else
  335. ldmia sp!,{r4-r12,lr}
  336. bx lr @ interoperable with Thumb ISA:-)
  337. #endif
  338. .size ecp_nistz256_div_by_2,.-ecp_nistz256_div_by_2
  339. .type __ecp_nistz256_div_by_2,%function
  340. .align 4
  341. __ecp_nistz256_div_by_2:
  342. @ ret = (a is odd ? a+mod : a) >> 1
  343. ldr $a0,[$a_ptr,#0]
  344. ldr $a1,[$a_ptr,#4]
  345. ldr $a2,[$a_ptr,#8]
  346. mov $ff,$a0,lsl#31 @ place least significant bit to most
  347. @ significant position, now arithmetic
  348. @ right shift by 31 will produce -1 or
  349. @ 0, while logical right shift 1 or 0,
  350. @ this is how modulus is conditionally
  351. @ synthesized in this case...
  352. ldr $a3,[$a_ptr,#12]
  353. adds $a0,$a0,$ff,asr#31
  354. ldr $a4,[$a_ptr,#16]
  355. adcs $a1,$a1,$ff,asr#31
  356. ldr $a5,[$a_ptr,#20]
  357. adcs $a2,$a2,$ff,asr#31
  358. ldr $a6,[$a_ptr,#24]
  359. adcs $a3,$a3,#0
  360. ldr $a7,[$a_ptr,#28]
  361. adcs $a4,$a4,#0
  362. mov $a0,$a0,lsr#1 @ a[0:7]>>=1, we can start early
  363. @ because it doesn't affect flags
  364. adcs $a5,$a5,#0
  365. orr $a0,$a0,$a1,lsl#31
  366. adcs $a6,$a6,$ff,lsr#31
  367. mov $b_ptr,#0
  368. adcs $a7,$a7,$ff,asr#31
  369. mov $a1,$a1,lsr#1
  370. adc $b_ptr,$b_ptr,#0 @ top-most carry bit from addition
  371. orr $a1,$a1,$a2,lsl#31
  372. mov $a2,$a2,lsr#1
  373. str $a0,[$r_ptr,#0]
  374. orr $a2,$a2,$a3,lsl#31
  375. mov $a3,$a3,lsr#1
  376. str $a1,[$r_ptr,#4]
  377. orr $a3,$a3,$a4,lsl#31
  378. mov $a4,$a4,lsr#1
  379. str $a2,[$r_ptr,#8]
  380. orr $a4,$a4,$a5,lsl#31
  381. mov $a5,$a5,lsr#1
  382. str $a3,[$r_ptr,#12]
  383. orr $a5,$a5,$a6,lsl#31
  384. mov $a6,$a6,lsr#1
  385. str $a4,[$r_ptr,#16]
  386. orr $a6,$a6,$a7,lsl#31
  387. mov $a7,$a7,lsr#1
  388. str $a5,[$r_ptr,#20]
  389. orr $a7,$a7,$b_ptr,lsl#31 @ don't forget the top-most carry bit
  390. str $a6,[$r_ptr,#24]
  391. str $a7,[$r_ptr,#28]
  392. mov pc,lr
  393. .size __ecp_nistz256_div_by_2,.-__ecp_nistz256_div_by_2
  394. @ void ecp_nistz256_sub(BN_ULONG r0[8],const BN_ULONG r1[8],
  395. @ const BN_ULONG r2[8]);
  396. .globl ecp_nistz256_sub
  397. .type ecp_nistz256_sub,%function
  398. .align 4
  399. ecp_nistz256_sub:
  400. stmdb sp!,{r4-r12,lr}
  401. bl __ecp_nistz256_sub
  402. #if __ARM_ARCH__>=5 || !defined(__thumb__)
  403. ldmia sp!,{r4-r12,pc}
  404. #else
  405. ldmia sp!,{r4-r12,lr}
  406. bx lr @ interoperable with Thumb ISA:-)
  407. #endif
  408. .size ecp_nistz256_sub,.-ecp_nistz256_sub
  409. .type __ecp_nistz256_sub,%function
  410. .align 4
  411. __ecp_nistz256_sub:
  412. str lr,[sp,#-4]! @ push lr
  413. ldr $a0,[$a_ptr,#0]
  414. ldr $a1,[$a_ptr,#4]
  415. ldr $a2,[$a_ptr,#8]
  416. ldr $a3,[$a_ptr,#12]
  417. ldr $a4,[$a_ptr,#16]
  418. ldr $t0,[$b_ptr,#0]
  419. ldr $a5,[$a_ptr,#20]
  420. ldr $t1,[$b_ptr,#4]
  421. ldr $a6,[$a_ptr,#24]
  422. ldr $t2,[$b_ptr,#8]
  423. ldr $a7,[$a_ptr,#28]
  424. ldr $t3,[$b_ptr,#12]
  425. subs $a0,$a0,$t0
  426. ldr $t0,[$b_ptr,#16]
  427. sbcs $a1,$a1,$t1
  428. ldr $t1,[$b_ptr,#20]
  429. sbcs $a2,$a2,$t2
  430. ldr $t2,[$b_ptr,#24]
  431. sbcs $a3,$a3,$t3
  432. ldr $t3,[$b_ptr,#28]
  433. sbcs $a4,$a4,$t0
  434. sbcs $a5,$a5,$t1
  435. sbcs $a6,$a6,$t2
  436. sbcs $a7,$a7,$t3
  437. sbc $ff,$ff,$ff @ broadcast borrow bit
  438. ldr lr,[sp],#4 @ pop lr
  439. .Lreduce_by_add:
  440. @ if a-b borrows, add modulus.
  441. @
  442. @ Note that because mod has special form, i.e. consists of
  443. @ 0xffffffff, 1 and 0s, we can conditionally synthesize it by
  444. @ broadcasting borrow bit to a register, $ff, and using it as
  445. @ a whole or extracting single bit.
  446. adds $a0,$a0,$ff @ add synthesized modulus
  447. adcs $a1,$a1,$ff
  448. str $a0,[$r_ptr,#0]
  449. adcs $a2,$a2,$ff
  450. str $a1,[$r_ptr,#4]
  451. adcs $a3,$a3,#0
  452. str $a2,[$r_ptr,#8]
  453. adcs $a4,$a4,#0
  454. str $a3,[$r_ptr,#12]
  455. adcs $a5,$a5,#0
  456. str $a4,[$r_ptr,#16]
  457. adcs $a6,$a6,$ff,lsr#31
  458. str $a5,[$r_ptr,#20]
  459. adcs $a7,$a7,$ff
  460. str $a6,[$r_ptr,#24]
  461. str $a7,[$r_ptr,#28]
  462. mov pc,lr
  463. .size __ecp_nistz256_sub,.-__ecp_nistz256_sub
  464. @ void ecp_nistz256_neg(BN_ULONG r0[8],const BN_ULONG r1[8]);
  465. .globl ecp_nistz256_neg
  466. .type ecp_nistz256_neg,%function
  467. .align 4
  468. ecp_nistz256_neg:
  469. stmdb sp!,{r4-r12,lr}
  470. bl __ecp_nistz256_neg
  471. #if __ARM_ARCH__>=5 || !defined(__thumb__)
  472. ldmia sp!,{r4-r12,pc}
  473. #else
  474. ldmia sp!,{r4-r12,lr}
  475. bx lr @ interoperable with Thumb ISA:-)
  476. #endif
  477. .size ecp_nistz256_neg,.-ecp_nistz256_neg
  478. .type __ecp_nistz256_neg,%function
  479. .align 4
  480. __ecp_nistz256_neg:
  481. ldr $a0,[$a_ptr,#0]
  482. eor $ff,$ff,$ff
  483. ldr $a1,[$a_ptr,#4]
  484. ldr $a2,[$a_ptr,#8]
  485. subs $a0,$ff,$a0
  486. ldr $a3,[$a_ptr,#12]
  487. sbcs $a1,$ff,$a1
  488. ldr $a4,[$a_ptr,#16]
  489. sbcs $a2,$ff,$a2
  490. ldr $a5,[$a_ptr,#20]
  491. sbcs $a3,$ff,$a3
  492. ldr $a6,[$a_ptr,#24]
  493. sbcs $a4,$ff,$a4
  494. ldr $a7,[$a_ptr,#28]
  495. sbcs $a5,$ff,$a5
  496. sbcs $a6,$ff,$a6
  497. sbcs $a7,$ff,$a7
  498. sbc $ff,$ff,$ff
  499. b .Lreduce_by_add
  500. .size __ecp_nistz256_neg,.-__ecp_nistz256_neg
  501. ___
  502. {
  503. my @acc=map("r$_",(3..11));
  504. my ($t0,$t1,$bj,$t2,$t3)=map("r$_",(0,1,2,12,14));
  505. $code.=<<___;
  506. @ void ecp_nistz256_sqr_mont(BN_ULONG r0[8],const BN_ULONG r1[8]);
  507. .globl ecp_nistz256_sqr_mont
  508. .type ecp_nistz256_sqr_mont,%function
  509. .align 4
  510. ecp_nistz256_sqr_mont:
  511. mov $b_ptr,$a_ptr
  512. b .Lecp_nistz256_mul_mont
  513. .size ecp_nistz256_sqr_mont,.-ecp_nistz256_sqr_mont
  514. @ void ecp_nistz256_mul_mont(BN_ULONG r0[8],const BN_ULONG r1[8],
  515. @ const BN_ULONG r2[8]);
  516. .globl ecp_nistz256_mul_mont
  517. .type ecp_nistz256_mul_mont,%function
  518. .align 4
  519. ecp_nistz256_mul_mont:
  520. .Lecp_nistz256_mul_mont:
  521. stmdb sp!,{r4-r12,lr}
  522. bl __ecp_nistz256_mul_mont
  523. #if __ARM_ARCH__>=5 || !defined(__thumb__)
  524. ldmia sp!,{r4-r12,pc}
  525. #else
  526. ldmia sp!,{r4-r12,lr}
  527. bx lr @ interoperable with Thumb ISA:-)
  528. #endif
  529. .size ecp_nistz256_mul_mont,.-ecp_nistz256_mul_mont
  530. .type __ecp_nistz256_mul_mont,%function
  531. .align 4
  532. __ecp_nistz256_mul_mont:
  533. stmdb sp!,{r0-r2,lr} @ make a copy of arguments too
  534. ldr $bj,[$b_ptr,#0] @ b[0]
  535. ldmia $a_ptr,{@acc[1]-@acc[8]}
  536. umull @acc[0],$t3,@acc[1],$bj @ r[0]=a[0]*b[0]
  537. stmdb sp!,{$acc[1]-@acc[8]} @ copy a[0-7] to stack, so
  538. @ that it can be addressed
  539. @ without spending register
  540. @ on address
  541. umull @acc[1],$t0,@acc[2],$bj @ r[1]=a[1]*b[0]
  542. umull @acc[2],$t1,@acc[3],$bj
  543. adds @acc[1],@acc[1],$t3 @ accumulate high part of mult
  544. umull @acc[3],$t2,@acc[4],$bj
  545. adcs @acc[2],@acc[2],$t0
  546. umull @acc[4],$t3,@acc[5],$bj
  547. adcs @acc[3],@acc[3],$t1
  548. umull @acc[5],$t0,@acc[6],$bj
  549. adcs @acc[4],@acc[4],$t2
  550. umull @acc[6],$t1,@acc[7],$bj
  551. adcs @acc[5],@acc[5],$t3
  552. umull @acc[7],$t2,@acc[8],$bj
  553. adcs @acc[6],@acc[6],$t0
  554. adcs @acc[7],@acc[7],$t1
  555. eor $t3,$t3,$t3 @ first overflow bit is zero
  556. adc @acc[8],$t2,#0
  557. ___
  558. for(my $i=1;$i<8;$i++) {
  559. my $t4=@acc[0];
  560. # Reduction iteration is normally performed by accumulating
  561. # result of multiplication of modulus by "magic" digit [and
  562. # omitting least significant word, which is guaranteed to
  563. # be 0], but thanks to special form of modulus and "magic"
  564. # digit being equal to least significant word, it can be
  565. # performed with additions and subtractions alone. Indeed:
  566. #
  567. # ffff.0001.0000.0000.0000.ffff.ffff.ffff
  568. # * abcd
  569. # + xxxx.xxxx.xxxx.xxxx.xxxx.xxxx.xxxx.xxxx.abcd
  570. #
  571. # Now observing that ff..ff*x = (2^n-1)*x = 2^n*x-x, we
  572. # rewrite above as:
  573. #
  574. # xxxx.xxxx.xxxx.xxxx.xxxx.xxxx.xxxx.xxxx.abcd
  575. # + abcd.0000.abcd.0000.0000.abcd.0000.0000.0000
  576. # - abcd.0000.0000.0000.0000.0000.0000.abcd
  577. #
  578. # or marking redundant operations:
  579. #
  580. # xxxx.xxxx.xxxx.xxxx.xxxx.xxxx.xxxx.xxxx.----
  581. # + abcd.0000.abcd.0000.0000.abcd.----.----.----
  582. # - abcd.----.----.----.----.----.----.----
  583. $code.=<<___;
  584. @ multiplication-less reduction $i
  585. adds @acc[3],@acc[3],@acc[0] @ r[3]+=r[0]
  586. ldr $bj,[sp,#40] @ restore b_ptr
  587. adcs @acc[4],@acc[4],#0 @ r[4]+=0
  588. adcs @acc[5],@acc[5],#0 @ r[5]+=0
  589. adcs @acc[6],@acc[6],@acc[0] @ r[6]+=r[0]
  590. ldr $t1,[sp,#0] @ load a[0]
  591. adcs @acc[7],@acc[7],#0 @ r[7]+=0
  592. ldr $bj,[$bj,#4*$i] @ load b[i]
  593. adcs @acc[8],@acc[8],@acc[0] @ r[8]+=r[0]
  594. eor $t0,$t0,$t0
  595. adc $t3,$t3,#0 @ overflow bit
  596. subs @acc[7],@acc[7],@acc[0] @ r[7]-=r[0]
  597. ldr $t2,[sp,#4] @ a[1]
  598. sbcs @acc[8],@acc[8],#0 @ r[8]-=0
  599. umlal @acc[1],$t0,$t1,$bj @ "r[0]"+=a[0]*b[i]
  600. eor $t1,$t1,$t1
  601. sbc @acc[0],$t3,#0 @ overflow bit, keep in mind
  602. @ that netto result is
  603. @ addition of a value which
  604. @ makes underflow impossible
  605. ldr $t3,[sp,#8] @ a[2]
  606. umlal @acc[2],$t1,$t2,$bj @ "r[1]"+=a[1]*b[i]
  607. str @acc[0],[sp,#36] @ temporarily offload overflow
  608. eor $t2,$t2,$t2
  609. ldr $t4,[sp,#12] @ a[3], $t4 is alias @acc[0]
  610. umlal @acc[3],$t2,$t3,$bj @ "r[2]"+=a[2]*b[i]
  611. eor $t3,$t3,$t3
  612. adds @acc[2],@acc[2],$t0 @ accumulate high part of mult
  613. ldr $t0,[sp,#16] @ a[4]
  614. umlal @acc[4],$t3,$t4,$bj @ "r[3]"+=a[3]*b[i]
  615. eor $t4,$t4,$t4
  616. adcs @acc[3],@acc[3],$t1
  617. ldr $t1,[sp,#20] @ a[5]
  618. umlal @acc[5],$t4,$t0,$bj @ "r[4]"+=a[4]*b[i]
  619. eor $t0,$t0,$t0
  620. adcs @acc[4],@acc[4],$t2
  621. ldr $t2,[sp,#24] @ a[6]
  622. umlal @acc[6],$t0,$t1,$bj @ "r[5]"+=a[5]*b[i]
  623. eor $t1,$t1,$t1
  624. adcs @acc[5],@acc[5],$t3
  625. ldr $t3,[sp,#28] @ a[7]
  626. umlal @acc[7],$t1,$t2,$bj @ "r[6]"+=a[6]*b[i]
  627. eor $t2,$t2,$t2
  628. adcs @acc[6],@acc[6],$t4
  629. ldr @acc[0],[sp,#36] @ restore overflow bit
  630. umlal @acc[8],$t2,$t3,$bj @ "r[7]"+=a[7]*b[i]
  631. eor $t3,$t3,$t3
  632. adcs @acc[7],@acc[7],$t0
  633. adcs @acc[8],@acc[8],$t1
  634. adcs @acc[0],$acc[0],$t2
  635. adc $t3,$t3,#0 @ new overflow bit
  636. ___
  637. push(@acc,shift(@acc)); # rotate registers, so that
  638. # "r[i]" becomes r[i]
  639. }
  640. $code.=<<___;
  641. @ last multiplication-less reduction
  642. adds @acc[3],@acc[3],@acc[0]
  643. ldr $r_ptr,[sp,#32] @ restore r_ptr
  644. adcs @acc[4],@acc[4],#0
  645. adcs @acc[5],@acc[5],#0
  646. adcs @acc[6],@acc[6],@acc[0]
  647. adcs @acc[7],@acc[7],#0
  648. adcs @acc[8],@acc[8],@acc[0]
  649. adc $t3,$t3,#0
  650. subs @acc[7],@acc[7],@acc[0]
  651. sbcs @acc[8],@acc[8],#0
  652. sbc @acc[0],$t3,#0 @ overflow bit
  653. @ Final step is "if result > mod, subtract mod", but we do it
  654. @ "other way around", namely subtract modulus from result
  655. @ and if it borrowed, add modulus back.
  656. adds @acc[1],@acc[1],#1 @ subs @acc[1],@acc[1],#-1
  657. adcs @acc[2],@acc[2],#0 @ sbcs @acc[2],@acc[2],#-1
  658. adcs @acc[3],@acc[3],#0 @ sbcs @acc[3],@acc[3],#-1
  659. sbcs @acc[4],@acc[4],#0
  660. sbcs @acc[5],@acc[5],#0
  661. sbcs @acc[6],@acc[6],#0
  662. sbcs @acc[7],@acc[7],#1
  663. adcs @acc[8],@acc[8],#0 @ sbcs @acc[8],@acc[8],#-1
  664. ldr lr,[sp,#44] @ restore lr
  665. sbc @acc[0],@acc[0],#0 @ broadcast borrow bit
  666. add sp,sp,#48
  667. @ Note that because mod has special form, i.e. consists of
  668. @ 0xffffffff, 1 and 0s, we can conditionally synthesize it by
  669. @ broadcasting borrow bit to a register, @acc[0], and using it as
  670. @ a whole or extracting single bit.
  671. adds @acc[1],@acc[1],@acc[0] @ add modulus or zero
  672. adcs @acc[2],@acc[2],@acc[0]
  673. str @acc[1],[$r_ptr,#0]
  674. adcs @acc[3],@acc[3],@acc[0]
  675. str @acc[2],[$r_ptr,#4]
  676. adcs @acc[4],@acc[4],#0
  677. str @acc[3],[$r_ptr,#8]
  678. adcs @acc[5],@acc[5],#0
  679. str @acc[4],[$r_ptr,#12]
  680. adcs @acc[6],@acc[6],#0
  681. str @acc[5],[$r_ptr,#16]
  682. adcs @acc[7],@acc[7],@acc[0],lsr#31
  683. str @acc[6],[$r_ptr,#20]
  684. adc @acc[8],@acc[8],@acc[0]
  685. str @acc[7],[$r_ptr,#24]
  686. str @acc[8],[$r_ptr,#28]
  687. mov pc,lr
  688. .size __ecp_nistz256_mul_mont,.-__ecp_nistz256_mul_mont
  689. ___
  690. }
  691. {
  692. my ($out,$inp,$index,$mask)=map("r$_",(0..3));
  693. $code.=<<___;
  694. @ void ecp_nistz256_scatter_w5(void *r0,const P256_POINT *r1,
  695. @ int r2);
  696. .globl ecp_nistz256_scatter_w5
  697. .type ecp_nistz256_scatter_w5,%function
  698. .align 5
  699. ecp_nistz256_scatter_w5:
  700. stmdb sp!,{r4-r11}
  701. add $out,$out,$index,lsl#2
  702. ldmia $inp!,{r4-r11} @ X
  703. str r4,[$out,#64*0-4]
  704. str r5,[$out,#64*1-4]
  705. str r6,[$out,#64*2-4]
  706. str r7,[$out,#64*3-4]
  707. str r8,[$out,#64*4-4]
  708. str r9,[$out,#64*5-4]
  709. str r10,[$out,#64*6-4]
  710. str r11,[$out,#64*7-4]
  711. add $out,$out,#64*8
  712. ldmia $inp!,{r4-r11} @ Y
  713. str r4,[$out,#64*0-4]
  714. str r5,[$out,#64*1-4]
  715. str r6,[$out,#64*2-4]
  716. str r7,[$out,#64*3-4]
  717. str r8,[$out,#64*4-4]
  718. str r9,[$out,#64*5-4]
  719. str r10,[$out,#64*6-4]
  720. str r11,[$out,#64*7-4]
  721. add $out,$out,#64*8
  722. ldmia $inp,{r4-r11} @ Z
  723. str r4,[$out,#64*0-4]
  724. str r5,[$out,#64*1-4]
  725. str r6,[$out,#64*2-4]
  726. str r7,[$out,#64*3-4]
  727. str r8,[$out,#64*4-4]
  728. str r9,[$out,#64*5-4]
  729. str r10,[$out,#64*6-4]
  730. str r11,[$out,#64*7-4]
  731. ldmia sp!,{r4-r11}
  732. #if __ARM_ARCH__>=5 || defined(__thumb__)
  733. bx lr
  734. #else
  735. mov pc,lr
  736. #endif
  737. .size ecp_nistz256_scatter_w5,.-ecp_nistz256_scatter_w5
  738. @ void ecp_nistz256_gather_w5(P256_POINT *r0,const void *r1,
  739. @ int r2);
  740. .globl ecp_nistz256_gather_w5
  741. .type ecp_nistz256_gather_w5,%function
  742. .align 5
  743. ecp_nistz256_gather_w5:
  744. stmdb sp!,{r4-r11}
  745. cmp $index,#0
  746. mov $mask,#0
  747. #ifdef __thumb2__
  748. itt ne
  749. #endif
  750. subne $index,$index,#1
  751. movne $mask,#-1
  752. add $inp,$inp,$index,lsl#2
  753. ldr r4,[$inp,#64*0]
  754. ldr r5,[$inp,#64*1]
  755. ldr r6,[$inp,#64*2]
  756. and r4,r4,$mask
  757. ldr r7,[$inp,#64*3]
  758. and r5,r5,$mask
  759. ldr r8,[$inp,#64*4]
  760. and r6,r6,$mask
  761. ldr r9,[$inp,#64*5]
  762. and r7,r7,$mask
  763. ldr r10,[$inp,#64*6]
  764. and r8,r8,$mask
  765. ldr r11,[$inp,#64*7]
  766. add $inp,$inp,#64*8
  767. and r9,r9,$mask
  768. and r10,r10,$mask
  769. and r11,r11,$mask
  770. stmia $out!,{r4-r11} @ X
  771. ldr r4,[$inp,#64*0]
  772. ldr r5,[$inp,#64*1]
  773. ldr r6,[$inp,#64*2]
  774. and r4,r4,$mask
  775. ldr r7,[$inp,#64*3]
  776. and r5,r5,$mask
  777. ldr r8,[$inp,#64*4]
  778. and r6,r6,$mask
  779. ldr r9,[$inp,#64*5]
  780. and r7,r7,$mask
  781. ldr r10,[$inp,#64*6]
  782. and r8,r8,$mask
  783. ldr r11,[$inp,#64*7]
  784. add $inp,$inp,#64*8
  785. and r9,r9,$mask
  786. and r10,r10,$mask
  787. and r11,r11,$mask
  788. stmia $out!,{r4-r11} @ Y
  789. ldr r4,[$inp,#64*0]
  790. ldr r5,[$inp,#64*1]
  791. ldr r6,[$inp,#64*2]
  792. and r4,r4,$mask
  793. ldr r7,[$inp,#64*3]
  794. and r5,r5,$mask
  795. ldr r8,[$inp,#64*4]
  796. and r6,r6,$mask
  797. ldr r9,[$inp,#64*5]
  798. and r7,r7,$mask
  799. ldr r10,[$inp,#64*6]
  800. and r8,r8,$mask
  801. ldr r11,[$inp,#64*7]
  802. and r9,r9,$mask
  803. and r10,r10,$mask
  804. and r11,r11,$mask
  805. stmia $out,{r4-r11} @ Z
  806. ldmia sp!,{r4-r11}
  807. #if __ARM_ARCH__>=5 || defined(__thumb__)
  808. bx lr
  809. #else
  810. mov pc,lr
  811. #endif
  812. .size ecp_nistz256_gather_w5,.-ecp_nistz256_gather_w5
  813. @ void ecp_nistz256_scatter_w7(void *r0,const P256_POINT_AFFINE *r1,
  814. @ int r2);
  815. .globl ecp_nistz256_scatter_w7
  816. .type ecp_nistz256_scatter_w7,%function
  817. .align 5
  818. ecp_nistz256_scatter_w7:
  819. add $out,$out,$index
  820. mov $index,#64/4
  821. .Loop_scatter_w7:
  822. ldr $mask,[$inp],#4
  823. subs $index,$index,#1
  824. strb $mask,[$out,#64*0]
  825. mov $mask,$mask,lsr#8
  826. strb $mask,[$out,#64*1]
  827. mov $mask,$mask,lsr#8
  828. strb $mask,[$out,#64*2]
  829. mov $mask,$mask,lsr#8
  830. strb $mask,[$out,#64*3]
  831. add $out,$out,#64*4
  832. bne .Loop_scatter_w7
  833. #if __ARM_ARCH__>=5 || defined(__thumb__)
  834. bx lr
  835. #else
  836. mov pc,lr
  837. #endif
  838. .size ecp_nistz256_scatter_w7,.-ecp_nistz256_scatter_w7
  839. @ void ecp_nistz256_gather_w7(P256_POINT_AFFINE *r0,const void *r1,
  840. @ int r2);
  841. .globl ecp_nistz256_gather_w7
  842. .type ecp_nistz256_gather_w7,%function
  843. .align 5
  844. ecp_nistz256_gather_w7:
  845. stmdb sp!,{r4-r7}
  846. cmp $index,#0
  847. mov $mask,#0
  848. #ifdef __thumb2__
  849. itt ne
  850. #endif
  851. subne $index,$index,#1
  852. movne $mask,#-1
  853. add $inp,$inp,$index
  854. mov $index,#64/4
  855. nop
  856. .Loop_gather_w7:
  857. ldrb r4,[$inp,#64*0]
  858. subs $index,$index,#1
  859. ldrb r5,[$inp,#64*1]
  860. ldrb r6,[$inp,#64*2]
  861. ldrb r7,[$inp,#64*3]
  862. add $inp,$inp,#64*4
  863. orr r4,r4,r5,lsl#8
  864. orr r4,r4,r6,lsl#16
  865. orr r4,r4,r7,lsl#24
  866. and r4,r4,$mask
  867. str r4,[$out],#4
  868. bne .Loop_gather_w7
  869. ldmia sp!,{r4-r7}
  870. #if __ARM_ARCH__>=5 || defined(__thumb__)
  871. bx lr
  872. #else
  873. mov pc,lr
  874. #endif
  875. .size ecp_nistz256_gather_w7,.-ecp_nistz256_gather_w7
  876. ___
  877. }
  878. if (0) {
  879. # In comparison to integer-only equivalent of below subroutine:
  880. #
  881. # Cortex-A8 +10%
  882. # Cortex-A9 -10%
  883. # Snapdragon S4 +5%
  884. #
  885. # As not all time is spent in multiplication, overall impact is deemed
  886. # too low to care about.
  887. my ($A0,$A1,$A2,$A3,$Bi,$zero,$temp)=map("d$_",(0..7));
  888. my $mask="q4";
  889. my $mult="q5";
  890. my @AxB=map("q$_",(8..15));
  891. my ($rptr,$aptr,$bptr,$toutptr)=map("r$_",(0..3));
  892. $code.=<<___;
  893. #if __ARM_ARCH__>=7
  894. .fpu neon
  895. .globl ecp_nistz256_mul_mont_neon
  896. .type ecp_nistz256_mul_mont_neon,%function
  897. .align 5
  898. ecp_nistz256_mul_mont_neon:
  899. mov ip,sp
  900. stmdb sp!,{r4-r9}
  901. vstmdb sp!,{q4-q5} @ ABI specification says so
  902. sub $toutptr,sp,#40
  903. vld1.32 {${Bi}[0]},[$bptr,:32]!
  904. veor $zero,$zero,$zero
  905. vld1.32 {$A0-$A3}, [$aptr] @ can't specify :32 :-(
  906. vzip.16 $Bi,$zero
  907. mov sp,$toutptr @ alloca
  908. vmov.i64 $mask,#0xffff
  909. vmull.u32 @AxB[0],$Bi,${A0}[0]
  910. vmull.u32 @AxB[1],$Bi,${A0}[1]
  911. vmull.u32 @AxB[2],$Bi,${A1}[0]
  912. vmull.u32 @AxB[3],$Bi,${A1}[1]
  913. vshr.u64 $temp,@AxB[0]#lo,#16
  914. vmull.u32 @AxB[4],$Bi,${A2}[0]
  915. vadd.u64 @AxB[0]#hi,@AxB[0]#hi,$temp
  916. vmull.u32 @AxB[5],$Bi,${A2}[1]
  917. vshr.u64 $temp,@AxB[0]#hi,#16 @ upper 32 bits of a[0]*b[0]
  918. vmull.u32 @AxB[6],$Bi,${A3}[0]
  919. vand.u64 @AxB[0],@AxB[0],$mask @ lower 32 bits of a[0]*b[0]
  920. vmull.u32 @AxB[7],$Bi,${A3}[1]
  921. ___
  922. for($i=1;$i<8;$i++) {
  923. $code.=<<___;
  924. vld1.32 {${Bi}[0]},[$bptr,:32]!
  925. veor $zero,$zero,$zero
  926. vadd.u64 @AxB[1]#lo,@AxB[1]#lo,$temp @ reduction
  927. vshl.u64 $mult,@AxB[0],#32
  928. vadd.u64 @AxB[3],@AxB[3],@AxB[0]
  929. vsub.u64 $mult,$mult,@AxB[0]
  930. vzip.16 $Bi,$zero
  931. vadd.u64 @AxB[6],@AxB[6],@AxB[0]
  932. vadd.u64 @AxB[7],@AxB[7],$mult
  933. ___
  934. push(@AxB,shift(@AxB));
  935. $code.=<<___;
  936. vmlal.u32 @AxB[0],$Bi,${A0}[0]
  937. vmlal.u32 @AxB[1],$Bi,${A0}[1]
  938. vmlal.u32 @AxB[2],$Bi,${A1}[0]
  939. vmlal.u32 @AxB[3],$Bi,${A1}[1]
  940. vshr.u64 $temp,@AxB[0]#lo,#16
  941. vmlal.u32 @AxB[4],$Bi,${A2}[0]
  942. vadd.u64 @AxB[0]#hi,@AxB[0]#hi,$temp
  943. vmlal.u32 @AxB[5],$Bi,${A2}[1]
  944. vshr.u64 $temp,@AxB[0]#hi,#16 @ upper 33 bits of a[0]*b[i]+t[0]
  945. vmlal.u32 @AxB[6],$Bi,${A3}[0]
  946. vand.u64 @AxB[0],@AxB[0],$mask @ lower 32 bits of a[0]*b[0]
  947. vmull.u32 @AxB[7],$Bi,${A3}[1]
  948. ___
  949. }
  950. $code.=<<___;
  951. vadd.u64 @AxB[1]#lo,@AxB[1]#lo,$temp @ last reduction
  952. vshl.u64 $mult,@AxB[0],#32
  953. vadd.u64 @AxB[3],@AxB[3],@AxB[0]
  954. vsub.u64 $mult,$mult,@AxB[0]
  955. vadd.u64 @AxB[6],@AxB[6],@AxB[0]
  956. vadd.u64 @AxB[7],@AxB[7],$mult
  957. vshr.u64 $temp,@AxB[1]#lo,#16 @ convert
  958. vadd.u64 @AxB[1]#hi,@AxB[1]#hi,$temp
  959. vshr.u64 $temp,@AxB[1]#hi,#16
  960. vzip.16 @AxB[1]#lo,@AxB[1]#hi
  961. ___
  962. foreach (2..7) {
  963. $code.=<<___;
  964. vadd.u64 @AxB[$_]#lo,@AxB[$_]#lo,$temp
  965. vst1.32 {@AxB[$_-1]#lo[0]},[$toutptr,:32]!
  966. vshr.u64 $temp,@AxB[$_]#lo,#16
  967. vadd.u64 @AxB[$_]#hi,@AxB[$_]#hi,$temp
  968. vshr.u64 $temp,@AxB[$_]#hi,#16
  969. vzip.16 @AxB[$_]#lo,@AxB[$_]#hi
  970. ___
  971. }
  972. $code.=<<___;
  973. vst1.32 {@AxB[7]#lo[0]},[$toutptr,:32]!
  974. vst1.32 {$temp},[$toutptr] @ upper 33 bits
  975. ldr r1,[sp,#0]
  976. ldr r2,[sp,#4]
  977. ldr r3,[sp,#8]
  978. subs r1,r1,#-1
  979. ldr r4,[sp,#12]
  980. sbcs r2,r2,#-1
  981. ldr r5,[sp,#16]
  982. sbcs r3,r3,#-1
  983. ldr r6,[sp,#20]
  984. sbcs r4,r4,#0
  985. ldr r7,[sp,#24]
  986. sbcs r5,r5,#0
  987. ldr r8,[sp,#28]
  988. sbcs r6,r6,#0
  989. ldr r9,[sp,#32] @ top-most bit
  990. sbcs r7,r7,#1
  991. sub sp,ip,#40+16
  992. sbcs r8,r8,#-1
  993. sbc r9,r9,#0
  994. vldmia sp!,{q4-q5}
  995. adds r1,r1,r9
  996. adcs r2,r2,r9
  997. str r1,[$rptr,#0]
  998. adcs r3,r3,r9
  999. str r2,[$rptr,#4]
  1000. adcs r4,r4,#0
  1001. str r3,[$rptr,#8]
  1002. adcs r5,r5,#0
  1003. str r4,[$rptr,#12]
  1004. adcs r6,r6,#0
  1005. str r5,[$rptr,#16]
  1006. adcs r7,r7,r9,lsr#31
  1007. str r6,[$rptr,#20]
  1008. adcs r8,r8,r9
  1009. str r7,[$rptr,#24]
  1010. str r8,[$rptr,#28]
  1011. ldmia sp!,{r4-r9}
  1012. bx lr
  1013. .size ecp_nistz256_mul_mont_neon,.-ecp_nistz256_mul_mont_neon
  1014. #endif
  1015. ___
  1016. }
  1017. {{{
  1018. ########################################################################
  1019. # Below $aN assignment matches order in which 256-bit result appears in
  1020. # register bank at return from __ecp_nistz256_mul_mont, so that we can
  1021. # skip over reloading it from memory. This means that below functions
  1022. # use custom calling sequence accepting 256-bit input in registers,
  1023. # output pointer in r0, $r_ptr, and optional pointer in r2, $b_ptr.
  1024. #
  1025. # See their "normal" counterparts for insights on calculations.
  1026. my ($a0,$a1,$a2,$a3,$a4,$a5,$a6,$a7,
  1027. $t0,$t1,$t2,$t3)=map("r$_",(11,3..10,12,14,1));
  1028. my $ff=$b_ptr;
  1029. $code.=<<___;
  1030. .type __ecp_nistz256_sub_from,%function
  1031. .align 5
  1032. __ecp_nistz256_sub_from:
  1033. str lr,[sp,#-4]! @ push lr
  1034. ldr $t0,[$b_ptr,#0]
  1035. ldr $t1,[$b_ptr,#4]
  1036. ldr $t2,[$b_ptr,#8]
  1037. ldr $t3,[$b_ptr,#12]
  1038. subs $a0,$a0,$t0
  1039. ldr $t0,[$b_ptr,#16]
  1040. sbcs $a1,$a1,$t1
  1041. ldr $t1,[$b_ptr,#20]
  1042. sbcs $a2,$a2,$t2
  1043. ldr $t2,[$b_ptr,#24]
  1044. sbcs $a3,$a3,$t3
  1045. ldr $t3,[$b_ptr,#28]
  1046. sbcs $a4,$a4,$t0
  1047. sbcs $a5,$a5,$t1
  1048. sbcs $a6,$a6,$t2
  1049. sbcs $a7,$a7,$t3
  1050. sbc $ff,$ff,$ff @ broadcast borrow bit
  1051. ldr lr,[sp],#4 @ pop lr
  1052. adds $a0,$a0,$ff @ add synthesized modulus
  1053. adcs $a1,$a1,$ff
  1054. str $a0,[$r_ptr,#0]
  1055. adcs $a2,$a2,$ff
  1056. str $a1,[$r_ptr,#4]
  1057. adcs $a3,$a3,#0
  1058. str $a2,[$r_ptr,#8]
  1059. adcs $a4,$a4,#0
  1060. str $a3,[$r_ptr,#12]
  1061. adcs $a5,$a5,#0
  1062. str $a4,[$r_ptr,#16]
  1063. adcs $a6,$a6,$ff,lsr#31
  1064. str $a5,[$r_ptr,#20]
  1065. adcs $a7,$a7,$ff
  1066. str $a6,[$r_ptr,#24]
  1067. str $a7,[$r_ptr,#28]
  1068. mov pc,lr
  1069. .size __ecp_nistz256_sub_from,.-__ecp_nistz256_sub_from
  1070. .type __ecp_nistz256_sub_morf,%function
  1071. .align 5
  1072. __ecp_nistz256_sub_morf:
  1073. str lr,[sp,#-4]! @ push lr
  1074. ldr $t0,[$b_ptr,#0]
  1075. ldr $t1,[$b_ptr,#4]
  1076. ldr $t2,[$b_ptr,#8]
  1077. ldr $t3,[$b_ptr,#12]
  1078. subs $a0,$t0,$a0
  1079. ldr $t0,[$b_ptr,#16]
  1080. sbcs $a1,$t1,$a1
  1081. ldr $t1,[$b_ptr,#20]
  1082. sbcs $a2,$t2,$a2
  1083. ldr $t2,[$b_ptr,#24]
  1084. sbcs $a3,$t3,$a3
  1085. ldr $t3,[$b_ptr,#28]
  1086. sbcs $a4,$t0,$a4
  1087. sbcs $a5,$t1,$a5
  1088. sbcs $a6,$t2,$a6
  1089. sbcs $a7,$t3,$a7
  1090. sbc $ff,$ff,$ff @ broadcast borrow bit
  1091. ldr lr,[sp],#4 @ pop lr
  1092. adds $a0,$a0,$ff @ add synthesized modulus
  1093. adcs $a1,$a1,$ff
  1094. str $a0,[$r_ptr,#0]
  1095. adcs $a2,$a2,$ff
  1096. str $a1,[$r_ptr,#4]
  1097. adcs $a3,$a3,#0
  1098. str $a2,[$r_ptr,#8]
  1099. adcs $a4,$a4,#0
  1100. str $a3,[$r_ptr,#12]
  1101. adcs $a5,$a5,#0
  1102. str $a4,[$r_ptr,#16]
  1103. adcs $a6,$a6,$ff,lsr#31
  1104. str $a5,[$r_ptr,#20]
  1105. adcs $a7,$a7,$ff
  1106. str $a6,[$r_ptr,#24]
  1107. str $a7,[$r_ptr,#28]
  1108. mov pc,lr
  1109. .size __ecp_nistz256_sub_morf,.-__ecp_nistz256_sub_morf
  1110. .type __ecp_nistz256_add_self,%function
  1111. .align 4
  1112. __ecp_nistz256_add_self:
  1113. adds $a0,$a0,$a0 @ a[0:7]+=a[0:7]
  1114. adcs $a1,$a1,$a1
  1115. adcs $a2,$a2,$a2
  1116. adcs $a3,$a3,$a3
  1117. adcs $a4,$a4,$a4
  1118. adcs $a5,$a5,$a5
  1119. adcs $a6,$a6,$a6
  1120. mov $ff,#0
  1121. adcs $a7,$a7,$a7
  1122. adc $ff,$ff,#0
  1123. @ if a+b >= modulus, subtract modulus.
  1124. @
  1125. @ But since comparison implies subtraction, we subtract
  1126. @ modulus and then add it back if subtraction borrowed.
  1127. subs $a0,$a0,#-1
  1128. sbcs $a1,$a1,#-1
  1129. sbcs $a2,$a2,#-1
  1130. sbcs $a3,$a3,#0
  1131. sbcs $a4,$a4,#0
  1132. sbcs $a5,$a5,#0
  1133. sbcs $a6,$a6,#1
  1134. sbcs $a7,$a7,#-1
  1135. sbc $ff,$ff,#0
  1136. @ Note that because mod has special form, i.e. consists of
  1137. @ 0xffffffff, 1 and 0s, we can conditionally synthesize it by
  1138. @ using value of borrow as a whole or extracting single bit.
  1139. @ Follow $ff register...
  1140. adds $a0,$a0,$ff @ add synthesized modulus
  1141. adcs $a1,$a1,$ff
  1142. str $a0,[$r_ptr,#0]
  1143. adcs $a2,$a2,$ff
  1144. str $a1,[$r_ptr,#4]
  1145. adcs $a3,$a3,#0
  1146. str $a2,[$r_ptr,#8]
  1147. adcs $a4,$a4,#0
  1148. str $a3,[$r_ptr,#12]
  1149. adcs $a5,$a5,#0
  1150. str $a4,[$r_ptr,#16]
  1151. adcs $a6,$a6,$ff,lsr#31
  1152. str $a5,[$r_ptr,#20]
  1153. adcs $a7,$a7,$ff
  1154. str $a6,[$r_ptr,#24]
  1155. str $a7,[$r_ptr,#28]
  1156. mov pc,lr
  1157. .size __ecp_nistz256_add_self,.-__ecp_nistz256_add_self
  1158. ___
  1159. ########################################################################
  1160. # following subroutines are "literal" implementation of those found in
  1161. # ecp_nistz256.c
  1162. #
  1163. ########################################################################
  1164. # void ecp_nistz256_point_double(P256_POINT *out,const P256_POINT *inp);
  1165. #
  1166. {
  1167. my ($S,$M,$Zsqr,$in_x,$tmp0)=map(32*$_,(0..4));
  1168. # above map() describes stack layout with 5 temporary
  1169. # 256-bit vectors on top. Then note that we push
  1170. # starting from r0, which means that we have copy of
  1171. # input arguments just below these temporary vectors.
  1172. $code.=<<___;
  1173. .globl ecp_nistz256_point_double
  1174. .type ecp_nistz256_point_double,%function
  1175. .align 5
  1176. ecp_nistz256_point_double:
  1177. stmdb sp!,{r0-r12,lr} @ push from r0, unusual, but intentional
  1178. sub sp,sp,#32*5
  1179. .Lpoint_double_shortcut:
  1180. add r3,sp,#$in_x
  1181. ldmia $a_ptr!,{r4-r11} @ copy in_x
  1182. stmia r3,{r4-r11}
  1183. add $r_ptr,sp,#$S
  1184. bl __ecp_nistz256_mul_by_2 @ p256_mul_by_2(S, in_y);
  1185. add $b_ptr,$a_ptr,#32
  1186. add $a_ptr,$a_ptr,#32
  1187. add $r_ptr,sp,#$Zsqr
  1188. bl __ecp_nistz256_mul_mont @ p256_sqr_mont(Zsqr, in_z);
  1189. add $a_ptr,sp,#$S
  1190. add $b_ptr,sp,#$S
  1191. add $r_ptr,sp,#$S
  1192. bl __ecp_nistz256_mul_mont @ p256_sqr_mont(S, S);
  1193. ldr $b_ptr,[sp,#32*5+4]
  1194. add $a_ptr,$b_ptr,#32
  1195. add $b_ptr,$b_ptr,#64
  1196. add $r_ptr,sp,#$tmp0
  1197. bl __ecp_nistz256_mul_mont @ p256_mul_mont(tmp0, in_z, in_y);
  1198. ldr $r_ptr,[sp,#32*5]
  1199. add $r_ptr,$r_ptr,#64
  1200. bl __ecp_nistz256_add_self @ p256_mul_by_2(res_z, tmp0);
  1201. add $a_ptr,sp,#$in_x
  1202. add $b_ptr,sp,#$Zsqr
  1203. add $r_ptr,sp,#$M
  1204. bl __ecp_nistz256_add @ p256_add(M, in_x, Zsqr);
  1205. add $a_ptr,sp,#$in_x
  1206. add $b_ptr,sp,#$Zsqr
  1207. add $r_ptr,sp,#$Zsqr
  1208. bl __ecp_nistz256_sub @ p256_sub(Zsqr, in_x, Zsqr);
  1209. add $a_ptr,sp,#$S
  1210. add $b_ptr,sp,#$S
  1211. add $r_ptr,sp,#$tmp0
  1212. bl __ecp_nistz256_mul_mont @ p256_sqr_mont(tmp0, S);
  1213. add $a_ptr,sp,#$Zsqr
  1214. add $b_ptr,sp,#$M
  1215. add $r_ptr,sp,#$M
  1216. bl __ecp_nistz256_mul_mont @ p256_mul_mont(M, M, Zsqr);
  1217. ldr $r_ptr,[sp,#32*5]
  1218. add $a_ptr,sp,#$tmp0
  1219. add $r_ptr,$r_ptr,#32
  1220. bl __ecp_nistz256_div_by_2 @ p256_div_by_2(res_y, tmp0);
  1221. add $a_ptr,sp,#$M
  1222. add $r_ptr,sp,#$M
  1223. bl __ecp_nistz256_mul_by_3 @ p256_mul_by_3(M, M);
  1224. add $a_ptr,sp,#$in_x
  1225. add $b_ptr,sp,#$S
  1226. add $r_ptr,sp,#$S
  1227. bl __ecp_nistz256_mul_mont @ p256_mul_mont(S, S, in_x);
  1228. add $r_ptr,sp,#$tmp0
  1229. bl __ecp_nistz256_add_self @ p256_mul_by_2(tmp0, S);
  1230. ldr $r_ptr,[sp,#32*5]
  1231. add $a_ptr,sp,#$M
  1232. add $b_ptr,sp,#$M
  1233. bl __ecp_nistz256_mul_mont @ p256_sqr_mont(res_x, M);
  1234. add $b_ptr,sp,#$tmp0
  1235. bl __ecp_nistz256_sub_from @ p256_sub(res_x, res_x, tmp0);
  1236. add $b_ptr,sp,#$S
  1237. add $r_ptr,sp,#$S
  1238. bl __ecp_nistz256_sub_morf @ p256_sub(S, S, res_x);
  1239. add $a_ptr,sp,#$M
  1240. add $b_ptr,sp,#$S
  1241. bl __ecp_nistz256_mul_mont @ p256_mul_mont(S, S, M);
  1242. ldr $r_ptr,[sp,#32*5]
  1243. add $b_ptr,$r_ptr,#32
  1244. add $r_ptr,$r_ptr,#32
  1245. bl __ecp_nistz256_sub_from @ p256_sub(res_y, S, res_y);
  1246. add sp,sp,#32*5+16 @ +16 means "skip even over saved r0-r3"
  1247. #if __ARM_ARCH__>=5 || !defined(__thumb__)
  1248. ldmia sp!,{r4-r12,pc}
  1249. #else
  1250. ldmia sp!,{r4-r12,lr}
  1251. bx lr @ interoperable with Thumb ISA:-)
  1252. #endif
  1253. .size ecp_nistz256_point_double,.-ecp_nistz256_point_double
  1254. ___
  1255. }
  1256. ########################################################################
  1257. # void ecp_nistz256_point_add(P256_POINT *out,const P256_POINT *in1,
  1258. # const P256_POINT *in2);
  1259. {
  1260. my ($res_x,$res_y,$res_z,
  1261. $in1_x,$in1_y,$in1_z,
  1262. $in2_x,$in2_y,$in2_z,
  1263. $H,$Hsqr,$R,$Rsqr,$Hcub,
  1264. $U1,$U2,$S1,$S2)=map(32*$_,(0..17));
  1265. my ($Z1sqr, $Z2sqr) = ($Hsqr, $Rsqr);
  1266. # above map() describes stack layout with 18 temporary
  1267. # 256-bit vectors on top. Then note that we push
  1268. # starting from r0, which means that we have copy of
  1269. # input arguments just below these temporary vectors.
  1270. # We use three of them for ~in1infty, ~in2infty and
  1271. # result of check for zero.
  1272. $code.=<<___;
  1273. .globl ecp_nistz256_point_add
  1274. .type ecp_nistz256_point_add,%function
  1275. .align 5
  1276. ecp_nistz256_point_add:
  1277. stmdb sp!,{r0-r12,lr} @ push from r0, unusual, but intentional
  1278. sub sp,sp,#32*18+16
  1279. ldmia $b_ptr!,{r4-r11} @ copy in2_x
  1280. add r3,sp,#$in2_x
  1281. stmia r3!,{r4-r11}
  1282. ldmia $b_ptr!,{r4-r11} @ copy in2_y
  1283. stmia r3!,{r4-r11}
  1284. ldmia $b_ptr,{r4-r11} @ copy in2_z
  1285. orr r12,r4,r5
  1286. orr r12,r12,r6
  1287. orr r12,r12,r7
  1288. orr r12,r12,r8
  1289. orr r12,r12,r9
  1290. orr r12,r12,r10
  1291. orr r12,r12,r11
  1292. cmp r12,#0
  1293. #ifdef __thumb2__
  1294. it ne
  1295. #endif
  1296. movne r12,#-1
  1297. stmia r3,{r4-r11}
  1298. str r12,[sp,#32*18+8] @ ~in2infty
  1299. ldmia $a_ptr!,{r4-r11} @ copy in1_x
  1300. add r3,sp,#$in1_x
  1301. stmia r3!,{r4-r11}
  1302. ldmia $a_ptr!,{r4-r11} @ copy in1_y
  1303. stmia r3!,{r4-r11}
  1304. ldmia $a_ptr,{r4-r11} @ copy in1_z
  1305. orr r12,r4,r5
  1306. orr r12,r12,r6
  1307. orr r12,r12,r7
  1308. orr r12,r12,r8
  1309. orr r12,r12,r9
  1310. orr r12,r12,r10
  1311. orr r12,r12,r11
  1312. cmp r12,#0
  1313. #ifdef __thumb2__
  1314. it ne
  1315. #endif
  1316. movne r12,#-1
  1317. stmia r3,{r4-r11}
  1318. str r12,[sp,#32*18+4] @ ~in1infty
  1319. add $a_ptr,sp,#$in2_z
  1320. add $b_ptr,sp,#$in2_z
  1321. add $r_ptr,sp,#$Z2sqr
  1322. bl __ecp_nistz256_mul_mont @ p256_sqr_mont(Z2sqr, in2_z);
  1323. add $a_ptr,sp,#$in1_z
  1324. add $b_ptr,sp,#$in1_z
  1325. add $r_ptr,sp,#$Z1sqr
  1326. bl __ecp_nistz256_mul_mont @ p256_sqr_mont(Z1sqr, in1_z);
  1327. add $a_ptr,sp,#$in2_z
  1328. add $b_ptr,sp,#$Z2sqr
  1329. add $r_ptr,sp,#$S1
  1330. bl __ecp_nistz256_mul_mont @ p256_mul_mont(S1, Z2sqr, in2_z);
  1331. add $a_ptr,sp,#$in1_z
  1332. add $b_ptr,sp,#$Z1sqr
  1333. add $r_ptr,sp,#$S2
  1334. bl __ecp_nistz256_mul_mont @ p256_mul_mont(S2, Z1sqr, in1_z);
  1335. add $a_ptr,sp,#$in1_y
  1336. add $b_ptr,sp,#$S1
  1337. add $r_ptr,sp,#$S1
  1338. bl __ecp_nistz256_mul_mont @ p256_mul_mont(S1, S1, in1_y);
  1339. add $a_ptr,sp,#$in2_y
  1340. add $b_ptr,sp,#$S2
  1341. add $r_ptr,sp,#$S2
  1342. bl __ecp_nistz256_mul_mont @ p256_mul_mont(S2, S2, in2_y);
  1343. add $b_ptr,sp,#$S1
  1344. add $r_ptr,sp,#$R
  1345. bl __ecp_nistz256_sub_from @ p256_sub(R, S2, S1);
  1346. orr $a0,$a0,$a1 @ see if result is zero
  1347. orr $a2,$a2,$a3
  1348. orr $a4,$a4,$a5
  1349. orr $a0,$a0,$a2
  1350. orr $a4,$a4,$a6
  1351. orr $a0,$a0,$a7
  1352. add $a_ptr,sp,#$in1_x
  1353. orr $a0,$a0,$a4
  1354. add $b_ptr,sp,#$Z2sqr
  1355. str $a0,[sp,#32*18+12]
  1356. add $r_ptr,sp,#$U1
  1357. bl __ecp_nistz256_mul_mont @ p256_mul_mont(U1, in1_x, Z2sqr);
  1358. add $a_ptr,sp,#$in2_x
  1359. add $b_ptr,sp,#$Z1sqr
  1360. add $r_ptr,sp,#$U2
  1361. bl __ecp_nistz256_mul_mont @ p256_mul_mont(U2, in2_x, Z1sqr);
  1362. add $b_ptr,sp,#$U1
  1363. add $r_ptr,sp,#$H
  1364. bl __ecp_nistz256_sub_from @ p256_sub(H, U2, U1);
  1365. orr $a0,$a0,$a1 @ see if result is zero
  1366. orr $a2,$a2,$a3
  1367. orr $a4,$a4,$a5
  1368. orr $a0,$a0,$a2
  1369. orr $a4,$a4,$a6
  1370. orr $a0,$a0,$a7
  1371. orr $a0,$a0,$a4 @ ~is_equal(U1,U2)
  1372. ldr $t0,[sp,#32*18+4] @ ~in1infty
  1373. ldr $t1,[sp,#32*18+8] @ ~in2infty
  1374. ldr $t2,[sp,#32*18+12] @ ~is_equal(S1,S2)
  1375. mvn $t0,$t0 @ -1/0 -> 0/-1
  1376. mvn $t1,$t1 @ -1/0 -> 0/-1
  1377. orr $a0,$a0,$t0
  1378. orr $a0,$a0,$t1
  1379. orrs $a0,$a0,$t2 @ set flags
  1380. @ if(~is_equal(U1,U2) | in1infty | in2infty | ~is_equal(S1,S2))
  1381. bne .Ladd_proceed
  1382. .Ladd_double:
  1383. ldr $a_ptr,[sp,#32*18+20]
  1384. add sp,sp,#32*(18-5)+16 @ difference in frame sizes
  1385. b .Lpoint_double_shortcut
  1386. .align 4
  1387. .Ladd_proceed:
  1388. add $a_ptr,sp,#$R
  1389. add $b_ptr,sp,#$R
  1390. add $r_ptr,sp,#$Rsqr
  1391. bl __ecp_nistz256_mul_mont @ p256_sqr_mont(Rsqr, R);
  1392. add $a_ptr,sp,#$H
  1393. add $b_ptr,sp,#$in1_z
  1394. add $r_ptr,sp,#$res_z
  1395. bl __ecp_nistz256_mul_mont @ p256_mul_mont(res_z, H, in1_z);
  1396. add $a_ptr,sp,#$H
  1397. add $b_ptr,sp,#$H
  1398. add $r_ptr,sp,#$Hsqr
  1399. bl __ecp_nistz256_mul_mont @ p256_sqr_mont(Hsqr, H);
  1400. add $a_ptr,sp,#$in2_z
  1401. add $b_ptr,sp,#$res_z
  1402. add $r_ptr,sp,#$res_z
  1403. bl __ecp_nistz256_mul_mont @ p256_mul_mont(res_z, res_z, in2_z);
  1404. add $a_ptr,sp,#$H
  1405. add $b_ptr,sp,#$Hsqr
  1406. add $r_ptr,sp,#$Hcub
  1407. bl __ecp_nistz256_mul_mont @ p256_mul_mont(Hcub, Hsqr, H);
  1408. add $a_ptr,sp,#$Hsqr
  1409. add $b_ptr,sp,#$U1
  1410. add $r_ptr,sp,#$U2
  1411. bl __ecp_nistz256_mul_mont @ p256_mul_mont(U2, U1, Hsqr);
  1412. add $r_ptr,sp,#$Hsqr
  1413. bl __ecp_nistz256_add_self @ p256_mul_by_2(Hsqr, U2);
  1414. add $b_ptr,sp,#$Rsqr
  1415. add $r_ptr,sp,#$res_x
  1416. bl __ecp_nistz256_sub_morf @ p256_sub(res_x, Rsqr, Hsqr);
  1417. add $b_ptr,sp,#$Hcub
  1418. bl __ecp_nistz256_sub_from @ p256_sub(res_x, res_x, Hcub);
  1419. add $b_ptr,sp,#$U2
  1420. add $r_ptr,sp,#$res_y
  1421. bl __ecp_nistz256_sub_morf @ p256_sub(res_y, U2, res_x);
  1422. add $a_ptr,sp,#$Hcub
  1423. add $b_ptr,sp,#$S1
  1424. add $r_ptr,sp,#$S2
  1425. bl __ecp_nistz256_mul_mont @ p256_mul_mont(S2, S1, Hcub);
  1426. add $a_ptr,sp,#$R
  1427. add $b_ptr,sp,#$res_y
  1428. add $r_ptr,sp,#$res_y
  1429. bl __ecp_nistz256_mul_mont @ p256_mul_mont(res_y, res_y, R);
  1430. add $b_ptr,sp,#$S2
  1431. bl __ecp_nistz256_sub_from @ p256_sub(res_y, res_y, S2);
  1432. ldr r11,[sp,#32*18+4] @ ~in1infty
  1433. ldr r12,[sp,#32*18+8] @ ~in2infty
  1434. add r1,sp,#$res_x
  1435. add r2,sp,#$in2_x
  1436. and r10,r11,r12 @ ~in1infty & ~in2infty
  1437. mvn r11,r11
  1438. add r3,sp,#$in1_x
  1439. and r11,r11,r12 @ in1infty & ~in2infty
  1440. mvn r12,r12 @ in2infty
  1441. ldr $r_ptr,[sp,#32*18+16]
  1442. ___
  1443. for($i=0;$i<96;$i+=8) { # conditional moves
  1444. $code.=<<___;
  1445. ldmia r1!,{r4-r5} @ res_x
  1446. ldmia r2!,{r6-r7} @ in2_x
  1447. ldmia r3!,{r8-r9} @ in1_x
  1448. and r4,r4,r10 @ ~in1infty & ~in2infty
  1449. and r5,r5,r10
  1450. and r6,r6,r11 @ in1infty & ~in2infty
  1451. and r7,r7,r11
  1452. and r8,r8,r12 @ in2infty
  1453. and r9,r9,r12
  1454. orr r4,r4,r6
  1455. orr r5,r5,r7
  1456. orr r4,r4,r8
  1457. orr r5,r5,r9
  1458. stmia $r_ptr!,{r4-r5}
  1459. ___
  1460. }
  1461. $code.=<<___;
  1462. .Ladd_done:
  1463. add sp,sp,#32*18+16+16 @ +16 means "skip even over saved r0-r3"
  1464. #if __ARM_ARCH__>=5 || !defined(__thumb__)
  1465. ldmia sp!,{r4-r12,pc}
  1466. #else
  1467. ldmia sp!,{r4-r12,lr}
  1468. bx lr @ interoperable with Thumb ISA:-)
  1469. #endif
  1470. .size ecp_nistz256_point_add,.-ecp_nistz256_point_add
  1471. ___
  1472. }
  1473. ########################################################################
  1474. # void ecp_nistz256_point_add_affine(P256_POINT *out,const P256_POINT *in1,
  1475. # const P256_POINT_AFFINE *in2);
  1476. {
  1477. my ($res_x,$res_y,$res_z,
  1478. $in1_x,$in1_y,$in1_z,
  1479. $in2_x,$in2_y,
  1480. $U2,$S2,$H,$R,$Hsqr,$Hcub,$Rsqr)=map(32*$_,(0..14));
  1481. my $Z1sqr = $S2;
  1482. # above map() describes stack layout with 18 temporary
  1483. # 256-bit vectors on top. Then note that we push
  1484. # starting from r0, which means that we have copy of
  1485. # input arguments just below these temporary vectors.
  1486. # We use two of them for ~in1infty, ~in2infty.
  1487. my @ONE_mont=(1,0,0,-1,-1,-1,-2,0);
  1488. $code.=<<___;
  1489. .globl ecp_nistz256_point_add_affine
  1490. .type ecp_nistz256_point_add_affine,%function
  1491. .align 5
  1492. ecp_nistz256_point_add_affine:
  1493. stmdb sp!,{r0-r12,lr} @ push from r0, unusual, but intentional
  1494. sub sp,sp,#32*15
  1495. ldmia $a_ptr!,{r4-r11} @ copy in1_x
  1496. add r3,sp,#$in1_x
  1497. stmia r3!,{r4-r11}
  1498. ldmia $a_ptr!,{r4-r11} @ copy in1_y
  1499. stmia r3!,{r4-r11}
  1500. ldmia $a_ptr,{r4-r11} @ copy in1_z
  1501. orr r12,r4,r5
  1502. orr r12,r12,r6
  1503. orr r12,r12,r7
  1504. orr r12,r12,r8
  1505. orr r12,r12,r9
  1506. orr r12,r12,r10
  1507. orr r12,r12,r11
  1508. cmp r12,#0
  1509. #ifdef __thumb2__
  1510. it ne
  1511. #endif
  1512. movne r12,#-1
  1513. stmia r3,{r4-r11}
  1514. str r12,[sp,#32*15+4] @ ~in1infty
  1515. ldmia $b_ptr!,{r4-r11} @ copy in2_x
  1516. add r3,sp,#$in2_x
  1517. orr r12,r4,r5
  1518. orr r12,r12,r6
  1519. orr r12,r12,r7
  1520. orr r12,r12,r8
  1521. orr r12,r12,r9
  1522. orr r12,r12,r10
  1523. orr r12,r12,r11
  1524. stmia r3!,{r4-r11}
  1525. ldmia $b_ptr!,{r4-r11} @ copy in2_y
  1526. orr r12,r12,r4
  1527. orr r12,r12,r5
  1528. orr r12,r12,r6
  1529. orr r12,r12,r7
  1530. orr r12,r12,r8
  1531. orr r12,r12,r9
  1532. orr r12,r12,r10
  1533. orr r12,r12,r11
  1534. stmia r3!,{r4-r11}
  1535. cmp r12,#0
  1536. #ifdef __thumb2__
  1537. it ne
  1538. #endif
  1539. movne r12,#-1
  1540. str r12,[sp,#32*15+8] @ ~in2infty
  1541. add $a_ptr,sp,#$in1_z
  1542. add $b_ptr,sp,#$in1_z
  1543. add $r_ptr,sp,#$Z1sqr
  1544. bl __ecp_nistz256_mul_mont @ p256_sqr_mont(Z1sqr, in1_z);
  1545. add $a_ptr,sp,#$Z1sqr
  1546. add $b_ptr,sp,#$in2_x
  1547. add $r_ptr,sp,#$U2
  1548. bl __ecp_nistz256_mul_mont @ p256_mul_mont(U2, Z1sqr, in2_x);
  1549. add $b_ptr,sp,#$in1_x
  1550. add $r_ptr,sp,#$H
  1551. bl __ecp_nistz256_sub_from @ p256_sub(H, U2, in1_x);
  1552. add $a_ptr,sp,#$Z1sqr
  1553. add $b_ptr,sp,#$in1_z
  1554. add $r_ptr,sp,#$S2
  1555. bl __ecp_nistz256_mul_mont @ p256_mul_mont(S2, Z1sqr, in1_z);
  1556. add $a_ptr,sp,#$H
  1557. add $b_ptr,sp,#$in1_z
  1558. add $r_ptr,sp,#$res_z
  1559. bl __ecp_nistz256_mul_mont @ p256_mul_mont(res_z, H, in1_z);
  1560. add $a_ptr,sp,#$in2_y
  1561. add $b_ptr,sp,#$S2
  1562. add $r_ptr,sp,#$S2
  1563. bl __ecp_nistz256_mul_mont @ p256_mul_mont(S2, S2, in2_y);
  1564. add $b_ptr,sp,#$in1_y
  1565. add $r_ptr,sp,#$R
  1566. bl __ecp_nistz256_sub_from @ p256_sub(R, S2, in1_y);
  1567. add $a_ptr,sp,#$H
  1568. add $b_ptr,sp,#$H
  1569. add $r_ptr,sp,#$Hsqr
  1570. bl __ecp_nistz256_mul_mont @ p256_sqr_mont(Hsqr, H);
  1571. add $a_ptr,sp,#$R
  1572. add $b_ptr,sp,#$R
  1573. add $r_ptr,sp,#$Rsqr
  1574. bl __ecp_nistz256_mul_mont @ p256_sqr_mont(Rsqr, R);
  1575. add $a_ptr,sp,#$H
  1576. add $b_ptr,sp,#$Hsqr
  1577. add $r_ptr,sp,#$Hcub
  1578. bl __ecp_nistz256_mul_mont @ p256_mul_mont(Hcub, Hsqr, H);
  1579. add $a_ptr,sp,#$Hsqr
  1580. add $b_ptr,sp,#$in1_x
  1581. add $r_ptr,sp,#$U2
  1582. bl __ecp_nistz256_mul_mont @ p256_mul_mont(U2, in1_x, Hsqr);
  1583. add $r_ptr,sp,#$Hsqr
  1584. bl __ecp_nistz256_add_self @ p256_mul_by_2(Hsqr, U2);
  1585. add $b_ptr,sp,#$Rsqr
  1586. add $r_ptr,sp,#$res_x
  1587. bl __ecp_nistz256_sub_morf @ p256_sub(res_x, Rsqr, Hsqr);
  1588. add $b_ptr,sp,#$Hcub
  1589. bl __ecp_nistz256_sub_from @ p256_sub(res_x, res_x, Hcub);
  1590. add $b_ptr,sp,#$U2
  1591. add $r_ptr,sp,#$res_y
  1592. bl __ecp_nistz256_sub_morf @ p256_sub(res_y, U2, res_x);
  1593. add $a_ptr,sp,#$Hcub
  1594. add $b_ptr,sp,#$in1_y
  1595. add $r_ptr,sp,#$S2
  1596. bl __ecp_nistz256_mul_mont @ p256_mul_mont(S2, in1_y, Hcub);
  1597. add $a_ptr,sp,#$R
  1598. add $b_ptr,sp,#$res_y
  1599. add $r_ptr,sp,#$res_y
  1600. bl __ecp_nistz256_mul_mont @ p256_mul_mont(res_y, res_y, R);
  1601. add $b_ptr,sp,#$S2
  1602. bl __ecp_nistz256_sub_from @ p256_sub(res_y, res_y, S2);
  1603. ldr r11,[sp,#32*15+4] @ ~in1infty
  1604. ldr r12,[sp,#32*15+8] @ ~in2infty
  1605. add r1,sp,#$res_x
  1606. add r2,sp,#$in2_x
  1607. and r10,r11,r12 @ ~in1infty & ~in2infty
  1608. mvn r11,r11
  1609. add r3,sp,#$in1_x
  1610. and r11,r11,r12 @ in1infty & ~in2infty
  1611. mvn r12,r12 @ in2infty
  1612. ldr $r_ptr,[sp,#32*15]
  1613. ___
  1614. for($i=0;$i<64;$i+=8) { # conditional moves
  1615. $code.=<<___;
  1616. ldmia r1!,{r4-r5} @ res_x
  1617. ldmia r2!,{r6-r7} @ in2_x
  1618. ldmia r3!,{r8-r9} @ in1_x
  1619. and r4,r4,r10 @ ~in1infty & ~in2infty
  1620. and r5,r5,r10
  1621. and r6,r6,r11 @ in1infty & ~in2infty
  1622. and r7,r7,r11
  1623. and r8,r8,r12 @ in2infty
  1624. and r9,r9,r12
  1625. orr r4,r4,r6
  1626. orr r5,r5,r7
  1627. orr r4,r4,r8
  1628. orr r5,r5,r9
  1629. stmia $r_ptr!,{r4-r5}
  1630. ___
  1631. }
  1632. for(;$i<96;$i+=8) {
  1633. my $j=($i-64)/4;
  1634. $code.=<<___;
  1635. ldmia r1!,{r4-r5} @ res_z
  1636. ldmia r3!,{r8-r9} @ in1_z
  1637. and r4,r4,r10
  1638. and r5,r5,r10
  1639. and r6,r11,#@ONE_mont[$j]
  1640. and r7,r11,#@ONE_mont[$j+1]
  1641. and r8,r8,r12
  1642. and r9,r9,r12
  1643. orr r4,r4,r6
  1644. orr r5,r5,r7
  1645. orr r4,r4,r8
  1646. orr r5,r5,r9
  1647. stmia $r_ptr!,{r4-r5}
  1648. ___
  1649. }
  1650. $code.=<<___;
  1651. add sp,sp,#32*15+16 @ +16 means "skip even over saved r0-r3"
  1652. #if __ARM_ARCH__>=5 || !defined(__thumb__)
  1653. ldmia sp!,{r4-r12,pc}
  1654. #else
  1655. ldmia sp!,{r4-r12,lr}
  1656. bx lr @ interoperable with Thumb ISA:-)
  1657. #endif
  1658. .size ecp_nistz256_point_add_affine,.-ecp_nistz256_point_add_affine
  1659. ___
  1660. } }}}
  1661. foreach (split("\n",$code)) {
  1662. s/\`([^\`]*)\`/eval $1/geo;
  1663. s/\bq([0-9]+)#(lo|hi)/sprintf "d%d",2*$1+($2 eq "hi")/geo;
  1664. print $_,"\n";
  1665. }
  1666. close STDOUT or die "error closing STDOUT: $!"; # enforce flush