armv4-mont.pl 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766
  1. #! /usr/bin/env perl
  2. # Copyright 2007-2020 The OpenSSL Project Authors. All Rights Reserved.
  3. #
  4. # Licensed under the Apache License 2.0 (the "License"). You may not use
  5. # this file except in compliance with the License. You can obtain a copy
  6. # in the file LICENSE in the source distribution or at
  7. # https://www.openssl.org/source/license.html
  8. # ====================================================================
  9. # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
  10. # project. The module is, however, dual licensed under OpenSSL and
  11. # CRYPTOGAMS licenses depending on where you obtain it. For further
  12. # details see http://www.openssl.org/~appro/cryptogams/.
  13. # ====================================================================
  14. # January 2007.
  15. # Montgomery multiplication for ARMv4.
  16. #
  17. # Performance improvement naturally varies among CPU implementations
  18. # and compilers. The code was observed to provide +65-35% improvement
  19. # [depending on key length, less for longer keys] on ARM920T, and
  20. # +115-80% on Intel IXP425. This is compared to pre-bn_mul_mont code
  21. # base and compiler generated code with in-lined umull and even umlal
  22. # instructions. The latter means that this code didn't really have an
  23. # "advantage" of utilizing some "secret" instruction.
  24. #
  25. # The code is interoperable with Thumb ISA and is rather compact, less
  26. # than 1/2KB. Windows CE port would be trivial, as it's exclusively
  27. # about decorations, ABI and instruction syntax are identical.
  28. # November 2013
  29. #
  30. # Add NEON code path, which handles lengths divisible by 8. RSA/DSA
  31. # performance improvement on Cortex-A8 is ~45-100% depending on key
  32. # length, more for longer keys. On Cortex-A15 the span is ~10-105%.
  33. # On Snapdragon S4 improvement was measured to vary from ~70% to
  34. # incredible ~380%, yes, 4.8x faster, for RSA4096 sign. But this is
  35. # rather because original integer-only code seems to perform
  36. # suboptimally on S4. Situation on Cortex-A9 is unfortunately
  37. # different. It's being looked into, but the trouble is that
  38. # performance for vectors longer than 256 bits is actually couple
  39. # of percent worse than for integer-only code. The code is chosen
  40. # for execution on all NEON-capable processors, because gain on
  41. # others outweighs the marginal loss on Cortex-A9.
  42. # September 2015
  43. #
  44. # Align Cortex-A9 performance with November 2013 improvements, i.e.
  45. # NEON code is now ~20-105% faster than integer-only one on this
  46. # processor. But this optimization further improved performance even
  47. # on other processors: NEON code path is ~45-180% faster than original
  48. # integer-only on Cortex-A8, ~10-210% on Cortex-A15, ~70-450% on
  49. # Snapdragon S4.
  50. # $output is the last argument if it looks like a file (it has an extension)
  51. # $flavour is the first argument if it doesn't look like a file
  52. my $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
  53. my $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
  54. if ($flavour && $flavour ne "void") {
  55. $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
  56. ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
  57. ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
  58. die "can't locate arm-xlate.pl";
  59. open STDOUT,"| \"$^X\" $xlate $flavour \"$output\""
  60. or die "can't call $xlate: $1";
  61. } else {
  62. $output and open STDOUT,">$output";
  63. }
  64. $num="r0"; # starts as num argument, but holds &tp[num-1]
  65. $ap="r1";
  66. $bp="r2"; $bi="r2"; $rp="r2";
  67. $np="r3";
  68. $tp="r4";
  69. $aj="r5";
  70. $nj="r6";
  71. $tj="r7";
  72. $n0="r8";
  73. ########### # r9 is reserved by ELF as platform specific, e.g. TLS pointer
  74. $alo="r10"; # sl, gcc uses it to keep @GOT
  75. $ahi="r11"; # fp
  76. $nlo="r12"; # ip
  77. ########### # r13 is stack pointer
  78. $nhi="r14"; # lr
  79. ########### # r15 is program counter
  80. #### argument block layout relative to &tp[num-1], a.k.a. $num
  81. $_rp="$num,#12*4";
  82. # ap permanently resides in r1
  83. $_bp="$num,#13*4";
  84. # np permanently resides in r3
  85. $_n0="$num,#14*4";
  86. $_num="$num,#15*4"; $_bpend=$_num;
  87. $code=<<___;
  88. #include "arm_arch.h"
  89. #if defined(__thumb2__)
  90. .syntax unified
  91. .thumb
  92. #else
  93. .code 32
  94. #endif
  95. .text
  96. #if __ARM_MAX_ARCH__>=7
  97. .align 5
  98. .LOPENSSL_armcap:
  99. # ifdef _WIN32
  100. .word OPENSSL_armcap_P
  101. # else
  102. .word OPENSSL_armcap_P-.Lbn_mul_mont
  103. # endif
  104. #endif
  105. .global bn_mul_mont
  106. .type bn_mul_mont,%function
  107. .align 5
  108. bn_mul_mont:
  109. .Lbn_mul_mont:
  110. ldr ip,[sp,#4] @ load num
  111. stmdb sp!,{r0,r2} @ sp points at argument block
  112. #if __ARM_MAX_ARCH__>=7
  113. tst ip,#7
  114. bne .Lialu
  115. ldr r0,.LOPENSSL_armcap
  116. #if !defined(_WIN32)
  117. adr r2,.Lbn_mul_mont
  118. ldr r0,[r0,r2]
  119. # endif
  120. # if defined(__APPLE__) || defined(_WIN32)
  121. ldr r0,[r0]
  122. # endif
  123. tst r0,#ARMV7_NEON @ NEON available?
  124. ldmia sp, {r0,r2}
  125. beq .Lialu
  126. add sp,sp,#8
  127. b bn_mul8x_mont_neon
  128. .align 4
  129. .Lialu:
  130. #endif
  131. cmp ip,#2
  132. mov $num,ip @ load num
  133. #ifdef __thumb2__
  134. ittt lt
  135. #endif
  136. movlt r0,#0
  137. addlt sp,sp,#2*4
  138. blt .Labrt
  139. stmdb sp!,{r4-r12,lr} @ save 10 registers
  140. mov $num,$num,lsl#2 @ rescale $num for byte count
  141. sub sp,sp,$num @ alloca(4*num)
  142. sub sp,sp,#4 @ +extra dword
  143. sub $num,$num,#4 @ "num=num-1"
  144. add $tp,$bp,$num @ &bp[num-1]
  145. add $num,sp,$num @ $num to point at &tp[num-1]
  146. ldr $n0,[$_n0] @ &n0
  147. ldr $bi,[$bp] @ bp[0]
  148. ldr $aj,[$ap],#4 @ ap[0],ap++
  149. ldr $nj,[$np],#4 @ np[0],np++
  150. ldr $n0,[$n0] @ *n0
  151. str $tp,[$_bpend] @ save &bp[num]
  152. umull $alo,$ahi,$aj,$bi @ ap[0]*bp[0]
  153. str $n0,[$_n0] @ save n0 value
  154. mul $n0,$alo,$n0 @ "tp[0]"*n0
  155. mov $nlo,#0
  156. umlal $alo,$nlo,$nj,$n0 @ np[0]*n0+"t[0]"
  157. mov $tp,sp
  158. .L1st:
  159. ldr $aj,[$ap],#4 @ ap[j],ap++
  160. mov $alo,$ahi
  161. ldr $nj,[$np],#4 @ np[j],np++
  162. mov $ahi,#0
  163. umlal $alo,$ahi,$aj,$bi @ ap[j]*bp[0]
  164. mov $nhi,#0
  165. umlal $nlo,$nhi,$nj,$n0 @ np[j]*n0
  166. adds $nlo,$nlo,$alo
  167. str $nlo,[$tp],#4 @ tp[j-1]=,tp++
  168. adc $nlo,$nhi,#0
  169. cmp $tp,$num
  170. bne .L1st
  171. adds $nlo,$nlo,$ahi
  172. ldr $tp,[$_bp] @ restore bp
  173. mov $nhi,#0
  174. ldr $n0,[$_n0] @ restore n0
  175. adc $nhi,$nhi,#0
  176. str $nlo,[$num] @ tp[num-1]=
  177. mov $tj,sp
  178. str $nhi,[$num,#4] @ tp[num]=
  179. .Louter:
  180. sub $tj,$num,$tj @ "original" $num-1 value
  181. sub $ap,$ap,$tj @ "rewind" ap to &ap[1]
  182. ldr $bi,[$tp,#4]! @ *(++bp)
  183. sub $np,$np,$tj @ "rewind" np to &np[1]
  184. ldr $aj,[$ap,#-4] @ ap[0]
  185. ldr $alo,[sp] @ tp[0]
  186. ldr $nj,[$np,#-4] @ np[0]
  187. ldr $tj,[sp,#4] @ tp[1]
  188. mov $ahi,#0
  189. umlal $alo,$ahi,$aj,$bi @ ap[0]*bp[i]+tp[0]
  190. str $tp,[$_bp] @ save bp
  191. mul $n0,$alo,$n0
  192. mov $nlo,#0
  193. umlal $alo,$nlo,$nj,$n0 @ np[0]*n0+"tp[0]"
  194. mov $tp,sp
  195. .Linner:
  196. ldr $aj,[$ap],#4 @ ap[j],ap++
  197. adds $alo,$ahi,$tj @ +=tp[j]
  198. ldr $nj,[$np],#4 @ np[j],np++
  199. mov $ahi,#0
  200. umlal $alo,$ahi,$aj,$bi @ ap[j]*bp[i]
  201. mov $nhi,#0
  202. umlal $nlo,$nhi,$nj,$n0 @ np[j]*n0
  203. adc $ahi,$ahi,#0
  204. ldr $tj,[$tp,#8] @ tp[j+1]
  205. adds $nlo,$nlo,$alo
  206. str $nlo,[$tp],#4 @ tp[j-1]=,tp++
  207. adc $nlo,$nhi,#0
  208. cmp $tp,$num
  209. bne .Linner
  210. adds $nlo,$nlo,$ahi
  211. mov $nhi,#0
  212. ldr $tp,[$_bp] @ restore bp
  213. adc $nhi,$nhi,#0
  214. ldr $n0,[$_n0] @ restore n0
  215. adds $nlo,$nlo,$tj
  216. ldr $tj,[$_bpend] @ restore &bp[num]
  217. adc $nhi,$nhi,#0
  218. str $nlo,[$num] @ tp[num-1]=
  219. str $nhi,[$num,#4] @ tp[num]=
  220. cmp $tp,$tj
  221. #ifdef __thumb2__
  222. itt ne
  223. #endif
  224. movne $tj,sp
  225. bne .Louter
  226. ldr $rp,[$_rp] @ pull rp
  227. mov $aj,sp
  228. add $num,$num,#4 @ $num to point at &tp[num]
  229. sub $aj,$num,$aj @ "original" num value
  230. mov $tp,sp @ "rewind" $tp
  231. mov $ap,$tp @ "borrow" $ap
  232. sub $np,$np,$aj @ "rewind" $np to &np[0]
  233. subs $tj,$tj,$tj @ "clear" carry flag
  234. .Lsub: ldr $tj,[$tp],#4
  235. ldr $nj,[$np],#4
  236. sbcs $tj,$tj,$nj @ tp[j]-np[j]
  237. str $tj,[$rp],#4 @ rp[j]=
  238. teq $tp,$num @ preserve carry
  239. bne .Lsub
  240. sbcs $nhi,$nhi,#0 @ upmost carry
  241. mov $tp,sp @ "rewind" $tp
  242. sub $rp,$rp,$aj @ "rewind" $rp
  243. .Lcopy: ldr $tj,[$tp] @ conditional copy
  244. ldr $aj,[$rp]
  245. str sp,[$tp],#4 @ zap tp
  246. #ifdef __thumb2__
  247. it cc
  248. #endif
  249. movcc $aj,$tj
  250. str $aj,[$rp],#4
  251. teq $tp,$num @ preserve carry
  252. bne .Lcopy
  253. mov sp,$num
  254. add sp,sp,#4 @ skip over tp[num+1]
  255. ldmia sp!,{r4-r12,lr} @ restore registers
  256. add sp,sp,#2*4 @ skip over {r0,r2}
  257. mov r0,#1
  258. .Labrt:
  259. #if __ARM_ARCH__>=5
  260. ret @ bx lr
  261. #else
  262. tst lr,#1
  263. moveq pc,lr @ be binary compatible with V4, yet
  264. bx lr @ interoperable with Thumb ISA:-)
  265. #endif
  266. .size bn_mul_mont,.-bn_mul_mont
  267. ___
  268. {
  269. my ($A0,$A1,$A2,$A3)=map("d$_",(0..3));
  270. my ($N0,$N1,$N2,$N3)=map("d$_",(4..7));
  271. my ($Z,$Temp)=("q4","q5");
  272. my @ACC=map("q$_",(6..13));
  273. my ($Bi,$Ni,$M0)=map("d$_",(28..31));
  274. my $zero="$Z#lo";
  275. my $temp="$Temp#lo";
  276. my ($rptr,$aptr,$bptr,$nptr,$n0,$num)=map("r$_",(0..5));
  277. my ($tinptr,$toutptr,$inner,$outer,$bnptr)=map("r$_",(6..11));
  278. $code.=<<___;
  279. #if __ARM_MAX_ARCH__>=7
  280. .arch armv7-a
  281. .fpu neon
  282. .type bn_mul8x_mont_neon,%function
  283. .align 5
  284. bn_mul8x_mont_neon:
  285. mov ip,sp
  286. stmdb sp!,{r4-r11}
  287. vstmdb sp!,{d8-d15} @ ABI specification says so
  288. ldmia ip,{r4-r5} @ load rest of parameter block
  289. mov ip,sp
  290. cmp $num,#8
  291. bhi .LNEON_8n
  292. @ special case for $num==8, everything is in register bank...
  293. vld1.32 {${Bi}[0]}, [$bptr,:32]!
  294. veor $zero,$zero,$zero
  295. sub $toutptr,sp,$num,lsl#4
  296. vld1.32 {$A0-$A3}, [$aptr]! @ can't specify :32 :-(
  297. and $toutptr,$toutptr,#-64
  298. vld1.32 {${M0}[0]}, [$n0,:32]
  299. mov sp,$toutptr @ alloca
  300. vzip.16 $Bi,$zero
  301. vmull.u32 @ACC[0],$Bi,${A0}[0]
  302. vmull.u32 @ACC[1],$Bi,${A0}[1]
  303. vmull.u32 @ACC[2],$Bi,${A1}[0]
  304. vshl.i64 $Ni,@ACC[0]#hi,#16
  305. vmull.u32 @ACC[3],$Bi,${A1}[1]
  306. vadd.u64 $Ni,$Ni,@ACC[0]#lo
  307. veor $zero,$zero,$zero
  308. vmul.u32 $Ni,$Ni,$M0
  309. vmull.u32 @ACC[4],$Bi,${A2}[0]
  310. vld1.32 {$N0-$N3}, [$nptr]!
  311. vmull.u32 @ACC[5],$Bi,${A2}[1]
  312. vmull.u32 @ACC[6],$Bi,${A3}[0]
  313. vzip.16 $Ni,$zero
  314. vmull.u32 @ACC[7],$Bi,${A3}[1]
  315. vmlal.u32 @ACC[0],$Ni,${N0}[0]
  316. sub $outer,$num,#1
  317. vmlal.u32 @ACC[1],$Ni,${N0}[1]
  318. vmlal.u32 @ACC[2],$Ni,${N1}[0]
  319. vmlal.u32 @ACC[3],$Ni,${N1}[1]
  320. vmlal.u32 @ACC[4],$Ni,${N2}[0]
  321. vmov $Temp,@ACC[0]
  322. vmlal.u32 @ACC[5],$Ni,${N2}[1]
  323. vmov @ACC[0],@ACC[1]
  324. vmlal.u32 @ACC[6],$Ni,${N3}[0]
  325. vmov @ACC[1],@ACC[2]
  326. vmlal.u32 @ACC[7],$Ni,${N3}[1]
  327. vmov @ACC[2],@ACC[3]
  328. vmov @ACC[3],@ACC[4]
  329. vshr.u64 $temp,$temp,#16
  330. vmov @ACC[4],@ACC[5]
  331. vmov @ACC[5],@ACC[6]
  332. vadd.u64 $temp,$temp,$Temp#hi
  333. vmov @ACC[6],@ACC[7]
  334. veor @ACC[7],@ACC[7]
  335. vshr.u64 $temp,$temp,#16
  336. b .LNEON_outer8
  337. .align 4
  338. .LNEON_outer8:
  339. vld1.32 {${Bi}[0]}, [$bptr,:32]!
  340. veor $zero,$zero,$zero
  341. vzip.16 $Bi,$zero
  342. vadd.u64 @ACC[0]#lo,@ACC[0]#lo,$temp
  343. vmlal.u32 @ACC[0],$Bi,${A0}[0]
  344. vmlal.u32 @ACC[1],$Bi,${A0}[1]
  345. vmlal.u32 @ACC[2],$Bi,${A1}[0]
  346. vshl.i64 $Ni,@ACC[0]#hi,#16
  347. vmlal.u32 @ACC[3],$Bi,${A1}[1]
  348. vadd.u64 $Ni,$Ni,@ACC[0]#lo
  349. veor $zero,$zero,$zero
  350. subs $outer,$outer,#1
  351. vmul.u32 $Ni,$Ni,$M0
  352. vmlal.u32 @ACC[4],$Bi,${A2}[0]
  353. vmlal.u32 @ACC[5],$Bi,${A2}[1]
  354. vmlal.u32 @ACC[6],$Bi,${A3}[0]
  355. vzip.16 $Ni,$zero
  356. vmlal.u32 @ACC[7],$Bi,${A3}[1]
  357. vmlal.u32 @ACC[0],$Ni,${N0}[0]
  358. vmlal.u32 @ACC[1],$Ni,${N0}[1]
  359. vmlal.u32 @ACC[2],$Ni,${N1}[0]
  360. vmlal.u32 @ACC[3],$Ni,${N1}[1]
  361. vmlal.u32 @ACC[4],$Ni,${N2}[0]
  362. vmov $Temp,@ACC[0]
  363. vmlal.u32 @ACC[5],$Ni,${N2}[1]
  364. vmov @ACC[0],@ACC[1]
  365. vmlal.u32 @ACC[6],$Ni,${N3}[0]
  366. vmov @ACC[1],@ACC[2]
  367. vmlal.u32 @ACC[7],$Ni,${N3}[1]
  368. vmov @ACC[2],@ACC[3]
  369. vmov @ACC[3],@ACC[4]
  370. vshr.u64 $temp,$temp,#16
  371. vmov @ACC[4],@ACC[5]
  372. vmov @ACC[5],@ACC[6]
  373. vadd.u64 $temp,$temp,$Temp#hi
  374. vmov @ACC[6],@ACC[7]
  375. veor @ACC[7],@ACC[7]
  376. vshr.u64 $temp,$temp,#16
  377. bne .LNEON_outer8
  378. vadd.u64 @ACC[0]#lo,@ACC[0]#lo,$temp
  379. mov $toutptr,sp
  380. vshr.u64 $temp,@ACC[0]#lo,#16
  381. mov $inner,$num
  382. vadd.u64 @ACC[0]#hi,@ACC[0]#hi,$temp
  383. add $tinptr,sp,#96
  384. vshr.u64 $temp,@ACC[0]#hi,#16
  385. vzip.16 @ACC[0]#lo,@ACC[0]#hi
  386. b .LNEON_tail_entry
  387. .align 4
  388. .LNEON_8n:
  389. veor @ACC[0],@ACC[0],@ACC[0]
  390. sub $toutptr,sp,#128
  391. veor @ACC[1],@ACC[1],@ACC[1]
  392. sub $toutptr,$toutptr,$num,lsl#4
  393. veor @ACC[2],@ACC[2],@ACC[2]
  394. and $toutptr,$toutptr,#-64
  395. veor @ACC[3],@ACC[3],@ACC[3]
  396. mov sp,$toutptr @ alloca
  397. veor @ACC[4],@ACC[4],@ACC[4]
  398. add $toutptr,$toutptr,#256
  399. veor @ACC[5],@ACC[5],@ACC[5]
  400. sub $inner,$num,#8
  401. veor @ACC[6],@ACC[6],@ACC[6]
  402. veor @ACC[7],@ACC[7],@ACC[7]
  403. .LNEON_8n_init:
  404. vst1.64 {@ACC[0]-@ACC[1]},[$toutptr,:256]!
  405. subs $inner,$inner,#8
  406. vst1.64 {@ACC[2]-@ACC[3]},[$toutptr,:256]!
  407. vst1.64 {@ACC[4]-@ACC[5]},[$toutptr,:256]!
  408. vst1.64 {@ACC[6]-@ACC[7]},[$toutptr,:256]!
  409. bne .LNEON_8n_init
  410. add $tinptr,sp,#256
  411. vld1.32 {$A0-$A3},[$aptr]!
  412. add $bnptr,sp,#8
  413. vld1.32 {${M0}[0]},[$n0,:32]
  414. mov $outer,$num
  415. b .LNEON_8n_outer
  416. .align 4
  417. .LNEON_8n_outer:
  418. vld1.32 {${Bi}[0]},[$bptr,:32]! @ *b++
  419. veor $zero,$zero,$zero
  420. vzip.16 $Bi,$zero
  421. add $toutptr,sp,#128
  422. vld1.32 {$N0-$N3},[$nptr]!
  423. vmlal.u32 @ACC[0],$Bi,${A0}[0]
  424. vmlal.u32 @ACC[1],$Bi,${A0}[1]
  425. veor $zero,$zero,$zero
  426. vmlal.u32 @ACC[2],$Bi,${A1}[0]
  427. vshl.i64 $Ni,@ACC[0]#hi,#16
  428. vmlal.u32 @ACC[3],$Bi,${A1}[1]
  429. vadd.u64 $Ni,$Ni,@ACC[0]#lo
  430. vmlal.u32 @ACC[4],$Bi,${A2}[0]
  431. vmul.u32 $Ni,$Ni,$M0
  432. vmlal.u32 @ACC[5],$Bi,${A2}[1]
  433. vst1.32 {$Bi},[sp,:64] @ put aside smashed b[8*i+0]
  434. vmlal.u32 @ACC[6],$Bi,${A3}[0]
  435. vzip.16 $Ni,$zero
  436. vmlal.u32 @ACC[7],$Bi,${A3}[1]
  437. ___
  438. for ($i=0; $i<7;) {
  439. $code.=<<___;
  440. vld1.32 {${Bi}[0]},[$bptr,:32]! @ *b++
  441. vmlal.u32 @ACC[0],$Ni,${N0}[0]
  442. veor $temp,$temp,$temp
  443. vmlal.u32 @ACC[1],$Ni,${N0}[1]
  444. vzip.16 $Bi,$temp
  445. vmlal.u32 @ACC[2],$Ni,${N1}[0]
  446. vshr.u64 @ACC[0]#lo,@ACC[0]#lo,#16
  447. vmlal.u32 @ACC[3],$Ni,${N1}[1]
  448. vmlal.u32 @ACC[4],$Ni,${N2}[0]
  449. vadd.u64 @ACC[0]#lo,@ACC[0]#lo,@ACC[0]#hi
  450. vmlal.u32 @ACC[5],$Ni,${N2}[1]
  451. vshr.u64 @ACC[0]#lo,@ACC[0]#lo,#16
  452. vmlal.u32 @ACC[6],$Ni,${N3}[0]
  453. vmlal.u32 @ACC[7],$Ni,${N3}[1]
  454. vadd.u64 @ACC[1]#lo,@ACC[1]#lo,@ACC[0]#lo
  455. vst1.32 {$Ni},[$bnptr,:64]! @ put aside smashed m[8*i+$i]
  456. ___
  457. push(@ACC,shift(@ACC)); $i++;
  458. $code.=<<___;
  459. vmlal.u32 @ACC[0],$Bi,${A0}[0]
  460. vld1.64 {@ACC[7]},[$tinptr,:128]!
  461. vmlal.u32 @ACC[1],$Bi,${A0}[1]
  462. veor $zero,$zero,$zero
  463. vmlal.u32 @ACC[2],$Bi,${A1}[0]
  464. vshl.i64 $Ni,@ACC[0]#hi,#16
  465. vmlal.u32 @ACC[3],$Bi,${A1}[1]
  466. vadd.u64 $Ni,$Ni,@ACC[0]#lo
  467. vmlal.u32 @ACC[4],$Bi,${A2}[0]
  468. vmul.u32 $Ni,$Ni,$M0
  469. vmlal.u32 @ACC[5],$Bi,${A2}[1]
  470. vst1.32 {$Bi},[$bnptr,:64]! @ put aside smashed b[8*i+$i]
  471. vmlal.u32 @ACC[6],$Bi,${A3}[0]
  472. vzip.16 $Ni,$zero
  473. vmlal.u32 @ACC[7],$Bi,${A3}[1]
  474. ___
  475. }
  476. $code.=<<___;
  477. vld1.32 {$Bi},[sp,:64] @ pull smashed b[8*i+0]
  478. vmlal.u32 @ACC[0],$Ni,${N0}[0]
  479. vld1.32 {$A0-$A3},[$aptr]!
  480. vmlal.u32 @ACC[1],$Ni,${N0}[1]
  481. vmlal.u32 @ACC[2],$Ni,${N1}[0]
  482. vshr.u64 @ACC[0]#lo,@ACC[0]#lo,#16
  483. vmlal.u32 @ACC[3],$Ni,${N1}[1]
  484. vmlal.u32 @ACC[4],$Ni,${N2}[0]
  485. vadd.u64 @ACC[0]#lo,@ACC[0]#lo,@ACC[0]#hi
  486. vmlal.u32 @ACC[5],$Ni,${N2}[1]
  487. vshr.u64 @ACC[0]#lo,@ACC[0]#lo,#16
  488. vmlal.u32 @ACC[6],$Ni,${N3}[0]
  489. vmlal.u32 @ACC[7],$Ni,${N3}[1]
  490. vadd.u64 @ACC[1]#lo,@ACC[1]#lo,@ACC[0]#lo
  491. vst1.32 {$Ni},[$bnptr,:64] @ put aside smashed m[8*i+$i]
  492. add $bnptr,sp,#8 @ rewind
  493. ___
  494. push(@ACC,shift(@ACC));
  495. $code.=<<___;
  496. sub $inner,$num,#8
  497. b .LNEON_8n_inner
  498. .align 4
  499. .LNEON_8n_inner:
  500. subs $inner,$inner,#8
  501. vmlal.u32 @ACC[0],$Bi,${A0}[0]
  502. vld1.64 {@ACC[7]},[$tinptr,:128]
  503. vmlal.u32 @ACC[1],$Bi,${A0}[1]
  504. vld1.32 {$Ni},[$bnptr,:64]! @ pull smashed m[8*i+0]
  505. vmlal.u32 @ACC[2],$Bi,${A1}[0]
  506. vld1.32 {$N0-$N3},[$nptr]!
  507. vmlal.u32 @ACC[3],$Bi,${A1}[1]
  508. it ne
  509. addne $tinptr,$tinptr,#16 @ don't advance in last iteration
  510. vmlal.u32 @ACC[4],$Bi,${A2}[0]
  511. vmlal.u32 @ACC[5],$Bi,${A2}[1]
  512. vmlal.u32 @ACC[6],$Bi,${A3}[0]
  513. vmlal.u32 @ACC[7],$Bi,${A3}[1]
  514. ___
  515. for ($i=1; $i<8; $i++) {
  516. $code.=<<___;
  517. vld1.32 {$Bi},[$bnptr,:64]! @ pull smashed b[8*i+$i]
  518. vmlal.u32 @ACC[0],$Ni,${N0}[0]
  519. vmlal.u32 @ACC[1],$Ni,${N0}[1]
  520. vmlal.u32 @ACC[2],$Ni,${N1}[0]
  521. vmlal.u32 @ACC[3],$Ni,${N1}[1]
  522. vmlal.u32 @ACC[4],$Ni,${N2}[0]
  523. vmlal.u32 @ACC[5],$Ni,${N2}[1]
  524. vmlal.u32 @ACC[6],$Ni,${N3}[0]
  525. vmlal.u32 @ACC[7],$Ni,${N3}[1]
  526. vst1.64 {@ACC[0]},[$toutptr,:128]!
  527. ___
  528. push(@ACC,shift(@ACC));
  529. $code.=<<___;
  530. vmlal.u32 @ACC[0],$Bi,${A0}[0]
  531. vld1.64 {@ACC[7]},[$tinptr,:128]
  532. vmlal.u32 @ACC[1],$Bi,${A0}[1]
  533. vld1.32 {$Ni},[$bnptr,:64]! @ pull smashed m[8*i+$i]
  534. vmlal.u32 @ACC[2],$Bi,${A1}[0]
  535. it ne
  536. addne $tinptr,$tinptr,#16 @ don't advance in last iteration
  537. vmlal.u32 @ACC[3],$Bi,${A1}[1]
  538. vmlal.u32 @ACC[4],$Bi,${A2}[0]
  539. vmlal.u32 @ACC[5],$Bi,${A2}[1]
  540. vmlal.u32 @ACC[6],$Bi,${A3}[0]
  541. vmlal.u32 @ACC[7],$Bi,${A3}[1]
  542. ___
  543. }
  544. $code.=<<___;
  545. it eq
  546. subeq $aptr,$aptr,$num,lsl#2 @ rewind
  547. vmlal.u32 @ACC[0],$Ni,${N0}[0]
  548. vld1.32 {$Bi},[sp,:64] @ pull smashed b[8*i+0]
  549. vmlal.u32 @ACC[1],$Ni,${N0}[1]
  550. vld1.32 {$A0-$A3},[$aptr]!
  551. vmlal.u32 @ACC[2],$Ni,${N1}[0]
  552. add $bnptr,sp,#8 @ rewind
  553. vmlal.u32 @ACC[3],$Ni,${N1}[1]
  554. vmlal.u32 @ACC[4],$Ni,${N2}[0]
  555. vmlal.u32 @ACC[5],$Ni,${N2}[1]
  556. vmlal.u32 @ACC[6],$Ni,${N3}[0]
  557. vst1.64 {@ACC[0]},[$toutptr,:128]!
  558. vmlal.u32 @ACC[7],$Ni,${N3}[1]
  559. bne .LNEON_8n_inner
  560. ___
  561. push(@ACC,shift(@ACC));
  562. $code.=<<___;
  563. add $tinptr,sp,#128
  564. vst1.64 {@ACC[0]-@ACC[1]},[$toutptr,:256]!
  565. veor q2,q2,q2 @ $N0-$N1
  566. vst1.64 {@ACC[2]-@ACC[3]},[$toutptr,:256]!
  567. veor q3,q3,q3 @ $N2-$N3
  568. vst1.64 {@ACC[4]-@ACC[5]},[$toutptr,:256]!
  569. vst1.64 {@ACC[6]},[$toutptr,:128]
  570. subs $outer,$outer,#8
  571. vld1.64 {@ACC[0]-@ACC[1]},[$tinptr,:256]!
  572. vld1.64 {@ACC[2]-@ACC[3]},[$tinptr,:256]!
  573. vld1.64 {@ACC[4]-@ACC[5]},[$tinptr,:256]!
  574. vld1.64 {@ACC[6]-@ACC[7]},[$tinptr,:256]!
  575. itt ne
  576. subne $nptr,$nptr,$num,lsl#2 @ rewind
  577. bne .LNEON_8n_outer
  578. add $toutptr,sp,#128
  579. vst1.64 {q2-q3}, [sp,:256]! @ start wiping stack frame
  580. vshr.u64 $temp,@ACC[0]#lo,#16
  581. vst1.64 {q2-q3},[sp,:256]!
  582. vadd.u64 @ACC[0]#hi,@ACC[0]#hi,$temp
  583. vst1.64 {q2-q3}, [sp,:256]!
  584. vshr.u64 $temp,@ACC[0]#hi,#16
  585. vst1.64 {q2-q3}, [sp,:256]!
  586. vzip.16 @ACC[0]#lo,@ACC[0]#hi
  587. mov $inner,$num
  588. b .LNEON_tail_entry
  589. .align 4
  590. .LNEON_tail:
  591. vadd.u64 @ACC[0]#lo,@ACC[0]#lo,$temp
  592. vshr.u64 $temp,@ACC[0]#lo,#16
  593. vld1.64 {@ACC[2]-@ACC[3]}, [$tinptr, :256]!
  594. vadd.u64 @ACC[0]#hi,@ACC[0]#hi,$temp
  595. vld1.64 {@ACC[4]-@ACC[5]}, [$tinptr, :256]!
  596. vshr.u64 $temp,@ACC[0]#hi,#16
  597. vld1.64 {@ACC[6]-@ACC[7]}, [$tinptr, :256]!
  598. vzip.16 @ACC[0]#lo,@ACC[0]#hi
  599. .LNEON_tail_entry:
  600. ___
  601. for ($i=1; $i<8; $i++) {
  602. $code.=<<___;
  603. vadd.u64 @ACC[1]#lo,@ACC[1]#lo,$temp
  604. vst1.32 {@ACC[0]#lo[0]}, [$toutptr, :32]!
  605. vshr.u64 $temp,@ACC[1]#lo,#16
  606. vadd.u64 @ACC[1]#hi,@ACC[1]#hi,$temp
  607. vshr.u64 $temp,@ACC[1]#hi,#16
  608. vzip.16 @ACC[1]#lo,@ACC[1]#hi
  609. ___
  610. push(@ACC,shift(@ACC));
  611. }
  612. push(@ACC,shift(@ACC));
  613. $code.=<<___;
  614. vld1.64 {@ACC[0]-@ACC[1]}, [$tinptr, :256]!
  615. subs $inner,$inner,#8
  616. vst1.32 {@ACC[7]#lo[0]}, [$toutptr, :32]!
  617. bne .LNEON_tail
  618. vst1.32 {${temp}[0]}, [$toutptr, :32] @ top-most bit
  619. sub $nptr,$nptr,$num,lsl#2 @ rewind $nptr
  620. subs $aptr,sp,#0 @ clear carry flag
  621. add $bptr,sp,$num,lsl#2
  622. .LNEON_sub:
  623. ldmia $aptr!, {r4-r7}
  624. ldmia $nptr!, {r8-r11}
  625. sbcs r8, r4,r8
  626. sbcs r9, r5,r9
  627. sbcs r10,r6,r10
  628. sbcs r11,r7,r11
  629. teq $aptr,$bptr @ preserves carry
  630. stmia $rptr!, {r8-r11}
  631. bne .LNEON_sub
  632. ldr r10, [$aptr] @ load top-most bit
  633. mov r11,sp
  634. veor q0,q0,q0
  635. sub r11,$bptr,r11 @ this is num*4
  636. veor q1,q1,q1
  637. mov $aptr,sp
  638. sub $rptr,$rptr,r11 @ rewind $rptr
  639. mov $nptr,$bptr @ second 3/4th of frame
  640. sbcs r10,r10,#0 @ result is carry flag
  641. .LNEON_copy_n_zap:
  642. ldmia $aptr!, {r4-r7}
  643. ldmia $rptr, {r8-r11}
  644. it cc
  645. movcc r8, r4
  646. vst1.64 {q0-q1}, [$nptr,:256]! @ wipe
  647. itt cc
  648. movcc r9, r5
  649. movcc r10,r6
  650. vst1.64 {q0-q1}, [$nptr,:256]! @ wipe
  651. it cc
  652. movcc r11,r7
  653. ldmia $aptr, {r4-r7}
  654. stmia $rptr!, {r8-r11}
  655. sub $aptr,$aptr,#16
  656. ldmia $rptr, {r8-r11}
  657. it cc
  658. movcc r8, r4
  659. vst1.64 {q0-q1}, [$aptr,:256]! @ wipe
  660. itt cc
  661. movcc r9, r5
  662. movcc r10,r6
  663. vst1.64 {q0-q1}, [$nptr,:256]! @ wipe
  664. it cc
  665. movcc r11,r7
  666. teq $aptr,$bptr @ preserves carry
  667. stmia $rptr!, {r8-r11}
  668. bne .LNEON_copy_n_zap
  669. mov sp,ip
  670. vldmia sp!,{d8-d15}
  671. ldmia sp!,{r4-r11}
  672. ret @ bx lr
  673. .size bn_mul8x_mont_neon,.-bn_mul8x_mont_neon
  674. #endif
  675. ___
  676. }
  677. $code.=<<___;
  678. .asciz "Montgomery multiplication for ARMv4/NEON, CRYPTOGAMS by <appro\@openssl.org>"
  679. .align 2
  680. #if __ARM_MAX_ARCH__>=7
  681. .comm OPENSSL_armcap_P,4,4
  682. #endif
  683. ___
  684. foreach (split("\n",$code)) {
  685. s/\`([^\`]*)\`/eval $1/ge;
  686. s/\bq([0-9]+)#(lo|hi)/sprintf "d%d",2*$1+($2 eq "hi")/ge or
  687. s/\bret\b/bx lr/g or
  688. s/\bbx\s+lr\b/.word\t0xe12fff1e/g; # make it possible to compile with -march=armv4
  689. print $_,"\n";
  690. }
  691. close STDOUT or die "error closing STDOUT: $!";