ghash-armv4.pl 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554
  1. #! /usr/bin/env perl
  2. # Copyright 2010-2020 The OpenSSL Project Authors. All Rights Reserved.
  3. #
  4. # Licensed under the Apache License 2.0 (the "License"). You may not use
  5. # this file except in compliance with the License. You can obtain a copy
  6. # in the file LICENSE in the source distribution or at
  7. # https://www.openssl.org/source/license.html
  8. #
  9. # ====================================================================
  10. # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
  11. # project. The module is, however, dual licensed under OpenSSL and
  12. # CRYPTOGAMS licenses depending on where you obtain it. For further
  13. # details see http://www.openssl.org/~appro/cryptogams/.
  14. # ====================================================================
  15. #
  16. # April 2010
  17. #
  18. # The module implements "4-bit" GCM GHASH function and underlying
  19. # single multiplication operation in GF(2^128). "4-bit" means that it
  20. # uses 256 bytes per-key table [+32 bytes shared table]. There is no
  21. # experimental performance data available yet. The only approximation
  22. # that can be made at this point is based on code size. Inner loop is
  23. # 32 instructions long and on single-issue core should execute in <40
  24. # cycles. Having verified that gcc 3.4 didn't unroll corresponding
  25. # loop, this assembler loop body was found to be ~3x smaller than
  26. # compiler-generated one...
  27. #
  28. # July 2010
  29. #
  30. # Rescheduling for dual-issue pipeline resulted in 8.5% improvement on
  31. # Cortex A8 core and ~25 cycles per processed byte (which was observed
  32. # to be ~3 times faster than gcc-generated code:-)
  33. #
  34. # February 2011
  35. #
  36. # Profiler-assisted and platform-specific optimization resulted in 7%
  37. # improvement on Cortex A8 core and ~23.5 cycles per byte.
  38. #
  39. # March 2011
  40. #
  41. # Add NEON implementation featuring polynomial multiplication, i.e. no
  42. # lookup tables involved. On Cortex A8 it was measured to process one
  43. # byte in 15 cycles or 55% faster than integer-only code.
  44. #
  45. # April 2014
  46. #
  47. # Switch to multiplication algorithm suggested in paper referred
  48. # below and combine it with reduction algorithm from x86 module.
  49. # Performance improvement over previous version varies from 65% on
  50. # Snapdragon S4 to 110% on Cortex A9. In absolute terms Cortex A8
  51. # processes one byte in 8.45 cycles, A9 - in 10.2, A15 - in 7.63,
  52. # Snapdragon S4 - in 9.33.
  53. #
  54. # Câmara, D.; Gouvêa, C. P. L.; López, J. & Dahab, R.: Fast Software
  55. # Polynomial Multiplication on ARM Processors using the NEON Engine.
  56. #
  57. # http://conradoplg.cryptoland.net/files/2010/12/mocrysen13.pdf
  58. # ====================================================================
  59. # Note about "528B" variant. In ARM case it makes lesser sense to
  60. # implement it for following reasons:
  61. #
  62. # - performance improvement won't be anywhere near 50%, because 128-
  63. # bit shift operation is neatly fused with 128-bit xor here, and
  64. # "538B" variant would eliminate only 4-5 instructions out of 32
  65. # in the inner loop (meaning that estimated improvement is ~15%);
  66. # - ARM-based systems are often embedded ones and extra memory
  67. # consumption might be unappreciated (for so little improvement);
  68. #
  69. # Byte order [in]dependence. =========================================
  70. #
  71. # Caller is expected to maintain specific *dword* order in Htable,
  72. # namely with *least* significant dword of 128-bit value at *lower*
  73. # address. This differs completely from C code and has everything to
  74. # do with ldm instruction and order in which dwords are "consumed" by
  75. # algorithm. *Byte* order within these dwords in turn is whatever
  76. # *native* byte order on current platform. See gcm128.c for working
  77. # example...
  78. # $output is the last argument if it looks like a file (it has an extension)
  79. # $flavour is the first argument if it doesn't look like a file
  80. $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
  81. $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
  82. if ($flavour && $flavour ne "void") {
  83. $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
  84. ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
  85. ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
  86. die "can't locate arm-xlate.pl";
  87. open STDOUT,"| \"$^X\" $xlate $flavour \"$output\""
  88. or die "can't call $xlate: $!";
  89. } else {
  90. $output and open STDOUT,">$output";
  91. }
  92. $Xi="r0"; # argument block
  93. $Htbl="r1";
  94. $inp="r2";
  95. $len="r3";
  96. $Zll="r4"; # variables
  97. $Zlh="r5";
  98. $Zhl="r6";
  99. $Zhh="r7";
  100. $Tll="r8";
  101. $Tlh="r9";
  102. $Thl="r10";
  103. $Thh="r11";
  104. $nlo="r12";
  105. ################# r13 is stack pointer
  106. $nhi="r14";
  107. ################# r15 is program counter
  108. $rem_4bit=$inp; # used in gcm_gmult_4bit
  109. $cnt=$len;
  110. sub Zsmash() {
  111. my $i=12;
  112. my @args=@_;
  113. for ($Zll,$Zlh,$Zhl,$Zhh) {
  114. $code.=<<___;
  115. #if __ARM_ARCH__>=7 && defined(__ARMEL__)
  116. rev $_,$_
  117. str $_,[$Xi,#$i]
  118. #elif defined(__ARMEB__)
  119. str $_,[$Xi,#$i]
  120. #else
  121. mov $Tlh,$_,lsr#8
  122. strb $_,[$Xi,#$i+3]
  123. mov $Thl,$_,lsr#16
  124. strb $Tlh,[$Xi,#$i+2]
  125. mov $Thh,$_,lsr#24
  126. strb $Thl,[$Xi,#$i+1]
  127. strb $Thh,[$Xi,#$i]
  128. #endif
  129. ___
  130. $code.="\t".shift(@args)."\n";
  131. $i-=4;
  132. }
  133. }
  134. $code=<<___;
  135. #include "arm_arch.h"
  136. #if defined(__thumb2__) || defined(__clang__)
  137. .syntax unified
  138. #define ldrplb ldrbpl
  139. #define ldrneb ldrbne
  140. #endif
  141. #if defined(__thumb2__)
  142. .thumb
  143. #else
  144. .code 32
  145. #endif
  146. .text
  147. .type rem_4bit,%object
  148. .align 5
  149. rem_4bit:
  150. .short 0x0000,0x1C20,0x3840,0x2460
  151. .short 0x7080,0x6CA0,0x48C0,0x54E0
  152. .short 0xE100,0xFD20,0xD940,0xC560
  153. .short 0x9180,0x8DA0,0xA9C0,0xB5E0
  154. .size rem_4bit,.-rem_4bit
  155. .type rem_4bit_get,%function
  156. rem_4bit_get:
  157. #if defined(__thumb2__)
  158. adr $rem_4bit,rem_4bit
  159. #else
  160. sub $rem_4bit,pc,#8+32 @ &rem_4bit
  161. #endif
  162. b .Lrem_4bit_got
  163. nop
  164. nop
  165. .size rem_4bit_get,.-rem_4bit_get
  166. .global gcm_ghash_4bit
  167. .type gcm_ghash_4bit,%function
  168. .align 4
  169. gcm_ghash_4bit:
  170. #if defined(__thumb2__)
  171. adr r12,rem_4bit
  172. #else
  173. sub r12,pc,#8+48 @ &rem_4bit
  174. #endif
  175. add $len,$inp,$len @ $len to point at the end
  176. stmdb sp!,{r3-r11,lr} @ save $len/end too
  177. ldmia r12,{r4-r11} @ copy rem_4bit ...
  178. stmdb sp!,{r4-r11} @ ... to stack
  179. ldrb $nlo,[$inp,#15]
  180. ldrb $nhi,[$Xi,#15]
  181. .Louter:
  182. eor $nlo,$nlo,$nhi
  183. and $nhi,$nlo,#0xf0
  184. and $nlo,$nlo,#0x0f
  185. mov $cnt,#14
  186. add $Zhh,$Htbl,$nlo,lsl#4
  187. ldmia $Zhh,{$Zll-$Zhh} @ load Htbl[nlo]
  188. add $Thh,$Htbl,$nhi
  189. ldrb $nlo,[$inp,#14]
  190. and $nhi,$Zll,#0xf @ rem
  191. ldmia $Thh,{$Tll-$Thh} @ load Htbl[nhi]
  192. add $nhi,$nhi,$nhi
  193. eor $Zll,$Tll,$Zll,lsr#4
  194. ldrh $Tll,[sp,$nhi] @ rem_4bit[rem]
  195. eor $Zll,$Zll,$Zlh,lsl#28
  196. ldrb $nhi,[$Xi,#14]
  197. eor $Zlh,$Tlh,$Zlh,lsr#4
  198. eor $Zlh,$Zlh,$Zhl,lsl#28
  199. eor $Zhl,$Thl,$Zhl,lsr#4
  200. eor $Zhl,$Zhl,$Zhh,lsl#28
  201. eor $Zhh,$Thh,$Zhh,lsr#4
  202. eor $nlo,$nlo,$nhi
  203. and $nhi,$nlo,#0xf0
  204. and $nlo,$nlo,#0x0f
  205. eor $Zhh,$Zhh,$Tll,lsl#16
  206. .Linner:
  207. add $Thh,$Htbl,$nlo,lsl#4
  208. and $nlo,$Zll,#0xf @ rem
  209. subs $cnt,$cnt,#1
  210. add $nlo,$nlo,$nlo
  211. ldmia $Thh,{$Tll-$Thh} @ load Htbl[nlo]
  212. eor $Zll,$Tll,$Zll,lsr#4
  213. eor $Zll,$Zll,$Zlh,lsl#28
  214. eor $Zlh,$Tlh,$Zlh,lsr#4
  215. eor $Zlh,$Zlh,$Zhl,lsl#28
  216. ldrh $Tll,[sp,$nlo] @ rem_4bit[rem]
  217. eor $Zhl,$Thl,$Zhl,lsr#4
  218. #ifdef __thumb2__
  219. it pl
  220. #endif
  221. ldrplb $nlo,[$inp,$cnt]
  222. eor $Zhl,$Zhl,$Zhh,lsl#28
  223. eor $Zhh,$Thh,$Zhh,lsr#4
  224. add $Thh,$Htbl,$nhi
  225. and $nhi,$Zll,#0xf @ rem
  226. eor $Zhh,$Zhh,$Tll,lsl#16 @ ^= rem_4bit[rem]
  227. add $nhi,$nhi,$nhi
  228. ldmia $Thh,{$Tll-$Thh} @ load Htbl[nhi]
  229. eor $Zll,$Tll,$Zll,lsr#4
  230. #ifdef __thumb2__
  231. it pl
  232. #endif
  233. ldrplb $Tll,[$Xi,$cnt]
  234. eor $Zll,$Zll,$Zlh,lsl#28
  235. eor $Zlh,$Tlh,$Zlh,lsr#4
  236. ldrh $Tlh,[sp,$nhi]
  237. eor $Zlh,$Zlh,$Zhl,lsl#28
  238. eor $Zhl,$Thl,$Zhl,lsr#4
  239. eor $Zhl,$Zhl,$Zhh,lsl#28
  240. #ifdef __thumb2__
  241. it pl
  242. #endif
  243. eorpl $nlo,$nlo,$Tll
  244. eor $Zhh,$Thh,$Zhh,lsr#4
  245. #ifdef __thumb2__
  246. itt pl
  247. #endif
  248. andpl $nhi,$nlo,#0xf0
  249. andpl $nlo,$nlo,#0x0f
  250. eor $Zhh,$Zhh,$Tlh,lsl#16 @ ^= rem_4bit[rem]
  251. bpl .Linner
  252. ldr $len,[sp,#32] @ re-load $len/end
  253. add $inp,$inp,#16
  254. mov $nhi,$Zll
  255. ___
  256. &Zsmash("cmp\t$inp,$len","\n".
  257. "#ifdef __thumb2__\n".
  258. " it ne\n".
  259. "#endif\n".
  260. " ldrneb $nlo,[$inp,#15]");
  261. $code.=<<___;
  262. bne .Louter
  263. add sp,sp,#36
  264. #if __ARM_ARCH__>=5
  265. ldmia sp!,{r4-r11,pc}
  266. #else
  267. ldmia sp!,{r4-r11,lr}
  268. tst lr,#1
  269. moveq pc,lr @ be binary compatible with V4, yet
  270. bx lr @ interoperable with Thumb ISA:-)
  271. #endif
  272. .size gcm_ghash_4bit,.-gcm_ghash_4bit
  273. .global gcm_gmult_4bit
  274. .type gcm_gmult_4bit,%function
  275. gcm_gmult_4bit:
  276. stmdb sp!,{r4-r11,lr}
  277. ldrb $nlo,[$Xi,#15]
  278. b rem_4bit_get
  279. .Lrem_4bit_got:
  280. and $nhi,$nlo,#0xf0
  281. and $nlo,$nlo,#0x0f
  282. mov $cnt,#14
  283. add $Zhh,$Htbl,$nlo,lsl#4
  284. ldmia $Zhh,{$Zll-$Zhh} @ load Htbl[nlo]
  285. ldrb $nlo,[$Xi,#14]
  286. add $Thh,$Htbl,$nhi
  287. and $nhi,$Zll,#0xf @ rem
  288. ldmia $Thh,{$Tll-$Thh} @ load Htbl[nhi]
  289. add $nhi,$nhi,$nhi
  290. eor $Zll,$Tll,$Zll,lsr#4
  291. ldrh $Tll,[$rem_4bit,$nhi] @ rem_4bit[rem]
  292. eor $Zll,$Zll,$Zlh,lsl#28
  293. eor $Zlh,$Tlh,$Zlh,lsr#4
  294. eor $Zlh,$Zlh,$Zhl,lsl#28
  295. eor $Zhl,$Thl,$Zhl,lsr#4
  296. eor $Zhl,$Zhl,$Zhh,lsl#28
  297. eor $Zhh,$Thh,$Zhh,lsr#4
  298. and $nhi,$nlo,#0xf0
  299. eor $Zhh,$Zhh,$Tll,lsl#16
  300. and $nlo,$nlo,#0x0f
  301. .Loop:
  302. add $Thh,$Htbl,$nlo,lsl#4
  303. and $nlo,$Zll,#0xf @ rem
  304. subs $cnt,$cnt,#1
  305. add $nlo,$nlo,$nlo
  306. ldmia $Thh,{$Tll-$Thh} @ load Htbl[nlo]
  307. eor $Zll,$Tll,$Zll,lsr#4
  308. eor $Zll,$Zll,$Zlh,lsl#28
  309. eor $Zlh,$Tlh,$Zlh,lsr#4
  310. eor $Zlh,$Zlh,$Zhl,lsl#28
  311. ldrh $Tll,[$rem_4bit,$nlo] @ rem_4bit[rem]
  312. eor $Zhl,$Thl,$Zhl,lsr#4
  313. #ifdef __thumb2__
  314. it pl
  315. #endif
  316. ldrplb $nlo,[$Xi,$cnt]
  317. eor $Zhl,$Zhl,$Zhh,lsl#28
  318. eor $Zhh,$Thh,$Zhh,lsr#4
  319. add $Thh,$Htbl,$nhi
  320. and $nhi,$Zll,#0xf @ rem
  321. eor $Zhh,$Zhh,$Tll,lsl#16 @ ^= rem_4bit[rem]
  322. add $nhi,$nhi,$nhi
  323. ldmia $Thh,{$Tll-$Thh} @ load Htbl[nhi]
  324. eor $Zll,$Tll,$Zll,lsr#4
  325. eor $Zll,$Zll,$Zlh,lsl#28
  326. eor $Zlh,$Tlh,$Zlh,lsr#4
  327. ldrh $Tll,[$rem_4bit,$nhi] @ rem_4bit[rem]
  328. eor $Zlh,$Zlh,$Zhl,lsl#28
  329. eor $Zhl,$Thl,$Zhl,lsr#4
  330. eor $Zhl,$Zhl,$Zhh,lsl#28
  331. eor $Zhh,$Thh,$Zhh,lsr#4
  332. #ifdef __thumb2__
  333. itt pl
  334. #endif
  335. andpl $nhi,$nlo,#0xf0
  336. andpl $nlo,$nlo,#0x0f
  337. eor $Zhh,$Zhh,$Tll,lsl#16 @ ^= rem_4bit[rem]
  338. bpl .Loop
  339. ___
  340. &Zsmash();
  341. $code.=<<___;
  342. #if __ARM_ARCH__>=5
  343. ldmia sp!,{r4-r11,pc}
  344. #else
  345. ldmia sp!,{r4-r11,lr}
  346. tst lr,#1
  347. moveq pc,lr @ be binary compatible with V4, yet
  348. bx lr @ interoperable with Thumb ISA:-)
  349. #endif
  350. .size gcm_gmult_4bit,.-gcm_gmult_4bit
  351. ___
  352. {
  353. my ($Xl,$Xm,$Xh,$IN)=map("q$_",(0..3));
  354. my ($t0,$t1,$t2,$t3)=map("q$_",(8..12));
  355. my ($Hlo,$Hhi,$Hhl,$k48,$k32,$k16)=map("d$_",(26..31));
  356. sub clmul64x64 {
  357. my ($r,$a,$b)=@_;
  358. $code.=<<___;
  359. vext.8 $t0#lo, $a, $a, #1 @ A1
  360. vmull.p8 $t0, $t0#lo, $b @ F = A1*B
  361. vext.8 $r#lo, $b, $b, #1 @ B1
  362. vmull.p8 $r, $a, $r#lo @ E = A*B1
  363. vext.8 $t1#lo, $a, $a, #2 @ A2
  364. vmull.p8 $t1, $t1#lo, $b @ H = A2*B
  365. vext.8 $t3#lo, $b, $b, #2 @ B2
  366. vmull.p8 $t3, $a, $t3#lo @ G = A*B2
  367. vext.8 $t2#lo, $a, $a, #3 @ A3
  368. veor $t0, $t0, $r @ L = E + F
  369. vmull.p8 $t2, $t2#lo, $b @ J = A3*B
  370. vext.8 $r#lo, $b, $b, #3 @ B3
  371. veor $t1, $t1, $t3 @ M = G + H
  372. vmull.p8 $r, $a, $r#lo @ I = A*B3
  373. veor $t0#lo, $t0#lo, $t0#hi @ t0 = (L) (P0 + P1) << 8
  374. vand $t0#hi, $t0#hi, $k48
  375. vext.8 $t3#lo, $b, $b, #4 @ B4
  376. veor $t1#lo, $t1#lo, $t1#hi @ t1 = (M) (P2 + P3) << 16
  377. vand $t1#hi, $t1#hi, $k32
  378. vmull.p8 $t3, $a, $t3#lo @ K = A*B4
  379. veor $t2, $t2, $r @ N = I + J
  380. veor $t0#lo, $t0#lo, $t0#hi
  381. veor $t1#lo, $t1#lo, $t1#hi
  382. veor $t2#lo, $t2#lo, $t2#hi @ t2 = (N) (P4 + P5) << 24
  383. vand $t2#hi, $t2#hi, $k16
  384. vext.8 $t0, $t0, $t0, #15
  385. veor $t3#lo, $t3#lo, $t3#hi @ t3 = (K) (P6 + P7) << 32
  386. vmov.i64 $t3#hi, #0
  387. vext.8 $t1, $t1, $t1, #14
  388. veor $t2#lo, $t2#lo, $t2#hi
  389. vmull.p8 $r, $a, $b @ D = A*B
  390. vext.8 $t3, $t3, $t3, #12
  391. vext.8 $t2, $t2, $t2, #13
  392. veor $t0, $t0, $t1
  393. veor $t2, $t2, $t3
  394. veor $r, $r, $t0
  395. veor $r, $r, $t2
  396. ___
  397. }
  398. $code.=<<___;
  399. #if __ARM_MAX_ARCH__>=7
  400. .arch armv7-a
  401. .fpu neon
  402. .global gcm_init_neon
  403. .type gcm_init_neon,%function
  404. .align 4
  405. gcm_init_neon:
  406. vld1.64 $IN#hi,[r1]! @ load H
  407. vmov.i8 $t0,#0xe1
  408. vld1.64 $IN#lo,[r1]
  409. vshl.i64 $t0#hi,#57
  410. vshr.u64 $t0#lo,#63 @ t0=0xc2....01
  411. vdup.8 $t1,$IN#hi[7]
  412. vshr.u64 $Hlo,$IN#lo,#63
  413. vshr.s8 $t1,#7 @ broadcast carry bit
  414. vshl.i64 $IN,$IN,#1
  415. vand $t0,$t0,$t1
  416. vorr $IN#hi,$Hlo @ H<<<=1
  417. veor $IN,$IN,$t0 @ twisted H
  418. vstmia r0,{$IN}
  419. ret @ bx lr
  420. .size gcm_init_neon,.-gcm_init_neon
  421. .global gcm_gmult_neon
  422. .type gcm_gmult_neon,%function
  423. .align 4
  424. gcm_gmult_neon:
  425. vld1.64 $IN#hi,[$Xi]! @ load Xi
  426. vld1.64 $IN#lo,[$Xi]!
  427. vmov.i64 $k48,#0x0000ffffffffffff
  428. vldmia $Htbl,{$Hlo-$Hhi} @ load twisted H
  429. vmov.i64 $k32,#0x00000000ffffffff
  430. #ifdef __ARMEL__
  431. vrev64.8 $IN,$IN
  432. #endif
  433. vmov.i64 $k16,#0x000000000000ffff
  434. veor $Hhl,$Hlo,$Hhi @ Karatsuba pre-processing
  435. mov $len,#16
  436. b .Lgmult_neon
  437. .size gcm_gmult_neon,.-gcm_gmult_neon
  438. .global gcm_ghash_neon
  439. .type gcm_ghash_neon,%function
  440. .align 4
  441. gcm_ghash_neon:
  442. vld1.64 $Xl#hi,[$Xi]! @ load Xi
  443. vld1.64 $Xl#lo,[$Xi]!
  444. vmov.i64 $k48,#0x0000ffffffffffff
  445. vldmia $Htbl,{$Hlo-$Hhi} @ load twisted H
  446. vmov.i64 $k32,#0x00000000ffffffff
  447. #ifdef __ARMEL__
  448. vrev64.8 $Xl,$Xl
  449. #endif
  450. vmov.i64 $k16,#0x000000000000ffff
  451. veor $Hhl,$Hlo,$Hhi @ Karatsuba pre-processing
  452. .Loop_neon:
  453. vld1.64 $IN#hi,[$inp]! @ load inp
  454. vld1.64 $IN#lo,[$inp]!
  455. #ifdef __ARMEL__
  456. vrev64.8 $IN,$IN
  457. #endif
  458. veor $IN,$Xl @ inp^=Xi
  459. .Lgmult_neon:
  460. ___
  461. &clmul64x64 ($Xl,$Hlo,"$IN#lo"); # H.lo·Xi.lo
  462. $code.=<<___;
  463. veor $IN#lo,$IN#lo,$IN#hi @ Karatsuba pre-processing
  464. ___
  465. &clmul64x64 ($Xm,$Hhl,"$IN#lo"); # (H.lo+H.hi)·(Xi.lo+Xi.hi)
  466. &clmul64x64 ($Xh,$Hhi,"$IN#hi"); # H.hi·Xi.hi
  467. $code.=<<___;
  468. veor $Xm,$Xm,$Xl @ Karatsuba post-processing
  469. veor $Xm,$Xm,$Xh
  470. veor $Xl#hi,$Xl#hi,$Xm#lo
  471. veor $Xh#lo,$Xh#lo,$Xm#hi @ Xh|Xl - 256-bit result
  472. @ equivalent of reduction_avx from ghash-x86_64.pl
  473. vshl.i64 $t1,$Xl,#57 @ 1st phase
  474. vshl.i64 $t2,$Xl,#62
  475. veor $t2,$t2,$t1 @
  476. vshl.i64 $t1,$Xl,#63
  477. veor $t2, $t2, $t1 @
  478. veor $Xl#hi,$Xl#hi,$t2#lo @
  479. veor $Xh#lo,$Xh#lo,$t2#hi
  480. vshr.u64 $t2,$Xl,#1 @ 2nd phase
  481. veor $Xh,$Xh,$Xl
  482. veor $Xl,$Xl,$t2 @
  483. vshr.u64 $t2,$t2,#6
  484. vshr.u64 $Xl,$Xl,#1 @
  485. veor $Xl,$Xl,$Xh @
  486. veor $Xl,$Xl,$t2 @
  487. subs $len,#16
  488. bne .Loop_neon
  489. #ifdef __ARMEL__
  490. vrev64.8 $Xl,$Xl
  491. #endif
  492. sub $Xi,#16
  493. vst1.64 $Xl#hi,[$Xi]! @ write out Xi
  494. vst1.64 $Xl#lo,[$Xi]
  495. ret @ bx lr
  496. .size gcm_ghash_neon,.-gcm_ghash_neon
  497. #endif
  498. ___
  499. }
  500. $code.=<<___;
  501. .asciz "GHASH for ARMv4/NEON, CRYPTOGAMS by <appro\@openssl.org>"
  502. .align 2
  503. ___
  504. foreach (split("\n",$code)) {
  505. s/\`([^\`]*)\`/eval $1/geo;
  506. s/\bq([0-9]+)#(lo|hi)/sprintf "d%d",2*$1+($2 eq "hi")/geo or
  507. s/\bret\b/bx lr/go or
  508. s/\bbx\s+lr\b/.word\t0xe12fff1e/go; # make it possible to compile with -march=armv4
  509. print $_,"\n";
  510. }
  511. close STDOUT or die "error closing STDOUT: $!"; # enforce flush