2
0

x86_64cpuid.pl 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522
  1. #! /usr/bin/env perl
  2. # Copyright 2005-2021 The OpenSSL Project Authors. All Rights Reserved.
  3. #
  4. # Licensed under the Apache License 2.0 (the "License"). You may not use
  5. # this file except in compliance with the License. You can obtain a copy
  6. # in the file LICENSE in the source distribution or at
  7. # https://www.openssl.org/source/license.html
  8. # $output is the last argument if it looks like a file (it has an extension)
  9. # $flavour is the first argument if it doesn't look like a file
  10. $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
  11. $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
  12. $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
  13. $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
  14. ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
  15. ( $xlate="${dir}perlasm/x86_64-xlate.pl" and -f $xlate) or
  16. die "can't locate x86_64-xlate.pl";
  17. open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\""
  18. or die "can't call $xlate: $!";
  19. *STDOUT=*OUT;
  20. ($arg1,$arg2,$arg3,$arg4)=$win64?("%rcx","%rdx","%r8", "%r9") : # Win64 order
  21. ("%rdi","%rsi","%rdx","%rcx"); # Unix order
  22. print<<___;
  23. .extern OPENSSL_cpuid_setup
  24. .hidden OPENSSL_cpuid_setup
  25. .section .init
  26. call OPENSSL_cpuid_setup
  27. .hidden OPENSSL_ia32cap_P
  28. .comm OPENSSL_ia32cap_P,16,4
  29. .text
  30. .globl OPENSSL_atomic_add
  31. .type OPENSSL_atomic_add,\@abi-omnipotent
  32. .align 16
  33. OPENSSL_atomic_add:
  34. .cfi_startproc
  35. endbranch
  36. movl ($arg1),%eax
  37. .Lspin: leaq ($arg2,%rax),%r8
  38. .byte 0xf0 # lock
  39. cmpxchgl %r8d,($arg1)
  40. jne .Lspin
  41. movl %r8d,%eax
  42. .byte 0x48,0x98 # cltq/cdqe
  43. ret
  44. .cfi_endproc
  45. .size OPENSSL_atomic_add,.-OPENSSL_atomic_add
  46. .globl OPENSSL_rdtsc
  47. .type OPENSSL_rdtsc,\@abi-omnipotent
  48. .align 16
  49. OPENSSL_rdtsc:
  50. .cfi_startproc
  51. endbranch
  52. rdtsc
  53. shl \$32,%rdx
  54. or %rdx,%rax
  55. ret
  56. .cfi_endproc
  57. .size OPENSSL_rdtsc,.-OPENSSL_rdtsc
  58. .globl OPENSSL_ia32_cpuid
  59. .type OPENSSL_ia32_cpuid,\@function,1
  60. .align 16
  61. OPENSSL_ia32_cpuid:
  62. .cfi_startproc
  63. endbranch
  64. mov %rbx,%r8 # save %rbx
  65. .cfi_register %rbx,%r8
  66. xor %eax,%eax
  67. mov %rax,8(%rdi) # clear extended feature flags
  68. cpuid
  69. mov %eax,%r11d # max value for standard query level
  70. xor %eax,%eax
  71. cmp \$0x756e6547,%ebx # "Genu"
  72. setne %al
  73. mov %eax,%r9d
  74. cmp \$0x49656e69,%edx # "ineI"
  75. setne %al
  76. or %eax,%r9d
  77. cmp \$0x6c65746e,%ecx # "ntel"
  78. setne %al
  79. or %eax,%r9d # 0 indicates Intel CPU
  80. jz .Lintel
  81. cmp \$0x68747541,%ebx # "Auth"
  82. setne %al
  83. mov %eax,%r10d
  84. cmp \$0x69746E65,%edx # "enti"
  85. setne %al
  86. or %eax,%r10d
  87. cmp \$0x444D4163,%ecx # "cAMD"
  88. setne %al
  89. or %eax,%r10d # 0 indicates AMD CPU
  90. jnz .Lintel
  91. # AMD specific
  92. mov \$0x80000000,%eax
  93. cpuid
  94. cmp \$0x80000001,%eax
  95. jb .Lintel
  96. mov %eax,%r10d
  97. mov \$0x80000001,%eax
  98. cpuid
  99. or %ecx,%r9d
  100. and \$0x00000801,%r9d # isolate AMD XOP bit, 1<<11
  101. cmp \$0x80000008,%r10d
  102. jb .Lintel
  103. mov \$0x80000008,%eax
  104. cpuid
  105. movzb %cl,%r10 # number of cores - 1
  106. inc %r10 # number of cores
  107. mov \$1,%eax
  108. cpuid
  109. bt \$28,%edx # test hyper-threading bit
  110. jnc .Lgeneric
  111. shr \$16,%ebx # number of logical processors
  112. cmp %r10b,%bl
  113. ja .Lgeneric
  114. and \$0xefffffff,%edx # ~(1<<28)
  115. jmp .Lgeneric
  116. .Lintel:
  117. cmp \$4,%r11d
  118. mov \$-1,%r10d
  119. jb .Lnocacheinfo
  120. mov \$4,%eax
  121. mov \$0,%ecx # query L1D
  122. cpuid
  123. mov %eax,%r10d
  124. shr \$14,%r10d
  125. and \$0xfff,%r10d # number of cores -1 per L1D
  126. .Lnocacheinfo:
  127. mov \$1,%eax
  128. cpuid
  129. movd %eax,%xmm0 # put aside processor id
  130. and \$0xbfefffff,%edx # force reserved bits to 0
  131. cmp \$0,%r9d
  132. jne .Lnotintel
  133. or \$0x40000000,%edx # set reserved bit#30 on Intel CPUs
  134. and \$15,%ah
  135. cmp \$15,%ah # examine Family ID
  136. jne .LnotP4
  137. or \$0x00100000,%edx # set reserved bit#20 to engage RC4_CHAR
  138. .LnotP4:
  139. cmp \$6,%ah
  140. jne .Lnotintel
  141. and \$0x0fff0ff0,%eax
  142. cmp \$0x00050670,%eax # Knights Landing
  143. je .Lknights
  144. cmp \$0x00080650,%eax # Knights Mill (according to sde)
  145. jne .Lnotintel
  146. .Lknights:
  147. and \$0xfbffffff,%ecx # clear XSAVE flag to mimic Silvermont
  148. .Lnotintel:
  149. bt \$28,%edx # test hyper-threading bit
  150. jnc .Lgeneric
  151. and \$0xefffffff,%edx # ~(1<<28)
  152. cmp \$0,%r10d
  153. je .Lgeneric
  154. or \$0x10000000,%edx # 1<<28
  155. shr \$16,%ebx
  156. cmp \$1,%bl # see if cache is shared
  157. ja .Lgeneric
  158. and \$0xefffffff,%edx # ~(1<<28)
  159. .Lgeneric:
  160. and \$0x00000800,%r9d # isolate AMD XOP flag
  161. and \$0xfffff7ff,%ecx
  162. or %ecx,%r9d # merge AMD XOP flag
  163. mov %edx,%r10d # %r9d:%r10d is copy of %ecx:%edx
  164. cmp \$7,%r11d
  165. jb .Lno_extended_info
  166. mov \$7,%eax
  167. xor %ecx,%ecx
  168. cpuid
  169. bt \$26,%r9d # check XSAVE bit, cleared on Knights
  170. jc .Lnotknights
  171. and \$0xfff7ffff,%ebx # clear ADCX/ADOX flag
  172. .Lnotknights:
  173. movd %xmm0,%eax # restore processor id
  174. and \$0x0fff0ff0,%eax
  175. cmp \$0x00050650,%eax # Skylake-X
  176. jne .Lnotskylakex
  177. and \$0xfffeffff,%ebx # ~(1<<16)
  178. # suppress AVX512F flag on Skylake-X
  179. .Lnotskylakex:
  180. mov %ebx,8(%rdi) # save extended feature flags
  181. mov %ecx,12(%rdi)
  182. .Lno_extended_info:
  183. bt \$27,%r9d # check OSXSAVE bit
  184. jnc .Lclear_avx
  185. xor %ecx,%ecx # XCR0
  186. .byte 0x0f,0x01,0xd0 # xgetbv
  187. and \$0xe6,%eax # isolate XMM, YMM and ZMM state support
  188. cmp \$0xe6,%eax
  189. je .Ldone
  190. andl \$0x3fdeffff,8(%rdi) # ~(1<<31|1<<30|1<<21|1<<16)
  191. # clear AVX512F+BW+VL+IFMA, all of
  192. # them are EVEX-encoded, which requires
  193. # ZMM state support even if one uses
  194. # only XMM and YMM :-(
  195. and \$6,%eax # isolate XMM and YMM state support
  196. cmp \$6,%eax
  197. je .Ldone
  198. .Lclear_avx:
  199. mov \$0xefffe7ff,%eax # ~(1<<28|1<<12|1<<11)
  200. and %eax,%r9d # clear AVX, FMA and AMD XOP bits
  201. mov \$0x3fdeffdf,%eax # ~(1<<31|1<<30|1<<21|1<<16|1<<5)
  202. and %eax,8(%rdi) # clear AVX2 and AVX512* bits
  203. .Ldone:
  204. shl \$32,%r9
  205. mov %r10d,%eax
  206. mov %r8,%rbx # restore %rbx
  207. .cfi_restore %rbx
  208. or %r9,%rax
  209. ret
  210. .cfi_endproc
  211. .size OPENSSL_ia32_cpuid,.-OPENSSL_ia32_cpuid
  212. .globl OPENSSL_cleanse
  213. .type OPENSSL_cleanse,\@abi-omnipotent
  214. .align 16
  215. OPENSSL_cleanse:
  216. .cfi_startproc
  217. endbranch
  218. xor %rax,%rax
  219. cmp \$15,$arg2
  220. jae .Lot
  221. cmp \$0,$arg2
  222. je .Lret
  223. .Little:
  224. mov %al,($arg1)
  225. sub \$1,$arg2
  226. lea 1($arg1),$arg1
  227. jnz .Little
  228. .Lret:
  229. ret
  230. .align 16
  231. .Lot:
  232. test \$7,$arg1
  233. jz .Laligned
  234. mov %al,($arg1)
  235. lea -1($arg2),$arg2
  236. lea 1($arg1),$arg1
  237. jmp .Lot
  238. .Laligned:
  239. mov %rax,($arg1)
  240. lea -8($arg2),$arg2
  241. test \$-8,$arg2
  242. lea 8($arg1),$arg1
  243. jnz .Laligned
  244. cmp \$0,$arg2
  245. jne .Little
  246. ret
  247. .cfi_endproc
  248. .size OPENSSL_cleanse,.-OPENSSL_cleanse
  249. .globl CRYPTO_memcmp
  250. .type CRYPTO_memcmp,\@abi-omnipotent
  251. .align 16
  252. CRYPTO_memcmp:
  253. .cfi_startproc
  254. endbranch
  255. xor %rax,%rax
  256. xor %r10,%r10
  257. cmp \$0,$arg3
  258. je .Lno_data
  259. cmp \$16,$arg3
  260. jne .Loop_cmp
  261. mov ($arg1),%r10
  262. mov 8($arg1),%r11
  263. mov \$1,$arg3
  264. xor ($arg2),%r10
  265. xor 8($arg2),%r11
  266. or %r11,%r10
  267. cmovnz $arg3,%rax
  268. ret
  269. .align 16
  270. .Loop_cmp:
  271. mov ($arg1),%r10b
  272. lea 1($arg1),$arg1
  273. xor ($arg2),%r10b
  274. lea 1($arg2),$arg2
  275. or %r10b,%al
  276. dec $arg3
  277. jnz .Loop_cmp
  278. neg %rax
  279. shr \$63,%rax
  280. .Lno_data:
  281. ret
  282. .cfi_endproc
  283. .size CRYPTO_memcmp,.-CRYPTO_memcmp
  284. ___
  285. print<<___ if (!$win64);
  286. .globl OPENSSL_wipe_cpu
  287. .type OPENSSL_wipe_cpu,\@abi-omnipotent
  288. .align 16
  289. OPENSSL_wipe_cpu:
  290. .cfi_startproc
  291. endbranch
  292. pxor %xmm0,%xmm0
  293. pxor %xmm1,%xmm1
  294. pxor %xmm2,%xmm2
  295. pxor %xmm3,%xmm3
  296. pxor %xmm4,%xmm4
  297. pxor %xmm5,%xmm5
  298. pxor %xmm6,%xmm6
  299. pxor %xmm7,%xmm7
  300. pxor %xmm8,%xmm8
  301. pxor %xmm9,%xmm9
  302. pxor %xmm10,%xmm10
  303. pxor %xmm11,%xmm11
  304. pxor %xmm12,%xmm12
  305. pxor %xmm13,%xmm13
  306. pxor %xmm14,%xmm14
  307. pxor %xmm15,%xmm15
  308. xorq %rcx,%rcx
  309. xorq %rdx,%rdx
  310. xorq %rsi,%rsi
  311. xorq %rdi,%rdi
  312. xorq %r8,%r8
  313. xorq %r9,%r9
  314. xorq %r10,%r10
  315. xorq %r11,%r11
  316. leaq 8(%rsp),%rax
  317. ret
  318. .cfi_endproc
  319. .size OPENSSL_wipe_cpu,.-OPENSSL_wipe_cpu
  320. ___
  321. print<<___ if ($win64);
  322. .globl OPENSSL_wipe_cpu
  323. .type OPENSSL_wipe_cpu,\@abi-omnipotent
  324. .align 16
  325. OPENSSL_wipe_cpu:
  326. pxor %xmm0,%xmm0
  327. pxor %xmm1,%xmm1
  328. pxor %xmm2,%xmm2
  329. pxor %xmm3,%xmm3
  330. pxor %xmm4,%xmm4
  331. pxor %xmm5,%xmm5
  332. xorq %rcx,%rcx
  333. xorq %rdx,%rdx
  334. xorq %r8,%r8
  335. xorq %r9,%r9
  336. xorq %r10,%r10
  337. xorq %r11,%r11
  338. leaq 8(%rsp),%rax
  339. ret
  340. .size OPENSSL_wipe_cpu,.-OPENSSL_wipe_cpu
  341. ___
  342. {
  343. my $out="%r10";
  344. my $cnt="%rcx";
  345. my $max="%r11";
  346. my $lasttick="%r8d";
  347. my $lastdiff="%r9d";
  348. my $redzone=win64?8:-8;
  349. print<<___;
  350. .globl OPENSSL_instrument_bus
  351. .type OPENSSL_instrument_bus,\@abi-omnipotent
  352. .align 16
  353. OPENSSL_instrument_bus:
  354. .cfi_startproc
  355. endbranch
  356. mov $arg1,$out # tribute to Win64
  357. mov $arg2,$cnt
  358. mov $arg2,$max
  359. rdtsc # collect 1st tick
  360. mov %eax,$lasttick # lasttick = tick
  361. mov \$0,$lastdiff # lastdiff = 0
  362. clflush ($out)
  363. .byte 0xf0 # lock
  364. add $lastdiff,($out)
  365. jmp .Loop
  366. .align 16
  367. .Loop: rdtsc
  368. mov %eax,%edx
  369. sub $lasttick,%eax
  370. mov %edx,$lasttick
  371. mov %eax,$lastdiff
  372. clflush ($out)
  373. .byte 0xf0 # lock
  374. add %eax,($out)
  375. lea 4($out),$out
  376. sub \$1,$cnt
  377. jnz .Loop
  378. mov $max,%rax
  379. ret
  380. .cfi_endproc
  381. .size OPENSSL_instrument_bus,.-OPENSSL_instrument_bus
  382. .globl OPENSSL_instrument_bus2
  383. .type OPENSSL_instrument_bus2,\@abi-omnipotent
  384. .align 16
  385. OPENSSL_instrument_bus2:
  386. .cfi_startproc
  387. endbranch
  388. mov $arg1,$out # tribute to Win64
  389. mov $arg2,$cnt
  390. mov $arg3,$max
  391. mov $cnt,$redzone(%rsp)
  392. rdtsc # collect 1st tick
  393. mov %eax,$lasttick # lasttick = tick
  394. mov \$0,$lastdiff # lastdiff = 0
  395. clflush ($out)
  396. .byte 0xf0 # lock
  397. add $lastdiff,($out)
  398. rdtsc # collect 1st diff
  399. mov %eax,%edx
  400. sub $lasttick,%eax # diff
  401. mov %edx,$lasttick # lasttick = tick
  402. mov %eax,$lastdiff # lastdiff = diff
  403. .Loop2:
  404. clflush ($out)
  405. .byte 0xf0 # lock
  406. add %eax,($out) # accumulate diff
  407. sub \$1,$max
  408. jz .Ldone2
  409. rdtsc
  410. mov %eax,%edx
  411. sub $lasttick,%eax # diff
  412. mov %edx,$lasttick # lasttick = tick
  413. cmp $lastdiff,%eax
  414. mov %eax,$lastdiff # lastdiff = diff
  415. mov \$0,%edx
  416. setne %dl
  417. sub %rdx,$cnt # conditional --$cnt
  418. lea ($out,%rdx,4),$out # conditional ++$out
  419. jnz .Loop2
  420. .Ldone2:
  421. mov $redzone(%rsp),%rax
  422. sub $cnt,%rax
  423. ret
  424. .cfi_endproc
  425. .size OPENSSL_instrument_bus2,.-OPENSSL_instrument_bus2
  426. ___
  427. }
  428. sub gen_random {
  429. my $rdop = shift;
  430. print<<___;
  431. .globl OPENSSL_ia32_${rdop}_bytes
  432. .type OPENSSL_ia32_${rdop}_bytes,\@abi-omnipotent
  433. .align 16
  434. OPENSSL_ia32_${rdop}_bytes:
  435. .cfi_startproc
  436. endbranch
  437. xor %rax, %rax # return value
  438. cmp \$0,$arg2
  439. je .Ldone_${rdop}_bytes
  440. mov \$8,%r11
  441. .Loop_${rdop}_bytes:
  442. ${rdop} %r10
  443. jc .Lbreak_${rdop}_bytes
  444. dec %r11
  445. jnz .Loop_${rdop}_bytes
  446. jmp .Ldone_${rdop}_bytes
  447. .align 16
  448. .Lbreak_${rdop}_bytes:
  449. cmp \$8,$arg2
  450. jb .Ltail_${rdop}_bytes
  451. mov %r10,($arg1)
  452. lea 8($arg1),$arg1
  453. add \$8,%rax
  454. sub \$8,$arg2
  455. jz .Ldone_${rdop}_bytes
  456. mov \$8,%r11
  457. jmp .Loop_${rdop}_bytes
  458. .align 16
  459. .Ltail_${rdop}_bytes:
  460. mov %r10b,($arg1)
  461. lea 1($arg1),$arg1
  462. inc %rax
  463. shr \$8,%r10
  464. dec $arg2
  465. jnz .Ltail_${rdop}_bytes
  466. .Ldone_${rdop}_bytes:
  467. xor %r10,%r10 # Clear sensitive data from register
  468. ret
  469. .cfi_endproc
  470. .size OPENSSL_ia32_${rdop}_bytes,.-OPENSSL_ia32_${rdop}_bytes
  471. ___
  472. }
  473. gen_random("rdrand");
  474. gen_random("rdseed");
  475. close STDOUT or die "error closing STDOUT: $!"; # flush