|
@@ -7,6 +7,8 @@
|
|
|
# details see http://www.openssl.org/~appro/cryptogams/.
|
|
|
# ====================================================================
|
|
|
#
|
|
|
+# July 2004
|
|
|
+#
|
|
|
# 2.22x RC4 tune-up:-) It should be noted though that my hand [as in
|
|
|
# "hand-coded assembler"] doesn't stand for the whole improvement
|
|
|
# coefficient. It turned out that eliminating RC4_CHAR from config
|
|
@@ -19,6 +21,8 @@
|
|
|
# to operate on partial registers, it turned out to be the best bet.
|
|
|
# At least for AMD... How IA32E would perform remains to be seen...
|
|
|
|
|
|
+# November 2004
|
|
|
+#
|
|
|
# As was shown by Marc Bevand reordering of couple of load operations
|
|
|
# results in even higher performance gain of 3.3x:-) At least on
|
|
|
# Opteron... For reference, 1x in this case is RC4_CHAR C-code
|
|
@@ -26,6 +30,8 @@
|
|
|
# Latter means that if you want to *estimate* what to expect from
|
|
|
# *your* Opteron, then multiply 54 by 3.3 and clock frequency in GHz.
|
|
|
|
|
|
+# November 2004
|
|
|
+#
|
|
|
# Intel P4 EM64T core was found to run the AMD64 code really slow...
|
|
|
# The only way to achieve comparable performance on P4 was to keep
|
|
|
# RC4_CHAR. Kind of ironic, huh? As it's apparently impossible to
|
|
@@ -33,10 +39,14 @@
|
|
|
# on either AMD and Intel platforms, I implement both cases. See
|
|
|
# rc4_skey.c for further details...
|
|
|
|
|
|
+# April 2005
|
|
|
+#
|
|
|
# P4 EM64T core appears to be "allergic" to 64-bit inc/dec. Replacing
|
|
|
# those with add/sub results in 50% performance improvement of folded
|
|
|
# loop...
|
|
|
|
|
|
+# May 2005
|
|
|
+#
|
|
|
# As was shown by Zou Nanhai loop unrolling can improve Intel EM64T
|
|
|
# performance by >30% [unlike P4 32-bit case that is]. But this is
|
|
|
# provided that loads are reordered even more aggressively! Both code
|
|
@@ -50,6 +60,8 @@
|
|
|
# is not implemented, then this final RC4_CHAR code-path should be
|
|
|
# preferred, as it provides better *all-round* performance].
|
|
|
|
|
|
+# March 2007
|
|
|
+#
|
|
|
# Intel Core2 was observed to perform poorly on both code paths:-( It
|
|
|
# apparently suffers from some kind of partial register stall, which
|
|
|
# occurs in 64-bit mode only [as virtually identical 32-bit loop was
|
|
@@ -58,10 +70,32 @@
|
|
|
# fit for Core2 and therefore the code was modified to skip cloop8 on
|
|
|
# this CPU.
|
|
|
|
|
|
+# May 2010
|
|
|
+#
|
|
|
# Intel Westmere was observed to perform suboptimally. Adding yet
|
|
|
# another movzb to cloop1 improved performance by almost 50%! Core2
|
|
|
# performance is improved too, but nominally...
|
|
|
|
|
|
+# May 2011
|
|
|
+#
|
|
|
+# The only code path that was not modified is P4-specific one. New
|
|
|
+# AMD code path is inspired by and Intel optimization is heavily
|
|
|
+# based on submission from Maxim Locktyukhin of Intel. Current
|
|
|
+# performance in cycles per processed byte (less is better) and
|
|
|
+# improvement coefficients relative to previous version of this
|
|
|
+# module are:
|
|
|
+#
|
|
|
+# Opteron 5.3/+0%
|
|
|
+# P4 6.5
|
|
|
+# Core2 6.2/+15%(*)
|
|
|
+# Westmere 4.2/+60%
|
|
|
+# Sandy Bridge 4.2/+120%
|
|
|
+# Atom 9.3/+80%
|
|
|
+#
|
|
|
+# (*) Note that this result is ~15% lower than result for 32-bit
|
|
|
+# code, meaning that it's possible to improve it, but it's
|
|
|
+# more than likely at the cost of the others...
|
|
|
+
|
|
|
$flavour = shift;
|
|
|
$output = shift;
|
|
|
if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
|
|
@@ -80,11 +114,7 @@ $len="%rsi"; # arg2
|
|
|
$inp="%rdx"; # arg3
|
|
|
$out="%rcx"; # arg4
|
|
|
|
|
|
-@XX=("%r8","%r10");
|
|
|
-@TX=("%r9","%r11");
|
|
|
-$YY="%r12";
|
|
|
-$TY="%r13";
|
|
|
-
|
|
|
+{
|
|
|
$code=<<___;
|
|
|
.text
|
|
|
|
|
@@ -99,48 +129,173 @@ RC4: or $len,$len
|
|
|
push %r12
|
|
|
push %r13
|
|
|
.Lprologue:
|
|
|
+ mov $len,%r11
|
|
|
+ mov $inp,%r12
|
|
|
+ mov $out,%r13
|
|
|
+___
|
|
|
+my $len="%r11"; # reassign input arguments
|
|
|
+my $inp="%r12";
|
|
|
+my $out="%r13";
|
|
|
+
|
|
|
+my @XX=("%r10","%rsi");
|
|
|
+my @TX=("%rax","%rbx");
|
|
|
+my $YY="%rcx";
|
|
|
+my $TY="%rdx";
|
|
|
|
|
|
- add \$8,$dat
|
|
|
- movl -8($dat),$XX[0]#d
|
|
|
- movl -4($dat),$YY#d
|
|
|
+$code.=<<___;
|
|
|
+ xor $XX[0],$XX[0]
|
|
|
+ xor $YY,$YY
|
|
|
+
|
|
|
+ lea 8($dat),$dat
|
|
|
+ mov -8($dat),$XX[0]#b
|
|
|
+ mov -4($dat),$YY#b
|
|
|
cmpl \$-1,256($dat)
|
|
|
je .LRC4_CHAR
|
|
|
+ mov OPENSSL_ia32cap_P(%rip),%r8d
|
|
|
+ xor $TX[1],$TX[1]
|
|
|
inc $XX[0]#b
|
|
|
+ sub $XX[0],$TX[1]
|
|
|
+ sub $inp,$out
|
|
|
movl ($dat,$XX[0],4),$TX[0]#d
|
|
|
- test \$-8,$len
|
|
|
+ test \$-16,$len
|
|
|
jz .Lloop1
|
|
|
- jmp .Lloop8
|
|
|
+ bt \$30,%r8d # Intel CPU Family 6
|
|
|
+ jc .L16x
|
|
|
+ and \$7,$TX[1]
|
|
|
+ lea 1($XX[0]),$XX[1]
|
|
|
+ jz .Loop8
|
|
|
+ sub $TX[1],$len
|
|
|
+.Loop8_warmup:
|
|
|
+ add $TX[0]#b,$YY#b
|
|
|
+ movl ($dat,$YY,4),$TY#d
|
|
|
+ movl $TX[0]#d,($dat,$YY,4)
|
|
|
+ movl $TY#d,($dat,$XX[0],4)
|
|
|
+ add $TY#b,$TX[0]#b
|
|
|
+ inc $XX[0]#b
|
|
|
+ movl ($dat,$TX[0],4),$TY#d
|
|
|
+ movl ($dat,$XX[0],4),$TX[0]#d
|
|
|
+ xorb ($inp),$TY#b
|
|
|
+ movb $TY#b,($out,$inp)
|
|
|
+ lea 1($inp),$inp
|
|
|
+ dec $TX[1]
|
|
|
+ jnz .Loop8_warmup
|
|
|
+
|
|
|
+ lea 1($XX[0]),$XX[1]
|
|
|
+ jmp .Loop8
|
|
|
.align 16
|
|
|
-.Lloop8:
|
|
|
+.Loop8:
|
|
|
___
|
|
|
for ($i=0;$i<8;$i++) {
|
|
|
+$code.=<<___ if ($i==7);
|
|
|
+ add \$8,$XX[1]#b
|
|
|
+___
|
|
|
$code.=<<___;
|
|
|
add $TX[0]#b,$YY#b
|
|
|
- mov $XX[0],$XX[1]
|
|
|
movl ($dat,$YY,4),$TY#d
|
|
|
- ror \$8,%rax # ror is redundant when $i=0
|
|
|
- inc $XX[1]#b
|
|
|
- movl ($dat,$XX[1],4),$TX[1]#d
|
|
|
- cmp $XX[1],$YY
|
|
|
movl $TX[0]#d,($dat,$YY,4)
|
|
|
- cmove $TX[0],$TX[1]
|
|
|
- movl $TY#d,($dat,$XX[0],4)
|
|
|
+ movl `4*($i==7?-1:$i)`($dat,$XX[1],4),$TX[1]#d
|
|
|
+ ror \$8,%r8 # ror is redundant when $i=0
|
|
|
+ movl $TY#d,4*$i($dat,$XX[0],4)
|
|
|
add $TX[0]#b,$TY#b
|
|
|
- movb ($dat,$TY,4),%al
|
|
|
+ movb ($dat,$TY,4),%r8b
|
|
|
___
|
|
|
-push(@TX,shift(@TX)); push(@XX,shift(@XX)); # "rotate" registers
|
|
|
+push(@TX,shift(@TX)); #push(@XX,shift(@XX)); # "rotate" registers
|
|
|
}
|
|
|
$code.=<<___;
|
|
|
- ror \$8,%rax
|
|
|
+ add \$8,$XX[0]#b
|
|
|
+ ror \$8,%r8
|
|
|
sub \$8,$len
|
|
|
|
|
|
- xor ($inp),%rax
|
|
|
- add \$8,$inp
|
|
|
- mov %rax,($out)
|
|
|
- add \$8,$out
|
|
|
+ xor ($inp),%r8
|
|
|
+ mov %r8,($out,$inp)
|
|
|
+ lea 8($inp),$inp
|
|
|
|
|
|
test \$-8,$len
|
|
|
- jnz .Lloop8
|
|
|
+ jnz .Loop8
|
|
|
+ cmp \$0,$len
|
|
|
+ jne .Lloop1
|
|
|
+ jmp .Lexit
|
|
|
+
|
|
|
+.align 16
|
|
|
+.L16x:
|
|
|
+ test \$-32,$len
|
|
|
+ jz .Lloop1
|
|
|
+ and \$15,$TX[1]
|
|
|
+ jz .Loop16_is_hot
|
|
|
+ sub $TX[1],$len
|
|
|
+.Loop16_warmup:
|
|
|
+ add $TX[0]#b,$YY#b
|
|
|
+ movl ($dat,$YY,4),$TY#d
|
|
|
+ movl $TX[0]#d,($dat,$YY,4)
|
|
|
+ movl $TY#d,($dat,$XX[0],4)
|
|
|
+ add $TY#b,$TX[0]#b
|
|
|
+ inc $XX[0]#b
|
|
|
+ movl ($dat,$TX[0],4),$TY#d
|
|
|
+ movl ($dat,$XX[0],4),$TX[0]#d
|
|
|
+ xorb ($inp),$TY#b
|
|
|
+ movb $TY#b,($out,$inp)
|
|
|
+ lea 1($inp),$inp
|
|
|
+ dec $TX[1]
|
|
|
+ jnz .Loop16_warmup
|
|
|
+
|
|
|
+ mov $YY,$TX[1]
|
|
|
+ xor $YY,$YY
|
|
|
+ mov $TX[1]#b,$YY#b
|
|
|
+
|
|
|
+.Loop16_is_hot:
|
|
|
+ lea ($dat,$XX[0],4),$XX[1]
|
|
|
+___
|
|
|
+sub RC4_loop {
|
|
|
+ my $i=shift;
|
|
|
+ my $j=$i<0?0:$i;
|
|
|
+ my $xmm="%xmm".($j&1);
|
|
|
+
|
|
|
+ $code.=" add \$16,$XX[0]#b\n" if ($i==15);
|
|
|
+ $code.=" movdqu ($inp),%xmm2\n" if ($i==15);
|
|
|
+ $code.=" add $TX[0]#b,$YY#b\n" if ($i<=0);
|
|
|
+ $code.=" movl ($dat,$YY,4),$TY#d\n";
|
|
|
+ $code.=" pxor %xmm0,%xmm2\n" if ($i==0);
|
|
|
+ $code.=" psllq \$8,%xmm1\n" if ($i==0);
|
|
|
+ $code.=" pxor $xmm,$xmm\n" if ($i<=1);
|
|
|
+ $code.=" movl $TX[0]#d,($dat,$YY,4)\n";
|
|
|
+ $code.=" add $TY#b,$TX[0]#b\n";
|
|
|
+ $code.=" movl `4*($j+1)`($XX[1]),$TX[1]#d\n" if ($i<15);
|
|
|
+ $code.=" movz $TX[0]#b,$TX[0]#d\n";
|
|
|
+ $code.=" movl $TY#d,4*$j($XX[1])\n";
|
|
|
+ $code.=" pxor %xmm1,%xmm2\n" if ($i==0);
|
|
|
+ $code.=" lea ($dat,$XX[0],4),$XX[1]\n" if ($i==15);
|
|
|
+ $code.=" add $TX[1]#b,$YY#b\n" if ($i<15);
|
|
|
+ $code.=" pinsrw \$`($j>>1)&7`,($dat,$TX[0],4),$xmm\n";
|
|
|
+ $code.=" movdqu %xmm2,($out,$inp)\n" if ($i==0);
|
|
|
+ $code.=" lea 16($inp),$inp\n" if ($i==0);
|
|
|
+ $code.=" movl ($XX[1]),$TX[1]#d\n" if ($i==15);
|
|
|
+}
|
|
|
+ RC4_loop(-1);
|
|
|
+$code.=<<___;
|
|
|
+ jmp .Loop16_enter
|
|
|
+.align 16
|
|
|
+.Loop16:
|
|
|
+___
|
|
|
+
|
|
|
+for ($i=0;$i<16;$i++) {
|
|
|
+ $code.=".Loop16_enter:\n" if ($i==1);
|
|
|
+ RC4_loop($i);
|
|
|
+ push(@TX,shift(@TX)); # "rotate" registers
|
|
|
+}
|
|
|
+$code.=<<___;
|
|
|
+ mov $YY,$TX[1]
|
|
|
+ xor $YY,$YY # keyword to partial register
|
|
|
+ sub \$16,$len
|
|
|
+ mov $TX[1]#b,$YY#b
|
|
|
+ test \$-16,$len
|
|
|
+ jnz .Loop16
|
|
|
+
|
|
|
+ psllq \$8,%xmm1
|
|
|
+ pxor %xmm0,%xmm2
|
|
|
+ pxor %xmm1,%xmm2
|
|
|
+ movdqu %xmm2,($out,$inp)
|
|
|
+ lea 16($inp),$inp
|
|
|
+
|
|
|
cmp \$0,$len
|
|
|
jne .Lloop1
|
|
|
jmp .Lexit
|
|
@@ -156,9 +311,8 @@ $code.=<<___;
|
|
|
movl ($dat,$TX[0],4),$TY#d
|
|
|
movl ($dat,$XX[0],4),$TX[0]#d
|
|
|
xorb ($inp),$TY#b
|
|
|
- inc $inp
|
|
|
- movb $TY#b,($out)
|
|
|
- inc $out
|
|
|
+ movb $TY#b,($out,$inp)
|
|
|
+ lea 1($inp),$inp
|
|
|
dec $len
|
|
|
jnz .Lloop1
|
|
|
jmp .Lexit
|
|
@@ -169,13 +323,11 @@ $code.=<<___;
|
|
|
movzb ($dat,$XX[0]),$TX[0]#d
|
|
|
test \$-8,$len
|
|
|
jz .Lcloop1
|
|
|
- cmpl \$0,260($dat)
|
|
|
- jnz .Lcloop1
|
|
|
jmp .Lcloop8
|
|
|
.align 16
|
|
|
.Lcloop8:
|
|
|
- mov ($inp),%eax
|
|
|
- mov 4($inp),%ebx
|
|
|
+ mov ($inp),%r8d
|
|
|
+ mov 4($inp),%r9d
|
|
|
___
|
|
|
# unroll 2x4-wise, because 64-bit rotates kill Intel P4...
|
|
|
for ($i=0;$i<4;$i++) {
|
|
@@ -192,8 +344,8 @@ $code.=<<___;
|
|
|
mov $TX[0],$TX[1]
|
|
|
.Lcmov$i:
|
|
|
add $TX[0]#b,$TY#b
|
|
|
- xor ($dat,$TY),%al
|
|
|
- ror \$8,%eax
|
|
|
+ xor ($dat,$TY),%r8b
|
|
|
+ ror \$8,%r8d
|
|
|
___
|
|
|
push(@TX,shift(@TX)); push(@XX,shift(@XX)); # "rotate" registers
|
|
|
}
|
|
@@ -211,16 +363,16 @@ $code.=<<___;
|
|
|
mov $TX[0],$TX[1]
|
|
|
.Lcmov$i:
|
|
|
add $TX[0]#b,$TY#b
|
|
|
- xor ($dat,$TY),%bl
|
|
|
- ror \$8,%ebx
|
|
|
+ xor ($dat,$TY),%r9b
|
|
|
+ ror \$8,%r9d
|
|
|
___
|
|
|
push(@TX,shift(@TX)); push(@XX,shift(@XX)); # "rotate" registers
|
|
|
}
|
|
|
$code.=<<___;
|
|
|
lea -8($len),$len
|
|
|
- mov %eax,($out)
|
|
|
+ mov %r8d,($out)
|
|
|
lea 8($inp),$inp
|
|
|
- mov %ebx,4($out)
|
|
|
+ mov %r9d,4($out)
|
|
|
lea 8($out),$out
|
|
|
|
|
|
test \$-8,$len
|
|
@@ -265,6 +417,7 @@ $code.=<<___;
|
|
|
ret
|
|
|
.size RC4,.-RC4
|
|
|
___
|
|
|
+}
|
|
|
|
|
|
$idx="%r8";
|
|
|
$ido="%r9";
|
|
@@ -285,12 +438,11 @@ RC4_set_key:
|
|
|
xor %r11,%r11
|
|
|
|
|
|
mov OPENSSL_ia32cap_P(%rip),$idx#d
|
|
|
- bt \$20,$idx#d
|
|
|
+ bt \$20,$idx#d # Intel CPU
|
|
|
jnc .Lw1stloop
|
|
|
- bt \$30,$idx#d
|
|
|
- setc $ido#b
|
|
|
- mov $ido#d,260($dat)
|
|
|
- jmp .Lc1stloop
|
|
|
+ bt \$30,$idx#d # Intel CPU Family 6
|
|
|
+ jnc .Lc1stloop
|
|
|
+ jmp .Lw1stloop
|
|
|
|
|
|
.align 16
|
|
|
.Lw1stloop:
|
|
@@ -364,7 +516,7 @@ RC4_options:
|
|
|
.Lopts:
|
|
|
.asciz "rc4(8x,int)"
|
|
|
.asciz "rc4(8x,char)"
|
|
|
-.asciz "rc4(1x,char)"
|
|
|
+.asciz "rc4(16x,int)"
|
|
|
.asciz "RC4 for x86_64, CRYPTOGAMS by <appro\@openssl.org>"
|
|
|
.align 64
|
|
|
.size RC4_options,.-RC4_options
|
|
@@ -502,7 +654,17 @@ key_se_handler:
|
|
|
___
|
|
|
}
|
|
|
|
|
|
-$code =~ s/#([bwd])/$1/gm;
|
|
|
+sub reg_part {
|
|
|
+my ($reg,$conv)=@_;
|
|
|
+ if ($reg =~ /%r[0-9]+/) { $reg .= $conv; }
|
|
|
+ elsif ($conv eq "b") { $reg =~ s/%[er]([^x]+)x?/%$1l/; }
|
|
|
+ elsif ($conv eq "w") { $reg =~ s/%[er](.+)/%$1/; }
|
|
|
+ elsif ($conv eq "d") { $reg =~ s/%[er](.+)/%e$1/; }
|
|
|
+ return $reg;
|
|
|
+}
|
|
|
+
|
|
|
+$code =~ s/(%[a-z0-9]+)#([bwd])/reg_part($1,$2)/gem;
|
|
|
+$code =~ s/\`([^\`]*)\`/eval $1/gem;
|
|
|
|
|
|
print $code;
|
|
|
|