openssl/crypto/modes/asm/ghash-c64xplus.pl

#! /usr/bin/env perl
# Copyright 2012-2020 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the Apache License 2.0 (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html

#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
# December 2011
#
# The module implements GCM GHASH function and underlying single
# multiplication operation in GF(2^128). Even though subroutines
# have _4bit suffix, they are not using any tables, but rely on
# hardware Galois Field Multiply support. Streamed GHASH processes
# byte in ~7 cycles, which is >6x faster than "4-bit" table-driven
# code compiled with TI's cl6x 6.0 with -mv6400+ -o2 flags. We are
# comparing apples vs. oranges, but compiler surely could have done
# better, because theoretical [though not necessarily achievable]
# estimate for "4-bit" table-driven implementation is ~12 cycles.

$output = pop and open STDOUT,">$output";

($Xip,$Htable,$inp,$len)=("A4","B4","A6","B6");	# arguments

($Z0,$Z1,$Z2,$Z3,	$H0, $H1, $H2, $H3,
			$H0x,$H1x,$H2x,$H3x)=map("A$_",(16..27));
($H01u,$H01y,$H2u,$H3u,	$H0y,$H1y,$H2y,$H3y,
			$H0z,$H1z,$H2z,$H3z)=map("B$_",(16..27));
($FF000000,$E10000)=("B30","B31");
($xip,$x0,$x1,$xib)=map("B$_",(6..9));	# $xip zaps $len
 $xia="A9";
($rem,$res)=("B4","B5");		# $rem zaps $Htable

$code.=<<___;
	.text

	.if	.ASSEMBLER_VERSION<7000000
	.asg	0,__TI_EABI__
	.endif
	.if	__TI_EABI__
	.asg	gcm_gmult_1bit,_gcm_gmult_1bit
	.asg	gcm_gmult_4bit,_gcm_gmult_4bit
	.asg	gcm_ghash_4bit,_gcm_ghash_4bit
	.endif

	.asg	B3,RA

	.if	0
	.global	_gcm_gmult_1bit
_gcm_gmult_1bit:
	ADDAD	$Htable,2,$Htable
	.endif
	.global	_gcm_gmult_4bit
_gcm_gmult_4bit:
	.asmfunc
	LDDW	*${Htable}[-1],$H1:$H0	; H.lo
	LDDW	*${Htable}[-2],$H3:$H2	; H.hi
||	MV	$Xip,${xip}		; reassign Xi
||	MVK	15,B1			; SPLOOPD constant

	MVK	0xE1,$E10000
||	LDBU	*++${xip}[15],$x1	; Xi[15]
	MVK	0xFF,$FF000000
||	LDBU	*--${xip},$x0		; Xi[14]
	SHL	$E10000,16,$E10000	; [pre-shifted] reduction polynomial
	SHL	$FF000000,24,$FF000000	; upper byte mask
||	BNOP	ghash_loop?
||	MVK	1,B0			; take a single spin

	PACKH2	$H0,$H1,$xia		; pack H0' and H1's upper bytes
	AND	$H2,$FF000000,$H2u	; H2's upper byte
	AND	$H3,$FF000000,$H3u	; H3's upper byte
||	SHRU	$H2u,8,$H2u
	SHRU	$H3u,8,$H3u
||	ZERO	$Z1:$Z0
	SHRU2	$xia,8,$H01u
||	ZERO	$Z3:$Z2
	.endasmfunc

	.global	_gcm_ghash_4bit
_gcm_ghash_4bit:
	.asmfunc
	LDDW	*${Htable}[-1],$H1:$H0	; H.lo
||	SHRU	$len,4,B0		; reassign len
	LDDW	*${Htable}[-2],$H3:$H2	; H.hi
||	MV	$Xip,${xip}		; reassign Xi
||	MVK	15,B1			; SPLOOPD constant

	MVK	0xE1,$E10000
|| [B0]	LDNDW	*${inp}[1],$H1x:$H0x
	MVK	0xFF,$FF000000
|| [B0]	LDNDW	*${inp}++[2],$H3x:$H2x
	SHL	$E10000,16,$E10000	; [pre-shifted] reduction polynomial
||	LDDW	*${xip}[1],$Z1:$Z0
	SHL	$FF000000,24,$FF000000	; upper byte mask
||	LDDW	*${xip}[0],$Z3:$Z2

	PACKH2	$H0,$H1,$xia		; pack H0' and H1's upper bytes
	AND	$H2,$FF000000,$H2u	; H2's upper byte
	AND	$H3,$FF000000,$H3u	; H3's upper byte
||	SHRU	$H2u,8,$H2u
	SHRU	$H3u,8,$H3u
	SHRU2	$xia,8,$H01u

|| [B0]	XOR	$H0x,$Z0,$Z0		; Xi^=inp
|| [B0]	XOR	$H1x,$Z1,$Z1
	.if	.LITTLE_ENDIAN
   [B0]	XOR	$H2x,$Z2,$Z2
|| [B0]	XOR	$H3x,$Z3,$Z3
|| [B0]	SHRU	$Z1,24,$xia		; Xi[15], avoid cross-path stall
	STDW	$Z1:$Z0,*${xip}[1]
|| [B0]	SHRU	$Z1,16,$x0		; Xi[14]
|| [B0]	ZERO	$Z1:$Z0
	.else
   [B0]	XOR	$H2x,$Z2,$Z2
|| [B0]	XOR	$H3x,$Z3,$Z3
|| [B0]	MV	$Z0,$xia		; Xi[15], avoid cross-path stall
	STDW	$Z1:$Z0,*${xip}[1]
|| [B0] SHRU	$Z0,8,$x0		; Xi[14]
|| [B0]	ZERO	$Z1:$Z0
	.endif
	STDW	$Z3:$Z2,*${xip}[0]
|| [B0]	ZERO	$Z3:$Z2
|| [B0]	MV	$xia,$x1
   [B0]	ADDK	14,${xip}

ghash_loop?:
	SPLOOPD	6			; 6*16+7
||	MVC	B1,ILC
|| [B0]	SUB	B0,1,B0
||	ZERO	A0
||	ADD	$x1,$x1,$xib		; SHL	$x1,1,$xib
||	SHL	$x1,1,$xia
___

########____________________________
#  0    D2.     M1          M2      |
#  1            M1                  |
#  2            M1          M2      |
#  3        D1. M1          M2      |
#  4        S1. L1                  |
#  5    S2  S1x L1          D2  L2  |____________________________
#  6/0          L1  S1      L2  S2x |D2.     M1          M2      |
#  7/1          L1  S1  D1x S2  M2  |        M1                  |
#  8/2              S1  L1x S2      |        M1          M2      |
#  9/3              S1  L1x         |    D1. M1          M2      |
# 10/4                  D1x         |    S1. L1                  |
# 11/5                              |S2  S1x L1          D2  L2  |____________
# 12/6/0                D1x       __|        L1  S1      L2  S2x |D2.     ....
#    7/1                                     L1  S1  D1x S2  M2  |        ....
#    8/2                                         S1  L1x S2      |        ....
#####...                                         ................|............
$code.=<<___;
	XORMPY	$H0,$xia,$H0x		; 0	; H·(Xi[i]<<1)
||	XORMPY	$H01u,$xib,$H01y
|| [A0]	LDBU	*--${xip},$x0
	XORMPY	$H1,$xia,$H1x		; 1
	XORMPY	$H2,$xia,$H2x		; 2
||	XORMPY	$H2u,$xib,$H2y
	XORMPY	$H3,$xia,$H3x		; 3
||	XORMPY	$H3u,$xib,$H3y
||[!A0]	MVK.D	15,A0				; *--${xip} counter
	XOR.L	$H0x,$Z0,$Z0		; 4	; Z^=H·(Xi[i]<<1)
|| [A0]	SUB.S	A0,1,A0
	XOR.L	$H1x,$Z1,$Z1		; 5
||	AND.D	$H01y,$FF000000,$H0z
||	SWAP2.L	$H01y,$H1y		;	; SHL	$H01y,16,$H1y
||	SHL	$x0,1,$xib
||	SHL	$x0,1,$xia

	XOR.L	$H2x,$Z2,$Z2		; 6/0	; [0,0] in epilogue
||	SHL	$Z0,1,$rem		;	; rem=Z<<1
||	SHRMB.S	$Z1,$Z0,$Z0		;	; Z>>=8
||	AND.L	$H1y,$FF000000,$H1z
	XOR.L	$H3x,$Z3,$Z3		; 7/1
||	SHRMB.S	$Z2,$Z1,$Z1
||	XOR.D	$H0z,$Z0,$Z0			; merge upper byte products
||	AND.S	$H2y,$FF000000,$H2z
||	XORMPY	$E10000,$rem,$res	;	; implicit rem&0x1FE
	XOR.L	$H1z,$Z1,$Z1		; 8/2
||	SHRMB.S	$Z3,$Z2,$Z2
||	AND.S	$H3y,$FF000000,$H3z
	XOR.L	$H2z,$Z2,$Z2		; 9/3
||	SHRU	$Z3,8,$Z3
	XOR.D	$H3z,$Z3,$Z3		; 10/4
	NOP				; 11/5

	SPKERNEL 0,2
||	XOR.D	$res,$Z3,$Z3		; 12/6/0; Z^=res

	; input pre-fetch is possible where D1 slot is available...
   [B0]	LDNDW	*${inp}[1],$H1x:$H0x	; 8/-
   [B0]	LDNDW	*${inp}++[2],$H3x:$H2x	; 9/-
	NOP				; 10/-
	.if	.LITTLE_ENDIAN
	SWAP2	$Z0,$Z1			; 11/-
||	SWAP4	$Z1,$Z0
	SWAP4	$Z1,$Z1			; 12/-
||	SWAP2	$Z0,$Z0
	SWAP2	$Z2,$Z3
||	SWAP4	$Z3,$Z2
||[!B0]	BNOP	RA
	SWAP4	$Z3,$Z3
||	SWAP2	$Z2,$Z2
|| [B0]	BNOP	ghash_loop?
   [B0]	XOR	$H0x,$Z0,$Z0		; Xi^=inp
|| [B0]	XOR	$H1x,$Z1,$Z1
   [B0]	XOR	$H2x,$Z2,$Z2
|| [B0]	XOR	$H3x,$Z3,$Z3
|| [B0]	SHRU	$Z1,24,$xia		; Xi[15], avoid cross-path stall
	STDW	$Z1:$Z0,*${xip}[1]
|| [B0]	SHRU	$Z1,16,$x0		; Xi[14]
|| [B0]	ZERO	$Z1:$Z0
	.else
  [!B0]	BNOP	RA			; 11/-
   [B0]	BNOP	ghash_loop?		; 12/-
   [B0]	XOR	$H0x,$Z0,$Z0		; Xi^=inp
|| [B0]	XOR	$H1x,$Z1,$Z1
   [B0]	XOR	$H2x,$Z2,$Z2
|| [B0]	XOR	$H3x,$Z3,$Z3
|| [B0]	MV	$Z0,$xia		; Xi[15], avoid cross-path stall
	STDW	$Z1:$Z0,*${xip}[1]
|| [B0] SHRU	$Z0,8,$x0		; Xi[14]
|| [B0]	ZERO	$Z1:$Z0
	.endif
	STDW	$Z3:$Z2,*${xip}[0]
|| [B0]	ZERO	$Z3:$Z2
|| [B0]	MV	$xia,$x1
   [B0]	ADDK	14,${xip}
	.endasmfunc

	.sect	.const
	.cstring "GHASH for C64x+, CRYPTOGAMS by <appro\@openssl.org>"
	.align	4
___

print $code;
close STDOUT or die "error closing STDOUT: $!";