openssl/crypto/poly1305/asm/poly1305-ia64.S

// ====================================================================
// Written by Andy Polyakov, @dot-asm, initially for use in the OpenSSL
// project.
// ====================================================================
//
// Poly1305 for Itanium.
//
// January 2019
//
// Performance was reported to be ~2.1 cycles per byte on Itanium 2.
// With exception for processors in 95xx family, which have higher
// floating-point instructions' latencies and deliver ~2.6 cpb.
// Comparison to compiler-generated code is not exactly fair, because
// of different radixes. But just for reference, it was observed to be
// >3x faster. Originally it was argued that floating-point base 2^32
// implementation would be optimal. Upon closer look estimate for below
// integer base 2^64 implementation turned to be approximately same on
// Itanium 2. But floating-point code would be larger, and have higher
// overhead, which would negatively affect small-block performance...

#if defined(_HPUX_SOURCE)
# if !defined(_LP64)
#  define ADDP  addp4
# else
#  define ADDP  add
# endif
# define RUM    rum
# define SUM    sum
#else
# define ADDP   add
# define RUM    nop
# define SUM    nop
#endif

.text
.explicit

.global	poly1305_init#
.proc	poly1305_init#
.align	64
poly1305_init:
	.prologue
	.save		ar.pfs,r2
{ .mmi;	alloc		r2=ar.pfs,2,0,0,0
	cmp.eq		p6,p7=0,r33		}	// key == NULL?
{ .mmi;	ADDP		r9=8,r32
	ADDP		r10=16,r32
	ADDP		r32=0,r32		};;
	.body
{ .mmi;	st8		[r32]=r0,24			// ctx->h0 = 0
	st8		[r9]=r0				// ctx->h1 = 0
(p7)	ADDP		r8=0,r33		}
{ .mib;	st8		[r10]=r0			// ctx->h2 = 0
(p6)	mov		r8=0
(p6)	br.ret.spnt	b0			};;

{ .mmi;	ADDP		r9=1,r33
	ADDP		r10=2,r33
	ADDP		r11=3,r33		};;
{ .mmi;	ld1		r16=[r8],4			// load key, little-endian
	ld1		r17=[r9],4		}
{ .mmi;	ld1		r18=[r10],4
	ld1		r19=[r11],4		};;
{ .mmi;	ld1		r20=[r8],4
	ld1		r21=[r9],4		}
{ .mmi;	ld1		r22=[r10],4
	ld1		r23=[r11],4
	and		r19=15,r19		};;
{ .mmi;	ld1		r24=[r8],4
	ld1		r25=[r9],4
	and		r20=-4,r20		}
{ .mmi;	ld1		r26=[r10],4
	ld1		r27=[r11],4
	and		r23=15,r23		};;
{ .mmi;	ld1		r28=[r8],4
	ld1		r29=[r9],4
	and		r24=-4,r24		}
{ .mmi;	ld1		r30=[r10],4
	ld1		r31=[r11],4
	and		r27=15,r27		};;

{ .mii;	and		r28=-4,r28
	dep		r16=r17,r16,8,8
	dep		r18=r19,r18,8,8		};;
{ .mii;	and		r31=15,r31
	dep		r16=r18,r16,16,16
	dep		r20=r21,r20,8,8		};;
{ .mii;	dep		r16=r20,r16,32,16
	dep		r22=r23,r22,8,8		};;
{ .mii;	dep		r16=r22,r16,48,16
	dep		r24=r25,r24,8,8		};;
{ .mii;	dep		r26=r27,r26,8,8
	dep		r28=r29,r28,8,8		};;
{ .mii;	dep		r24=r26,r24,16,16
	dep		r30=r31,r30,8,8		};;
{ .mii;	st8		[r32]=r16,8			// ctx->r0
	dep		r24=r28,r24,32,16;;
	dep		r24=r30,r24,48,16	};;
{ .mii;	st8		[r32]=r24,8			// ctx->r1
	shr.u		r25=r24,2;;
	add		r25=r25,r24		};;
{ .mib; st8		[r32]=r25			// ctx->s1
	mov		r8=0
	br.ret.sptk	b0			};;
.endp	poly1305_init#

h0=r17;  h1=r18;  h2=r19;
i0=r20;  i1=r21;
HF0=f8;  HF1=f9;  HF2=f10;
RF0=f11; RF1=f12; SF1=f13;

.global	poly1305_blocks#
.proc	poly1305_blocks#
.align	64
poly1305_blocks:
	.prologue
	.save		ar.pfs,r2
{ .mii;	alloc		r2=ar.pfs,4,1,0,0
	.save		ar.lc,r3
	mov		r3=ar.lc
	.save		pr,r36
	mov		r36=pr			}

	.body
{ .mmi;	ADDP		r8=0,r32
	ADDP		r9=8,r32
	and		r29=7,r33		};;
{ .mmi;	ld8		h0=[r8],16
	ld8		h1=[r9],16
	and		r33=-8,r33		};;
{ .mmi;	ld8		h2=[r8],16
	ldf8		RF0=[r9],16
	shr.u		r34=r34,4		};;
{ .mmi;	ldf8		RF1=[r8],-32
	ldf8		SF1=[r9],-32
	cmp.ltu		p16,p17=1,r34		};;
{ .mmi;
(p16)	add		r34=-2,r34
(p17)	mov		r34=0
	ADDP		r10=0,r33		}
{ .mii;	ADDP		r11=8,r33
(p16)	mov		ar.ec=2
(p17)	mov		ar.ec=1			};;
{ .mib;	RUM		1<<1				// go little-endian
	mov		ar.lc=r34
	brp.loop.imp	.Loop,.Lcend-16		}

{ .mmi;	cmp.eq		p8,p7=0,r29
	cmp.eq		p9,p0=1,r29
	cmp.eq		p10,p0=2,r29		}
{ .mmi;	cmp.eq		p11,p0=3,r29
	cmp.eq		p12,p0=4,r29
	cmp.eq		p13,p0=5,r29		}
{ .mmi;	cmp.eq		p14,p0=6,r29
	cmp.eq		p15,p0=7,r29
	add		r16=16,r10		};;

{ .mmb;
(p8)	ld8		i0=[r10],16			// aligned input
(p8)	ld8		i1=[r11],16
(p8)	br.cond.sptk	.Loop			};;

	// align first block
	.pred.rel	"mutex",p8,p9,p10,p11,p12,p13,p14,p15
{ .mmi;	(p7)	ld8		r14=[r10],24
	(p7)	ld8		r15=[r11],24		}

{ .mii;	(p7)	ld8		r16=[r16]
		nop.i		0;;
	(p15)	shrp		i0=r15,r14,56		}
{ .mii;	(p15)	shrp		i1=r16,r15,56
	(p14)	shrp		i0=r15,r14,48		}
{ .mii;	(p14)	shrp		i1=r16,r15,48
	(p13)	shrp		i0=r15,r14,40		}
{ .mii;	(p13)	shrp		i1=r16,r15,40
	(p12)	shrp		i0=r15,r14,32		}
{ .mii;	(p12)	shrp		i1=r16,r15,32
	(p11)	shrp		i0=r15,r14,24		}
{ .mii;	(p11)	shrp		i1=r16,r15,24
	(p10)	shrp		i0=r15,r14,16		}
{ .mii;	(p10)	shrp		i1=r16,r15,16
	(p9)	shrp		i0=r15,r14,8		}
{ .mii;	(p9)	shrp		i1=r16,r15,8
		mov		r14=r16			};;

.Loop:
		.pred.rel	"mutex",p8,p9,p10,p11,p12,p13,p14,p15
{ .mmi;		add		h0=h0,i0
		add		h1=h1,i1
		add		h2=h2,r35		};;
{ .mmi;		setf.sig	HF0=h0
		cmp.ltu		p6,p0=h0,i0
		cmp.ltu		p7,p0=h1,i1		};;
{ .mmi;	(p6)	add		h1=1,h1;;
		setf.sig	HF1=h1
	(p6)	cmp.eq.or	p7,p0=0,h1		};;
{ .mmi;	(p7)	add		h2=1,h2;;
		setf.sig	HF2=h2			};;

{ .mfi;	(p16)	ld8		r15=[r10],16
		xmpy.lu		f32=HF0,RF0		}
{ .mfi;	(p16)	ld8		r16=[r11],16
		xmpy.hu		f33=HF0,RF0		}
{ .mfi;		xmpy.lu		f36=HF0,RF1		}
{ .mfi;		xmpy.hu		f37=HF0,RF1		};;
{ .mfi;		xmpy.lu		f34=HF1,SF1
	(p15)	shrp		i0=r15,r14,56		}
{ .mfi;		xmpy.hu		f35=HF1,SF1		}
{ .mfi;		xmpy.lu		f38=HF1,RF0
	(p15)	shrp		i1=r16,r15,56		}
{ .mfi;		xmpy.hu		f39=HF1,RF0		}
{ .mfi;		xmpy.lu		f40=HF2,SF1
	(p14)	shrp		i0=r15,r14,48		}
{ .mfi;		xmpy.lu		f41=HF2,RF0		};;

{ .mmi;		getf.sig	r22=f32
		getf.sig	r23=f33
	(p14)	shrp		i1=r16,r15,48		}
{ .mmi;		getf.sig	r24=f34
		getf.sig	r25=f35
	(p13)	shrp		i0=r15,r14,40		}
{ .mmi;		getf.sig	r26=f36
		getf.sig	r27=f37
	(p13)	shrp		i1=r16,r15,40		}
{ .mmi;		getf.sig	r28=f38
		getf.sig	r29=f39
	(p12)	shrp		i0=r15,r14,32		}
{ .mmi;		getf.sig	r30=f40
		getf.sig	r31=f41			};;

{ .mmi;		add		h0=r22,r24
		add		r23=r23,r25
	(p12)	shrp		i1=r16,r15,32		}
{ .mmi;		add		h1=r26,r28
		add		r27=r27,r29
	(p11)	shrp		i0=r15,r14,24		};;
{ .mmi;		cmp.ltu		p6,p0=h0,r24
		cmp.ltu		p7,p0=h1,r28
		add		r23=r23,r30		};;
{ .mmi;	(p6)	add		r23=1,r23
	(p7)	add		r27=1,r27
	(p11)	shrp		i1=r16,r15,24		};;
{ .mmi;		add		h1=h1,r23;;
		cmp.ltu		p6,p7=h1,r23
	(p10)	shrp		i0=r15,r14,16		};;
{ .mmi;	(p6)	add		h2=r31,r27,1
	(p7)	add		h2=r31,r27
	(p10)	shrp		i1=r16,r15,16		};;

{ .mmi;	(p8)	mov		i0=r15
		and		r22=-4,h2
		shr.u		r23=h2,2		};;
{ .mmi;		add		r22=r22,r23
		and		h2=3,h2
	(p9)	shrp		i0=r15,r14,8		};;

{ .mmi;		add		h0=h0,r22;;
		cmp.ltu		p6,p0=h0,r22
	(p9)	shrp		i1=r16,r15,8		};;
{ .mmi;	(p8)	mov		i1=r16
	(p6)	cmp.eq.unc	p7,p0=-1,h1
	(p6)	add		h1=1,h1			};;
{ .mmb;	(p7)	add		h2=1,h2
		mov		r14=r16
		br.ctop.sptk	.Loop			};;
.Lcend:

{ .mii;	SUM		1<<1				// back to big-endian
	mov		ar.lc=r3		};;

{ .mmi;	st8		[r8]=h0,16
	st8		[r9]=h1
	mov		pr=r36,0x1ffff		};;
{ .mmb;	st8		[r8]=h2
	rum		1<<5
	br.ret.sptk	b0			};;
.endp	poly1305_blocks#

.global	poly1305_emit#
.proc	poly1305_emit#
.align	64
poly1305_emit:
	.prologue
	.save		ar.pfs,r2
{ .mmi;	alloc		r2=ar.pfs,3,0,0,0
	ADDP		r8=0,r32
	ADDP		r9=8,r32		};;

	.body
{ .mmi;	ld8		r16=[r8],16			// load hash
	ld8		r17=[r9]
	ADDP		r10=0,r34		};;
{ .mmi;	ld8		r18=[r8]
	ld4		r24=[r10],8			// load nonce
	ADDP		r11=4,r34		};;

{ .mmi;	ld4		r25=[r11],8
	ld4		r26=[r10]
	add		r20=5,r16		};;

{ .mmi;	ld4		r27=[r11]
	cmp.ltu		p6,p7=r20,r16
	shl		r25=r25,32		};;
{ .mmi;
(p6)	add		r21=1,r17
(p7)	add		r21=0,r17
(p6)	cmp.eq.or.andcm	p6,p7=-1,r17		};;
{ .mmi;
(p6)	add		r22=1,r18
(p7)	add		r22=0,r18
	shl		r27=r27,32		};;
{ .mmi;	or		r24=r24,r25
	or		r26=r26,r27
	cmp.leu		p6,p7=4,r22		};;
{ .mmi;
(p6)	add		r16=r20,r24
(p7)	add		r16=r16,r24
(p6)	add		r17=r21,r26		};;
{ .mii;
(p7)	add		r17=r17,r26
	cmp.ltu		p6,p7=r16,r24;;
(p6)	add		r17=1,r17		};;

{ .mmi;	ADDP		r8=0,r33
	ADDP		r9=4,r33
	shr.u		r20=r16,32		}
{ .mmi;	ADDP		r10=8,r33
	ADDP		r11=12,r33
	shr.u		r21=r17,32		};;

{ .mmi;	st1		[r8]=r16,1			// write mac, little-endian
	st1		[r9]=r20,1
	shr.u		r16=r16,8		}
{ .mii;	st1		[r10]=r17,1
	shr.u		r20=r20,8
	shr.u		r17=r17,8		}
{ .mmi;	st1		[r11]=r21,1
	shr.u		r21=r21,8		};;

{ .mmi;	st1		[r8]=r16,1
	st1		[r9]=r20,1
	shr.u		r16=r16,8		}
{ .mii;	st1		[r10]=r17,1
	shr.u		r20=r20,8
	shr.u		r17=r17,8		}
{ .mmi;	st1		[r11]=r21,1
	shr.u		r21=r21,8		};;

{ .mmi;	st1		[r8]=r16,1
	st1		[r9]=r20,1
	shr.u		r16=r16,8		}
{ .mii;	st1		[r10]=r17,1
	shr.u		r20=r20,8
	shr.u		r17=r17,8		}
{ .mmi;	st1		[r11]=r21,1
	shr.u		r21=r21,8		};;

{ .mmi;	st1		[r8]=r16
	st1		[r9]=r20		}
{ .mmb;	st1		[r10]=r17
	st1		[r11]=r21
	br.ret.sptk	b0			};;
.endp	poly1305_emit#

stringz	"Poly1305 for IA64, CRYPTOGAMS by \@dot-asm"