openssl/crypto/poly1305/asm/poly1305-ppc.pl

#! /usr/bin/env perl
# Copyright 2016-2018 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the Apache License 2.0 (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html

#
# ====================================================================
# Written by Andy Polyakov, @dot-asm, initially for use in the OpenSSL
# project. The module is dual licensed under OpenSSL and CRYPTOGAMS
# licenses depending on where you obtain it. For further details see
# https://github.com/dot-asm/cryptogams/.
# ====================================================================
#
# This module implements Poly1305 hash for PowerPC.
#
# June 2015
#
# Numbers are cycles per processed byte with poly1305_blocks alone,
# and improvement coefficients relative to gcc-generated code.
#
#			-m32		-m64
#
# Freescale e300	14.8/+80%	-
# PPC74x0		7.60/+60%	-
# PPC970		7.00/+114%	3.51/+205%
# POWER7		3.75/+260%	1.93/+100%
# POWER8		-		2.03/+200%
# POWER9		-		2.00/+150%
#
# Do we need floating-point implementation for PPC? Results presented
# in poly1305_ieee754.c are tricky to compare to, because they are for
# compiler-generated code. On the other hand it's known that floating-
# point performance can be dominated by FPU latency, which means that
# there is limit even for ideally optimized (and even vectorized) code.
# And this limit is estimated to be higher than above -m64 results. Or
# in other words floating-point implementation can be meaningful to
# consider only in 32-bit application context. We probably have to
# recognize that 32-bit builds are getting less popular on high-end
# systems and therefore tend to target embedded ones, which might not
# even have FPU...
#
# On side note, Power ISA 2.07 enables vector base 2^26 implementation,
# and POWER8 might have capacity to break 1.0 cycle per byte barrier...
#
# January 2019
#
# ... Unfortunately not:-( Estimate was a projection of ARM result,
# but ARM has vector multiply-n-add instruction, while PowerISA does
# not, not one usable in the context. Improvement is ~40% over -m64
# result above and is ~1.43 on little-endian systems.

# $output is the last argument if it looks like a file (it has an extension)
# $flavour is the first argument if it doesn't look like a file
$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
$flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;

if ($flavour =~ /64/) {
	$SIZE_T	=8;
	$LRSAVE	=2*$SIZE_T;
	$UCMP	="cmpld";
	$STU	="stdu";
	$POP	="ld";
	$PUSH	="std";
} elsif ($flavour =~ /32/) {
	$SIZE_T	=4;
	$LRSAVE	=$SIZE_T;
	$UCMP	="cmplw";
	$STU	="stwu";
	$POP	="lwz";
	$PUSH	="stw";
} else { die "nonsense $flavour"; }

# Define endianness based on flavour
# i.e.: linux64le
$LITTLE_ENDIAN = ($flavour=~/le$/) ? $SIZE_T : 0;

$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
die "can't locate ppc-xlate.pl";

open STDOUT,"| $^X $xlate $flavour \"$output\""
    or die "can't call $xlate: $!";

$FRAME=24*$SIZE_T;

$sp="r1";
my ($ctx,$inp,$len,$padbit) = map("r$_",(3..6));
my ($mac,$nonce)=($inp,$len);
my $mask = "r0";

$code=<<___;
.machine	"any"
.text
___
							if ($flavour =~ /64/) {
###############################################################################
# base 2^64 implementation

my ($h0,$h1,$h2,$d0,$d1,$d2, $r0,$r1,$s1, $t0,$t1) = map("r$_",(7..12,27..31));

$code.=<<___;
.globl	.poly1305_init_int
.align	4
.poly1305_init_int:
	xor	r0,r0,r0
	std	r0,0($ctx)		# zero hash value
	std	r0,8($ctx)
	std	r0,16($ctx)
	stw	r0,24($ctx)		# clear is_base2_26

	$UCMP	$inp,r0
	beq-	Lno_key
___
$code.=<<___	if ($LITTLE_ENDIAN);
	ld	$d0,0($inp)		# load key material
	ld	$d1,8($inp)
___
$code.=<<___	if (!$LITTLE_ENDIAN);
	li	$h0,4
	lwbrx	$d0,0,$inp		# load key material
	li	$d1,8
	lwbrx	$h0,$h0,$inp
	li	$h1,12
	lwbrx	$d1,$d1,$inp
	lwbrx	$h1,$h1,$inp
	insrdi	$d0,$h0,32,0
	insrdi	$d1,$h1,32,0
___
$code.=<<___;
	lis	$h1,0xfff		# 0x0fff0000
	ori	$h1,$h1,0xfffc		# 0x0ffffffc
	insrdi	$h1,$h1,32,0		# 0x0ffffffc0ffffffc
	ori	$h0,$h1,3		# 0x0ffffffc0fffffff

	and	$d0,$d0,$h0
	and	$d1,$d1,$h1

	std	$d0,32($ctx)		# store key
	std	$d1,40($ctx)

Lno_key:
	xor	r3,r3,r3
	blr
	.long	0
	.byte	0,12,0x14,0,0,0,2,0
.size	.poly1305_init_int,.-.poly1305_init_int

.globl	.poly1305_blocks
.align	4
.poly1305_blocks:
Lpoly1305_blocks:
	srdi.	$len,$len,4
	beq-	Labort

	$STU	$sp,-$FRAME($sp)
	mflr	r0
	$PUSH	r27,`$FRAME-$SIZE_T*5`($sp)
	$PUSH	r28,`$FRAME-$SIZE_T*4`($sp)
	$PUSH	r29,`$FRAME-$SIZE_T*3`($sp)
	$PUSH	r30,`$FRAME-$SIZE_T*2`($sp)
	$PUSH	r31,`$FRAME-$SIZE_T*1`($sp)
	$PUSH	r0,`$FRAME+$LRSAVE`($sp)

	ld	$r0,32($ctx)		# load key
	ld	$r1,40($ctx)

	ld	$h0,0($ctx)		# load hash value
	ld	$h1,8($ctx)
	ld	$h2,16($ctx)

	srdi	$s1,$r1,2
	mtctr	$len
	add	$s1,$s1,$r1		# s1 = r1 + r1>>2
	li	$mask,3
	b	Loop

.align	4
Loop:
___
$code.=<<___	if ($LITTLE_ENDIAN);
	ld	$t0,0($inp)		# load input
	ld	$t1,8($inp)
___
$code.=<<___	if (!$LITTLE_ENDIAN);
	li	$d0,4
	lwbrx	$t0,0,$inp		# load input
	li	$t1,8
	lwbrx	$d0,$d0,$inp
	li	$d1,12
	lwbrx	$t1,$t1,$inp
	lwbrx	$d1,$d1,$inp
	insrdi	$t0,$d0,32,0
	insrdi	$t1,$d1,32,0
___
$code.=<<___;
	addi	$inp,$inp,16

	addc	$h0,$h0,$t0		# accumulate input
	adde	$h1,$h1,$t1

	mulld	$d0,$h0,$r0		# h0*r0
	mulhdu	$d1,$h0,$r0
	adde	$h2,$h2,$padbit

	mulld	$t0,$h1,$s1		# h1*5*r1
	mulhdu	$t1,$h1,$s1
	addc	$d0,$d0,$t0
	adde	$d1,$d1,$t1

	mulld	$t0,$h0,$r1		# h0*r1
	mulhdu	$d2,$h0,$r1
	addc	$d1,$d1,$t0
	addze	$d2,$d2

	mulld	$t0,$h1,$r0		# h1*r0
	mulhdu	$t1,$h1,$r0
	addc	$d1,$d1,$t0
	adde	$d2,$d2,$t1

	mulld	$t0,$h2,$s1		# h2*5*r1
	mulld	$t1,$h2,$r0		# h2*r0
	addc	$d1,$d1,$t0
	adde	$d2,$d2,$t1

	andc	$t0,$d2,$mask		# final reduction step
	and	$h2,$d2,$mask
	srdi	$t1,$t0,2
	add	$t0,$t0,$t1
	addc	$h0,$d0,$t0
	addze	$h1,$d1
	addze	$h2,$h2

	bdnz	Loop

	std	$h0,0($ctx)		# store hash value
	std	$h1,8($ctx)
	std	$h2,16($ctx)

	$POP	r27,`$FRAME-$SIZE_T*5`($sp)
	$POP	r28,`$FRAME-$SIZE_T*4`($sp)
	$POP	r29,`$FRAME-$SIZE_T*3`($sp)
	$POP	r30,`$FRAME-$SIZE_T*2`($sp)
	$POP	r31,`$FRAME-$SIZE_T*1`($sp)
	addi	$sp,$sp,$FRAME
Labort:
	blr
	.long	0
	.byte	0,12,4,1,0x80,5,4,0
.size	.poly1305_blocks,.-.poly1305_blocks
___
{
my ($h0,$h1,$h2,$h3,$h4,$t0) = map("r$_",(7..12));

$code.=<<___;
.globl	.poly1305_emit
.align	5
.poly1305_emit:
	lwz	$h0,0($ctx)	# load hash value base 2^26
	lwz	$h1,4($ctx)
	lwz	$h2,8($ctx)
	lwz	$h3,12($ctx)
	lwz	$h4,16($ctx)
	lwz	r0,24($ctx)	# is_base2_26

	sldi	$h1,$h1,26	# base 2^26 -> base 2^64
	sldi	$t0,$h2,52
	srdi	$h2,$h2,12
	sldi	$h3,$h3,14
	add	$h0,$h0,$h1
	addc	$h0,$h0,$t0
	sldi	$t0,$h4,40
	srdi	$h4,$h4,24
	adde	$h1,$h2,$h3
	addc	$h1,$h1,$t0
	addze	$h2,$h4

	ld	$h3,0($ctx)	# load hash value base 2^64
	ld	$h4,8($ctx)
	ld	$t0,16($ctx)

	neg	r0,r0
	xor	$h0,$h0,$h3	# choose between radixes
	xor	$h1,$h1,$h4
	xor	$h2,$h2,$t0
	and	$h0,$h0,r0
	and	$h1,$h1,r0
	and	$h2,$h2,r0
	xor	$h0,$h0,$h3
	xor	$h1,$h1,$h4
	xor	$h2,$h2,$t0

	addic	$h3,$h0,5	# compare to modulus
	addze	$h4,$h1
	addze	$t0,$h2

	srdi	$t0,$t0,2	# see if it carried/borrowed
	neg	$t0,$t0

	andc	$h0,$h0,$t0
	and	$h3,$h3,$t0
	andc	$h1,$h1,$t0
	and	$h4,$h4,$t0
	or	$h0,$h0,$h3
	or	$h1,$h1,$h4

	lwz	$t0,4($nonce)
	lwz	$h2,12($nonce)
	lwz	$h3,0($nonce)
	lwz	$h4,8($nonce)

	insrdi	$h3,$t0,32,0
	insrdi	$h4,$h2,32,0

	addc	$h0,$h0,$h3	# accumulate nonce
	adde	$h1,$h1,$h4

	addi	$ctx,$mac,-1
	addi	$mac,$mac,7

	stbu	$h0,1($ctx)	# write [little-endian] result
	srdi	$h0,$h0,8
	stbu	$h1,1($mac)
	srdi	$h1,$h1,8

	stbu	$h0,1($ctx)
	srdi	$h0,$h0,8
	stbu	$h1,1($mac)
	srdi	$h1,$h1,8

	stbu	$h0,1($ctx)
	srdi	$h0,$h0,8
	stbu	$h1,1($mac)
	srdi	$h1,$h1,8

	stbu	$h0,1($ctx)
	srdi	$h0,$h0,8
	stbu	$h1,1($mac)
	srdi	$h1,$h1,8

	stbu	$h0,1($ctx)
	srdi	$h0,$h0,8
	stbu	$h1,1($mac)
	srdi	$h1,$h1,8

	stbu	$h0,1($ctx)
	srdi	$h0,$h0,8
	stbu	$h1,1($mac)
	srdi	$h1,$h1,8

	stbu	$h0,1($ctx)
	srdi	$h0,$h0,8
	stbu	$h1,1($mac)
	srdi	$h1,$h1,8

	stbu	$h0,1($ctx)
	stbu	$h1,1($mac)

	blr
	.long	0
	.byte	0,12,0x14,0,0,0,3,0
.size	.poly1305_emit,.-.poly1305_emit
___
}							} else {
###############################################################################
# base 2^32 implementation

my ($h0,$h1,$h2,$h3,$h4, $r0,$r1,$r2,$r3, $s1,$s2,$s3,
    $t0,$t1,$t2,$t3, $D0,$D1,$D2,$D3, $d0,$d1,$d2,$d3
   ) = map("r$_",(7..12,14..31));

$code.=<<___;
.globl	.poly1305_init_int
.align	4
.poly1305_init_int:
	xor	r0,r0,r0
	stw	r0,0($ctx)		# zero hash value
	stw	r0,4($ctx)
	stw	r0,8($ctx)
	stw	r0,12($ctx)
	stw	r0,16($ctx)
	stw	r0,24($ctx)		# clear is_base2_26

	$UCMP	$inp,r0
	beq-	Lno_key
___
$code.=<<___	if ($LITTLE_ENDIAN);
	lw	$h0,0($inp)		# load key material
	lw	$h1,4($inp)
	lw	$h2,8($inp)
	lw	$h3,12($inp)
___
$code.=<<___	if (!$LITTLE_ENDIAN);
	li	$h1,4
	lwbrx	$h0,0,$inp		# load key material
	li	$h2,8
	lwbrx	$h1,$h1,$inp
	li	$h3,12
	lwbrx	$h2,$h2,$inp
	lwbrx	$h3,$h3,$inp
___
$code.=<<___;
	lis	$mask,0xf000		# 0xf0000000
	li	$r0,-4
	andc	$r0,$r0,$mask		# 0x0ffffffc

	andc	$h0,$h0,$mask
	and	$h1,$h1,$r0
	and	$h2,$h2,$r0
	and	$h3,$h3,$r0

	stw	$h0,32($ctx)		# store key
	stw	$h1,36($ctx)
	stw	$h2,40($ctx)
	stw	$h3,44($ctx)

Lno_key:
	xor	r3,r3,r3
	blr
	.long	0
	.byte	0,12,0x14,0,0,0,2,0
.size	.poly1305_init_int,.-.poly1305_init_int

.globl	.poly1305_blocks
.align	4
.poly1305_blocks:
Lpoly1305_blocks:
	srwi.	$len,$len,4
	beq-	Labort

	$STU	$sp,-$FRAME($sp)
	mflr	r0
	$PUSH	r14,`$FRAME-$SIZE_T*18`($sp)
	$PUSH	r15,`$FRAME-$SIZE_T*17`($sp)
	$PUSH	r16,`$FRAME-$SIZE_T*16`($sp)
	$PUSH	r17,`$FRAME-$SIZE_T*15`($sp)
	$PUSH	r18,`$FRAME-$SIZE_T*14`($sp)
	$PUSH	r19,`$FRAME-$SIZE_T*13`($sp)
	$PUSH	r20,`$FRAME-$SIZE_T*12`($sp)
	$PUSH	r21,`$FRAME-$SIZE_T*11`($sp)
	$PUSH	r22,`$FRAME-$SIZE_T*10`($sp)
	$PUSH	r23,`$FRAME-$SIZE_T*9`($sp)
	$PUSH	r24,`$FRAME-$SIZE_T*8`($sp)
	$PUSH	r25,`$FRAME-$SIZE_T*7`($sp)
	$PUSH	r26,`$FRAME-$SIZE_T*6`($sp)
	$PUSH	r27,`$FRAME-$SIZE_T*5`($sp)
	$PUSH	r28,`$FRAME-$SIZE_T*4`($sp)
	$PUSH	r29,`$FRAME-$SIZE_T*3`($sp)
	$PUSH	r30,`$FRAME-$SIZE_T*2`($sp)
	$PUSH	r31,`$FRAME-$SIZE_T*1`($sp)
	$PUSH	r0,`$FRAME+$LRSAVE`($sp)

	lwz	$r0,32($ctx)		# load key
	lwz	$r1,36($ctx)
	lwz	$r2,40($ctx)
	lwz	$r3,44($ctx)

	lwz	$h0,0($ctx)		# load hash value
	lwz	$h1,4($ctx)
	lwz	$h2,8($ctx)
	lwz	$h3,12($ctx)
	lwz	$h4,16($ctx)

	srwi	$s1,$r1,2
	srwi	$s2,$r2,2
	srwi	$s3,$r3,2
	add	$s1,$s1,$r1		# si = ri + ri>>2
	add	$s2,$s2,$r2
	add	$s3,$s3,$r3
	mtctr	$len
	li	$mask,3
	b	Loop

.align	4
Loop:
___
$code.=<<___	if ($LITTLE_ENDIAN);
	lwz	$d0,0($inp)		# load input
	lwz	$d1,4($inp)
	lwz	$d2,8($inp)
	lwz	$d3,12($inp)
___
$code.=<<___	if (!$LITTLE_ENDIAN);
	li	$d1,4
	lwbrx	$d0,0,$inp		# load input
	li	$d2,8
	lwbrx	$d1,$d1,$inp
	li	$d3,12
	lwbrx	$d2,$d2,$inp
	lwbrx	$d3,$d3,$inp
___
$code.=<<___;
	addi	$inp,$inp,16

	addc	$h0,$h0,$d0		# accumulate input
	adde	$h1,$h1,$d1
	adde	$h2,$h2,$d2

	mullw	$d0,$h0,$r0		# h0*r0
	mulhwu	$D0,$h0,$r0

	mullw	$d1,$h0,$r1		# h0*r1
	mulhwu	$D1,$h0,$r1

	mullw	$d2,$h0,$r2		# h0*r2
	mulhwu	$D2,$h0,$r2

	 adde	$h3,$h3,$d3
	 adde	$h4,$h4,$padbit

	mullw	$d3,$h0,$r3		# h0*r3
	mulhwu	$D3,$h0,$r3

	mullw	$t0,$h1,$s3		# h1*s3
	mulhwu	$t1,$h1,$s3

	mullw	$t2,$h1,$r0		# h1*r0
	mulhwu	$t3,$h1,$r0
	 addc	$d0,$d0,$t0
	 adde	$D0,$D0,$t1

	mullw	$t0,$h1,$r1		# h1*r1
	mulhwu	$t1,$h1,$r1
	 addc	$d1,$d1,$t2
	 adde	$D1,$D1,$t3

	mullw	$t2,$h1,$r2		# h1*r2
	mulhwu	$t3,$h1,$r2
	 addc	$d2,$d2,$t0
	 adde	$D2,$D2,$t1

	mullw	$t0,$h2,$s2		# h2*s2
	mulhwu	$t1,$h2,$s2
	 addc	$d3,$d3,$t2
	 adde	$D3,$D3,$t3

	mullw	$t2,$h2,$s3		# h2*s3
	mulhwu	$t3,$h2,$s3
	 addc	$d0,$d0,$t0
	 adde	$D0,$D0,$t1

	mullw	$t0,$h2,$r0		# h2*r0
	mulhwu	$t1,$h2,$r0
	 addc	$d1,$d1,$t2
	 adde	$D1,$D1,$t3

	mullw	$t2,$h2,$r1		# h2*r1
	mulhwu	$t3,$h2,$r1
	 addc	$d2,$d2,$t0
	 adde	$D2,$D2,$t1

	mullw	$t0,$h3,$s1		# h3*s1
	mulhwu	$t1,$h3,$s1
	 addc	$d3,$d3,$t2
	 adde	$D3,$D3,$t3

	mullw	$t2,$h3,$s2		# h3*s2
	mulhwu	$t3,$h3,$s2
	 addc	$d0,$d0,$t0
	 adde	$D0,$D0,$t1

	mullw	$t0,$h3,$s3		# h3*s3
	mulhwu	$t1,$h3,$s3
	 addc	$d1,$d1,$t2
	 adde	$D1,$D1,$t3

	mullw	$t2,$h3,$r0		# h3*r0
	mulhwu	$t3,$h3,$r0
	 addc	$d2,$d2,$t0
	 adde	$D2,$D2,$t1

	mullw	$t0,$h4,$s1		# h4*s1
	 addc	$d3,$d3,$t2
	 adde	$D3,$D3,$t3
	addc	$d1,$d1,$t0

	mullw	$t1,$h4,$s2		# h4*s2
	 addze	$D1,$D1
	addc	$d2,$d2,$t1
	addze	$D2,$D2

	mullw	$t2,$h4,$s3		# h4*s3
	addc	$d3,$d3,$t2
	addze	$D3,$D3

	mullw	$h4,$h4,$r0		# h4*r0

	addc	$h1,$d1,$D0
	adde	$h2,$d2,$D1
	adde	$h3,$d3,$D2
	adde	$h4,$h4,$D3

	andc	$D0,$h4,$mask		# final reduction step
	and	$h4,$h4,$mask
	srwi	$D1,$D0,2
	add	$D0,$D0,$D1
	addc	$h0,$d0,$D0
	addze	$h1,$h1
	addze	$h2,$h2
	addze	$h3,$h3
	addze	$h4,$h4

	bdnz	Loop

	stw	$h0,0($ctx)		# store hash value
	stw	$h1,4($ctx)
	stw	$h2,8($ctx)
	stw	$h3,12($ctx)
	stw	$h4,16($ctx)

	$POP	r14,`$FRAME-$SIZE_T*18`($sp)
	$POP	r15,`$FRAME-$SIZE_T*17`($sp)
	$POP	r16,`$FRAME-$SIZE_T*16`($sp)
	$POP	r17,`$FRAME-$SIZE_T*15`($sp)
	$POP	r18,`$FRAME-$SIZE_T*14`($sp)
	$POP	r19,`$FRAME-$SIZE_T*13`($sp)
	$POP	r20,`$FRAME-$SIZE_T*12`($sp)
	$POP	r21,`$FRAME-$SIZE_T*11`($sp)
	$POP	r22,`$FRAME-$SIZE_T*10`($sp)
	$POP	r23,`$FRAME-$SIZE_T*9`($sp)
	$POP	r24,`$FRAME-$SIZE_T*8`($sp)
	$POP	r25,`$FRAME-$SIZE_T*7`($sp)
	$POP	r26,`$FRAME-$SIZE_T*6`($sp)
	$POP	r27,`$FRAME-$SIZE_T*5`($sp)
	$POP	r28,`$FRAME-$SIZE_T*4`($sp)
	$POP	r29,`$FRAME-$SIZE_T*3`($sp)
	$POP	r30,`$FRAME-$SIZE_T*2`($sp)
	$POP	r31,`$FRAME-$SIZE_T*1`($sp)
	addi	$sp,$sp,$FRAME
Labort:
	blr
	.long	0
	.byte	0,12,4,1,0x80,18,4,0
.size	.poly1305_blocks,.-.poly1305_blocks
___
{
my ($h0,$h1,$h2,$h3,$h4,$t0,$t1) = map("r$_",(6..12));

$code.=<<___;
.globl	.poly1305_emit
.align	5
.poly1305_emit:
	lwz	r0,24($ctx)	# is_base2_26
	lwz	$h0,0($ctx)	# load hash value
	lwz	$h1,4($ctx)
	lwz	$h2,8($ctx)
	lwz	$h3,12($ctx)
	lwz	$h4,16($ctx)
	cmplwi	r0,0
	beq	Lemit_base2_32

	slwi	$t0,$h1,26	# base 2^26 -> base 2^32
	srwi	$h1,$h1,6
	slwi	$t1,$h2,20
	srwi	$h2,$h2,12
	addc	$h0,$h0,$t0
	slwi	$t0,$h3,14
	srwi	$h3,$h3,18
	adde	$h1,$h1,$t1
	slwi	$t1,$h4,8
	srwi	$h4,$h4,24
	adde	$h2,$h2,$t0
	adde	$h3,$h3,$t1
	addze	$h4,$h4

Lemit_base2_32:
	addic	r0,$h0,5	# compare to modulus
	addze	r0,$h1
	addze	r0,$h2
	addze	r0,$h3
	addze	r0,$h4

	srwi	r0,r0,2		# see if it carried/borrowed
	neg	r0,r0
	andi.	r0,r0,5

	addc	$h0,$h0,r0
	lwz	r0,0($nonce)
	addze	$h1,$h1
	lwz	$t0,4($nonce)
	addze	$h2,$h2
	lwz	$t1,8($nonce)
	addze	$h3,$h3
	lwz	$h4,12($nonce)

	addc	$h0,$h0,r0	# accumulate nonce
	adde	$h1,$h1,$t0
	adde	$h2,$h2,$t1
	adde	$h3,$h3,$h4

	addi	$ctx,$mac,-1
	addi	$mac,$mac,7

	stbu	$h0,1($ctx)	# write [little-endian] result
	srwi	$h0,$h0,8
	stbu	$h2,1($mac)
	srwi	$h2,$h2,8

	stbu	$h0,1($ctx)
	srwi	$h0,$h0,8
	stbu	$h2,1($mac)
	srwi	$h2,$h2,8

	stbu	$h0,1($ctx)
	srwi	$h0,$h0,8
	stbu	$h2,1($mac)
	srwi	$h2,$h2,8

	stbu	$h0,1($ctx)
	stbu	$h2,1($mac)

	stbu	$h1,1($ctx)
	srwi	$h1,$h1,8
	stbu	$h3,1($mac)
	srwi	$h3,$h3,8

	stbu	$h1,1($ctx)
	srwi	$h1,$h1,8
	stbu	$h3,1($mac)
	srwi	$h3,$h3,8

	stbu	$h1,1($ctx)
	srwi	$h1,$h1,8
	stbu	$h3,1($mac)
	srwi	$h3,$h3,8

	stbu	$h1,1($ctx)
	stbu	$h3,1($mac)

	blr
	.long	0
	.byte	0,12,0x14,0,0,0,3,0
.size	.poly1305_emit,.-.poly1305_emit
___
}							}
{{{
########################################################################
# PowerISA 2.07/VSX section                                            #
########################################################################

my $LOCALS= 6*$SIZE_T;
my $VSXFRAME = $LOCALS + 6*$SIZE_T;
   $VSXFRAME += 128;	# local variables
   $VSXFRAME += 13*16;	# v20-v31 offload

my $BIG_ENDIAN = ($flavour !~ /le/) ? 4 : 0;

########################################################################
# Layout of opaque area is following:
#
#	unsigned __int32 h[5];		# current hash value base 2^26
#	unsigned __int32 pad;
#	unsigned __int32 is_base2_26, pad;
#	unsigned __int64 r[2];		# key value base 2^64
#	struct { unsigned __int32 r^2, r^4, r^1, r^3; } r[9];
#
# where r^n are base 2^26 digits of powers of multiplier key. There are
# 5 digits, but last four are interleaved with multiples of 5, totalling
# in 9 elements: r0, r1, 5*r1, r2, 5*r2, r3, 5*r3, r4, 5*r4. Order of
# powers is as they appear in register, not memory.

my ($H0, $H1, $H2, $H3, $H4) = map("v$_",(0..4));
my ($I0, $I1, $I2, $I3, $I4) = map("v$_",(5..9));
my ($R0, $R1, $S1, $R2, $S2) = map("v$_",(10..14));
my      ($R3, $S3, $R4, $S4) = ($R1, $S1, $R2, $S2);
my ($ACC0, $ACC1, $ACC2, $ACC3, $ACC4) = map("v$_",(15..19));
my ($T0, $T1, $T2, $T3, $T4) = map("v$_",(20..24));
my ($_26,$_4,$_40,$_14,$mask26,$padbits,$I2perm) = map("v$_",(25..31));
my ($x00,$x60,$x70,$x10,$x20,$x30,$x40,$x50) = (0, map("r$_",(7,8,27..31)));
my ($ctx_,$_ctx,$const) = map("r$_",(10..12));

							if ($flavour =~ /64/) {
###############################################################################
# setup phase of poly1305_blocks_vsx is different on 32- and 64-bit platforms,
# but the base 2^26 computational part is same...

my ($h0,$h1,$h2,$d0,$d1,$d2, $r0,$r1,$s1, $t0,$t1) = map("r$_",(6..11,27..31));
my $mask = "r0";

$code.=<<___;
.globl	.poly1305_blocks_vsx
.align	5
.poly1305_blocks_vsx:
	lwz	r7,24($ctx)		# is_base2_26
	cmpldi	$len,128
	bge	__poly1305_blocks_vsx

	neg	r0,r7			# is_base2_26 as mask
	lwz	r7,0($ctx)		# load hash base 2^26
	lwz	r8,4($ctx)
	lwz	r9,8($ctx)
	lwz	r10,12($ctx)
	lwz	r11,16($ctx)

	sldi	r8,r8,26		# base 2^26 -> base 2^64
	sldi	r12,r9,52
	add	r7,r7,r8
	srdi	r9,r9,12
	sldi	r10,r10,14
	addc	r7,r7,r12
	sldi	r8,r11,40
	adde	r9,r9,r10
	srdi	r11,r11,24
	addc	r9,r9,r8
	addze	r11,r11

	ld	r8,0($ctx)		# load hash base 2^64
	ld	r10,8($ctx)
	ld	r12,16($ctx)

	xor	r7,r7,r8		# select between radixes
	xor	r9,r9,r10
	xor	r11,r11,r12
	and	r7,r7,r0
	and	r9,r9,r0
	and	r11,r11,r0
	xor	r7,r7,r8
	xor	r9,r9,r10
	xor	r11,r11,r12

	li	r0,0
	std	r7,0($ctx)		# store hash base 2^64
	std	r9,8($ctx)
	std	r11,16($ctx)
	stw	r0,24($ctx)		# clear is_base2_26

	b	Lpoly1305_blocks
	.long	0
	.byte	0,12,0x14,0,0,0,4,0
.size	.poly1305_blocks_vsx,.-.poly1305_blocks_vsx

.align	5
__poly1305_mul:
	mulld	$d0,$h0,$r0		# h0*r0
	mulhdu	$d1,$h0,$r0

	mulld	$t0,$h1,$s1		# h1*5*r1
	mulhdu	$t1,$h1,$s1
	addc	$d0,$d0,$t0
	adde	$d1,$d1,$t1

	mulld	$t0,$h0,$r1		# h0*r1
	mulhdu	$d2,$h0,$r1
	addc	$d1,$d1,$t0
	addze	$d2,$d2

	mulld	$t0,$h1,$r0		# h1*r0
	mulhdu	$t1,$h1,$r0
	addc	$d1,$d1,$t0
	adde	$d2,$d2,$t1

	mulld	$t0,$h2,$s1		# h2*5*r1
	mulld	$t1,$h2,$r0		# h2*r0
	addc	$d1,$d1,$t0
	adde	$d2,$d2,$t1

	andc	$t0,$d2,$mask		# final reduction step
	and	$h2,$d2,$mask
	srdi	$t1,$t0,2
	add	$t0,$t0,$t1
	addc	$h0,$d0,$t0
	addze	$h1,$d1
	addze	$h2,$h2

	blr
	.long	0
	.byte	0,12,0x14,0,0,0,0,0
.size	__poly1305_mul,.-__poly1305_mul

.align	5
__poly1305_splat:
	extrdi	$d0,$h0,26,38
	extrdi	$d1,$h0,26,12
	stw	$d0,0x00($t1)

	extrdi	$d2,$h0,12,0
	slwi	$d0,$d1,2
	stw	$d1,0x10($t1)
	add	$d0,$d0,$d1		# * 5
	stw	$d0,0x20($t1)

	insrdi	$d2,$h1,14,38
	slwi	$d0,$d2,2
	stw	$d2,0x30($t1)
	add	$d0,$d0,$d2		# * 5
	stw	$d0,0x40($t1)

	extrdi	$d1,$h1,26,24
	extrdi	$d2,$h1,24,0
	slwi	$d0,$d1,2
	stw	$d1,0x50($t1)
	add	$d0,$d0,$d1		# * 5
	stw	$d0,0x60($t1)

	insrdi	$d2,$h2,3,37
	slwi	$d0,$d2,2
	stw	$d2,0x70($t1)
	add	$d0,$d0,$d2		# * 5
	stw	$d0,0x80($t1)

	blr
	.long	0
	.byte	0,12,0x14,0,0,0,0,0
.size	__poly1305_splat,.-__poly1305_splat

.align	5
__poly1305_blocks_vsx:
	$STU	$sp,-$VSXFRAME($sp)
	mflr	r0
	li	r10,`15+$LOCALS+128`
	li	r11,`31+$LOCALS+128`
	mfspr	r12,256
	stvx	v20,r10,$sp
	addi	r10,r10,32
	stvx	v21,r11,$sp
	addi	r11,r11,32
	stvx	v22,r10,$sp
	addi	r10,r10,32
	stvx	v23,r10,$sp
	addi	r10,r10,32
	stvx	v24,r11,$sp
	addi	r11,r11,32
	stvx	v25,r10,$sp
	addi	r10,r10,32
	stvx	v26,r10,$sp
	addi	r10,r10,32
	stvx	v27,r11,$sp
	addi	r11,r11,32
	stvx	v28,r10,$sp
	addi	r10,r10,32
	stvx	v29,r11,$sp
	addi	r11,r11,32
	stvx	v30,r10,$sp
	stvx	v31,r11,$sp
	stw	r12,`$VSXFRAME-$SIZE_T*5-4`($sp)# save vrsave
	li	r12,-1
	mtspr	256,r12			# preserve all AltiVec registers
	$PUSH	r27,`$VSXFRAME-$SIZE_T*5`($sp)
	$PUSH	r28,`$VSXFRAME-$SIZE_T*4`($sp)
	$PUSH	r29,`$VSXFRAME-$SIZE_T*3`($sp)
	$PUSH	r30,`$VSXFRAME-$SIZE_T*2`($sp)
	$PUSH	r31,`$VSXFRAME-$SIZE_T*1`($sp)
	$PUSH	r0,`$VSXFRAME+$LRSAVE`($sp)

	bl	LPICmeup

	li	$x10,0x10
	li	$x20,0x20
	li	$x30,0x30
	li	$x40,0x40
	li	$x50,0x50
	lvx_u	$mask26,$x00,$const
	lvx_u	$_26,$x10,$const
	lvx_u	$_40,$x20,$const
	lvx_u	$I2perm,$x30,$const
	lvx_u	$padbits,$x40,$const

	cmplwi	r7,0			# is_base2_26?
	bne	Lskip_init_vsx

	ld	$r0,32($ctx)		# load key base 2^64
	ld	$r1,40($ctx)
	srdi	$s1,$r1,2
	li	$mask,3
	add	$s1,$s1,$r1		# s1 = r1 + r1>>2

	mr	$h0,$r0			# "calculate" r^1
	mr	$h1,$r1
	li	$h2,0
	addi	$t1,$ctx,`48+(12^$BIG_ENDIAN)`
	bl	__poly1305_splat

	bl	__poly1305_mul		# calculate r^2
	addi	$t1,$ctx,`48+(4^$BIG_ENDIAN)`
	bl	__poly1305_splat

	bl	__poly1305_mul		# calculate r^3
	addi	$t1,$ctx,`48+(8^$BIG_ENDIAN)`
	bl	__poly1305_splat

	bl	__poly1305_mul		# calculate r^4
	addi	$t1,$ctx,`48+(0^$BIG_ENDIAN)`
	bl	__poly1305_splat

	ld	$h0,0($ctx)		# load hash
	ld	$h1,8($ctx)
	ld	$h2,16($ctx)

	extrdi	$d0,$h0,26,38		# base 2^64 -> base 2^26
	extrdi	$d1,$h0,26,12
	extrdi	$d2,$h0,12,0
	mtvrwz	$H0,$d0
	insrdi	$d2,$h1,14,38
	mtvrwz	$H1,$d1
	extrdi	$d1,$h1,26,24
	mtvrwz	$H2,$d2
	extrdi	$d2,$h1,24,0
	mtvrwz	$H3,$d1
	insrdi	$d2,$h2,3,37
	mtvrwz	$H4,$d2
___
							} else {
###############################################################################
# 32-bit initialization

my ($h0,$h1,$h2,$h3,$h4,$t0,$t1) = map("r$_",(7..11,0,12));
my ($R3,$S3,$R4,$S4)=($I1,$I2,$I3,$I4);

$code.=<<___;
.globl	.poly1305_blocks_vsx
.align	5
.poly1305_blocks_vsx:
	lwz	r7,24($ctx)		# is_base2_26
	cmplwi	$len,128
	bge	__poly1305_blocks_vsx
	cmplwi	r7,0
	beq	Lpoly1305_blocks

	lwz	$h0,0($ctx)		# load hash
	lwz	$h1,4($ctx)
	lwz	$h2,8($ctx)
	lwz	$h3,12($ctx)
	lwz	$h4,16($ctx)

	slwi	$t0,$h1,26		# base 2^26 -> base 2^32
	srwi	$h1,$h1,6
	slwi	$t1,$h2,20
	srwi	$h2,$h2,12
	addc	$h0,$h0,$t0
	slwi	$t0,$h3,14
	srwi	$h3,$h3,18
	adde	$h1,$h1,$t1
	slwi	$t1,$h4,8
	srwi	$h4,$h4,24
	adde	$h2,$h2,$t0
	li	$t0,0
	adde	$h3,$h3,$t1
	addze	$h4,$h4

	stw	$h0,0($ctx)		# store hash base 2^32
	stw	$h1,4($ctx)
	stw	$h2,8($ctx)
	stw	$h3,12($ctx)
	stw	$h4,16($ctx)
	stw	$t0,24($ctx)		# clear is_base2_26

	b	Lpoly1305_blocks
	.long	0
	.byte	0,12,0x14,0,0,0,4,0
.size	.poly1305_blocks_vsx,.-.poly1305_blocks_vsx

.align	5
__poly1305_mul:
	vmulouw		$ACC0,$H0,$R0
	vmulouw		$ACC1,$H1,$R0
	vmulouw		$ACC2,$H2,$R0
	vmulouw		$ACC3,$H3,$R0
	vmulouw		$ACC4,$H4,$R0

	vmulouw		$T0,$H4,$S1
	vaddudm		$ACC0,$ACC0,$T0
	vmulouw		$T0,$H0,$R1
	vaddudm		$ACC1,$ACC1,$T0
	vmulouw		$T0,$H1,$R1
	vaddudm		$ACC2,$ACC2,$T0
	vmulouw		$T0,$H2,$R1
	vaddudm		$ACC3,$ACC3,$T0
	vmulouw		$T0,$H3,$R1
	vaddudm		$ACC4,$ACC4,$T0

	vmulouw		$T0,$H3,$S2
	vaddudm		$ACC0,$ACC0,$T0
	vmulouw		$T0,$H4,$S2
	vaddudm		$ACC1,$ACC1,$T0
	vmulouw		$T0,$H0,$R2
	vaddudm		$ACC2,$ACC2,$T0
	vmulouw		$T0,$H1,$R2
	vaddudm		$ACC3,$ACC3,$T0
	vmulouw		$T0,$H2,$R2
	vaddudm		$ACC4,$ACC4,$T0

	vmulouw		$T0,$H2,$S3
	vaddudm		$ACC0,$ACC0,$T0
	vmulouw		$T0,$H3,$S3
	vaddudm		$ACC1,$ACC1,$T0
	vmulouw		$T0,$H4,$S3
	vaddudm		$ACC2,$ACC2,$T0
	vmulouw		$T0,$H0,$R3
	vaddudm		$ACC3,$ACC3,$T0
	vmulouw		$T0,$H1,$R3
	vaddudm		$ACC4,$ACC4,$T0

	vmulouw		$T0,$H1,$S4
	vaddudm		$ACC0,$ACC0,$T0
	vmulouw		$T0,$H2,$S4
	vaddudm		$ACC1,$ACC1,$T0
	vmulouw		$T0,$H3,$S4
	vaddudm		$ACC2,$ACC2,$T0
	vmulouw		$T0,$H4,$S4
	vaddudm		$ACC3,$ACC3,$T0
	vmulouw		$T0,$H0,$R4
	vaddudm		$ACC4,$ACC4,$T0

	################################################################
	# lazy reduction

	vspltisb	$T0,2
	vsrd		$H4,$ACC3,$_26
	vsrd		$H1,$ACC0,$_26
	vand		$H3,$ACC3,$mask26
	vand		$H0,$ACC0,$mask26
	vaddudm		$H4,$H4,$ACC4		# h3 -> h4
	vaddudm		$H1,$H1,$ACC1		# h0 -> h1

	vsrd		$ACC4,$H4,$_26
	vsrd		$ACC1,$H1,$_26
	vand		$H4,$H4,$mask26
	vand		$H1,$H1,$mask26
	vaddudm		$H0,$H0,$ACC4
	vaddudm		$H2,$ACC2,$ACC1		# h1 -> h2

	vsld		$ACC4,$ACC4,$T0		# <<2
	vsrd		$ACC2,$H2,$_26
	vand		$H2,$H2,$mask26
	vaddudm		$H0,$H0,$ACC4		# h4 -> h0
	vaddudm		$H3,$H3,$ACC2		# h2 -> h3

	vsrd		$ACC0,$H0,$_26
	vsrd		$ACC3,$H3,$_26
	vand		$H0,$H0,$mask26
	vand		$H3,$H3,$mask26
	vaddudm		$H1,$H1,$ACC0		# h0 -> h1
	vaddudm		$H4,$H4,$ACC3		# h3 -> h4

	blr
	.long	0
	.byte	0,12,0x14,0,0,0,0,0
.size	__poly1305_mul,.-__poly1305_mul

.align	5
__poly1305_blocks_vsx:
	$STU	$sp,-$VSXFRAME($sp)
	mflr	r0
	li	r10,`15+$LOCALS+128`
	li	r11,`31+$LOCALS+128`
	mfspr	r12,256
	stvx	v20,r10,$sp
	addi	r10,r10,32
	stvx	v21,r11,$sp
	addi	r11,r11,32
	stvx	v22,r10,$sp
	addi	r10,r10,32
	stvx	v23,r10,$sp
	addi	r10,r10,32
	stvx	v24,r11,$sp
	addi	r11,r11,32
	stvx	v25,r10,$sp
	addi	r10,r10,32
	stvx	v26,r10,$sp
	addi	r10,r10,32
	stvx	v27,r11,$sp
	addi	r11,r11,32
	stvx	v28,r10,$sp
	addi	r10,r10,32
	stvx	v29,r11,$sp
	addi	r11,r11,32
	stvx	v30,r10,$sp
	stvx	v31,r11,$sp
	stw	r12,`$VSXFRAME-$SIZE_T*5-4`($sp)# save vrsave
	li	r12,-1
	mtspr	256,r12			# preserve all AltiVec registers
	$PUSH	r27,`$VSXFRAME-$SIZE_T*5`($sp)
	$PUSH	r28,`$VSXFRAME-$SIZE_T*4`($sp)
	$PUSH	r29,`$VSXFRAME-$SIZE_T*3`($sp)
	$PUSH	r30,`$VSXFRAME-$SIZE_T*2`($sp)
	$PUSH	r31,`$VSXFRAME-$SIZE_T*1`($sp)
	$PUSH	r0,`$VSXFRAME+$LRSAVE`($sp)

	bl	LPICmeup

	li	$x10,0x10
	li	$x20,0x20
	li	$x30,0x30
	li	$x40,0x40
	li	$x50,0x50
	lvx_u	$mask26,$x00,$const
	lvx_u	$_26,$x10,$const
	lvx_u	$_40,$x20,$const
	lvx_u	$I2perm,$x30,$const
	lvx_u	$padbits,$x40,$const

	cmplwi	r7,0			# is_base2_26?
	bne	Lskip_init_vsx

	lwz	$h1,32($ctx)		# load key base 2^32
	lwz	$h2,36($ctx)
	lwz	$h3,40($ctx)
	lwz	$h4,44($ctx)

	extrwi	$h0,$h1,26,6		# base 2^32 -> base 2^26
	extrwi	$h1,$h1,6,0
	insrwi	$h1,$h2,20,6
	extrwi	$h2,$h2,12,0
	insrwi	$h2,$h3,14,6
	extrwi	$h3,$h3,18,0
	insrwi	$h3,$h4,8,6
	extrwi	$h4,$h4,24,0

	mtvrwz	$R0,$h0
	slwi	$h0,$h1,2
	mtvrwz	$R1,$h1
	add	$h1,$h1,$h0
	mtvrwz	$S1,$h1
	slwi	$h1,$h2,2
	mtvrwz	$R2,$h2
	add	$h2,$h2,$h1
	mtvrwz	$S2,$h2
	slwi	$h2,$h3,2
	mtvrwz	$R3,$h3
	add	$h3,$h3,$h2
	mtvrwz	$S3,$h3
	slwi	$h3,$h4,2
	mtvrwz	$R4,$h4
	add	$h4,$h4,$h3
	mtvrwz	$S4,$h4

	vmr	$H0,$R0
	vmr	$H1,$R1
	vmr	$H2,$R2
	vmr	$H3,$R3
	vmr	$H4,$R4

	bl	__poly1305_mul		# r^1:- * r^1:-

	vpermdi	$R0,$H0,$R0,0b00
	vpermdi	$R1,$H1,$R1,0b00
	vpermdi	$R2,$H2,$R2,0b00
	vpermdi	$R3,$H3,$R3,0b00
	vpermdi	$R4,$H4,$R4,0b00
	vpermdi	$H0,$H0,$H0,0b00
	vpermdi	$H1,$H1,$H1,0b00
	vpermdi	$H2,$H2,$H2,0b00
	vpermdi	$H3,$H3,$H3,0b00
	vpermdi	$H4,$H4,$H4,0b00
	vsld	$S1,$R1,$T0		# <<2
	vsld	$S2,$R2,$T0
	vsld	$S3,$R3,$T0
	vsld	$S4,$R4,$T0
	vaddudm	$S1,$S1,$R1
	vaddudm	$S2,$S2,$R2
	vaddudm	$S3,$S3,$R3
	vaddudm	$S4,$S4,$R4

	bl	__poly1305_mul		# r^2:r^2 * r^2:r^1

	addi	$h0,$ctx,0x60
	lwz	$h1,0($ctx)		# load hash
	lwz	$h2,4($ctx)
	lwz	$h3,8($ctx)
	lwz	$h4,12($ctx)
	lwz	$t0,16($ctx)

	vmrgow	$R0,$R0,$H0		# r^2:r^4:r^1:r^3
	vmrgow	$R1,$R1,$H1
	vmrgow	$R2,$R2,$H2
	vmrgow	$R3,$R3,$H3
	vmrgow	$R4,$R4,$H4
	vslw	$S1,$R1,$T0		# <<2
	vslw	$S2,$R2,$T0
	vslw	$S3,$R3,$T0
	vslw	$S4,$R4,$T0
	vadduwm	$S1,$S1,$R1
	vadduwm	$S2,$S2,$R2
	vadduwm	$S3,$S3,$R3
	vadduwm	$S4,$S4,$R4

	stvx_u	$R0,$x30,$ctx
	stvx_u	$R1,$x40,$ctx
	stvx_u	$S1,$x50,$ctx
	stvx_u	$R2,$x00,$h0
	stvx_u	$S2,$x10,$h0
	stvx_u	$R3,$x20,$h0
	stvx_u	$S3,$x30,$h0
	stvx_u	$R4,$x40,$h0
	stvx_u	$S4,$x50,$h0

	extrwi	$h0,$h1,26,6		# base 2^32 -> base 2^26
	extrwi	$h1,$h1,6,0
	mtvrwz	$H0,$h0
	insrwi	$h1,$h2,20,6
	extrwi	$h2,$h2,12,0
	mtvrwz	$H1,$h1
	insrwi	$h2,$h3,14,6
	extrwi	$h3,$h3,18,0
	mtvrwz	$H2,$h2
	insrwi	$h3,$h4,8,6
	extrwi	$h4,$h4,24,0
	mtvrwz	$H3,$h3
	insrwi	$h4,$t0,3,5
	mtvrwz	$H4,$h4
___
							}
$code.=<<___;
	li	r0,1
	stw	r0,24($ctx)		# set is_base2_26
	b	Loaded_vsx

.align	4
Lskip_init_vsx:
	li		$x10,4
	li		$x20,8
	li		$x30,12
	li		$x40,16
	lvwzx_u		$H0,$x00,$ctx
	lvwzx_u		$H1,$x10,$ctx
	lvwzx_u		$H2,$x20,$ctx
	lvwzx_u		$H3,$x30,$ctx
	lvwzx_u		$H4,$x40,$ctx

Loaded_vsx:
	li		$x10,0x10
	li		$x20,0x20
	li		$x30,0x30
	li		$x40,0x40
	li		$x50,0x50
	li		$x60,0x60
	li		$x70,0x70
	addi		$ctx_,$ctx,64		# &ctx->r[1]
	addi		$_ctx,$sp,`$LOCALS+15`	# &ctx->r[1], r^2:r^4 shadow

	vxor		$T0,$T0,$T0		# ensure second half is zero
	vpermdi		$H0,$H0,$T0,0b00
	vpermdi		$H1,$H1,$T0,0b00
	vpermdi		$H2,$H2,$T0,0b00
	vpermdi		$H3,$H3,$T0,0b00
	vpermdi		$H4,$H4,$T0,0b00

	be?lvx_u	$_4,$x50,$const		# byte swap mask
	lvx_u		$T1,$x00,$inp		# load first input block
	lvx_u		$T2,$x10,$inp
	lvx_u		$T3,$x20,$inp
	lvx_u		$T4,$x30,$inp
	be?vperm	$T1,$T1,$T1,$_4
	be?vperm	$T2,$T2,$T2,$_4
	be?vperm	$T3,$T3,$T3,$_4
	be?vperm	$T4,$T4,$T4,$_4

	vpermdi		$I0,$T1,$T2,0b00	# smash input to base 2^26
	vspltisb	$_4,4
	vperm		$I2,$T1,$T2,$I2perm	# 0x...0e0f0001...1e1f1011
	vspltisb	$_14,14
	vpermdi		$I3,$T1,$T2,0b11

	vsrd		$I1,$I0,$_26
	vsrd		$I2,$I2,$_4
	vsrd		$I4,$I3,$_40
	vsrd		$I3,$I3,$_14
	vand		$I0,$I0,$mask26
	vand		$I1,$I1,$mask26
	vand		$I2,$I2,$mask26
	vand		$I3,$I3,$mask26

	vpermdi		$T1,$T3,$T4,0b00
	vperm		$T2,$T3,$T4,$I2perm	# 0x...0e0f0001...1e1f1011
	vpermdi		$T3,$T3,$T4,0b11

	vsrd		$T0,$T1,$_26
	vsrd		$T2,$T2,$_4
	vsrd		$T4,$T3,$_40
	vsrd		$T3,$T3,$_14
	vand		$T1,$T1,$mask26
	vand		$T0,$T0,$mask26
	vand		$T2,$T2,$mask26
	vand		$T3,$T3,$mask26

	# inp[2]:inp[0]:inp[3]:inp[1]
	vmrgow		$I4,$T4,$I4
	vmrgow		$I0,$T1,$I0
	vmrgow		$I1,$T0,$I1
	vmrgow		$I2,$T2,$I2
	vmrgow		$I3,$T3,$I3
	vor		$I4,$I4,$padbits

	lvx_splt	$R0,$x30,$ctx		# taking lvx_vsplt out of loop
	lvx_splt	$R1,$x00,$ctx_		# gives ~8% improvement
	lvx_splt	$S1,$x10,$ctx_
	lvx_splt	$R2,$x20,$ctx_
	lvx_splt	$S2,$x30,$ctx_
	lvx_splt	$T1,$x40,$ctx_
	lvx_splt	$T2,$x50,$ctx_
	lvx_splt	$T3,$x60,$ctx_
	lvx_splt	$T4,$x70,$ctx_
	stvx		$R1,$x00,$_ctx
	stvx		$S1,$x10,$_ctx
	stvx		$R2,$x20,$_ctx
	stvx		$S2,$x30,$_ctx
	stvx		$T1,$x40,$_ctx
	stvx		$T2,$x50,$_ctx
	stvx		$T3,$x60,$_ctx
	stvx		$T4,$x70,$_ctx

	addi		$inp,$inp,0x40
	addi		$const,$const,0x50
	addi		r0,$len,-64
	srdi		r0,r0,6
	mtctr		r0
	b		Loop_vsx

.align	4
Loop_vsx:
	################################################################
	## ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2
	## ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r
	##   \___________________/
	##
	## Note that we start with inp[2:3]*r^2. This is because it
	## doesn't depend on reduction in previous iteration.
	################################################################
	## d4 = h4*r0 + h3*r1   + h2*r2   + h1*r3   + h0*r4
	## d3 = h3*r0 + h2*r1   + h1*r2   + h0*r3   + h4*5*r4
	## d2 = h2*r0 + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4
	## d1 = h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3 + h2*5*r4
	## d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4

	vmuleuw		$ACC0,$I0,$R0
	vmuleuw		$ACC1,$I0,$R1
	vmuleuw		$ACC2,$I0,$R2
	vmuleuw		$ACC3,$I1,$R2

	vmuleuw		$T0,$I1,$R0
	vaddudm		$ACC1,$ACC1,$T0
	vmuleuw		$T0,$I1,$R1
	vaddudm		$ACC2,$ACC2,$T0
	 vmuleuw	$ACC4,$I2,$R2
	vmuleuw		$T0,$I4,$S1
	vaddudm		$ACC0,$ACC0,$T0
	vmuleuw		$T0,$I2,$R1
	vaddudm		$ACC3,$ACC3,$T0
	lvx		$S3,$x50,$_ctx
	vmuleuw		$T0,$I3,$R1
	vaddudm		$ACC4,$ACC4,$T0
	lvx		$R3,$x40,$_ctx

	 vaddudm	$H2,$H2,$I2
	 vaddudm	$H0,$H0,$I0
	 vaddudm	$H3,$H3,$I3
	 vaddudm	$H1,$H1,$I1
	 vaddudm	$H4,$H4,$I4

	vmuleuw		$T0,$I3,$S2
	vaddudm		$ACC0,$ACC0,$T0
	vmuleuw		$T0,$I4,$S2
	vaddudm		$ACC1,$ACC1,$T0
	vmuleuw		$T0,$I2,$R0
	vaddudm		$ACC2,$ACC2,$T0
	vmuleuw		$T0,$I3,$R0
	vaddudm		$ACC3,$ACC3,$T0
	lvx		$S4,$x70,$_ctx
	vmuleuw		$T0,$I4,$R0
	vaddudm		$ACC4,$ACC4,$T0
	lvx		$R4,$x60,$_ctx

	vmuleuw		$T0,$I2,$S3
	vaddudm		$ACC0,$ACC0,$T0
	vmuleuw		$T0,$I3,$S3
	vaddudm		$ACC1,$ACC1,$T0
	vmuleuw		$T0,$I4,$S3
	vaddudm		$ACC2,$ACC2,$T0
	vmuleuw		$T0,$I0,$R3
	vaddudm		$ACC3,$ACC3,$T0
	vmuleuw		$T0,$I1,$R3
	vaddudm		$ACC4,$ACC4,$T0

	 be?lvx_u	$_4,$x00,$const		# byte swap mask
	 lvx_u		$T1,$x00,$inp		# load next input block
	 lvx_u		$T2,$x10,$inp
	 lvx_u		$T3,$x20,$inp
	 lvx_u		$T4,$x30,$inp
	 be?vperm	$T1,$T1,$T1,$_4
	 be?vperm	$T2,$T2,$T2,$_4
	 be?vperm	$T3,$T3,$T3,$_4
	 be?vperm	$T4,$T4,$T4,$_4

	vmuleuw		$T0,$I1,$S4
	vaddudm		$ACC0,$ACC0,$T0
	vmuleuw		$T0,$I2,$S4
	vaddudm		$ACC1,$ACC1,$T0
	vmuleuw		$T0,$I3,$S4
	vaddudm		$ACC2,$ACC2,$T0
	vmuleuw		$T0,$I4,$S4
	vaddudm		$ACC3,$ACC3,$T0
	vmuleuw		$T0,$I0,$R4
	vaddudm		$ACC4,$ACC4,$T0

	 vpermdi	$I0,$T1,$T2,0b00	# smash input to base 2^26
	 vspltisb	$_4,4
	 vperm		$I2,$T1,$T2,$I2perm	# 0x...0e0f0001...1e1f1011
	 vpermdi	$I3,$T1,$T2,0b11

	# (hash + inp[0:1]) * r^4
	vmulouw		$T0,$H0,$R0
	vaddudm		$ACC0,$ACC0,$T0
	vmulouw		$T0,$H1,$R0
	vaddudm		$ACC1,$ACC1,$T0
	vmulouw		$T0,$H2,$R0
	vaddudm		$ACC2,$ACC2,$T0
	vmulouw		$T0,$H3,$R0
	vaddudm		$ACC3,$ACC3,$T0
	vmulouw		$T0,$H4,$R0
	vaddudm		$ACC4,$ACC4,$T0

	 vpermdi	$T1,$T3,$T4,0b00
	 vperm		$T2,$T3,$T4,$I2perm	# 0x...0e0f0001...1e1f1011
	 vpermdi	$T3,$T3,$T4,0b11

	vmulouw		$T0,$H2,$S3
	vaddudm		$ACC0,$ACC0,$T0
	vmulouw		$T0,$H3,$S3
	vaddudm		$ACC1,$ACC1,$T0
	vmulouw		$T0,$H4,$S3
	vaddudm		$ACC2,$ACC2,$T0
	vmulouw		$T0,$H0,$R3
	vaddudm		$ACC3,$ACC3,$T0
	lvx		$S1,$x10,$_ctx
	vmulouw		$T0,$H1,$R3
	vaddudm		$ACC4,$ACC4,$T0
	lvx		$R1,$x00,$_ctx

	 vsrd		$I1,$I0,$_26
	 vsrd		$I2,$I2,$_4
	 vsrd		$I4,$I3,$_40
	 vsrd		$I3,$I3,$_14

	vmulouw		$T0,$H1,$S4
	vaddudm		$ACC0,$ACC0,$T0
	vmulouw		$T0,$H2,$S4
	vaddudm		$ACC1,$ACC1,$T0
	vmulouw		$T0,$H3,$S4
	vaddudm		$ACC2,$ACC2,$T0
	vmulouw		$T0,$H4,$S4
	vaddudm		$ACC3,$ACC3,$T0
	lvx		$S2,$x30,$_ctx
	vmulouw		$T0,$H0,$R4
	vaddudm		$ACC4,$ACC4,$T0
	lvx		$R2,$x20,$_ctx

	 vand		$I0,$I0,$mask26
	 vand		$I1,$I1,$mask26
	 vand		$I2,$I2,$mask26
	 vand		$I3,$I3,$mask26

	vmulouw		$T0,$H4,$S1
	vaddudm		$ACC0,$ACC0,$T0
	vmulouw		$T0,$H0,$R1
	vaddudm		$ACC1,$ACC1,$T0
	vmulouw		$T0,$H1,$R1
	vaddudm		$ACC2,$ACC2,$T0
	vmulouw		$T0,$H2,$R1
	vaddudm		$ACC3,$ACC3,$T0
	vmulouw		$T0,$H3,$R1
	vaddudm		$ACC4,$ACC4,$T0

	 vsrd		$T2,$T2,$_4
	 vsrd		$_4,$T1,$_26
	 vsrd		$T4,$T3,$_40
	 vsrd		$T3,$T3,$_14

	vmulouw		$T0,$H3,$S2
	vaddudm		$ACC0,$ACC0,$T0
	vmulouw		$T0,$H4,$S2
	vaddudm		$ACC1,$ACC1,$T0
	vmulouw		$T0,$H0,$R2
	vaddudm		$ACC2,$ACC2,$T0
	vmulouw		$T0,$H1,$R2
	vaddudm		$ACC3,$ACC3,$T0
	vmulouw		$T0,$H2,$R2
	vaddudm		$ACC4,$ACC4,$T0

	 vand		$T1,$T1,$mask26
	 vand		$_4,$_4,$mask26
	 vand		$T2,$T2,$mask26
	 vand		$T3,$T3,$mask26

	################################################################
	# lazy reduction as discussed in "NEON crypto" by D.J. Bernstein
	# and P. Schwabe

	vspltisb	$T0,2
	vsrd		$H4,$ACC3,$_26
	vsrd		$H1,$ACC0,$_26
	vand		$H3,$ACC3,$mask26
	vand		$H0,$ACC0,$mask26
	vaddudm		$H4,$H4,$ACC4		# h3 -> h4
	vaddudm		$H1,$H1,$ACC1		# h0 -> h1

	 vmrgow		$I4,$T4,$I4
	 vmrgow		$I0,$T1,$I0
	 vmrgow		$I1,$_4,$I1
	 vmrgow		$I2,$T2,$I2
	 vmrgow		$I3,$T3,$I3
	 vor		$I4,$I4,$padbits

	vsrd		$ACC4,$H4,$_26
	vsrd		$ACC1,$H1,$_26
	vand		$H4,$H4,$mask26
	vand		$H1,$H1,$mask26
	vaddudm		$H0,$H0,$ACC4
	vaddudm		$H2,$ACC2,$ACC1		# h1 -> h2

	vsld		$ACC4,$ACC4,$T0		# <<2
	vsrd		$ACC2,$H2,$_26
	vand		$H2,$H2,$mask26
	vaddudm		$H0,$H0,$ACC4		# h4 -> h0
	vaddudm		$H3,$H3,$ACC2		# h2 -> h3

	vsrd		$ACC0,$H0,$_26
	vsrd		$ACC3,$H3,$_26
	vand		$H0,$H0,$mask26
	vand		$H3,$H3,$mask26
	vaddudm		$H1,$H1,$ACC0		# h0 -> h1
	vaddudm		$H4,$H4,$ACC3		# h3 -> h4

	addi		$inp,$inp,0x40
	bdnz		Loop_vsx

	neg		$len,$len
	andi.		$len,$len,0x30
	sub		$inp,$inp,$len

	lvx_u		$R0,$x30,$ctx		# load all powers
	lvx_u		$R1,$x00,$ctx_
	lvx_u		$S1,$x10,$ctx_
	lvx_u		$R2,$x20,$ctx_
	lvx_u		$S2,$x30,$ctx_

Last_vsx:
	vmuleuw		$ACC0,$I0,$R0
	vmuleuw		$ACC1,$I1,$R0
	vmuleuw		$ACC2,$I2,$R0
	vmuleuw		$ACC3,$I3,$R0
	vmuleuw		$ACC4,$I4,$R0

	vmuleuw		$T0,$I4,$S1
	vaddudm		$ACC0,$ACC0,$T0
	vmuleuw		$T0,$I0,$R1
	vaddudm		$ACC1,$ACC1,$T0
	vmuleuw		$T0,$I1,$R1
	vaddudm		$ACC2,$ACC2,$T0
	vmuleuw		$T0,$I2,$R1
	vaddudm		$ACC3,$ACC3,$T0
	lvx_u		$S3,$x50,$ctx_
	vmuleuw		$T0,$I3,$R1
	vaddudm		$ACC4,$ACC4,$T0
	lvx_u		$R3,$x40,$ctx_

	 vaddudm	$H2,$H2,$I2
	 vaddudm	$H0,$H0,$I0
	 vaddudm	$H3,$H3,$I3
	 vaddudm	$H1,$H1,$I1
	 vaddudm	$H4,$H4,$I4

	vmuleuw		$T0,$I3,$S2
	vaddudm		$ACC0,$ACC0,$T0
	vmuleuw		$T0,$I4,$S2
	vaddudm		$ACC1,$ACC1,$T0
	vmuleuw		$T0,$I0,$R2
	vaddudm		$ACC2,$ACC2,$T0
	vmuleuw		$T0,$I1,$R2
	vaddudm		$ACC3,$ACC3,$T0
	lvx_u		$S4,$x70,$ctx_
	vmuleuw		$T0,$I2,$R2
	vaddudm		$ACC4,$ACC4,$T0
	lvx_u		$R4,$x60,$ctx_

	vmuleuw		$T0,$I2,$S3
	vaddudm		$ACC0,$ACC0,$T0
	vmuleuw		$T0,$I3,$S3
	vaddudm		$ACC1,$ACC1,$T0
	vmuleuw		$T0,$I4,$S3
	vaddudm		$ACC2,$ACC2,$T0
	vmuleuw		$T0,$I0,$R3
	vaddudm		$ACC3,$ACC3,$T0
	vmuleuw		$T0,$I1,$R3
	vaddudm		$ACC4,$ACC4,$T0

	vmuleuw		$T0,$I1,$S4
	vaddudm		$ACC0,$ACC0,$T0
	vmuleuw		$T0,$I2,$S4
	vaddudm		$ACC1,$ACC1,$T0
	vmuleuw		$T0,$I3,$S4
	vaddudm		$ACC2,$ACC2,$T0
	vmuleuw		$T0,$I4,$S4
	vaddudm		$ACC3,$ACC3,$T0
	vmuleuw		$T0,$I0,$R4
	vaddudm		$ACC4,$ACC4,$T0

	# (hash + inp[0:1]) * r^4
	vmulouw		$T0,$H0,$R0
	vaddudm		$ACC0,$ACC0,$T0
	vmulouw		$T0,$H1,$R0
	vaddudm		$ACC1,$ACC1,$T0
	vmulouw		$T0,$H2,$R0
	vaddudm		$ACC2,$ACC2,$T0
	vmulouw		$T0,$H3,$R0
	vaddudm		$ACC3,$ACC3,$T0
	vmulouw		$T0,$H4,$R0
	vaddudm		$ACC4,$ACC4,$T0

	vmulouw		$T0,$H2,$S3
	vaddudm		$ACC0,$ACC0,$T0
	vmulouw		$T0,$H3,$S3
	vaddudm		$ACC1,$ACC1,$T0
	vmulouw		$T0,$H4,$S3
	vaddudm		$ACC2,$ACC2,$T0
	vmulouw		$T0,$H0,$R3
	vaddudm		$ACC3,$ACC3,$T0
	lvx_u		$S1,$x10,$ctx_
	vmulouw		$T0,$H1,$R3
	vaddudm		$ACC4,$ACC4,$T0
	lvx_u		$R1,$x00,$ctx_

	vmulouw		$T0,$H1,$S4
	vaddudm		$ACC0,$ACC0,$T0
	vmulouw		$T0,$H2,$S4
	vaddudm		$ACC1,$ACC1,$T0
	vmulouw		$T0,$H3,$S4
	vaddudm		$ACC2,$ACC2,$T0
	vmulouw		$T0,$H4,$S4
	vaddudm		$ACC3,$ACC3,$T0
	lvx_u		$S2,$x30,$ctx_
	vmulouw		$T0,$H0,$R4
	vaddudm		$ACC4,$ACC4,$T0
	lvx_u		$R2,$x20,$ctx_

	vmulouw		$T0,$H4,$S1
	vaddudm		$ACC0,$ACC0,$T0
	vmulouw		$T0,$H0,$R1
	vaddudm		$ACC1,$ACC1,$T0
	vmulouw		$T0,$H1,$R1
	vaddudm		$ACC2,$ACC2,$T0
	vmulouw		$T0,$H2,$R1
	vaddudm		$ACC3,$ACC3,$T0
	vmulouw		$T0,$H3,$R1
	vaddudm		$ACC4,$ACC4,$T0

	vmulouw		$T0,$H3,$S2
	vaddudm		$ACC0,$ACC0,$T0
	vmulouw		$T0,$H4,$S2
	vaddudm		$ACC1,$ACC1,$T0
	vmulouw		$T0,$H0,$R2
	vaddudm		$ACC2,$ACC2,$T0
	vmulouw		$T0,$H1,$R2
	vaddudm		$ACC3,$ACC3,$T0
	vmulouw		$T0,$H2,$R2
	vaddudm		$ACC4,$ACC4,$T0

	################################################################
	# horizontal addition

	vpermdi		$H0,$ACC0,$ACC0,0b10
	vpermdi		$H1,$ACC1,$ACC1,0b10
	vpermdi		$H2,$ACC2,$ACC2,0b10
	vpermdi		$H3,$ACC3,$ACC3,0b10
	vpermdi		$H4,$ACC4,$ACC4,0b10
	vaddudm		$ACC0,$ACC0,$H0
	vaddudm		$ACC1,$ACC1,$H1
	vaddudm		$ACC2,$ACC2,$H2
	vaddudm		$ACC3,$ACC3,$H3
	vaddudm		$ACC4,$ACC4,$H4

	################################################################
	# lazy reduction

	vspltisb	$T0,2
	vsrd		$H4,$ACC3,$_26
	vsrd		$H1,$ACC0,$_26
	vand		$H3,$ACC3,$mask26
	vand		$H0,$ACC0,$mask26
	vaddudm		$H4,$H4,$ACC4		# h3 -> h4
	vaddudm		$H1,$H1,$ACC1		# h0 -> h1

	vsrd		$ACC4,$H4,$_26
	vsrd		$ACC1,$H1,$_26
	vand		$H4,$H4,$mask26
	vand		$H1,$H1,$mask26
	vaddudm		$H0,$H0,$ACC4
	vaddudm		$H2,$ACC2,$ACC1		# h1 -> h2

	vsld		$ACC4,$ACC4,$T0		# <<2
	vsrd		$ACC2,$H2,$_26
	vand		$H2,$H2,$mask26
	vaddudm		$H0,$H0,$ACC4		# h4 -> h0
	vaddudm		$H3,$H3,$ACC2		# h2 -> h3

	vsrd		$ACC0,$H0,$_26
	vsrd		$ACC3,$H3,$_26
	vand		$H0,$H0,$mask26
	vand		$H3,$H3,$mask26
	vaddudm		$H1,$H1,$ACC0		# h0 -> h1
	vaddudm		$H4,$H4,$ACC3		# h3 -> h4

	beq		Ldone_vsx

	add		r6,$const,$len

	be?lvx_u	$_4,$x00,$const		# byte swap mask
	lvx_u		$T1,$x00,$inp		# load last partial input block
	lvx_u		$T2,$x10,$inp
	lvx_u		$T3,$x20,$inp
	lvx_u		$T4,$x30,$inp
	be?vperm	$T1,$T1,$T1,$_4
	be?vperm	$T2,$T2,$T2,$_4
	be?vperm	$T3,$T3,$T3,$_4
	be?vperm	$T4,$T4,$T4,$_4

	vpermdi		$I0,$T1,$T2,0b00	# smash input to base 2^26
	vspltisb	$_4,4
	vperm		$I2,$T1,$T2,$I2perm	# 0x...0e0f0001...1e1f1011
	vpermdi		$I3,$T1,$T2,0b11

	vsrd		$I1,$I0,$_26
	vsrd		$I2,$I2,$_4
	vsrd		$I4,$I3,$_40
	vsrd		$I3,$I3,$_14
	vand		$I0,$I0,$mask26
	vand		$I1,$I1,$mask26
	vand		$I2,$I2,$mask26
	vand		$I3,$I3,$mask26

	vpermdi		$T0,$T3,$T4,0b00
	vperm		$T1,$T3,$T4,$I2perm	# 0x...0e0f0001...1e1f1011
	vpermdi		$T2,$T3,$T4,0b11

	lvx_u		$ACC0,$x00,r6
	lvx_u		$ACC1,$x30,r6

	vsrd		$T3,$T0,$_26
	vsrd		$T1,$T1,$_4
	vsrd		$T4,$T2,$_40
	vsrd		$T2,$T2,$_14
	vand		$T0,$T0,$mask26
	vand		$T3,$T3,$mask26
	vand		$T1,$T1,$mask26
	vand		$T2,$T2,$mask26

	# inp[2]:inp[0]:inp[3]:inp[1]
	vmrgow		$I4,$T4,$I4
	vmrgow		$I0,$T0,$I0
	vmrgow		$I1,$T3,$I1
	vmrgow		$I2,$T1,$I2
	vmrgow		$I3,$T2,$I3
	vor		$I4,$I4,$padbits

	vperm		$H0,$H0,$H0,$ACC0	# move hash to right lane
	vand		$I0,$I0,    $ACC1	# mask redundant input lane[s]
	vperm		$H1,$H1,$H1,$ACC0
	vand		$I1,$I1,    $ACC1
	vperm		$H2,$H2,$H2,$ACC0
	vand		$I2,$I2,    $ACC1
	vperm		$H3,$H3,$H3,$ACC0
	vand		$I3,$I3,    $ACC1
	vperm		$H4,$H4,$H4,$ACC0
	vand		$I4,$I4,    $ACC1

	vaddudm		$I0,$I0,$H0		# accumulate hash
	vxor		$H0,$H0,$H0		# wipe hash value
	vaddudm		$I1,$I1,$H1
	vxor		$H1,$H1,$H1
	vaddudm		$I2,$I2,$H2
	vxor		$H2,$H2,$H2
	vaddudm		$I3,$I3,$H3
	vxor		$H3,$H3,$H3
	vaddudm		$I4,$I4,$H4
	vxor		$H4,$H4,$H4

	xor.		$len,$len,$len
	b		Last_vsx

.align	4
Ldone_vsx:
	$POP	r0,`$VSXFRAME+$LRSAVE`($sp)
	li	$x10,4
	li	$x20,8
	li	$x30,12
	li	$x40,16
	stvwx_u	$H0,$x00,$ctx			# store hash
	stvwx_u	$H1,$x10,$ctx
	stvwx_u	$H2,$x20,$ctx
	stvwx_u	$H3,$x30,$ctx
	stvwx_u	$H4,$x40,$ctx

	lwz	r12,`$VSXFRAME-$SIZE_T*5-4`($sp)# pull vrsave
	mtlr	r0
	li	r10,`15+$LOCALS+128`
	li	r11,`31+$LOCALS+128`
	mtspr	256,r12				# restore vrsave
	lvx	v20,r10,$sp
	addi	r10,r10,32
	lvx	v21,r10,$sp
	addi	r10,r10,32
	lvx	v22,r11,$sp
	addi	r11,r11,32
	lvx	v23,r10,$sp
	addi	r10,r10,32
	lvx	v24,r11,$sp
	addi	r11,r11,32
	lvx	v25,r10,$sp
	addi	r10,r10,32
	lvx	v26,r11,$sp
	addi	r11,r11,32
	lvx	v27,r10,$sp
	addi	r10,r10,32
	lvx	v28,r11,$sp
	addi	r11,r11,32
	lvx	v29,r10,$sp
	addi	r10,r10,32
	lvx	v30,r11,$sp
	lvx	v31,r10,$sp
	$POP	r27,`$VSXFRAME-$SIZE_T*5`($sp)
	$POP	r28,`$VSXFRAME-$SIZE_T*4`($sp)
	$POP	r29,`$VSXFRAME-$SIZE_T*3`($sp)
	$POP	r30,`$VSXFRAME-$SIZE_T*2`($sp)
	$POP	r31,`$VSXFRAME-$SIZE_T*1`($sp)
	addi	$sp,$sp,$VSXFRAME
	blr
	.long	0
	.byte	0,12,0x04,1,0x80,5,4,0
	.long	0
.size	__poly1305_blocks_vsx,.-__poly1305_blocks_vsx

.align	6
LPICmeup:
	mflr	r0
	bcl	20,31,\$+4
	mflr	$const      # vvvvvv "distance" between . and 1st data entry
	addi	$const,$const,`64-8`
	mtlr	r0
	blr
	.long	0
	.byte	0,12,0x14,0,0,0,0,0
	.space	`64-9*4`

.quad	0x0000000003ffffff,0x0000000003ffffff	# mask26
.quad	0x000000000000001a,0x000000000000001a	# _26
.quad	0x0000000000000028,0x0000000000000028	# _40
.quad	0x000000000e0f0001,0x000000001e1f1011	# I2perm
.quad	0x0100000001000000,0x0100000001000000	# padbits
.quad	0x0706050403020100,0x0f0e0d0c0b0a0908	# byte swap for big-endian

.quad	0x0000000000000000,0x0000000004050607	# magic tail masks
.quad	0x0405060700000000,0x0000000000000000
.quad	0x0000000000000000,0x0405060700000000

.quad	0xffffffff00000000,0xffffffffffffffff
.quad	0xffffffff00000000,0xffffffff00000000
.quad	0x0000000000000000,0xffffffff00000000
___
}}}
$code.=<<___;
.asciz	"Poly1305 for PPC, CRYPTOGAMS by \@dot-asm"
___

foreach (split("\n",$code)) {
	s/\`([^\`]*)\`/eval($1)/ge;

	# instructions prefixed with '?' are endian-specific and need
	# to be adjusted accordingly...
	if ($flavour !~ /le$/) {	# big-endian
	    s/be\?//		or
	    s/le\?/#le#/
	} else {			# little-endian
	    s/le\?//		or
	    s/be\?/#be#/
	}

	print $_,"\n";
}
close STDOUT or die "error closing STDOUT";