openssl/crypto/chacha/asm/chacha-s390x.pl

#! /usr/bin/env perl
# Copyright 2016-2019 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the Apache License 2.0 (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html

#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
# December 2015
#
# ChaCha20 for s390x.
#
# 3 times faster than compiler-generated code.

#
# August 2018
#
# Add vx code path: 4x"vertical".
#
# Copyright IBM Corp. 2018
# Author: Patrick Steuer <patrick.steuer@de.ibm.com>

#
# February 2019
#
# Add 6x"horizontal" VX implementation. It's ~25% faster than IBM's
# 4x"vertical" submission [on z13] and >3 faster than scalar code.
# But to harness overheads revert to transliteration of VSX code path
# from chacha-ppc module, which is also 4x"vertical", to handle inputs
# not longer than 256 bytes.

use strict;
use FindBin qw($Bin);
use lib "$Bin/../..";
use perlasm::s390x qw(:DEFAULT :VX :EI AUTOLOAD LABEL INCLUDE);

# $output is the last argument if it looks like a file (it has an extension)
# $flavour is the first argument if it doesn't look like a file
my $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
my $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;

my ($z,$SIZE_T);
if ($flavour =~ /3[12]/) {
	$z=0;	# S/390 ABI
	$SIZE_T=4;
} else {
	$z=1;	# zSeries ABI
	$SIZE_T=8;
}

my $sp="%r15";
my $stdframe=16*$SIZE_T+4*8;

sub ROUND {
my @x=map("%r$_",(0..7,"x","x","x","x",(10..13)));
my @t=map("%r$_",(8,9));
my ($a0,$b0,$c0,$d0)=@_;
my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0));
my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1));
my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2));
my ($xc,$xc_)=map("$_",@t);

	# Consider order in which variables are addressed by their
	# index:
	#
	#	a   b   c   d
	#
	#	0   4   8  12 < even round
	#	1   5   9  13
	#	2   6  10  14
	#	3   7  11  15
	#	0   5  10  15 < odd round
	#	1   6  11  12
	#	2   7   8  13
	#	3   4   9  14
	#
	# 'a', 'b' and 'd's are permanently allocated in registers,
	# @x[0..7,12..15], while 'c's are maintained in memory. If
	# you observe 'c' column, you'll notice that pair of 'c's is
	# invariant between rounds. This means that we have to reload
	# them once per round, in the middle. This is why you'll see
	# 'c' stores and loads in the middle, but none in the beginning
	# or end.

	alr	(@x[$a0],@x[$b0]);	# Q1
	 alr	(@x[$a1],@x[$b1]);	# Q2
	xr	(@x[$d0],@x[$a0]);
	 xr	(@x[$d1],@x[$a1]);
	rll	(@x[$d0],@x[$d0],16);
	 rll	(@x[$d1],@x[$d1],16);

	alr	($xc,@x[$d0]);
	 alr	($xc_,@x[$d1]);
	xr	(@x[$b0],$xc);
	 xr	(@x[$b1],$xc_);
	rll	(@x[$b0],@x[$b0],12);
	 rll	(@x[$b1],@x[$b1],12);

	alr	(@x[$a0],@x[$b0]);
	 alr	(@x[$a1],@x[$b1]);
	xr	(@x[$d0],@x[$a0]);
	 xr	(@x[$d1],@x[$a1]);
	rll	(@x[$d0],@x[$d0],8);
	 rll	(@x[$d1],@x[$d1],8);

	alr	($xc,@x[$d0]);
	 alr	($xc_,@x[$d1]);
	xr	(@x[$b0],$xc);
	 xr	(@x[$b1],$xc_);
	rll	(@x[$b0],@x[$b0],7);
	 rll	(@x[$b1],@x[$b1],7);

	stm	($xc,$xc_,"$stdframe+4*8+4*$c0($sp)");	# reload pair of 'c's
	lm	($xc,$xc_,"$stdframe+4*8+4*$c2($sp)");

	alr	(@x[$a2],@x[$b2]);	# Q3
	 alr	(@x[$a3],@x[$b3]);	# Q4
	xr	(@x[$d2],@x[$a2]);
	 xr	(@x[$d3],@x[$a3]);
	rll	(@x[$d2],@x[$d2],16);
	 rll	(@x[$d3],@x[$d3],16);

	alr	($xc,@x[$d2]);
	 alr	($xc_,@x[$d3]);
	xr	(@x[$b2],$xc);
	 xr	(@x[$b3],$xc_);
	rll	(@x[$b2],@x[$b2],12);
	 rll	(@x[$b3],@x[$b3],12);

	alr	(@x[$a2],@x[$b2]);
	 alr	(@x[$a3],@x[$b3]);
	xr	(@x[$d2],@x[$a2]);
	 xr	(@x[$d3],@x[$a3]);
	rll	(@x[$d2],@x[$d2],8);
	 rll	(@x[$d3],@x[$d3],8);

	alr	($xc,@x[$d2]);
	 alr	($xc_,@x[$d3]);
	xr	(@x[$b2],$xc);
	 xr	(@x[$b3],$xc_);
	rll	(@x[$b2],@x[$b2],7);
	 rll	(@x[$b3],@x[$b3],7);
}

sub VX_lane_ROUND {
my ($a0,$b0,$c0,$d0)=@_;
my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0));
my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1));
my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2));
my @x=map("%v$_",(0..15));

	vaf	(@x[$a0],@x[$a0],@x[$b0]);	# Q1
	vx	(@x[$d0],@x[$d0],@x[$a0]);
	verllf	(@x[$d0],@x[$d0],16);
	vaf	(@x[$a1],@x[$a1],@x[$b1]);	# Q2
	vx	(@x[$d1],@x[$d1],@x[$a1]);
	verllf	(@x[$d1],@x[$d1],16);
	vaf	(@x[$a2],@x[$a2],@x[$b2]);	# Q3
	vx	(@x[$d2],@x[$d2],@x[$a2]);
	verllf	(@x[$d2],@x[$d2],16);
	vaf	(@x[$a3],@x[$a3],@x[$b3]);	# Q4
	vx	(@x[$d3],@x[$d3],@x[$a3]);
	verllf	(@x[$d3],@x[$d3],16);

	vaf	(@x[$c0],@x[$c0],@x[$d0]);
	vx	(@x[$b0],@x[$b0],@x[$c0]);
	verllf	(@x[$b0],@x[$b0],12);
	vaf	(@x[$c1],@x[$c1],@x[$d1]);
	vx	(@x[$b1],@x[$b1],@x[$c1]);
	verllf	(@x[$b1],@x[$b1],12);
	vaf	(@x[$c2],@x[$c2],@x[$d2]);
	vx	(@x[$b2],@x[$b2],@x[$c2]);
	verllf	(@x[$b2],@x[$b2],12);
	vaf	(@x[$c3],@x[$c3],@x[$d3]);
	vx	(@x[$b3],@x[$b3],@x[$c3]);
	verllf	(@x[$b3],@x[$b3],12);

	vaf	(@x[$a0],@x[$a0],@x[$b0]);
	vx	(@x[$d0],@x[$d0],@x[$a0]);
	verllf	(@x[$d0],@x[$d0],8);
	vaf	(@x[$a1],@x[$a1],@x[$b1]);
	vx	(@x[$d1],@x[$d1],@x[$a1]);
	verllf	(@x[$d1],@x[$d1],8);
	vaf	(@x[$a2],@x[$a2],@x[$b2]);
	vx	(@x[$d2],@x[$d2],@x[$a2]);
	verllf	(@x[$d2],@x[$d2],8);
	vaf	(@x[$a3],@x[$a3],@x[$b3]);
	vx	(@x[$d3],@x[$d3],@x[$a3]);
	verllf	(@x[$d3],@x[$d3],8);

	vaf	(@x[$c0],@x[$c0],@x[$d0]);
	vx	(@x[$b0],@x[$b0],@x[$c0]);
	verllf	(@x[$b0],@x[$b0],7);
	vaf	(@x[$c1],@x[$c1],@x[$d1]);
	vx	(@x[$b1],@x[$b1],@x[$c1]);
	verllf	(@x[$b1],@x[$b1],7);
	vaf	(@x[$c2],@x[$c2],@x[$d2]);
	vx	(@x[$b2],@x[$b2],@x[$c2]);
	verllf	(@x[$b2],@x[$b2],7);
	vaf	(@x[$c3],@x[$c3],@x[$d3]);
	vx	(@x[$b3],@x[$b3],@x[$c3]);
	verllf	(@x[$b3],@x[$b3],7);
}

sub VX_ROUND {
my @a=@_[0..5];
my @b=@_[6..11];
my @c=@_[12..17];
my @d=@_[18..23];
my $odd=@_[24];

	vaf		(@a[$_],@a[$_],@b[$_]) for (0..5);
	vx		(@d[$_],@d[$_],@a[$_]) for (0..5);
	verllf		(@d[$_],@d[$_],16) for (0..5);

	vaf		(@c[$_],@c[$_],@d[$_]) for (0..5);
	vx		(@b[$_],@b[$_],@c[$_]) for (0..5);
	verllf		(@b[$_],@b[$_],12) for (0..5);

	vaf		(@a[$_],@a[$_],@b[$_]) for (0..5);
	vx		(@d[$_],@d[$_],@a[$_]) for (0..5);
	verllf		(@d[$_],@d[$_],8) for (0..5);

	vaf		(@c[$_],@c[$_],@d[$_]) for (0..5);
	vx		(@b[$_],@b[$_],@c[$_]) for (0..5);
	verllf		(@b[$_],@b[$_],7) for (0..5);

	vsldb		(@c[$_],@c[$_],@c[$_],8) for (0..5);
	vsldb		(@b[$_],@b[$_],@b[$_],$odd?12:4) for (0..5);
	vsldb		(@d[$_],@d[$_],@d[$_],$odd?4:12) for (0..5);
}

PERLASM_BEGIN($output);

INCLUDE	("s390x_arch.h");
TEXT	();

################
# void ChaCha20_ctr32(unsigned char *out, const unsigned char *inp, size_t len,
#                     const unsigned int key[8], const unsigned int counter[4])
my ($out,$inp,$len,$key,$counter)=map("%r$_",(2..6));
{
my $frame=$stdframe+4*20;
my @x=map("%r$_",(0..7,"x","x","x","x",(10..13)));
my @t=map("%r$_",(8,9));

GLOBL	("ChaCha20_ctr32");
TYPE	("ChaCha20_ctr32","\@function");
ALIGN	(32);
LABEL	("ChaCha20_ctr32");
	larl	("%r1","OPENSSL_s390xcap_P");

	lghi	("%r0",64);
&{$z?	\&ltgr:\&ltr}	($len,$len);		# len==0?
	bzr	("%r14");
	lg	("%r1","S390X_STFLE+16(%r1)");
&{$z?	\&clgr:\&clr}	($len,"%r0");
	jle	(".Lshort");

	tmhh	("%r1",0x4000);			# check for vx bit
	jnz	(".LChaCha20_ctr32_vx");

LABEL	(".Lshort");
&{$z?	\&aghi:\&ahi}	($len,-64);
&{$z?	\&lghi:\&lhi}	("%r1",-$frame);
&{$z?	\&stmg:\&stm}	("%r6","%r15","6*$SIZE_T($sp)");
&{$z?	\&slgr:\&slr}	($out,$inp);	# difference
	la	($len,"0($inp,$len)");	# end of input minus 64
	larl	("%r7",".Lsigma");
	lgr	("%r0",$sp);
	la	($sp,"0(%r1,$sp)");
&{$z?	\&stg:\&st}	("%r0","0($sp)");

	lmg	("%r8","%r11","0($key)");	# load key
	lmg	("%r12","%r13","0($counter)");	# load counter
	lmg	("%r6","%r7","0(%r7)");	# load sigma constant

	la	("%r14","0($inp)");
&{$z?	\&stg:\&st}	($out,"$frame+3*$SIZE_T($sp)");
&{$z?	\&stg:\&st}	($len,"$frame+4*$SIZE_T($sp)");
	stmg	("%r6","%r13","$stdframe($sp)");# copy key schedule to stack
	srlg	(@x[12],"%r12",32);	# 32-bit counter value
	j	(".Loop_outer");

ALIGN	(16);
LABEL	(".Loop_outer");
	lm	(@x[0],@x[7],"$stdframe+4*0($sp)");	# load x[0]-x[7]
	lm	(@t[0],@t[1],"$stdframe+4*10($sp)");	# load x[10]-x[11]
	lm	(@x[13],@x[15],"$stdframe+4*13($sp)");	# load x[13]-x[15]
	stm	(@t[0],@t[1],"$stdframe+4*8+4*10($sp)");# offload x[10]-x[11]
	lm	(@t[0],@t[1],"$stdframe+4*8($sp)");	# load x[8]-x[9]
	st	(@x[12],"$stdframe+4*12($sp)");	# save counter
&{$z?	\&stg:\&st}	("%r14","$frame+2*$SIZE_T($sp)");# save input pointer
	lhi	("%r14",10);
	j	(".Loop");

ALIGN	(4);
LABEL	(".Loop");
	ROUND	(0, 4, 8,12);
	ROUND	(0, 5,10,15);
	brct	("%r14",".Loop");

&{$z?	\&lg:\&l}	("%r14","$frame+2*$SIZE_T($sp)");# pull input pointer
	stm	(@t[0],@t[1],"$stdframe+4*8+4*8($sp)");	# offload x[8]-x[9]
&{$z?	\&lmg:\&lm}	(@t[0],@t[1],"$frame+3*$SIZE_T($sp)");

	al	(@x[0],"$stdframe+4*0($sp)");	# accumulate key schedule
	al	(@x[1],"$stdframe+4*1($sp)");
	al	(@x[2],"$stdframe+4*2($sp)");
	al	(@x[3],"$stdframe+4*3($sp)");
	al	(@x[4],"$stdframe+4*4($sp)");
	al	(@x[5],"$stdframe+4*5($sp)");
	al	(@x[6],"$stdframe+4*6($sp)");
	al	(@x[7],"$stdframe+4*7($sp)");
	lrvr	(@x[0],@x[0]);
	lrvr	(@x[1],@x[1]);
	lrvr	(@x[2],@x[2]);
	lrvr	(@x[3],@x[3]);
	lrvr	(@x[4],@x[4]);
	lrvr	(@x[5],@x[5]);
	lrvr	(@x[6],@x[6]);
	lrvr	(@x[7],@x[7]);
	al	(@x[12],"$stdframe+4*12($sp)");
	al	(@x[13],"$stdframe+4*13($sp)");
	al	(@x[14],"$stdframe+4*14($sp)");
	al	(@x[15],"$stdframe+4*15($sp)");
	lrvr	(@x[12],@x[12]);
	lrvr	(@x[13],@x[13]);
	lrvr	(@x[14],@x[14]);
	lrvr	(@x[15],@x[15]);

	la	(@t[0],"0(@t[0],%r14)");	# reconstruct output pointer
&{$z?	\&clgr:\&clr}	("%r14",@t[1]);
	jh	(".Ltail");

	x	(@x[0],"4*0(%r14)");	# xor with input
	x	(@x[1],"4*1(%r14)");
	st	(@x[0],"4*0(@t[0])");	# store output
	x	(@x[2],"4*2(%r14)");
	st	(@x[1],"4*1(@t[0])");
	x	(@x[3],"4*3(%r14)");
	st	(@x[2],"4*2(@t[0])");
	x	(@x[4],"4*4(%r14)");
	st	(@x[3],"4*3(@t[0])");
	 lm	(@x[0],@x[3],"$stdframe+4*8+4*8($sp)");	# load x[8]-x[11]
	x	(@x[5],"4*5(%r14)");
	st	(@x[4],"4*4(@t[0])");
	x	(@x[6],"4*6(%r14)");
	 al	(@x[0],"$stdframe+4*8($sp)");
	st	(@x[5],"4*5(@t[0])");
	x	(@x[7],"4*7(%r14)");
	 al	(@x[1],"$stdframe+4*9($sp)");
	st	(@x[6],"4*6(@t[0])");
	x	(@x[12],"4*12(%r14)");
	 al	(@x[2],"$stdframe+4*10($sp)");
	st	(@x[7],"4*7(@t[0])");
	x	(@x[13],"4*13(%r14)");
	 al	(@x[3],"$stdframe+4*11($sp)");
	st	(@x[12],"4*12(@t[0])");
	x	(@x[14],"4*14(%r14)");
	st	(@x[13],"4*13(@t[0])");
	x	(@x[15],"4*15(%r14)");
	st	(@x[14],"4*14(@t[0])");
	 lrvr	(@x[0],@x[0]);
	st	(@x[15],"4*15(@t[0])");
	 lrvr	(@x[1],@x[1]);
	 lrvr	(@x[2],@x[2]);
	 lrvr	(@x[3],@x[3]);
	lhi	(@x[12],1);
	 x	(@x[0],"4*8(%r14)");
	al	(@x[12],"$stdframe+4*12($sp)");	# increment counter
	 x	(@x[1],"4*9(%r14)");
	 st	(@x[0],"4*8(@t[0])");
	 x	(@x[2],"4*10(%r14)");
	 st	(@x[1],"4*9(@t[0])");
	 x	(@x[3],"4*11(%r14)");
	 st	(@x[2],"4*10(@t[0])");
	 st	(@x[3],"4*11(@t[0])");

&{$z?	\&clgr:\&clr}	("%r14",@t[1]);	# done yet?
	la	("%r14","64(%r14)");
	jl	(".Loop_outer");

LABEL	(".Ldone");
	xgr	("%r0","%r0");
	xgr	("%r1","%r1");
	xgr	("%r2","%r2");
	xgr	("%r3","%r3");
	stmg	("%r0","%r3","$stdframe+4*4($sp)");	# wipe key copy
	stmg	("%r0","%r3","$stdframe+4*12($sp)");

&{$z?	\&lmg:\&lm}	("%r6","%r15","$frame+6*$SIZE_T($sp)");
	br	("%r14");

ALIGN	(16);
LABEL	(".Ltail");
	la	(@t[1],"64($t[1])");
	stm	(@x[0],@x[7],"$stdframe+4*0($sp)");
&{$z?	\&slgr:\&slr}	(@t[1],"%r14");
	lm	(@x[0],@x[3],"$stdframe+4*8+4*8($sp)");
&{$z?	\&lghi:\&lhi}	(@x[6],0);
	stm	(@x[12],@x[15],"$stdframe+4*12($sp)");
	al	(@x[0],"$stdframe+4*8($sp)");
	al	(@x[1],"$stdframe+4*9($sp)");
	al	(@x[2],"$stdframe+4*10($sp)");
	al	(@x[3],"$stdframe+4*11($sp)");
	lrvr	(@x[0],@x[0]);
	lrvr	(@x[1],@x[1]);
	lrvr	(@x[2],@x[2]);
	lrvr	(@x[3],@x[3]);
	stm	(@x[0],@x[3],"$stdframe+4*8($sp)");

LABEL	(".Loop_tail");
	llgc	(@x[4],"0(@x[6],%r14)");
	llgc	(@x[5],"$stdframe(@x[6],$sp)");
	xr	(@x[5],@x[4]);
	stc	(@x[5],"0(@x[6],@t[0])");
	la	(@x[6],"1(@x[6])");
	brct	(@t[1],".Loop_tail");

	j	(".Ldone");
SIZE	("ChaCha20_ctr32",".-ChaCha20_ctr32");
}

########################################################################
# 4x"vertical" layout minimizes amount of instructions, but pipeline
# runs underutilized [because of vector instructions' high latency].
# On the other hand minimum amount of data it takes to fully utilize
# the pipeline is higher, so that effectively, short inputs would be
# processed slower. Hence this code path targeting <=256 bytes lengths.
#
{
my ($xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3,
    $xc0,$xc1,$xc2,$xc3, $xd0,$xd1,$xd2,$xd3)=map("%v$_",(0..15));
my @K=map("%v$_",(16..19));
my $CTR="%v26";
my ($xt0,$xt1,$xt2,$xt3)=map("%v$_",(27..30));
my $beperm="%v31";
my ($x00,$x10,$x20,$x30)=(0,map("r$_",(8..10)));
my $FRAME=$stdframe+4*16;

ALIGN	(32);
LABEL	("ChaCha20_ctr32_4x");
LABEL	(".LChaCha20_ctr32_4x");
&{$z?	\&stmg:\&stm}	("%r6","%r7","6*$SIZE_T($sp)");
if (!$z) {
	std	("%f4","16*$SIZE_T+2*8($sp)");
	std	("%f6","16*$SIZE_T+3*8($sp)");
}
&{$z?	\&lghi:\&lhi}	("%r1",-$FRAME);
	lgr	("%r0",$sp);
	la	($sp,"0(%r1,$sp)");
&{$z?	\&stg:\&st}	("%r0","0($sp)");	# back-chain
if ($z) {
	std	("%f8","$stdframe+8*0($sp)");
	std	("%f9","$stdframe+8*1($sp)");
	std	("%f10","$stdframe+8*2($sp)");
	std	("%f11","$stdframe+8*3($sp)");
	std	("%f12","$stdframe+8*4($sp)");
	std	("%f13","$stdframe+8*5($sp)");
	std	("%f14","$stdframe+8*6($sp)");
	std	("%f15","$stdframe+8*7($sp)");
}
	larl	("%r7",".Lsigma");
	lhi	("%r0",10);
	lhi	("%r1",0);

	vl	(@K[0],"0(%r7)");		# load sigma
	vl	(@K[1],"0($key)");		# load key
	vl	(@K[2],"16($key)");
	vl	(@K[3],"0($counter)");		# load counter

	vl	($beperm,"0x40(%r7)");
	vl	($xt1,"0x50(%r7)");
	vrepf	($CTR,@K[3],0);
	vlvgf	(@K[3],"%r1",0);		# clear @K[3].word[0]
	vaf	($CTR,$CTR,$xt1);

#LABEL	(".Loop_outer_4x");
	vlm	($xa0,$xa3,"0x60(%r7)");	# load [smashed] sigma

	vrepf	($xb0,@K[1],0);			# smash the key
	vrepf	($xb1,@K[1],1);
	vrepf	($xb2,@K[1],2);
	vrepf	($xb3,@K[1],3);

	vrepf	($xc0,@K[2],0);
	vrepf	($xc1,@K[2],1);
	vrepf	($xc2,@K[2],2);
	vrepf	($xc3,@K[2],3);

	vlr	($xd0,$CTR);
	vrepf	($xd1,@K[3],1);
	vrepf	($xd2,@K[3],2);
	vrepf	($xd3,@K[3],3);

LABEL	(".Loop_4x");
	VX_lane_ROUND(0, 4, 8,12);
	VX_lane_ROUND(0, 5,10,15);
	brct	("%r0",".Loop_4x");

	vaf	($xd0,$xd0,$CTR);

	vmrhf	($xt0,$xa0,$xa1);		# transpose data
	vmrhf	($xt1,$xa2,$xa3);
	vmrlf	($xt2,$xa0,$xa1);
	vmrlf	($xt3,$xa2,$xa3);
	vpdi	($xa0,$xt0,$xt1,0b0000);
	vpdi	($xa1,$xt0,$xt1,0b0101);
	vpdi	($xa2,$xt2,$xt3,0b0000);
	vpdi	($xa3,$xt2,$xt3,0b0101);

	vmrhf	($xt0,$xb0,$xb1);
	vmrhf	($xt1,$xb2,$xb3);
	vmrlf	($xt2,$xb0,$xb1);
	vmrlf	($xt3,$xb2,$xb3);
	vpdi	($xb0,$xt0,$xt1,0b0000);
	vpdi	($xb1,$xt0,$xt1,0b0101);
	vpdi	($xb2,$xt2,$xt3,0b0000);
	vpdi	($xb3,$xt2,$xt3,0b0101);

	vmrhf	($xt0,$xc0,$xc1);
	vmrhf	($xt1,$xc2,$xc3);
	vmrlf	($xt2,$xc0,$xc1);
	vmrlf	($xt3,$xc2,$xc3);
	vpdi	($xc0,$xt0,$xt1,0b0000);
	vpdi	($xc1,$xt0,$xt1,0b0101);
	vpdi	($xc2,$xt2,$xt3,0b0000);
	vpdi	($xc3,$xt2,$xt3,0b0101);

	vmrhf	($xt0,$xd0,$xd1);
	vmrhf	($xt1,$xd2,$xd3);
	vmrlf	($xt2,$xd0,$xd1);
	vmrlf	($xt3,$xd2,$xd3);
	vpdi	($xd0,$xt0,$xt1,0b0000);
	vpdi	($xd1,$xt0,$xt1,0b0101);
	vpdi	($xd2,$xt2,$xt3,0b0000);
	vpdi	($xd3,$xt2,$xt3,0b0101);

	#vrepif	($xt0,4);
	#vaf	($CTR,$CTR,$xt0);		# next counter value

	vaf	($xa0,$xa0,@K[0]);
	vaf	($xb0,$xb0,@K[1]);
	vaf	($xc0,$xc0,@K[2]);
	vaf	($xd0,$xd0,@K[3]);

	vperm	($xa0,$xa0,$xa0,$beperm);
	vperm	($xb0,$xb0,$xb0,$beperm);
	vperm	($xc0,$xc0,$xc0,$beperm);
	vperm	($xd0,$xd0,$xd0,$beperm);

	#&{$z?	\&clgfi:\&clfi} ($len,0x40);
	#jl	(".Ltail_4x");

	vlm	($xt0,$xt3,"0($inp)");

	vx	($xt0,$xt0,$xa0);
	vx	($xt1,$xt1,$xb0);
	vx	($xt2,$xt2,$xc0);
	vx	($xt3,$xt3,$xd0);

	vstm	($xt0,$xt3,"0($out)");

	la	($inp,"0x40($inp)");
	la	($out,"0x40($out)");
&{$z?	\&aghi:\&ahi}	($len,-0x40);
	#je	(".Ldone_4x");

	vaf	($xa0,$xa1,@K[0]);
	vaf	($xb0,$xb1,@K[1]);
	vaf	($xc0,$xc1,@K[2]);
	vaf	($xd0,$xd1,@K[3]);

	vperm	($xa0,$xa0,$xa0,$beperm);
	vperm	($xb0,$xb0,$xb0,$beperm);
	vperm	($xc0,$xc0,$xc0,$beperm);
	vperm	($xd0,$xd0,$xd0,$beperm);

&{$z?	\&clgfi:\&clfi} ($len,0x40);
	jl	(".Ltail_4x");

	vlm	($xt0,$xt3,"0($inp)");

	vx	($xt0,$xt0,$xa0);
	vx	($xt1,$xt1,$xb0);
	vx	($xt2,$xt2,$xc0);
	vx	($xt3,$xt3,$xd0);

	vstm	($xt0,$xt3,"0($out)");

	la	($inp,"0x40($inp)");
	la	($out,"0x40($out)");
&{$z?	\&aghi:\&ahi}	($len,-0x40);
	je	(".Ldone_4x");

	vaf	($xa0,$xa2,@K[0]);
	vaf	($xb0,$xb2,@K[1]);
	vaf	($xc0,$xc2,@K[2]);
	vaf	($xd0,$xd2,@K[3]);

	vperm	($xa0,$xa0,$xa0,$beperm);
	vperm	($xb0,$xb0,$xb0,$beperm);
	vperm	($xc0,$xc0,$xc0,$beperm);
	vperm	($xd0,$xd0,$xd0,$beperm);

&{$z?	\&clgfi:\&clfi} ($len,0x40);
	jl	(".Ltail_4x");

	vlm	($xt0,$xt3,"0($inp)");

	vx	($xt0,$xt0,$xa0);
	vx	($xt1,$xt1,$xb0);
	vx	($xt2,$xt2,$xc0);
	vx	($xt3,$xt3,$xd0);

	vstm	($xt0,$xt3,"0($out)");

	la	($inp,"0x40($inp)");
	la	($out,"0x40($out)");
&{$z?	\&aghi:\&ahi}	($len,-0x40);
	je	(".Ldone_4x");

	vaf	($xa0,$xa3,@K[0]);
	vaf	($xb0,$xb3,@K[1]);
	vaf	($xc0,$xc3,@K[2]);
	vaf	($xd0,$xd3,@K[3]);

	vperm	($xa0,$xa0,$xa0,$beperm);
	vperm	($xb0,$xb0,$xb0,$beperm);
	vperm	($xc0,$xc0,$xc0,$beperm);
	vperm	($xd0,$xd0,$xd0,$beperm);

&{$z?	\&clgfi:\&clfi} ($len,0x40);
	jl	(".Ltail_4x");

	vlm	($xt0,$xt3,"0($inp)");

	vx	($xt0,$xt0,$xa0);
	vx	($xt1,$xt1,$xb0);
	vx	($xt2,$xt2,$xc0);
	vx	($xt3,$xt3,$xd0);

	vstm	($xt0,$xt3,"0($out)");

	#la	$inp,0x40($inp));
	#la	$out,0x40($out));
	#lhi	%r0,10);
	#&{$z?	\&aghi:\&ahi}	$len,-0x40);
	#jne	.Loop_outer_4x);

LABEL	(".Ldone_4x");
if (!$z) {
	ld	("%f4","$FRAME+16*$SIZE_T+2*8($sp)");
	ld	("%f6","$FRAME+16*$SIZE_T+3*8($sp)");
} else {
	ld	("%f8","$stdframe+8*0($sp)");
	ld	("%f9","$stdframe+8*1($sp)");
	ld	("%f10","$stdframe+8*2($sp)");
	ld	("%f11","$stdframe+8*3($sp)");
	ld	("%f12","$stdframe+8*4($sp)");
	ld	("%f13","$stdframe+8*5($sp)");
	ld	("%f14","$stdframe+8*6($sp)");
	ld	("%f15","$stdframe+8*7($sp)");
}
&{$z?	\&lmg:\&lm}	("%r6","%r7","$FRAME+6*$SIZE_T($sp)");
	la	($sp,"$FRAME($sp)");
	br	("%r14");

ALIGN	(16);
LABEL	(".Ltail_4x");
if (!$z) {
	vlr	($xt0,$xb0);
	ld	("%f4","$FRAME+16*$SIZE_T+2*8($sp)");
	ld	("%f6","$FRAME+16*$SIZE_T+3*8($sp)");

	vst	($xa0,"$stdframe+0x00($sp)");
	vst	($xt0,"$stdframe+0x10($sp)");
	vst	($xc0,"$stdframe+0x20($sp)");
	vst	($xd0,"$stdframe+0x30($sp)");
} else {
	vlr	($xt0,$xc0);
	ld	("%f8","$stdframe+8*0($sp)");
	ld	("%f9","$stdframe+8*1($sp)");
	ld	("%f10","$stdframe+8*2($sp)");
	ld	("%f11","$stdframe+8*3($sp)");
	vlr	($xt1,$xd0);
	ld	("%f12","$stdframe+8*4($sp)");
	ld	("%f13","$stdframe+8*5($sp)");
	ld	("%f14","$stdframe+8*6($sp)");
	ld	("%f15","$stdframe+8*7($sp)");

	vst	($xa0,"$stdframe+0x00($sp)");
	vst	($xb0,"$stdframe+0x10($sp)");
	vst	($xt0,"$stdframe+0x20($sp)");
	vst	($xt1,"$stdframe+0x30($sp)");
}
	lghi	("%r1",0);

LABEL	(".Loop_tail_4x");
	llgc	("%r5","0(%r1,$inp)");
	llgc	("%r6","$stdframe(%r1,$sp)");
	xr	("%r6","%r5");
	stc	("%r6","0(%r1,$out)");
	la	("%r1","1(%r1)");
	brct	($len,".Loop_tail_4x");

&{$z?	\&lmg:\&lm}	("%r6","%r7","$FRAME+6*$SIZE_T($sp)");
	la	($sp,"$FRAME($sp)");
	br	("%r14");
SIZE	("ChaCha20_ctr32_4x",".-ChaCha20_ctr32_4x");
}

########################################################################
# 6x"horizontal" layout is optimal fit for the platform in its current
# shape, more specifically for given vector instructions' latency. Well,
# computational part of 8x"vertical" would be faster, but it consumes
# all registers and dealing with that will diminish the return...
#
{
my ($a0,$b0,$c0,$d0, $a1,$b1,$c1,$d1,
    $a2,$b2,$c2,$d2, $a3,$b3,$c3,$d3,
    $a4,$b4,$c4,$d4, $a5,$b5,$c5,$d5)=map("%v$_",(0..23));
my @K=map("%v$_",(27,24..26));
my ($t0,$t1,$t2,$t3)=map("%v$_",27..30);
my $beperm="%v31";
my $FRAME=$stdframe + 4*16;

GLOBL	("ChaCha20_ctr32_vx");
ALIGN	(32);
LABEL	("ChaCha20_ctr32_vx");
LABEL	(".LChaCha20_ctr32_vx");
&{$z?	\&clgfi:\&clfi}	($len,256);
	jle	(".LChaCha20_ctr32_4x");
&{$z?	\&stmg:\&stm}	("%r6","%r7","6*$SIZE_T($sp)");
if (!$z) {
	std	("%f4","16*$SIZE_T+2*8($sp)");
	std	("%f6","16*$SIZE_T+3*8($sp)");
}
&{$z?	\&lghi:\&lhi}	("%r1",-$FRAME);
	lgr	("%r0",$sp);
	la	($sp,"0(%r1,$sp)");
&{$z?	\&stg:\&st}	("%r0","0($sp)");	# back-chain
if ($z) {
	std	("%f8","$FRAME-8*8($sp)");
	std	("%f9","$FRAME-8*7($sp)");
	std	("%f10","$FRAME-8*6($sp)");
	std	("%f11","$FRAME-8*5($sp)");
	std	("%f12","$FRAME-8*4($sp)");
	std	("%f13","$FRAME-8*3($sp)");
	std	("%f14","$FRAME-8*2($sp)");
	std	("%f15","$FRAME-8*1($sp)");
}
	larl	("%r7",".Lsigma");
	lhi	("%r0",10);

	vlm	(@K[1],@K[2],"0($key)");	# load key
	vl	(@K[3],"0($counter)");		# load counter

	vlm	(@K[0],"$beperm","0(%r7)");	# load sigma, increments, ...

LABEL	(".Loop_outer_vx");
	vlr	($a0,@K[0]);
	vlr	($b0,@K[1]);
	vlr	($a1,@K[0]);
	vlr	($b1,@K[1]);
	vlr	($a2,@K[0]);
	vlr	($b2,@K[1]);
	vlr	($a3,@K[0]);
	vlr	($b3,@K[1]);
	vlr	($a4,@K[0]);
	vlr	($b4,@K[1]);
	vlr	($a5,@K[0]);
	vlr	($b5,@K[1]);

	vlr	($d0,@K[3]);
	vaf	($d1,@K[3],$t1);		# K[3]+1
	vaf	($d2,@K[3],$t2);		# K[3]+2
	vaf	($d3,@K[3],$t3);		# K[3]+3
	vaf	($d4,$d2,$t2);			# K[3]+4
	vaf	($d5,$d2,$t3);			# K[3]+5

	vlr	($c0,@K[2]);
	vlr	($c1,@K[2]);
	vlr	($c2,@K[2]);
	vlr	($c3,@K[2]);
	vlr	($c4,@K[2]);
	vlr	($c5,@K[2]);

	vlr	($t1,$d1);
	vlr	($t2,$d2);
	vlr	($t3,$d3);

ALIGN	(4);
LABEL	(".Loop_vx");

	VX_ROUND($a0,$a1,$a2,$a3,$a4,$a5,
		 $b0,$b1,$b2,$b3,$b4,$b5,
		 $c0,$c1,$c2,$c3,$c4,$c5,
		 $d0,$d1,$d2,$d3,$d4,$d5,
		 0);

	VX_ROUND($a0,$a1,$a2,$a3,$a4,$a5,
		 $b0,$b1,$b2,$b3,$b4,$b5,
		 $c0,$c1,$c2,$c3,$c4,$c5,
		 $d0,$d1,$d2,$d3,$d4,$d5,
		 1);

	brct	("%r0",".Loop_vx");

	vaf	($a0,$a0,@K[0]);
	vaf	($b0,$b0,@K[1]);
	vaf	($c0,$c0,@K[2]);
	vaf	($d0,$d0,@K[3]);
	vaf	($a1,$a1,@K[0]);
	vaf	($d1,$d1,$t1);			# +K[3]+1

	vperm	($a0,$a0,$a0,$beperm);
	vperm	($b0,$b0,$b0,$beperm);
	vperm	($c0,$c0,$c0,$beperm);
	vperm	($d0,$d0,$d0,$beperm);

&{$z?	\&clgfi:\&clfi}	($len,0x40);
	jl	(".Ltail_vx");

	vaf	($d2,$d2,$t2);			# +K[3]+2
	vaf	($d3,$d3,$t3);			# +K[3]+3
	vlm	($t0,$t3,"0($inp)");

	vx	($a0,$a0,$t0);
	vx	($b0,$b0,$t1);
	vx	($c0,$c0,$t2);
	vx	($d0,$d0,$t3);

	vlm	(@K[0],$t3,"0(%r7)");		# re-load sigma and increments

	vstm	($a0,$d0,"0($out)");

	la	($inp,"0x40($inp)");
	la	($out,"0x40($out)");
&{$z?	\&aghi:\&ahi}	($len,-0x40);
	je	(".Ldone_vx");

	vaf	($b1,$b1,@K[1]);
	vaf	($c1,$c1,@K[2]);

	vperm	($a0,$a1,$a1,$beperm);
	vperm	($b0,$b1,$b1,$beperm);
	vperm	($c0,$c1,$c1,$beperm);
	vperm	($d0,$d1,$d1,$beperm);

&{$z?	\&clgfi:\&clfi} ($len,0x40);
	jl	(".Ltail_vx");

	vlm	($a1,$d1,"0($inp)");

	vx	($a0,$a0,$a1);
	vx	($b0,$b0,$b1);
	vx	($c0,$c0,$c1);
	vx	($d0,$d0,$d1);

	vstm	($a0,$d0,"0($out)");

	la	($inp,"0x40($inp)");
	la	($out,"0x40($out)");
&{$z?	\&aghi:\&ahi}	($len,-0x40);
	je	(".Ldone_vx");

	vaf	($a2,$a2,@K[0]);
	vaf	($b2,$b2,@K[1]);
	vaf	($c2,$c2,@K[2]);

	vperm	($a0,$a2,$a2,$beperm);
	vperm	($b0,$b2,$b2,$beperm);
	vperm	($c0,$c2,$c2,$beperm);
	vperm	($d0,$d2,$d2,$beperm);

&{$z?	\&clgfi:\&clfi}	($len,0x40);
	jl	(".Ltail_vx");

	vlm	($a1,$d1,"0($inp)");

	vx	($a0,$a0,$a1);
	vx	($b0,$b0,$b1);
	vx	($c0,$c0,$c1);
	vx	($d0,$d0,$d1);

	vstm	($a0,$d0,"0($out)");

	la	($inp,"0x40($inp)");
	la	($out,"0x40($out)");
&{$z?	\&aghi:\&ahi}	($len,-0x40);
	je	(".Ldone_vx");

	vaf	($a3,$a3,@K[0]);
	vaf	($b3,$b3,@K[1]);
	vaf	($c3,$c3,@K[2]);
	vaf	($d2,@K[3],$t3);		# K[3]+3

	vperm	($a0,$a3,$a3,$beperm);
	vperm	($b0,$b3,$b3,$beperm);
	vperm	($c0,$c3,$c3,$beperm);
	vperm	($d0,$d3,$d3,$beperm);

&{$z?	\&clgfi:\&clfi}	($len,0x40);
	jl	(".Ltail_vx");

	vaf	($d3,$d2,$t1);			# K[3]+4
	vlm	($a1,$d1,"0($inp)");

	vx	($a0,$a0,$a1);
	vx	($b0,$b0,$b1);
	vx	($c0,$c0,$c1);
	vx	($d0,$d0,$d1);

	vstm	($a0,$d0,"0($out)");

	la	($inp,"0x40($inp)");
	la	($out,"0x40($out)");
&{$z?	\&aghi:\&ahi}	($len,-0x40);
	je	(".Ldone_vx");

	vaf	($a4,$a4,@K[0]);
	vaf	($b4,$b4,@K[1]);
	vaf	($c4,$c4,@K[2]);
	vaf	($d4,$d4,$d3);			# +K[3]+4
	vaf	($d3,$d3,$t1);			# K[3]+5
	vaf	(@K[3],$d2,$t3);		# K[3]+=6

	vperm	($a0,$a4,$a4,$beperm);
	vperm	($b0,$b4,$b4,$beperm);
	vperm	($c0,$c4,$c4,$beperm);
	vperm	($d0,$d4,$d4,$beperm);

&{$z?	\&clgfi:\&clfi}	($len,0x40);
	jl	(".Ltail_vx");

	vlm	($a1,$d1,"0($inp)");

	vx	($a0,$a0,$a1);
	vx	($b0,$b0,$b1);
	vx	($c0,$c0,$c1);
	vx	($d0,$d0,$d1);

	vstm	($a0,$d0,"0($out)");

	la	($inp,"0x40($inp)");
	la	($out,"0x40($out)");
&{$z?	\&aghi:\&ahi}	($len,-0x40);
	je	(".Ldone_vx");

	vaf	($a5,$a5,@K[0]);
	vaf	($b5,$b5,@K[1]);
	vaf	($c5,$c5,@K[2]);
	vaf	($d5,$d5,$d3);			# +K[3]+5

	vperm	($a0,$a5,$a5,$beperm);
	vperm	($b0,$b5,$b5,$beperm);
	vperm	($c0,$c5,$c5,$beperm);
	vperm	($d0,$d5,$d5,$beperm);

&{$z?	\&clgfi:\&clfi} ($len,0x40);
	jl	(".Ltail_vx");

	vlm	($a1,$d1,"0($inp)");

	vx	($a0,$a0,$a1);
	vx	($b0,$b0,$b1);
	vx	($c0,$c0,$c1);
	vx	($d0,$d0,$d1);

	vstm	($a0,$d0,"0($out)");

	la	($inp,"0x40($inp)");
	la	($out,"0x40($out)");
	lhi	("%r0",10);
&{$z?	\&aghi:\&ahi}	($len,-0x40);
	jne	(".Loop_outer_vx");

LABEL	(".Ldone_vx");
if (!$z) {
	ld	("%f4","$FRAME+16*$SIZE_T+2*8($sp)");
	ld	("%f6","$FRAME+16*$SIZE_T+3*8($sp)");
} else {
	ld	("%f8","$FRAME-8*8($sp)");
	ld	("%f9","$FRAME-8*7($sp)");
	ld	("%f10","$FRAME-8*6($sp)");
	ld	("%f11","$FRAME-8*5($sp)");
	ld	("%f12","$FRAME-8*4($sp)");
	ld	("%f13","$FRAME-8*3($sp)");
	ld	("%f14","$FRAME-8*2($sp)");
	ld	("%f15","$FRAME-8*1($sp)");
}
&{$z?	\&lmg:\&lm}	("%r6","%r7","$FRAME+6*$SIZE_T($sp)");
	la	($sp,"$FRAME($sp)");
	br	("%r14");

ALIGN	(16);
LABEL	(".Ltail_vx");
if (!$z) {
	ld	("%f4","$FRAME+16*$SIZE_T+2*8($sp)");
	ld	("%f6","$FRAME+16*$SIZE_T+3*8($sp)");
} else {
	ld	("%f8","$FRAME-8*8($sp)");
	ld	("%f9","$FRAME-8*7($sp)");
	ld	("%f10","$FRAME-8*6($sp)");
	ld	("%f11","$FRAME-8*5($sp)");
	ld	("%f12","$FRAME-8*4($sp)");
	ld	("%f13","$FRAME-8*3($sp)");
	ld	("%f14","$FRAME-8*2($sp)");
	ld	("%f15","$FRAME-8*1($sp)");
}
	vstm	($a0,$d0,"$stdframe($sp)");
	lghi	("%r1",0);

LABEL	(".Loop_tail_vx");
	llgc	("%r5","0(%r1,$inp)");
	llgc	("%r6","$stdframe(%r1,$sp)");
	xr	("%r6","%r5");
	stc	("%r6","0(%r1,$out)");
	la	("%r1","1(%r1)");
	brct	($len,".Loop_tail_vx");

&{$z?	\&lmg:\&lm}	("%r6","%r7","$FRAME+6*$SIZE_T($sp)");
	la	($sp,"$FRAME($sp)");
	br	("%r14");
SIZE	("ChaCha20_ctr32_vx",".-ChaCha20_ctr32_vx");
}
################

ALIGN	(32);
LABEL	(".Lsigma");
LONG	(0x61707865,0x3320646e,0x79622d32,0x6b206574);	# endian-neutral sigma
LONG	(1,0,0,0);
LONG	(2,0,0,0);
LONG	(3,0,0,0);
LONG	(0x03020100,0x07060504,0x0b0a0908,0x0f0e0d0c);	# byte swap

LONG	(0,1,2,3);
LONG	(0x61707865,0x61707865,0x61707865,0x61707865);	# smashed sigma
LONG	(0x3320646e,0x3320646e,0x3320646e,0x3320646e);
LONG	(0x79622d32,0x79622d32,0x79622d32,0x79622d32);
LONG	(0x6b206574,0x6b206574,0x6b206574,0x6b206574);

ASCIZ	("\"ChaCha20 for s390x, CRYPTOGAMS by <appro\@openssl.org>\"");
ALIGN	(4);

PERLASM_END();