openssl/crypto/aes/asm/aesv8-armx.pl

#! /usr/bin/env perl
# Copyright 2014-2023 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the Apache License 2.0 (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html

#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
# This module implements support for ARMv8 AES instructions. The
# module is endian-agnostic in sense that it supports both big- and
# little-endian cases. As does it support both 32- and 64-bit modes
# of operation. Latter is achieved by limiting amount of utilized
# registers to 16, which implies additional NEON load and integer
# instructions. This has no effect on mighty Apple A7, where results
# are literally equal to the theoretical estimates based on AES
# instruction latencies and issue rates. On Cortex-A53, an in-order
# execution core, this costs up to 10-15%, which is partially
# compensated by implementing dedicated code path for 128-bit
# CBC encrypt case. On Cortex-A57 parallelizable mode performance
# seems to be limited by sheer amount of NEON instructions...
#
# April 2019
#
# Key to performance of parallelize-able modes is round instruction
# interleaving. But which factor to use? There is optimal one for
# each combination of instruction latency and issue rate, beyond
# which increasing interleave factor doesn't pay off. While on cons
# side we have code size increase and resource waste on platforms for
# which interleave factor is too high. In other words you want it to
# be just right. So far interleave factor of 3x was serving well all
# platforms. But for ThunderX2 optimal interleave factor was measured
# to be 5x...
#
# Performance in cycles per byte processed with 128-bit key:
#
#		CBC enc		CBC dec		CTR
# Apple A7	2.39		1.20		1.20
# Cortex-A53	1.32		1.17/1.29(**)	1.36/1.46
# Cortex-A57(*)	1.95		0.82/0.85	0.89/0.93
# Cortex-A72	1.33		0.85/0.88	0.92/0.96
# Denver	1.96		0.65/0.86	0.76/0.80
# Mongoose	1.33		1.23/1.20	1.30/1.20
# Kryo		1.26		0.87/0.94	1.00/1.00
# ThunderX2	5.95		1.25		1.30
#
# (*)	original 3.64/1.34/1.32 results were for r0p0 revision
#	and are still same even for updated module;
# (**)	numbers after slash are for 32-bit code, which is 3x-
#	interleaved;

# $output is the last argument if it looks like a file (it has an extension)
# $flavour is the first argument if it doesn't look like a file
$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
$flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;

$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
die "can't locate arm-xlate.pl";

open OUT,"| \"$^X\" $xlate $flavour \"$output\""
    or die "can't call $xlate: $!";
*STDOUT=*OUT;

$prefix="aes_v8";

$_byte = ($flavour =~ /win/ ? "DCB" : ".byte");

$code=<<___;
#include "arm_arch.h"

#if __ARM_MAX_ARCH__>=7
___
$code.=".arch	armv8-a+crypto\n.text\n"		if ($flavour =~ /64/);
$code.=<<___						if ($flavour !~ /64/);
.arch	armv7-a	// don't confuse not-so-latest binutils with argv8 :-)
.fpu	neon
#ifdef	__thumb2__
.syntax	unified
.thumb
# define INST(a,b,c,d)	$_byte	c,d|0xc,a,b
#else
.code	32
# define INST(a,b,c,d)	$_byte	a,b,c,d
#endif

.text
___

# Assembler mnemonics are an eclectic mix of 32- and 64-bit syntax,
# NEON is mostly 32-bit mnemonics, integer - mostly 64. Goal is to
# maintain both 32- and 64-bit codes within single module and
# transliterate common code to either flavour with regex vodoo.
#
{{{
my ($inp,$bits,$out,$ptr,$rounds)=("x0","w1","x2","x3","w12");
my ($zero,$rcon,$mask,$in0,$in1,$tmp,$key)=
	$flavour=~/64/? map("q$_",(0..6)) : map("q$_",(0..3,8..10));


$code.=<<___;
.align	5
.Lrcon:
.long	0x01,0x01,0x01,0x01
.long	0x0c0f0e0d,0x0c0f0e0d,0x0c0f0e0d,0x0c0f0e0d	// rotate-n-splat
.long	0x1b,0x1b,0x1b,0x1b

.globl	${prefix}_set_encrypt_key
.type	${prefix}_set_encrypt_key,%function
.align	5
${prefix}_set_encrypt_key:
.Lenc_key:
___
$code.=<<___	if ($flavour =~ /64/);
	AARCH64_VALID_CALL_TARGET
	// Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later.
	stp	x29,x30,[sp,#-16]!
	add	x29,sp,#0
___
$code.=<<___;
	mov	$ptr,#-1
	cmp	$inp,#0
	b.eq	.Lenc_key_abort
	cmp	$out,#0
	b.eq	.Lenc_key_abort
	mov	$ptr,#-2
	cmp	$bits,#128
	b.lt	.Lenc_key_abort
	cmp	$bits,#256
	b.gt	.Lenc_key_abort
	tst	$bits,#0x3f
	b.ne	.Lenc_key_abort

	adr	$ptr,.Lrcon
	cmp	$bits,#192

	veor	$zero,$zero,$zero
	vld1.8	{$in0},[$inp],#16
	mov	$bits,#8		// reuse $bits
	vld1.32	{$rcon,$mask},[$ptr],#32

	b.lt	.Loop128
	b.eq	.L192
	b	.L256

.align	4
.Loop128:
	vtbl.8	$key,{$in0},$mask
	vext.8	$tmp,$zero,$in0,#12
	vst1.32	{$in0},[$out],#16
	aese	$key,$zero
	subs	$bits,$bits,#1

	veor	$in0,$in0,$tmp
	vext.8	$tmp,$zero,$tmp,#12
	veor	$in0,$in0,$tmp
	vext.8	$tmp,$zero,$tmp,#12
	 veor	$key,$key,$rcon
	veor	$in0,$in0,$tmp
	vshl.u8	$rcon,$rcon,#1
	veor	$in0,$in0,$key
	b.ne	.Loop128

	vld1.32	{$rcon},[$ptr]

	vtbl.8	$key,{$in0},$mask
	vext.8	$tmp,$zero,$in0,#12
	vst1.32	{$in0},[$out],#16
	aese	$key,$zero

	veor	$in0,$in0,$tmp
	vext.8	$tmp,$zero,$tmp,#12
	veor	$in0,$in0,$tmp
	vext.8	$tmp,$zero,$tmp,#12
	 veor	$key,$key,$rcon
	veor	$in0,$in0,$tmp
	vshl.u8	$rcon,$rcon,#1
	veor	$in0,$in0,$key

	vtbl.8	$key,{$in0},$mask
	vext.8	$tmp,$zero,$in0,#12
	vst1.32	{$in0},[$out],#16
	aese	$key,$zero

	veor	$in0,$in0,$tmp
	vext.8	$tmp,$zero,$tmp,#12
	veor	$in0,$in0,$tmp
	vext.8	$tmp,$zero,$tmp,#12
	 veor	$key,$key,$rcon
	veor	$in0,$in0,$tmp
	veor	$in0,$in0,$key
	vst1.32	{$in0},[$out]
	add	$out,$out,#0x50

	mov	$rounds,#10
	b	.Ldone

.align	4
.L192:
	vld1.8	{$in1},[$inp],#8
	vmov.i8	$key,#8			// borrow $key
	vst1.32	{$in0},[$out],#16
	vsub.i8	$mask,$mask,$key	// adjust the mask

.Loop192:
	vtbl.8	$key,{$in1},$mask
	vext.8	$tmp,$zero,$in0,#12
#ifdef __ARMEB__
	vst1.32	{$in1},[$out],#16
	sub	$out,$out,#8
#else
	vst1.32	{$in1},[$out],#8
#endif
	aese	$key,$zero
	subs	$bits,$bits,#1

	veor	$in0,$in0,$tmp
	vext.8	$tmp,$zero,$tmp,#12
	veor	$in0,$in0,$tmp
	vext.8	$tmp,$zero,$tmp,#12
	veor	$in0,$in0,$tmp

	vdup.32	$tmp,${in0}[3]
	veor	$tmp,$tmp,$in1
	 veor	$key,$key,$rcon
	vext.8	$in1,$zero,$in1,#12
	vshl.u8	$rcon,$rcon,#1
	veor	$in1,$in1,$tmp
	veor	$in0,$in0,$key
	veor	$in1,$in1,$key
	vst1.32	{$in0},[$out],#16
	b.ne	.Loop192

	mov	$rounds,#12
	add	$out,$out,#0x20
	b	.Ldone

.align	4
.L256:
	vld1.8	{$in1},[$inp]
	mov	$bits,#7
	mov	$rounds,#14
	vst1.32	{$in0},[$out],#16

.Loop256:
	vtbl.8	$key,{$in1},$mask
	vext.8	$tmp,$zero,$in0,#12
	vst1.32	{$in1},[$out],#16
	aese	$key,$zero
	subs	$bits,$bits,#1

	veor	$in0,$in0,$tmp
	vext.8	$tmp,$zero,$tmp,#12
	veor	$in0,$in0,$tmp
	vext.8	$tmp,$zero,$tmp,#12
	 veor	$key,$key,$rcon
	veor	$in0,$in0,$tmp
	vshl.u8	$rcon,$rcon,#1
	veor	$in0,$in0,$key
	vst1.32	{$in0},[$out],#16
	b.eq	.Ldone

	vdup.32	$key,${in0}[3]		// just splat
	vext.8	$tmp,$zero,$in1,#12
	aese	$key,$zero

	veor	$in1,$in1,$tmp
	vext.8	$tmp,$zero,$tmp,#12
	veor	$in1,$in1,$tmp
	vext.8	$tmp,$zero,$tmp,#12
	veor	$in1,$in1,$tmp

	veor	$in1,$in1,$key
	b	.Loop256

.Ldone:
	str	$rounds,[$out]
	mov	$ptr,#0

.Lenc_key_abort:
	mov	x0,$ptr			// return value
	`"ldr	x29,[sp],#16"		if ($flavour =~ /64/)`
	ret
.size	${prefix}_set_encrypt_key,.-${prefix}_set_encrypt_key

.globl	${prefix}_set_decrypt_key
.type	${prefix}_set_decrypt_key,%function
.align	5
${prefix}_set_decrypt_key:
___
$code.=<<___	if ($flavour =~ /64/);
	AARCH64_SIGN_LINK_REGISTER
	stp	x29,x30,[sp,#-16]!
	add	x29,sp,#0
___
$code.=<<___	if ($flavour !~ /64/);
	stmdb	sp!,{r4,lr}
___
$code.=<<___;
	bl	.Lenc_key

	cmp	x0,#0
	b.ne	.Ldec_key_abort

	sub	$out,$out,#240		// restore original $out
	mov	x4,#-16
	add	$inp,$out,x12,lsl#4	// end of key schedule

	vld1.32	{v0.16b},[$out]
	vld1.32	{v1.16b},[$inp]
	vst1.32	{v0.16b},[$inp],x4
	vst1.32	{v1.16b},[$out],#16

.Loop_imc:
	vld1.32	{v0.16b},[$out]
	vld1.32	{v1.16b},[$inp]
	aesimc	v0.16b,v0.16b
	aesimc	v1.16b,v1.16b
	vst1.32	{v0.16b},[$inp],x4
	vst1.32	{v1.16b},[$out],#16
	cmp	$inp,$out
	b.hi	.Loop_imc

	vld1.32	{v0.16b},[$out]
	aesimc	v0.16b,v0.16b
	vst1.32	{v0.16b},[$inp]

	eor	x0,x0,x0		// return value
.Ldec_key_abort:
___
$code.=<<___	if ($flavour !~ /64/);
	ldmia	sp!,{r4,pc}
___
$code.=<<___	if ($flavour =~ /64/);
	ldp	x29,x30,[sp],#16
	AARCH64_VALIDATE_LINK_REGISTER
	ret
___
$code.=<<___;
.size	${prefix}_set_decrypt_key,.-${prefix}_set_decrypt_key
___
}}}
{{{
sub gen_block () {
my $dir = shift;
my ($e,$mc) = $dir eq "en" ? ("e","mc") : ("d","imc");
my ($inp,$out,$key)=map("x$_",(0..2));
my $rounds="w3";
my ($rndkey0,$rndkey1,$inout)=map("q$_",(0..3));

$code.=<<___;
.globl	${prefix}_${dir}crypt
.type	${prefix}_${dir}crypt,%function
.align	5
${prefix}_${dir}crypt:
___
$code.=<<___	if ($flavour =~ /64/);
	AARCH64_VALID_CALL_TARGET
___
$code.=<<___;
	ldr	$rounds,[$key,#240]
	vld1.32	{$rndkey0},[$key],#16
	vld1.8	{$inout},[$inp]
	sub	$rounds,$rounds,#2
	vld1.32	{$rndkey1},[$key],#16

.Loop_${dir}c:
	aes$e	$inout,$rndkey0
	aes$mc	$inout,$inout
	vld1.32	{$rndkey0},[$key],#16
	subs	$rounds,$rounds,#2
	aes$e	$inout,$rndkey1
	aes$mc	$inout,$inout
	vld1.32	{$rndkey1},[$key],#16
	b.gt	.Loop_${dir}c

	aes$e	$inout,$rndkey0
	aes$mc	$inout,$inout
	vld1.32	{$rndkey0},[$key]
	aes$e	$inout,$rndkey1
	veor	$inout,$inout,$rndkey0

	vst1.8	{$inout},[$out]
	ret
.size	${prefix}_${dir}crypt,.-${prefix}_${dir}crypt
___
}
&gen_block("en");
&gen_block("de");
}}}

# Performance in cycles per byte.
# Processed with AES-ECB different key size.
# It shows the value before and after optimization as below:
# (before/after):
#
#		AES-128-ECB		AES-192-ECB		AES-256-ECB
# Cortex-A57	1.85/0.82		2.16/0.96		2.47/1.10
# Cortex-A72	1.64/0.85		1.82/0.99		2.13/1.14

# Optimization is implemented by loop unrolling and interleaving.
# Commonly, we choose the unrolling factor as 5, if the input
# data size smaller than 5 blocks, but not smaller than 3 blocks,
# choose 3 as the unrolling factor.
# If the input data size dsize >= 5*16 bytes, then take 5 blocks
# as one iteration, every loop the left size lsize -= 5*16.
# If 5*16 > lsize >= 3*16 bytes, take 3 blocks as one iteration,
# every loop lsize -=3*16.
# If lsize < 3*16 bytes, treat them as the tail, interleave the
# two blocks AES instructions.
# There is one special case, if the original input data size dsize
# = 16 bytes, we will treat it separately to improve the
# performance: one independent code block without LR, FP load and
# store, just looks like what the original ECB implementation does.

{{{
my ($inp,$out,$len,$key)=map("x$_",(0..3));
my ($enc,$rounds,$cnt,$key_,$step)=("w4","w5","w6","x7","x8");
my ($dat0,$dat1,$in0,$in1,$tmp0,$tmp1,$tmp2,$rndlast)=map("q$_",(0..7));

my ($dat,$tmp,$rndzero_n_last)=($dat0,$tmp0,$tmp1);

### q7	last round key
### q10-q15	q7 Last 7 round keys
### q8-q9	preloaded round keys except last 7 keys for big size
### q5, q6, q8-q9	preloaded round keys except last 7 keys for only 16 byte

{
my ($dat2,$in2,$tmp2)=map("q$_",(10,11,9));

my ($dat3,$in3,$tmp3);	# used only in 64-bit mode
my ($dat4,$in4,$tmp4);
if ($flavour =~ /64/) {
    ($dat2,$dat3,$dat4,$in2,$in3,$in4,$tmp3,$tmp4)=map("q$_",(16..23));
}

$code.=<<___;
.globl	${prefix}_ecb_encrypt
.type	${prefix}_ecb_encrypt,%function
.align	5
${prefix}_ecb_encrypt:
___
$code.=<<___	if ($flavour =~ /64/);
	AARCH64_VALID_CALL_TARGET
	subs	$len,$len,#16
	// Original input data size bigger than 16, jump to big size processing.
	b.ne    .Lecb_big_size
	vld1.8	{$dat0},[$inp]
	cmp	$enc,#0					// en- or decrypting?
	ldr	$rounds,[$key,#240]
	vld1.32	{q5-q6},[$key],#32			// load key schedule...

	b.eq .Lecb_small_dec
	aese	$dat0,q5
	aesmc	$dat0,$dat0
	vld1.32	{q8-q9},[$key],#32			// load key schedule...
	aese	$dat0,q6
	aesmc	$dat0,$dat0
	subs	$rounds,$rounds,#10			// if rounds==10, jump to aes-128-ecb processing
	b.eq    .Lecb_128_enc
.Lecb_round_loop:
	aese	$dat0,q8
	aesmc	$dat0,$dat0
	vld1.32	{q8},[$key],#16				// load key schedule...
	aese	$dat0,q9
	aesmc	$dat0,$dat0
	vld1.32	{q9},[$key],#16				// load key schedule...
	subs	$rounds,$rounds,#2			// bias
	b.gt    .Lecb_round_loop
.Lecb_128_enc:
	vld1.32	{q10-q11},[$key],#32		// load key schedule...
	aese	$dat0,q8
	aesmc	$dat0,$dat0
	aese	$dat0,q9
	aesmc	$dat0,$dat0
	vld1.32	{q12-q13},[$key],#32		// load key schedule...
	aese	$dat0,q10
	aesmc	$dat0,$dat0
	aese	$dat0,q11
	aesmc	$dat0,$dat0
	vld1.32	{q14-q15},[$key],#32		// load key schedule...
	aese	$dat0,q12
	aesmc	$dat0,$dat0
	aese	$dat0,q13
	aesmc	$dat0,$dat0
	vld1.32	{$rndlast},[$key]
	aese	$dat0,q14
	aesmc	$dat0,$dat0
	aese	$dat0,q15
	veor	$dat0,$dat0,$rndlast
	vst1.8	{$dat0},[$out]
	b	.Lecb_Final_abort
.Lecb_small_dec:
	aesd	$dat0,q5
	aesimc	$dat0,$dat0
	vld1.32	{q8-q9},[$key],#32			// load key schedule...
	aesd	$dat0,q6
	aesimc	$dat0,$dat0
	subs	$rounds,$rounds,#10			// bias
	b.eq    .Lecb_128_dec
.Lecb_dec_round_loop:
	aesd	$dat0,q8
	aesimc	$dat0,$dat0
	vld1.32	{q8},[$key],#16				// load key schedule...
	aesd	$dat0,q9
	aesimc	$dat0,$dat0
	vld1.32	{q9},[$key],#16				// load key schedule...
	subs	$rounds,$rounds,#2			// bias
	b.gt    .Lecb_dec_round_loop
.Lecb_128_dec:
	vld1.32	{q10-q11},[$key],#32		// load key schedule...
	aesd	$dat0,q8
	aesimc	$dat0,$dat0
	aesd	$dat0,q9
	aesimc	$dat0,$dat0
	vld1.32	{q12-q13},[$key],#32		// load key schedule...
	aesd	$dat0,q10
	aesimc	$dat0,$dat0
	aesd	$dat0,q11
	aesimc	$dat0,$dat0
	vld1.32	{q14-q15},[$key],#32		// load key schedule...
	aesd	$dat0,q12
	aesimc	$dat0,$dat0
	aesd	$dat0,q13
	aesimc	$dat0,$dat0
	vld1.32	{$rndlast},[$key]
	aesd	$dat0,q14
	aesimc	$dat0,$dat0
	aesd	$dat0,q15
	veor	$dat0,$dat0,$rndlast
	vst1.8	{$dat0},[$out]
	b	.Lecb_Final_abort
.Lecb_big_size:
___
$code.=<<___	if ($flavour =~ /64/);
	stp	x29,x30,[sp,#-16]!
	add	x29,sp,#0
___
$code.=<<___	if ($flavour !~ /64/);
	mov	ip,sp
	stmdb	sp!,{r4-r8,lr}
	vstmdb	sp!,{d8-d15}			@ ABI specification says so
	ldmia	ip,{r4-r5}			@ load remaining args
	subs	$len,$len,#16
___
$code.=<<___;
	mov	$step,#16
	b.lo	.Lecb_done
	cclr	$step,eq

	cmp	$enc,#0					// en- or decrypting?
	ldr	$rounds,[$key,#240]
	and	$len,$len,#-16
	vld1.8	{$dat},[$inp],$step

	vld1.32	{q8-q9},[$key]				// load key schedule...
	sub	$rounds,$rounds,#6
	add	$key_,$key,x5,lsl#4				// pointer to last 7 round keys
	sub	$rounds,$rounds,#2
	vld1.32	{q10-q11},[$key_],#32
	vld1.32	{q12-q13},[$key_],#32
	vld1.32	{q14-q15},[$key_],#32
	vld1.32	{$rndlast},[$key_]

	add	$key_,$key,#32
	mov	$cnt,$rounds
	b.eq	.Lecb_dec

	vld1.8	{$dat1},[$inp],#16
	subs	$len,$len,#32				// bias
	add	$cnt,$rounds,#2
	vorr	$in1,$dat1,$dat1
	vorr	$dat2,$dat1,$dat1
	vorr	$dat1,$dat,$dat
	b.lo	.Lecb_enc_tail

	vorr	$dat1,$in1,$in1
	vld1.8	{$dat2},[$inp],#16
___
$code.=<<___	if ($flavour =~ /64/);
	cmp	$len,#32
	b.lo	.Loop3x_ecb_enc

	vld1.8	{$dat3},[$inp],#16
	vld1.8	{$dat4},[$inp],#16
	sub	$len,$len,#32				// bias
	mov	$cnt,$rounds

.Loop5x_ecb_enc:
	aese	$dat0,q8
	aesmc	$dat0,$dat0
	aese	$dat1,q8
	aesmc	$dat1,$dat1
	aese	$dat2,q8
	aesmc	$dat2,$dat2
	aese	$dat3,q8
	aesmc	$dat3,$dat3
	aese	$dat4,q8
	aesmc	$dat4,$dat4
	vld1.32	{q8},[$key_],#16
	subs	$cnt,$cnt,#2
	aese	$dat0,q9
	aesmc	$dat0,$dat0
	aese	$dat1,q9
	aesmc	$dat1,$dat1
	aese	$dat2,q9
	aesmc	$dat2,$dat2
	aese	$dat3,q9
	aesmc	$dat3,$dat3
	aese	$dat4,q9
	aesmc	$dat4,$dat4
	vld1.32	{q9},[$key_],#16
	b.gt	.Loop5x_ecb_enc

	aese	$dat0,q8
	aesmc	$dat0,$dat0
	aese	$dat1,q8
	aesmc	$dat1,$dat1
	aese	$dat2,q8
	aesmc	$dat2,$dat2
	aese	$dat3,q8
	aesmc	$dat3,$dat3
	aese	$dat4,q8
	aesmc	$dat4,$dat4
	cmp	$len,#0x40					// because .Lecb_enc_tail4x
	sub	$len,$len,#0x50

	aese	$dat0,q9
	aesmc	$dat0,$dat0
	aese	$dat1,q9
	aesmc	$dat1,$dat1
	aese	$dat2,q9
	aesmc	$dat2,$dat2
	aese	$dat3,q9
	aesmc	$dat3,$dat3
	aese	$dat4,q9
	aesmc	$dat4,$dat4
	csel	x6,xzr,$len,gt			// borrow x6, $cnt, "gt" is not typo
	mov	$key_,$key

	aese	$dat0,q10
	aesmc	$dat0,$dat0
	aese	$dat1,q10
	aesmc	$dat1,$dat1
	aese	$dat2,q10
	aesmc	$dat2,$dat2
	aese	$dat3,q10
	aesmc	$dat3,$dat3
	aese	$dat4,q10
	aesmc	$dat4,$dat4
	add	$inp,$inp,x6				// $inp is adjusted in such way that
							// at exit from the loop $dat1-$dat4
							// are loaded with last "words"
	add	x6,$len,#0x60		    // because .Lecb_enc_tail4x

	aese	$dat0,q11
	aesmc	$dat0,$dat0
	aese	$dat1,q11
	aesmc	$dat1,$dat1
	aese	$dat2,q11
	aesmc	$dat2,$dat2
	aese	$dat3,q11
	aesmc	$dat3,$dat3
	aese	$dat4,q11
	aesmc	$dat4,$dat4

	aese	$dat0,q12
	aesmc	$dat0,$dat0
	aese	$dat1,q12
	aesmc	$dat1,$dat1
	aese	$dat2,q12
	aesmc	$dat2,$dat2
	aese	$dat3,q12
	aesmc	$dat3,$dat3
	aese	$dat4,q12
	aesmc	$dat4,$dat4

	aese	$dat0,q13
	aesmc	$dat0,$dat0
	aese	$dat1,q13
	aesmc	$dat1,$dat1
	aese	$dat2,q13
	aesmc	$dat2,$dat2
	aese	$dat3,q13
	aesmc	$dat3,$dat3
	aese	$dat4,q13
	aesmc	$dat4,$dat4

	aese	$dat0,q14
	aesmc	$dat0,$dat0
	aese	$dat1,q14
	aesmc	$dat1,$dat1
	aese	$dat2,q14
	aesmc	$dat2,$dat2
	aese	$dat3,q14
	aesmc	$dat3,$dat3
	aese	$dat4,q14
	aesmc	$dat4,$dat4

	aese	$dat0,q15
	vld1.8	{$in0},[$inp],#16
	aese	$dat1,q15
	vld1.8	{$in1},[$inp],#16
	aese	$dat2,q15
	vld1.8	{$in2},[$inp],#16
	aese	$dat3,q15
	vld1.8	{$in3},[$inp],#16
	aese	$dat4,q15
	vld1.8	{$in4},[$inp],#16
	cbz	x6,.Lecb_enc_tail4x
	vld1.32 {q8},[$key_],#16			// re-pre-load rndkey[0]
	veor	$tmp0,$rndlast,$dat0
	vorr	$dat0,$in0,$in0
	veor	$tmp1,$rndlast,$dat1
	vorr	$dat1,$in1,$in1
	veor	$tmp2,$rndlast,$dat2
	vorr	$dat2,$in2,$in2
	veor	$tmp3,$rndlast,$dat3
	vorr	$dat3,$in3,$in3
	veor	$tmp4,$rndlast,$dat4
	vst1.8	{$tmp0},[$out],#16
	vorr	$dat4,$in4,$in4
	vst1.8	{$tmp1},[$out],#16
	mov	$cnt,$rounds
	vst1.8	{$tmp2},[$out],#16
	vld1.32 {q9},[$key_],#16			// re-pre-load rndkey[1]
	vst1.8	{$tmp3},[$out],#16
	vst1.8	{$tmp4},[$out],#16
	b.hs	.Loop5x_ecb_enc

	add	$len,$len,#0x50
	cbz	$len,.Lecb_done

	add	$cnt,$rounds,#2
	subs	$len,$len,#0x30
	vorr	$dat0,$in2,$in2
	vorr	$dat1,$in3,$in3
	vorr	$dat2,$in4,$in4
	b.lo	.Lecb_enc_tail

	b	.Loop3x_ecb_enc

.align	4
.Lecb_enc_tail4x:
	veor	$tmp1,$rndlast,$dat1
	veor	$tmp2,$rndlast,$dat2
	veor	$tmp3,$rndlast,$dat3
	veor	$tmp4,$rndlast,$dat4
	vst1.8	{$tmp1},[$out],#16
	vst1.8	{$tmp2},[$out],#16
	vst1.8	{$tmp3},[$out],#16
	vst1.8	{$tmp4},[$out],#16

	b	.Lecb_done
.align	4
___
$code.=<<___;
.Loop3x_ecb_enc:
	aese	$dat0,q8
	aesmc	$dat0,$dat0
	aese	$dat1,q8
	aesmc	$dat1,$dat1
	aese	$dat2,q8
	aesmc	$dat2,$dat2
	vld1.32	{q8},[$key_],#16
	subs	$cnt,$cnt,#2
	aese	$dat0,q9
	aesmc	$dat0,$dat0
	aese	$dat1,q9
	aesmc	$dat1,$dat1
	aese	$dat2,q9
	aesmc	$dat2,$dat2
	vld1.32	{q9},[$key_],#16
	b.gt	.Loop3x_ecb_enc

	aese	$dat0,q8
	aesmc	$dat0,$dat0
	aese	$dat1,q8
	aesmc	$dat1,$dat1
	aese	$dat2,q8
	aesmc	$dat2,$dat2
	subs	$len,$len,#0x30
	mov.lo	x6,$len				// x6, $cnt, is zero at this point
	aese	$dat0,q9
	aesmc	$dat0,$dat0
	aese	$dat1,q9
	aesmc	$dat1,$dat1
	aese	$dat2,q9
	aesmc	$dat2,$dat2
	add	$inp,$inp,x6			// $inp is adjusted in such way that
						// at exit from the loop $dat1-$dat2
						// are loaded with last "words"
	mov	$key_,$key
	aese	$dat0,q12
	aesmc	$dat0,$dat0
	aese	$dat1,q12
	aesmc	$dat1,$dat1
	aese	$dat2,q12
	aesmc	$dat2,$dat2
	vld1.8	{$in0},[$inp],#16
	aese	$dat0,q13
	aesmc	$dat0,$dat0
	aese	$dat1,q13
	aesmc	$dat1,$dat1
	aese	$dat2,q13
	aesmc	$dat2,$dat2
	vld1.8	{$in1},[$inp],#16
	aese	$dat0,q14
	aesmc	$dat0,$dat0
	aese	$dat1,q14
	aesmc	$dat1,$dat1
	aese	$dat2,q14
	aesmc	$dat2,$dat2
	vld1.8	{$in2},[$inp],#16
	aese	$dat0,q15
	aese	$dat1,q15
	aese	$dat2,q15
	vld1.32 {q8},[$key_],#16		// re-pre-load rndkey[0]
	add	$cnt,$rounds,#2
	veor	$tmp0,$rndlast,$dat0
	veor	$tmp1,$rndlast,$dat1
	veor	$dat2,$dat2,$rndlast
	vld1.32 {q9},[$key_],#16		// re-pre-load rndkey[1]
	vst1.8	{$tmp0},[$out],#16
	vorr	$dat0,$in0,$in0
	vst1.8	{$tmp1},[$out],#16
	vorr	$dat1,$in1,$in1
	vst1.8	{$dat2},[$out],#16
	vorr	$dat2,$in2,$in2
	b.hs	.Loop3x_ecb_enc

	cmn	$len,#0x30
	b.eq	.Lecb_done
	nop

.Lecb_enc_tail:
	aese	$dat1,q8
	aesmc	$dat1,$dat1
	aese	$dat2,q8
	aesmc	$dat2,$dat2
	vld1.32	{q8},[$key_],#16
	subs	$cnt,$cnt,#2
	aese	$dat1,q9
	aesmc	$dat1,$dat1
	aese	$dat2,q9
	aesmc	$dat2,$dat2
	vld1.32	{q9},[$key_],#16
	b.gt	.Lecb_enc_tail

	aese	$dat1,q8
	aesmc	$dat1,$dat1
	aese	$dat2,q8
	aesmc	$dat2,$dat2
	aese	$dat1,q9
	aesmc	$dat1,$dat1
	aese	$dat2,q9
	aesmc	$dat2,$dat2
	aese	$dat1,q12
	aesmc	$dat1,$dat1
	aese	$dat2,q12
	aesmc	$dat2,$dat2
	cmn	$len,#0x20
	aese	$dat1,q13
	aesmc	$dat1,$dat1
	aese	$dat2,q13
	aesmc	$dat2,$dat2
	aese	$dat1,q14
	aesmc	$dat1,$dat1
	aese	$dat2,q14
	aesmc	$dat2,$dat2
	aese	$dat1,q15
	aese	$dat2,q15
	b.eq	.Lecb_enc_one
	veor	$tmp1,$rndlast,$dat1
	veor	$tmp2,$rndlast,$dat2
	vst1.8	{$tmp1},[$out],#16
	vst1.8	{$tmp2},[$out],#16
	b	.Lecb_done

.Lecb_enc_one:
	veor	$tmp1,$rndlast,$dat2
	vst1.8	{$tmp1},[$out],#16
	b	.Lecb_done
___

$code.=<<___;
.align	5
.Lecb_dec:
	vld1.8	{$dat1},[$inp],#16
	subs	$len,$len,#32			// bias
	add	$cnt,$rounds,#2
	vorr	$in1,$dat1,$dat1
	vorr	$dat2,$dat1,$dat1
	vorr	$dat1,$dat,$dat
	b.lo	.Lecb_dec_tail

	vorr	$dat1,$in1,$in1
	vld1.8	{$dat2},[$inp],#16
___
$code.=<<___	if ($flavour =~ /64/);
	cmp	$len,#32
	b.lo	.Loop3x_ecb_dec

	vld1.8	{$dat3},[$inp],#16
	vld1.8	{$dat4},[$inp],#16
	sub	$len,$len,#32				// bias
	mov	$cnt,$rounds

.Loop5x_ecb_dec:
	aesd	$dat0,q8
	aesimc	$dat0,$dat0
	aesd	$dat1,q8
	aesimc	$dat1,$dat1
	aesd	$dat2,q8
	aesimc	$dat2,$dat2
	aesd	$dat3,q8
	aesimc	$dat3,$dat3
	aesd	$dat4,q8
	aesimc	$dat4,$dat4
	vld1.32	{q8},[$key_],#16
	subs	$cnt,$cnt,#2
	aesd	$dat0,q9
	aesimc	$dat0,$dat0
	aesd	$dat1,q9
	aesimc	$dat1,$dat1
	aesd	$dat2,q9
	aesimc	$dat2,$dat2
	aesd	$dat3,q9
	aesimc	$dat3,$dat3
	aesd	$dat4,q9
	aesimc	$dat4,$dat4
	vld1.32	{q9},[$key_],#16
	b.gt	.Loop5x_ecb_dec

	aesd	$dat0,q8
	aesimc	$dat0,$dat0
	aesd	$dat1,q8
	aesimc	$dat1,$dat1
	aesd	$dat2,q8
	aesimc	$dat2,$dat2
	aesd	$dat3,q8
	aesimc	$dat3,$dat3
	aesd	$dat4,q8
	aesimc	$dat4,$dat4
	cmp	$len,#0x40				// because .Lecb_tail4x
	sub	$len,$len,#0x50

	aesd	$dat0,q9
	aesimc	$dat0,$dat0
	aesd	$dat1,q9
	aesimc	$dat1,$dat1
	aesd	$dat2,q9
	aesimc	$dat2,$dat2
	aesd	$dat3,q9
	aesimc	$dat3,$dat3
	aesd	$dat4,q9
	aesimc	$dat4,$dat4
	csel	x6,xzr,$len,gt		// borrow x6, $cnt, "gt" is not typo
	mov	$key_,$key

	aesd	$dat0,q10
	aesimc	$dat0,$dat0
	aesd	$dat1,q10
	aesimc	$dat1,$dat1
	aesd	$dat2,q10
	aesimc	$dat2,$dat2
	aesd	$dat3,q10
	aesimc	$dat3,$dat3
	aesd	$dat4,q10
	aesimc	$dat4,$dat4
	add	$inp,$inp,x6				// $inp is adjusted in such way that
							// at exit from the loop $dat1-$dat4
							// are loaded with last "words"
	add	x6,$len,#0x60			// because .Lecb_tail4x

	aesd	$dat0,q11
	aesimc	$dat0,$dat0
	aesd	$dat1,q11
	aesimc	$dat1,$dat1
	aesd	$dat2,q11
	aesimc	$dat2,$dat2
	aesd	$dat3,q11
	aesimc	$dat3,$dat3
	aesd	$dat4,q11
	aesimc	$dat4,$dat4

	aesd	$dat0,q12
	aesimc	$dat0,$dat0
	aesd	$dat1,q12
	aesimc	$dat1,$dat1
	aesd	$dat2,q12
	aesimc	$dat2,$dat2
	aesd	$dat3,q12
	aesimc	$dat3,$dat3
	aesd	$dat4,q12
	aesimc	$dat4,$dat4

	aesd	$dat0,q13
	aesimc	$dat0,$dat0
	aesd	$dat1,q13
	aesimc	$dat1,$dat1
	aesd	$dat2,q13
	aesimc	$dat2,$dat2
	aesd	$dat3,q13
	aesimc	$dat3,$dat3
	aesd	$dat4,q13
	aesimc	$dat4,$dat4

	aesd	$dat0,q14
	aesimc	$dat0,$dat0
	aesd	$dat1,q14
	aesimc	$dat1,$dat1
	aesd	$dat2,q14
	aesimc	$dat2,$dat2
	aesd	$dat3,q14
	aesimc	$dat3,$dat3
	aesd	$dat4,q14
	aesimc	$dat4,$dat4

	aesd	$dat0,q15
	vld1.8	{$in0},[$inp],#16
	aesd	$dat1,q15
	vld1.8	{$in1},[$inp],#16
	aesd	$dat2,q15
	vld1.8	{$in2},[$inp],#16
	aesd	$dat3,q15
	vld1.8	{$in3},[$inp],#16
	aesd	$dat4,q15
	vld1.8	{$in4},[$inp],#16
	cbz	x6,.Lecb_tail4x
	vld1.32 {q8},[$key_],#16			// re-pre-load rndkey[0]
	veor	$tmp0,$rndlast,$dat0
	vorr	$dat0,$in0,$in0
	veor	$tmp1,$rndlast,$dat1
	vorr	$dat1,$in1,$in1
	veor	$tmp2,$rndlast,$dat2
	vorr	$dat2,$in2,$in2
	veor	$tmp3,$rndlast,$dat3
	vorr	$dat3,$in3,$in3
	veor	$tmp4,$rndlast,$dat4
	vst1.8	{$tmp0},[$out],#16
	vorr	$dat4,$in4,$in4
	vst1.8	{$tmp1},[$out],#16
	mov	$cnt,$rounds
	vst1.8	{$tmp2},[$out],#16
	vld1.32 {q9},[$key_],#16			// re-pre-load rndkey[1]
	vst1.8	{$tmp3},[$out],#16
	vst1.8	{$tmp4},[$out],#16
	b.hs	.Loop5x_ecb_dec

	add	$len,$len,#0x50
	cbz	$len,.Lecb_done

	add	$cnt,$rounds,#2
	subs	$len,$len,#0x30
	vorr	$dat0,$in2,$in2
	vorr	$dat1,$in3,$in3
	vorr	$dat2,$in4,$in4
	b.lo	.Lecb_dec_tail

	b	.Loop3x_ecb_dec

.align	4
.Lecb_tail4x:
	veor	$tmp1,$rndlast,$dat1
	veor	$tmp2,$rndlast,$dat2
	veor	$tmp3,$rndlast,$dat3
	veor	$tmp4,$rndlast,$dat4
	vst1.8	{$tmp1},[$out],#16
	vst1.8	{$tmp2},[$out],#16
	vst1.8	{$tmp3},[$out],#16
	vst1.8	{$tmp4},[$out],#16

	b	.Lecb_done
.align	4
___
$code.=<<___;
.Loop3x_ecb_dec:
	aesd	$dat0,q8
	aesimc	$dat0,$dat0
	aesd	$dat1,q8
	aesimc	$dat1,$dat1
	aesd	$dat2,q8
	aesimc	$dat2,$dat2
	vld1.32	{q8},[$key_],#16
	subs	$cnt,$cnt,#2
	aesd	$dat0,q9
	aesimc	$dat0,$dat0
	aesd	$dat1,q9
	aesimc	$dat1,$dat1
	aesd	$dat2,q9
	aesimc	$dat2,$dat2
	vld1.32	{q9},[$key_],#16
	b.gt	.Loop3x_ecb_dec

	aesd	$dat0,q8
	aesimc	$dat0,$dat0
	aesd	$dat1,q8
	aesimc	$dat1,$dat1
	aesd	$dat2,q8
	aesimc	$dat2,$dat2
	subs	$len,$len,#0x30
	mov.lo	x6,$len				// x6, $cnt, is zero at this point
	aesd	$dat0,q9
	aesimc	$dat0,$dat0
	aesd	$dat1,q9
	aesimc	$dat1,$dat1
	aesd	$dat2,q9
	aesimc	$dat2,$dat2
	add	$inp,$inp,x6 			// $inp is adjusted in such way that
						// at exit from the loop $dat1-$dat2
						// are loaded with last "words"
	mov	$key_,$key
	aesd	$dat0,q12
	aesimc	$dat0,$dat0
	aesd	$dat1,q12
	aesimc	$dat1,$dat1
	aesd	$dat2,q12
	aesimc	$dat2,$dat2
	vld1.8	{$in0},[$inp],#16
	aesd	$dat0,q13
	aesimc	$dat0,$dat0
	aesd	$dat1,q13
	aesimc	$dat1,$dat1
	aesd	$dat2,q13
	aesimc	$dat2,$dat2
	vld1.8	{$in1},[$inp],#16
	aesd	$dat0,q14
	aesimc	$dat0,$dat0
	aesd	$dat1,q14
	aesimc	$dat1,$dat1
	aesd	$dat2,q14
	aesimc	$dat2,$dat2
	vld1.8	{$in2},[$inp],#16
	aesd	$dat0,q15
	aesd	$dat1,q15
	aesd	$dat2,q15
	vld1.32 {q8},[$key_],#16			// re-pre-load rndkey[0]
	add	$cnt,$rounds,#2
	veor	$tmp0,$rndlast,$dat0
	veor	$tmp1,$rndlast,$dat1
	veor	$dat2,$dat2,$rndlast
	vld1.32 {q9},[$key_],#16			// re-pre-load rndkey[1]
	vst1.8	{$tmp0},[$out],#16
	vorr	$dat0,$in0,$in0
	vst1.8	{$tmp1},[$out],#16
	vorr	$dat1,$in1,$in1
	vst1.8	{$dat2},[$out],#16
	vorr	$dat2,$in2,$in2
	b.hs	.Loop3x_ecb_dec

	cmn	$len,#0x30
	b.eq	.Lecb_done
	nop

.Lecb_dec_tail:
	aesd	$dat1,q8
	aesimc	$dat1,$dat1
	aesd	$dat2,q8
	aesimc	$dat2,$dat2
	vld1.32	{q8},[$key_],#16
	subs	$cnt,$cnt,#2
	aesd	$dat1,q9
	aesimc	$dat1,$dat1
	aesd	$dat2,q9
	aesimc	$dat2,$dat2
	vld1.32	{q9},[$key_],#16
	b.gt	.Lecb_dec_tail

	aesd	$dat1,q8
	aesimc	$dat1,$dat1
	aesd	$dat2,q8
	aesimc	$dat2,$dat2
	aesd	$dat1,q9
	aesimc	$dat1,$dat1
	aesd	$dat2,q9
	aesimc	$dat2,$dat2
	aesd	$dat1,q12
	aesimc	$dat1,$dat1
	aesd	$dat2,q12
	aesimc	$dat2,$dat2
	cmn	$len,#0x20
	aesd	$dat1,q13
	aesimc	$dat1,$dat1
	aesd	$dat2,q13
	aesimc	$dat2,$dat2
	aesd	$dat1,q14
	aesimc	$dat1,$dat1
	aesd	$dat2,q14
	aesimc	$dat2,$dat2
	aesd	$dat1,q15
	aesd	$dat2,q15
	b.eq	.Lecb_dec_one
	veor	$tmp1,$rndlast,$dat1
	veor	$tmp2,$rndlast,$dat2
	vst1.8	{$tmp1},[$out],#16
	vst1.8	{$tmp2},[$out],#16
	b	.Lecb_done

.Lecb_dec_one:
	veor	$tmp1,$rndlast,$dat2
	vst1.8	{$tmp1},[$out],#16

.Lecb_done:
___
}
$code.=<<___	if ($flavour !~ /64/);
	vldmia	sp!,{d8-d15}
	ldmia	sp!,{r4-r8,pc}
___
$code.=<<___	if ($flavour =~ /64/);
	ldr	x29,[sp],#16
___
$code.=<<___	if ($flavour =~ /64/);
.Lecb_Final_abort:
	ret
___
$code.=<<___;
.size	${prefix}_ecb_encrypt,.-${prefix}_ecb_encrypt
___
}}}
{{{
my ($inp,$out,$len,$key,$ivp)=map("x$_",(0..4)); my $enc="w5";
my ($rounds,$cnt,$key_,$step,$step1)=($enc,"w6","x7","x8","x12");
my ($dat0,$dat1,$in0,$in1,$tmp0,$tmp1,$ivec,$rndlast)=map("q$_",(0..7));

my ($dat,$tmp,$rndzero_n_last)=($dat0,$tmp0,$tmp1);
my ($key4,$key5,$key6,$key7)=("x6","x12","x14",$key);

### q8-q15	preloaded key schedule

$code.=<<___;
.globl	${prefix}_cbc_encrypt
.type	${prefix}_cbc_encrypt,%function
.align	5
${prefix}_cbc_encrypt:
___
$code.=<<___	if ($flavour =~ /64/);
	AARCH64_VALID_CALL_TARGET
	// Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later.
	stp	x29,x30,[sp,#-16]!
	add	x29,sp,#0
___
$code.=<<___	if ($flavour !~ /64/);
	mov	ip,sp
	stmdb	sp!,{r4-r8,lr}
	vstmdb	sp!,{d8-d15}            @ ABI specification says so
	ldmia	ip,{r4-r5}		@ load remaining args
___
$code.=<<___;
	subs	$len,$len,#16
	mov	$step,#16
	b.lo	.Lcbc_abort
	cclr	$step,eq

	cmp	$enc,#0			// en- or decrypting?
	ldr	$rounds,[$key,#240]
	and	$len,$len,#-16
	vld1.8	{$ivec},[$ivp]
	vld1.8	{$dat},[$inp],$step

	vld1.32	{q8-q9},[$key]		// load key schedule...
	sub	$rounds,$rounds,#6
	add	$key_,$key,x5,lsl#4	// pointer to last 7 round keys
	sub	$rounds,$rounds,#2
	vld1.32	{q10-q11},[$key_],#32
	vld1.32	{q12-q13},[$key_],#32
	vld1.32	{q14-q15},[$key_],#32
	vld1.32	{$rndlast},[$key_]

	add	$key_,$key,#32
	mov	$cnt,$rounds
	b.eq	.Lcbc_dec

	cmp	$rounds,#2
	veor	$dat,$dat,$ivec
	veor	$rndzero_n_last,q8,$rndlast
	b.eq	.Lcbc_enc128

	vld1.32	{$in0-$in1},[$key_]
	add	$key_,$key,#16
	add	$key4,$key,#16*4
	add	$key5,$key,#16*5
	aese	$dat,q8
	aesmc	$dat,$dat
	add	$key6,$key,#16*6
	add	$key7,$key,#16*7
	b	.Lenter_cbc_enc

.align	4
.Loop_cbc_enc:
	aese	$dat,q8
	aesmc	$dat,$dat
	 vst1.8	{$ivec},[$out],#16
.Lenter_cbc_enc:
	aese	$dat,q9
	aesmc	$dat,$dat
	aese	$dat,$in0
	aesmc	$dat,$dat
	vld1.32	{q8},[$key4]
	cmp	$rounds,#4
	aese	$dat,$in1
	aesmc	$dat,$dat
	vld1.32	{q9},[$key5]
	b.eq	.Lcbc_enc192

	aese	$dat,q8
	aesmc	$dat,$dat
	vld1.32	{q8},[$key6]
	aese	$dat,q9
	aesmc	$dat,$dat
	vld1.32	{q9},[$key7]
	nop

.Lcbc_enc192:
	aese	$dat,q8
	aesmc	$dat,$dat
	 subs	$len,$len,#16
	aese	$dat,q9
	aesmc	$dat,$dat
	 cclr	$step,eq
	aese	$dat,q10
	aesmc	$dat,$dat
	aese	$dat,q11
	aesmc	$dat,$dat
	 vld1.8	{q8},[$inp],$step
	aese	$dat,q12
	aesmc	$dat,$dat
	 veor	q8,q8,$rndzero_n_last
	aese	$dat,q13
	aesmc	$dat,$dat
	 vld1.32 {q9},[$key_]		// re-pre-load rndkey[1]
	aese	$dat,q14
	aesmc	$dat,$dat
	aese	$dat,q15
	veor	$ivec,$dat,$rndlast
	b.hs	.Loop_cbc_enc

	vst1.8	{$ivec},[$out],#16
	b	.Lcbc_done

.align	5
.Lcbc_enc128:
	vld1.32	{$in0-$in1},[$key_]
	aese	$dat,q8
	aesmc	$dat,$dat
	b	.Lenter_cbc_enc128
.Loop_cbc_enc128:
	aese	$dat,q8
	aesmc	$dat,$dat
	 vst1.8	{$ivec},[$out],#16
.Lenter_cbc_enc128:
	aese	$dat,q9
	aesmc	$dat,$dat
	 subs	$len,$len,#16
	aese	$dat,$in0
	aesmc	$dat,$dat
	 cclr	$step,eq
	aese	$dat,$in1
	aesmc	$dat,$dat
	aese	$dat,q10
	aesmc	$dat,$dat
	aese	$dat,q11
	aesmc	$dat,$dat
	 vld1.8	{q8},[$inp],$step
	aese	$dat,q12
	aesmc	$dat,$dat
	aese	$dat,q13
	aesmc	$dat,$dat
	aese	$dat,q14
	aesmc	$dat,$dat
	 veor	q8,q8,$rndzero_n_last
	aese	$dat,q15
	veor	$ivec,$dat,$rndlast
	b.hs	.Loop_cbc_enc128

	vst1.8	{$ivec},[$out],#16
	b	.Lcbc_done
___
{
my ($dat2,$in2,$tmp2)=map("q$_",(10,11,9));

my ($dat3,$in3,$tmp3);	# used only in 64-bit mode
my ($dat4,$in4,$tmp4);
if ($flavour =~ /64/) {
    ($dat2,$dat3,$dat4,$in2,$in3,$in4,$tmp3,$tmp4)=map("q$_",(16..23));
}

$code.=<<___;
.align	5
.Lcbc_dec:
	vld1.8	{$dat2},[$inp],#16
	subs	$len,$len,#32		// bias
	add	$cnt,$rounds,#2
	vorr	$in1,$dat,$dat
	vorr	$dat1,$dat,$dat
	vorr	$in2,$dat2,$dat2
	b.lo	.Lcbc_dec_tail

	vorr	$dat1,$dat2,$dat2
	vld1.8	{$dat2},[$inp],#16
	vorr	$in0,$dat,$dat
	vorr	$in1,$dat1,$dat1
	vorr	$in2,$dat2,$dat2
___
$code.=<<___	if ($flavour =~ /64/);
	cmp	$len,#32
	b.lo	.Loop3x_cbc_dec

	vld1.8	{$dat3},[$inp],#16
	vld1.8	{$dat4},[$inp],#16
	sub	$len,$len,#32		// bias
	mov	$cnt,$rounds
	vorr	$in3,$dat3,$dat3
	vorr	$in4,$dat4,$dat4

.Loop5x_cbc_dec:
	aesd	$dat0,q8
	aesimc	$dat0,$dat0
	aesd	$dat1,q8
	aesimc	$dat1,$dat1
	aesd	$dat2,q8
	aesimc	$dat2,$dat2
	aesd	$dat3,q8
	aesimc	$dat3,$dat3
	aesd	$dat4,q8
	aesimc	$dat4,$dat4
	vld1.32	{q8},[$key_],#16
	subs	$cnt,$cnt,#2
	aesd	$dat0,q9
	aesimc	$dat0,$dat0
	aesd	$dat1,q9
	aesimc	$dat1,$dat1
	aesd	$dat2,q9
	aesimc	$dat2,$dat2
	aesd	$dat3,q9
	aesimc	$dat3,$dat3
	aesd	$dat4,q9
	aesimc	$dat4,$dat4
	vld1.32	{q9},[$key_],#16
	b.gt	.Loop5x_cbc_dec

	aesd	$dat0,q8
	aesimc	$dat0,$dat0
	aesd	$dat1,q8
	aesimc	$dat1,$dat1
	aesd	$dat2,q8
	aesimc	$dat2,$dat2
	aesd	$dat3,q8
	aesimc	$dat3,$dat3
	aesd	$dat4,q8
	aesimc	$dat4,$dat4
	 cmp	$len,#0x40		// because .Lcbc_tail4x
	 sub	$len,$len,#0x50

	aesd	$dat0,q9
	aesimc	$dat0,$dat0
	aesd	$dat1,q9
	aesimc	$dat1,$dat1
	aesd	$dat2,q9
	aesimc	$dat2,$dat2
	aesd	$dat3,q9
	aesimc	$dat3,$dat3
	aesd	$dat4,q9
	aesimc	$dat4,$dat4
	 csel	x6,xzr,$len,gt		// borrow x6, $cnt, "gt" is not typo
	 mov	$key_,$key

	aesd	$dat0,q10
	aesimc	$dat0,$dat0
	aesd	$dat1,q10
	aesimc	$dat1,$dat1
	aesd	$dat2,q10
	aesimc	$dat2,$dat2
	aesd	$dat3,q10
	aesimc	$dat3,$dat3
	aesd	$dat4,q10
	aesimc	$dat4,$dat4
	 add	$inp,$inp,x6		// $inp is adjusted in such way that
					// at exit from the loop $dat1-$dat4
					// are loaded with last "words"
	 add	x6,$len,#0x60		// because .Lcbc_tail4x

	aesd	$dat0,q11
	aesimc	$dat0,$dat0
	aesd	$dat1,q11
	aesimc	$dat1,$dat1
	aesd	$dat2,q11
	aesimc	$dat2,$dat2
	aesd	$dat3,q11
	aesimc	$dat3,$dat3
	aesd	$dat4,q11
	aesimc	$dat4,$dat4

	aesd	$dat0,q12
	aesimc	$dat0,$dat0
	aesd	$dat1,q12
	aesimc	$dat1,$dat1
	aesd	$dat2,q12
	aesimc	$dat2,$dat2
	aesd	$dat3,q12
	aesimc	$dat3,$dat3
	aesd	$dat4,q12
	aesimc	$dat4,$dat4

	aesd	$dat0,q13
	aesimc	$dat0,$dat0
	aesd	$dat1,q13
	aesimc	$dat1,$dat1
	aesd	$dat2,q13
	aesimc	$dat2,$dat2
	aesd	$dat3,q13
	aesimc	$dat3,$dat3
	aesd	$dat4,q13
	aesimc	$dat4,$dat4

	aesd	$dat0,q14
	aesimc	$dat0,$dat0
	aesd	$dat1,q14
	aesimc	$dat1,$dat1
	aesd	$dat2,q14
	aesimc	$dat2,$dat2
	aesd	$dat3,q14
	aesimc	$dat3,$dat3
	aesd	$dat4,q14
	aesimc	$dat4,$dat4

	 veor	$tmp0,$ivec,$rndlast
	aesd	$dat0,q15
	 veor	$tmp1,$in0,$rndlast
	 vld1.8	{$in0},[$inp],#16
	aesd	$dat1,q15
	 veor	$tmp2,$in1,$rndlast
	 vld1.8	{$in1},[$inp],#16
	aesd	$dat2,q15
	 veor	$tmp3,$in2,$rndlast
	 vld1.8	{$in2},[$inp],#16
	aesd	$dat3,q15
	 veor	$tmp4,$in3,$rndlast
	 vld1.8	{$in3},[$inp],#16
	aesd	$dat4,q15
	 vorr	$ivec,$in4,$in4
	 vld1.8	{$in4},[$inp],#16
	cbz	x6,.Lcbc_tail4x
	 vld1.32 {q8},[$key_],#16	// re-pre-load rndkey[0]
	veor	$tmp0,$tmp0,$dat0
	 vorr	$dat0,$in0,$in0
	veor	$tmp1,$tmp1,$dat1
	 vorr	$dat1,$in1,$in1
	veor	$tmp2,$tmp2,$dat2
	 vorr	$dat2,$in2,$in2
	veor	$tmp3,$tmp3,$dat3
	 vorr	$dat3,$in3,$in3
	veor	$tmp4,$tmp4,$dat4
	vst1.8	{$tmp0},[$out],#16
	 vorr	$dat4,$in4,$in4
	vst1.8	{$tmp1},[$out],#16
	 mov	$cnt,$rounds
	vst1.8	{$tmp2},[$out],#16
	 vld1.32 {q9},[$key_],#16	// re-pre-load rndkey[1]
	vst1.8	{$tmp3},[$out],#16
	vst1.8	{$tmp4},[$out],#16
	b.hs	.Loop5x_cbc_dec

	add	$len,$len,#0x50
	cbz	$len,.Lcbc_done

	add	$cnt,$rounds,#2
	subs	$len,$len,#0x30
	vorr	$dat0,$in2,$in2
	vorr	$in0,$in2,$in2
	vorr	$dat1,$in3,$in3
	vorr	$in1,$in3,$in3
	vorr	$dat2,$in4,$in4
	vorr	$in2,$in4,$in4
	b.lo	.Lcbc_dec_tail

	b	.Loop3x_cbc_dec

.align	4
.Lcbc_tail4x:
	veor	$tmp1,$tmp0,$dat1
	veor	$tmp2,$tmp2,$dat2
	veor	$tmp3,$tmp3,$dat3
	veor	$tmp4,$tmp4,$dat4
	vst1.8	{$tmp1},[$out],#16
	vst1.8	{$tmp2},[$out],#16
	vst1.8	{$tmp3},[$out],#16
	vst1.8	{$tmp4},[$out],#16

	b	.Lcbc_done
.align	4
___
$code.=<<___;
.Loop3x_cbc_dec:
	aesd	$dat0,q8
	aesimc	$dat0,$dat0
	aesd	$dat1,q8
	aesimc	$dat1,$dat1
	aesd	$dat2,q8
	aesimc	$dat2,$dat2
	vld1.32	{q8},[$key_],#16
	subs	$cnt,$cnt,#2
	aesd	$dat0,q9
	aesimc	$dat0,$dat0
	aesd	$dat1,q9
	aesimc	$dat1,$dat1
	aesd	$dat2,q9
	aesimc	$dat2,$dat2
	vld1.32	{q9},[$key_],#16
	b.gt	.Loop3x_cbc_dec

	aesd	$dat0,q8
	aesimc	$dat0,$dat0
	aesd	$dat1,q8
	aesimc	$dat1,$dat1
	aesd	$dat2,q8
	aesimc	$dat2,$dat2
	 veor	$tmp0,$ivec,$rndlast
	 subs	$len,$len,#0x30
	 veor	$tmp1,$in0,$rndlast
	 mov.lo	x6,$len			// x6, $cnt, is zero at this point
	aesd	$dat0,q9
	aesimc	$dat0,$dat0
	aesd	$dat1,q9
	aesimc	$dat1,$dat1
	aesd	$dat2,q9
	aesimc	$dat2,$dat2
	 veor	$tmp2,$in1,$rndlast
	 add	$inp,$inp,x6		// $inp is adjusted in such way that
					// at exit from the loop $dat1-$dat2
					// are loaded with last "words"
	 vorr	$ivec,$in2,$in2
	 mov	$key_,$key
	aesd	$dat0,q12
	aesimc	$dat0,$dat0
	aesd	$dat1,q12
	aesimc	$dat1,$dat1
	aesd	$dat2,q12
	aesimc	$dat2,$dat2
	 vld1.8	{$in0},[$inp],#16
	aesd	$dat0,q13
	aesimc	$dat0,$dat0
	aesd	$dat1,q13
	aesimc	$dat1,$dat1
	aesd	$dat2,q13
	aesimc	$dat2,$dat2
	 vld1.8	{$in1},[$inp],#16
	aesd	$dat0,q14
	aesimc	$dat0,$dat0
	aesd	$dat1,q14
	aesimc	$dat1,$dat1
	aesd	$dat2,q14
	aesimc	$dat2,$dat2
	 vld1.8	{$in2},[$inp],#16
	aesd	$dat0,q15
	aesd	$dat1,q15
	aesd	$dat2,q15
	 vld1.32 {q8},[$key_],#16	// re-pre-load rndkey[0]
	 add	$cnt,$rounds,#2
	veor	$tmp0,$tmp0,$dat0
	veor	$tmp1,$tmp1,$dat1
	veor	$dat2,$dat2,$tmp2
	 vld1.32 {q9},[$key_],#16	// re-pre-load rndkey[1]
	vst1.8	{$tmp0},[$out],#16
	 vorr	$dat0,$in0,$in0
	vst1.8	{$tmp1},[$out],#16
	 vorr	$dat1,$in1,$in1
	vst1.8	{$dat2},[$out],#16
	 vorr	$dat2,$in2,$in2
	b.hs	.Loop3x_cbc_dec

	cmn	$len,#0x30
	b.eq	.Lcbc_done
	nop

.Lcbc_dec_tail:
	aesd	$dat1,q8
	aesimc	$dat1,$dat1
	aesd	$dat2,q8
	aesimc	$dat2,$dat2
	vld1.32	{q8},[$key_],#16
	subs	$cnt,$cnt,#2
	aesd	$dat1,q9
	aesimc	$dat1,$dat1
	aesd	$dat2,q9
	aesimc	$dat2,$dat2
	vld1.32	{q9},[$key_],#16
	b.gt	.Lcbc_dec_tail

	aesd	$dat1,q8
	aesimc	$dat1,$dat1
	aesd	$dat2,q8
	aesimc	$dat2,$dat2
	aesd	$dat1,q9
	aesimc	$dat1,$dat1
	aesd	$dat2,q9
	aesimc	$dat2,$dat2
	aesd	$dat1,q12
	aesimc	$dat1,$dat1
	aesd	$dat2,q12
	aesimc	$dat2,$dat2
	 cmn	$len,#0x20
	aesd	$dat1,q13
	aesimc	$dat1,$dat1
	aesd	$dat2,q13
	aesimc	$dat2,$dat2
	 veor	$tmp1,$ivec,$rndlast
	aesd	$dat1,q14
	aesimc	$dat1,$dat1
	aesd	$dat2,q14
	aesimc	$dat2,$dat2
	 veor	$tmp2,$in1,$rndlast
	aesd	$dat1,q15
	aesd	$dat2,q15
	b.eq	.Lcbc_dec_one
	veor	$tmp1,$tmp1,$dat1
	veor	$tmp2,$tmp2,$dat2
	 vorr	$ivec,$in2,$in2
	vst1.8	{$tmp1},[$out],#16
	vst1.8	{$tmp2},[$out],#16
	b	.Lcbc_done

.Lcbc_dec_one:
	veor	$tmp1,$tmp1,$dat2
	 vorr	$ivec,$in2,$in2
	vst1.8	{$tmp1},[$out],#16

.Lcbc_done:
	vst1.8	{$ivec},[$ivp]
.Lcbc_abort:
___
}
$code.=<<___	if ($flavour !~ /64/);
	vldmia	sp!,{d8-d15}
	ldmia	sp!,{r4-r8,pc}
___
$code.=<<___	if ($flavour =~ /64/);
	ldr	x29,[sp],#16
	ret
___
$code.=<<___;
.size	${prefix}_cbc_encrypt,.-${prefix}_cbc_encrypt
___
}}}
{{{
my ($inp,$out,$len,$key,$ivp)=map("x$_",(0..4));
my ($rounds,$cnt,$key_)=("w5","w6","x7");
my ($ctr,$tctr0,$tctr1,$tctr2)=map("w$_",(8..10,12));
my $step="x12";		# aliases with $tctr2

my ($dat0,$dat1,$in0,$in1,$tmp0,$tmp1,$ivec,$rndlast)=map("q$_",(0..7));
my ($dat2,$in2,$tmp2)=map("q$_",(10,11,9));

# used only in 64-bit mode...
my ($dat3,$dat4,$in3,$in4)=map("q$_",(16..23));

my ($dat,$tmp)=($dat0,$tmp0);

### q8-q15	preloaded key schedule

$code.=<<___;
.globl	${prefix}_ctr32_encrypt_blocks
.type	${prefix}_ctr32_encrypt_blocks,%function
.align	5
${prefix}_ctr32_encrypt_blocks:
___
$code.=<<___	if ($flavour =~ /64/);
	AARCH64_VALID_CALL_TARGET
	// Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later.
	stp		x29,x30,[sp,#-16]!
	add		x29,sp,#0
___
$code.=<<___	if ($flavour !~ /64/);
	mov		ip,sp
	stmdb		sp!,{r4-r10,lr}
	vstmdb		sp!,{d8-d15}            @ ABI specification says so
	ldr		r4, [ip]		@ load remaining arg
___
$code.=<<___;
	ldr		$rounds,[$key,#240]

	ldr		$ctr, [$ivp, #12]
#ifdef __ARMEB__
	vld1.8		{$dat0},[$ivp]
#else
	vld1.32		{$dat0},[$ivp]
#endif
	vld1.32		{q8-q9},[$key]		// load key schedule...
	sub		$rounds,$rounds,#4
	mov		$step,#16
	cmp		$len,#2
	add		$key_,$key,x5,lsl#4	// pointer to last 5 round keys
	sub		$rounds,$rounds,#2
	vld1.32		{q12-q13},[$key_],#32
	vld1.32		{q14-q15},[$key_],#32
	vld1.32		{$rndlast},[$key_]
	add		$key_,$key,#32
	mov		$cnt,$rounds
	cclr		$step,lo
#ifndef __ARMEB__
	rev		$ctr, $ctr
#endif
___
$code.=<<___	if ($flavour =~ /64/);
	vorr		$dat1,$dat0,$dat0
	add		$tctr1, $ctr, #1
	vorr		$dat2,$dat0,$dat0
	add		$ctr, $ctr, #2
	vorr		$ivec,$dat0,$dat0
	rev		$tctr1, $tctr1
	vmov.32		${dat1}[3],$tctr1
	b.ls		.Lctr32_tail
	rev		$tctr2, $ctr
	sub		$len,$len,#3		// bias
	vmov.32		${dat2}[3],$tctr2
___
$code.=<<___	if ($flavour !~ /64/);
	add		$tctr1, $ctr, #1
	vorr		$ivec,$dat0,$dat0
	rev		$tctr1, $tctr1
	vmov.32		${ivec}[3],$tctr1
	add		$ctr, $ctr, #2
	vorr		$dat1,$ivec,$ivec
	b.ls		.Lctr32_tail
	rev		$tctr2, $ctr
	vmov.32		${ivec}[3],$tctr2
	sub		$len,$len,#3		// bias
	vorr		$dat2,$ivec,$ivec
___
$code.=<<___	if ($flavour =~ /64/);
	cmp		$len,#32
	b.lo		.Loop3x_ctr32

	add		w13,$ctr,#1
	add		w14,$ctr,#2
	vorr		$dat3,$dat0,$dat0
	rev		w13,w13
	vorr		$dat4,$dat0,$dat0
	rev		w14,w14
	vmov.32		${dat3}[3],w13
	sub		$len,$len,#2		// bias
	vmov.32		${dat4}[3],w14
	add		$ctr,$ctr,#2
	b		.Loop5x_ctr32

.align	4
.Loop5x_ctr32:
	aese		$dat0,q8
	aesmc		$dat0,$dat0
	aese		$dat1,q8
	aesmc		$dat1,$dat1
	aese		$dat2,q8
	aesmc		$dat2,$dat2
	aese		$dat3,q8
	aesmc		$dat3,$dat3
	aese		$dat4,q8
	aesmc		$dat4,$dat4
	vld1.32		{q8},[$key_],#16
	subs		$cnt,$cnt,#2
	aese		$dat0,q9
	aesmc		$dat0,$dat0
	aese		$dat1,q9
	aesmc		$dat1,$dat1
	aese		$dat2,q9
	aesmc		$dat2,$dat2
	aese		$dat3,q9
	aesmc		$dat3,$dat3
	aese		$dat4,q9
	aesmc		$dat4,$dat4
	vld1.32		{q9},[$key_],#16
	b.gt		.Loop5x_ctr32

	mov		$key_,$key
	aese		$dat0,q8
	aesmc		$dat0,$dat0
	aese		$dat1,q8
	aesmc		$dat1,$dat1
	aese		$dat2,q8
	aesmc		$dat2,$dat2
	aese		$dat3,q8
	aesmc		$dat3,$dat3
	aese		$dat4,q8
	aesmc		$dat4,$dat4
	vld1.32	 	{q8},[$key_],#16	// re-pre-load rndkey[0]

	aese		$dat0,q9
	aesmc		$dat0,$dat0
	aese		$dat1,q9
	aesmc		$dat1,$dat1
	aese		$dat2,q9
	aesmc		$dat2,$dat2
	aese		$dat3,q9
	aesmc		$dat3,$dat3
	aese		$dat4,q9
	aesmc		$dat4,$dat4
	vld1.32	 	{q9},[$key_],#16	// re-pre-load rndkey[1]

	aese		$dat0,q12
	aesmc		$dat0,$dat0
	 add		$tctr0,$ctr,#1
	 add		$tctr1,$ctr,#2
	aese		$dat1,q12
	aesmc		$dat1,$dat1
	 add		$tctr2,$ctr,#3
	 add		w13,$ctr,#4
	aese		$dat2,q12
	aesmc		$dat2,$dat2
	 add		w14,$ctr,#5
	 rev		$tctr0,$tctr0
	aese		$dat3,q12
	aesmc		$dat3,$dat3
	 rev		$tctr1,$tctr1
	 rev		$tctr2,$tctr2
	aese		$dat4,q12
	aesmc		$dat4,$dat4
	 rev		w13,w13
	 rev		w14,w14

	aese		$dat0,q13
	aesmc		$dat0,$dat0
	aese		$dat1,q13
	aesmc		$dat1,$dat1
	aese		$dat2,q13
	aesmc		$dat2,$dat2
	aese		$dat3,q13
	aesmc		$dat3,$dat3
	aese		$dat4,q13
	aesmc		$dat4,$dat4

	aese		$dat0,q14
	aesmc		$dat0,$dat0
	 vld1.8		{$in0},[$inp],#16
	aese		$dat1,q14
	aesmc		$dat1,$dat1
	 vld1.8		{$in1},[$inp],#16
	aese		$dat2,q14
	aesmc		$dat2,$dat2
	 vld1.8		{$in2},[$inp],#16
	aese		$dat3,q14
	aesmc		$dat3,$dat3
	 vld1.8		{$in3},[$inp],#16
	aese		$dat4,q14
	aesmc		$dat4,$dat4
	 vld1.8		{$in4},[$inp],#16

	aese		$dat0,q15
	 veor		$in0,$in0,$rndlast
	aese		$dat1,q15
	 veor		$in1,$in1,$rndlast
	aese		$dat2,q15
	 veor		$in2,$in2,$rndlast
	aese		$dat3,q15
	 veor		$in3,$in3,$rndlast
	aese		$dat4,q15
	 veor		$in4,$in4,$rndlast

	veor		$in0,$in0,$dat0
	 vorr		$dat0,$ivec,$ivec
	veor		$in1,$in1,$dat1
	 vorr		$dat1,$ivec,$ivec
	veor		$in2,$in2,$dat2
	 vorr		$dat2,$ivec,$ivec
	veor		$in3,$in3,$dat3
	 vorr		$dat3,$ivec,$ivec
	veor		$in4,$in4,$dat4
	 vorr		$dat4,$ivec,$ivec

	vst1.8		{$in0},[$out],#16
	 vmov.32	${dat0}[3],$tctr0
	vst1.8		{$in1},[$out],#16
	 vmov.32	${dat1}[3],$tctr1
	vst1.8		{$in2},[$out],#16
	 vmov.32	${dat2}[3],$tctr2
	vst1.8		{$in3},[$out],#16
	 vmov.32	${dat3}[3],w13
	vst1.8		{$in4},[$out],#16
	 vmov.32	${dat4}[3],w14

	mov		$cnt,$rounds
	cbz		$len,.Lctr32_done

	add		$ctr,$ctr,#5
	subs		$len,$len,#5
	b.hs		.Loop5x_ctr32

	add		$len,$len,#5
	sub		$ctr,$ctr,#5

	cmp		$len,#2
	mov		$step,#16
	cclr		$step,lo
	b.ls		.Lctr32_tail

	sub		$len,$len,#3		// bias
	add		$ctr,$ctr,#3
___
$code.=<<___;
	b		.Loop3x_ctr32

.align	4
.Loop3x_ctr32:
	aese		$dat0,q8
	aesmc		$dat0,$dat0
	aese		$dat1,q8
	aesmc		$dat1,$dat1
	aese		$dat2,q8
	aesmc		$dat2,$dat2
	vld1.32		{q8},[$key_],#16
	subs		$cnt,$cnt,#2
	aese		$dat0,q9
	aesmc		$dat0,$dat0
	aese		$dat1,q9
	aesmc		$dat1,$dat1
	aese		$dat2,q9
	aesmc		$dat2,$dat2
	vld1.32		{q9},[$key_],#16
	b.gt		.Loop3x_ctr32

	aese		$dat0,q8
	aesmc		$tmp0,$dat0
	aese		$dat1,q8
	aesmc		$tmp1,$dat1
	 vld1.8		{$in0},[$inp],#16
___
$code.=<<___	if ($flavour =~ /64/);
	 vorr		$dat0,$ivec,$ivec
___
$code.=<<___	if ($flavour !~ /64/);
	 add		$tctr0,$ctr,#1
___
$code.=<<___;
	aese		$dat2,q8
	aesmc		$dat2,$dat2
	 vld1.8		{$in1},[$inp],#16
___
$code.=<<___	if ($flavour =~ /64/);
	 vorr		$dat1,$ivec,$ivec
___
$code.=<<___	if ($flavour !~ /64/);
	 rev		$tctr0,$tctr0
___
$code.=<<___;
	aese		$tmp0,q9
	aesmc		$tmp0,$tmp0
	aese		$tmp1,q9
	aesmc		$tmp1,$tmp1
	 vld1.8		{$in2},[$inp],#16
	 mov		$key_,$key
	aese		$dat2,q9
	aesmc		$tmp2,$dat2
___
$code.=<<___	if ($flavour =~ /64/);
	 vorr		$dat2,$ivec,$ivec
	 add		$tctr0,$ctr,#1
___
$code.=<<___;
	aese		$tmp0,q12
	aesmc		$tmp0,$tmp0
	aese		$tmp1,q12
	aesmc		$tmp1,$tmp1
	 veor		$in0,$in0,$rndlast
	 add		$tctr1,$ctr,#2
	aese		$tmp2,q12
	aesmc		$tmp2,$tmp2
	 veor		$in1,$in1,$rndlast
	 add		$ctr,$ctr,#3
	aese		$tmp0,q13
	aesmc		$tmp0,$tmp0
	aese		$tmp1,q13
	aesmc		$tmp1,$tmp1
	 veor		$in2,$in2,$rndlast
___
$code.=<<___	if ($flavour =~ /64/);
	 rev		$tctr0,$tctr0
	aese		$tmp2,q13
	aesmc		$tmp2,$tmp2
	 vmov.32	${dat0}[3], $tctr0
___
$code.=<<___	if ($flavour !~ /64/);
	 vmov.32	${ivec}[3], $tctr0
	aese		$tmp2,q13
	aesmc		$tmp2,$tmp2
	 vorr		$dat0,$ivec,$ivec
___
$code.=<<___;
	 rev		$tctr1,$tctr1
	aese		$tmp0,q14
	aesmc		$tmp0,$tmp0
___
$code.=<<___	if ($flavour !~ /64/);
	 vmov.32	${ivec}[3], $tctr1
	 rev		$tctr2,$ctr
___
$code.=<<___;
	aese		$tmp1,q14
	aesmc		$tmp1,$tmp1
___
$code.=<<___	if ($flavour =~ /64/);
	 vmov.32	${dat1}[3], $tctr1
	 rev		$tctr2,$ctr
	aese		$tmp2,q14
	aesmc		$tmp2,$tmp2
	 vmov.32	${dat2}[3], $tctr2
___
$code.=<<___	if ($flavour !~ /64/);
	 vorr		$dat1,$ivec,$ivec
	 vmov.32	${ivec}[3], $tctr2
	aese		$tmp2,q14
	aesmc		$tmp2,$tmp2
	 vorr		$dat2,$ivec,$ivec
___
$code.=<<___;
	 subs		$len,$len,#3
	aese		$tmp0,q15
	aese		$tmp1,q15
	aese		$tmp2,q15

	veor		$in0,$in0,$tmp0
	 vld1.32	 {q8},[$key_],#16	// re-pre-load rndkey[0]
	vst1.8		{$in0},[$out],#16
	veor		$in1,$in1,$tmp1
	 mov		$cnt,$rounds
	vst1.8		{$in1},[$out],#16
	veor		$in2,$in2,$tmp2
	 vld1.32	 {q9},[$key_],#16	// re-pre-load rndkey[1]
	vst1.8		{$in2},[$out],#16
	b.hs		.Loop3x_ctr32

	adds		$len,$len,#3
	b.eq		.Lctr32_done
	cmp		$len,#1
	mov		$step,#16
	cclr		$step,eq

.Lctr32_tail:
	aese		$dat0,q8
	aesmc		$dat0,$dat0
	aese		$dat1,q8
	aesmc		$dat1,$dat1
	vld1.32		{q8},[$key_],#16
	subs		$cnt,$cnt,#2
	aese		$dat0,q9
	aesmc		$dat0,$dat0
	aese		$dat1,q9
	aesmc		$dat1,$dat1
	vld1.32		{q9},[$key_],#16
	b.gt		.Lctr32_tail

	aese		$dat0,q8
	aesmc		$dat0,$dat0
	aese		$dat1,q8
	aesmc		$dat1,$dat1
	aese		$dat0,q9
	aesmc		$dat0,$dat0
	aese		$dat1,q9
	aesmc		$dat1,$dat1
	 vld1.8		{$in0},[$inp],$step
	aese		$dat0,q12
	aesmc		$dat0,$dat0
	aese		$dat1,q12
	aesmc		$dat1,$dat1
	 vld1.8		{$in1},[$inp]
	aese		$dat0,q13
	aesmc		$dat0,$dat0
	aese		$dat1,q13
	aesmc		$dat1,$dat1
	 veor		$in0,$in0,$rndlast
	aese		$dat0,q14
	aesmc		$dat0,$dat0
	aese		$dat1,q14
	aesmc		$dat1,$dat1
	 veor		$in1,$in1,$rndlast
	aese		$dat0,q15
	aese		$dat1,q15

	cmp		$len,#1
	veor		$in0,$in0,$dat0
	veor		$in1,$in1,$dat1
	vst1.8		{$in0},[$out],#16
	b.eq		.Lctr32_done
	vst1.8		{$in1},[$out]

.Lctr32_done:
___
$code.=<<___	if ($flavour !~ /64/);
	vldmia		sp!,{d8-d15}
	ldmia		sp!,{r4-r10,pc}
___
$code.=<<___	if ($flavour =~ /64/);
	ldr		x29,[sp],#16
	ret
___
$code.=<<___;
.size	${prefix}_ctr32_encrypt_blocks,.-${prefix}_ctr32_encrypt_blocks
___
}}}
# Performance in cycles per byte.
# Processed with AES-XTS different key size.
# It shows the value before and after optimization as below:
# (before/after):
#
#		AES-128-XTS		AES-256-XTS
# Cortex-A57	3.36/1.09		4.02/1.37
# Cortex-A72	3.03/1.02		3.28/1.33

# Optimization is implemented by loop unrolling and interleaving.
# Commonly, we choose the unrolling factor as 5, if the input
# data size smaller than 5 blocks, but not smaller than 3 blocks,
# choose 3 as the unrolling factor.
# If the input data size dsize >= 5*16 bytes, then take 5 blocks
# as one iteration, every loop the left size lsize -= 5*16.
# If lsize < 5*16 bytes, treat them as the tail. Note: left 4*16 bytes
# will be processed specially, which be integrated into the 5*16 bytes
# loop to improve the efficiency.
# There is one special case, if the original input data size dsize
# = 16 bytes, we will treat it separately to improve the
# performance: one independent code block without LR, FP load and
# store.
# Encryption will process the (length -tailcnt) bytes as mentioned
# previously, then encrypt the composite block as last second
# cipher block.
# Decryption will process the (length -tailcnt -1) bytes as mentioned
# previously, then decrypt the last second cipher block to get the
# last plain block(tail), decrypt the composite block as last second
# plain text block.

{{{
my ($inp,$out,$len,$key1,$key2,$ivp)=map("x$_",(0..5));
my ($rounds0,$rounds,$key_,$step,$ivl,$ivh)=("w5","w6","x7","x8","x9","x10");
my ($tmpoutp,$loutp,$l2outp,$tmpinp)=("x13","w14","w15","x20");
my ($tailcnt,$midnum,$midnumx,$constnum,$constnumx)=("x21","w22","x22","w19","x19");
my ($xoffset,$tmpmx,$tmpmw)=("x6","x11","w11");
my ($dat0,$dat1,$in0,$in1,$tmp0,$tmp1,$tmp2,$rndlast)=map("q$_",(0..7));
my ($iv0,$iv1,$iv2,$iv3,$iv4)=("v6.16b","v8.16b","v9.16b","v10.16b","v11.16b");
my ($ivd00,$ivd01,$ivd20,$ivd21)=("d6","v6.d[1]","d9","v9.d[1]");
my ($ivd10,$ivd11,$ivd30,$ivd31,$ivd40,$ivd41)=("d8","v8.d[1]","d10","v10.d[1]","d11","v11.d[1]");

my ($tmpin)=("v26.16b");
my ($dat,$tmp,$rndzero_n_last)=($dat0,$tmp0,$tmp1);

# q7	last round key
# q10-q15, q7	Last 7 round keys
# q8-q9	preloaded round keys except last 7 keys for big size
# q20, q21, q8-q9	preloaded round keys except last 7 keys for only 16 byte


my ($dat2,$in2,$tmp2)=map("q$_",(10,11,9));

my ($dat3,$in3,$tmp3);	# used only in 64-bit mode
my ($dat4,$in4,$tmp4);
if ($flavour =~ /64/) {
    ($dat2,$dat3,$dat4,$in2,$in3,$in4,$tmp3,$tmp4)=map("q$_",(16..23));
}

$code.=<<___	if ($flavour =~ /64/);
.globl	${prefix}_xts_encrypt
.type	${prefix}_xts_encrypt,%function
.align	5
${prefix}_xts_encrypt:
___
$code.=<<___	if ($flavour =~ /64/);
	AARCH64_VALID_CALL_TARGET
	cmp	$len,#16
	// Original input data size bigger than 16, jump to big size processing.
	b.ne	.Lxts_enc_big_size
	// Encrypt the iv with key2, as the first XEX iv.
	ldr	$rounds,[$key2,#240]
	vld1.32	{$dat},[$key2],#16
	vld1.8	{$iv0},[$ivp]
	sub	$rounds,$rounds,#2
	vld1.32	{$dat1},[$key2],#16

.Loop_enc_iv_enc:
	aese	$iv0,$dat
	aesmc	$iv0,$iv0
	vld1.32	{$dat},[$key2],#16
	subs	$rounds,$rounds,#2
	aese	$iv0,$dat1
	aesmc	$iv0,$iv0
	vld1.32	{$dat1},[$key2],#16
	b.gt	.Loop_enc_iv_enc

	aese	$iv0,$dat
	aesmc	$iv0,$iv0
	vld1.32	{$dat},[$key2]
	aese	$iv0,$dat1
	veor	$iv0,$iv0,$dat

	vld1.8	{$dat0},[$inp]
	veor	$dat0,$iv0,$dat0

	ldr	$rounds,[$key1,#240]
	vld1.32	{q20-q21},[$key1],#32		// load key schedule...

	aese	$dat0,q20
	aesmc	$dat0,$dat0
	vld1.32	{q8-q9},[$key1],#32		// load key schedule...
	aese	$dat0,q21
	aesmc	$dat0,$dat0
	subs	$rounds,$rounds,#10		// if rounds==10, jump to aes-128-xts processing
	b.eq	.Lxts_128_enc
.Lxts_enc_round_loop:
	aese	$dat0,q8
	aesmc	$dat0,$dat0
	vld1.32	{q8},[$key1],#16		// load key schedule...
	aese	$dat0,q9
	aesmc	$dat0,$dat0
	vld1.32	{q9},[$key1],#16		// load key schedule...
	subs	$rounds,$rounds,#2		// bias
	b.gt	.Lxts_enc_round_loop
.Lxts_128_enc:
	vld1.32	{q10-q11},[$key1],#32		// load key schedule...
	aese	$dat0,q8
	aesmc	$dat0,$dat0
	aese	$dat0,q9
	aesmc	$dat0,$dat0
	vld1.32	{q12-q13},[$key1],#32		// load key schedule...
	aese	$dat0,q10
	aesmc	$dat0,$dat0
	aese	$dat0,q11
	aesmc	$dat0,$dat0
	vld1.32	{q14-q15},[$key1],#32		// load key schedule...
	aese	$dat0,q12
	aesmc	$dat0,$dat0
	aese	$dat0,q13
	aesmc	$dat0,$dat0
	vld1.32	{$rndlast},[$key1]
	aese	$dat0,q14
	aesmc	$dat0,$dat0
	aese	$dat0,q15
	veor	$dat0,$dat0,$rndlast
	veor	$dat0,$dat0,$iv0
	vst1.8	{$dat0},[$out]
	b	.Lxts_enc_final_abort

.align	4
.Lxts_enc_big_size:
___
$code.=<<___	if ($flavour =~ /64/);
	stp	$constnumx,$tmpinp,[sp,#-64]!
	stp	$tailcnt,$midnumx,[sp,#48]
	stp	$ivd10,$ivd20,[sp,#32]
	stp	$ivd30,$ivd40,[sp,#16]

	// tailcnt store the tail value of length%16.
	and	$tailcnt,$len,#0xf
	and	$len,$len,#-16
	subs	$len,$len,#16
	mov	$step,#16
	b.lo	.Lxts_abort
	csel	$step,xzr,$step,eq

	// Firstly, encrypt the iv with key2, as the first iv of XEX.
	ldr	$rounds,[$key2,#240]
	vld1.32	{$dat},[$key2],#16
	vld1.8	{$iv0},[$ivp]
	sub	$rounds,$rounds,#2
	vld1.32	{$dat1},[$key2],#16

.Loop_iv_enc:
	aese	$iv0,$dat
	aesmc	$iv0,$iv0
	vld1.32	{$dat},[$key2],#16
	subs	$rounds,$rounds,#2
	aese	$iv0,$dat1
	aesmc	$iv0,$iv0
	vld1.32	{$dat1},[$key2],#16
	b.gt	.Loop_iv_enc

	aese	$iv0,$dat
	aesmc	$iv0,$iv0
	vld1.32	{$dat},[$key2]
	aese	$iv0,$dat1
	veor	$iv0,$iv0,$dat

	// The iv for second block
	// $ivl- iv(low), $ivh - iv(high)
	// the five ivs stored into, $iv0,$iv1,$iv2,$iv3,$iv4
	fmov	$ivl,$ivd00
	fmov	$ivh,$ivd01
	mov	$constnum,#0x87
	extr	$midnumx,$ivh,$ivh,#32
	extr	$ivh,$ivh,$ivl,#63
	and	$tmpmw,$constnum,$midnum,asr#31
	eor	$ivl,$tmpmx,$ivl,lsl#1
	fmov	$ivd10,$ivl
	fmov	$ivd11,$ivh

	ldr	$rounds0,[$key1,#240]		// next starting point
	vld1.8	{$dat},[$inp],$step

	vld1.32	{q8-q9},[$key1]			// load key schedule...
	sub	$rounds0,$rounds0,#6
	add	$key_,$key1,$ivp,lsl#4		// pointer to last 7 round keys
	sub	$rounds0,$rounds0,#2
	vld1.32	{q10-q11},[$key_],#32
	vld1.32	{q12-q13},[$key_],#32
	vld1.32	{q14-q15},[$key_],#32
	vld1.32	{$rndlast},[$key_]

	add	$key_,$key1,#32
	mov	$rounds,$rounds0

	// Encryption
.Lxts_enc:
	vld1.8	{$dat2},[$inp],#16
	subs	$len,$len,#32			// bias
	add	$rounds,$rounds0,#2
	vorr	$in1,$dat,$dat
	vorr	$dat1,$dat,$dat
	vorr	$in3,$dat,$dat
	vorr	$in2,$dat2,$dat2
	vorr	$in4,$dat2,$dat2
	b.lo	.Lxts_inner_enc_tail
	veor	$dat,$dat,$iv0			// before encryption, xor with iv
	veor	$dat2,$dat2,$iv1

	// The iv for third block
	extr	$midnumx,$ivh,$ivh,#32
	extr	$ivh,$ivh,$ivl,#63
	and	$tmpmw,$constnum,$midnum,asr#31
	eor	$ivl,$tmpmx,$ivl,lsl#1
	fmov	$ivd20,$ivl
	fmov	$ivd21,$ivh


	vorr	$dat1,$dat2,$dat2
	vld1.8	{$dat2},[$inp],#16
	vorr	$in0,$dat,$dat
	vorr	$in1,$dat1,$dat1
	veor	$in2,$dat2,$iv2 		// the third block
	veor	$dat2,$dat2,$iv2
	cmp	$len,#32
	b.lo	.Lxts_outer_enc_tail

	// The iv for fourth block
	extr	$midnumx,$ivh,$ivh,#32
	extr	$ivh,$ivh,$ivl,#63
	and	$tmpmw,$constnum,$midnum,asr#31
	eor	$ivl,$tmpmx,$ivl,lsl#1
	fmov	$ivd30,$ivl
	fmov	$ivd31,$ivh

	vld1.8	{$dat3},[$inp],#16
	// The iv for fifth block
	extr	$midnumx,$ivh,$ivh,#32
	extr	$ivh,$ivh,$ivl,#63
	and	$tmpmw,$constnum,$midnum,asr#31
	eor	$ivl,$tmpmx,$ivl,lsl#1
	fmov	$ivd40,$ivl
	fmov	$ivd41,$ivh

	vld1.8	{$dat4},[$inp],#16
	veor	$dat3,$dat3,$iv3		// the fourth block
	veor	$dat4,$dat4,$iv4
	sub	$len,$len,#32			// bias
	mov	$rounds,$rounds0
	b	.Loop5x_xts_enc

.align	4
.Loop5x_xts_enc:
	aese	$dat0,q8
	aesmc	$dat0,$dat0
	aese	$dat1,q8
	aesmc	$dat1,$dat1
	aese	$dat2,q8
	aesmc	$dat2,$dat2
	aese	$dat3,q8
	aesmc	$dat3,$dat3
	aese	$dat4,q8
	aesmc	$dat4,$dat4
	vld1.32	{q8},[$key_],#16
	subs	$rounds,$rounds,#2
	aese	$dat0,q9
	aesmc	$dat0,$dat0
	aese	$dat1,q9
	aesmc	$dat1,$dat1
	aese	$dat2,q9
	aesmc	$dat2,$dat2
	aese	$dat3,q9
	aesmc	$dat3,$dat3
	aese	$dat4,q9
	aesmc	$dat4,$dat4
	vld1.32	{q9},[$key_],#16
	b.gt	.Loop5x_xts_enc

	aese	$dat0,q8
	aesmc	$dat0,$dat0
	aese	$dat1,q8
	aesmc	$dat1,$dat1
	aese	$dat2,q8
	aesmc	$dat2,$dat2
	aese	$dat3,q8
	aesmc	$dat3,$dat3
	aese	$dat4,q8
	aesmc	$dat4,$dat4
	subs	$len,$len,#0x50			// because .Lxts_enc_tail4x

	aese	$dat0,q9
	aesmc	$dat0,$dat0
	aese	$dat1,q9
	aesmc	$dat1,$dat1
	aese	$dat2,q9
	aesmc	$dat2,$dat2
	aese	$dat3,q9
	aesmc	$dat3,$dat3
	aese	$dat4,q9
	aesmc	$dat4,$dat4
	csel	$xoffset,xzr,$len,gt		// borrow x6, w6, "gt" is not typo
	mov	$key_,$key1

	aese	$dat0,q10
	aesmc	$dat0,$dat0
	aese	$dat1,q10
	aesmc	$dat1,$dat1
	aese	$dat2,q10
	aesmc	$dat2,$dat2
	aese	$dat3,q10
	aesmc	$dat3,$dat3
	aese	$dat4,q10
	aesmc	$dat4,$dat4
	add	$inp,$inp,$xoffset		// x0 is adjusted in such way that
						// at exit from the loop v1.16b-v26.16b
						// are loaded with last "words"
	add	$xoffset,$len,#0x60		// because .Lxts_enc_tail4x

	aese	$dat0,q11
	aesmc	$dat0,$dat0
	aese	$dat1,q11
	aesmc	$dat1,$dat1
	aese	$dat2,q11
	aesmc	$dat2,$dat2
	aese	$dat3,q11
	aesmc	$dat3,$dat3
	aese	$dat4,q11
	aesmc	$dat4,$dat4

	aese	$dat0,q12
	aesmc	$dat0,$dat0
	aese	$dat1,q12
	aesmc	$dat1,$dat1
	aese	$dat2,q12
	aesmc	$dat2,$dat2
	aese	$dat3,q12
	aesmc	$dat3,$dat3
	aese	$dat4,q12
	aesmc	$dat4,$dat4

	aese	$dat0,q13
	aesmc	$dat0,$dat0
	aese	$dat1,q13
	aesmc	$dat1,$dat1
	aese	$dat2,q13
	aesmc	$dat2,$dat2
	aese	$dat3,q13
	aesmc	$dat3,$dat3
	aese	$dat4,q13
	aesmc	$dat4,$dat4

	aese	$dat0,q14
	aesmc	$dat0,$dat0
	aese	$dat1,q14
	aesmc	$dat1,$dat1
	aese	$dat2,q14
	aesmc	$dat2,$dat2
	aese	$dat3,q14
	aesmc	$dat3,$dat3
	aese	$dat4,q14
	aesmc	$dat4,$dat4

	veor	$tmp0,$rndlast,$iv0
	aese	$dat0,q15
	// The iv for first block of one iteration
	extr	$midnumx,$ivh,$ivh,#32
	extr	$ivh,$ivh,$ivl,#63
	and	$tmpmw,$constnum,$midnum,asr#31
	eor	$ivl,$tmpmx,$ivl,lsl#1
	fmov	$ivd00,$ivl
	fmov	$ivd01,$ivh
	veor	$tmp1,$rndlast,$iv1
	vld1.8	{$in0},[$inp],#16
	aese	$dat1,q15
	// The iv for second block
	extr	$midnumx,$ivh,$ivh,#32
	extr	$ivh,$ivh,$ivl,#63
	and	$tmpmw,$constnum,$midnum,asr#31
	eor	$ivl,$tmpmx,$ivl,lsl#1
	fmov	$ivd10,$ivl
	fmov	$ivd11,$ivh
	veor	$tmp2,$rndlast,$iv2
	vld1.8	{$in1},[$inp],#16
	aese	$dat2,q15
	// The iv for third block
	extr	$midnumx,$ivh,$ivh,#32
	extr	$ivh,$ivh,$ivl,#63
	and	$tmpmw,$constnum,$midnum,asr#31
	eor	$ivl,$tmpmx,$ivl,lsl#1
	fmov	$ivd20,$ivl
	fmov	$ivd21,$ivh
	veor	$tmp3,$rndlast,$iv3
	vld1.8	{$in2},[$inp],#16
	aese	$dat3,q15
	// The iv for fourth block
	extr	$midnumx,$ivh,$ivh,#32
	extr	$ivh,$ivh,$ivl,#63
	and	$tmpmw,$constnum,$midnum,asr#31
	eor	$ivl,$tmpmx,$ivl,lsl#1
	fmov	$ivd30,$ivl
	fmov	$ivd31,$ivh
	veor	$tmp4,$rndlast,$iv4
	vld1.8	{$in3},[$inp],#16
	aese	$dat4,q15

	// The iv for fifth block
	extr	$midnumx,$ivh,$ivh,#32
	extr	$ivh,$ivh,$ivl,#63
	and	$tmpmw,$constnum,$midnum,asr #31
	eor	$ivl,$tmpmx,$ivl,lsl #1
	fmov	$ivd40,$ivl
	fmov	$ivd41,$ivh

	vld1.8	{$in4},[$inp],#16
	cbz	$xoffset,.Lxts_enc_tail4x
	vld1.32 {q8},[$key_],#16		// re-pre-load rndkey[0]
	veor	$tmp0,$tmp0,$dat0
	veor	$dat0,$in0,$iv0
	veor	$tmp1,$tmp1,$dat1
	veor	$dat1,$in1,$iv1
	veor	$tmp2,$tmp2,$dat2
	veor	$dat2,$in2,$iv2
	veor	$tmp3,$tmp3,$dat3
	veor	$dat3,$in3,$iv3
	veor	$tmp4,$tmp4,$dat4
	vst1.8	{$tmp0},[$out],#16
	veor	$dat4,$in4,$iv4
	vst1.8	{$tmp1},[$out],#16
	mov	$rounds,$rounds0
	vst1.8	{$tmp2},[$out],#16
	vld1.32	{q9},[$key_],#16		// re-pre-load rndkey[1]
	vst1.8	{$tmp3},[$out],#16
	vst1.8	{$tmp4},[$out],#16
	b.hs	.Loop5x_xts_enc


	// If left 4 blocks, borrow the five block's processing.
	cmn	$len,#0x10
	b.ne	.Loop5x_enc_after
	vorr	$iv4,$iv3,$iv3
	vorr	$iv3,$iv2,$iv2
	vorr	$iv2,$iv1,$iv1
	vorr	$iv1,$iv0,$iv0
	fmov	$ivl,$ivd40
	fmov	$ivh,$ivd41
	veor	$dat0,$iv0,$in0
	veor	$dat1,$iv1,$in1
	veor	$dat2,$in2,$iv2
	veor	$dat3,$in3,$iv3
	veor	$dat4,$in4,$iv4
	b.eq	.Loop5x_xts_enc

.Loop5x_enc_after:
	add	$len,$len,#0x50
	cbz	$len,.Lxts_enc_done

	add	$rounds,$rounds0,#2
	subs	$len,$len,#0x30
	b.lo	.Lxts_inner_enc_tail

	veor	$dat0,$iv0,$in2
	veor	$dat1,$iv1,$in3
	veor	$dat2,$in4,$iv2
	b	.Lxts_outer_enc_tail

.align	4
.Lxts_enc_tail4x:
	add	$inp,$inp,#16
	veor	$tmp1,$dat1,$tmp1
	vst1.8	{$tmp1},[$out],#16
	veor	$tmp2,$dat2,$tmp2
	vst1.8	{$tmp2},[$out],#16
	veor	$tmp3,$dat3,$tmp3
	veor	$tmp4,$dat4,$tmp4
	vst1.8	{$tmp3-$tmp4},[$out],#32

	b	.Lxts_enc_done
.align	4
.Lxts_outer_enc_tail:
	aese	$dat0,q8
	aesmc	$dat0,$dat0
	aese	$dat1,q8
	aesmc	$dat1,$dat1
	aese	$dat2,q8
	aesmc	$dat2,$dat2
	vld1.32	{q8},[$key_],#16
	subs	$rounds,$rounds,#2
	aese	$dat0,q9
	aesmc	$dat0,$dat0
	aese	$dat1,q9
	aesmc	$dat1,$dat1
	aese	$dat2,q9
	aesmc	$dat2,$dat2
	vld1.32	{q9},[$key_],#16
	b.gt	.Lxts_outer_enc_tail

	aese	$dat0,q8
	aesmc	$dat0,$dat0
	aese	$dat1,q8
	aesmc	$dat1,$dat1
	aese	$dat2,q8
	aesmc	$dat2,$dat2
	veor	$tmp0,$iv0,$rndlast
	subs	$len,$len,#0x30
	// The iv for first block
	fmov	$ivl,$ivd20
	fmov	$ivh,$ivd21
	//mov	$constnum,#0x87
	extr	$midnumx,$ivh,$ivh,#32
	extr	$ivh,$ivh,$ivl,#63
	and	$tmpmw,$constnum,$midnum,asr#31
	eor	$ivl,$tmpmx,$ivl,lsl#1
	fmov	$ivd00,$ivl
	fmov	$ivd01,$ivh
	veor	$tmp1,$iv1,$rndlast
	csel	$xoffset,$len,$xoffset,lo       // x6, w6, is zero at this point
	aese	$dat0,q9
	aesmc	$dat0,$dat0
	aese	$dat1,q9
	aesmc	$dat1,$dat1
	aese	$dat2,q9
	aesmc	$dat2,$dat2
	veor	$tmp2,$iv2,$rndlast

	add	$xoffset,$xoffset,#0x20
	add	$inp,$inp,$xoffset
	mov	$key_,$key1

	aese	$dat0,q12
	aesmc	$dat0,$dat0
	aese	$dat1,q12
	aesmc	$dat1,$dat1
	aese	$dat2,q12
	aesmc	$dat2,$dat2
	aese	$dat0,q13
	aesmc	$dat0,$dat0
	aese	$dat1,q13
	aesmc	$dat1,$dat1
	aese	$dat2,q13
	aesmc	$dat2,$dat2
	aese	$dat0,q14
	aesmc	$dat0,$dat0
	aese	$dat1,q14
	aesmc	$dat1,$dat1
	aese	$dat2,q14
	aesmc	$dat2,$dat2
	aese	$dat0,q15
	aese	$dat1,q15
	aese	$dat2,q15
	vld1.8	{$in2},[$inp],#16
	add	$rounds,$rounds0,#2
	vld1.32	{q8},[$key_],#16                // re-pre-load rndkey[0]
	veor	$tmp0,$tmp0,$dat0
	veor	$tmp1,$tmp1,$dat1
	veor	$dat2,$dat2,$tmp2
	vld1.32	{q9},[$key_],#16                // re-pre-load rndkey[1]
	vst1.8	{$tmp0},[$out],#16
	vst1.8	{$tmp1},[$out],#16
	vst1.8	{$dat2},[$out],#16
	cmn	$len,#0x30
	b.eq	.Lxts_enc_done
.Lxts_encxor_one:
	vorr	$in3,$in1,$in1
	vorr	$in4,$in2,$in2
	nop

.Lxts_inner_enc_tail:
	cmn	$len,#0x10
	veor	$dat1,$in3,$iv0
	veor	$dat2,$in4,$iv1
	b.eq	.Lxts_enc_tail_loop
	veor	$dat2,$in4,$iv0
.Lxts_enc_tail_loop:
	aese	$dat1,q8
	aesmc	$dat1,$dat1
	aese	$dat2,q8
	aesmc	$dat2,$dat2
	vld1.32	{q8},[$key_],#16
	subs	$rounds,$rounds,#2
	aese	$dat1,q9
	aesmc	$dat1,$dat1
	aese	$dat2,q9
	aesmc	$dat2,$dat2
	vld1.32	{q9},[$key_],#16
	b.gt	.Lxts_enc_tail_loop

	aese	$dat1,q8
	aesmc	$dat1,$dat1
	aese	$dat2,q8
	aesmc	$dat2,$dat2
	aese	$dat1,q9
	aesmc	$dat1,$dat1
	aese	$dat2,q9
	aesmc	$dat2,$dat2
	aese	$dat1,q12
	aesmc	$dat1,$dat1
	aese	$dat2,q12
	aesmc	$dat2,$dat2
	cmn	$len,#0x20
	aese	$dat1,q13
	aesmc	$dat1,$dat1
	aese	$dat2,q13
	aesmc	$dat2,$dat2
	veor	$tmp1,$iv0,$rndlast
	aese	$dat1,q14
	aesmc	$dat1,$dat1
	aese	$dat2,q14
	aesmc	$dat2,$dat2
	veor	$tmp2,$iv1,$rndlast
	aese	$dat1,q15
	aese	$dat2,q15
	b.eq	.Lxts_enc_one
	veor	$tmp1,$tmp1,$dat1
	vst1.8	{$tmp1},[$out],#16
	veor	$tmp2,$tmp2,$dat2
	vorr	$iv0,$iv1,$iv1
	vst1.8	{$tmp2},[$out],#16
	fmov	$ivl,$ivd10
	fmov	$ivh,$ivd11
	mov	$constnum,#0x87
	extr	$midnumx,$ivh,$ivh,#32
	extr	$ivh,$ivh,$ivl,#63
	and	$tmpmw,$constnum,$midnum,asr #31
	eor	$ivl,$tmpmx,$ivl,lsl #1
	fmov	$ivd00,$ivl
	fmov	$ivd01,$ivh
	b	.Lxts_enc_done

.Lxts_enc_one:
	veor	$tmp1,$tmp1,$dat2
	vorr	$iv0,$iv0,$iv0
	vst1.8	{$tmp1},[$out],#16
	fmov	$ivl,$ivd00
	fmov	$ivh,$ivd01
	mov	$constnum,#0x87
	extr	$midnumx,$ivh,$ivh,#32
	extr	$ivh,$ivh,$ivl,#63
	and	$tmpmw,$constnum,$midnum,asr #31
	eor	$ivl,$tmpmx,$ivl,lsl #1
	fmov	$ivd00,$ivl
	fmov	$ivd01,$ivh
	b	.Lxts_enc_done
.align	5
.Lxts_enc_done:
	// Process the tail block with cipher stealing.
	tst	$tailcnt,#0xf
	b.eq	.Lxts_abort

	mov	$tmpinp,$inp
	mov	$tmpoutp,$out
	sub	$out,$out,#16
.composite_enc_loop:
	subs	$tailcnt,$tailcnt,#1
	ldrb	$l2outp,[$out,$tailcnt]
	ldrb	$loutp,[$tmpinp,$tailcnt]
	strb	$l2outp,[$tmpoutp,$tailcnt]
	strb	$loutp,[$out,$tailcnt]
	b.gt	.composite_enc_loop
.Lxts_enc_load_done:
	vld1.8	{$tmpin},[$out]
	veor	$tmpin,$tmpin,$iv0

	// Encrypt the composite block to get the last second encrypted text block
	ldr	$rounds,[$key1,#240]		// load key schedule...
	vld1.32	{$dat},[$key1],#16
	sub	$rounds,$rounds,#2
	vld1.32	{$dat1},[$key1],#16		// load key schedule...
.Loop_final_enc:
	aese	$tmpin,$dat0
	aesmc	$tmpin,$tmpin
	vld1.32	{$dat0},[$key1],#16
	subs	$rounds,$rounds,#2
	aese	$tmpin,$dat1
	aesmc	$tmpin,$tmpin
	vld1.32	{$dat1},[$key1],#16
	b.gt	.Loop_final_enc

	aese	$tmpin,$dat0
	aesmc	$tmpin,$tmpin
	vld1.32	{$dat0},[$key1]
	aese	$tmpin,$dat1
	veor	$tmpin,$tmpin,$dat0
	veor	$tmpin,$tmpin,$iv0
	vst1.8	{$tmpin},[$out]

.Lxts_abort:
	ldp	$tailcnt,$midnumx,[sp,#48]
	ldp	$ivd10,$ivd20,[sp,#32]
	ldp	$ivd30,$ivd40,[sp,#16]
	ldp	$constnumx,$tmpinp,[sp],#64
.Lxts_enc_final_abort:
	ret
.size	${prefix}_xts_encrypt,.-${prefix}_xts_encrypt
___

}}}
{{{
my ($inp,$out,$len,$key1,$key2,$ivp)=map("x$_",(0..5));
my ($rounds0,$rounds,$key_,$step,$ivl,$ivh)=("w5","w6","x7","x8","x9","x10");
my ($tmpoutp,$loutp,$l2outp,$tmpinp)=("x13","w14","w15","x20");
my ($tailcnt,$midnum,$midnumx,$constnum,$constnumx)=("x21","w22","x22","w19","x19");
my ($xoffset,$tmpmx,$tmpmw)=("x6","x11","w11");
my ($dat0,$dat1,$in0,$in1,$tmp0,$tmp1,$tmp2,$rndlast)=map("q$_",(0..7));
my ($iv0,$iv1,$iv2,$iv3,$iv4,$tmpin)=("v6.16b","v8.16b","v9.16b","v10.16b","v11.16b","v26.16b");
my ($ivd00,$ivd01,$ivd20,$ivd21)=("d6","v6.d[1]","d9","v9.d[1]");
my ($ivd10,$ivd11,$ivd30,$ivd31,$ivd40,$ivd41)=("d8","v8.d[1]","d10","v10.d[1]","d11","v11.d[1]");

my ($dat,$tmp,$rndzero_n_last)=($dat0,$tmp0,$tmp1);

# q7	last round key
# q10-q15, q7	Last 7 round keys
# q8-q9	preloaded round keys except last 7 keys for big size
# q20, q21, q8-q9	preloaded round keys except last 7 keys for only 16 byte

{
my ($dat2,$in2,$tmp2)=map("q$_",(10,11,9));

my ($dat3,$in3,$tmp3);	# used only in 64-bit mode
my ($dat4,$in4,$tmp4);
if ($flavour =~ /64/) {
    ($dat2,$dat3,$dat4,$in2,$in3,$in4,$tmp3,$tmp4)=map("q$_",(16..23));
}

$code.=<<___	if ($flavour =~ /64/);
.globl	${prefix}_xts_decrypt
.type	${prefix}_xts_decrypt,%function
.align	5
${prefix}_xts_decrypt:
	AARCH64_VALID_CALL_TARGET
___
$code.=<<___	if ($flavour =~ /64/);
	cmp	$len,#16
	// Original input data size bigger than 16, jump to big size processing.
	b.ne	.Lxts_dec_big_size
	// Encrypt the iv with key2, as the first XEX iv.
	ldr	$rounds,[$key2,#240]
	vld1.32	{$dat},[$key2],#16
	vld1.8	{$iv0},[$ivp]
	sub	$rounds,$rounds,#2
	vld1.32	{$dat1},[$key2],#16

.Loop_dec_small_iv_enc:
	aese	$iv0,$dat
	aesmc	$iv0,$iv0
	vld1.32	{$dat},[$key2],#16
	subs	$rounds,$rounds,#2
	aese	$iv0,$dat1
	aesmc	$iv0,$iv0
	vld1.32	{$dat1},[$key2],#16
	b.gt	.Loop_dec_small_iv_enc

	aese	$iv0,$dat
	aesmc	$iv0,$iv0
	vld1.32	{$dat},[$key2]
	aese	$iv0,$dat1
	veor	$iv0,$iv0,$dat

	vld1.8	{$dat0},[$inp]
	veor	$dat0,$iv0,$dat0

	ldr	$rounds,[$key1,#240]
	vld1.32	{q20-q21},[$key1],#32			// load key schedule...

	aesd	$dat0,q20
	aesimc	$dat0,$dat0
	vld1.32	{q8-q9},[$key1],#32			// load key schedule...
	aesd	$dat0,q21
	aesimc	$dat0,$dat0
	subs	$rounds,$rounds,#10			// bias
	b.eq	.Lxts_128_dec
.Lxts_dec_round_loop:
	aesd	$dat0,q8
	aesimc	$dat0,$dat0
	vld1.32	{q8},[$key1],#16			// load key schedule...
	aesd	$dat0,q9
	aesimc	$dat0,$dat0
	vld1.32	{q9},[$key1],#16			// load key schedule...
	subs	$rounds,$rounds,#2			// bias
	b.gt	.Lxts_dec_round_loop
.Lxts_128_dec:
	vld1.32	{q10-q11},[$key1],#32			// load key schedule...
	aesd	$dat0,q8
	aesimc	$dat0,$dat0
	aesd	$dat0,q9
	aesimc	$dat0,$dat0
	vld1.32	{q12-q13},[$key1],#32			// load key schedule...
	aesd	$dat0,q10
	aesimc	$dat0,$dat0
	aesd	$dat0,q11
	aesimc	$dat0,$dat0
	vld1.32	{q14-q15},[$key1],#32			// load key schedule...
	aesd	$dat0,q12
	aesimc	$dat0,$dat0
	aesd	$dat0,q13
	aesimc	$dat0,$dat0
	vld1.32	{$rndlast},[$key1]
	aesd	$dat0,q14
	aesimc	$dat0,$dat0
	aesd	$dat0,q15
	veor	$dat0,$dat0,$rndlast
	veor	$dat0,$iv0,$dat0
	vst1.8	{$dat0},[$out]
	b	.Lxts_dec_final_abort
.Lxts_dec_big_size:
___
$code.=<<___	if ($flavour =~ /64/);
	stp	$constnumx,$tmpinp,[sp,#-64]!
	stp	$tailcnt,$midnumx,[sp,#48]
	stp	$ivd10,$ivd20,[sp,#32]
	stp	$ivd30,$ivd40,[sp,#16]

	and	$tailcnt,$len,#0xf
	and	$len,$len,#-16
	subs	$len,$len,#16
	mov	$step,#16
	b.lo	.Lxts_dec_abort

	// Encrypt the iv with key2, as the first XEX iv
	ldr	$rounds,[$key2,#240]
	vld1.32	{$dat},[$key2],#16
	vld1.8	{$iv0},[$ivp]
	sub	$rounds,$rounds,#2
	vld1.32	{$dat1},[$key2],#16

.Loop_dec_iv_enc:
	aese	$iv0,$dat
	aesmc	$iv0,$iv0
	vld1.32	{$dat},[$key2],#16
	subs	$rounds,$rounds,#2
	aese	$iv0,$dat1
	aesmc	$iv0,$iv0
	vld1.32	{$dat1},[$key2],#16
	b.gt	.Loop_dec_iv_enc

	aese	$iv0,$dat
	aesmc	$iv0,$iv0
	vld1.32	{$dat},[$key2]
	aese	$iv0,$dat1
	veor	$iv0,$iv0,$dat

	// The iv for second block
	// $ivl- iv(low), $ivh - iv(high)
	// the five ivs stored into, $iv0,$iv1,$iv2,$iv3,$iv4
	fmov	$ivl,$ivd00
	fmov	$ivh,$ivd01
	mov	$constnum,#0x87
	extr	$midnumx,$ivh,$ivh,#32
	extr	$ivh,$ivh,$ivl,#63
	and	$tmpmw,$constnum,$midnum,asr #31
	eor	$ivl,$tmpmx,$ivl,lsl #1
	fmov	$ivd10,$ivl
	fmov	$ivd11,$ivh

	ldr	$rounds0,[$key1,#240]		// load rounds number

	// The iv for third block
	extr	$midnumx,$ivh,$ivh,#32
	extr	$ivh,$ivh,$ivl,#63
	and	$tmpmw,$constnum,$midnum,asr #31
	eor	$ivl,$tmpmx,$ivl,lsl #1
	fmov	$ivd20,$ivl
	fmov	$ivd21,$ivh

	vld1.32	{q8-q9},[$key1]			// load key schedule...
	sub	$rounds0,$rounds0,#6
	add	$key_,$key1,$ivp,lsl#4		// pointer to last 7 round keys
	sub	$rounds0,$rounds0,#2
	vld1.32	{q10-q11},[$key_],#32		// load key schedule...
	vld1.32	{q12-q13},[$key_],#32
	vld1.32	{q14-q15},[$key_],#32
	vld1.32	{$rndlast},[$key_]

	// The iv for fourth block
	extr	$midnumx,$ivh,$ivh,#32
	extr	$ivh,$ivh,$ivl,#63
	and	$tmpmw,$constnum,$midnum,asr #31
	eor	$ivl,$tmpmx,$ivl,lsl #1
	fmov	$ivd30,$ivl
	fmov	$ivd31,$ivh

	add	$key_,$key1,#32
	mov	$rounds,$rounds0
	b	.Lxts_dec

	// Decryption
.align	5
.Lxts_dec:
	tst	$tailcnt,#0xf
	b.eq	.Lxts_dec_begin
	subs	$len,$len,#16
	csel	$step,xzr,$step,eq
	vld1.8	{$dat},[$inp],#16
	b.lo	.Lxts_done
	sub	$inp,$inp,#16
.Lxts_dec_begin:
	vld1.8	{$dat},[$inp],$step
	subs	$len,$len,#32			// bias
	add	$rounds,$rounds0,#2
	vorr	$in1,$dat,$dat
	vorr	$dat1,$dat,$dat
	vorr	$in3,$dat,$dat
	vld1.8	{$dat2},[$inp],#16
	vorr	$in2,$dat2,$dat2
	vorr	$in4,$dat2,$dat2
	b.lo	.Lxts_inner_dec_tail
	veor	$dat,$dat,$iv0			// before decryt, xor with iv
	veor	$dat2,$dat2,$iv1

	vorr	$dat1,$dat2,$dat2
	vld1.8	{$dat2},[$inp],#16
	vorr	$in0,$dat,$dat
	vorr	$in1,$dat1,$dat1
	veor	$in2,$dat2,$iv2			// third block xox with third iv
	veor	$dat2,$dat2,$iv2
	cmp	$len,#32
	b.lo	.Lxts_outer_dec_tail

	vld1.8	{$dat3},[$inp],#16

	// The iv for fifth block
	extr	$midnumx,$ivh,$ivh,#32
	extr	$ivh,$ivh,$ivl,#63
	and	$tmpmw,$constnum,$midnum,asr #31
	eor	$ivl,$tmpmx,$ivl,lsl #1
	fmov	$ivd40,$ivl
	fmov	$ivd41,$ivh

	vld1.8	{$dat4},[$inp],#16
	veor	$dat3,$dat3,$iv3		// the fourth block
	veor	$dat4,$dat4,$iv4
	sub $len,$len,#32			// bias
	mov	$rounds,$rounds0
	b	.Loop5x_xts_dec

.align	4
.Loop5x_xts_dec:
	aesd	$dat0,q8
	aesimc	$dat0,$dat0
	aesd	$dat1,q8
	aesimc	$dat1,$dat1
	aesd	$dat2,q8
	aesimc	$dat2,$dat2
	aesd	$dat3,q8
	aesimc	$dat3,$dat3
	aesd	$dat4,q8
	aesimc	$dat4,$dat4
	vld1.32	{q8},[$key_],#16		// load key schedule...
	subs	$rounds,$rounds,#2
	aesd	$dat0,q9
	aesimc	$dat0,$dat0
	aesd	$dat1,q9
	aesimc	$dat1,$dat1
	aesd	$dat2,q9
	aesimc	$dat2,$dat2
	aesd	$dat3,q9
	aesimc	$dat3,$dat3
	aesd	$dat4,q9
	aesimc	$dat4,$dat4
	vld1.32	{q9},[$key_],#16		// load key schedule...
	b.gt	.Loop5x_xts_dec

	aesd	$dat0,q8
	aesimc	$dat0,$dat0
	aesd	$dat1,q8
	aesimc	$dat1,$dat1
	aesd	$dat2,q8
	aesimc	$dat2,$dat2
	aesd	$dat3,q8
	aesimc	$dat3,$dat3
	aesd	$dat4,q8
	aesimc	$dat4,$dat4
	subs	$len,$len,#0x50			// because .Lxts_dec_tail4x

	aesd	$dat0,q9
	aesimc	$dat0,$dat
	aesd	$dat1,q9
	aesimc	$dat1,$dat1
	aesd	$dat2,q9
	aesimc	$dat2,$dat2
	aesd	$dat3,q9
	aesimc	$dat3,$dat3
	aesd	$dat4,q9
	aesimc	$dat4,$dat4
	csel	$xoffset,xzr,$len,gt		// borrow x6, w6, "gt" is not typo
	mov	$key_,$key1

	aesd	$dat0,q10
	aesimc	$dat0,$dat0
	aesd	$dat1,q10
	aesimc	$dat1,$dat1
	aesd	$dat2,q10
	aesimc	$dat2,$dat2
	aesd	$dat3,q10
	aesimc	$dat3,$dat3
	aesd	$dat4,q10
	aesimc	$dat4,$dat4
	add	$inp,$inp,$xoffset		// x0 is adjusted in such way that
						// at exit from the loop v1.16b-v26.16b
						// are loaded with last "words"
	add	$xoffset,$len,#0x60		// because .Lxts_dec_tail4x

	aesd	$dat0,q11
	aesimc	$dat0,$dat0
	aesd	$dat1,q11
	aesimc	$dat1,$dat1
	aesd	$dat2,q11
	aesimc	$dat2,$dat2
	aesd	$dat3,q11
	aesimc	$dat3,$dat3
	aesd	$dat4,q11
	aesimc	$dat4,$dat4

	aesd	$dat0,q12
	aesimc	$dat0,$dat0
	aesd	$dat1,q12
	aesimc	$dat1,$dat1
	aesd	$dat2,q12
	aesimc	$dat2,$dat2
	aesd	$dat3,q12
	aesimc	$dat3,$dat3
	aesd	$dat4,q12
	aesimc	$dat4,$dat4

	aesd	$dat0,q13
	aesimc	$dat0,$dat0
	aesd	$dat1,q13
	aesimc	$dat1,$dat1
	aesd	$dat2,q13
	aesimc	$dat2,$dat2
	aesd	$dat3,q13
	aesimc	$dat3,$dat3
	aesd	$dat4,q13
	aesimc	$dat4,$dat4

	aesd	$dat0,q14
	aesimc	$dat0,$dat0
	aesd	$dat1,q14
	aesimc	$dat1,$dat1
	aesd	$dat2,q14
	aesimc	$dat2,$dat2
	aesd	$dat3,q14
	aesimc	$dat3,$dat3
	aesd	$dat4,q14
	aesimc	$dat4,$dat4

	veor	$tmp0,$rndlast,$iv0
	aesd	$dat0,q15
	// The iv for first block of next iteration.
	extr	$midnumx,$ivh,$ivh,#32
	extr	$ivh,$ivh,$ivl,#63
	and	$tmpmw,$constnum,$midnum,asr #31
	eor	$ivl,$tmpmx,$ivl,lsl #1
	fmov	$ivd00,$ivl
	fmov	$ivd01,$ivh
	veor	$tmp1,$rndlast,$iv1
	vld1.8	{$in0},[$inp],#16
	aesd	$dat1,q15
	// The iv for second block
	extr	$midnumx,$ivh,$ivh,#32
	extr	$ivh,$ivh,$ivl,#63
	and	$tmpmw,$constnum,$midnum,asr #31
	eor	$ivl,$tmpmx,$ivl,lsl #1
	fmov	$ivd10,$ivl
	fmov	$ivd11,$ivh
	veor	$tmp2,$rndlast,$iv2
	vld1.8	{$in1},[$inp],#16
	aesd	$dat2,q15
	// The iv for third block
	extr	$midnumx,$ivh,$ivh,#32
	extr	$ivh,$ivh,$ivl,#63
	and	$tmpmw,$constnum,$midnum,asr #31
	eor	$ivl,$tmpmx,$ivl,lsl #1
	fmov	$ivd20,$ivl
	fmov	$ivd21,$ivh
	veor	$tmp3,$rndlast,$iv3
	vld1.8	{$in2},[$inp],#16
	aesd	$dat3,q15
	// The iv for fourth block
	extr	$midnumx,$ivh,$ivh,#32
	extr	$ivh,$ivh,$ivl,#63
	and	$tmpmw,$constnum,$midnum,asr #31
	eor	$ivl,$tmpmx,$ivl,lsl #1
	fmov	$ivd30,$ivl
	fmov	$ivd31,$ivh
	veor	$tmp4,$rndlast,$iv4
	vld1.8	{$in3},[$inp],#16
	aesd	$dat4,q15

	// The iv for fifth block
	extr	$midnumx,$ivh,$ivh,#32
	extr	$ivh,$ivh,$ivl,#63
	and	$tmpmw,$constnum,$midnum,asr #31
	eor	$ivl,$tmpmx,$ivl,lsl #1
	fmov	$ivd40,$ivl
	fmov	$ivd41,$ivh

	vld1.8	{$in4},[$inp],#16
	cbz	$xoffset,.Lxts_dec_tail4x
	vld1.32	{q8},[$key_],#16		// re-pre-load rndkey[0]
	veor	$tmp0,$tmp0,$dat0
	veor	$dat0,$in0,$iv0
	veor	$tmp1,$tmp1,$dat1
	veor	$dat1,$in1,$iv1
	veor	$tmp2,$tmp2,$dat2
	veor	$dat2,$in2,$iv2
	veor	$tmp3,$tmp3,$dat3
	veor	$dat3,$in3,$iv3
	veor	$tmp4,$tmp4,$dat4
	vst1.8	{$tmp0},[$out],#16
	veor	$dat4,$in4,$iv4
	vst1.8	{$tmp1},[$out],#16
	mov	$rounds,$rounds0
	vst1.8	{$tmp2},[$out],#16
	vld1.32	{q9},[$key_],#16		// re-pre-load rndkey[1]
	vst1.8	{$tmp3},[$out],#16
	vst1.8	{$tmp4},[$out],#16
	b.hs	.Loop5x_xts_dec

	cmn	$len,#0x10
	b.ne	.Loop5x_dec_after
	// If x2($len) equal to -0x10, the left blocks is 4.
	// After specially processing, utilize the five blocks processing again.
	// It will use the following IVs: $iv0,$iv0,$iv1,$iv2,$iv3.
	vorr	$iv4,$iv3,$iv3
	vorr	$iv3,$iv2,$iv2
	vorr	$iv2,$iv1,$iv1
	vorr	$iv1,$iv0,$iv0
	fmov	$ivl,$ivd40
	fmov	$ivh,$ivd41
	veor	$dat0,$iv0,$in0
	veor	$dat1,$iv1,$in1
	veor	$dat2,$in2,$iv2
	veor	$dat3,$in3,$iv3
	veor	$dat4,$in4,$iv4
	b.eq	.Loop5x_xts_dec

.Loop5x_dec_after:
	add	$len,$len,#0x50
	cbz	$len,.Lxts_done

	add	$rounds,$rounds0,#2
	subs	$len,$len,#0x30
	b.lo	.Lxts_inner_dec_tail

	veor	$dat0,$iv0,$in2
	veor	$dat1,$iv1,$in3
	veor	$dat2,$in4,$iv2
	b	.Lxts_outer_dec_tail

.align	4
.Lxts_dec_tail4x:
	add	$inp,$inp,#16
	tst	$tailcnt,#0xf
	veor	$tmp1,$dat1,$tmp0
	vst1.8	{$tmp1},[$out],#16
	veor	$tmp2,$dat2,$tmp2
	vst1.8	{$tmp2},[$out],#16
	veor	$tmp3,$dat3,$tmp3
	veor	$tmp4,$dat4,$tmp4
	vst1.8	{$tmp3-$tmp4},[$out],#32

	b.eq	.Lxts_dec_abort
	vld1.8	{$dat0},[$inp],#16
	b	.Lxts_done
.align	4
.Lxts_outer_dec_tail:
	aesd	$dat0,q8
	aesimc	$dat0,$dat0
	aesd	$dat1,q8
	aesimc	$dat1,$dat1
	aesd	$dat2,q8
	aesimc	$dat2,$dat2
	vld1.32	{q8},[$key_],#16
	subs	$rounds,$rounds,#2
	aesd	$dat0,q9
	aesimc	$dat0,$dat0
	aesd	$dat1,q9
	aesimc	$dat1,$dat1
	aesd	$dat2,q9
	aesimc	$dat2,$dat2
	vld1.32	{q9},[$key_],#16
	b.gt	.Lxts_outer_dec_tail

	aesd	$dat0,q8
	aesimc	$dat0,$dat0
	aesd	$dat1,q8
	aesimc	$dat1,$dat1
	aesd	$dat2,q8
	aesimc	$dat2,$dat2
	veor	$tmp0,$iv0,$rndlast
	subs	$len,$len,#0x30
	// The iv for first block
	fmov	$ivl,$ivd20
	fmov	$ivh,$ivd21
	mov	$constnum,#0x87
	extr	$midnumx,$ivh,$ivh,#32
	extr	$ivh,$ivh,$ivl,#63
	and	$tmpmw,$constnum,$midnum,asr #31
	eor	$ivl,$tmpmx,$ivl,lsl #1
	fmov	$ivd00,$ivl
	fmov	$ivd01,$ivh
	veor	$tmp1,$iv1,$rndlast
	csel	$xoffset,$len,$xoffset,lo	// x6, w6, is zero at this point
	aesd	$dat0,q9
	aesimc	$dat0,$dat0
	aesd	$dat1,q9
	aesimc	$dat1,$dat1
	aesd	$dat2,q9
	aesimc	$dat2,$dat2
	veor	$tmp2,$iv2,$rndlast
	// The iv for second block
	extr	$midnumx,$ivh,$ivh,#32
	extr	$ivh,$ivh,$ivl,#63
	and	$tmpmw,$constnum,$midnum,asr #31
	eor	$ivl,$tmpmx,$ivl,lsl #1
	fmov	$ivd10,$ivl
	fmov	$ivd11,$ivh

	add	$xoffset,$xoffset,#0x20
	add	$inp,$inp,$xoffset		// $inp is adjusted to the last data

	mov	$key_,$key1

	// The iv for third block
	extr	$midnumx,$ivh,$ivh,#32
	extr	$ivh,$ivh,$ivl,#63
	and	$tmpmw,$constnum,$midnum,asr #31
	eor	$ivl,$tmpmx,$ivl,lsl #1
	fmov	$ivd20,$ivl
	fmov	$ivd21,$ivh

	aesd	$dat0,q12
	aesimc	$dat0,$dat0
	aesd	$dat1,q12
	aesimc	$dat1,$dat1
	aesd	$dat2,q12
	aesimc	$dat2,$dat2
	aesd	$dat0,q13
	aesimc	$dat0,$dat0
	aesd	$dat1,q13
	aesimc	$dat1,$dat1
	aesd	$dat2,q13
	aesimc	$dat2,$dat2
	aesd	$dat0,q14
	aesimc	$dat0,$dat0
	aesd	$dat1,q14
	aesimc	$dat1,$dat1
	aesd	$dat2,q14
	aesimc	$dat2,$dat2
	vld1.8	{$in2},[$inp],#16
	aesd	$dat0,q15
	aesd	$dat1,q15
	aesd	$dat2,q15
	vld1.32	{q8},[$key_],#16		// re-pre-load rndkey[0]
	add	$rounds,$rounds0,#2
	veor	$tmp0,$tmp0,$dat0
	veor	$tmp1,$tmp1,$dat1
	veor	$dat2,$dat2,$tmp2
	vld1.32	{q9},[$key_],#16		// re-pre-load rndkey[1]
	vst1.8	{$tmp0},[$out],#16
	vst1.8	{$tmp1},[$out],#16
	vst1.8	{$dat2},[$out],#16

	cmn	$len,#0x30
	add	$len,$len,#0x30
	b.eq	.Lxts_done
	sub	$len,$len,#0x30
	vorr	$in3,$in1,$in1
	vorr	$in4,$in2,$in2
	nop

.Lxts_inner_dec_tail:
	// $len == -0x10 means two blocks left.
	cmn	$len,#0x10
	veor	$dat1,$in3,$iv0
	veor	$dat2,$in4,$iv1
	b.eq	.Lxts_dec_tail_loop
	veor	$dat2,$in4,$iv0
.Lxts_dec_tail_loop:
	aesd	$dat1,q8
	aesimc	$dat1,$dat1
	aesd	$dat2,q8
	aesimc	$dat2,$dat2
	vld1.32	{q8},[$key_],#16
	subs	$rounds,$rounds,#2
	aesd	$dat1,q9
	aesimc	$dat1,$dat1
	aesd	$dat2,q9
	aesimc	$dat2,$dat2
	vld1.32	{q9},[$key_],#16
	b.gt	.Lxts_dec_tail_loop

	aesd	$dat1,q8
	aesimc	$dat1,$dat1
	aesd	$dat2,q8
	aesimc	$dat2,$dat2
	aesd	$dat1,q9
	aesimc	$dat1,$dat1
	aesd	$dat2,q9
	aesimc	$dat2,$dat2
	aesd	$dat1,q12
	aesimc	$dat1,$dat1
	aesd	$dat2,q12
	aesimc	$dat2,$dat2
	cmn	$len,#0x20
	aesd	$dat1,q13
	aesimc	$dat1,$dat1
	aesd	$dat2,q13
	aesimc	$dat2,$dat2
	veor	$tmp1,$iv0,$rndlast
	aesd	$dat1,q14
	aesimc	$dat1,$dat1
	aesd	$dat2,q14
	aesimc	$dat2,$dat2
	veor	$tmp2,$iv1,$rndlast
	aesd	$dat1,q15
	aesd	$dat2,q15
	b.eq	.Lxts_dec_one
	veor	$tmp1,$tmp1,$dat1
	veor	$tmp2,$tmp2,$dat2
	vorr	$iv0,$iv2,$iv2
	vorr	$iv1,$iv3,$iv3
	vst1.8	{$tmp1},[$out],#16
	vst1.8	{$tmp2},[$out],#16
	add	$len,$len,#16
	b	.Lxts_done

.Lxts_dec_one:
	veor	$tmp1,$tmp1,$dat2
	vorr	$iv0,$iv1,$iv1
	vorr	$iv1,$iv2,$iv2
	vst1.8	{$tmp1},[$out],#16
	add	$len,$len,#32

.Lxts_done:
	tst	$tailcnt,#0xf
	b.eq	.Lxts_dec_abort
	// Processing the last two blocks with cipher stealing.
	mov	x7,x3
	cbnz	x2,.Lxts_dec_1st_done
	vld1.8	{$dat0},[$inp],#16

	// Decrypt the last second block to get the last plain text block
.Lxts_dec_1st_done:
	eor	$tmpin,$dat0,$iv1
	ldr	$rounds,[$key1,#240]
	vld1.32	{$dat0},[$key1],#16
	sub	$rounds,$rounds,#2
	vld1.32	{$dat1},[$key1],#16
.Loop_final_2nd_dec:
	aesd	$tmpin,$dat0
	aesimc	$tmpin,$tmpin
	vld1.32	{$dat0},[$key1],#16		// load key schedule...
	subs	$rounds,$rounds,#2
	aesd	$tmpin,$dat1
	aesimc	$tmpin,$tmpin
	vld1.32	{$dat1},[$key1],#16		// load key schedule...
	b.gt	.Loop_final_2nd_dec

	aesd	$tmpin,$dat0
	aesimc	$tmpin,$tmpin
	vld1.32	{$dat0},[$key1]
	aesd	$tmpin,$dat1
	veor	$tmpin,$tmpin,$dat0
	veor	$tmpin,$tmpin,$iv1
	vst1.8	{$tmpin},[$out]

	mov	$tmpinp,$inp
	add	$tmpoutp,$out,#16

	// Composite the tailcnt "16 byte not aligned block" into the last second plain blocks
	// to get the last encrypted block.
.composite_dec_loop:
	subs	$tailcnt,$tailcnt,#1
	ldrb	$l2outp,[$out,$tailcnt]
	ldrb	$loutp,[$tmpinp,$tailcnt]
	strb	$l2outp,[$tmpoutp,$tailcnt]
	strb	$loutp,[$out,$tailcnt]
	b.gt	.composite_dec_loop
.Lxts_dec_load_done:
	vld1.8	{$tmpin},[$out]
	veor	$tmpin,$tmpin,$iv0

	// Decrypt the composite block to get the last second plain text block
	ldr	$rounds,[$key_,#240]
	vld1.32	{$dat},[$key_],#16
	sub	$rounds,$rounds,#2
	vld1.32	{$dat1},[$key_],#16
.Loop_final_dec:
	aesd	$tmpin,$dat0
	aesimc	$tmpin,$tmpin
	vld1.32	{$dat0},[$key_],#16		// load key schedule...
	subs	$rounds,$rounds,#2
	aesd	$tmpin,$dat1
	aesimc	$tmpin,$tmpin
	vld1.32	{$dat1},[$key_],#16		// load key schedule...
	b.gt	.Loop_final_dec

	aesd	$tmpin,$dat0
	aesimc	$tmpin,$tmpin
	vld1.32	{$dat0},[$key_]
	aesd	$tmpin,$dat1
	veor	$tmpin,$tmpin,$dat0
	veor	$tmpin,$tmpin,$iv0
	vst1.8	{$tmpin},[$out]

.Lxts_dec_abort:
	ldp	$tailcnt,$midnumx,[sp,#48]
	ldp	$ivd10,$ivd20,[sp,#32]
	ldp	$ivd30,$ivd40,[sp,#16]
	ldp	$constnumx,$tmpinp,[sp],#64

.Lxts_dec_final_abort:
	ret
.size	${prefix}_xts_decrypt,.-${prefix}_xts_decrypt
___
}
}}}
$code.=<<___;
#endif
___
########################################
if ($flavour =~ /64/) {			######## 64-bit code
    my %opcode = (
	"aesd"	=>	0x4e285800,	"aese"	=>	0x4e284800,
	"aesimc"=>	0x4e287800,	"aesmc"	=>	0x4e286800	);

    local *unaes = sub {
	my ($mnemonic,$arg)=@_;

	$arg =~ m/[qv]([0-9]+)[^,]*,\s*[qv]([0-9]+)/o	&&
	sprintf ".inst\t0x%08x\t//%s %s",
			$opcode{$mnemonic}|$1|($2<<5),
			$mnemonic,$arg;
    };

    foreach(split("\n",$code)) {
	s/\`([^\`]*)\`/eval($1)/geo;

	s/\bq([0-9]+)\b/"v".($1<8?$1:$1+8).".16b"/geo;	# old->new registers
	s/@\s/\/\//o;			# old->new style commentary

	#s/[v]?(aes\w+)\s+([qv].*)/unaes($1,$2)/geo	or
	s/cclr\s+([wx])([^,]+),\s*([a-z]+)/csel	$1$2,$1zr,$1$2,$3/o	or
	s/mov\.([a-z]+)\s+([wx][0-9]+),\s*([wx][0-9]+)/csel	$2,$3,$2,$1/o	or
	s/vmov\.i8/movi/o	or	# fix up legacy mnemonics
	s/vext\.8/ext/o		or
	s/vrev32\.8/rev32/o	or
	s/vtst\.8/cmtst/o	or
	s/vshr/ushr/o		or
	s/^(\s+)v/$1/o		or	# strip off v prefix
	s/\bbx\s+lr\b/ret/o;

	# fix up remaining legacy suffixes
	s/\.[ui]?8//o;
	m/\],#8/o and s/\.16b/\.8b/go;
	s/\.[ui]?32//o and s/\.16b/\.4s/go;
	s/\.[ui]?64//o and s/\.16b/\.2d/go;
	s/\.[42]([sd])\[([0-3])\]/\.$1\[$2\]/o;

	# Switch preprocessor checks to aarch64 versions.
	s/__ARME([BL])__/__AARCH64E$1__/go;

	print $_,"\n";
    }
} else {				######## 32-bit code
    my %opcode = (
	"aesd"	=>	0xf3b00340,	"aese"	=>	0xf3b00300,
	"aesimc"=>	0xf3b003c0,	"aesmc"	=>	0xf3b00380	);

    local *unaes = sub {
	my ($mnemonic,$arg)=@_;

	if ($arg =~ m/[qv]([0-9]+)[^,]*,\s*[qv]([0-9]+)/o) {
	    my $word = $opcode{$mnemonic}|(($1&7)<<13)|(($1&8)<<19)
					 |(($2&7)<<1) |(($2&8)<<2);
	    # since ARMv7 instructions are always encoded little-endian.
	    # correct solution is to use .inst directive, but older
	    # assemblers don't implement it:-(
	    sprintf "INST(0x%02x,0x%02x,0x%02x,0x%02x)\t@ %s %s",
			$word&0xff,($word>>8)&0xff,
			($word>>16)&0xff,($word>>24)&0xff,
			$mnemonic,$arg;
	}
    };

    sub unvtbl {
	my $arg=shift;

	$arg =~ m/q([0-9]+),\s*\{q([0-9]+)\},\s*q([0-9]+)/o &&
	sprintf	"vtbl.8	d%d,{q%d},d%d\n\t".
		"vtbl.8	d%d,{q%d},d%d", 2*$1,$2,2*$3, 2*$1+1,$2,2*$3+1;
    }

    sub unvdup32 {
	my $arg=shift;

	$arg =~ m/q([0-9]+),\s*q([0-9]+)\[([0-3])\]/o &&
	sprintf	"vdup.32	q%d,d%d[%d]",$1,2*$2+($3>>1),$3&1;
    }

    sub unvmov32 {
	my $arg=shift;

	$arg =~ m/q([0-9]+)\[([0-3])\],(.*)/o &&
	sprintf	"vmov.32	d%d[%d],%s",2*$1+($2>>1),$2&1,$3;
    }

    foreach(split("\n",$code)) {
	s/\`([^\`]*)\`/eval($1)/geo;

	s/\b[wx]([0-9]+)\b/r$1/go;		# new->old registers
	s/\bv([0-9])\.[12468]+[bsd]\b/q$1/go;	# new->old registers
	s/\/\/\s?/@ /o;				# new->old style commentary

	# fix up remaining new-style suffixes
	s/\{q([0-9]+)\},\s*\[(.+)\],#8/sprintf "{d%d},[$2]!",2*$1/eo	or
	s/\],#[0-9]+/]!/o;

	s/[v]?(aes\w+)\s+([qv].*)/unaes($1,$2)/geo	or
	s/cclr\s+([^,]+),\s*([a-z]+)/mov.$2	$1,#0/o	or
	s/vtbl\.8\s+(.*)/unvtbl($1)/geo			or
	s/vdup\.32\s+(.*)/unvdup32($1)/geo		or
	s/vmov\.32\s+(.*)/unvmov32($1)/geo		or
	s/^(\s+)b\./$1b/o				or
	s/^(\s+)ret/$1bx\tlr/o;

	if (s/^(\s+)mov\.([a-z]+)/$1mov$2/) {
	    print "	it	$2\n";
	}

	print $_,"\n";
    }
}

close STDOUT or die "error closing STDOUT: $!";