openssl/crypto/modes/asm/aes-gcm-riscv64-zvkb-zvkg-zvkned.pl

#! /usr/bin/env perl
# This file is dual-licensed, meaning that you can use it under your
# choice of either of the following two licenses:
#
# Copyright 2023 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the Apache License 2.0 (the "License"). You can obtain
# a copy in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html
#
# or
#
# Copyright (c) 2023, Jerry Shih <jerry.shih@sifive.com>
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

# - RV64I
# - RISC-V Vector ('V') with VLEN >= 128
# - RISC-V Vector Cryptography Bit-manipulation extension ('Zvkb')
# - RISC-V Vector GCM/GMAC extension ('Zvkg')
# - RISC-V Vector AES Block Cipher extension ('Zvkned')
# - RISC-V Zicclsm(Main memory supports misaligned loads/stores)

# Reference: https://github.com/riscv/riscv-crypto/issues/192#issuecomment-1270447575
#
# Assume we have 12 GCM blocks and we try to parallelize GCM computation for 4 blocks.
# Tag = M0*H^12 + M1*H^11 + M2*H^10 + M3*H^9 +
#       M4*H^8  + M5*H^7  + M6*H^6  + M7*H^5 +
#       M8*H^4  + M9*H^3  + M10*H^2 + M11*H^1
# We could rewrite the formula into:
# T0 = 0
# T1 = (T0+M1)*H^4   T2 = (T0+M2)*H^4    T3 = (T0+M3)*H^4    T4 = (T0+M4)*H^4
# T5 = (T1+M5)*H^4   T6 = (T2+M6)*H^4    T7 = (T3+M7)*H^4    T8 = (T4+M8)*H^4
# T9 = (T5+M9)*H^4  T10 = (T6+M10)*H^3  T11 = (T7+M11)*H^2  T12 = (T8+M12)*H^1
#
# We will multiply with [H^4, H^4, H^4, H^4] in each steps except the last iteration.
# The last iteration will multiply with [H^4, H^3, H^2, H^1].

use strict;
use warnings;

use FindBin qw($Bin);
use lib "$Bin";
use lib "$Bin/../../perlasm";
use riscv;

# $output is the last argument if it looks like a file (it has an extension)
# $flavour is the first argument if it doesn't look like a file
my $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
my $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;

$output and open STDOUT,">$output";

my $code=<<___;
.text
___

{
my ($INP, $OUTP, $LEN, $KEYP, $IVP, $XIP) = ("a0", "a1", "a2", "a3", "a4", "a5");
my ($T0, $T1, $T2, $T3) = ("t0", "t1", "t2", "t3");
my ($PADDING_LEN32) = ("t4");
my ($LEN32) = ("t5");
my ($CTR) = ("t6");
my ($FULL_BLOCK_LEN32) = ("a6");
my ($ORIGINAL_LEN32) = ("a7");
my ($PROCESSED_LEN) = ("a0");
my ($CTR_MASK) = ("v0");
my ($INPUT_PADDING_MASK) = ("v0");
my ($V0, $V1, $V2, $V3, $V4, $V5, $V6, $V7,
    $V8, $V9, $V10, $V11, $V12, $V13, $V14, $V15,
    $V16, $V17, $V18, $V19, $V20, $V21, $V22, $V23,
    $V24, $V25, $V26, $V27, $V28, $V29, $V30, $V31,
) = map("v$_",(0..31));

# Do aes-128 enc.
sub aes_128_cipher_body {
    my $code=<<___;
    @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}
    @{[vaesz_vs $V28, $V1]}
    @{[vaesem_vs $V28, $V2]}
    @{[vaesem_vs $V28, $V3]}
    @{[vaesem_vs $V28, $V4]}
    @{[vaesem_vs $V28, $V5]}
    @{[vaesem_vs $V28, $V6]}
    @{[vaesem_vs $V28, $V7]}
    @{[vaesem_vs $V28, $V8]}
    @{[vaesem_vs $V28, $V9]}
    @{[vaesem_vs $V28, $V10]}
    @{[vaesef_vs $V28, $V11]}
___

    return $code;
}

# Do aes-192 enc.
sub aes_192_cipher_body {
    my $TMP_REG = shift;

    my $code=<<___;
    # Load key 4
    @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
    addi $TMP_REG, $KEYP, 48
    @{[vle32_v $V11, $TMP_REG]}
    @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}
    @{[vaesz_vs $V28, $V1]}
    @{[vaesem_vs $V28, $V2]}
    @{[vaesem_vs $V28, $V3]}
    @{[vaesem_vs $V28, $V11]}
    # Load key 8
    @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
    addi $TMP_REG, $KEYP, 112
    @{[vle32_v $V11, $TMP_REG]}
    @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}
    @{[vaesem_vs $V28, $V4]}
    @{[vaesem_vs $V28, $V5]}
    @{[vaesem_vs $V28, $V6]}
    @{[vaesem_vs $V28, $V11]}
    # Load key 13
    @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
    addi $TMP_REG, $KEYP, 192
    @{[vle32_v $V11, $TMP_REG]}
    @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}
    @{[vaesem_vs $V28, $V7]}
    @{[vaesem_vs $V28, $V8]}
    @{[vaesem_vs $V28, $V9]}
    @{[vaesem_vs $V28, $V10]}
    @{[vaesef_vs $V28, $V11]}
___

    return $code;
}

# Do aes-256 enc.
sub aes_256_cipher_body {
    my $TMP_REG = shift;

    my $code=<<___;
    # Load key 3
    @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
    addi $TMP_REG, $KEYP, 32
    @{[vle32_v $V11, $TMP_REG]}
    @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}
    @{[vaesz_vs $V28, $V1]}
    @{[vaesem_vs $V28, $V2]}
    @{[vaesem_vs $V28, $V11]}
    # Load key 6
    @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
    addi $TMP_REG, $KEYP, 80
    @{[vle32_v $V11, $TMP_REG]}
    @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}
    @{[vaesem_vs $V28, $V3]}
    @{[vaesem_vs $V28, $V4]}
    @{[vaesem_vs $V28, $V11]}
    # Load key 9
    @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
    addi $TMP_REG, $KEYP, 128
    @{[vle32_v $V11, $TMP_REG]}
    @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}
    @{[vaesem_vs $V28, $V5]}
    @{[vaesem_vs $V28, $V6]}
    @{[vaesem_vs $V28, $V11]}
    # Load key 12
    @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
    addi $TMP_REG, $KEYP, 176
    @{[vle32_v $V11, $TMP_REG]}
    @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}
    @{[vaesem_vs $V28, $V7]}
    @{[vaesem_vs $V28, $V8]}
    @{[vaesem_vs $V28, $V11]}
    # Load key 15
    @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
    addi $TMP_REG, $KEYP, 224
    @{[vle32_v $V11, $TMP_REG]}
    @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}
    @{[vaesem_vs $V28, $V9]}
    @{[vaesem_vs $V28, $V10]}
    @{[vaesef_vs $V28, $V11]}
___

    return $code;
}

sub handle_padding_in_first_round {
    my $TMP_REG = shift;

    my $code=<<___;
    bnez $PADDING_LEN32, 1f

    ## without padding
    # Store ciphertext/plaintext
    @{[vse32_v $V28, $OUTP]}
    j 2f

    ## with padding
1:
    # Store ciphertext/plaintext using mask
    @{[vse32_v $V28, $OUTP, $INPUT_PADDING_MASK]}

    # Fill zero for the padding blocks
    @{[vsetvli "zero", $PADDING_LEN32, "e32", "m4", "tu", "ma"]}
    @{[vmv_v_i $V28, 0]}

    # We have used mask register for `INPUT_PADDING_MASK` before. We need to
    # setup the ctr mask back.
    # ctr mask : [000100010001....]
    @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e8", "m1", "ta", "ma"]}
    li $TMP_REG, 0b10001000
    @{[vmv_v_x $CTR_MASK, $TMP_REG]}
2:

___

    return $code;
}


# Do aes-128 enc for first round.
sub aes_128_first_round {
    my $PTR_OFFSET_REG = shift;
    my $TMP_REG = shift;

    my $code=<<___;
    # Load all 11 aes round keys to v1-v11 registers.
    @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
    @{[vle32_v $V1, $KEYP]}
    addi $KEYP, $KEYP, 16
    @{[vle32_v $V2, $KEYP]}
    addi $KEYP, $KEYP, 16
    @{[vle32_v $V3, $KEYP]}
    addi $KEYP, $KEYP, 16
    @{[vle32_v $V4, $KEYP]}
    addi $KEYP, $KEYP, 16
    @{[vle32_v $V5, $KEYP]}
    addi $KEYP, $KEYP, 16
    @{[vle32_v $V6, $KEYP]}
    addi $KEYP, $KEYP, 16
    @{[vle32_v $V7, $KEYP]}
    addi $KEYP, $KEYP, 16
    @{[vle32_v $V8, $KEYP]}
    addi $KEYP, $KEYP, 16
    @{[vle32_v $V9, $KEYP]}
    addi $KEYP, $KEYP, 16
    @{[vle32_v $V10, $KEYP]}
    addi $KEYP, $KEYP, 16
    @{[vle32_v $V11, $KEYP]}

    # We already have the ciphertext/plaintext and ctr data for the first round.
    @{[aes_128_cipher_body]}

    # Compute AES ctr result.
    @{[vxor_vv $V28, $V28, $V24]}

    @{[handle_padding_in_first_round $TMP_REG]}

    add $INP, $INP, $PTR_OFFSET_REG
    add $OUTP, $OUTP, $PTR_OFFSET_REG
___

    return $code;
}

# Do aes-192 enc for first round.
sub aes_192_first_round {
    my $PTR_OFFSET_REG = shift;
    my $TMP_REG = shift;

    my $code=<<___;
    # We run out of 32 vector registers, so we just preserve some round keys
    # and load the remaining round keys inside the aes body.
    # We keep the round keys for:
    #   1, 2, 3, 5, 6, 7, 9, 10, 11 and 12th keys.
    # The following keys will be loaded in the aes body:
    #   4, 8 and 13th keys.
    @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
    # key 1
    @{[vle32_v $V1, $KEYP]}
    # key 2
    addi $TMP_REG, $KEYP, 16
    @{[vle32_v $V2, $TMP_REG]}
    # key 3
    addi $TMP_REG, $KEYP, 32
    @{[vle32_v $V3, $TMP_REG]}
    # key 5
    addi $TMP_REG, $KEYP, 64
    @{[vle32_v $V4, $TMP_REG]}
    # key 6
    addi $TMP_REG, $KEYP, 80
    @{[vle32_v $V5, $TMP_REG]}
    # key 7
    addi $TMP_REG, $KEYP, 96
    @{[vle32_v $V6, $TMP_REG]}
    # key 9
    addi $TMP_REG, $KEYP, 128
    @{[vle32_v $V7, $TMP_REG]}
    # key 10
    addi $TMP_REG, $KEYP, 144
    @{[vle32_v $V8, $TMP_REG]}
    # key 11
    addi $TMP_REG, $KEYP, 160
    @{[vle32_v $V9, $TMP_REG]}
    # key 12
    addi $TMP_REG, $KEYP, 176
    @{[vle32_v $V10, $TMP_REG]}

    # We already have the ciphertext/plaintext and ctr data for the first round.
    @{[aes_192_cipher_body $TMP_REG]}

    # Compute AES ctr result.
    @{[vxor_vv $V28, $V28, $V24]}

    @{[handle_padding_in_first_round $TMP_REG]}

    add $INP, $INP, $PTR_OFFSET_REG
    add $OUTP, $OUTP, $PTR_OFFSET_REG
___

    return $code;
}

# Do aes-256 enc for first round.
sub aes_256_first_round {
    my $PTR_OFFSET_REG = shift;
    my $TMP_REG = shift;

    my $code=<<___;
    # We run out of 32 vector registers, so we just preserve some round keys
    # and load the remaining round keys inside the aes body.
    # We keep the round keys for:
    #   1, 2, 4, 5, 7, 8, 10, 11, 13 and 14th keys.
    # The following keys will be loaded in the aes body:
    #   3, 6, 9, 12 and 15th keys.
    @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
    # key 1
    @{[vle32_v $V1, $KEYP]}
    # key 2
    addi $TMP_REG, $KEYP, 16
    @{[vle32_v $V2, $TMP_REG]}
    # key 4
    addi $TMP_REG, $KEYP, 48
    @{[vle32_v $V3, $TMP_REG]}
    # key 5
    addi $TMP_REG, $KEYP, 64
    @{[vle32_v $V4, $TMP_REG]}
    # key 7
    addi $TMP_REG, $KEYP, 96
    @{[vle32_v $V5, $TMP_REG]}
    # key 8
    addi $TMP_REG, $KEYP, 112
    @{[vle32_v $V6, $TMP_REG]}
    # key 10
    addi $TMP_REG, $KEYP, 144
    @{[vle32_v $V7, $TMP_REG]}
    # key 11
    addi $TMP_REG, $KEYP, 160
    @{[vle32_v $V8, $TMP_REG]}
    # key 13
    addi $TMP_REG, $KEYP, 192
    @{[vle32_v $V9, $TMP_REG]}
    # key 14
    addi $TMP_REG, $KEYP, 208
    @{[vle32_v $V10, $TMP_REG]}

    # We already have the ciphertext/plaintext and ctr data for the first round.
    @{[aes_256_cipher_body $TMP_REG]}

    # Compute AES ctr result.
    @{[vxor_vv $V28, $V28, $V24]}

    @{[handle_padding_in_first_round $TMP_REG]}

    add $INP, $INP, $PTR_OFFSET_REG
    add $OUTP, $OUTP, $PTR_OFFSET_REG
___

    return $code;
}

sub aes_gcm_init {
    my $code=<<___;
    # Compute the AES-GCM full-block e32 length for `LMUL=4`. We will handle
    # the multiple AES-GCM blocks at the same time within `LMUL=4` register.
    # The AES-GCM's SEW is e32 and EGW is 128 bits.
    #   FULL_BLOCK_LEN32 = (VLEN*LMUL)/(EGW) * (EGW/SEW) = (VLEN*4)/(32*4) * 4
    #                    = (VLEN*4)/32
    # We could get the block_num using the VL value of `vsetvli with e32, m4`.
    @{[vsetvli $FULL_BLOCK_LEN32, "zero", "e32", "m4", "ta", "ma"]}
    # If `LEN32 % FULL_BLOCK_LEN32` is not equal to zero, we could fill the
    # zero padding data to make sure we could always handle FULL_BLOCK_LEN32
    # blocks for all iterations.

    ## Prepare the H^n multiplier in v16 for GCM multiplier. The `n` is the gcm
    ## block number in a LMUL=4 register group.
    ##   n = ((VLEN*LMUL)/(32*4)) = ((VLEN*4)/(32*4))
    ##     = (VLEN/32)
    ## We could use vsetvli with `e32, m1` to compute the `n` number.
    @{[vsetvli $T0, "zero", "e32", "m1", "ta", "ma"]}

    # The H is at `gcm128_context.Htable[0]`(addr(Xi)+16*2).
    addi $T1, $XIP, 32
    @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
    @{[vle32_v $V31, $T1]}

    # Compute the H^n
    li $T1, 1
1:
    @{[vgmul_vv $V31, $V31]}
    slli $T1, $T1, 1
    bltu $T1, $T0, 1b

    @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}
    @{[vmv_v_i $V16, 0]}
    @{[vaesz_vs $V16, $V31]}

    #### Load plaintext into v24 and handle padding. We also load the init tag
    #### data into v20 and prepare the AES ctr input data into v12 and v28.
    @{[vmv_v_i $V20, 0]}

    ## Prepare the AES ctr input data into v12.
    # Setup ctr input mask.
    # ctr mask : [000100010001....]
    # Note: The actual vl should be `FULL_BLOCK_LEN32/4 * 2`, but we just use
    #   `FULL_BLOCK_LEN32` here.
    @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e8", "m1", "ta", "ma"]}
    li $T0, 0b10001000
    @{[vmv_v_x $CTR_MASK, $T0]}
    # Load IV.
    @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
    @{[vle32_v $V31, $IVP]}
    # Convert the big-endian counter into little-endian.
    @{[vsetivli "zero", 4, "e32", "m1", "ta", "mu"]}
    @{[vrev8_v $V31, $V31, $CTR_MASK]}
    # Splat the `single block of IV` to v12
    @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}
    @{[vmv_v_i $V12, 0]}
    @{[vaesz_vs $V12, $V31]}
    # Prepare the ctr counter into v8
    # v8: [x, x, x, 0, x, x, x, 1, x, x, x, 2, ...]
    @{[viota_m $V8, $CTR_MASK, $CTR_MASK]}
    # Merge IV and ctr counter into v12.
    # v12:[x, x, x, count+0, x, x, x, count+1, ...]
    @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "mu"]}
    @{[vadd_vv $V12, $V12, $V8, $CTR_MASK]}

    li $PADDING_LEN32, 0
    # Get the SEW32 size in the first round.
    # If we have the non-zero value for `LEN32&(FULL_BLOCK_LEN32-1)`, then
    # we will have the leading padding zero.
    addi $T0, $FULL_BLOCK_LEN32, -1
    and $T0, $T0, $LEN32
    beqz $T0, 1f

    ## with padding
    sub $LEN32, $LEN32, $T0
    sub $PADDING_LEN32, $FULL_BLOCK_LEN32, $T0
    # padding block size
    srli $T1, $PADDING_LEN32, 2
    # padding byte size
    slli $T2, $PADDING_LEN32, 2

    # Adjust the ctr counter to make the counter start from `counter+0` for the
    # first non-padding block.
    @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "mu"]}
    @{[vsub_vx $V12, $V12, $T1, $CTR_MASK]}
    # Prepare the AES ctr input into v28.
    # The ctr data uses big-endian form.
    @{[vmv_v_v $V28, $V12]}
    @{[vrev8_v $V28, $V28, $CTR_MASK]}

    # Prepare the mask for input loading in the first round. We use
    # `VL=FULL_BLOCK_LEN32` with the mask in the first round.
    # Adjust input ptr.
    sub $INP, $INP, $T2
    # Adjust output ptr.
    sub $OUTP, $OUTP, $T2
    @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e16", "m2", "ta", "ma"]}
    @{[vid_v $V2]}
    # We don't use the pseudo instruction `vmsgeu` here. Use `vmsgtu` instead.
    # The original code is:
    #   vmsgeu.vx $INPUT_PADDING_MASK, $V2, $PADDING_LEN32
    addi $T0, $PADDING_LEN32, -1
    @{[vmsgtu_vx $INPUT_PADDING_MASK, $V2, $T0]}
    @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}
    @{[vmv_v_i $V24, 0]}
    # Load the input for length FULL_BLOCK_LEN32 with mask.
    @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "mu"]}
    @{[vle32_v $V24, $INP, $INPUT_PADDING_MASK]}

    # Load the init `Xi` data to v20 with preceding zero padding.
    # Adjust Xi ptr.
    sub $T0, $XIP, $T2
    # Load for length `zero-padding-e32-length + 4`.
    addi $T1, $PADDING_LEN32, 4
    @{[vsetvli "zero", $T1, "e32", "m4", "tu", "mu"]}
    @{[vle32_v $V20, $T0, $INPUT_PADDING_MASK]}
    j 2f

1:
    ## without padding
    sub $LEN32, $LEN32, $FULL_BLOCK_LEN32

    @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}
    @{[vle32_v $V24, $INP]}

    # Load the init Xi data to v20.
    @{[vsetivli "zero", 4, "e32", "m1", "tu", "ma"]}
    @{[vle32_v $V20, $XIP]}

    # Prepare the AES ctr input into v28.
    # The ctr data uses big-endian form.
    @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "mu"]}
    @{[vmv_v_v $V28, $V12]}
    @{[vrev8_v $V28, $V28, $CTR_MASK]}
2:
___

    return $code;
}

sub prepare_input_and_ctr {
    my $PTR_OFFSET_REG = shift;

    my $code=<<___;
    @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "mu"]}
    # Increase ctr in v12.
    @{[vadd_vx $V12, $V12, $CTR, $CTR_MASK]}
    sub $LEN32, $LEN32, $FULL_BLOCK_LEN32
    # Load plaintext into v24
    @{[vsetvli "zero", "zero", "e32", "m4", "ta", "ma"]}
    @{[vle32_v $V24, $INP]}
    # Prepare the AES ctr input into v28.
    # The ctr data uses big-endian form.
    @{[vmv_v_v $V28, $V12]}
    add $INP, $INP, $PTR_OFFSET_REG
    @{[vsetvli "zero", "zero", "e32", "m4", "ta", "mu"]}
    @{[vrev8_v $V28, $V28, $CTR_MASK]}
___

    return $code;
}

# Store the current CTR back to IV buffer.
sub store_current_ctr {
    my $code=<<___;
    @{[vsetivli "zero", 4, "e32", "m4", "ta", "ma"]}
    # Update current ctr value to v12
    @{[vadd_vx $V12, $V12, $CTR, $CTR_MASK]}
    # Convert ctr to big-endian counter.
    @{[vrev8_v $V12, $V12, $CTR_MASK]}
    @{[vse32_v $V12, $IVP, $CTR_MASK]}
___

    return $code;
}

# Compute the final tag into v0 from the partial tag v20.
sub compute_final_tag {
    my $TMP_REG = shift;

    my $code=<<___;
    # The H is at `gcm128_context.Htable[0]` (addr(Xi)+16*2).
    # Load H to v1
    addi $TMP_REG, $XIP, 32
    @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
    @{[vle32_v $V1, $TMP_REG]}
    # Multiply H for each partial tag and XOR them together.
    # Handle 1st partial tag
    @{[vmv_v_v $V0, $V20]}
    @{[vgmul_vv $V0, $V1]}
    # Handle 2nd to N-th partial tags
    li $TMP_REG, 4
1:
    @{[vsetivli "zero", 4, "e32", "m4", "ta", "ma"]}
    @{[vslidedown_vx $V4, $V20, $TMP_REG]}
    @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
    @{[vghsh_vv $V0, $V1, $V4]}
    addi $TMP_REG, $TMP_REG, 4
    blt $TMP_REG, $FULL_BLOCK_LEN32, 1b
___

    return $code;
}

################################################################################
# size_t rv64i_zvkb_zvkg_zvkned_aes_gcm_encrypt(const unsigned char *in,
#                                               unsigned char *out, size_t len,
#                                               const void *key,
#                                               unsigned char ivec[16], u64 *Xi);
{
$code .= <<___;
.p2align 3
.globl rv64i_zvkb_zvkg_zvkned_aes_gcm_encrypt
.type rv64i_zvkb_zvkg_zvkned_aes_gcm_encrypt,\@function
rv64i_zvkb_zvkg_zvkned_aes_gcm_encrypt:
    srli $T0, $LEN, 4
    beqz $T0, .Lenc_end
    slli $LEN32, $T0, 2

    mv $ORIGINAL_LEN32, $LEN32

    @{[aes_gcm_init]}

    # Load number of rounds
    lwu $T0, 240($KEYP)
    li $T1, 14
    li $T2, 12
    li $T3, 10

    beq $T0, $T1, aes_gcm_enc_blocks_256
    beq $T0, $T2, aes_gcm_enc_blocks_192
    beq $T0, $T3, aes_gcm_enc_blocks_128

.Lenc_end:
    li $PROCESSED_LEN, 0
    ret

.size rv64i_zvkb_zvkg_zvkned_aes_gcm_encrypt,.-rv64i_zvkb_zvkg_zvkned_aes_gcm_encrypt
___

$code .= <<___;
.p2align 3
aes_gcm_enc_blocks_128:
    srli $CTR, $FULL_BLOCK_LEN32, 2
    slli $T0, $FULL_BLOCK_LEN32, 2

    @{[aes_128_first_round $T0, $T1]}

    @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}

.Lenc_blocks_128:
    # Compute the partial tags.
    # The partial tags will multiply with [H^n, H^n, ..., H^n]
    #   [tag0, tag1, ...] =
    #     ([tag0, tag1, ...] + [ciphertext0, ciphertext1, ...] * [H^n, H^n, ..., H^n]
    # We will skip the [H^n, H^n, ..., H^n] multiplication for the last round.
    beqz $LEN32, .Lenc_blocks_128_end
    @{[vghsh_vv $V20, $V16, $V28]}

    @{[prepare_input_and_ctr $T0]}

    @{[aes_128_cipher_body]}

    # Compute AES ctr ciphertext result.
    @{[vxor_vv $V28, $V28, $V24]}

    # Store ciphertext
    @{[vse32_v $V28, $OUTP]}
    add $OUTP, $OUTP, $T0

    j .Lenc_blocks_128
.Lenc_blocks_128_end:

    # Add ciphertext into partial tag
    @{[vxor_vv $V20, $V20, $V28]}

    @{[store_current_ctr]}

    @{[compute_final_tag $T1]}

    # Save the final tag
    @{[vse32_v $V0, $XIP]}

    # return the processed size.
    slli $PROCESSED_LEN, $ORIGINAL_LEN32, 2
    ret
.size aes_gcm_enc_blocks_128,.-aes_gcm_enc_blocks_128
___

$code .= <<___;
.p2align 3
aes_gcm_enc_blocks_192:
    srli $CTR, $FULL_BLOCK_LEN32, 2
    slli $T0, $FULL_BLOCK_LEN32, 2

    @{[aes_192_first_round $T0, $T1]}

    @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}

.Lenc_blocks_192:
    # Compute the partial tags.
    # The partial tags will multiply with [H^n, H^n, ..., H^n]
    #   [tag0, tag1, ...] =
    #     ([tag0, tag1, ...] + [ciphertext0, ciphertext1, ...] * [H^n, H^n, ..., H^n]
    # We will skip the [H^n, H^n, ..., H^n] multiplication for the last round.
    beqz $LEN32, .Lenc_blocks_192_end
    @{[vghsh_vv $V20, $V16, $V28]}

    @{[prepare_input_and_ctr $T0]}

    @{[aes_192_cipher_body $T1]}

    # Compute AES ctr ciphertext result.
    @{[vxor_vv $V28, $V28, $V24]}

    # Store ciphertext
    @{[vse32_v $V28, $OUTP]}
    add $OUTP, $OUTP, $T0

    j .Lenc_blocks_192
.Lenc_blocks_192_end:

    # Add ciphertext into partial tag
    @{[vxor_vv $V20, $V20, $V28]}

    @{[store_current_ctr]}

    @{[compute_final_tag $T1]}

    # Save the final tag
    @{[vse32_v $V0, $XIP]}

    # return the processed size.
    slli $PROCESSED_LEN, $ORIGINAL_LEN32, 2
    ret
.size aes_gcm_enc_blocks_192,.-aes_gcm_enc_blocks_192
___

$code .= <<___;
.p2align 3
aes_gcm_enc_blocks_256:
    srli $CTR, $FULL_BLOCK_LEN32, 2
    slli $T0, $FULL_BLOCK_LEN32, 2

    @{[aes_256_first_round $T0, $T1]}

    @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}

.Lenc_blocks_256:
    # Compute the partial tags.
    # The partial tags will multiply with [H^n, H^n, ..., H^n]
    #   [tag0, tag1, ...] =
    #     ([tag0, tag1, ...] + [ciphertext0, ciphertext1, ...] * [H^n, H^n, ..., H^n]
    # We will skip the [H^n, H^n, ..., H^n] multiplication for the last round.
    beqz $LEN32, .Lenc_blocks_256_end
    @{[vghsh_vv $V20, $V16, $V28]}

    @{[prepare_input_and_ctr $T0]}

    @{[aes_256_cipher_body $T1]}

    # Compute AES ctr ciphertext result.
    @{[vxor_vv $V28, $V28, $V24]}

    # Store ciphertext
    @{[vse32_v $V28, $OUTP]}
    add $OUTP, $OUTP, $T0

    j .Lenc_blocks_256
.Lenc_blocks_256_end:

    # Add ciphertext into partial tag
    @{[vxor_vv $V20, $V20, $V28]}

    @{[store_current_ctr]}

    @{[compute_final_tag $T1]}

    # Save the final tag
    @{[vse32_v $V0, $XIP]}

    # return the processed size.
    slli $PROCESSED_LEN, $ORIGINAL_LEN32, 2
    ret
.size aes_gcm_enc_blocks_256,.-aes_gcm_enc_blocks_256
___

}

################################################################################
# size_t rv64i_zvkb_zvkg_zvkned_aes_gcm_decrypt(const unsigned char *in,
#                                               unsigned char *out, size_t len,
#                                               const void *key,
#                                               unsigned char ivec[16], u64 *Xi);
{
$code .= <<___;
.p2align 3
.globl rv64i_zvkb_zvkg_zvkned_aes_gcm_decrypt
.type rv64i_zvkb_zvkg_zvkned_aes_gcm_decrypt,\@function
rv64i_zvkb_zvkg_zvkned_aes_gcm_decrypt:
    srli $T0, $LEN, 4
    beqz $T0, .Ldec_end
    slli $LEN32, $T0, 2

    mv $ORIGINAL_LEN32, $LEN32

    @{[aes_gcm_init]}

    # Load number of rounds
    lwu $T0, 240($KEYP)
    li $T1, 14
    li $T2, 12
    li $T3, 10

    beq $T0, $T1, aes_gcm_dec_blocks_256
    beq $T0, $T2, aes_gcm_dec_blocks_192
    beq $T0, $T3, aes_gcm_dec_blocks_128

.Ldec_end:
    li $PROCESSED_LEN, 0
    ret
.size rv64i_zvkb_zvkg_zvkned_aes_gcm_decrypt,.-rv64i_zvkb_zvkg_zvkned_aes_gcm_decrypt
___

$code .= <<___;
.p2align 3
aes_gcm_dec_blocks_128:
    srli $CTR, $FULL_BLOCK_LEN32, 2
    slli $T0, $FULL_BLOCK_LEN32, 2

    @{[aes_128_first_round $T0, $T1]}

    @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}

.Ldec_blocks_128:
    # Compute the partial tags.
    # The partial tags will multiply with [H^n, H^n, ..., H^n]
    #   [tag0, tag1, ...] =
    #     ([tag0, tag1, ...] + [ciphertext0, ciphertext1, ...] * [H^n, H^n, ..., H^n]
    # We will skip the [H^n, H^n, ..., H^n] multiplication for the last round.
    beqz $LEN32, .Ldec_blocks_256_end
    @{[vghsh_vv $V20, $V16, $V24]}

    @{[prepare_input_and_ctr $T0]}

    @{[aes_128_cipher_body]}

    # Compute AES ctr plaintext result.
    @{[vxor_vv $V28, $V28, $V24]}

    # Store plaintext
    @{[vse32_v $V28, $OUTP]}
    add $OUTP, $OUTP, $T0

    j .Ldec_blocks_128
.Ldec_blocks_128_end:

    # Add ciphertext into partial tag
    @{[vxor_vv $V20, $V20, $V24]}

    @{[store_current_ctr]}

    @{[compute_final_tag $T1]}

    # Save the final tag
    @{[vse32_v $V0, $XIP]}

    # return the processed size.
    slli $PROCESSED_LEN, $ORIGINAL_LEN32, 2
    ret
.size aes_gcm_dec_blocks_128,.-aes_gcm_dec_blocks_128
___

$code .= <<___;
.p2align 3
aes_gcm_dec_blocks_192:
    srli $CTR, $FULL_BLOCK_LEN32, 2
    slli $T0, $FULL_BLOCK_LEN32, 2

    @{[aes_192_first_round $T0, $T1]}

    @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}

.Ldec_blocks_192:
    # Compute the partial tags.
    # The partial tags will multiply with [H^n, H^n, ..., H^n]
    #   [tag0, tag1, ...] =
    #     ([tag0, tag1, ...] + [ciphertext0, ciphertext1, ...] * [H^n, H^n, ..., H^n]
    # We will skip the [H^n, H^n, ..., H^n] multiplication for the last round.
    beqz $LEN32, .Ldec_blocks_192_end
    @{[vghsh_vv $V20, $V16, $V24]}

    @{[prepare_input_and_ctr $T0]}

    @{[aes_192_cipher_body $T1]}

    # Compute AES ctr plaintext result.
    @{[vxor_vv $V28, $V28, $V24]}

    # Store plaintext
    @{[vse32_v $V28, $OUTP]}
    add $OUTP, $OUTP, $T0

    j .Ldec_blocks_192
.Ldec_blocks_192_end:

    # Add ciphertext into partial tag
    @{[vxor_vv $V20, $V20, $V24]}

    @{[store_current_ctr]}

    @{[compute_final_tag $T1]}

    # Save the final tag
    @{[vse32_v $V0, $XIP]}

    # return the processed size.
    slli $PROCESSED_LEN, $ORIGINAL_LEN32, 2
    ret
.size aes_gcm_dec_blocks_192,.-aes_gcm_dec_blocks_192
___

$code .= <<___;
.p2align 3
aes_gcm_dec_blocks_256:
    srli $CTR, $FULL_BLOCK_LEN32, 2
    slli $T0, $FULL_BLOCK_LEN32, 2

    @{[aes_256_first_round $T0, $T1]}

    @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}

.Ldec_blocks_256:
    # Compute the partial tags.
    # The partial tags will multiply with [H^n, H^n, ..., H^n]
    #   [tag0, tag1, ...] =
    #     ([tag0, tag1, ...] + [ciphertext0, ciphertext1, ...] * [H^n, H^n, ..., H^n]
    # We will skip the [H^n, H^n, ..., H^n] multiplication for the last round.
    beqz $LEN32, .Ldec_blocks_256_end
    @{[vghsh_vv $V20, $V16, $V24]}

    @{[prepare_input_and_ctr $T0]}

    @{[aes_256_cipher_body $T1]}

    # Compute AES ctr plaintext result.
    @{[vxor_vv $V28, $V28, $V24]}

    # Store plaintext
    @{[vse32_v $V28, $OUTP]}
    add $OUTP, $OUTP, $T0

    j .Ldec_blocks_256
.Ldec_blocks_256_end:

    # Add ciphertext into partial tag
    @{[vxor_vv $V20, $V20, $V24]}

    @{[store_current_ctr]}

    @{[compute_final_tag $T1]}

    # Save the final tag
    @{[vse32_v $V0, $XIP]}

    # return the processed size.
    slli $PROCESSED_LEN, $ORIGINAL_LEN32, 2
    ret
.size aes_gcm_dec_blocks_256,.-aes_gcm_dec_blocks_256
___

}
}

print $code;

close STDOUT or die "error closing STDOUT: $!";