Viewing file:  aes-gcm-armv8-unroll8_64.pl (316.98 KB) -rw-r--r--
Select action/file-type:
 (+) |  (+) |  (+) | Code (+) | Session (+) |  (+) | SDB (+) |  (+) |  (+) |  (+) |  (+) |  (+) |
#! /usr/bin/env perl
# Copyright 2020-2022 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the Apache License 2.0 (the "License"). You may not use
# this file except in compliance with the License. You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html
#
#========================================================================
# Written by Xiaokang Qian <xiaokang.qian@arm.com> for the OpenSSL project,
# derived from https://github.com/ARM-software/AArch64cryptolib, original
# author Samuel Lee <Samuel.Lee@arm.com>. The module is, however, dual
# licensed under OpenSSL and SPDX BSD-3-Clause licenses depending on where you
# obtain it.
#========================================================================
#
# Approach - We want to reload constants as we have plenty of spare ASIMD slots around crypto units for loading
# Unroll x8 in main loop, main loop to act on 8 16B blocks per iteration, and then do modulo of the accumulated
# intermediate hashesfrom the 8 blocks.
#
# ____________________________________________________
# | |
# | PRE |
# |____________________________________________________|
# | | | |
# | CTR block 8k+13| AES block 8k+8 | GHASH block 8k+0 |
# |________________|________________|__________________|
# | | | |
# | CTR block 8k+14| AES block 8k+9 | GHASH block 8k+1 |
# |________________|________________|__________________|
# | | | |
# | CTR block 8k+15| AES block 8k+10| GHASH block 8k+2 |
# |________________|________________|__________________|
# | | | |
# | CTR block 8k+16| AES block 8k+11| GHASH block 8k+3 |
# |________________|________________|__________________|
# | | | |
# | CTR block 8k+17| AES block 8k+12| GHASH block 8k+4 |
# |________________|________________|__________________|
# | | | |
# | CTR block 8k+18| AES block 8k+13| GHASH block 8k+5 |
# |________________|________________|__________________|
# | | | |
# | CTR block 8k+19| AES block 8k+14| GHASH block 8k+6 |
# |________________|________________|__________________|
# | | | |
# | CTR block 8k+20| AES block 8k+15| GHASH block 8k+7 |
# |________________|____(mostly)____|__________________|
# | |
# | MODULO |
# |____________________________________________________|
#
# PRE:
# Ensure previous generated intermediate hash is aligned and merged with result for GHASH 4k+0
# EXT low_acc, low_acc, low_acc, #8
# EOR res_curr (8k+0), res_curr (4k+0), low_acc
#
# CTR block:
# Increment and byte reverse counter in scalar registers and transfer to SIMD registers
# REV ctr32, rev_ctr32
# ORR ctr64, constctr96_top32, ctr32, LSL #32
# INS ctr_next.d[0], constctr96_bottom64 // Keeping this in scalar registers to free up space in SIMD RF
# INS ctr_next.d[1], ctr64X
# ADD rev_ctr32, #1
#
# AES block:
# Do AES encryption/decryption on CTR block X and EOR it with input block X. Take 256 bytes key below for example.
# Doing small trick here of loading input in scalar registers, EORing with last key and then transferring
# Given we are very constrained in our ASIMD registers this is quite important
#
# Encrypt:
# LDR input_low, [ input_ptr ], #8
# LDR input_high, [ input_ptr ], #8
# EOR input_low, k14_low
# EOR input_high, k14_high
# INS res_curr.d[0], input_low
# INS res_curr.d[1], input_high
# AESE ctr_curr, k0; AESMC ctr_curr, ctr_curr
# AESE ctr_curr, k1; AESMC ctr_curr, ctr_curr
# AESE ctr_curr, k2; AESMC ctr_curr, ctr_curr
# AESE ctr_curr, k3; AESMC ctr_curr, ctr_curr
# AESE ctr_curr, k4; AESMC ctr_curr, ctr_curr
# AESE ctr_curr, k5; AESMC ctr_curr, ctr_curr
# AESE ctr_curr, k6; AESMC ctr_curr, ctr_curr
# AESE ctr_curr, k7; AESMC ctr_curr, ctr_curr
# AESE ctr_curr, k8; AESMC ctr_curr, ctr_curr
# AESE ctr_curr, k9; AESMC ctr_curr, ctr_curr
# AESE ctr_curr, k10; AESMC ctr_curr, ctr_curr
# AESE ctr_curr, k11; AESMC ctr_curr, ctr_curr
# AESE ctr_curr, k12; AESMC ctr_curr, ctr_curr
# AESE ctr_curr, k13
# EOR res_curr, res_curr, ctr_curr
# ST1 { res_curr.16b }, [ output_ptr ], #16
#
# Decrypt:
# AESE ctr_curr, k0; AESMC ctr_curr, ctr_curr
# AESE ctr_curr, k1; AESMC ctr_curr, ctr_curr
# AESE ctr_curr, k2; AESMC ctr_curr, ctr_curr
# AESE ctr_curr, k3; AESMC ctr_curr, ctr_curr
# AESE ctr_curr, k4; AESMC ctr_curr, ctr_curr
# AESE ctr_curr, k5; AESMC ctr_curr, ctr_curr
# AESE ctr_curr, k6; AESMC ctr_curr, ctr_curr
# AESE ctr_curr, k7; AESMC ctr_curr, ctr_curr
# AESE ctr_curr, k8; AESMC ctr_curr, ctr_curr
# AESE ctr_curr, k9; AESMC ctr_curr, ctr_curr
# AESE ctr_curr, k10; AESMC ctr_curr, ctr_curr
# AESE ctr_curr, k11; AESMC ctr_curr, ctr_curr
# AESE ctr_curr, k12; AESMC ctr_curr, ctr_curr
# AESE ctr_curr, k13
# LDR res_curr, [ input_ptr ], #16
# EOR res_curr, res_curr, ctr_curr
# MOV output_low, res_curr.d[0]
# MOV output_high, res_curr.d[1]
# EOR output_low, k14_low
# EOR output_high, k14_high
# STP output_low, output_high, [ output_ptr ], #16
# GHASH block X:
# Do 128b karatsuba polynomial multiplication on block
# We only have 64b->128b polynomial multipliers, naively that means we need to do 4 64b multiplies to generate a 128b
#
# multiplication:
# Pmull(A,B) == (Pmull(Ah,Bh)<<128 | Pmull(Al,Bl)) ^ (Pmull(Ah,Bl) ^ Pmull(Al,Bh))<<64
#
# The idea behind Karatsuba multiplication is that we can do just 3 64b multiplies:
# Pmull(A,B) == (Pmull(Ah,Bh)<<128 | Pmull(Al,Bl)) ^ (Pmull(Ah^Al,Bh^Bl) ^ Pmull(Ah,Bh) ^ Pmull(Al,Bl))<<64
#
# There is some complication here because the bit order of GHASH's PMULL is reversed compared to elsewhere, so we are
# multiplying with "twisted" powers of H
#
# Note: We can PMULL directly into the acc_x in first GHASH of the loop
# Note: For scheduling big cores we want to split the processing to happen over two loop iterations - otherwise the critical
# path latency dominates the performance
#
# This has a knock on effect on register pressure, so we have to be a bit more clever with our temporary registers
# than indicated here
# REV64 res_curr, res_curr
# INS t_m.d[0], res_curr.d[1]
# EOR t_m.8B, t_m.8B, res_curr.8B
# PMULL2 t_h, res_curr, HX
# PMULL t_l, res_curr, HX
# PMULL t_m, t_m, HX_k
# EOR acc_h, acc_h, t_h
# EOR acc_l, acc_l, t_l
# EOR acc_m, acc_m, t_m
#
# MODULO: take the partial accumulators (~representing sum of 256b multiplication results), from GHASH and do modulo reduction on them
# There is some complication here because the bit order of GHASH's PMULL is reversed compared to elsewhere, so we are doing modulo
# with a reversed constant
# EOR3 acc_m, acc_m, acc_l, acc_h // Finish off karatsuba processing
# PMULL t_mod, acc_h, mod_constant
# EXT acc_h, acc_h, acc_h, #8
# EOR3 acc_m, acc_m, t_mod, acc_h
# PMULL acc_h, acc_m, mod_constant
# EXT acc_m, acc_m, acc_m, #8
# EOR3 acc_l, acc_l, acc_m, acc_h
$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
$flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate ) or
die "can't locate arm-xlate.pl";
die "only for 64 bit" if $flavour !~ /64/;
open OUT,"| \"$^X\" $xlate $flavour $output";
*STDOUT=*OUT;
$code=<<___;
#include "arm_arch.h"
#if __ARM_MAX_ARCH__>=8
___
$code.=".arch armv8.2-a+crypto\n.text\n";
$input_ptr="x0"; #argument block
$bit_length="x1";
$output_ptr="x2";
$current_tag="x3";
$counter="x16";
$constant_temp="x15";
$modulo_constant="x10";
$cc="x8";
{
my ($end_input_ptr,$main_end_input_ptr,$temp0_x,$temp1_x)=map("x$_",(4..7));
my ($temp2_x,$temp3_x)=map("x$_",(13..14));
my ($ctr0b,$ctr1b,$ctr2b,$ctr3b,$ctr4b,$ctr5b,$ctr6b,$ctr7b,$res0b,$res1b,$res2b,$res3b,$res4b,$res5b,$res6b,$res7b)=map("v$_.16b",(0..15));
my ($ctr0,$ctr1,$ctr2,$ctr3,$ctr4,$ctr5,$ctr6,$ctr7,$res0,$res1,$res2,$res3,$res4,$res5,$res6,$res7)=map("v$_",(0..15));
my ($ctr0d,$ctr1d,$ctr2d,$ctr3d,$ctr4d,$ctr5d,$ctr6d,$ctr7d)=map("d$_",(0..7));
my ($ctr0q,$ctr1q,$ctr2q,$ctr3q,$ctr4q,$ctr5q,$ctr6q,$ctr7q)=map("q$_",(0..7));
my ($res0q,$res1q,$res2q,$res3q,$res4q,$res5q,$res6q,$res7q)=map("q$_",(8..15));
my ($ctr_t0,$ctr_t1,$ctr_t2,$ctr_t3,$ctr_t4,$ctr_t5,$ctr_t6,$ctr_t7)=map("v$_",(8..15));
my ($ctr_t0b,$ctr_t1b,$ctr_t2b,$ctr_t3b,$ctr_t4b,$ctr_t5b,$ctr_t6b,$ctr_t7b)=map("v$_.16b",(8..15));
my ($ctr_t0q,$ctr_t1q,$ctr_t2q,$ctr_t3q,$ctr_t4q,$ctr_t5q,$ctr_t6q,$ctr_t7q)=map("q$_",(8..15));
my ($acc_hb,$acc_mb,$acc_lb)=map("v$_.16b",(17..19));
my ($acc_h,$acc_m,$acc_l)=map("v$_",(17..19));
my ($h1,$h12k,$h2,$h3,$h34k,$h4)=map("v$_",(20..25));
my ($h5,$h56k,$h6,$h7,$h78k,$h8)=map("v$_",(20..25));
my ($h1q,$h12kq,$h2q,$h3q,$h34kq,$h4q)=map("q$_",(20..25));
my ($h5q,$h56kq,$h6q,$h7q,$h78kq,$h8q)=map("q$_",(20..25));
my $t0="v16";
my $t0d="d16";
my $t1="v29";
my $t2=$res1;
my $t3=$t1;
my $t4=$res0;
my $t5=$res2;
my $t6=$t0;
my $t7=$res3;
my $t8=$res4;
my $t9=$res5;
my $t10=$res6;
my $t11="v21";
my $t12=$t1;
my $rtmp_ctr="v30";
my $rtmp_ctrq="q30";
my $rctr_inc="v31";
my $rctr_incd="d31";
my $mod_constantd=$t0d;
my $mod_constant=$t0;
my ($rk0,$rk1,$rk2)=map("v$_.16b",(26..28));
my ($rk3,$rk4,$rk5)=map("v$_.16b",(26..28));
my ($rk6,$rk7,$rk8)=map("v$_.16b",(26..28));
my ($rk9,$rk10,$rk11)=map("v$_.16b",(26..28));
my ($rk12,$rk13,$rk14)=map("v$_.16b",(26..28));
my ($rk0q,$rk1q,$rk2q)=map("q$_",(26..28));
my ($rk3q,$rk4q,$rk5q)=map("q$_",(26..28));
my ($rk6q,$rk7q,$rk8q)=map("q$_",(26..28));
my ($rk9q,$rk10q,$rk11q)=map("q$_",(26..28));
my ($rk12q,$rk13q,$rk14q)=map("q$_",(26..28));
my $rk2q1="v28.1q";
my $rk3q1="v26.1q";
my $rk4v="v27";
#########################################################################################
# size_t unroll8_eor3_aes_gcm_enc_128_kernel(const unsigned char *in,
# size_t len,
# unsigned char *out,
# const void *key,
# unsigned char ivec[16],
# u64 *Xi);
#
$code.=<<___;
.global unroll8_eor3_aes_gcm_enc_128_kernel
.type unroll8_eor3_aes_gcm_enc_128_kernel,%function
.align 4
unroll8_eor3_aes_gcm_enc_128_kernel:
AARCH64_VALID_CALL_TARGET
cbz x1, .L128_enc_ret
stp d8, d9, [sp, #-80]!
mov $counter, x4
mov $cc, x5
stp d10, d11, [sp, #16]
stp d12, d13, [sp, #32]
stp d14, d15, [sp, #48]
mov x5, #0xc200000000000000
stp x5, xzr, [sp, #64]
add $modulo_constant, sp, #64
mov $constant_temp, #0x100000000 @ set up counter increment
movi $rctr_inc.16b, #0x0
mov $rctr_inc.d[1], $constant_temp
lsr $main_end_input_ptr, $bit_length, #3 @ byte_len
ld1 { $ctr0b}, [$counter] @ CTR block 0
sub $main_end_input_ptr, $main_end_input_ptr, #1 @ byte_len - 1
and $main_end_input_ptr, $main_end_input_ptr, #0xffffffffffffff80 @ number of bytes to be processed in main loop (at least 1 byte must be handled by tail)
rev32 $rtmp_ctr.16b, $ctr0.16b @ set up reversed counter
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 0
rev32 $ctr1.16b, $rtmp_ctr.16b @ CTR block 1
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 1
rev32 $ctr2.16b, $rtmp_ctr.16b @ CTR block 2
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 2
rev32 $ctr3.16b, $rtmp_ctr.16b @ CTR block 3
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 3
rev32 $ctr4.16b, $rtmp_ctr.16b @ CTR block 4
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 4
rev32 $ctr5.16b, $rtmp_ctr.16b @ CTR block 5
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 5
ldp $rk0q, $rk1q, [$cc, #0] @ load rk0, rk1
rev32 $ctr6.16b, $rtmp_ctr.16b @ CTR block 6
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 6
rev32 $ctr7.16b, $rtmp_ctr.16b @ CTR block 7
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 7
aese $ctr4b, $rk0 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 0
aese $ctr6b, $rk0 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 0
aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 0
aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 0
aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 0
aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 0
aese $ctr7b, $rk0 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 0
aese $ctr5b, $rk0 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 0
ldp $rk2q, $rk3q, [$cc, #32] @ load rk2, rk3
aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 1
aese $ctr7b, $rk1 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 1
aese $ctr5b, $rk1 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 1
aese $ctr4b, $rk1 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 1
aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 1
aese $ctr6b, $rk1 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 1
aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 1
aese $ctr5b, $rk2 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 2
aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 1
aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 2
aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 2
aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 2
aese $ctr7b, $rk2 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 2
aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 2
aese $ctr6b, $rk2 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 2
aese $ctr4b, $rk2 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 2
aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 3
ldp $rk4q, $rk5q, [$cc, #64] @ load rk4, rk5
aese $ctr5b, $rk3 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 3
aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 3
aese $ctr4b, $rk3 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 3
aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 3
aese $ctr6b, $rk3 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 3
aese $ctr7b, $rk3 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 3
aese $ctr6b, $rk4 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 4
aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 3
aese $ctr5b, $rk4 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 4
aese $ctr7b, $rk4 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 4
aese $ctr4b, $rk4 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 4
aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 4
aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 4
aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 4
aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 4
aese $ctr7b, $rk5 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 5
aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 5
ldp $rk6q, $rk7q, [$cc, #96] @ load rk6, rk7
aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 5
aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 5
aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 5
aese $ctr4b, $rk5 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 5
aese $ctr5b, $rk5 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 5
aese $ctr6b, $rk5 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 5
aese $ctr4b, $rk6 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 6
aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 6
aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 6
aese $ctr7b, $rk6 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 6
aese $ctr6b, $rk6 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 6
aese $ctr5b, $rk6 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 6
aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 6
aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 6
ldp $rk8q, $rk9q, [$cc, #128] @ load rk8, rk9
aese $ctr5b, $rk7 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 7
ld1 { $acc_lb}, [$current_tag]
ext $acc_lb, $acc_lb, $acc_lb, #8
rev64 $acc_lb, $acc_lb
aese $ctr7b, $rk7 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 7
aese $ctr4b, $rk7 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 7
aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 7
aese $ctr6b, $rk7 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 7
aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 7
aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 7
aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 7
aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 8
aese $ctr6b, $rk8 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 8
aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 8
aese $ctr7b, $rk8 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 8
aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 8
ldr $rk10q, [$cc, #160] @ load rk10
aese $ctr3b, $rk9 @ AES block 8k+11 - round 9
aese $ctr4b, $rk8 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 8
aese $ctr2b, $rk9 @ AES block 8k+10 - round 9
aese $ctr5b, $rk8 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 8
aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 8
aese $ctr6b, $rk9 @ AES block 8k+14 - round 9
aese $ctr4b, $rk9 @ AES block 8k+12 - round 9
add $main_end_input_ptr, $main_end_input_ptr, $input_ptr
aese $ctr0b, $rk9 @ AES block 8k+8 - round 9
aese $ctr7b, $rk9 @ AES block 8k+15 - round 9
aese $ctr5b, $rk9 @ AES block 8k+13 - round 9
aese $ctr1b, $rk9 @ AES block 8k+9 - round 9
add $end_input_ptr, $input_ptr, $bit_length, lsr #3 @ end_input_ptr
cmp $input_ptr, $main_end_input_ptr @ check if we have <= 8 blocks
b.ge .L128_enc_tail @ handle tail
ldp $ctr_t0q, $ctr_t1q, [$input_ptr], #32 @ AES block 0, 1 - load plaintext
ldp $ctr_t2q, $ctr_t3q, [$input_ptr], #32 @ AES block 2, 3 - load plaintext
ldp $ctr_t4q, $ctr_t5q, [$input_ptr], #32 @ AES block 4, 5 - load plaintext
ldp $ctr_t6q, $ctr_t7q, [$input_ptr], #32 @ AES block 6, 7 - load plaintext
cmp $input_ptr, $main_end_input_ptr @ check if we have <= 8 blocks
eor3 $res0b, $ctr_t0b, $ctr0b, $rk10 @ AES block 0 - result
rev32 $ctr0.16b, $rtmp_ctr.16b @ CTR block 8
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8
eor3 $res1b, $ctr_t1b, $ctr1b, $rk10 @ AES block 1 - result
stp $res0q, $res1q, [$output_ptr], #32 @ AES block 0, 1 - store result
rev32 $ctr1.16b, $rtmp_ctr.16b @ CTR block 9
eor3 $res5b, $ctr_t5b, $ctr5b, $rk10 @ AES block 5 - result
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 9
eor3 $res2b, $ctr_t2b, $ctr2b, $rk10 @ AES block 2 - result
eor3 $res6b, $ctr_t6b, $ctr6b, $rk10 @ AES block 6 - result
eor3 $res4b, $ctr_t4b, $ctr4b, $rk10 @ AES block 4 - result
rev32 $ctr2.16b, $rtmp_ctr.16b @ CTR block 10
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 10
eor3 $res3b, $ctr_t3b, $ctr3b, $rk10 @ AES block 3 - result
eor3 $res7b, $ctr_t7b, $ctr7b,$rk10 @ AES block 7 - result
stp $res2q, $res3q, [$output_ptr], #32 @ AES block 2, 3 - store result
rev32 $ctr3.16b, $rtmp_ctr.16b @ CTR block 11
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 11
stp $res4q, $res5q, [$output_ptr], #32 @ AES block 4, 5 - store result
stp $res6q, $res7q, [$output_ptr], #32 @ AES block 6, 7 - store result
rev32 $ctr4.16b, $rtmp_ctr.16b @ CTR block 12
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 12
b.ge .L128_enc_prepretail @ do prepretail
.L128_enc_main_loop: @ main loop start
rev32 $ctr5.16b, $rtmp_ctr.16b @ CTR block 8k+13
ldr $h5q, [$current_tag, #128] @ load h5l | h5h
ext $h5.16b, $h5.16b, $h5.16b, #8
ldr $h6q, [$current_tag, #160] @ load h6l | h6h
ext $h6.16b, $h6.16b, $h6.16b, #8
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+13
rev64 $res1b, $res1b @ GHASH block 8k+1
rev64 $res0b, $res0b @ GHASH block 8k
ldr $h7q, [$current_tag, #176] @ load h7l | h7h
ext $h7.16b, $h7.16b, $h7.16b, #8
ldr $h8q, [$current_tag, #208] @ load h8l | h8h
ext $h8.16b, $h8.16b, $h8.16b, #8
rev32 $ctr6.16b, $rtmp_ctr.16b @ CTR block 8k+14
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+14
ext $acc_lb, $acc_lb, $acc_lb, #8 @ PRE 0
ldr $h56kq, [$current_tag, #144] @ load h6k | h5k
ldr $h78kq, [$current_tag, #192] @ load h8k | h7k
rev64 $res5b, $res5b @ GHASH block 8k+5 (t0, t1, t2 and t3 free)
rev64 $res3b, $res3b @ GHASH block 8k+3
ldp $rk0q, $rk1q, [$cc, #0] @ load rk0, rk1
eor $res0b, $res0b, $acc_lb @ PRE 1
rev32 $ctr7.16b, $rtmp_ctr.16b @ CTR block 8k+15
rev64 $res7b, $res7b @ GHASH block 8k+7 (t0, t1, t2 and t3 free)
pmull2 $t0.1q, $res1.2d, $h7.2d @ GHASH block 8k+1 - high
rev64 $res2b, $res2b @ GHASH block 8k+2
pmull2 $acc_h.1q, $res0.2d, $h8.2d @ GHASH block 8k - high
pmull $h7.1q, $res1.1d, $h7.1d @ GHASH block 8k+1 - low
trn1 $acc_m.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid
pmull $acc_l.1q, $res0.1d, $h8.1d @ GHASH block 8k - low
trn2 $res0.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid
pmull2 $t1.1q, $res2.2d, $h6.2d @ GHASH block 8k+2 - high
pmull2 $t2.1q, $res3.2d, $h5.2d @ GHASH block 8k+3 - high
eor $acc_lb, $acc_lb, $h7.16b @ GHASH block 8k+1 - low
ldr $h3q, [$current_tag, #80] @ load h3l | h3h
ext $h3.16b, $h3.16b, $h3.16b, #8
ldr $h4q, [$current_tag, #112] @ load h3l | h3h
ext $h4.16b, $h4.16b, $h4.16b, #8
aese $ctr5b, $rk0 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 0
aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 0
aese $ctr4b, $rk0 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 0
eor $acc_hb, $acc_hb, $t0.16b @ GHASH block 8k+1 - high
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+15
aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 0
eor $res0.16b, $res0.16b, $acc_m.16b @ GHASH block 8k, 8k+1 - mid
aese $ctr6b, $rk0 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 0
aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 1
aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 0
aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 1
aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 0
pmull $h6.1q, $res2.1d, $h6.1d @ GHASH block 8k+2 - low
aese $ctr5b, $rk1 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 1
aese $ctr7b, $rk0 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 0
aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 1
eor3 $acc_hb, $acc_hb, $t1.16b,$t2.16b @ GHASH block 8k+2, 8k+3 - high
trn1 $t3.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid
trn2 $res2.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid
ldp $rk2q, $rk3q, [$cc, #32] @ load rk2, rk3
aese $ctr4b, $rk1 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 1
aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 1
pmull $h5.1q, $res3.1d, $h5.1d @ GHASH block 8k+3 - low
aese $ctr7b, $rk1 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 1
aese $ctr6b, $rk1 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 1
pmull2 $acc_m.1q, $res0.2d, $h78k.2d @ GHASH block 8k - mid
eor $res2.16b, $res2.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid
pmull $h78k.1q, $res0.1d, $h78k.1d @ GHASH block 8k+1 - mid
rev64 $res6b, $res6b @ GHASH block 8k+6 (t0, t1, and t2 free)
eor3 $acc_lb, $acc_lb, $h6.16b, $h5.16b @ GHASH block 8k+2, 8k+3 - low
pmull2 $t3.1q, $res2.2d, $h56k.2d @ GHASH block 8k+2 - mid
eor $acc_mb, $acc_mb, $h78k.16b @ GHASH block 8k+1 - mid
pmull $h56k.1q, $res2.1d, $h56k.1d @ GHASH block 8k+3 - mid
aese $ctr5b, $rk2 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 2
aese $ctr4b, $rk2 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 2
aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 2
aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 2
eor3 $acc_mb, $acc_mb, $h56k.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid
aese $ctr6b, $rk2 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 2
aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 2
aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 2
aese $ctr7b, $rk2 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 2
aese $ctr6b, $rk3 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 3
ldr $h12kq, [$current_tag, #48] @ load h2k | h1k
ldr $h34kq, [$current_tag, #96] @ load h4k | h3k
rev64 $res4b, $res4b @ GHASH block 8k+4 (t0, t1, and t2 free)
ldp $rk4q, $rk5q, [$cc, #64] @ load rk4, rk5
aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 3
aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 3
ldr $h1q, [$current_tag, #32] @ load h1l | h1h
ext $h1.16b, $h1.16b, $h1.16b, #8
ldr $h2q, [$current_tag, #64] @ load h1l | h1h
ext $h2.16b, $h2.16b, $h2.16b, #8
pmull2 $t4.1q, $res4.2d, $h4.2d @ GHASH block 8k+4 - high
pmull $h4.1q, $res4.1d, $h4.1d @ GHASH block 8k+4 - low
trn1 $t6.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid
trn2 $res4.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid
aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 3
aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 3
aese $ctr7b, $rk3 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 3
aese $ctr4b, $rk3 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 3
aese $ctr5b, $rk3 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 3
aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 4
aese $ctr7b, $rk4 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 4
aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 4
aese $ctr4b, $rk4 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 4
aese $ctr5b, $rk4 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 4
aese $ctr6b, $rk4 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 4
aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 4
pmull2 $t5.1q, $res5.2d, $h3.2d @ GHASH block 8k+5 - high
eor $res4.16b, $res4.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid
pmull $h3.1q, $res5.1d, $h3.1d @ GHASH block 8k+5 - low
aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 4
ldp $rk6q, $rk7q, [$cc, #96] @ load rk6, rk7
trn1 $t9.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid
pmull2 $t6.1q, $res4.2d, $h34k.2d @ GHASH block 8k+4 - mid
pmull $h34k.1q, $res4.1d, $h34k.1d @ GHASH block 8k+5 - mid
pmull2 $t7.1q, $res6.2d, $h2.2d @ GHASH block 8k+6 - high
pmull2 $t8.1q, $res7.2d, $h1.2d @ GHASH block 8k+7 - high
aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 5
aese $ctr5b, $rk5 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 5
pmull $h2.1q, $res6.1d, $h2.1d @ GHASH block 8k+6 - low
eor3 $acc_hb, $acc_hb, $t4.16b, $t5.16b @ GHASH block 8k+4, 8k+5 - high
trn2 $res6.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid
eor3 $acc_lb, $acc_lb, $h4.16b, $h3.16b @ GHASH block 8k+4, 8k+5 - low
aese $ctr6b, $rk5 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 5
eor $res6.16b, $res6.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid
aese $ctr7b, $rk5 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 5
aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 5
aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 5
aese $ctr4b, $rk5 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 5
aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 5
eor3 $acc_mb, $acc_mb, $h34k.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid
ldr $mod_constantd, [$modulo_constant] @ MODULO - load modulo constant
pmull $h1.1q, $res7.1d, $h1.1d @ GHASH block 8k+7 - low
aese $ctr7b, $rk6 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 6
aese $ctr5b, $rk6 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 6
pmull2 $t9.1q, $res6.2d, $h12k.2d @ GHASH block 8k+6 - mid
aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 6
aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 6
pmull $h12k.1q, $res6.1d, $h12k.1d @ GHASH block 8k+7 - mid
eor3 $acc_lb, $acc_lb, $h2.16b, $h1.16b @ GHASH block 8k+6, 8k+7 - low
ldp $ctr_t0q, $ctr_t1q, [$input_ptr], #32 @ AES block 8k+8, 8k+9 - load plaintext
aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 6
rev32 $h1.16b, $rtmp_ctr.16b @ CTR block 8k+16
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+16
aese $ctr4b, $rk6 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 6
aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 6
aese $ctr6b, $rk6 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 6
eor3 $acc_mb, $acc_mb, $h12k.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid
ldp $rk8q, $rk9q, [$cc, #128] @ load rk8, rk9
eor3 $acc_hb, $acc_hb, $t7.16b, $t8.16b @ GHASH block 8k+6, 8k+7 - high
aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 7
aese $ctr7b, $rk7 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 7
ldp $ctr_t2q, $ctr_t3q, [$input_ptr], #32 @ AES block 8k+10, 8k+11 - load plaintext
aese $ctr5b, $rk7 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 7
aese $ctr6b, $rk7 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 7
aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 7
pmull $t11.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid
aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 7
aese $ctr4b, $rk7 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 7
rev32 $h2.16b, $rtmp_ctr.16b @ CTR block 8k+17
aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 7
aese $ctr5b, $rk8 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 8
ldp $ctr_t4q, $ctr_t5q, [$input_ptr], #32 @ AES block 8k+12, 8k+13 - load plaintext
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+17
aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 8
aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 8
aese $ctr7b, $rk8 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 8
aese $ctr4b, $rk8 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 8
eor3 $acc_mb, $acc_mb, $acc_hb, $acc_lb @ MODULO - karatsuba tidy up
ldr $rk10q, [$cc, #160] @ load rk10
ext $t12.16b, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment
rev32 $h3.16b, $rtmp_ctr.16b @ CTR block 8k+18
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+18
aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 8
aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 8
eor3 $acc_mb, $acc_mb, $t12.16b, $t11.16b @ MODULO - fold into mid
aese $ctr6b, $rk8 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 8
aese $ctr2b, $rk9 @ AES block 8k+10 - round 9
aese $ctr4b, $rk9 @ AES block 8k+12 - round 9
aese $ctr1b, $rk9 @ AES block 8k+9 - round 9
ldp $ctr_t6q, $ctr_t7q, [$input_ptr], #32 @ AES block 8k+14, 8k+15 - load plaintext
rev32 $h4.16b, $rtmp_ctr.16b @ CTR block 8k+19
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+19
cmp $input_ptr, $main_end_input_ptr @ LOOP CONTROL
eor3 $res4b, $ctr_t4b, $ctr4b, $rk10 @ AES block 4 - result
aese $ctr7b, $rk9 @ AES block 8k+15 - round 9
aese $ctr6b, $rk9 @ AES block 8k+14 - round 9
aese $ctr3b, $rk9 @ AES block 8k+11 - round 9
eor3 $res2b, $ctr_t2b, $ctr2b, $rk10 @ AES block 8k+10 - result
mov $ctr2.16b, $h3.16b @ CTR block 8k+18
aese $ctr0b, $rk9 @ AES block 8k+8 - round 9
rev32 $ctr4.16b, $rtmp_ctr.16b @ CTR block 8k+20
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+20
eor3 $res7b, $ctr_t7b, $ctr7b, $rk10 @ AES block 7 - result
aese $ctr5b, $rk9 @ AES block 8k+13 - round 9
pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low
eor3 $res1b, $ctr_t1b, $ctr1b, $rk10 @ AES block 8k+9 - result
eor3 $res3b, $ctr_t3b, $ctr3b, $rk10 @ AES block 8k+11 - result
mov $ctr3.16b, $h4.16b @ CTR block 8k+19
ext $t11.16b, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment
eor3 $res5b, $ctr_t5b, $ctr5b, $rk10 @ AES block 5 - result
mov $ctr1.16b, $h2.16b @ CTR block 8k+17
eor3 $res0b, $ctr_t0b, $ctr0b, $rk10 @ AES block 8k+8 - result
mov $ctr0.16b, $h1.16b @ CTR block 8k+16
stp $res0q, $res1q, [$output_ptr], #32 @ AES block 8k+8, 8k+9 - store result
stp $res2q, $res3q, [$output_ptr], #32 @ AES block 8k+10, 8k+11 - store result
eor3 $res6b, $ctr_t6b, $ctr6b, $rk10 @ AES block 6 - result
stp $res4q, $res5q, [$output_ptr], #32 @ AES block 8k+12, 8k+13 - store result
eor3 $acc_lb, $acc_lb, $acc_hb, $t11.16b @ MODULO - fold into low
stp $res6q, $res7q, [$output_ptr], #32 @ AES block 8k+14, 8k+15 - store result
b.lt .L128_enc_main_loop
.L128_enc_prepretail: @ PREPRETAIL
rev32 $ctr5.16b, $rtmp_ctr.16b @ CTR block 8k+13
ldr $h7q, [$current_tag, #176] @ load h7l | h7h
ext $h7.16b, $h7.16b, $h7.16b, #8
ldr $h8q, [$current_tag, #208] @ load h8l | h8h
ext $h8.16b, $h8.16b, $h8.16b, #8
ext $acc_lb, $acc_lb, $acc_lb, #8 @ PRE 0
ldr $h5q, [$current_tag, #128] @ load h5l | h5h
ext $h5.16b, $h5.16b, $h5.16b, #8
ldr $h6q, [$current_tag, #160] @ load h6l | h6h
ext $h6.16b, $h6.16b, $h6.16b, #8
rev64 $res0b, $res0b @ GHASH block 8k
rev64 $res1b, $res1b @ GHASH block 8k+1
ldr $h56kq, [$current_tag, #144] @ load h6k | h5k
ldr $h78kq, [$current_tag, #192] @ load h6k | h5k
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+13
rev64 $res3b, $res3b @ GHASH block 8k+3
rev64 $res2b, $res2b @ GHASH block 8k+2
eor $res0b, $res0b, $acc_lb @ PRE 1
rev32 $ctr6.16b, $rtmp_ctr.16b @ CTR block 8k+14
pmull2 $t0.1q, $res1.2d, $h7.2d @ GHASH block 8k+1 - high
pmull $acc_l.1q, $res0.1d, $h8.1d @ GHASH block 8k - low
pmull2 $acc_h.1q, $res0.2d, $h8.2d @ GHASH block 8k - high
rev64 $res5b, $res5b @ GHASH block 8k+5 (t0, t1, t2 and t3 free)
trn1 $acc_m.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid
pmull $h7.1q, $res1.1d, $h7.1d @ GHASH block 8k+1 - low
eor $acc_hb, $acc_hb, $t0.16b @ GHASH block 8k+1 - high
trn2 $res0.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid
eor $acc_lb, $acc_lb, $h7.16b @ GHASH block 8k+1 - low
eor $res0.16b, $res0.16b, $acc_m.16b @ GHASH block 8k, 8k+1 - mid
ldp $rk0q, $rk1q, [$cc, #0] @ load rk0, rk1
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+14
pmull2 $acc_m.1q, $res0.2d, $h78k.2d @ GHASH block 8k - mid
pmull $h78k.1q, $res0.1d, $h78k.1d @ GHASH block 8k+1 - mid
rev64 $res4b, $res4b @ GHASH block 8k+4 (t0, t1, and t2 free)
rev64 $res7b, $res7b @ GHASH block 8k+7 (t0, t1, t2 and t3 free)
eor $acc_mb, $acc_mb, $h78k.16b @ GHASH block 8k+1 - mid
rev32 $ctr7.16b, $rtmp_ctr.16b @ CTR block 8k+15
rev64 $res6b, $res6b @ GHASH block 8k+6 (t0, t1, and t2 free)
aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 0
pmull2 $t2.1q, $res3.2d, $h5.2d @ GHASH block 8k+3 - high
pmull2 $t1.1q, $res2.2d, $h6.2d @ GHASH block 8k+2 - high
aese $ctr6b, $rk0 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 0
aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 0
pmull $h6.1q, $res2.1d, $h6.1d @ GHASH block 8k+2 - low
aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 0
eor3 $acc_hb, $acc_hb, $t1.16b, $t2.16b @ GHASH block 8k+2, 8k+3 - high
trn1 $t3.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid
trn2 $res2.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid
aese $ctr5b, $rk0 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 0
aese $ctr7b, $rk0 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 0
eor $res2.16b, $res2.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid
aese $ctr4b, $rk0 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 0
aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 0
aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 1
pmull $h5.1q, $res3.1d, $h5.1d @ GHASH block 8k+3 - low
ldr $h3q, [$current_tag, #80] @ load h3l | h3h
ext $h3.16b, $h3.16b, $h3.16b, #8
ldr $h4q, [$current_tag, #112] @ load h4l | h4h
ext $h4.16b, $h4.16b, $h4.16b, #8
ldp $rk2q, $rk3q, [$cc, #32] @ load rk2, rk3
aese $ctr5b, $rk1 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 1
pmull2 $t3.1q, $res2.2d, $h56k.2d @ GHASH block 8k+2 - mid
eor3 $acc_lb, $acc_lb, $h6.16b, $h5.16b @ GHASH block 8k+2, 8k+3 - low
pmull $h56k.1q, $res2.1d, $h56k.1d @ GHASH block 8k+3 - mid
aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 1
aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 1
eor3 $acc_mb, $acc_mb, $h56k.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid
ldr $h12kq, [$current_tag, #48] @ load h2k | h1k
ldr $h34kq, [$current_tag, #96] @ load h4k | h3k
aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 1
aese $ctr4b, $rk1 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 1
aese $ctr7b, $rk1 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 1
aese $ctr5b, $rk2 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 2
aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 2
aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 2
aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 2
aese $ctr6b, $rk1 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 1
aese $ctr4b, $rk2 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 2
aese $ctr5b, $rk3 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 3
aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 2
aese $ctr6b, $rk2 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 2
aese $ctr7b, $rk2 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 2
ldp $rk4q, $rk5q, [$cc, #64] @ load rk4, rk5
ldr $h1q, [$current_tag, #32] @ load h1l | h1h
ext $h1.16b, $h1.16b, $h1.16b, #8
ldr $h2q, [$current_tag, #64] @ load h1l | h1h
ext $h2.16b, $h2.16b, $h2.16b, #8
trn1 $t6.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid
aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 3
pmull2 $t4.1q, $res4.2d, $h4.2d @ GHASH block 8k+4 - high
aese $ctr6b, $rk3 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 3
aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 3
pmull $h4.1q, $res4.1d, $h4.1d @ GHASH block 8k+4 - low
trn2 $res4.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid
pmull2 $t5.1q, $res5.2d, $h3.2d @ GHASH block 8k+5 - high
aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 3
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+15
aese $ctr7b, $rk3 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 3
aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 3
eor $res4.16b, $res4.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid
pmull $h3.1q, $res5.1d, $h3.1d @ GHASH block 8k+5 - low
aese $ctr4b, $rk3 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 3
pmull2 $t7.1q, $res6.2d, $h2.2d @ GHASH block 8k+6 - high
trn1 $t9.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid
pmull $h2.1q, $res6.1d, $h2.1d @ GHASH block 8k+6 - low
trn2 $res6.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid
aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 4
aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 4
eor3 $acc_hb, $acc_hb, $t4.16b, $t5.16b @ GHASH block 8k+4, 8k+5 - high
eor3 $acc_lb, $acc_lb, $h4.16b, $h3.16b @ GHASH block 8k+4, 8k+5 - low
eor $res6.16b, $res6.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid
pmull2 $t6.1q, $res4.2d, $h34k.2d @ GHASH block 8k+4 - mid
aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 5
aese $ctr6b, $rk4 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 4
aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 4
aese $ctr7b, $rk4 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 4
aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 4
pmull $h34k.1q, $res4.1d, $h34k.1d @ GHASH block 8k+5 - mid
aese $ctr4b, $rk4 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 4
aese $ctr5b, $rk4 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 4
pmull2 $t8.1q, $res7.2d, $h1.2d @ GHASH block 8k+7 - high
ldp $rk6q, $rk7q, [$cc, #96] @ load rk6, rk7
pmull $h1.1q, $res7.1d, $h1.1d @ GHASH block 8k+7 - low
eor3 $acc_mb, $acc_mb, $h34k.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid
pmull2 $t9.1q, $res6.2d, $h12k.2d @ GHASH block 8k+6 - mid
pmull $h12k.1q, $res6.1d, $h12k.1d @ GHASH block 8k+7 - mid
aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 5
aese $ctr7b, $rk5 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 5
ldr $mod_constantd, [$modulo_constant] @ MODULO - load modulo constant
aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 5
aese $ctr4b, $rk5 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 5
eor3 $acc_hb, $acc_hb, $t7.16b, $t8.16b @ GHASH block 8k+6, 8k+7 - high
aese $ctr5b, $rk5 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 5
aese $ctr6b, $rk5 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 5
aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 5
aese $ctr4b, $rk6 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 6
aese $ctr5b, $rk6 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 6
aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 6
aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 6
aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 6
eor3 $acc_lb, $acc_lb, $h2.16b, $h1.16b @ GHASH block 8k+6, 8k+7 - low
eor3 $acc_mb, $acc_mb, $h12k.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid
aese $ctr6b, $rk6 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 6
aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 6
aese $ctr7b, $rk6 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 6
pmull $t11.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid
eor3 $acc_mb, $acc_mb, $acc_hb, $acc_lb @ MODULO - karatsuba tidy up
ldp $rk8q, $rk9q, [$cc, #128] @ load rk8, rk9
aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 7
aese $ctr6b, $rk7 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 7
aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 7
ext $t12.16b, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment
aese $ctr5b, $rk7 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 7
aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 7
eor3 $acc_mb, $acc_mb, $t12.16b, $t11.16b @ MODULO - fold into mid
aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 7
aese $ctr7b, $rk7 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 7
pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low
aese $ctr4b, $rk7 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 7
aese $ctr7b, $rk8 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 8
aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 8
aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 8
ext $acc_mb, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment
aese $ctr6b, $rk8 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 8
eor3 $acc_lb, $acc_lb, $acc_hb, $acc_mb @ MODULO - fold into low
aese $ctr4b, $rk8 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 8
aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 8
aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 8
aese $ctr5b, $rk8 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 8
ldr $rk10q, [$cc, #160] @ load rk10
aese $ctr6b, $rk9 @ AES block 8k+14 - round 9
aese $ctr2b, $rk9 @ AES block 8k+10 - round 9
aese $ctr0b, $rk9 @ AES block 8k+8 - round 9
aese $ctr1b, $rk9 @ AES block 8k+9 - round 9
aese $ctr3b, $rk9 @ AES block 8k+11 - round 9
aese $ctr5b, $rk9 @ AES block 8k+13 - round 9
aese $ctr4b, $rk9 @ AES block 8k+12 - round 9
aese $ctr7b, $rk9 @ AES block 8k+15 - round 9
.L128_enc_tail: @ TAIL
sub $main_end_input_ptr, $end_input_ptr, $input_ptr @ main_end_input_ptr is number of bytes left to process
ldr $ctr_t0q, [$input_ptr], #16 @ AES block 8k+8 - load plaintext
mov $t1.16b, $rk10
ldp $h5q, $h56kq, [$current_tag, #128] @ load h5l | h5h
ext $h5.16b, $h5.16b, $h5.16b, #8
eor3 $res1b, $ctr_t0b, $ctr0b, $t1.16b @ AES block 8k+8 - result
ext $t0.16b, $acc_lb, $acc_lb, #8 @ prepare final partial tag
ldp $h6q, $h7q, [$current_tag, #160] @ load h6l | h6h
ext $h6.16b, $h6.16b, $h6.16b, #8
ext $h7.16b, $h7.16b, $h7.16b, #8
ldp $h78kq, $h8q, [$current_tag, #192] @ load h8k | h7k
ext $h8.16b, $h8.16b, $h8.16b, #8
cmp $main_end_input_ptr, #112
b.gt .L128_enc_blocks_more_than_7
mov $ctr7b, $ctr6b
mov $ctr6b, $ctr5b
movi $acc_h.8b, #0
cmp $main_end_input_ptr, #96
sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
mov $ctr5b, $ctr4b
mov $ctr4b, $ctr3b
mov $ctr3b, $ctr2b
mov $ctr2b, $ctr1b
movi $acc_l.8b, #0
movi $acc_m.8b, #0
b.gt .L128_enc_blocks_more_than_6
mov $ctr7b, $ctr6b
cmp $main_end_input_ptr, #80
sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
mov $ctr6b, $ctr5b
mov $ctr5b, $ctr4b
mov $ctr4b, $ctr3b
mov $ctr3b, $ctr1b
b.gt .L128_enc_blocks_more_than_5
cmp $main_end_input_ptr, #64
sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
mov $ctr7b, $ctr6b
mov $ctr6b, $ctr5b
mov $ctr5b, $ctr4b
mov $ctr4b, $ctr1b
b.gt .L128_enc_blocks_more_than_4
mov $ctr7b, $ctr6b
sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
mov $ctr6b, $ctr5b
mov $ctr5b, $ctr1b
cmp $main_end_input_ptr, #48
b.gt .L128_enc_blocks_more_than_3
sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
mov $ctr7b, $ctr6b
mov $ctr6b, $ctr1b
cmp $main_end_input_ptr, #32
ldr $h34kq, [$current_tag, #96] @ load h4k | h3k
b.gt .L128_enc_blocks_more_than_2
cmp $main_end_input_ptr, #16
sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
mov $ctr7b, $ctr1b
b.gt .L128_enc_blocks_more_than_1
ldr $h12kq, [$current_tag, #48] @ load h2k | h1k
sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
b .L128_enc_blocks_less_than_1
.L128_enc_blocks_more_than_7: @ blocks left > 7
st1 { $res1b}, [$output_ptr], #16 @ AES final-7 block - store result
rev64 $res0b, $res1b @ GHASH final-7 block
ldr $ctr_t1q, [$input_ptr], #16 @ AES final-6 block - load plaintext
eor $res0b, $res0b, $t0.16b @ feed in partial tag
ins $rk4v.d[0], $res0.d[1] @ GHASH final-7 block - mid
pmull2 $acc_h.1q, $res0.2d, $h8.2d @ GHASH final-7 block - high
ins $acc_m.d[0], $h78k.d[1] @ GHASH final-7 block - mid
eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-7 block - mid
movi $t0.8b, #0 @ supress further partial tag feed in
eor3 $res1b, $ctr_t1b, $ctr1b, $t1.16b @ AES final-6 block - result
pmull $acc_m.1q, $rk4v.1d, $acc_m.1d @ GHASH final-7 block - mid
pmull $acc_l.1q, $res0.1d, $h8.1d @ GHASH final-7 block - low
.L128_enc_blocks_more_than_6: @ blocks left > 6
st1 { $res1b}, [$output_ptr], #16 @ AES final-6 block - store result
rev64 $res0b, $res1b @ GHASH final-6 block
ldr $ctr_t1q, [$input_ptr], #16 @ AES final-5 block - load plaintext
eor $res0b, $res0b, $t0.16b @ feed in partial tag
ins $rk4v.d[0], $res0.d[1] @ GHASH final-6 block - mid
eor3 $res1b, $ctr_t1b, $ctr2b, $t1.16b @ AES final-5 block - result
pmull $rk3q1, $res0.1d, $h7.1d @ GHASH final-6 block - low
eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-6 block - mid
movi $t0.8b, #0 @ supress further partial tag feed in
pmull $rk4v.1q, $rk4v.1d, $h78k.1d @ GHASH final-6 block - mid
pmull2 $rk2q1, $res0.2d, $h7.2d @ GHASH final-6 block - high
eor $acc_lb, $acc_lb, $rk3 @ GHASH final-6 block - low
eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-6 block - mid
eor $acc_hb, $acc_hb, $rk2 @ GHASH final-6 block - high
.L128_enc_blocks_more_than_5: @ blocks left > 5
st1 { $res1b}, [$output_ptr], #16 @ AES final-5 block - store result
rev64 $res0b, $res1b @ GHASH final-5 block
eor $res0b, $res0b, $t0.16b @ feed in partial tag
ins $rk4v.d[0], $res0.d[1] @ GHASH final-5 block - mid
ldr $ctr_t1q, [$input_ptr], #16 @ AES final-4 block - load plaintext
pmull2 $rk2q1, $res0.2d, $h6.2d @ GHASH final-5 block - high
eor $acc_hb, $acc_hb, $rk2 @ GHASH final-5 block - high
eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-5 block - mid
ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-5 block - mid
eor3 $res1b, $ctr_t1b, $ctr3b, $t1.16b @ AES final-4 block - result
pmull $rk3q1, $res0.1d, $h6.1d @ GHASH final-5 block - low
movi $t0.8b, #0 @ supress further partial tag feed in
pmull2 $rk4v.1q, $rk4v.2d, $h56k.2d @ GHASH final-5 block - mid
eor $acc_lb, $acc_lb, $rk3 @ GHASH final-5 block - low
eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-5 block - mid
.L128_enc_blocks_more_than_4: @ blocks left > 4
st1 { $res1b}, [$output_ptr], #16 @ AES final-4 block - store result
rev64 $res0b, $res1b @ GHASH final-4 block
ldr $ctr_t1q, [$input_ptr], #16 @ AES final-3 block - load plaintext
eor $res0b, $res0b, $t0.16b @ feed in partial tag
ins $rk4v.d[0], $res0.d[1] @ GHASH final-4 block - mid
movi $t0.8b, #0 @ supress further partial tag feed in
pmull2 $rk2q1, $res0.2d, $h5.2d @ GHASH final-4 block - high
eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-4 block - mid
pmull $rk3q1, $res0.1d, $h5.1d @ GHASH final-4 block - low
eor $acc_hb, $acc_hb, $rk2 @ GHASH final-4 block - high
pmull $rk4v.1q, $rk4v.1d, $h56k.1d @ GHASH final-4 block - mid
eor $acc_lb, $acc_lb, $rk3 @ GHASH final-4 block - low
eor3 $res1b, $ctr_t1b, $ctr4b, $t1.16b @ AES final-3 block - result
eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-4 block - mid
.L128_enc_blocks_more_than_3: @ blocks left > 3
st1 { $res1b}, [$output_ptr], #16 @ AES final-3 block - store result
ldr $h4q, [$current_tag, #112] @ load h4l | h4h
ext $h4.16b, $h4.16b, $h4.16b, #8
rev64 $res0b, $res1b @ GHASH final-3 block
eor $res0b, $res0b, $t0.16b @ feed in partial tag
movi $t0.8b, #0 @ supress further partial tag feed in
ins $rk4v.d[0], $res0.d[1] @ GHASH final-3 block - mid
ldr $h34kq, [$current_tag, #96] @ load h4k | h3k
pmull $rk3q1, $res0.1d, $h4.1d @ GHASH final-3 block - low
ldr $ctr_t1q, [$input_ptr], #16 @ AES final-2 block - load plaintext
eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-3 block - mid
ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-3 block - mid
eor $acc_lb, $acc_lb, $rk3 @ GHASH final-3 block - low
eor3 $res1b, $ctr_t1b, $ctr5b, $t1.16b @ AES final-2 block - result
pmull2 $rk4v.1q, $rk4v.2d, $h34k.2d @ GHASH final-3 block - mid
pmull2 $rk2q1, $res0.2d, $h4.2d @ GHASH final-3 block - high
eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-3 block - mid
eor $acc_hb, $acc_hb, $rk2 @ GHASH final-3 block - high
.L128_enc_blocks_more_than_2: @ blocks left > 2
st1 { $res1b}, [$output_ptr], #16 @ AES final-2 block - store result
rev64 $res0b, $res1b @ GHASH final-2 block
eor $res0b, $res0b, $t0.16b @ feed in partial tag
ldr $ctr_t1q, [$input_ptr], #16 @ AES final-1 block - load plaintext
ins $rk4v.d[0], $res0.d[1] @ GHASH final-2 block - mid
ldr $h3q, [$current_tag, #80] @ load h3l | h3h
ext $h3.16b, $h3.16b, $h3.16b, #8
movi $t0.8b, #0 @ supress further partial tag feed in
eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-2 block - mid
eor3 $res1b, $ctr_t1b, $ctr6b, $t1.16b @ AES final-1 block - result
pmull2 $rk2q1, $res0.2d, $h3.2d @ GHASH final-2 block - high
pmull $rk3q1, $res0.1d, $h3.1d @ GHASH final-2 block - low
pmull $rk4v.1q, $rk4v.1d, $h34k.1d @ GHASH final-2 block - mid
eor $acc_hb, $acc_hb, $rk2 @ GHASH final-2 block - high
eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-2 block - mid
eor $acc_lb, $acc_lb, $rk3 @ GHASH final-2 block - low
.L128_enc_blocks_more_than_1: @ blocks left > 1
st1 { $res1b}, [$output_ptr], #16 @ AES final-1 block - store result
ldr $h2q, [$current_tag, #64] @ load h2l | h2h
ext $h2.16b, $h2.16b, $h2.16b, #8
rev64 $res0b, $res1b @ GHASH final-1 block
ldr $ctr_t1q, [$input_ptr], #16 @ AES final block - load plaintext
eor $res0b, $res0b, $t0.16b @ feed in partial tag
movi $t0.8b, #0 @ supress further partial tag feed in
ins $rk4v.d[0], $res0.d[1] @ GHASH final-1 block - mid
eor3 $res1b, $ctr_t1b, $ctr7b, $t1.16b @ AES final block - result
pmull2 $rk2q1, $res0.2d, $h2.2d @ GHASH final-1 block - high
eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-1 block - mid
ldr $h12kq, [$current_tag, #48] @ load h2k | h1k
ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-1 block - mid
pmull $rk3q1, $res0.1d, $h2.1d @ GHASH final-1 block - low
pmull2 $rk4v.1q, $rk4v.2d, $h12k.2d @ GHASH final-1 block - mid
eor $acc_hb, $acc_hb, $rk2 @ GHASH final-1 block - high
eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-1 block - mid
eor $acc_lb, $acc_lb, $rk3 @ GHASH final-1 block - low
.L128_enc_blocks_less_than_1: @ blocks left <= 1
rev32 $rtmp_ctr.16b, $rtmp_ctr.16b
str $rtmp_ctrq, [$counter] @ store the updated counter
and $bit_length, $bit_length, #127 @ bit_length %= 128
sub $bit_length, $bit_length, #128 @ bit_length -= 128
neg $bit_length, $bit_length @ bit_length = 128 - #bits in input (in range [1,128])
mvn $temp0_x, xzr @ temp0_x = 0xffffffffffffffff
ld1 { $rk0}, [$output_ptr] @ load existing bytes where the possibly partial last block is to be stored
and $bit_length, $bit_length, #127 @ bit_length %= 128
lsr $temp0_x, $temp0_x, $bit_length @ temp0_x is mask for top 64b of last block
mvn $temp1_x, xzr @ temp1_x = 0xffffffffffffffff
cmp $bit_length, #64
csel $temp2_x, $temp1_x, $temp0_x, lt
csel $temp3_x, $temp0_x, xzr, lt
mov $ctr0.d[1], $temp3_x
mov $ctr0.d[0], $temp2_x @ ctr0b is mask for last block
and $res1b, $res1b, $ctr0b @ possibly partial last block has zeroes in highest bits
rev64 $res0b, $res1b @ GHASH final block
bif $res1b, $rk0, $ctr0b @ insert existing bytes in top end of result before storing
st1 { $res1b}, [$output_ptr] @ store all 16B
eor $res0b, $res0b, $t0.16b @ feed in partial tag
ins $t0.d[0], $res0.d[1] @ GHASH final block - mid
eor $t0.8b, $t0.8b, $res0.8b @ GHASH final block - mid
ldr $h1q, [$current_tag, #32] @ load h1l | h1h
ext $h1.16b, $h1.16b, $h1.16b, #8
pmull $t0.1q, $t0.1d, $h12k.1d @ GHASH final block - mid
pmull2 $rk2q1, $res0.2d, $h1.2d @ GHASH final block - high
eor $acc_mb, $acc_mb, $t0.16b @ GHASH final block - mid
ldr $mod_constantd, [$modulo_constant] @ MODULO - load modulo constant
pmull $rk3q1, $res0.1d, $h1.1d @ GHASH final block - low
eor $acc_hb, $acc_hb, $rk2 @ GHASH final block - high
eor $acc_lb, $acc_lb, $rk3 @ GHASH final block - low
ext $t11.16b, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment
pmull $t12.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid
eor3 $acc_mb, $acc_mb, $acc_hb, $acc_lb @ MODULO - karatsuba tidy up
eor3 $acc_mb, $acc_mb, $t12.16b, $t11.16b @ MODULO - fold into mid
pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low
ext $t11.16b, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment
eor3 $acc_lb, $acc_lb, $acc_hb, $t11.16b @ MODULO - fold into low
ext $acc_lb, $acc_lb, $acc_lb, #8
rev64 $acc_lb, $acc_lb
st1 { $acc_l.16b }, [$current_tag]
lsr x0, $bit_length, #3 @ return sizes
ldp d10, d11, [sp, #16]
ldp d12, d13, [sp, #32]
ldp d14, d15, [sp, #48]
ldp d8, d9, [sp], #80
ret
.L128_enc_ret:
mov w0, #0x0
ret
.size unroll8_eor3_aes_gcm_enc_128_kernel,.-unroll8_eor3_aes_gcm_enc_128_kernel
___
#########################################################################################
# size_t unroll8_eor3_aes_gcm_dec_128_kernel(const unsigned char *in,
# size_t len,
# unsigned char *out,
# u64 *Xi,
# unsigned char ivec[16],
# const void *key);
#
$code.=<<___;
.global unroll8_eor3_aes_gcm_dec_128_kernel
.type unroll8_eor3_aes_gcm_dec_128_kernel,%function
.align 4
unroll8_eor3_aes_gcm_dec_128_kernel:
AARCH64_VALID_CALL_TARGET
cbz x1, .L128_dec_ret
stp d8, d9, [sp, #-80]!
mov $counter, x4
mov $cc, x5
stp d10, d11, [sp, #16]
stp d12, d13, [sp, #32]
stp d14, d15, [sp, #48]
mov x5, #0xc200000000000000
stp x5, xzr, [sp, #64]
add $modulo_constant, sp, #64
lsr $main_end_input_ptr, $bit_length, #3 @ byte_len
ld1 { $ctr0b}, [$counter] @ CTR block 0
ldp $rk0q, $rk1q, [$cc, #0] @ load rk0, rk1
sub $main_end_input_ptr, $main_end_input_ptr, #1 @ byte_len - 1
mov $constant_temp, #0x100000000 @ set up counter increment
movi $rctr_inc.16b, #0x0
mov $rctr_inc.d[1], $constant_temp
ld1 { $acc_lb}, [$current_tag]
ext $acc_lb, $acc_lb, $acc_lb, #8
rev64 $acc_lb, $acc_lb
rev32 $rtmp_ctr.16b, $ctr0.16b @ set up reversed counter
aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 0
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 0
rev32 $ctr1.16b, $rtmp_ctr.16b @ CTR block 1
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 1
and $main_end_input_ptr, $main_end_input_ptr, #0xffffffffffffff80 @ number of bytes to be processed in main loop (at least 1 byte must be handled by tail)
rev32 $ctr2.16b, $rtmp_ctr.16b @ CTR block 2
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 2
aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 0
rev32 $ctr3.16b, $rtmp_ctr.16b @ CTR block 3
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 3
aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 1
aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 1
rev32 $ctr4.16b, $rtmp_ctr.16b @ CTR block 4
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 4
rev32 $ctr5.16b, $rtmp_ctr.16b @ CTR block 5
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 5
aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 0
rev32 $ctr6.16b, $rtmp_ctr.16b @ CTR block 6
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 6
aese $ctr5b, $rk0 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 0
aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 0
aese $ctr4b, $rk0 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 0
rev32 $ctr7.16b, $rtmp_ctr.16b @ CTR block 7
aese $ctr6b, $rk0 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 0
aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 1
aese $ctr7b, $rk0 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 0
ldp $rk2q, $rk3q, [$cc, #32] @ load rk2, rk3
aese $ctr6b, $rk1 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 1
aese $ctr5b, $rk1 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 1
aese $ctr4b, $rk1 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 1
aese $ctr7b, $rk1 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 1
aese $ctr7b, $rk2 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 2
aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 2
aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 1
aese $ctr6b, $rk2 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 2
aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 2
aese $ctr5b, $rk2 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 2
aese $ctr4b, $rk2 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 2
aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 2
aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 2
aese $ctr6b, $rk3 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 3
aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 3
ldp $rk4q, $rk5q, [$cc, #64] @ load rk4, rk5
aese $ctr5b, $rk3 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 3
aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 3
aese $ctr7b, $rk3 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 3
aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 3
aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 3
aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 4
aese $ctr7b, $rk4 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 4
aese $ctr4b, $rk3 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 3
aese $ctr6b, $rk4 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 4
aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 4
aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 4
aese $ctr5b, $rk4 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 4
aese $ctr4b, $rk4 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 4
aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 4
ldp $rk6q, $rk7q, [$cc, #96] @ load rk6, rk7
aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 5
aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 5
aese $ctr6b, $rk5 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 5
aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 5
aese $ctr7b, $rk5 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 5
aese $ctr5b, $rk5 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 5
aese $ctr4b, $rk5 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 5
aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 6
aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 6
aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 5
aese $ctr5b, $rk6 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 6
aese $ctr4b, $rk6 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 6
aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 6
aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 6
aese $ctr7b, $rk6 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 6
aese $ctr6b, $rk6 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 6
aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 7
aese $ctr4b, $rk7 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 7
aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 7
aese $ctr7b, $rk7 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 7
aese $ctr5b, $rk7 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 7
ldp $rk8q, $rk9q, [$cc, #128] @ load rk8, rk9
aese $ctr6b, $rk7 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 7
aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 7
aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 7
add $main_end_input_ptr, $main_end_input_ptr, $input_ptr
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 7
aese $ctr6b, $rk8 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 8
aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 8
aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 8
aese $ctr7b, $rk8 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 8
aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 8
aese $ctr5b, $rk8 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 8
aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 8
aese $ctr4b, $rk8 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 8
aese $ctr0b, $rk9 @ AES block 0 - round 9
aese $ctr1b, $rk9 @ AES block 1 - round 9
aese $ctr6b, $rk9 @ AES block 6 - round 9
ldr $rk10q, [$cc, #160] @ load rk10
aese $ctr4b, $rk9 @ AES block 4 - round 9
aese $ctr3b, $rk9 @ AES block 3 - round 9
aese $ctr2b, $rk9 @ AES block 2 - round 9
aese $ctr5b, $rk9 @ AES block 5 - round 9
aese $ctr7b, $rk9 @ AES block 7 - round 9
add $end_input_ptr, $input_ptr, $bit_length, lsr #3 @ end_input_ptr
cmp $input_ptr, $main_end_input_ptr @ check if we have <= 8 blocks
b.ge .L128_dec_tail @ handle tail
ldp $res0q, $res1q, [$input_ptr], #32 @ AES block 0, 1 - load ciphertext
eor3 $ctr0b, $res0b, $ctr0b, $rk10 @ AES block 0 - result
eor3 $ctr1b, $res1b, $ctr1b, $rk10 @ AES block 1 - result
stp $ctr0q, $ctr1q, [$output_ptr], #32 @ AES block 0, 1 - store result
rev32 $ctr0.16b, $rtmp_ctr.16b @ CTR block 8
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8
ldp $res2q, $res3q, [$input_ptr], #32 @ AES block 2, 3 - load ciphertext
ldp $res4q, $res5q, [$input_ptr], #32 @ AES block 4, 5 - load ciphertext
rev32 $ctr1.16b, $rtmp_ctr.16b @ CTR block 9
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 9
ldp $res6q, $res7q, [$input_ptr], #32 @ AES block 6, 7 - load ciphertext
eor3 $ctr3b, $res3b, $ctr3b, $rk10 @ AES block 3 - result
eor3 $ctr2b, $res2b, $ctr2b, $rk10 @ AES block 2 - result
stp $ctr2q, $ctr3q, [$output_ptr], #32 @ AES block 2, 3 - store result
rev32 $ctr2.16b, $rtmp_ctr.16b @ CTR block 10
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 10
eor3 $ctr6b, $res6b, $ctr6b, $rk10 @ AES block 6 - result
rev32 $ctr3.16b, $rtmp_ctr.16b @ CTR block 11
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 11
eor3 $ctr4b, $res4b, $ctr4b, $rk10 @ AES block 4 - result
eor3 $ctr5b, $res5b, $ctr5b, $rk10 @ AES block 5 - result
stp $ctr4q, $ctr5q, [$output_ptr], #32 @ AES block 4, 5 - store result
eor3 $ctr7b, $res7b, $ctr7b, $rk10 @ AES block 7 - result
stp $ctr6q, $ctr7q, [$output_ptr], #32 @ AES block 6, 7 - store result
rev32 $ctr4.16b, $rtmp_ctr.16b @ CTR block 12
cmp $input_ptr, $main_end_input_ptr @ check if we have <= 8 blocks
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 12
b.ge .L128_dec_prepretail @ do prepretail
.L128_dec_main_loop: @ main loop start
ldr $h7q, [$current_tag, #176] @ load h7l | h7h
ext $h7.16b, $h7.16b, $h7.16b, #8
ldr $h8q, [$current_tag, #208] @ load h8l | h8h
ext $h8.16b, $h8.16b, $h8.16b, #8
rev64 $res1b, $res1b @ GHASH block 8k+1
rev64 $res0b, $res0b @ GHASH block 8k
ext $acc_lb, $acc_lb, $acc_lb, #8 @ PRE 0
rev64 $res6b, $res6b @ GHASH block 8k+6
ldr $h5q, [$current_tag, #128] @ load h5l | h5h
ext $h5.16b, $h5.16b, $h5.16b, #8
ldr $h6q, [$current_tag, #160] @ load h6l | h6h
ext $h6.16b, $h6.16b, $h6.16b, #8
eor $res0b, $res0b, $acc_lb @ PRE 1
rev32 $ctr5.16b, $rtmp_ctr.16b @ CTR block 8k+13
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+13
rev64 $res2b, $res2b @ GHASH block 8k+2
rev64 $res4b, $res4b @ GHASH block 8k+4
ldp $rk0q, $rk1q, [$cc, #0] @ load rk0, rk1
rev32 $ctr6.16b, $rtmp_ctr.16b @ CTR block 8k+14
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+14
ldr $h56kq, [$current_tag, #144] @ load h6k | h5k
ldr $h78kq, [$current_tag, #192] @ load h8k | h7k
pmull2 $t0.1q, $res1.2d, $h7.2d @ GHASH block 8k+1 - high
pmull2 $acc_h.1q, $res0.2d, $h8.2d @ GHASH block 8k - high
rev64 $res3b, $res3b @ GHASH block 8k+3
rev32 $ctr7.16b, $rtmp_ctr.16b @ CTR block 8k+15
trn1 $acc_m.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid
rev64 $res5b, $res5b @ GHASH block 8k+5
pmull $h7.1q, $res1.1d, $h7.1d @ GHASH block 8k+1 - low
pmull $acc_l.1q, $res0.1d, $h8.1d @ GHASH block 8k - low
trn2 $res0.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid
pmull2 $t1.1q, $res2.2d, $h6.2d @ GHASH block 8k+2 - high
aese $ctr4b, $rk0 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 0
pmull2 $t2.1q, $res3.2d, $h5.2d @ GHASH block 8k+3 - high
aese $ctr6b, $rk0 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 0
aese $ctr5b, $rk0 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 0
aese $ctr7b, $rk0 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 0
aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 0
aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 0
eor $acc_hb, $acc_hb, $t0.16b @ GHASH block 8k+1 - high
aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 0
eor $res0.16b, $res0.16b, $acc_m.16b @ GHASH block 8k, 8k+1 - mid
aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 0
aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 1
eor $acc_lb, $acc_lb, $h7.16b @ GHASH block 8k+1 - low
eor3 $acc_hb, $acc_hb, $t1.16b, $t2.16b @ GHASH block 8k+2, 8k+3 - high
ldp $rk2q, $rk3q, [$cc, #32] @ load rk2, rk3
trn1 $t3.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid
aese $ctr7b, $rk1 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 1
pmull $h6.1q, $res2.1d, $h6.1d @ GHASH block 8k+2 - low
trn2 $res2.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid
pmull2 $acc_m.1q, $res0.2d, $h78k.2d @ GHASH block 8k - mid
ldr $h3q, [$current_tag, #80] @ load h3l | h3h
ext $h3.16b, $h3.16b, $h3.16b, #8
ldr $h4q, [$current_tag, #112] @ load h4l | h4h
ext $h4.16b, $h4.16b, $h4.16b, #8
pmull $h78k.1q, $res0.1d, $h78k.1d @ GHASH block 8k+1 - mid
aese $ctr6b, $rk1 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 1
aese $ctr4b, $rk1 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 1
aese $ctr5b, $rk1 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 1
pmull $h5.1q, $res3.1d, $h5.1d @ GHASH block 8k+3 - low
aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 1
aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 1
aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 1
aese $ctr7b, $rk2 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 2
aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 2
eor3 $acc_lb, $acc_lb, $h6.16b, $h5.16b @ GHASH block 8k+2, 8k+3 - low
aese $ctr4b, $rk2 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 2
eor $res2.16b, $res2.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid
ldr $h1q, [$current_tag, #32] @ load h1l | h1h
ext $h1.16b, $h1.16b, $h1.16b, #8
ldr $h2q, [$current_tag, #64] @ load h2l | h2h
ext $h2.16b, $h2.16b, $h2.16b, #8
eor $acc_mb, $acc_mb, $h78k.16b @ GHASH block 8k+1 - mid
aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 2
aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 2
trn1 $t6.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid
aese $ctr5b, $rk2 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 2
aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 2
aese $ctr6b, $rk2 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 2
pmull2 $t3.1q, $res2.2d, $h56k.2d @ GHASH block 8k+2 - mid
pmull $h56k.1q, $res2.1d, $h56k.1d @ GHASH block 8k+3 - mid
aese $ctr7b, $rk3 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 3
rev64 $res7b, $res7b @ GHASH block 8k+7
pmull2 $t4.1q, $res4.2d, $h4.2d @ GHASH block 8k+4 - high
ldp $rk4q, $rk5q, [$cc, #64] @ load rk4, rk5
pmull $h4.1q, $res4.1d, $h4.1d @ GHASH block 8k+4 - low
eor3 $acc_mb, $acc_mb, $h56k.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid
ldr $h12kq, [$current_tag, #48] @ load h2k | h1k
ldr $h34kq, [$current_tag, #96] @ load h4k | h3k
aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 3
trn2 $res4.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid
aese $ctr4b, $rk3 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 3
aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 3
aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 3
aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 3
aese $ctr6b, $rk3 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 3
aese $ctr5b, $rk3 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 3
pmull2 $t5.1q, $res5.2d, $h3.2d @ GHASH block 8k+5 - high
pmull $h3.1q, $res5.1d, $h3.1d @ GHASH block 8k+5 - low
pmull2 $t7.1q, $res6.2d, $h2.2d @ GHASH block 8k+6 - high
pmull $h2.1q, $res6.1d, $h2.1d @ GHASH block 8k+6 - low
aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 4
aese $ctr7b, $rk4 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 4
eor $res4.16b, $res4.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid
trn1 $t9.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid
aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 4
aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 4
aese $ctr5b, $rk4 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 4
aese $ctr6b, $rk4 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 4
aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 4
aese $ctr4b, $rk4 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 4
trn2 $res6.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid
ldp $rk6q, $rk7q, [$cc, #96] @ load rk6, rk7
aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 5
pmull2 $t6.1q, $res4.2d, $h34k.2d @ GHASH block 8k+4 - mid
aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 5
eor $res6.16b, $res6.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid
aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 5
pmull $h34k.1q, $res4.1d, $h34k.1d @ GHASH block 8k+5 - mid
aese $ctr6b, $rk5 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 5
aese $ctr7b, $rk5 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 5
aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 5
aese $ctr5b, $rk5 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 5
aese $ctr4b, $rk5 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 5
pmull2 $t8.1q, $res7.2d, $h1.2d @ GHASH block 8k+7 - high
eor3 $acc_mb, $acc_mb, $h34k.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid
eor3 $acc_lb, $acc_lb, $h4.16b, $h3.16b @ GHASH block 8k+4, 8k+5 - low
aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 6
eor3 $acc_hb, $acc_hb, $t4.16b, $t5.16b @ GHASH block 8k+4, 8k+5 - high
aese $ctr7b, $rk6 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 6
aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 6
pmull2 $t9.1q, $res6.2d, $h12k.2d @ GHASH block 8k+6 - mid
aese $ctr6b, $rk6 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 6
aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 6
aese $ctr5b, $rk6 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 6
pmull $h1.1q, $res7.1d, $h1.1d @ GHASH block 8k+7 - low
pmull $h12k.1q, $res6.1d, $h12k.1d @ GHASH block 8k+7 - mid
aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 6
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+15
eor3 $acc_hb, $acc_hb, $t7.16b, $t8.16b @ GHASH block 8k+6, 8k+7 - high
aese $ctr4b, $rk6 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 6
ldp $rk8q, $rk9q, [$cc, #128] @ load rk8, rk9
ldr $mod_constantd, [$modulo_constant] @ MODULO - load modulo constant
eor3 $acc_lb, $acc_lb, $h2.16b, $h1.16b @ GHASH block 8k+6, 8k+7 - low
aese $ctr5b, $rk7 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 7
rev32 $h1.16b, $rtmp_ctr.16b @ CTR block 8k+16
eor3 $acc_mb, $acc_mb, $h12k.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+16
aese $ctr6b, $rk7 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 7
aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 7
aese $ctr7b, $rk7 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 7
aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 7
aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 7
rev32 $h2.16b, $rtmp_ctr.16b @ CTR block 8k+17
aese $ctr4b, $rk7 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 7
ext $t11.16b, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment
pmull $t12.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid
eor3 $acc_mb, $acc_mb, $acc_hb, $acc_lb @ MODULO - karatsuba tidy up
aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 7
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+17
aese $ctr5b, $rk8 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 8
aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 8
ldp $res0q, $res1q, [$input_ptr], #32 @ AES block 8k+8, 8k+9 - load ciphertext
ldp $res2q, $res3q, [$input_ptr], #32 @ AES block 8k+10, 8k+11 - load ciphertext
aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 8
rev32 $h3.16b, $rtmp_ctr.16b @ CTR block 8k+18
ldp $res4q, $res5q, [$input_ptr], #32 @ AES block 8k+12, 8k+13 - load ciphertext
aese $ctr4b, $rk8 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 8
eor3 $acc_mb, $acc_mb, $t12.16b, $t11.16b @ MODULO - fold into mid
ldp $res6q, $res7q, [$input_ptr], #32 @ AES block 8k+14, 8k+15 - load ciphertext
aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 8
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+18
aese $ctr7b, $rk8 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 8
aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 8
aese $ctr6b, $rk8 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 8
aese $ctr0b, $rk9 @ AES block 8k+8 - round 9
aese $ctr1b, $rk9 @ AES block 8k+9 - round 9
ldr $rk10q, [$cc, #160] @ load rk10
aese $ctr6b, $rk9 @ AES block 8k+14 - round 9
pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low
aese $ctr2b, $rk9 @ AES block 8k+10 - round 9
aese $ctr7b, $rk9 @ AES block 8k+15 - round 9
aese $ctr4b, $rk9 @ AES block 8k+12 - round 9
ext $t11.16b, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment
rev32 $h4.16b, $rtmp_ctr.16b @ CTR block 8k+19
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+19
aese $ctr3b, $rk9 @ AES block 8k+11 - round 9
aese $ctr5b, $rk9 @ AES block 8k+13 - round 9
eor3 $ctr1b, $res1b, $ctr1b, $rk10 @ AES block 8k+9 - result
eor3 $ctr0b, $res0b, $ctr0b, $rk10 @ AES block 8k+8 - result
eor3 $ctr7b, $res7b, $ctr7b, $rk10 @ AES block 8k+15 - result
eor3 $ctr6b, $res6b, $ctr6b, $rk10 @ AES block 8k+14 - result
eor3 $ctr2b, $res2b, $ctr2b, $rk10 @ AES block 8k+10 - result
stp $ctr0q, $ctr1q, [$output_ptr], #32 @ AES block 8k+8, 8k+9 - store result
mov $ctr1.16b, $h2.16b @ CTR block 8k+17
eor3 $ctr4b, $res4b, $ctr4b, $rk10 @ AES block 8k+12 - result
eor3 $acc_lb, $acc_lb, $acc_hb, $t11.16b @ MODULO - fold into low
mov $ctr0.16b, $h1.16b @ CTR block 8k+16
eor3 $ctr3b, $res3b, $ctr3b, $rk10 @ AES block 8k+11 - result
cmp $input_ptr, $main_end_input_ptr @ LOOP CONTROL
stp $ctr2q, $ctr3q, [$output_ptr], #32 @ AES block 8k+10, 8k+11 - store result
eor3 $ctr5b, $res5b, $ctr5b, $rk10 @ AES block 8k+13 - result
mov $ctr2.16b, $h3.16b @ CTR block 8k+18
stp $ctr4q, $ctr5q, [$output_ptr], #32 @ AES block 8k+12, 8k+13 - store result
rev32 $ctr4.16b, $rtmp_ctr.16b @ CTR block 8k+20
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+20
stp $ctr6q, $ctr7q, [$output_ptr], #32 @ AES block 8k+14, 8k+15 - store result
mov $ctr3.16b, $h4.16b @ CTR block 8k+19
b.lt .L128_dec_main_loop
.L128_dec_prepretail: @ PREPRETAIL
rev64 $res3b, $res3b @ GHASH block 8k+3
ext $acc_lb, $acc_lb, $acc_lb, #8 @ PRE 0
rev64 $res0b, $res0b @ GHASH block 8k
rev64 $res2b, $res2b @ GHASH block 8k+2
rev32 $ctr5.16b, $rtmp_ctr.16b @ CTR block 8k+13
ldp $rk0q, $rk1q, [$cc, #0] @ load rk0, rk1
ldr $h7q, [$current_tag, #176] @ load h7l | h7h
ext $h7.16b, $h7.16b, $h7.16b, #8
ldr $h8q, [$current_tag, #208] @ load h8l | h8h
ext $h8.16b, $h8.16b, $h8.16b, #8
eor $res0b, $res0b, $acc_lb @ PRE 1
rev64 $res1b, $res1b @ GHASH block 8k+1
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+13
ldr $h5q, [$current_tag, #128] @ load h5l | h5h
ext $h5.16b, $h5.16b, $h5.16b, #8
ldr $h6q, [$current_tag, #160] @ load h6l | h6h
ext $h6.16b, $h6.16b, $h6.16b, #8
rev64 $res5b, $res5b @ GHASH block 8k+5
rev64 $res4b, $res4b @ GHASH block 8k+4
rev64 $res6b, $res6b @ GHASH block 8k+6
ldr $h56kq, [$current_tag, #144] @ load h6k | h5k
ldr $h78kq, [$current_tag, #192] @ load h8k | h7k
rev32 $ctr6.16b, $rtmp_ctr.16b @ CTR block 8k+14
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+14
pmull2 $t0.1q, $res1.2d, $h7.2d @ GHASH block 8k+1 - high
pmull $acc_l.1q, $res0.1d, $h8.1d @ GHASH block 8k - low
pmull2 $acc_h.1q, $res0.2d, $h8.2d @ GHASH block 8k - high
trn1 $acc_m.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid
trn2 $res0.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid
pmull2 $t1.1q, $res2.2d, $h6.2d @ GHASH block 8k+2 - high
pmull $h7.1q, $res1.1d, $h7.1d @ GHASH block 8k+1 - low
pmull2 $t2.1q, $res3.2d, $h5.2d @ GHASH block 8k+3 - high
aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 0
eor $acc_hb, $acc_hb, $t0.16b @ GHASH block 8k+1 - high
aese $ctr4b, $rk0 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 0
eor $res0.16b, $res0.16b, $acc_m.16b @ GHASH block 8k, 8k+1 - mid
pmull $h6.1q, $res2.1d, $h6.1d @ GHASH block 8k+2 - low
rev32 $ctr7.16b, $rtmp_ctr.16b @ CTR block 8k+15
aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 0
eor3 $acc_hb, $acc_hb, $t1.16b, $t2.16b @ GHASH block 8k+2, 8k+3 - high
trn1 $t3.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid
trn2 $res2.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid
aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 0
aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 0
aese $ctr5b, $rk0 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 0
pmull2 $acc_m.1q, $res0.2d, $h78k.2d @ GHASH block 8k - mid
pmull $h78k.1q, $res0.1d, $h78k.1d @ GHASH block 8k+1 - mid
pmull $h5.1q, $res3.1d, $h5.1d @ GHASH block 8k+3 - low
aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 1
aese $ctr7b, $rk0 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 0
aese $ctr6b, $rk0 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 0
eor $acc_lb, $acc_lb, $h7.16b @ GHASH block 8k+1 - low
eor $res2.16b, $res2.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid
eor $acc_mb, $acc_mb, $h78k.16b @ GHASH block 8k+1 - mid
aese $ctr6b, $rk1 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 1
aese $ctr4b, $rk1 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 1
aese $ctr5b, $rk1 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 1
ldp $rk2q, $rk3q, [$cc, #32] @ load rk2, rk3
eor3 $acc_lb, $acc_lb, $h6.16b, $h5.16b @ GHASH block 8k+2, 8k+3 - low
pmull2 $t3.1q, $res2.2d, $h56k.2d @ GHASH block 8k+2 - mid
ldr $h3q, [$current_tag, #80] @ load h3l | h3h
ext $h3.16b, $h3.16b, $h3.16b, #8
ldr $h4q, [$current_tag, #112] @ load h4l | h4h
ext $h4.16b, $h4.16b, $h4.16b, #8
aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 1
pmull $h56k.1q, $res2.1d, $h56k.1d @ GHASH block 8k+3 - mid
aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 1
aese $ctr7b, $rk1 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 1
aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 1
ldr $h1q, [$current_tag, #32] @ load h1l | h1h
ext $h1.16b, $h1.16b, $h1.16b, #8
ldr $h2q, [$current_tag, #64] @ load h2l | h2h
ext $h2.16b, $h2.16b, $h2.16b, #8
eor3 $acc_mb, $acc_mb, $h56k.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid
aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 2
aese $ctr6b, $rk2 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 2
aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 2
aese $ctr4b, $rk2 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 2
trn1 $t6.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid
aese $ctr7b, $rk2 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 2
aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 2
aese $ctr5b, $rk2 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 2
aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 2
pmull2 $t4.1q, $res4.2d, $h4.2d @ GHASH block 8k+4 - high
pmull $h4.1q, $res4.1d, $h4.1d @ GHASH block 8k+4 - low
trn2 $res4.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid
ldp $rk4q, $rk5q, [$cc, #64] @ load rk4, rk5
rev64 $res7b, $res7b @ GHASH block 8k+7
aese $ctr6b, $rk3 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 3
ldr $h12kq, [$current_tag, #48] @ load h2k | h1k
ldr $h34kq, [$current_tag, #96] @ load h4k | h3k
pmull2 $t5.1q, $res5.2d, $h3.2d @ GHASH block 8k+5 - high
pmull $h3.1q, $res5.1d, $h3.1d @ GHASH block 8k+5 - low
aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 3
aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 3
trn1 $t9.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid
pmull2 $t7.1q, $res6.2d, $h2.2d @ GHASH block 8k+6 - high
pmull $h2.1q, $res6.1d, $h2.1d @ GHASH block 8k+6 - low
trn2 $res6.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid
aese $ctr4b, $rk3 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 3
aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 3
aese $ctr7b, $rk3 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 3
aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 3
aese $ctr5b, $rk3 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 3
eor $res4.16b, $res4.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid
eor3 $acc_hb, $acc_hb, $t4.16b, $t5.16b @ GHASH block 8k+4, 8k+5 - high
aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 4
aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 4
eor $res6.16b, $res6.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid
aese $ctr5b, $rk4 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 4
pmull2 $t6.1q, $res4.2d, $h34k.2d @ GHASH block 8k+4 - mid
aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 4
aese $ctr6b, $rk4 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 4
aese $ctr4b, $rk4 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 4
aese $ctr7b, $rk4 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 4
aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 4
pmull $h34k.1q, $res4.1d, $h34k.1d @ GHASH block 8k+5 - mid
pmull2 $t8.1q, $res7.2d, $h1.2d @ GHASH block 8k+7 - high
pmull2 $t9.1q, $res6.2d, $h12k.2d @ GHASH block 8k+6 - mid
pmull $h12k.1q, $res6.1d, $h12k.1d @ GHASH block 8k+7 - mid
ldp $rk6q, $rk7q, [$cc, #96] @ load rk6, rk7
eor3 $acc_mb, $acc_mb, $h34k.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid
aese $ctr6b, $rk5 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 5
ldr $mod_constantd, [$modulo_constant] @ MODULO - load modulo constant
pmull $h1.1q, $res7.1d, $h1.1d @ GHASH block 8k+7 - low
eor3 $acc_lb, $acc_lb, $h4.16b, $h3.16b @ GHASH block 8k+4, 8k+5 - low
aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 5
aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 5
aese $ctr4b, $rk5 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 5
aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 5
aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 5
aese $ctr5b, $rk5 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 5
aese $ctr7b, $rk5 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 5
eor3 $acc_mb, $acc_mb, $h12k.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid
eor3 $acc_lb, $acc_lb, $h2.16b, $h1.16b @ GHASH block 8k+6, 8k+7 - low
aese $ctr4b, $rk6 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 6
aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 6
aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 6
eor3 $acc_hb, $acc_hb, $t7.16b, $t8.16b @ GHASH block 8k+6, 8k+7 - high
aese $ctr5b, $rk6 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 6
aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 6
aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 6
aese $ctr6b, $rk6 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 6
aese $ctr7b, $rk6 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 6
aese $ctr4b, $rk7 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 7
eor3 $acc_mb, $acc_mb, $acc_hb, $acc_lb @ MODULO - karatsuba tidy up
ldp $rk8q, $rk9q, [$cc, #128] @ load rk8, rk9
pmull $t12.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid
aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 7
ext $t11.16b, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment
aese $ctr5b, $rk7 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 7
aese $ctr6b, $rk7 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 7
aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 7
aese $ctr7b, $rk7 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 7
aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 7
aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 7
eor3 $acc_mb, $acc_mb, $t12.16b, $t11.16b @ MODULO - fold into mid
ldr $rk10q, [$cc, #160] @ load rk10
aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 8
aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 8
pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low
aese $ctr6b, $rk8 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 8
ext $t11.16b, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment
aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 8
aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 8
aese $ctr7b, $rk8 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 8
aese $ctr6b, $rk9 @ AES block 8k+14 - round 9
aese $ctr5b, $rk8 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 8
aese $ctr4b, $rk8 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 8
eor3 $acc_lb, $acc_lb, $acc_hb, $t11.16b @ MODULO - fold into low
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+15
aese $ctr2b, $rk9 @ AES block 8k+10 - round 9
aese $ctr3b, $rk9 @ AES block 8k+11 - round 9
aese $ctr5b, $rk9 @ AES block 8k+13 - round 9
aese $ctr0b, $rk9 @ AES block 8k+8 - round 9
aese $ctr4b, $rk9 @ AES block 8k+12 - round 9
aese $ctr1b, $rk9 @ AES block 8k+9 - round 9
aese $ctr7b, $rk9 @ AES block 8k+15 - round 9
.L128_dec_tail: @ TAIL
mov $t1.16b, $rk10
sub $main_end_input_ptr, $end_input_ptr, $input_ptr @ main_end_input_ptr is number of bytes left to process
cmp $main_end_input_ptr, #112
ldp $h78kq, $h8q, [$current_tag, #192] @ load h8k | h7k
ext $h8.16b, $h8.16b, $h8.16b, #8
ldr $res1q, [$input_ptr], #16 @ AES block 8k+8 - load ciphertext
ldp $h5q, $h56kq, [$current_tag, #128] @ load h5l | h5h
ext $h5.16b, $h5.16b, $h5.16b, #8
ext $t0.16b, $acc_lb, $acc_lb, #8 @ prepare final partial tag
ldp $h6q, $h7q, [$current_tag, #160] @ load h6l | h6h
ext $h6.16b, $h6.16b, $h6.16b, #8
ext $h7.16b, $h7.16b, $h7.16b, #8
eor3 $res4b, $res1b, $ctr0b, $t1.16b @ AES block 8k+8 - result
b.gt .L128_dec_blocks_more_than_7
cmp $main_end_input_ptr, #96
mov $ctr7b, $ctr6b
movi $acc_l.8b, #0
movi $acc_h.8b, #0
mov $ctr6b, $ctr5b
mov $ctr5b, $ctr4b
mov $ctr4b, $ctr3b
mov $ctr3b, $ctr2b
mov $ctr2b, $ctr1b
movi $acc_m.8b, #0
sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
b.gt .L128_dec_blocks_more_than_6
cmp $main_end_input_ptr, #80
sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
mov $ctr7b, $ctr6b
mov $ctr6b, $ctr5b
mov $ctr5b, $ctr4b
mov $ctr4b, $ctr3b
mov $ctr3b, $ctr1b
b.gt .L128_dec_blocks_more_than_5
cmp $main_end_input_ptr, #64
mov $ctr7b, $ctr6b
mov $ctr6b, $ctr5b
mov $ctr5b, $ctr4b
mov $ctr4b, $ctr1b
sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
b.gt .L128_dec_blocks_more_than_4
sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
mov $ctr7b, $ctr6b
mov $ctr6b, $ctr5b
mov $ctr5b, $ctr1b
cmp $main_end_input_ptr, #48
b.gt .L128_dec_blocks_more_than_3
sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
mov $ctr7b, $ctr6b
cmp $main_end_input_ptr, #32
ldr $h34kq, [$current_tag, #96] @ load h4k | h3k
mov $ctr6b, $ctr1b
b.gt .L128_dec_blocks_more_than_2
cmp $main_end_input_ptr, #16
mov $ctr7b, $ctr1b
sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
b.gt L128_dec_blocks_more_than_1
sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
ldr $h12kq, [$current_tag, #48] @ load h2k | h1k
b .L128_dec_blocks_less_than_1
.L128_dec_blocks_more_than_7: @ blocks left > 7
rev64 $res0b, $res1b @ GHASH final-7 block
eor $res0b, $res0b, $t0.16b @ feed in partial tag
ins $acc_m.d[0], $h78k.d[1] @ GHASH final-7 block - mid
pmull $acc_l.1q, $res0.1d, $h8.1d @ GHASH final-7 block - low
ins $rk4v.d[0], $res0.d[1] @ GHASH final-7 block - mid
movi $t0.8b, #0 @ supress further partial tag feed in
ldr $res1q, [$input_ptr], #16 @ AES final-6 block - load ciphertext
eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-7 block - mid
pmull2 $acc_h.1q, $res0.2d, $h8.2d @ GHASH final-7 block - high
st1 { $res4b}, [$output_ptr], #16 @ AES final-7 block - store result
eor3 $res4b, $res1b, $ctr1b, $t1.16b @ AES final-6 block - result
pmull $acc_m.1q, $rk4v.1d, $acc_m.1d @ GHASH final-7 block - mid
.L128_dec_blocks_more_than_6: @ blocks left > 6
rev64 $res0b, $res1b @ GHASH final-6 block
eor $res0b, $res0b, $t0.16b @ feed in partial tag
ins $rk4v.d[0], $res0.d[1] @ GHASH final-6 block - mid
eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-6 block - mid
pmull $rk3q1, $res0.1d, $h7.1d @ GHASH final-6 block - low
ldr $res1q, [$input_ptr], #16 @ AES final-5 block - load ciphertext
movi $t0.8b, #0 @ supress further partial tag feed in
pmull $rk4v.1q, $rk4v.1d, $h78k.1d @ GHASH final-6 block - mid
st1 { $res4b}, [$output_ptr], #16 @ AES final-6 block - store result
pmull2 $rk2q1, $res0.2d, $h7.2d @ GHASH final-6 block - high
eor $acc_lb, $acc_lb, $rk3 @ GHASH final-6 block - low
eor $acc_hb, $acc_hb, $rk2 @ GHASH final-6 block - high
eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-6 block - mid
eor3 $res4b, $res1b, $ctr2b, $t1.16b @ AES final-5 block - result
.L128_dec_blocks_more_than_5: @ blocks left > 5
rev64 $res0b, $res1b @ GHASH final-5 block
ldr $res1q, [$input_ptr], #16 @ AES final-4 block - load ciphertext
st1 { $res4b}, [$output_ptr], #16 @ AES final-5 block - store result
eor $res0b, $res0b, $t0.16b @ feed in partial tag
ins $rk4v.d[0], $res0.d[1] @ GHASH final-5 block - mid
eor3 $res4b, $res1b, $ctr3b, $t1.16b @ AES final-4 block - result
eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-5 block - mid
ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-5 block - mid
pmull $rk3q1, $res0.1d, $h6.1d @ GHASH final-5 block - low
movi $t0.8b, #0 @ supress further partial tag feed in
pmull2 $rk4v.1q, $rk4v.2d, $h56k.2d @ GHASH final-5 block - mid
pmull2 $rk2q1, $res0.2d, $h6.2d @ GHASH final-5 block - high
eor $acc_lb, $acc_lb, $rk3 @ GHASH final-5 block - low
eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-5 block - mid
eor $acc_hb, $acc_hb, $rk2 @ GHASH final-5 block - high
.L128_dec_blocks_more_than_4: @ blocks left > 4
rev64 $res0b, $res1b @ GHASH final-4 block
eor $res0b, $res0b, $t0.16b @ feed in partial tag
ldr $res1q, [$input_ptr], #16 @ AES final-3 block - load ciphertext
ins $rk4v.d[0], $res0.d[1] @ GHASH final-4 block - mid
movi $t0.8b, #0 @ supress further partial tag feed in
pmull2 $rk2q1, $res0.2d, $h5.2d @ GHASH final-4 block - high
pmull $rk3q1, $res0.1d, $h5.1d @ GHASH final-4 block - low
eor $acc_hb, $acc_hb, $rk2 @ GHASH final-4 block - high
st1 { $res4b}, [$output_ptr], #16 @ AES final-4 block - store result
eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-4 block - mid
eor3 $res4b, $res1b, $ctr4b, $t1.16b @ AES final-3 block - result
eor $acc_lb, $acc_lb, $rk3 @ GHASH final-4 block - low
pmull $rk4v.1q, $rk4v.1d, $h56k.1d @ GHASH final-4 block - mid
eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-4 block - mid
.L128_dec_blocks_more_than_3: @ blocks left > 3
st1 { $res4b}, [$output_ptr], #16 @ AES final-3 block - store result
rev64 $res0b, $res1b @ GHASH final-3 block
eor $res0b, $res0b, $t0.16b @ feed in partial tag
ins $rk4v.d[0], $res0.d[1] @ GHASH final-3 block - mid
ldr $h4q, [$current_tag, #112] @ load h4l | h4h
ext $h4.16b, $h4.16b, $h4.16b, #8
ldr $h34kq, [$current_tag, #96] @ load h4k | h3k
eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-3 block - mid
ldr $res1q, [$input_ptr], #16 @ AES final-2 block - load ciphertext
ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-3 block - mid
pmull $rk3q1, $res0.1d, $h4.1d @ GHASH final-3 block - low
pmull2 $rk2q1, $res0.2d, $h4.2d @ GHASH final-3 block - high
movi $t0.8b, #0 @ supress further partial tag feed in
eor3 $res4b, $res1b, $ctr5b, $t1.16b @ AES final-2 block - result
eor $acc_lb, $acc_lb, $rk3 @ GHASH final-3 block - low
pmull2 $rk4v.1q, $rk4v.2d, $h34k.2d @ GHASH final-3 block - mid
eor $acc_hb, $acc_hb, $rk2 @ GHASH final-3 block - high
eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-3 block - mid
.L128_dec_blocks_more_than_2: @ blocks left > 2
rev64 $res0b, $res1b @ GHASH final-2 block
st1 { $res4b}, [$output_ptr], #16 @ AES final-2 block - store result
eor $res0b, $res0b, $t0.16b @ feed in partial tag
ldr $h3q, [$current_tag, #80] @ load h3l | h3h
ext $h3.16b, $h3.16b, $h3.16b, #8
movi $t0.8b, #0 @ supress further partial tag feed in
ins $rk4v.d[0], $res0.d[1] @ GHASH final-2 block - mid
eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-2 block - mid
pmull $rk3q1, $res0.1d, $h3.1d @ GHASH final-2 block - low
pmull2 $rk2q1, $res0.2d, $h3.2d @ GHASH final-2 block - high
pmull $rk4v.1q, $rk4v.1d, $h34k.1d @ GHASH final-2 block - mid
ldr $res1q, [$input_ptr], #16 @ AES final-1 block - load ciphertext
eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-2 block - mid
eor $acc_lb, $acc_lb, $rk3 @ GHASH final-2 block - low
eor3 $res4b, $res1b, $ctr6b, $t1.16b @ AES final-1 block - result
eor $acc_hb, $acc_hb, $rk2 @ GHASH final-2 block - high
.L128_dec_blocks_more_than_1: @ blocks left > 1
st1 { $res4b}, [$output_ptr], #16 @ AES final-1 block - store result
rev64 $res0b, $res1b @ GHASH final-1 block
ldr $h2q, [$current_tag, #64] @ load h2l | h2h
ext $h2.16b, $h2.16b, $h2.16b, #8
eor $res0b, $res0b, $t0.16b @ feed in partial tag
movi $t0.8b, #0 @ supress further partial tag feed in
ins $rk4v.d[0], $res0.d[1] @ GHASH final-1 block - mid
ldr $res1q, [$input_ptr], #16 @ AES final block - load ciphertext
pmull2 $rk2q1, $res0.2d, $h2.2d @ GHASH final-1 block - high
eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-1 block - mid
eor $acc_hb, $acc_hb, $rk2 @ GHASH final-1 block - high
ldr $h12kq, [$current_tag, #48] @ load h2k | h1k
ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-1 block - mid
eor3 $res4b, $res1b, $ctr7b, $t1.16b @ AES final block - result
pmull $rk3q1, $res0.1d, $h2.1d @ GHASH final-1 block - low
pmull2 $rk4v.1q, $rk4v.2d, $h12k.2d @ GHASH final-1 block - mid
eor $acc_lb, $acc_lb, $rk3 @ GHASH final-1 block - low
eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-1 block - mid
.L128_dec_blocks_less_than_1: @ blocks left <= 1
and $bit_length, $bit_length, #127 @ bit_length %= 128
sub $bit_length, $bit_length, #128 @ bit_length -= 128
neg $bit_length, $bit_length @ bit_length = 128 - #bits in input (in range [1,128])
mvn $temp0_x, xzr @ temp0_x = 0xffffffffffffffff
and $bit_length, $bit_length, #127 @ bit_length %= 128
lsr $temp0_x, $temp0_x, $bit_length @ temp0_x is mask for top 64b of last block
cmp $bit_length, #64
mvn $temp1_x, xzr @ temp1_x = 0xffffffffffffffff
csel $temp2_x, $temp1_x, $temp0_x, lt
csel $temp3_x, $temp0_x, xzr, lt
mov $ctr0.d[1], $temp3_x
mov $ctr0.d[0], $temp2_x @ ctr0b is mask for last block
ldr $h1q, [$current_tag, #32] @ load h1l | h1h
ext $h1.16b, $h1.16b, $h1.16b, #8
ld1 { $rk0}, [$output_ptr] @ load existing bytes where the possibly partial last block is to be stored
and $res1b, $res1b, $ctr0b @ possibly partial last block has zeroes in highest bits
rev64 $res0b, $res1b @ GHASH final block
eor $res0b, $res0b, $t0.16b @ feed in partial tag
pmull2 $rk2q1, $res0.2d, $h1.2d @ GHASH final block - high
ins $t0.d[0], $res0.d[1] @ GHASH final block - mid
eor $acc_hb, $acc_hb, $rk2 @ GHASH final block - high
eor $t0.8b, $t0.8b, $res0.8b @ GHASH final block - mid
bif $res4b, $rk0, $ctr0b @ insert existing bytes in top end of result before storing
pmull $t0.1q, $t0.1d, $h12k.1d @ GHASH final block - mid
st1 { $res4b}, [$output_ptr] @ store all 16B
pmull $rk3q1, $res0.1d, $h1.1d @ GHASH final block - low
eor $acc_mb, $acc_mb, $t0.16b @ GHASH final block - mid
ldr $mod_constantd, [$modulo_constant] @ MODULO - load modulo constant
eor $acc_lb, $acc_lb, $rk3 @ GHASH final block - low
eor $t10.16b, $acc_hb, $acc_lb @ MODULO - karatsuba tidy up
pmull $t11.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid
ext $acc_hb, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment
eor $acc_mb, $acc_mb, $t10.16b @ MODULO - karatsuba tidy up
eor3 $acc_mb, $acc_mb, $acc_hb, $t11.16b @ MODULO - fold into mid
pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low
ext $acc_mb, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment
eor3 $acc_lb, $acc_lb, $acc_mb, $acc_hb @ MODULO - fold into low
ext $acc_lb, $acc_lb, $acc_lb, #8
rev64 $acc_lb, $acc_lb
st1 { $acc_l.16b }, [$current_tag]
rev32 $rtmp_ctr.16b, $rtmp_ctr.16b
str $rtmp_ctrq, [$counter] @ store the updated counter
lsr x0, $bit_length, #3
ldp d10, d11, [sp, #16]
ldp d12, d13, [sp, #32]
ldp d14, d15, [sp, #48]
ldp d8, d9, [sp], #80
ret
.L128_dec_ret:
mov w0, #0x0
ret
.size unroll8_eor3_aes_gcm_dec_128_kernel,.-unroll8_eor3_aes_gcm_dec_128_kernel
___
}
{
my ($end_input_ptr,$main_end_input_ptr,$temp0_x,$temp1_x)=map("x$_",(4..7));
my ($temp2_x,$temp3_x)=map("x$_",(13..14));
my ($ctr0b,$ctr1b,$ctr2b,$ctr3b,$ctr4b,$ctr5b,$ctr6b,$ctr7b,$res0b,$res1b,$res2b,$res3b,$res4b,$res5b,$res6b,$res7b)=map("v$_.16b",(0..15));
my ($ctr0,$ctr1,$ctr2,$ctr3,$ctr4,$ctr5,$ctr6,$ctr7,$res0,$res1,$res2,$res3,$res4,$res5,$res6,$res7)=map("v$_",(0..15));
my ($ctr0d,$ctr1d,$ctr2d,$ctr3d,$ctr4d,$ctr5d,$ctr6d,$ctr7d)=map("d$_",(0..7));
my ($ctr0q,$ctr1q,$ctr2q,$ctr3q,$ctr4q,$ctr5q,$ctr6q,$ctr7q)=map("q$_",(0..7));
my ($res0q,$res1q,$res2q,$res3q,$res4q,$res5q,$res6q,$res7q)=map("q$_",(8..15));
my ($ctr_t0,$ctr_t1,$ctr_t2,$ctr_t3,$ctr_t4,$ctr_t5,$ctr_t6,$ctr_t7)=map("v$_",(8..15));
my ($ctr_t0b,$ctr_t1b,$ctr_t2b,$ctr_t3b,$ctr_t4b,$ctr_t5b,$ctr_t6b,$ctr_t7b)=map("v$_.16b",(8..15));
my ($ctr_t0q,$ctr_t1q,$ctr_t2q,$ctr_t3q,$ctr_t4q,$ctr_t5q,$ctr_t6q,$ctr_t7q)=map("q$_",(8..15));
my ($acc_hb,$acc_mb,$acc_lb)=map("v$_.16b",(17..19));
my ($acc_h,$acc_m,$acc_l)=map("v$_",(17..19));
my ($h1,$h12k,$h2,$h3,$h34k,$h4)=map("v$_",(20..25));
my ($h5,$h56k,$h6,$h7,$h78k,$h8)=map("v$_",(20..25));
my ($h1q,$h12kq,$h2q,$h3q,$h34kq,$h4q)=map("q$_",(20..25));
my ($h5q,$h56kq,$h6q,$h7q,$h78kq,$h8q)=map("q$_",(20..25));
my $t0="v16";
my $t0d="d16";
my $t1="v29";
my $t2=$res1;
my $t3=$t1;
my $t4=$res0;
my $t5=$res2;
my $t6=$t0;
my $t7=$res3;
my $t8=$res4;
my $t9=$res5;
my $t10=$res6;
my $t11="v21";
my $t12=$t1;
my $rtmp_ctr="v30";
my $rtmp_ctrq="q30";
my $rctr_inc="v31";
my $rctr_incd="d31";
my $mod_constantd=$t0d;
my $mod_constant=$t0;
my ($rk0,$rk1,$rk2)=map("v$_.16b",(26..28));
my ($rk3,$rk4,$rk5)=map("v$_.16b",(26..28));
my ($rk6,$rk7,$rk8)=map("v$_.16b",(26..28));
my ($rk9,$rk10,$rk11)=map("v$_.16b",(26..28));
my ($rk12,$rk13,$rk14)=map("v$_.16b",(26..28));
my ($rk0q,$rk1q,$rk2q)=map("q$_",(26..28));
my ($rk3q,$rk4q,$rk5q)=map("q$_",(26..28));
my ($rk6q,$rk7q,$rk8q)=map("q$_",(26..28));
my ($rk9q,$rk10q,$rk11q)=map("q$_",(26..28));
my ($rk12q,$rk13q,$rk14q)=map("q$_",(26..28));
my $rk2q1="v28.1q";
my $rk3q1="v26.1q";
my $rk4v="v27";
#########################################################################################
# size_t unroll8_eor3_aes_gcm_enc_192_kernel(const unsigned char *in,
# size_t len,
# unsigned char *out,
# const void *key,
# unsigned char ivec[16],
# u64 *Xi);
#
$code.=<<___;
.global unroll8_eor3_aes_gcm_enc_192_kernel
.type unroll8_eor3_aes_gcm_enc_192_kernel,%function
.align 4
unroll8_eor3_aes_gcm_enc_192_kernel:
AARCH64_VALID_CALL_TARGET
cbz x1, .L192_enc_ret
stp d8, d9, [sp, #-80]!
mov $counter, x4
mov $cc, x5
stp d10, d11, [sp, #16]
stp d12, d13, [sp, #32]
stp d14, d15, [sp, #48]
mov x5, #0xc200000000000000
stp x5, xzr, [sp, #64]
add $modulo_constant, sp, #64
lsr $main_end_input_ptr, $bit_length, #3 @ byte_len
ld1 { $ctr0b}, [$counter] @ CTR block 0
mov $constant_temp, #0x100000000 @ set up counter increment
movi $rctr_inc.16b, #0x0
mov $rctr_inc.d[1], $constant_temp
rev32 $rtmp_ctr.16b, $ctr0.16b @ set up reversed counter
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 0
rev32 $ctr1.16b, $rtmp_ctr.16b @ CTR block 1
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 1
rev32 $ctr2.16b, $rtmp_ctr.16b @ CTR block 2
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 2
rev32 $ctr3.16b, $rtmp_ctr.16b @ CTR block 3
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 3
rev32 $ctr4.16b, $rtmp_ctr.16b @ CTR block 4
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 4
sub $main_end_input_ptr, $main_end_input_ptr, #1 @ byte_len - 1
and $main_end_input_ptr, $main_end_input_ptr, #0xffffffffffffff80 @ number of bytes to be processed in main loop (at least 1 byte must be handled by tail)
rev32 $ctr5.16b, $rtmp_ctr.16b @ CTR block 5
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 5
ldp $rk0q, $rk1q, [$cc, #0] @ load rk0, rk1
add $main_end_input_ptr, $main_end_input_ptr, $input_ptr
rev32 $ctr6.16b, $rtmp_ctr.16b @ CTR block 6
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 6
rev32 $ctr7.16b, $rtmp_ctr.16b @ CTR block 7
aese $ctr5b, $rk0 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 0
aese $ctr4b, $rk0 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 0
aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 0
aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 0
aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 0
aese $ctr7b, $rk0 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 0
aese $ctr6b, $rk0 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 0
aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 0
ldp $rk2q, $rk3q, [$cc, #32] @ load rk2, rk3
aese $ctr5b, $rk1 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 1
aese $ctr7b, $rk1 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 1
aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 1
aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 1
aese $ctr6b, $rk1 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 1
aese $ctr5b, $rk2 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 2
aese $ctr4b, $rk1 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 1
aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 1
aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 1
aese $ctr7b, $rk2 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 2
aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 2
aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 2
aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 2
aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 2
aese $ctr4b, $rk2 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 2
aese $ctr6b, $rk2 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 2
ldp $rk4q, $rk5q, [$cc, #64] @ load rk4, rk5
aese $ctr4b, $rk3 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 3
aese $ctr7b, $rk3 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 3
aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 3
aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 3
aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 3
aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 3
aese $ctr6b, $rk3 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 3
aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 4
aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 4
aese $ctr5b, $rk3 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 3
aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 4
aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 4
aese $ctr4b, $rk4 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 4
aese $ctr6b, $rk4 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 4
aese $ctr7b, $rk4 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 4
aese $ctr5b, $rk4 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 4
aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 5
ldp $rk6q, $rk7q, [$cc, #96] @ load rk6, rk7
aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 5
aese $ctr4b, $rk5 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 5
aese $ctr7b, $rk5 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 5
aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 5
aese $ctr5b, $rk5 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 5
aese $ctr6b, $rk5 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 5
aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 5
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 7
aese $ctr5b, $rk6 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 6
aese $ctr4b, $rk6 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 6
aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 6
aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 6
aese $ctr6b, $rk6 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 6
aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 6
aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 6
aese $ctr7b, $rk6 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 6
ldp $rk8q, $rk9q, [$cc, #128] @ load rk8, rk9
aese $ctr6b, $rk7 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 7
aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 7
aese $ctr4b, $rk7 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 7
aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 7
aese $ctr7b, $rk7 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 7
aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 7
aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 7
aese $ctr5b, $rk7 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 7
aese $ctr7b, $rk8 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 8
aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 8
aese $ctr4b, $rk8 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 8
aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 8
aese $ctr5b, $rk8 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 8
aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 8
aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 8
aese $ctr6b, $rk8 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 8
add $end_input_ptr, $input_ptr, $bit_length, lsr #3 @ end_input_ptr
cmp $input_ptr, $main_end_input_ptr @ check if we have <= 8 blocks
aese $ctr3b, $rk9 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 9
ld1 { $acc_lb}, [$current_tag]
ext $acc_lb, $acc_lb, $acc_lb, #8
rev64 $acc_lb, $acc_lb
ldp $rk10q, $rk11q, [$cc, #160] @ load rk10, rk11
aese $ctr6b, $rk9 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 9
aese $ctr1b, $rk9 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 9
aese $ctr5b, $rk9 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 9
aese $ctr2b, $rk9 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 9
aese $ctr0b, $rk9 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 9
aese $ctr4b, $rk9 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 9
aese $ctr6b, $rk10 \n aesmc $ctr6b, $ctr6b @ AES block 14 - round 10
aese $ctr7b, $rk9 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 9
aese $ctr3b, $rk10 \n aesmc $ctr3b, $ctr3b @ AES block 11 - round 10
aese $ctr1b, $rk10 \n aesmc $ctr1b, $ctr1b @ AES block 9 - round 10
aese $ctr5b, $rk10 \n aesmc $ctr5b, $ctr5b @ AES block 13 - round 10
aese $ctr4b, $rk10 \n aesmc $ctr4b, $ctr4b @ AES block 12 - round 10
aese $ctr0b, $rk10 \n aesmc $ctr0b, $ctr0b @ AES block 8 - round 10
aese $ctr2b, $rk10 \n aesmc $ctr2b, $ctr2b @ AES block 10 - round 10
aese $ctr7b, $rk10 \n aesmc $ctr7b, $ctr7b @ AES block 15 - round 10
aese $ctr6b, $rk11 @ AES block 14 - round 11
aese $ctr3b, $rk11 @ AES block 11 - round 11
aese $ctr4b, $rk11 @ AES block 12 - round 11
aese $ctr7b, $rk11 @ AES block 15 - round 11
ldr $rk12q, [$cc, #192] @ load rk12
aese $ctr1b, $rk11 @ AES block 9 - round 11
aese $ctr5b, $rk11 @ AES block 13 - round 11
aese $ctr2b, $rk11 @ AES block 10 - round 11
aese $ctr0b, $rk11 @ AES block 8 - round 11
b.ge .L192_enc_tail @ handle tail
ldp $ctr_t0q, $ctr_t1q, [$input_ptr], #32 @ AES block 0, 1 - load plaintext
ldp $ctr_t2q, $ctr_t3q, [$input_ptr], #32 @ AES block 2, 3 - load plaintext
ldp $ctr_t4q, $ctr_t5q, [$input_ptr], #32 @ AES block 4, 5 - load plaintext
ldp $ctr_t6q, $ctr_t7q, [$input_ptr], #32 @ AES block 6, 7 - load plaintext
eor3 $res0b, $ctr_t0b, $ctr0b, $rk12 @ AES block 0 - result
rev32 $ctr0.16b, $rtmp_ctr.16b @ CTR block 8
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8
eor3 $res3b, $ctr_t3b, $ctr3b, $rk12 @ AES block 3 - result
eor3 $res1b, $ctr_t1b, $ctr1b, $rk12 @ AES block 1 - result
rev32 $ctr1.16b, $rtmp_ctr.16b @ CTR block 9
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 9
eor3 $res4b, $ctr_t4b, $ctr4b, $rk12 @ AES block 4 - result
eor3 $res5b, $ctr_t5b, $ctr5b, $rk12 @ AES block 5 - result
eor3 $res7b, $ctr_t7b, $ctr7b, $rk12 @ AES block 7 - result
stp $res0q, $res1q, [$output_ptr], #32 @ AES block 0, 1 - store result
eor3 $res2b, $ctr_t2b, $ctr2b, $rk12 @ AES block 2 - result
rev32 $ctr2.16b, $rtmp_ctr.16b @ CTR block 10
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 10
stp $res2q, $res3q, [$output_ptr], #32 @ AES block 2, 3 - store result
cmp $input_ptr, $main_end_input_ptr @ check if we have <= 8 blocks
rev32 $ctr3.16b, $rtmp_ctr.16b @ CTR block 11
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 11
eor3 $res6b, $ctr_t6b, $ctr6b, $rk12 @ AES block 6 - result
stp $res4q, $res5q, [$output_ptr], #32 @ AES block 4, 5 - store result
rev32 $ctr4.16b, $rtmp_ctr.16b @ CTR block 12
stp $res6q, $res7q, [$output_ptr], #32 @ AES block 6, 7 - store result
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 12
b.ge .L192_enc_prepretail @ do prepretail
.L192_enc_main_loop: @ main loop start
rev64 $res4b, $res4b @ GHASH block 8k+4 (t0, t1, and t2 free)
ldp $rk0q, $rk1q, [$cc, #0] @ load rk0, rk1
rev64 $res2b, $res2b @ GHASH block 8k+2
rev32 $ctr5.16b, $rtmp_ctr.16b @ CTR block 8k+13
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+13
ldr $h7q, [$current_tag, #176] @ load h7l | h7h
ext $h7.16b, $h7.16b, $h7.16b, #8
ldr $h8q, [$current_tag, #208] @ load h8l | h8h
ext $h8.16b, $h8.16b, $h8.16b, #8
ext $acc_lb, $acc_lb, $acc_lb, #8 @ PRE 0
rev64 $res0b, $res0b @ GHASH block 8k
ldr $h5q, [$current_tag, #128] @ load h5l | h5h
ext $h5.16b, $h5.16b, $h5.16b, #8
ldr $h6q, [$current_tag, #160] @ load h6l | h6h
ext $h6.16b, $h6.16b, $h6.16b, #8
rev64 $res1b, $res1b @ GHASH block 8k+1
rev32 $ctr6.16b, $rtmp_ctr.16b @ CTR block 8k+14
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+14
eor $res0b, $res0b, $acc_lb @ PRE 1
rev64 $res3b, $res3b @ GHASH block 8k+3
rev64 $res5b, $res5b @ GHASH block 8k+5 (t0, t1, t2 and t3 free)
aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 0
rev32 $ctr7.16b, $rtmp_ctr.16b @ CTR block 8k+15
aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 0
aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 0
aese $ctr5b, $rk0 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 0
aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 0
aese $ctr7b, $rk0 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 0
aese $ctr4b, $rk0 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 0
aese $ctr6b, $rk0 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 0
ldp $rk2q, $rk3q, [$cc, #32] @ load rk2, rk3
pmull2 $acc_h.1q, $res0.2d, $h8.2d @ GHASH block 8k - high
aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 1
aese $ctr4b, $rk1 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 1
pmull2 $t0.1q, $res1.2d, $h7.2d @ GHASH block 8k+1 - high
pmull $h7.1q, $res1.1d, $h7.1d @ GHASH block 8k+1 - low
trn1 $acc_m.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid
aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 1
ldr $h56kq, [$current_tag, #144] @ load h6k | h5k
ldr $h78kq, [$current_tag, #192] @ load h8k | h7k
pmull2 $t1.1q, $res2.2d, $h6.2d @ GHASH block 8k+2 - high
pmull $acc_l.1q, $res0.1d, $h8.1d @ GHASH block 8k - low
trn2 $res0.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid
aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 1
aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 1
aese $ctr5b, $rk1 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 1
eor $acc_hb, $acc_hb, $t0.16b @ GHASH block 8k+1 - high
aese $ctr6b, $rk1 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 1
aese $ctr7b, $rk1 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 1
pmull2 $t2.1q, $res3.2d, $h5.2d @ GHASH block 8k+3 - high
eor $res0.16b, $res0.16b, $acc_m.16b @ GHASH block 8k, 8k+1 - mid
aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 2
aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 2
aese $ctr4b, $rk2 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 2
aese $ctr6b, $rk2 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 2
aese $ctr5b, $rk2 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 2
aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 3
eor3 $acc_hb, $acc_hb, $t1.16b, $t2.16b @ GHASH block 8k+2, 8k+3 - high
pmull $h6.1q, $res2.1d, $h6.1d @ GHASH block 8k+2 - low
aese $ctr7b, $rk2 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 2
aese $ctr4b, $rk3 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 3
aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 2
trn1 $t3.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid
aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 2
trn2 $res2.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid
aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 3
ldp $rk4q, $rk5q, [$cc, #64] @ load rk4, rk5
aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 3
eor $acc_lb, $acc_lb, $h7.16b @ GHASH block 8k+1 - low
ldr $h3q, [$current_tag, #80] @ load h3l | h3h
ext $h3.16b, $h3.16b, $h3.16b, #8
ldr $h4q, [$current_tag, #112] @ load h4l | h4h
ext $h4.16b, $h4.16b, $h4.16b, #8
pmull2 $acc_m.1q, $res0.2d, $h78k.2d @ GHASH block 8k - mid
pmull $h78k.1q, $res0.1d, $h78k.1d @ GHASH block 8k+1 - mid
pmull $h5.1q, $res3.1d, $h5.1d @ GHASH block 8k+3 - low
aese $ctr5b, $rk3 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 3
eor $res2.16b, $res2.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid
trn1 $t6.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid
eor $acc_mb, $acc_mb, $h78k.16b @ GHASH block 8k+1 - mid
aese $ctr6b, $rk3 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 3
eor3 $acc_lb, $acc_lb, $h6.16b, $h5.16b @ GHASH block 8k+2, 8k+3 - low
aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 4
aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 4
aese $ctr7b, $rk3 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 3
pmull2 $t3.1q, $res2.2d, $h56k.2d @ GHASH block 8k+2 - mid
aese $ctr6b, $rk4 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 4
aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 3
pmull $h56k.1q, $res2.1d, $h56k.1d @ GHASH block 8k+3 - mid
aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 4
aese $ctr4b, $rk4 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 4
aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 4
aese $ctr5b, $rk4 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 4
aese $ctr7b, $rk4 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 4
eor3 $acc_mb, $acc_mb, $h56k.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid
aese $ctr4b, $rk5 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 5
ldr $h1q, [$current_tag, #32] @ load h1l | h1h
ext $h1.16b, $h1.16b, $h1.16b, #8
ldr $h2q, [$current_tag, #64] @ load h2l | h2h
ext $h2.16b, $h2.16b, $h2.16b, #8
ldp $rk6q, $rk7q, [$cc, #96] @ load rk6, rk7
aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 5
rev64 $res7b, $res7b @ GHASH block 8k+7 (t0, t1, t2 and t3 free)
rev64 $res6b, $res6b @ GHASH block 8k+6 (t0, t1, and t2 free)
pmull2 $t4.1q, $res4.2d, $h4.2d @ GHASH block 8k+4 - high
pmull $h4.1q, $res4.1d, $h4.1d @ GHASH block 8k+4 - low
aese $ctr5b, $rk5 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 5
trn2 $res4.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid
aese $ctr6b, $rk5 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 5
ldr $h12kq, [$current_tag, #48] @ load h2k | h1k
ldr $h34kq, [$current_tag, #96] @ load h4k | h3k
aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 5
pmull2 $t5.1q, $res5.2d, $h3.2d @ GHASH block 8k+5 - high
eor $res4.16b, $res4.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid
aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 5
aese $ctr7b, $rk5 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 5
aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 5
pmull $h3.1q, $res5.1d, $h3.1d @ GHASH block 8k+5 - low
aese $ctr4b, $rk6 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 6
trn1 $t9.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid
aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 6
aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 6
pmull2 $t7.1q, $res6.2d, $h2.2d @ GHASH block 8k+6 - high
pmull $h2.1q, $res6.1d, $h2.1d @ GHASH block 8k+6 - low
trn2 $res6.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid
aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 6
aese $ctr6b, $rk6 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 6
aese $ctr5b, $rk6 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 6
aese $ctr7b, $rk6 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 6
aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 7
aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 6
aese $ctr6b, $rk7 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 7
eor $res6.16b, $res6.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid
pmull2 $t6.1q, $res4.2d, $h34k.2d @ GHASH block 8k+4 - mid
ldp $rk8q, $rk9q, [$cc, #128] @ load rk8, rk9
pmull $h34k.1q, $res4.1d, $h34k.1d @ GHASH block 8k+5 - mid
aese $ctr4b, $rk7 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 7
pmull2 $t8.1q, $res7.2d, $h1.2d @ GHASH block 8k+7 - high
aese $ctr5b, $rk7 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 7
eor3 $acc_mb, $acc_mb, $h34k.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid
aese $ctr7b, $rk7 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 7
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+15
ldr $mod_constantd, [$modulo_constant] @ MODULO - load modulo constant
eor3 $acc_hb, $acc_hb, $t4.16b, $t5.16b @ GHASH block 8k+4, 8k+5 - high
aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 7
pmull2 $t9.1q, $res6.2d, $h12k.2d @ GHASH block 8k+6 - mid
pmull $h1.1q, $res7.1d, $h1.1d @ GHASH block 8k+7 - low
aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 7
aese $ctr5b, $rk8 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 8
aese $ctr4b, $rk8 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 8
aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 8
aese $ctr6b, $rk8 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 8
eor3 $acc_lb, $acc_lb, $h4.16b, $h3.16b @ GHASH block 8k+4, 8k+5 - low
aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 7
aese $ctr7b, $rk8 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 8
aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 8
pmull $h12k.1q, $res6.1d, $h12k.1d @ GHASH block 8k+7 - mid
aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 8
aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 8
ldp $rk10q, $rk11q, [$cc, #160] @ load rk10, rk11
eor3 $acc_lb, $acc_lb, $h2.16b, $h1.16b @ GHASH block 8k+6, 8k+7 - low
rev32 $h1.16b, $rtmp_ctr.16b @ CTR block 8k+16
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+16
aese $ctr2b, $rk9 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 9
eor3 $acc_mb, $acc_mb, $h12k.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid
eor3 $acc_hb, $acc_hb, $t7.16b, $t8.16b @ GHASH block 8k+6, 8k+7 - high
aese $ctr6b, $rk9 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 9
aese $ctr3b, $rk9 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 9
ldp $ctr_t0q, $ctr_t1q, [$input_ptr], #32 @ AES block 8k+8, 8k+9 - load plaintext
pmull $t11.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid
rev32 $h2.16b, $rtmp_ctr.16b @ CTR block 8k+17
aese $ctr0b, $rk9 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 9
aese $ctr4b, $rk9 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 9
aese $ctr1b, $rk9 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 9
aese $ctr7b, $rk9 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 9
eor3 $acc_mb, $acc_mb, $acc_hb, $acc_lb @ MODULO - karatsuba tidy up
aese $ctr5b, $rk9 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 9
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+17
aese $ctr2b, $rk10 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 10
aese $ctr4b, $rk10 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 10
ldr $rk12q, [$cc, #192] @ load rk12
ext $t12.16b, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment
aese $ctr0b, $rk10 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 10
aese $ctr7b, $rk10 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 10
ldp $ctr_t2q, $ctr_t3q, [$input_ptr], #32 @ AES block 8k+10, 8k+11 - load plaintext
aese $ctr4b, $rk11 @ AES block 8k+12 - round 11
eor3 $acc_mb, $acc_mb, $t12.16b, $t11.16b @ MODULO - fold into mid
ldp $ctr_t4q, $ctr_t5q, [$input_ptr], #32 @ AES block 8k+12, 8k+13 - load plaintext
ldp $ctr_t6q, $ctr_t7q, [$input_ptr], #32 @ AES block 8k+14, 8k+15 - load plaintext
aese $ctr2b, $rk11 @ AES block 8k+10 - round 11
aese $ctr1b, $rk10 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 10
rev32 $h3.16b, $rtmp_ctr.16b @ CTR block 8k+18
aese $ctr5b, $rk10 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 10
aese $ctr3b, $rk10 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 10
pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low
aese $ctr6b, $rk10 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 10
aese $ctr5b, $rk11 @ AES block 8k+13 - round 11
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+18
aese $ctr7b, $rk11 @ AES block 8k+15 - round 11
aese $ctr0b, $rk11 @ AES block 8k+8 - round 11
eor3 $res4b, $ctr_t4b, $ctr4b, $rk12 @ AES block 4 - result
aese $ctr6b, $rk11 @ AES block 8k+14 - round 11
aese $ctr3b, $rk11 @ AES block 8k+11 - round 11
aese $ctr1b, $rk11 @ AES block 8k+9 - round 11
rev32 $h4.16b, $rtmp_ctr.16b @ CTR block 8k+19
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+19
eor3 $res7b, $ctr_t7b, $ctr7b, $rk12 @ AES block 7 - result
eor3 $res2b, $ctr_t2b, $ctr2b, $rk12 @ AES block 8k+10 - result
eor3 $res0b, $ctr_t0b, $ctr0b, $rk12 @ AES block 8k+8 - result
mov $ctr2.16b, $h3.16b @ CTR block 8k+18
eor3 $res1b, $ctr_t1b, $ctr1b, $rk12 @ AES block 8k+9 - result
mov $ctr1.16b, $h2.16b @ CTR block 8k+17
stp $res0q, $res1q, [$output_ptr], #32 @ AES block 8k+8, 8k+9 - store result
ext $t11.16b, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment
eor3 $res6b, $ctr_t6b, $ctr6b, $rk12 @ AES block 6 - result
mov $ctr0.16b, $h1.16b @ CTR block 8k+16
rev32 $ctr4.16b, $rtmp_ctr.16b @ CTR block 8k+20
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+20
eor3 $res5b, $ctr_t5b, $ctr5b, $rk12 @ AES block 5 - result
eor3 $acc_lb, $acc_lb, $acc_hb, $t11.16b @ MODULO - fold into low
eor3 $res3b, $ctr_t3b, $ctr3b, $rk12 @ AES block 8k+11 - result
mov $ctr3.16b, $h4.16b @ CTR block 8k+19
stp $res2q, $res3q, [$output_ptr], #32 @ AES block 8k+10, 8k+11 - store result
stp $res4q, $res5q, [$output_ptr], #32 @ AES block 8k+12, 8k+13 - store result
cmp $input_ptr, $main_end_input_ptr @ LOOP CONTROL
stp $res6q, $res7q, [$output_ptr], #32 @ AES block 8k+14, 8k+15 - store result
b.lt .L192_enc_main_loop
.L192_enc_prepretail: @ PREPRETAIL
rev32 $ctr5.16b, $rtmp_ctr.16b @ CTR block 8k+13
ldp $rk0q, $rk1q, [$cc, #0] @ load rk0, rk1
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+13
ldr $h7q, [$current_tag, #176] @ load h7l | h7h
ext $h7.16b, $h7.16b, $h7.16b, #8
ldr $h8q, [$current_tag, #208] @ load h8l | h8h
ext $h8.16b, $h8.16b, $h8.16b, #8
rev64 $res0b, $res0b @ GHASH block 8k
ext $acc_lb, $acc_lb, $acc_lb, #8 @ PRE 0
rev32 $ctr6.16b, $rtmp_ctr.16b @ CTR block 8k+14
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+14
ldr $h56kq, [$current_tag, #144] @ load h6k | h5k
ldr $h78kq, [$current_tag, #192] @ load h8k | h7k
rev64 $res3b, $res3b @ GHASH block 8k+3
rev64 $res2b, $res2b @ GHASH block 8k+2
ldr $h5q, [$current_tag, #128] @ load h5l | h5h
ext $h5.16b, $h5.16b, $h5.16b, #8
ldr $h6q, [$current_tag, #160] @ load h6l | h6h
ext $h6.16b, $h6.16b, $h6.16b, #8
eor $res0b, $res0b, $acc_lb @ PRE 1
rev32 $ctr7.16b, $rtmp_ctr.16b @ CTR block 8k+15
rev64 $res1b, $res1b @ GHASH block 8k+1
aese $ctr5b, $rk0 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 0
aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 0
aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 0
pmull2 $t0.1q, $res1.2d, $h7.2d @ GHASH block 8k+1 - high
aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 0
aese $ctr6b, $rk0 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 0
aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 0
aese $ctr4b, $rk0 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 0
pmull2 $acc_h.1q, $res0.2d, $h8.2d @ GHASH block 8k - high
aese $ctr6b, $rk1 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 1
pmull $acc_l.1q, $res0.1d, $h8.1d @ GHASH block 8k - low
trn1 $acc_m.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid
trn2 $res0.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid
aese $ctr7b, $rk0 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 0
ldp $rk2q, $rk3q, [$cc, #32] @ load rk2, rk3
pmull $h7.1q, $res1.1d, $h7.1d @ GHASH block 8k+1 - low
eor $acc_hb, $acc_hb, $t0.16b @ GHASH block 8k+1 - high
aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 1
aese $ctr5b, $rk1 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 1
eor $res0.16b, $res0.16b, $acc_m.16b @ GHASH block 8k, 8k+1 - mid
aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 1
aese $ctr7b, $rk1 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 1
pmull2 $t2.1q, $res3.2d, $h5.2d @ GHASH block 8k+3 - high
pmull2 $t1.1q, $res2.2d, $h6.2d @ GHASH block 8k+2 - high
aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 1
aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 1
aese $ctr4b, $rk1 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 1
pmull $h6.1q, $res2.1d, $h6.1d @ GHASH block 8k+2 - low
aese $ctr5b, $rk2 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 2
eor $acc_lb, $acc_lb, $h7.16b @ GHASH block 8k+1 - low
pmull $h5.1q, $res3.1d, $h5.1d @ GHASH block 8k+3 - low
aese $ctr7b, $rk2 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 2
eor3 $acc_hb, $acc_hb, $t1.16b, $t2.16b @ GHASH block 8k+2, 8k+3 - high
aese $ctr5b, $rk3 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 3
trn1 $t3.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid
aese $ctr6b, $rk2 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 2
aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 2
pmull2 $acc_m.1q, $res0.2d, $h78k.2d @ GHASH block 8k - mid
trn2 $res2.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid
aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 2
rev64 $res5b, $res5b @ GHASH block 8k+5 (t0, t1, t2 and t3 free)
rev64 $res6b, $res6b @ GHASH block 8k+6 (t0, t1, and t2 free)
aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 2
aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 2
aese $ctr4b, $rk2 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 2
eor $res2.16b, $res2.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid
pmull $h78k.1q, $res0.1d, $h78k.1d @ GHASH block 8k+1 - mid
ldp $rk4q, $rk5q, [$cc, #64] @ load rk4, rk5
aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 3
aese $ctr6b, $rk3 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 3
aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 3
eor $acc_mb, $acc_mb, $h78k.16b @ GHASH block 8k+1 - mid
eor3 $acc_lb, $acc_lb, $h6.16b, $h5.16b @ GHASH block 8k+2, 8k+3 - low
aese $ctr7b, $rk3 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 3
ldr $h3q, [$current_tag, #80] @ load h3l | h3h
ext $h3.16b, $h3.16b, $h3.16b, #8
ldr $h4q, [$current_tag, #112] @ load h4l | h4h
ext $h4.16b, $h4.16b, $h4.16b, #8
aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 3
pmull2 $t3.1q, $res2.2d, $h56k.2d @ GHASH block 8k+2 - mid
ldr $h1q, [$current_tag, #32] @ load h1l | h1h
ext $h1.16b, $h1.16b, $h1.16b, #8
ldr $h2q, [$current_tag, #64] @ load h2l | h2h
ext $h2.16b, $h2.16b, $h2.16b, #8
aese $ctr4b, $rk3 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 3
rev64 $res4b, $res4b @ GHASH block 8k+4 (t0, t1, and t2 free)
aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 3
pmull $h56k.1q, $res2.1d, $h56k.1d @ GHASH block 8k+3 - mid
aese $ctr6b, $rk4 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 4
trn1 $t6.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid
aese $ctr7b, $rk4 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 4
aese $ctr5b, $rk4 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 4
eor3 $acc_mb, $acc_mb, $h56k.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid
aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 4
aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 4
aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 4
aese $ctr4b, $rk4 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 4
aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 4
aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 5
rev64 $res7b, $res7b @ GHASH block 8k+7 (t0, t1, t2 and t3 free)
ldr $h12kq, [$current_tag, #48] @ load h2k | h1k
ldr $h34kq, [$current_tag, #96] @ load h4k | h3k
aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 5
aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 5
ldp $rk6q, $rk7q, [$cc, #96] @ load rk6, rk7
pmull2 $t7.1q, $res6.2d, $h2.2d @ GHASH block 8k+6 - high
pmull2 $t4.1q, $res4.2d, $h4.2d @ GHASH block 8k+4 - high
pmull $h4.1q, $res4.1d, $h4.1d @ GHASH block 8k+4 - low
aese $ctr4b, $rk5 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 5
trn2 $res4.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid
pmull2 $t5.1q, $res5.2d, $h3.2d @ GHASH block 8k+5 - high
pmull $h3.1q, $res5.1d, $h3.1d @ GHASH block 8k+5 - low
pmull $h2.1q, $res6.1d, $h2.1d @ GHASH block 8k+6 - low
trn1 $t9.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid
eor $res4.16b, $res4.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid
trn2 $res6.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid
aese $ctr5b, $rk5 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 5
aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 6
aese $ctr7b, $rk5 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 5
aese $ctr6b, $rk5 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 5
eor $res6.16b, $res6.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid
aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 5
pmull2 $t6.1q, $res4.2d, $h34k.2d @ GHASH block 8k+4 - mid
pmull $h34k.1q, $res4.1d, $h34k.1d @ GHASH block 8k+5 - mid
aese $ctr4b, $rk6 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 6
aese $ctr5b, $rk6 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 6
aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 7
aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 6
aese $ctr7b, $rk6 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 6
eor3 $acc_mb, $acc_mb, $h34k.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid
aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 6
eor3 $acc_hb, $acc_hb, $t4.16b, $t5.16b @ GHASH block 8k+4, 8k+5 - high
aese $ctr5b, $rk7 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 7
aese $ctr6b, $rk6 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 6
ldr $mod_constantd, [$modulo_constant] @ MODULO - load modulo constant
aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 6
pmull2 $t9.1q, $res6.2d, $h12k.2d @ GHASH block 8k+6 - mid
aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 7
eor3 $acc_lb, $acc_lb, $h4.16b, $h3.16b @ GHASH block 8k+4, 8k+5 - low
pmull2 $t8.1q, $res7.2d, $h1.2d @ GHASH block 8k+7 - high
pmull $h12k.1q, $res6.1d, $h12k.1d @ GHASH block 8k+7 - mid
pmull $h1.1q, $res7.1d, $h1.1d @ GHASH block 8k+7 - low
aese $ctr4b, $rk7 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 7
aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 7
ldp $rk8q, $rk9q, [$cc, #128] @ load rk8, rk9
aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 7
eor3 $acc_mb, $acc_mb, $h12k.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid
eor3 $acc_lb, $acc_lb, $h2.16b, $h1.16b @ GHASH block 8k+6, 8k+7 - low
eor3 $acc_hb, $acc_hb, $t7.16b, $t8.16b @ GHASH block 8k+6, 8k+7 - high
eor3 $acc_mb, $acc_mb, $acc_hb, $acc_lb @ MODULO - karatsuba tidy up
ext $t12.16b, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment
aese $ctr7b, $rk7 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 7
pmull $t11.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid
aese $ctr5b, $rk8 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 8
aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 8
aese $ctr6b, $rk7 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 7
aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 8
eor3 $acc_mb, $acc_mb, $t12.16b, $t11.16b @ MODULO - fold into mid
aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 8
aese $ctr5b, $rk9 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 9
aese $ctr4b, $rk8 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 8
aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 8
aese $ctr7b, $rk8 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 8
aese $ctr6b, $rk8 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 8
aese $ctr3b, $rk9 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 9
ldp $rk10q, $rk11q, [$cc, #160] @ load rk10, rk11
aese $ctr4b, $rk9 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 9
aese $ctr2b, $rk9 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 9
aese $ctr7b, $rk9 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 9
ext $t11.16b, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment
aese $ctr6b, $rk9 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 9
aese $ctr0b, $rk9 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 9
aese $ctr1b, $rk9 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 9
pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low
ldr $rk12q, [$cc, #192] @ load rk12
aese $ctr7b, $rk10 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 10
aese $ctr1b, $rk10 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 10
aese $ctr2b, $rk10 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 10
eor3 $acc_lb, $acc_lb, $acc_hb, $t11.16b @ MODULO - fold into low
aese $ctr0b, $rk10 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 10
aese $ctr3b, $rk10 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 10
aese $ctr1b, $rk11 @ AES block 8k+9 - round 11
aese $ctr7b, $rk11 @ AES block 8k+15 - round 11
aese $ctr4b, $rk10 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 10
aese $ctr3b, $rk11 @ AES block 8k+11 - round 11
aese $ctr5b, $rk10 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 10
aese $ctr6b, $rk10 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 10
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+15
aese $ctr2b, $rk11 @ AES block 8k+10 - round 11
aese $ctr0b, $rk11 @ AES block 8k+8 - round 11
aese $ctr6b, $rk11 @ AES block 8k+14 - round 11
aese $ctr4b, $rk11 @ AES block 8k+12 - round 11
aese $ctr5b, $rk11 @ AES block 8k+13 - round 11
.L192_enc_tail: @ TAIL
ldp $h5q, $h56kq, [$current_tag, #128] @ load h5l | h5h
ext $h5.16b, $h5.16b, $h5.16b, #8
sub $main_end_input_ptr, $end_input_ptr, $input_ptr @ main_end_input_ptr is number of bytes left to process
ldr $ctr_t0q, [$input_ptr], #16 @ AES block 8k+8 - l3ad plaintext
ldp $h78kq, $h8q, [$current_tag, #192] @ load h8k | h7k
ext $h8.16b, $h8.16b, $h8.16b, #8
mov $t1.16b, $rk12
ldp $h6q, $h7q, [$current_tag, #160] @ load h6l | h6h
ext $h6.16b, $h6.16b, $h6.16b, #8
ext $h7.16b, $h7.16b, $h7.16b, #8
cmp $main_end_input_ptr, #112
eor3 $res1b, $ctr_t0b, $ctr0b, $t1.16b @ AES block 8k+8 - result
ext $t0.16b, $acc_lb, $acc_lb, #8 @ prepare final partial tag
b.gt .L192_enc_blocks_more_than_7
cmp $main_end_input_ptr, #96
mov $ctr7b, $ctr6b
movi $acc_h.8b, #0
mov $ctr6b, $ctr5b
movi $acc_l.8b, #0
sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
mov $ctr5b, $ctr4b
mov $ctr4b, $ctr3b
mov $ctr3b, $ctr2b
mov $ctr2b, $ctr1b
movi $acc_m.8b, #0
b.gt .L192_enc_blocks_more_than_6
mov $ctr7b, $ctr6b
cmp $main_end_input_ptr, #80
mov $ctr6b, $ctr5b
mov $ctr5b, $ctr4b
mov $ctr4b, $ctr3b
mov $ctr3b, $ctr1b
sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
b.gt .L192_enc_blocks_more_than_5
cmp $main_end_input_ptr, #64
sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
mov $ctr7b, $ctr6b
mov $ctr6b, $ctr5b
mov $ctr5b, $ctr4b
mov $ctr4b, $ctr1b
b.gt .L192_enc_blocks_more_than_4
mov $ctr7b, $ctr6b
mov $ctr6b, $ctr5b
mov $ctr5b, $ctr1b
sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
cmp $main_end_input_ptr, #48
b.gt .L192_enc_blocks_more_than_3
mov $ctr7b, $ctr6b
mov $ctr6b, $ctr1b
sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
ldr $h34kq, [$current_tag, #96] @ load h4k | h3k
cmp $main_end_input_ptr, #32
b.gt .L192_enc_blocks_more_than_2
sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
cmp $main_end_input_ptr, #16
mov $ctr7b, $ctr1b
b.gt .L192_enc_blocks_more_than_1
sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
ldr $h12kq, [$current_tag, #48] @ load h2k | h1k
b .L192_enc_blocks_less_than_1
.L192_enc_blocks_more_than_7: @ blocks left > 7
st1 { $res1b}, [$output_ptr], #16 @ AES final-7 block - store result
rev64 $res0b, $res1b @ GHASH final-7 block
ins $acc_m.d[0], $h78k.d[1] @ GHASH final-7 block - mid
eor $res0b, $res0b, $t0.16b @ feed in partial tag
ins $rk4v.d[0], $res0.d[1] @ GHASH final-7 block - mid
ldr $ctr_t1q, [$input_ptr], #16 @ AES final-6 block - load plaintext
eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-7 block - mid
movi $t0.8b, #0 @ supress further partial tag feed in
pmull $acc_l.1q, $res0.1d, $h8.1d @ GHASH final-7 block - low
pmull2 $acc_h.1q, $res0.2d, $h8.2d @ GHASH final-7 block - high
pmull $acc_m.1q, $rk4v.1d, $acc_m.1d @ GHASH final-7 block - mid
eor3 $res1b, $ctr_t1b, $ctr1b, $t1.16b @ AES final-6 block - result
.L192_enc_blocks_more_than_6: @ blocks left > 6
st1 { $res1b}, [$output_ptr], #16 @ AES final-6 block - store result
rev64 $res0b, $res1b @ GHASH final-6 block
ldr $ctr_t1q, [$input_ptr], #16 @ AES final-5 block - load plaintext
eor $res0b, $res0b, $t0.16b @ feed in partial tag
ins $rk4v.d[0], $res0.d[1] @ GHASH final-6 block - mid
pmull $rk3q1, $res0.1d, $h7.1d @ GHASH final-6 block - low
eor3 $res1b, $ctr_t1b, $ctr2b, $t1.16b @ AES final-5 block - result
movi $t0.8b, #0 @ supress further partial tag feed in
pmull2 $rk2q1, $res0.2d, $h7.2d @ GHASH final-6 block - high
eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-6 block - mid
pmull $rk4v.1q, $rk4v.1d, $h78k.1d @ GHASH final-6 block - mid
eor $acc_hb, $acc_hb, $rk2 @ GHASH final-6 block - high
eor $acc_lb, $acc_lb, $rk3 @ GHASH final-6 block - low
eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-6 block - mid
.L192_enc_blocks_more_than_5: @ blocks left > 5
st1 { $res1b}, [$output_ptr], #16 @ AES final-5 block - store result
rev64 $res0b, $res1b @ GHASH final-5 block
eor $res0b, $res0b, $t0.16b @ feed in partial tag
ins $rk4v.d[0], $res0.d[1] @ GHASH final-5 block - mid
ldr $ctr_t1q, [$input_ptr], #16 @ AES final-4 block - load plaintext
pmull2 $rk2q1, $res0.2d, $h6.2d @ GHASH final-5 block - high
eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-5 block - mid
eor $acc_hb, $acc_hb, $rk2 @ GHASH final-5 block - high
ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-5 block - mid
pmull $rk3q1, $res0.1d, $h6.1d @ GHASH final-5 block - low
eor $acc_lb, $acc_lb, $rk3 @ GHASH final-5 block - low
pmull2 $rk4v.1q, $rk4v.2d, $h56k.2d @ GHASH final-5 block - mid
eor3 $res1b, $ctr_t1b, $ctr3b, $t1.16b @ AES final-4 block - result
movi $t0.8b, #0 @ supress further partial tag feed in
eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-5 block - mid
.L192_enc_blocks_more_than_4: @ blocks left > 4
st1 { $res1b}, [$output_ptr], #16 @ AES final-4 block - store result
rev64 $res0b, $res1b @ GHASH final-4 block
eor $res0b, $res0b, $t0.16b @ feed in partial tag
ldr $ctr_t1q, [$input_ptr], #16 @ AES final-3 block - load plaintext
pmull2 $rk2q1, $res0.2d, $h5.2d @ GHASH final-4 block - high
ins $rk4v.d[0], $res0.d[1] @ GHASH final-4 block - mid
pmull $rk3q1, $res0.1d, $h5.1d @ GHASH final-4 block - low
eor $acc_hb, $acc_hb, $rk2 @ GHASH final-4 block - high
eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-4 block - mid
movi $t0.8b, #0 @ supress further partial tag feed in
eor $acc_lb, $acc_lb, $rk3 @ GHASH final-4 block - low
pmull $rk4v.1q, $rk4v.1d, $h56k.1d @ GHASH final-4 block - mid
eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-4 block - mid
eor3 $res1b, $ctr_t1b, $ctr4b, $t1.16b @ AES final-3 block - result
.L192_enc_blocks_more_than_3: @ blocks left > 3
ldr $h34kq, [$current_tag, #96] @ load h4k | h3k
st1 { $res1b}, [$output_ptr], #16 @ AES final-3 block - store result
rev64 $res0b, $res1b @ GHASH final-3 block
eor $res0b, $res0b, $t0.16b @ feed in partial tag
movi $t0.8b, #0 @ supress further partial tag feed in
ldr $ctr_t1q, [$input_ptr], #16 @ AES final-2 block - load plaintext
ldr $h4q, [$current_tag, #112] @ load h4l | h4h
ext $h4.16b, $h4.16b, $h4.16b, #8
ins $rk4v.d[0], $res0.d[1] @ GHASH final-3 block - mid
eor3 $res1b, $ctr_t1b, $ctr5b, $t1.16b @ AES final-2 block - result
eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-3 block - mid
ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-3 block - mid
pmull $rk3q1, $res0.1d, $h4.1d @ GHASH final-3 block - low
pmull2 $rk2q1, $res0.2d, $h4.2d @ GHASH final-3 block - high
pmull2 $rk4v.1q, $rk4v.2d, $h34k.2d @ GHASH final-3 block - mid
eor $acc_lb, $acc_lb, $rk3 @ GHASH final-3 block - low
eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-3 block - mid
eor $acc_hb, $acc_hb, $rk2 @ GHASH final-3 block - high
.L192_enc_blocks_more_than_2: @ blocks left > 2
st1 { $res1b}, [$output_ptr], #16 @ AES final-2 block - store result
rev64 $res0b, $res1b @ GHASH final-2 block
ldr $h3q, [$current_tag, #80] @ load h3l | h3h
ext $h3.16b, $h3.16b, $h3.16b, #8
eor $res0b, $res0b, $t0.16b @ feed in partial tag
ldr $ctr_t1q, [$input_ptr], #16 @ AES final-1 block - load plaintext
ins $rk4v.d[0], $res0.d[1] @ GHASH final-2 block - mid
eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-2 block - mid
pmull $rk3q1, $res0.1d, $h3.1d @ GHASH final-2 block - low
pmull2 $rk2q1, $res0.2d, $h3.2d @ GHASH final-2 block - high
movi $t0.8b, #0 @ supress further partial tag feed in
pmull $rk4v.1q, $rk4v.1d, $h34k.1d @ GHASH final-2 block - mid
eor $acc_lb, $acc_lb, $rk3 @ GHASH final-2 block - low
eor $acc_hb, $acc_hb, $rk2 @ GHASH final-2 block - high
eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-2 block - mid
eor3 $res1b, $ctr_t1b, $ctr6b, $t1.16b @ AES final-1 block - result
.L192_enc_blocks_more_than_1: @ blocks left > 1
ldr $h2q, [$current_tag, #64] @ load h1l | h1h
ext $h2.16b, $h2.16b, $h2.16b, #8
st1 { $res1b}, [$output_ptr], #16 @ AES final-1 block - store result
rev64 $res0b, $res1b @ GHASH final-1 block
eor $res0b, $res0b, $t0.16b @ feed in partial tag
ins $rk4v.d[0], $res0.d[1] @ GHASH final-1 block - mid
pmull $rk3q1, $res0.1d, $h2.1d @ GHASH final-1 block - low
eor $acc_lb, $acc_lb, $rk3 @ GHASH final-1 block - low
pmull2 $rk2q1, $res0.2d, $h2.2d @ GHASH final-1 block - high
eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-1 block - mid
ldr $ctr_t1q, [$input_ptr], #16 @ AES final block - load plaintext
ldr $h12kq, [$current_tag, #48] @ load h2k | h1k
ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-1 block - mid
eor3 $res1b, $ctr_t1b, $ctr7b, $t1.16b @ AES final block - result
pmull2 $rk4v.1q, $rk4v.2d, $h12k.2d @ GHASH final-1 block - mid
movi $t0.8b, #0 @ supress further partial tag feed in
eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-1 block - mid
eor $acc_hb, $acc_hb, $rk2 @ GHASH final-1 block - high
.L192_enc_blocks_less_than_1: @ blocks left <= 1
mvn $temp0_x, xzr @ temp0_x = 0xffffffffffffffff
and $bit_length, $bit_length, #127 @ bit_length %= 128
sub $bit_length, $bit_length, #128 @ bit_length -= 128
neg $bit_length, $bit_length @ bit_length = 128 - #bits in input (in range [1,128])
and $bit_length, $bit_length, #127 @ bit_length %= 128
lsr $temp0_x, $temp0_x, $bit_length @ temp0_x is mask for top 64b of last block
cmp $bit_length, #64
mvn $temp1_x, xzr @ temp1_x = 0xffffffffffffffff
csel $temp2_x, $temp1_x, $temp0_x, lt
csel $temp3_x, $temp0_x, xzr, lt
mov $ctr0.d[1], $temp3_x
ldr $h1q, [$current_tag, #32] @ load h1l | h1h
ext $h1.16b, $h1.16b, $h1.16b, #8
ld1 { $rk0}, [$output_ptr] @ load existing bytes where the possibly partial last block is to be stored
mov $ctr0.d[0], $temp2_x @ ctr0b is mask for last block
and $res1b, $res1b, $ctr0b @ possibly partial last block has zeroes in highest bits
rev64 $res0b, $res1b @ GHASH final block
bif $res1b, $rk0, $ctr0b @ insert existing bytes in top end of result before storing
st1 { $res1b}, [$output_ptr] @ store all 16B
eor $res0b, $res0b, $t0.16b @ feed in partial tag
ins $t0.d[0], $res0.d[1] @ GHASH final block - mid
pmull2 $rk2q1, $res0.2d, $h1.2d @ GHASH final block - high
eor $acc_hb, $acc_hb, $rk2 @ GHASH final block - high
pmull $rk3q1, $res0.1d, $h1.1d @ GHASH final block - low
eor $t0.8b, $t0.8b, $res0.8b @ GHASH final block - mid
pmull $t0.1q, $t0.1d, $h12k.1d @ GHASH final block - mid
eor $acc_mb, $acc_mb, $t0.16b @ GHASH final block - mid
ldr $mod_constantd, [$modulo_constant] @ MODULO - load modulo constant
eor $acc_lb, $acc_lb, $rk3 @ GHASH final block - low
ext $t11.16b, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment
rev32 $rtmp_ctr.16b, $rtmp_ctr.16b
str $rtmp_ctrq, [$counter] @ store the updated counter
eor3 $acc_mb, $acc_mb, $acc_hb, $acc_lb @ MODULO - karatsuba tidy up
pmull $t12.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid
eor3 $acc_mb, $acc_mb, $t12.16b, $t11.16b @ MODULO - fold into mid
pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low
ext $t11.16b, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment
eor3 $acc_lb, $acc_lb, $acc_hb, $t11.16b @ MODULO - fold into low
ext $acc_lb, $acc_lb, $acc_lb, #8
rev64 $acc_lb, $acc_lb
st1 { $acc_l.16b }, [$current_tag]
lsr x0, $bit_length, #3 @ return sizes
ldp d10, d11, [sp, #16]
ldp d12, d13, [sp, #32]
ldp d14, d15, [sp, #48]
ldp d8, d9, [sp], #80
ret
.L192_enc_ret:
mov w0, #0x0
ret
.size unroll8_eor3_aes_gcm_enc_192_kernel,.-unroll8_eor3_aes_gcm_enc_192_kernel
___
#########################################################################################
# size_t unroll8_eor3_aes_gcm_dec_192_kernel(const unsigned char *in,
# size_t len,
# unsigned char *out,
# const void *key,
# unsigned char ivec[16],
# u64 *Xi);
#
$code.=<<___;
.global unroll8_eor3_aes_gcm_dec_192_kernel
.type unroll8_eor3_aes_gcm_dec_192_kernel,%function
.align 4
unroll8_eor3_aes_gcm_dec_192_kernel:
AARCH64_VALID_CALL_TARGET
cbz x1, .L192_dec_ret
stp d8, d9, [sp, #-80]!
mov $counter, x4
mov $cc, x5
stp d10, d11, [sp, #16]
stp d12, d13, [sp, #32]
stp d14, d15, [sp, #48]
mov x5, #0xc200000000000000
stp x5, xzr, [sp, #64]
add $modulo_constant, sp, #64
lsr $main_end_input_ptr, $bit_length, #3 @ byte_len
ld1 { $ctr0b}, [$counter] @ CTR block 0
ld1 { $acc_lb}, [$current_tag]
mov $constant_temp, #0x100000000 @ set up counter increment
movi $rctr_inc.16b, #0x0
mov $rctr_inc.d[1], $constant_temp
rev32 $rtmp_ctr.16b, $ctr0.16b @ set up reversed counter
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 0
rev32 $ctr1.16b, $rtmp_ctr.16b @ CTR block 1
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 1
rev32 $ctr2.16b, $rtmp_ctr.16b @ CTR block 2
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 2
rev32 $ctr3.16b, $rtmp_ctr.16b @ CTR block 3
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 3
rev32 $ctr4.16b, $rtmp_ctr.16b @ CTR block 4
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 4
rev32 $ctr5.16b, $rtmp_ctr.16b @ CTR block 5
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 5
ldp $rk0q, $rk1q, [$cc, #0] @ load rk0, rk1
rev32 $ctr6.16b, $rtmp_ctr.16b @ CTR block 6
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 6
rev32 $ctr7.16b, $rtmp_ctr.16b @ CTR block 7
aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 0
aese $ctr6b, $rk0 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 0
aese $ctr5b, $rk0 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 0
aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 0
aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 0
aese $ctr7b, $rk0 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 0
aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 0
aese $ctr4b, $rk0 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 0
ldp $rk2q, $rk3q, [$cc, #32] @ load rk2, rk3
aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 1
aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 1
aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 1
aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 1
aese $ctr7b, $rk1 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 1
aese $ctr5b, $rk1 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 1
aese $ctr6b, $rk1 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 1
aese $ctr7b, $rk2 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 2
aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 2
aese $ctr4b, $rk1 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 1
aese $ctr5b, $rk2 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 2
aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 2
aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 2
aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 2
aese $ctr4b, $rk2 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 2
aese $ctr6b, $rk2 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 2
aese $ctr7b, $rk3 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 3
ldp $rk4q, $rk5q, [$cc, #64] @ load rk4, rk5
aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 3
aese $ctr5b, $rk3 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 3
aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 3
aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 3
aese $ctr4b, $rk3 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 3
aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 3
aese $ctr6b, $rk3 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 3
aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 4
aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 4
aese $ctr5b, $rk4 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 4
aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 4
aese $ctr7b, $rk4 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 4
aese $ctr6b, $rk4 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 4
aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 4
aese $ctr5b, $rk5 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 5
aese $ctr4b, $rk4 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 4
aese $ctr6b, $rk5 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 5
ldp $rk6q, $rk7q, [$cc, #96] @ load rk6, rk7
aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 5
aese $ctr4b, $rk5 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 5
aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 5
aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 5
aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 5
aese $ctr7b, $rk5 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 5
sub $main_end_input_ptr, $main_end_input_ptr, #1 @ byte_len - 1
aese $ctr4b, $rk6 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 6
aese $ctr5b, $rk6 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 6
aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 6
aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 6
aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 6
aese $ctr6b, $rk6 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 6
aese $ctr7b, $rk6 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 6
aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 6
ldp $rk8q, $rk9q, [$cc, #128] @ load rk8, rk9
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 7
aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 7
aese $ctr7b, $rk7 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 7
aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 7
aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 7
aese $ctr4b, $rk7 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 7
aese $ctr6b, $rk7 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 7
aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 7
aese $ctr5b, $rk7 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 7
aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 8
aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 8
and $main_end_input_ptr, $main_end_input_ptr, #0xffffffffffffff80 @ number of bytes to be processed in main loop (at least 1 byte must be handled by tail)
aese $ctr7b, $rk8 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 8
aese $ctr6b, $rk8 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 8
aese $ctr5b, $rk8 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 8
aese $ctr4b, $rk8 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 8
aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 8
aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 8
add $end_input_ptr, $input_ptr, $bit_length, lsr #3 @ end_input_ptr
aese $ctr6b, $rk9 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 9
ld1 { $acc_lb}, [$current_tag]
ext $acc_lb, $acc_lb, $acc_lb, #8
rev64 $acc_lb, $acc_lb
ldp $rk10q, $rk11q, [$cc, #160] @ load rk10, rk11
aese $ctr0b, $rk9 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 9
add $main_end_input_ptr, $main_end_input_ptr, $input_ptr
aese $ctr1b, $rk9 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 9
aese $ctr7b, $rk9 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 9
aese $ctr4b, $rk9 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 9
cmp $input_ptr, $main_end_input_ptr @ check if we have <= 8 blocks
aese $ctr3b, $rk9 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 9
aese $ctr5b, $rk9 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 9
aese $ctr2b, $rk9 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 9
aese $ctr3b, $rk10 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 10
aese $ctr1b, $rk10 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 10
aese $ctr7b, $rk10 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 10
aese $ctr4b, $rk10 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 10
aese $ctr0b, $rk10 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 10
aese $ctr2b, $rk10 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 10
aese $ctr6b, $rk10 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 10
aese $ctr5b, $rk10 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 10
ldr $rk12q, [$cc, #192] @ load rk12
aese $ctr0b, $rk11 @ AES block 0 - round 11
aese $ctr1b, $rk11 @ AES block 1 - round 11
aese $ctr4b, $rk11 @ AES block 4 - round 11
aese $ctr6b, $rk11 @ AES block 6 - round 11
aese $ctr5b, $rk11 @ AES block 5 - round 11
aese $ctr7b, $rk11 @ AES block 7 - round 11
aese $ctr2b, $rk11 @ AES block 2 - round 11
aese $ctr3b, $rk11 @ AES block 3 - round 11
b.ge .L192_dec_tail @ handle tail
ldp $res0q, $res1q, [$input_ptr], #32 @ AES block 0, 1 - load ciphertext
ldp $res2q, $res3q, [$input_ptr], #32 @ AES block 2, 3 - load ciphertext
ldp $res4q, $res5q, [$input_ptr], #32 @ AES block 4, 5 - load ciphertext
eor3 $ctr1b, $res1b, $ctr1b, $rk12 @ AES block 1 - result
eor3 $ctr0b, $res0b, $ctr0b, $rk12 @ AES block 0 - result
stp $ctr0q, $ctr1q, [$output_ptr], #32 @ AES block 0, 1 - store result
rev32 $ctr0.16b, $rtmp_ctr.16b @ CTR block 8
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8
rev32 $ctr1.16b, $rtmp_ctr.16b @ CTR block 9
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 9
eor3 $ctr3b, $res3b, $ctr3b, $rk12 @ AES block 3 - result
eor3 $ctr2b, $res2b, $ctr2b, $rk12 @ AES block 2 - result
stp $ctr2q, $ctr3q, [$output_ptr], #32 @ AES block 2, 3 - store result
ldp $res6q, $res7q, [$input_ptr], #32 @ AES block 6, 7 - load ciphertext
rev32 $ctr2.16b, $rtmp_ctr.16b @ CTR block 10
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 10
eor3 $ctr4b, $res4b, $ctr4b, $rk12 @ AES block 4 - result
rev32 $ctr3.16b, $rtmp_ctr.16b @ CTR block 11
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 11
eor3 $ctr5b, $res5b, $ctr5b, $rk12 @ AES block 5 - result
stp $ctr4q, $ctr5q, [$output_ptr], #32 @ AES block 4, 5 - store result
cmp $input_ptr, $main_end_input_ptr @ check if we have <= 8 blocks
eor3 $ctr6b, $res6b, $ctr6b, $rk12 @ AES block 6 - result
eor3 $ctr7b, $res7b, $ctr7b, $rk12 @ AES block 7 - result
rev32 $ctr4.16b, $rtmp_ctr.16b @ CTR block 12
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 12
stp $ctr6q, $ctr7q, [$output_ptr], #32 @ AES block 6, 7 - store result
b.ge .L192_dec_prepretail @ do prepretail
.L192_dec_main_loop: @ main loop start
rev64 $res1b, $res1b @ GHASH block 8k+1
ldp $rk0q, $rk1q, [$cc, #0] @ load rk0, rk1
ext $acc_lb, $acc_lb, $acc_lb, #8 @ PRE 0
rev64 $res0b, $res0b @ GHASH block 8k
rev32 $ctr5.16b, $rtmp_ctr.16b @ CTR block 8k+13
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+13
ldr $h7q, [$current_tag, #176] @ load h7l | h7h
ext $h7.16b, $h7.16b, $h7.16b, #8
ldr $h8q, [$current_tag, #208] @ load h8l | h8h
ext $h8.16b, $h8.16b, $h8.16b, #8
rev64 $res4b, $res4b @ GHASH block 8k+4
rev64 $res3b, $res3b @ GHASH block 8k+3
eor $res0b, $res0b, $acc_lb @ PRE 1
rev32 $ctr6.16b, $rtmp_ctr.16b @ CTR block 8k+14
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+14
rev64 $res5b, $res5b @ GHASH block 8k+5
rev32 $ctr7.16b, $rtmp_ctr.16b @ CTR block 8k+15
aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 0
aese $ctr6b, $rk0 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 0
aese $ctr5b, $rk0 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 0
aese $ctr4b, $rk0 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 0
aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 0
aese $ctr7b, $rk0 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 0
aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 0
aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 0
pmull $acc_l.1q, $res0.1d, $h8.1d @ GHASH block 8k - low
pmull2 $t0.1q, $res1.2d, $h7.2d @ GHASH block 8k+1 - high
ldp $rk2q, $rk3q, [$cc, #32] @ load rk2, rk3
aese $ctr6b, $rk1 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 1
pmull $h7.1q, $res1.1d, $h7.1d @ GHASH block 8k+1 - low
ldr $h5q, [$current_tag, #128] @ load h5l | h5h
ext $h5.16b, $h5.16b, $h5.16b, #8
ldr $h6q, [$current_tag, #160] @ load h6l | h6h
ext $h6.16b, $h6.16b, $h6.16b, #8
aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 1
aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 1
aese $ctr7b, $rk1 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 1
pmull2 $acc_h.1q, $res0.2d, $h8.2d @ GHASH block 8k - high
aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 1
aese $ctr4b, $rk1 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 1
trn1 $acc_m.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid
rev64 $res2b, $res2b @ GHASH block 8k+2
aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 1
aese $ctr5b, $rk1 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 1
ldr $h56kq, [$current_tag, #144] @ load h6k | h5k
ldr $h78kq, [$current_tag, #192] @ load h8k | h7k
trn2 $res0.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid
eor $acc_hb, $acc_hb, $t0.16b @ GHASH block 8k+1 - high
pmull2 $t2.1q, $res3.2d, $h5.2d @ GHASH block 8k+3 - high
pmull2 $t1.1q, $res2.2d, $h6.2d @ GHASH block 8k+2 - high
eor $res0.16b, $res0.16b, $acc_m.16b @ GHASH block 8k, 8k+1 - mid
eor $acc_lb, $acc_lb, $h7.16b @ GHASH block 8k+1 - low
aese $ctr6b, $rk2 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 2
aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 2
pmull $h5.1q, $res3.1d, $h5.1d @ GHASH block 8k+3 - low
eor3 $acc_hb, $acc_hb, $t1.16b, $t2.16b @ GHASH block 8k+2, 8k+3 - high
aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 2
aese $ctr6b, $rk3 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 3
aese $ctr4b, $rk2 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 2
aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 2
aese $ctr7b, $rk2 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 2
aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 2
ldr $h3q, [$current_tag, #80] @ load h3l | h3h
ext $h3.16b, $h3.16b, $h3.16b, #8
ldr $h4q, [$current_tag, #112] @ load h4l | h4h
ext $h4.16b, $h4.16b, $h4.16b, #8
aese $ctr5b, $rk2 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 2
aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 3
pmull $h6.1q, $res2.1d, $h6.1d @ GHASH block 8k+2 - low
trn1 $t3.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid
trn2 $res2.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid
aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 3
aese $ctr4b, $rk3 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 3
aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 3
aese $ctr7b, $rk3 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 3
ldp $rk4q, $rk5q, [$cc, #64] @ load rk4, rk5
eor $res2.16b, $res2.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid
eor3 $acc_lb, $acc_lb, $h6.16b, $h5.16b @ GHASH block 8k+2, 8k+3 - low
aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 3
trn1 $t6.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+15
pmull2 $t3.1q, $res2.2d, $h56k.2d @ GHASH block 8k+2 - mid
pmull2 $acc_m.1q, $res0.2d, $h78k.2d @ GHASH block 8k - mid
pmull $h78k.1q, $res0.1d, $h78k.1d @ GHASH block 8k+1 - mid
aese $ctr5b, $rk3 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 3
pmull $h56k.1q, $res2.1d, $h56k.1d @ GHASH block 8k+3 - mid
pmull2 $t4.1q, $res4.2d, $h4.2d @ GHASH block 8k+4 - high
aese $ctr4b, $rk4 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 4
aese $ctr6b, $rk4 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 4
eor $acc_mb, $acc_mb, $h78k.16b @ GHASH block 8k+1 - mid
aese $ctr5b, $rk4 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 4
aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 4
aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 4
aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 4
aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 4
aese $ctr7b, $rk4 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 4
ldr $h1q, [$current_tag, #32] @ load h1l | h1h
ext $h1.16b, $h1.16b, $h1.16b, #8
ldr $h2q, [$current_tag, #64] @ load h2l | h2h
ext $h2.16b, $h2.16b, $h2.16b, #8
aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 5
aese $ctr5b, $rk5 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 5
ldp $rk6q, $rk7q, [$cc, #96] @ load rk6, rk7
aese $ctr7b, $rk5 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 5
rev64 $res7b, $res7b @ GHASH block 8k+7
aese $ctr4b, $rk5 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 5
eor3 $acc_mb, $acc_mb, $h56k.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid
aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 5
pmull $h4.1q, $res4.1d, $h4.1d @ GHASH block 8k+4 - low
trn2 $res4.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid
aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 5
aese $ctr6b, $rk5 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 5
aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 5
rev64 $res6b, $res6b @ GHASH block 8k+6
ldr $h12kq, [$current_tag, #48] @ load h2k | h1k
ldr $h34kq, [$current_tag, #96] @ load h4k | h3k
pmull2 $t5.1q, $res5.2d, $h3.2d @ GHASH block 8k+5 - high
pmull $h3.1q, $res5.1d, $h3.1d @ GHASH block 8k+5 - low
aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 6
eor $res4.16b, $res4.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid
trn1 $t9.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid
aese $ctr7b, $rk6 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 6
aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 6
aese $ctr6b, $rk6 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 6
pmull2 $t7.1q, $res6.2d, $h2.2d @ GHASH block 8k+6 - high
aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 6
aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 6
aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 7
aese $ctr6b, $rk7 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 7
aese $ctr5b, $rk6 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 6
pmull2 $t6.1q, $res4.2d, $h34k.2d @ GHASH block 8k+4 - mid
eor3 $acc_hb, $acc_hb, $t4.16b, $t5.16b @ GHASH block 8k+4, 8k+5 - high
eor3 $acc_lb, $acc_lb, $h4.16b, $h3.16b @ GHASH block 8k+4, 8k+5 - low
pmull $h2.1q, $res6.1d, $h2.1d @ GHASH block 8k+6 - low
trn2 $res6.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid
aese $ctr4b, $rk6 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 6
aese $ctr5b, $rk7 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 7
ldp $rk8q, $rk9q, [$cc, #128] @ load rk8, rk9
aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 7
eor $res6.16b, $res6.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid
pmull $h34k.1q, $res4.1d, $h34k.1d @ GHASH block 8k+5 - mid
aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 7
aese $ctr4b, $rk7 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 7
aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 7
aese $ctr7b, $rk7 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 7
eor3 $acc_mb, $acc_mb, $h34k.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid
pmull2 $t9.1q, $res6.2d, $h12k.2d @ GHASH block 8k+6 - mid
pmull2 $t8.1q, $res7.2d, $h1.2d @ GHASH block 8k+7 - high
pmull $h12k.1q, $res6.1d, $h12k.1d @ GHASH block 8k+7 - mid
ldr $mod_constantd, [$modulo_constant] @ MODULO - load modulo constant
pmull $h1.1q, $res7.1d, $h1.1d @ GHASH block 8k+7 - low
aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 8
aese $ctr5b, $rk8 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 8
aese $ctr7b, $rk8 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 8
aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 8
aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 8
eor3 $acc_lb, $acc_lb, $h2.16b, $h1.16b @ GHASH block 8k+6, 8k+7 - low
aese $ctr4b, $rk8 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 8
aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 8
aese $ctr6b, $rk8 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 8
eor3 $acc_hb, $acc_hb, $t7.16b, $t8.16b @ GHASH block 8k+6, 8k+7 - high
rev32 $h1.16b, $rtmp_ctr.16b @ CTR block 8k+16
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+16
aese $ctr5b, $rk9 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 9
eor3 $acc_mb, $acc_mb, $h12k.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid
aese $ctr1b, $rk9 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 9
aese $ctr3b, $rk9 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 9
aese $ctr7b, $rk9 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 9
ldp $rk10q, $rk11q, [$cc, #160] @ load rk10, rk11
eor3 $acc_mb, $acc_mb, $acc_hb, $acc_lb @ MODULO - karatsuba tidy up
ldp $res0q, $res1q, [$input_ptr], #32 @ AES block 8k+8, 8k+9 - load ciphertext
aese $ctr2b, $rk9 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 9
aese $ctr0b, $rk9 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 9
ldp $res2q, $res3q, [$input_ptr], #32 @ AES block 8k+10, 8k+11 - load ciphertext
rev32 $h2.16b, $rtmp_ctr.16b @ CTR block 8k+17
pmull $t12.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+17
aese $ctr6b, $rk9 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 9
aese $ctr4b, $rk9 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 9
ext $t11.16b, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment
aese $ctr3b, $rk10 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 10
aese $ctr7b, $rk10 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 10
ldp $res4q, $res5q, [$input_ptr], #32 @ AES block 8k+12, 8k+13 - load ciphertext
rev32 $h3.16b, $rtmp_ctr.16b @ CTR block 8k+18
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+18
eor3 $acc_mb, $acc_mb, $t12.16b, $t11.16b @ MODULO - fold into mid
aese $ctr0b, $rk10 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 10
aese $ctr1b, $rk10 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 10
ldr $rk12q, [$cc, #192] @ load rk12
ldp $res6q, $res7q, [$input_ptr], #32 @ AES block 8k+14, 8k+15 - load ciphertext
aese $ctr4b, $rk10 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 10
aese $ctr6b, $rk10 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 10
aese $ctr0b, $rk11 @ AES block 8k+8 - round 11
ext $t11.16b, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment
aese $ctr1b, $rk11 @ AES block 8k+9 - round 11
aese $ctr2b, $rk10 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 10
aese $ctr6b, $rk11 @ AES block 8k+14 - round 11
aese $ctr3b, $rk11 @ AES block 8k+11 - round 11
eor3 $ctr0b, $res0b, $ctr0b, $rk12 @ AES block 8k+8 - result
rev32 $h4.16b, $rtmp_ctr.16b @ CTR block 8k+19
aese $ctr5b, $rk10 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 10
aese $ctr4b, $rk11 @ AES block 8k+12 - round 11
aese $ctr2b, $rk11 @ AES block 8k+10 - round 11
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+19
aese $ctr7b, $rk11 @ AES block 8k+15 - round 11
aese $ctr5b, $rk11 @ AES block 8k+13 - round 11
pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low
eor3 $ctr1b, $res1b, $ctr1b, $rk12 @ AES block 8k+9 - result
stp $ctr0q, $ctr1q, [$output_ptr], #32 @ AES block 8k+8, 8k+9 - store result
eor3 $ctr3b, $res3b, $ctr3b, $rk12 @ AES block 8k+11 - result
eor3 $ctr2b, $res2b, $ctr2b, $rk12 @ AES block 8k+10 - result
eor3 $ctr7b, $res7b, $ctr7b, $rk12 @ AES block 8k+15 - result
stp $ctr2q, $ctr3q, [$output_ptr], #32 @ AES block 8k+10, 8k+11 - store result
eor3 $ctr5b, $res5b, $ctr5b, $rk12 @ AES block 8k+13 - result
eor3 $acc_lb, $acc_lb, $acc_hb, $t11.16b @ MODULO - fold into low
mov $ctr3.16b, $h4.16b @ CTR block 8k+19
eor3 $ctr4b, $res4b, $ctr4b, $rk12 @ AES block 8k+12 - result
stp $ctr4q, $ctr5q, [$output_ptr], #32 @ AES block 8k+12, 8k+13 - store result
cmp $input_ptr, $main_end_input_ptr @ LOOP CONTROL
eor3 $ctr6b, $res6b, $ctr6b, $rk12 @ AES block 8k+14 - result
stp $ctr6q, $ctr7q, [$output_ptr], #32 @ AES block 8k+14, 8k+15 - store result
mov $ctr0.16b, $h1.16b @ CTR block 8k+16
mov $ctr1.16b, $h2.16b @ CTR block 8k+17
mov $ctr2.16b, $h3.16b @ CTR block 8k+18
rev32 $ctr4.16b, $rtmp_ctr.16b @ CTR block 8k+20
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+20
b.lt .L192_dec_main_loop
.L192_dec_prepretail: @ PREPRETAIL
ldp $rk0q, $rk1q, [$cc, #0] @ load rk0, rk1
rev32 $ctr5.16b, $rtmp_ctr.16b @ CTR block 8k+13
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+13
ldr $h7q, [$current_tag, #176] @ load h7l | h7h
ext $h7.16b, $h7.16b, $h7.16b, #8
ldr $h8q, [$current_tag, #208] @ load h8l | h8h
ext $h8.16b, $h8.16b, $h8.16b, #8
rev64 $res0b, $res0b @ GHASH block 8k
ext $acc_lb, $acc_lb, $acc_lb, #8 @ PRE 0
rev64 $res3b, $res3b @ GHASH block 8k+3
rev32 $ctr6.16b, $rtmp_ctr.16b @ CTR block 8k+14
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+14
eor $res0b, $res0b, $acc_lb @ PRE 1
rev64 $res2b, $res2b @ GHASH block 8k+2
rev64 $res1b, $res1b @ GHASH block 8k+1
ldr $h5q, [$current_tag, #128] @ load h5l | h5h
ext $h5.16b, $h5.16b, $h5.16b, #8
ldr $h6q, [$current_tag, #160] @ load h6l | h6h
ext $h6.16b, $h6.16b, $h6.16b, #8
rev32 $ctr7.16b, $rtmp_ctr.16b @ CTR block 8k+15
aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 0
aese $ctr6b, $rk0 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 0
aese $ctr5b, $rk0 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 0
aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 0
aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 0
pmull2 $t0.1q, $res1.2d, $h7.2d @ GHASH block 8k+1 - high
aese $ctr4b, $rk0 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 0
pmull2 $acc_h.1q, $res0.2d, $h8.2d @ GHASH block 8k - high
aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 0
aese $ctr6b, $rk1 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 1
aese $ctr7b, $rk0 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 0
ldp $rk2q, $rk3q, [$cc, #32] @ load rk2, rk3
aese $ctr4b, $rk1 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 1
pmull2 $t1.1q, $res2.2d, $h6.2d @ GHASH block 8k+2 - high
pmull $h6.1q, $res2.1d, $h6.1d @ GHASH block 8k+2 - low
pmull $h7.1q, $res1.1d, $h7.1d @ GHASH block 8k+1 - low
eor $acc_hb, $acc_hb, $t0.16b @ GHASH block 8k+1 - high
aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 1
pmull $acc_l.1q, $res0.1d, $h8.1d @ GHASH block 8k - low
aese $ctr7b, $rk1 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 1
aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 1
trn1 $acc_m.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid
trn2 $res0.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid
pmull2 $t2.1q, $res3.2d, $h5.2d @ GHASH block 8k+3 - high
aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 1
aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 1
aese $ctr5b, $rk1 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 1
ldr $h56kq, [$current_tag, #144] @ load h6k | h5k
ldr $h78kq, [$current_tag, #192] @ load h8k | h7k
aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 2
eor $res0.16b, $res0.16b, $acc_m.16b @ GHASH block 8k, 8k+1 - mid
aese $ctr6b, $rk2 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 2
rev64 $res5b, $res5b @ GHASH block 8k+5
pmull $h5.1q, $res3.1d, $h5.1d @ GHASH block 8k+3 - low
eor3 $acc_hb, $acc_hb, $t1.16b, $t2.16b @ GHASH block 8k+2, 8k+3 - high
aese $ctr4b, $rk2 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 2
aese $ctr5b, $rk2 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 2
trn1 $t3.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid
aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 3
aese $ctr7b, $rk2 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 2
aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 2
aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 2
trn2 $res2.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid
pmull2 $acc_m.1q, $res0.2d, $h78k.2d @ GHASH block 8k - mid
aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 2
pmull $h78k.1q, $res0.1d, $h78k.1d @ GHASH block 8k+1 - mid
aese $ctr5b, $rk3 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 3
eor $res2.16b, $res2.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid
eor $acc_lb, $acc_lb, $h7.16b @ GHASH block 8k+1 - low
aese $ctr7b, $rk3 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 3
aese $ctr6b, $rk3 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 3
aese $ctr4b, $rk3 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 3
eor3 $acc_lb, $acc_lb, $h6.16b, $h5.16b @ GHASH block 8k+2, 8k+3 - low
ldp $rk4q, $rk5q, [$cc, #64] @ load rk4, rk5
aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 3
ldr $h3q, [$current_tag, #80] @ load h3l | h3h
ext $h3.16b, $h3.16b, $h3.16b, #8
ldr $h4q, [$current_tag, #112] @ load h4l | h4h
ext $h4.16b, $h4.16b, $h4.16b, #8
pmull2 $t3.1q, $res2.2d, $h56k.2d @ GHASH block 8k+2 - mid
pmull $h56k.1q, $res2.1d, $h56k.1d @ GHASH block 8k+3 - mid
ldr $h1q, [$current_tag, #32] @ load h1l | h1h
ext $h1.16b, $h1.16b, $h1.16b, #8
ldr $h2q, [$current_tag, #64] @ load h2l | h2h
ext $h2.16b, $h2.16b, $h2.16b, #8
eor $acc_mb, $acc_mb, $h78k.16b @ GHASH block 8k+1 - mid
aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 3
rev64 $res7b, $res7b @ GHASH block 8k+7
eor3 $acc_mb, $acc_mb, $h56k.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid
rev64 $res4b, $res4b @ GHASH block 8k+4
aese $ctr5b, $rk4 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 4
aese $ctr4b, $rk4 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 4
aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 3
aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 4
aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 4
aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 4
aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 4
aese $ctr6b, $rk4 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 4
aese $ctr7b, $rk4 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 4
rev64 $res6b, $res6b @ GHASH block 8k+6
ldr $h12kq, [$current_tag, #48] @ load h2k | h1k
ldr $h34kq, [$current_tag, #96] @ load h4k | h3k
trn1 $t6.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid
aese $ctr7b, $rk5 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 5
aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 5
aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 5
ldp $rk6q, $rk7q, [$cc, #96] @ load rk6, rk7
aese $ctr6b, $rk5 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 5
aese $ctr5b, $rk5 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 5
pmull2 $t7.1q, $res6.2d, $h2.2d @ GHASH block 8k+6 - high
pmull2 $t4.1q, $res4.2d, $h4.2d @ GHASH block 8k+4 - high
pmull $h2.1q, $res6.1d, $h2.1d @ GHASH block 8k+6 - low
aese $ctr4b, $rk5 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 5
pmull $h4.1q, $res4.1d, $h4.1d @ GHASH block 8k+4 - low
trn2 $res4.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid
pmull2 $t5.1q, $res5.2d, $h3.2d @ GHASH block 8k+5 - high
pmull $h3.1q, $res5.1d, $h3.1d @ GHASH block 8k+5 - low
trn1 $t9.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid
aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 5
trn2 $res6.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid
aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 5
eor $res4.16b, $res4.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid
aese $ctr4b, $rk6 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 6
aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 6
eor $res6.16b, $res6.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid
aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 6
aese $ctr7b, $rk6 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 6
pmull2 $t6.1q, $res4.2d, $h34k.2d @ GHASH block 8k+4 - mid
pmull $h34k.1q, $res4.1d, $h34k.1d @ GHASH block 8k+5 - mid
aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 6
pmull2 $t9.1q, $res6.2d, $h12k.2d @ GHASH block 8k+6 - mid
aese $ctr5b, $rk6 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 6
pmull2 $t8.1q, $res7.2d, $h1.2d @ GHASH block 8k+7 - high
eor3 $acc_mb, $acc_mb, $h34k.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid
aese $ctr4b, $rk7 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 7
eor3 $acc_lb, $acc_lb, $h4.16b, $h3.16b @ GHASH block 8k+4, 8k+5 - low
aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 6
aese $ctr6b, $rk6 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 6
aese $ctr5b, $rk7 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 7
ldp $rk8q, $rk9q, [$cc, #128] @ load rk8, rk9
pmull $h12k.1q, $res6.1d, $h12k.1d @ GHASH block 8k+7 - mid
aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 7
ldr $mod_constantd, [$modulo_constant] @ MODULO - load modulo constant
eor3 $acc_hb, $acc_hb, $t4.16b, $t5.16b @ GHASH block 8k+4, 8k+5 - high
pmull $h1.1q, $res7.1d, $h1.1d @ GHASH block 8k+7 - low
aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 7
aese $ctr7b, $rk7 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 7
aese $ctr6b, $rk7 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 7
eor3 $acc_hb, $acc_hb, $t7.16b, $t8.16b @ GHASH block 8k+6, 8k+7 - high
eor3 $acc_lb, $acc_lb, $h2.16b, $h1.16b @ GHASH block 8k+6, 8k+7 - low
eor3 $acc_mb, $acc_mb, $h12k.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid
aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 7
aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 7
eor3 $acc_mb, $acc_mb, $acc_hb, $acc_lb @ MODULO - karatsuba tidy up
ext $t11.16b, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment
aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 8
aese $ctr6b, $rk8 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 8
aese $ctr7b, $rk8 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 8
aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 8
aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 8
pmull $t12.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid
aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 8
aese $ctr5b, $rk8 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 8
aese $ctr4b, $rk8 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 8
ldp $rk10q, $rk11q, [$cc, #160] @ load rk10, rk11
eor3 $acc_mb, $acc_mb, $t12.16b, $t11.16b @ MODULO - fold into mid
aese $ctr7b, $rk9 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 9
aese $ctr6b, $rk9 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 9
aese $ctr5b, $rk9 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 9
aese $ctr2b, $rk9 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 9
aese $ctr3b, $rk9 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 9
aese $ctr0b, $rk9 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 9
aese $ctr1b, $rk9 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 9
aese $ctr4b, $rk9 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 9
pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low
ldr $rk12q, [$cc, #192] @ load rk12
ext $t11.16b, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment
aese $ctr2b, $rk10 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 10
aese $ctr5b, $rk10 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 10
aese $ctr0b, $rk10 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 10
aese $ctr4b, $rk10 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 10
aese $ctr6b, $rk10 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 10
aese $ctr7b, $rk10 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 10
aese $ctr0b, $rk11 @ AES block 8k+8 - round 11
eor3 $acc_lb, $acc_lb, $acc_hb, $t11.16b @ MODULO - fold into low
aese $ctr5b, $rk11 @ AES block 8k+13 - round 11
aese $ctr2b, $rk11 @ AES block 8k+10 - round 11
aese $ctr3b, $rk10 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 10
aese $ctr1b, $rk10 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 10
aese $ctr6b, $rk11 @ AES block 8k+14 - round 11
aese $ctr4b, $rk11 @ AES block 8k+12 - round 11
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+15
aese $ctr3b, $rk11 @ AES block 8k+11 - round 11
aese $ctr1b, $rk11 @ AES block 8k+9 - round 11
aese $ctr7b, $rk11 @ AES block 8k+15 - round 11
.L192_dec_tail: @ TAIL
sub $main_end_input_ptr, $end_input_ptr, $input_ptr @ main_end_input_ptr is number of bytes left to process
ldp $h5q, $h56kq, [$current_tag, #128] @ load h5l | h5h
ext $h5.16b, $h5.16b, $h5.16b, #8
ldr $res1q, [$input_ptr], #16 @ AES block 8k+8 - load ciphertext
ldp $h78kq, $h8q, [$current_tag, #192] @ load h8k | h7k
ext $h8.16b, $h8.16b, $h8.16b, #8
mov $t1.16b, $rk12
ldp $h6q, $h7q, [$current_tag, #160] @ load h6l | h6h
ext $h6.16b, $h6.16b, $h6.16b, #8
ext $h7.16b, $h7.16b, $h7.16b, #8
ext $t0.16b, $acc_lb, $acc_lb, #8 @ prepare final partial tag
eor3 $res4b, $res1b, $ctr0b, $t1.16b @ AES block 8k+8 - result
cmp $main_end_input_ptr, #112
b.gt .L192_dec_blocks_more_than_7
mov $ctr7b, $ctr6b
movi $acc_h.8b, #0
sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
mov $ctr6b, $ctr5b
mov $ctr5b, $ctr4b
mov $ctr4b, $ctr3b
cmp $main_end_input_ptr, #96
movi $acc_l.8b, #0
mov $ctr3b, $ctr2b
mov $ctr2b, $ctr1b
movi $acc_m.8b, #0
b.gt .L192_dec_blocks_more_than_6
mov $ctr7b, $ctr6b
mov $ctr6b, $ctr5b
mov $ctr5b, $ctr4b
mov $ctr4b, $ctr3b
mov $ctr3b, $ctr1b
sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
cmp $main_end_input_ptr, #80
b.gt .L192_dec_blocks_more_than_5
mov $ctr7b, $ctr6b
mov $ctr6b, $ctr5b
mov $ctr5b, $ctr4b
mov $ctr4b, $ctr1b
cmp $main_end_input_ptr, #64
sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
b.gt .L192_dec_blocks_more_than_4
sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
mov $ctr7b, $ctr6b
mov $ctr6b, $ctr5b
mov $ctr5b, $ctr1b
cmp $main_end_input_ptr, #48
b.gt .L192_dec_blocks_more_than_3
sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
mov $ctr7b, $ctr6b
cmp $main_end_input_ptr, #32
mov $ctr6b, $ctr1b
ldr $h34kq, [$current_tag, #96] @ load h4k | h3k
b.gt .L192_dec_blocks_more_than_2
sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
mov $ctr7b, $ctr1b
cmp $main_end_input_ptr, #16
b.gt .L192_dec_blocks_more_than_1
sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
ldr $h12kq, [$current_tag, #48] @ load h2k | h1k
b .L192_dec_blocks_less_than_1
.L192_dec_blocks_more_than_7: @ blocks left > 7
rev64 $res0b, $res1b @ GHASH final-7 block
ins $acc_m.d[0], $h78k.d[1] @ GHASH final-7 block - mid
eor $res0b, $res0b, $t0.16b @ feed in partial tag
pmull2 $acc_h.1q, $res0.2d, $h8.2d @ GHASH final-7 block - high
ins $rk4v.d[0], $res0.d[1] @ GHASH final-7 block - mid
ldr $res1q, [$input_ptr], #16 @ AES final-6 block - load ciphertext
pmull $acc_l.1q, $res0.1d, $h8.1d @ GHASH final-7 block - low
eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-7 block - mid
st1 { $res4b}, [$output_ptr], #16 @ AES final-7 block - store result
eor3 $res4b, $res1b, $ctr1b, $t1.16b @ AES final-6 block - result
pmull $acc_m.1q, $rk4v.1d, $acc_m.1d @ GHASH final-7 block - mid
movi $t0.8b, #0 @ supress further partial tag feed in
.L192_dec_blocks_more_than_6: @ blocks left > 6
rev64 $res0b, $res1b @ GHASH final-6 block
eor $res0b, $res0b, $t0.16b @ feed in partial tag
ldr $res1q, [$input_ptr], #16 @ AES final-5 block - load ciphertext
ins $rk4v.d[0], $res0.d[1] @ GHASH final-6 block - mid
eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-6 block - mid
movi $t0.8b, #0 @ supress further partial tag feed in
pmull2 $rk2q1, $res0.2d, $h7.2d @ GHASH final-6 block - high
st1 { $res4b}, [$output_ptr], #16 @ AES final-6 block - store result
eor3 $res4b, $res1b, $ctr2b, $t1.16b @ AES final-5 block - result
eor $acc_hb, $acc_hb, $rk2 @ GHASH final-6 block - high
pmull $rk4v.1q, $rk4v.1d, $h78k.1d @ GHASH final-6 block - mid
pmull $rk3q1, $res0.1d, $h7.1d @ GHASH final-6 block - low
eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-6 block - mid
eor $acc_lb, $acc_lb, $rk3 @ GHASH final-6 block - low
.L192_dec_blocks_more_than_5: @ blocks left > 5
rev64 $res0b, $res1b @ GHASH final-5 block
eor $res0b, $res0b, $t0.16b @ feed in partial tag
ins $rk4v.d[0], $res0.d[1] @ GHASH final-5 block - mid
eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-5 block - mid
ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-5 block - mid
pmull2 $rk2q1, $res0.2d, $h6.2d @ GHASH final-5 block - high
ldr $res1q, [$input_ptr], #16 @ AES final-4 block - load ciphertext
eor $acc_hb, $acc_hb, $rk2 @ GHASH final-5 block - high
pmull $rk3q1, $res0.1d, $h6.1d @ GHASH final-5 block - low
pmull2 $rk4v.1q, $rk4v.2d, $h56k.2d @ GHASH final-5 block - mid
eor $acc_lb, $acc_lb, $rk3 @ GHASH final-5 block - low
movi $t0.8b, #0 @ supress further partial tag feed in
st1 { $res4b}, [$output_ptr], #16 @ AES final-5 block - store result
eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-5 block - mid
eor3 $res4b, $res1b, $ctr3b, $t1.16b @ AES final-4 block - result
.L192_dec_blocks_more_than_4: @ blocks left > 4
rev64 $res0b, $res1b @ GHASH final-4 block
eor $res0b, $res0b, $t0.16b @ feed in partial tag
movi $t0.8b, #0 @ supress further partial tag feed in
ldr $res1q, [$input_ptr], #16 @ AES final-3 block - load ciphertext
ins $rk4v.d[0], $res0.d[1] @ GHASH final-4 block - mid
pmull $rk3q1, $res0.1d, $h5.1d @ GHASH final-4 block - low
eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-4 block - mid
eor $acc_lb, $acc_lb, $rk3 @ GHASH final-4 block - low
pmull $rk4v.1q, $rk4v.1d, $h56k.1d @ GHASH final-4 block - mid
st1 { $res4b}, [$output_ptr], #16 @ AES final-4 block - store result
pmull2 $rk2q1, $res0.2d, $h5.2d @ GHASH final-4 block - high
eor3 $res4b, $res1b, $ctr4b, $t1.16b @ AES final-3 block - result
eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-4 block - mid
eor $acc_hb, $acc_hb, $rk2 @ GHASH final-4 block - high
.L192_dec_blocks_more_than_3: @ blocks left > 3
ldr $h4q, [$current_tag, #112] @ load h4l | h4h
ext $h4.16b, $h4.16b, $h4.16b, #8
rev64 $res0b, $res1b @ GHASH final-3 block
ldr $res1q, [$input_ptr], #16 @ AES final-2 block - load ciphertext
eor $res0b, $res0b, $t0.16b @ feed in partial tag
ins $rk4v.d[0], $res0.d[1] @ GHASH final-3 block - mid
pmull2 $rk2q1, $res0.2d, $h4.2d @ GHASH final-3 block - high
eor $acc_hb, $acc_hb, $rk2 @ GHASH final-3 block - high
movi $t0.8b, #0 @ supress further partial tag feed in
pmull $rk3q1, $res0.1d, $h4.1d @ GHASH final-3 block - low
st1 { $res4b}, [$output_ptr], #16 @ AES final-3 block - store result
eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-3 block - mid
eor3 $res4b, $res1b, $ctr5b, $t1.16b @ AES final-2 block - result
eor $acc_lb, $acc_lb, $rk3 @ GHASH final-3 block - low
ldr $h34kq, [$current_tag, #96] @ load h4k | h3k
ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-3 block - mid
pmull2 $rk4v.1q, $rk4v.2d, $h34k.2d @ GHASH final-3 block - mid
eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-3 block - mid
.L192_dec_blocks_more_than_2: @ blocks left > 2
rev64 $res0b, $res1b @ GHASH final-2 block
ldr $h3q, [$current_tag, #80] @ load h3l | h3h
ext $h3.16b, $h3.16b, $h3.16b, #8
eor $res0b, $res0b, $t0.16b @ feed in partial tag
ins $rk4v.d[0], $res0.d[1] @ GHASH final-2 block - mid
ldr $res1q, [$input_ptr], #16 @ AES final-1 block - load ciphertext
pmull2 $rk2q1, $res0.2d, $h3.2d @ GHASH final-2 block - high
eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-2 block - mid
eor $acc_hb, $acc_hb, $rk2 @ GHASH final-2 block - high
pmull $rk3q1, $res0.1d, $h3.1d @ GHASH final-2 block - low
pmull $rk4v.1q, $rk4v.1d, $h34k.1d @ GHASH final-2 block - mid
movi $t0.8b, #0 @ supress further partial tag feed in
eor $acc_lb, $acc_lb, $rk3 @ GHASH final-2 block - low
st1 { $res4b}, [$output_ptr], #16 @ AES final-2 block - store result
eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-2 block - mid
eor3 $res4b, $res1b, $ctr6b, $t1.16b @ AES final-1 block - result
.L192_dec_blocks_more_than_1: @ blocks left > 1
rev64 $res0b, $res1b @ GHASH final-1 block
ldr $res1q, [$input_ptr], #16 @ AES final block - load ciphertext
ldr $h2q, [$current_tag, #64] @ load h1l | h1h
ext $h2.16b, $h2.16b, $h2.16b, #8
eor $res0b, $res0b, $t0.16b @ feed in partial tag
movi $t0.8b, #0 @ supress further partial tag feed in
ldr $h12kq, [$current_tag, #48] @ load h2k | h1k
pmull $rk3q1, $res0.1d, $h2.1d @ GHASH final-1 block - low
ins $rk4v.d[0], $res0.d[1] @ GHASH final-1 block - mid
st1 { $res4b}, [$output_ptr], #16 @ AES final-1 block - store result
pmull2 $rk2q1, $res0.2d, $h2.2d @ GHASH final-1 block - high
eor3 $res4b, $res1b, $ctr7b, $t1.16b @ AES final block - result
eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-1 block - mid
ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-1 block - mid
pmull2 $rk4v.1q, $rk4v.2d, $h12k.2d @ GHASH final-1 block - mid
eor $acc_lb, $acc_lb, $rk3 @ GHASH final-1 block - low
eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-1 block - mid
eor $acc_hb, $acc_hb, $rk2 @ GHASH final-1 block - high
.L192_dec_blocks_less_than_1: @ blocks left <= 1
rev32 $rtmp_ctr.16b, $rtmp_ctr.16b
and $bit_length, $bit_length, #127 @ bit_length %= 128
sub $bit_length, $bit_length, #128 @ bit_length -= 128
str $rtmp_ctrq, [$counter] @ store the updated counter
neg $bit_length, $bit_length @ bit_length = 128 - #bits in input (in range [1,128])
mvn $temp0_x, xzr @ temp0_x = 0xffffffffffffffff
and $bit_length, $bit_length, #127 @ bit_length %= 128
mvn $temp1_x, xzr @ temp1_x = 0xffffffffffffffff
lsr $temp0_x, $temp0_x, $bit_length @ temp0_x is mask for top 64b of last block
cmp $bit_length, #64
csel $temp2_x, $temp1_x, $temp0_x, lt
csel $temp3_x, $temp0_x, xzr, lt
ldr $h1q, [$current_tag, #32] @ load h1l | h1h
ext $h1.16b, $h1.16b, $h1.16b, #8
mov $ctr0.d[1], $temp3_x
ld1 { $rk0}, [$output_ptr] @ load existing bytes where the possibly partial last block is to be stored
mov $ctr0.d[0], $temp2_x @ ctr0b is mask for last block
and $res1b, $res1b, $ctr0b @ possibly partial last block has zeroes in highest bits
bif $res4b, $rk0, $ctr0b @ insert existing bytes in top end of result before storing
rev64 $res0b, $res1b @ GHASH final block
st1 { $res4b}, [$output_ptr] @ store all 16B
eor $res0b, $res0b, $t0.16b @ feed in partial tag
ins $t0.d[0], $res0.d[1] @ GHASH final block - mid
pmull $rk3q1, $res0.1d, $h1.1d @ GHASH final block - low
eor $t0.8b, $t0.8b, $res0.8b @ GHASH final block - mid
pmull2 $rk2q1, $res0.2d, $h1.2d @ GHASH final block - high
eor $acc_lb, $acc_lb, $rk3 @ GHASH final block - low
pmull $t0.1q, $t0.1d, $h12k.1d @ GHASH final block - mid
eor $acc_hb, $acc_hb, $rk2 @ GHASH final block - high
eor $t10.16b, $acc_hb, $acc_lb @ MODULO - karatsuba tidy up
eor $acc_mb, $acc_mb, $t0.16b @ GHASH final block - mid
ldr $mod_constantd, [$modulo_constant] @ MODULO - load modulo constant
pmull $t11.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid
ext $acc_hb, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment
eor $acc_mb, $acc_mb, $t10.16b @ MODULO - karatsuba tidy up
eor3 $acc_mb, $acc_mb, $acc_hb, $t11.16b @ MODULO - fold into mid
pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low
ext $acc_mb, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment
eor3 $acc_lb, $acc_lb, $acc_mb, $acc_hb @ MODULO - fold into low
ext $acc_lb, $acc_lb, $acc_lb, #8
rev64 $acc_lb, $acc_lb
st1 { $acc_l.16b }, [$current_tag]
ldp d10, d11, [sp, #16]
ldp d12, d13, [sp, #32]
ldp d14, d15, [sp, #48]
ldp d8, d9, [sp], #80
ret
.L192_dec_ret:
mov w0, #0x0
ret
.size unroll8_eor3_aes_gcm_dec_192_kernel,.-unroll8_eor3_aes_gcm_dec_192_kernel
___
}
{
my ($end_input_ptr,$main_end_input_ptr,$temp0_x,$temp1_x)=map("x$_",(4..7));
my ($temp2_x,$temp3_x)=map("x$_",(13..14));
my ($ctr0b,$ctr1b,$ctr2b,$ctr3b,$ctr4b,$ctr5b,$ctr6b,$ctr7b,$res0b,$res1b,$res2b,$res3b,$res4b,$res5b,$res6b,$res7b)=map("v$_.16b",(0..15));
my ($ctr0,$ctr1,$ctr2,$ctr3,$ctr4,$ctr5,$ctr6,$ctr7,$res0,$res1,$res2,$res3,$res4,$res5,$res6,$res7)=map("v$_",(0..15));
my ($ctr0d,$ctr1d,$ctr2d,$ctr3d,$ctr4d,$ctr5d,$ctr6d,$ctr7d)=map("d$_",(0..7));
my ($ctr0q,$ctr1q,$ctr2q,$ctr3q,$ctr4q,$ctr5q,$ctr6q,$ctr7q)=map("q$_",(0..7));
my ($res0q,$res1q,$res2q,$res3q,$res4q,$res5q,$res6q,$res7q)=map("q$_",(8..15));
my ($ctr_t0,$ctr_t1,$ctr_t2,$ctr_t3,$ctr_t4,$ctr_t5,$ctr_t6,$ctr_t7)=map("v$_",(8..15));
my ($ctr_t0b,$ctr_t1b,$ctr_t2b,$ctr_t3b,$ctr_t4b,$ctr_t5b,$ctr_t6b,$ctr_t7b)=map("v$_.16b",(8..15));
my ($ctr_t0q,$ctr_t1q,$ctr_t2q,$ctr_t3q,$ctr_t4q,$ctr_t5q,$ctr_t6q,$ctr_t7q)=map("q$_",(8..15));
my ($acc_hb,$acc_mb,$acc_lb)=map("v$_.16b",(17..19));
my ($acc_h,$acc_m,$acc_l)=map("v$_",(17..19));
my ($h1,$h12k,$h2,$h3,$h34k,$h4)=map("v$_",(20..25));
my ($h5,$h56k,$h6,$h7,$h78k,$h8)=map("v$_",(20..25));
my ($h1q,$h12kq,$h2q,$h3q,$h34kq,$h4q)=map("q$_",(20..25));
my ($h5q,$h56kq,$h6q,$h7q,$h78kq,$h8q)=map("q$_",(20..25));
my $t0="v16";
my $t0d="d16";
my $t1="v29";
my $t2=$res1;
my $t3=$t1;
my $t4=$res0;
my $t5=$res2;
my $t6=$t0;
my $t7=$res3;
my $t8=$res4;
my $t9=$res5;
my $t10=$res6;
my $t11="v21";
my $t12=$t1;
my $rtmp_ctr="v30";
my $rtmp_ctrq="q30";
my $rctr_inc="v31";
my $rctr_incd="d31";
my $mod_constantd=$t0d;
my $mod_constant=$t0;
my ($rk0,$rk1,$rk2)=map("v$_.16b",(26..28));
my ($rk3,$rk4,$rk5)=map("v$_.16b",(26..28));
my ($rk6,$rk7,$rk8)=map("v$_.16b",(26..28));
my ($rk9,$rk10,$rk11)=map("v$_.16b",(26..28));
my ($rk12,$rk13,$rk14)=map("v$_.16b",(26..28));
my ($rk0q,$rk1q,$rk2q)=map("q$_",(26..28));
my ($rk3q,$rk4q,$rk5q)=map("q$_",(26..28));
my ($rk6q,$rk7q,$rk8q)=map("q$_",(26..28));
my ($rk9q,$rk10q,$rk11q)=map("q$_",(26..28));
my ($rk12q,$rk13q,$rk14q)=map("q$_",(26..28));
my $rk2q1="v28.1q";
my $rk3q1="v26.1q";
my $rk4v="v27";
#########################################################################################
# size_t unroll8_eor3_aes_gcm_enc_256_kernel(const unsigned char *in,
# size_t len,
# unsigned char *out,
# const void *key,
# unsigned char ivec[16],
# u64 *Xi);
#
$code.=<<___;
.global unroll8_eor3_aes_gcm_enc_256_kernel
.type unroll8_eor3_aes_gcm_enc_256_kernel,%function
.align 4
unroll8_eor3_aes_gcm_enc_256_kernel:
AARCH64_VALID_CALL_TARGET
cbz x1, .L256_enc_ret
stp d8, d9, [sp, #-80]!
mov $counter, x4
mov $cc, x5
stp d10, d11, [sp, #16]
stp d12, d13, [sp, #32]
stp d14, d15, [sp, #48]
mov x5, #0xc200000000000000
stp x5, xzr, [sp, #64]
add $modulo_constant, sp, #64
ld1 { $ctr0b}, [$counter] @ CTR block 0
lsr $main_end_input_ptr, $bit_length, #3 @ byte_len
mov $constant_temp, #0x100000000 @ set up counter increment
movi $rctr_inc.16b, #0x0
mov $rctr_inc.d[1], $constant_temp
sub $main_end_input_ptr, $main_end_input_ptr, #1 @ byte_len - 1
and $main_end_input_ptr, $main_end_input_ptr, #0xffffffffffffff80 @ number of bytes to be processed in main loop (at least 1 byte must be handled by tail)
add $main_end_input_ptr, $main_end_input_ptr, $input_ptr
rev32 $rtmp_ctr.16b, $ctr0.16b @ set up reversed counter
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 0
rev32 $ctr1.16b, $rtmp_ctr.16b @ CTR block 1
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 1
rev32 $ctr2.16b, $rtmp_ctr.16b @ CTR block 2
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 2
rev32 $ctr3.16b, $rtmp_ctr.16b @ CTR block 3
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 3
rev32 $ctr4.16b, $rtmp_ctr.16b @ CTR block 4
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 4
rev32 $ctr5.16b, $rtmp_ctr.16b @ CTR block 5
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 5
ldp $rk0q, $rk1q, [$cc, #0] @ load rk0, rk1
rev32 $ctr6.16b, $rtmp_ctr.16b @ CTR block 6
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 6
rev32 $ctr7.16b, $rtmp_ctr.16b @ CTR block 7
aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 0
aese $ctr4b, $rk0 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 0
aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 0
aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 0
aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 0
aese $ctr6b, $rk0 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 0
aese $ctr5b, $rk0 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 0
aese $ctr7b, $rk0 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 0
ldp $rk2q, $rk3q, [$cc, #32] @ load rk2, rk3
aese $ctr4b, $rk1 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 1
aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 1
aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 1
aese $ctr6b, $rk1 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 1
aese $ctr5b, $rk1 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 1
aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 1
aese $ctr7b, $rk1 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 1
aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 2
aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 2
aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 1
aese $ctr7b, $rk2 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 2
aese $ctr6b, $rk2 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 2
aese $ctr5b, $rk2 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 2
aese $ctr4b, $rk2 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 2
aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 2
aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 2
aese $ctr5b, $rk3 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 3
aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 3
ldp $rk4q, $rk5q, [$cc, #64] @ load rk4, rk5
aese $ctr4b, $rk3 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 3
aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 3
aese $ctr6b, $rk3 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 3
aese $ctr7b, $rk3 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 3
aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 3
aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 3
aese $ctr4b, $rk4 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 4
aese $ctr6b, $rk4 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 4
aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 4
aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 4
aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 4
aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 4
aese $ctr7b, $rk4 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 4
aese $ctr5b, $rk4 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 4
aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 5
aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 5
ldp $rk6q, $rk7q, [$cc, #96] @ load rk6, rk7
aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 5
aese $ctr4b, $rk5 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 5
aese $ctr5b, $rk5 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 5
aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 5
aese $ctr6b, $rk5 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 5
aese $ctr7b, $rk5 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 5
aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 6
aese $ctr5b, $rk6 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 6
aese $ctr4b, $rk6 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 6
aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 6
aese $ctr6b, $rk6 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 6
aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 6
aese $ctr7b, $rk6 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 6
aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 6
ldp $rk8q, $rk9q, [$cc, #128] @ load rk8, rk9
aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 7
aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 7
aese $ctr7b, $rk7 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 7
aese $ctr6b, $rk7 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 7
aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 7
aese $ctr5b, $rk7 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 7
aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 7
aese $ctr4b, $rk7 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 7
aese $ctr6b, $rk8 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 8
aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 8
aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 8
aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 8
aese $ctr7b, $rk8 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 8
aese $ctr5b, $rk8 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 8
aese $ctr4b, $rk8 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 8
aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 8
ld1 { $acc_lb}, [$current_tag]
ext $acc_lb, $acc_lb, $acc_lb, #8
rev64 $acc_lb, $acc_lb
ldp $rk10q, $rk11q, [$cc, #160] @ load rk10, rk11
aese $ctr6b, $rk9 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 9
aese $ctr7b, $rk9 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 9
aese $ctr3b, $rk9 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 9
aese $ctr4b, $rk9 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 9
aese $ctr5b, $rk9 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 9
aese $ctr2b, $rk9 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 9
aese $ctr1b, $rk9 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 9
aese $ctr7b, $rk10 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 10
aese $ctr4b, $rk10 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 10
aese $ctr0b, $rk9 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 9
aese $ctr1b, $rk10 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 10
aese $ctr5b, $rk10 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 10
aese $ctr3b, $rk10 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 10
aese $ctr2b, $rk10 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 10
aese $ctr0b, $rk10 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 10
aese $ctr6b, $rk10 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 10
aese $ctr4b, $rk11 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 11
ldp $rk12q, $rk13q, [$cc, #192] @ load rk12, rk13
aese $ctr5b, $rk11 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 11
aese $ctr2b, $rk11 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 11
aese $ctr6b, $rk11 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 11
aese $ctr1b, $rk11 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 11
aese $ctr0b, $rk11 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 11
aese $ctr3b, $rk11 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 11
aese $ctr7b, $rk11 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 11
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 7
ldr $rk14q, [$cc, #224] @ load rk14
aese $ctr4b, $rk12 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 12
aese $ctr2b, $rk12 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 12
aese $ctr1b, $rk12 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 12
aese $ctr0b, $rk12 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 12
aese $ctr5b, $rk12 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 12
aese $ctr3b, $rk12 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 12
aese $ctr2b, $rk13 @ AES block 2 - round 13
aese $ctr1b, $rk13 @ AES block 1 - round 13
aese $ctr4b, $rk13 @ AES block 4 - round 13
aese $ctr6b, $rk12 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 12
aese $ctr7b, $rk12 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 12
aese $ctr0b, $rk13 @ AES block 0 - round 13
aese $ctr5b, $rk13 @ AES block 5 - round 13
aese $ctr6b, $rk13 @ AES block 6 - round 13
aese $ctr7b, $rk13 @ AES block 7 - round 13
aese $ctr3b, $rk13 @ AES block 3 - round 13
add $end_input_ptr, $input_ptr, $bit_length, lsr #3 @ end_input_ptr
cmp $input_ptr, $main_end_input_ptr @ check if we have <= 8 blocks
b.ge .L256_enc_tail @ handle tail
ldp $ctr_t0q, $ctr_t1q, [$input_ptr], #32 @ AES block 0, 1 - load plaintext
ldp $ctr_t2q, $ctr_t3q, [$input_ptr], #32 @ AES block 2, 3 - load plaintext
eor3 $res0b, $ctr_t0b, $ctr0b, $rk14 @ AES block 0 - result
rev32 $ctr0.16b, $rtmp_ctr.16b @ CTR block 8
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8
eor3 $res1b, $ctr_t1b, $ctr1b, $rk14 @ AES block 1 - result
eor3 $res3b, $ctr_t3b, $ctr3b, $rk14 @ AES block 3 - result
rev32 $ctr1.16b, $rtmp_ctr.16b @ CTR block 9
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 9
ldp $ctr_t4q, $ctr_t5q, [$input_ptr], #32 @ AES block 4, 5 - load plaintext
ldp $ctr_t6q, $ctr_t7q, [$input_ptr], #32 @ AES block 6, 7 - load plaintext
eor3 $res2b, $ctr_t2b, $ctr2b, $rk14 @ AES block 2 - result
cmp $input_ptr, $main_end_input_ptr @ check if we have <= 8 blocks
rev32 $ctr2.16b, $rtmp_ctr.16b @ CTR block 10
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 10
stp $res0q, $res1q, [$output_ptr], #32 @ AES block 0, 1 - store result
stp $res2q, $res3q, [$output_ptr], #32 @ AES block 2, 3 - store result
rev32 $ctr3.16b, $rtmp_ctr.16b @ CTR block 11
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 11
eor3 $res4b, $ctr_t4b, $ctr4b, $rk14 @ AES block 4 - result
eor3 $res7b, $ctr_t7b, $ctr7b, $rk14 @ AES block 7 - result
eor3 $res6b, $ctr_t6b, $ctr6b, $rk14 @ AES block 6 - result
eor3 $res5b, $ctr_t5b, $ctr5b, $rk14 @ AES block 5 - result
stp $res4q, $res5q, [$output_ptr], #32 @ AES block 4, 5 - store result
rev32 $ctr4.16b, $rtmp_ctr.16b @ CTR block 12
stp $res6q, $res7q, [$output_ptr], #32 @ AES block 6, 7 - store result
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 12
b.ge .L256_enc_prepretail @ do prepretail
.L256_enc_main_loop: @ main loop start
ldp $rk0q, $rk1q, [$cc, #0] @ load rk0, rk1
rev32 $ctr5.16b, $rtmp_ctr.16b @ CTR block 8k+13
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+13
ldr $h56kq, [$current_tag, #144] @ load h6k | h5k
ldr $h78kq, [$current_tag, #192] @ load h8k | h7k
rev64 $res3b, $res3b @ GHASH block 8k+3
ldr $h5q, [$current_tag, #128] @ load h5l | h5h
ext $h5.16b, $h5.16b, $h5.16b, #8
ldr $h6q, [$current_tag, #160] @ load h6l | h6h
ext $h6.16b, $h6.16b, $h6.16b, #8
rev64 $res1b, $res1b @ GHASH block 8k+1
rev32 $ctr6.16b, $rtmp_ctr.16b @ CTR block 8k+14
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+14
rev64 $res0b, $res0b @ GHASH block 8k
rev64 $res4b, $res4b @ GHASH block 8k+4
ext $acc_lb, $acc_lb, $acc_lb, #8 @ PRE 0
ldr $h7q, [$current_tag, #176] @ load h7l | h7h
ext $h7.16b, $h7.16b, $h7.16b, #8
ldr $h8q, [$current_tag, #208] @ load h8l | h8h
ext $h8.16b, $h8.16b, $h8.16b, #8
aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 0
aese $ctr5b, $rk0 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 0
rev32 $ctr7.16b, $rtmp_ctr.16b @ CTR block 8k+15
aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 0
aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 0
aese $ctr6b, $rk0 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 0
aese $ctr7b, $rk0 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 0
aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 0
aese $ctr4b, $rk0 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 0
ldp $rk2q, $rk3q, [$cc, #32] @ load rk2, rk3
eor $res0b, $res0b, $acc_lb @ PRE 1
aese $ctr6b, $rk1 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 1
aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 1
aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 1
aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 1
aese $ctr4b, $rk1 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 1
aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 1
aese $ctr5b, $rk1 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 1
pmull2 $acc_h.1q, $res0.2d, $h8.2d @ GHASH block 8k - high
pmull $acc_l.1q, $res0.1d, $h8.1d @ GHASH block 8k - low
pmull2 $t0.1q, $res1.2d, $h7.2d @ GHASH block 8k+1 - high
trn1 $acc_m.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid
trn2 $res0.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid
aese $ctr7b, $rk1 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 1
aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 2
aese $ctr5b, $rk2 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 2
aese $ctr6b, $rk2 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 2
aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 2
pmull $h7.1q, $res1.1d, $h7.1d @ GHASH block 8k+1 - low
aese $ctr4b, $rk2 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 2
aese $ctr5b, $rk3 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 3
aese $ctr6b, $rk3 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 3
aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 2
aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 3
aese $ctr7b, $rk2 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 2
aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 2
aese $ctr4b, $rk3 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 3
rev64 $res6b, $res6b @ GHASH block 8k+6
pmull2 $t2.1q, $res3.2d, $h5.2d @ GHASH block 8k+3 - high
aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 3
ldp $rk4q, $rk5q, [$cc, #64] @ load rk4, rk5
rev64 $res2b, $res2b @ GHASH block 8k+2
aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 3
aese $ctr7b, $rk3 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 3
aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 3
eor $acc_hb, $acc_hb, $t0.16b @ GHASH block 8k+1 - high
pmull2 $t1.1q, $res2.2d, $h6.2d @ GHASH block 8k+2 - high
rev64 $res5b, $res5b @ GHASH block 8k+5
pmull $h5.1q, $res3.1d, $h5.1d @ GHASH block 8k+3 - low
eor $acc_lb, $acc_lb, $h7.16b @ GHASH block 8k+1 - low
ldr $h3q, [$current_tag, #80] @ load h3l | h3h
ext $h3.16b, $h3.16b, $h3.16b, #8
ldr $h4q, [$current_tag, #112] @ load h4l | h4h
ext $h4.16b, $h4.16b, $h4.16b, #8
trn1 $t6.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid
eor3 $acc_hb, $acc_hb, $t1.16b, $t2.16b @ GHASH block 8k+2, 8k+3 - high
pmull $h6.1q, $res2.1d, $h6.1d @ GHASH block 8k+2 - low
aese $ctr4b, $rk4 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 4
aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 4
aese $ctr5b, $rk4 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 4
aese $ctr7b, $rk4 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 4
aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 4
aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 4
trn1 $t3.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid
aese $ctr6b, $rk4 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 4
aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 4
trn2 $res2.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid
eor $res0.16b, $res0.16b, $acc_m.16b @ GHASH block 8k, 8k+1 - mid
ldp $rk6q, $rk7q, [$cc, #96] @ load rk6, rk7
aese $ctr5b, $rk5 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 5
aese $ctr7b, $rk5 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 5
aese $ctr4b, $rk5 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 5
eor $res2.16b, $res2.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid
aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 5
rev64 $res7b, $res7b @ GHASH block 8k+7
aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 5
aese $ctr6b, $rk5 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 5
aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 5
pmull2 $t3.1q, $res2.2d, $h56k.2d @ GHASH block 8k+2 - mid
pmull2 $acc_m.1q, $res0.2d, $h78k.2d @ GHASH block 8k - mid
aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 5
pmull $h78k.1q, $res0.1d, $h78k.1d @ GHASH block 8k+1 - mid
aese $ctr4b, $rk6 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 6
aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 6
aese $ctr6b, $rk6 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 6
aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 6
aese $ctr7b, $rk6 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 6
eor $acc_mb, $acc_mb, $h78k.16b @ GHASH block 8k+1 - mid
pmull $h56k.1q, $res2.1d, $h56k.1d @ GHASH block 8k+3 - mid
aese $ctr5b, $rk6 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 6
eor3 $acc_lb, $acc_lb, $h6.16b, $h5.16b @ GHASH block 8k+2, 8k+3 - low
aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 6
aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 6
ldp $rk8q, $rk9q, [$cc, #128] @ load rk8, rk9
pmull2 $t4.1q, $res4.2d, $h4.2d @ GHASH block 8k+4 - high
aese $ctr5b, $rk7 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 7
ldr $h1q, [$current_tag, #32] @ load h1l | h1h
ext $h1.16b, $h1.16b, $h1.16b, #8
ldr $h2q, [$current_tag, #64] @ load h2l | h2h
ext $h2.16b, $h2.16b, $h2.16b, #8
aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 7
eor3 $acc_mb, $acc_mb, $h56k.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid
ldr $h12kq, [$current_tag, #48] @ load h2k | h1k
ldr $h34kq, [$current_tag, #96] @ load h4k | h3k
aese $ctr6b, $rk7 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 7
aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 7
aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 7
aese $ctr7b, $rk7 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 7
pmull $h4.1q, $res4.1d, $h4.1d @ GHASH block 8k+4 - low
trn2 $res4.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid
aese $ctr4b, $rk7 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 7
aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 7
pmull2 $t5.1q, $res5.2d, $h3.2d @ GHASH block 8k+5 - high
aese $ctr7b, $rk8 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 8
aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 8
pmull $h3.1q, $res5.1d, $h3.1d @ GHASH block 8k+5 - low
trn1 $t9.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid
eor $res4.16b, $res4.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid
aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 8
aese $ctr0b, $rk9 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 9
aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 8
pmull2 $t6.1q, $res4.2d, $h34k.2d @ GHASH block 8k+4 - mid
pmull $h34k.1q, $res4.1d, $h34k.1d @ GHASH block 8k+5 - mid
aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 8
aese $ctr5b, $rk8 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 8
pmull2 $t7.1q, $res6.2d, $h2.2d @ GHASH block 8k+6 - high
pmull $h2.1q, $res6.1d, $h2.1d @ GHASH block 8k+6 - low
aese $ctr6b, $rk8 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 8
trn2 $res6.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid
aese $ctr4b, $rk8 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 8
eor3 $acc_mb, $acc_mb, $h34k.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid
aese $ctr7b, $rk9 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 9
aese $ctr5b, $rk9 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 9
eor $res6.16b, $res6.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid
aese $ctr6b, $rk9 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 9
aese $ctr4b, $rk9 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 9
ldp $rk10q, $rk11q, [$cc, #160] @ load rk10, rk11
aese $ctr2b, $rk9 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 9
aese $ctr3b, $rk9 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 9
pmull2 $t8.1q, $res7.2d, $h1.2d @ GHASH block 8k+7 - high
eor3 $acc_lb, $acc_lb, $h4.16b, $h3.16b @ GHASH block 8k+4, 8k+5 - low
pmull $h1.1q, $res7.1d, $h1.1d @ GHASH block 8k+7 - low
ldr $mod_constantd, [$modulo_constant] @ MODULO - load modulo constant
pmull2 $t9.1q, $res6.2d, $h12k.2d @ GHASH block 8k+6 - mid
pmull $h12k.1q, $res6.1d, $h12k.1d @ GHASH block 8k+7 - mid
aese $ctr1b, $rk9 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 9
eor3 $acc_mb, $acc_mb, $h12k.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid
eor3 $acc_lb, $acc_lb, $h2.16b, $h1.16b @ GHASH block 8k+6, 8k+7 - low
eor3 $acc_hb, $acc_hb, $t4.16b, $t5.16b @ GHASH block 8k+4, 8k+5 - high
aese $ctr4b, $rk10 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 10
aese $ctr3b, $rk10 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 10
aese $ctr5b, $rk10 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 10
aese $ctr0b, $rk10 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 10
aese $ctr2b, $rk10 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 10
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+15
aese $ctr1b, $rk10 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 10
aese $ctr7b, $rk10 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 10
aese $ctr6b, $rk10 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 10
eor3 $acc_hb, $acc_hb, $t7.16b, $t8.16b @ GHASH block 8k+6, 8k+7 - high
ldp $rk12q, $rk13q, [$cc, #192] @ load rk12, rk13
rev32 $h1.16b, $rtmp_ctr.16b @ CTR block 8k+16
ext $t11.16b, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment
ldp $ctr_t0q, $ctr_t1q, [$input_ptr], #32 @ AES block 8k+8, 8k+9 - load plaintext
aese $ctr2b, $rk11 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 11
aese $ctr6b, $rk11 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 11
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+16
aese $ctr3b, $rk11 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 11
aese $ctr0b, $rk11 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 11
aese $ctr7b, $rk11 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 11
pmull $t12.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid
aese $ctr1b, $rk11 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 11
aese $ctr7b, $rk12 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 12
aese $ctr5b, $rk11 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 11
aese $ctr3b, $rk12 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 12
aese $ctr6b, $rk12 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 12
rev32 $h2.16b, $rtmp_ctr.16b @ CTR block 8k+17
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+17
aese $ctr4b, $rk11 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 11
eor3 $acc_mb, $acc_mb, $acc_hb, $acc_lb @ MODULO - karatsuba tidy up
aese $ctr5b, $rk12 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 12
ldr $rk14q, [$cc, #224] @ load rk14
aese $ctr7b, $rk13 @ AES block 8k+15 - round 13
ldp $ctr_t2q, $ctr_t3q, [$input_ptr], #32 @ AES block 8k+10, 8k+11 - load plaintext
aese $ctr2b, $rk12 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 12
aese $ctr4b, $rk12 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 12
eor3 $acc_mb, $acc_mb, $t12.16b, $t11.16b @ MODULO - fold into mid
aese $ctr1b, $rk12 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 12
ldp $ctr_t4q, $ctr_t5q, [$input_ptr], #32 @ AES block 4, 5 - load plaintext
ldp $ctr_t6q, $ctr_t7q, [$input_ptr], #32 @ AES block 6, 7 - load plaintext
aese $ctr2b, $rk13 @ AES block 8k+10 - round 13
aese $ctr4b, $rk13 @ AES block 8k+12 - round 13
rev32 $h3.16b, $rtmp_ctr.16b @ CTR block 8k+18
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+18
aese $ctr5b, $rk13 @ AES block 8k+13 - round 13
aese $ctr0b, $rk12 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 12
aese $ctr3b, $rk13 @ AES block 8k+11 - round 13
cmp $input_ptr, $main_end_input_ptr @ LOOP CONTROL
eor3 $res2b, $ctr_t2b, $ctr2b, $rk14 @ AES block 8k+10 - result
rev32 $h4.16b, $rtmp_ctr.16b @ CTR block 8k+19
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+19
aese $ctr0b, $rk13 @ AES block 8k+8 - round 13
aese $ctr6b, $rk13 @ AES block 8k+14 - round 13
eor3 $res5b, $ctr_t5b, $ctr5b, $rk14 @ AES block 5 - result
ext $t11.16b, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment
pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low
aese $ctr1b, $rk13 @ AES block 8k+9 - round 13
eor3 $res4b, $ctr_t4b, $ctr4b, $rk14 @ AES block 4 - result
rev32 $ctr4.16b, $rtmp_ctr.16b @ CTR block 8k+20
eor3 $res3b, $ctr_t3b, $ctr3b, $rk14 @ AES block 8k+11 - result
mov $ctr3.16b, $h4.16b @ CTR block 8k+19
eor3 $res1b, $ctr_t1b, $ctr1b, $rk14 @ AES block 8k+9 - result
eor3 $res0b, $ctr_t0b, $ctr0b, $rk14 @ AES block 8k+8 - result
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+20
stp $res0q, $res1q, [$output_ptr], #32 @ AES block 8k+8, 8k+9 - store result
mov $ctr2.16b, $h3.16b @ CTR block 8k+18
eor3 $res7b, $ctr_t7b, $ctr7b, $rk14 @ AES block 7 - result
eor3 $acc_lb, $acc_lb, $t11.16b, $acc_hb @ MODULO - fold into low
stp $res2q, $res3q, [$output_ptr], #32 @ AES block 8k+10, 8k+11 - store result
eor3 $res6b, $ctr_t6b, $ctr6b, $rk14 @ AES block 6 - result
mov $ctr1.16b, $h2.16b @ CTR block 8k+17
stp $res4q, $res5q, [$output_ptr], #32 @ AES block 4, 5 - store result
stp $res6q, $res7q, [$output_ptr], #32 @ AES block 6, 7 - store result
mov $ctr0.16b, $h1.16b @ CTR block 8k+16
b.lt .L256_enc_main_loop
.L256_enc_prepretail: @ PREPRETAIL
rev32 $ctr5.16b, $rtmp_ctr.16b @ CTR block 8k+13
ldp $rk0q, $rk1q, [$cc, #0] @ load rk0, rk1
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+13
rev64 $res2b, $res2b @ GHASH block 8k+2
rev32 $ctr6.16b, $rtmp_ctr.16b @ CTR block 8k+14
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+14
rev64 $res5b, $res5b @ GHASH block 8k+5
ldr $h56kq, [$current_tag, #144] @ load h6k | h5k
ldr $h78kq, [$current_tag, #192] @ load h8k | h7k
rev32 $ctr7.16b, $rtmp_ctr.16b @ CTR block 8k+15
aese $ctr6b, $rk0 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 0
aese $ctr4b, $rk0 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 0
aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 0
aese $ctr5b, $rk0 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 0
aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 0
aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 0
aese $ctr7b, $rk0 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 0
aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 0
ext $acc_lb, $acc_lb, $acc_lb, #8 @ PRE 0
rev64 $res0b, $res0b @ GHASH block 8k
aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 1
rev64 $res1b, $res1b @ GHASH block 8k+1
ldp $rk2q, $rk3q, [$cc, #32] @ load rk2, rk3
aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 1
ldr $h7q, [$current_tag, #176] @ load h7l | h7h
ext $h7.16b, $h7.16b, $h7.16b, #8
ldr $h8q, [$current_tag, #208] @ load h8l | h8h
ext $h8.16b, $h8.16b, $h8.16b, #8
aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 1
ldr $h5q, [$current_tag, #128] @ load h5l | h5h
ext $h5.16b, $h5.16b, $h5.16b, #8
ldr $h6q, [$current_tag, #160] @ load h6l | h6h
ext $h6.16b, $h6.16b, $h6.16b, #8
aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 1
aese $ctr5b, $rk1 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 1
aese $ctr4b, $rk1 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 1
eor $res0b, $res0b, $acc_lb @ PRE 1
rev64 $res3b, $res3b @ GHASH block 8k+3
aese $ctr6b, $rk1 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 1
aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 2
aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 2
aese $ctr7b, $rk1 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 1
aese $ctr4b, $rk2 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 2
aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 2
aese $ctr6b, $rk2 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 2
aese $ctr5b, $rk2 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 2
aese $ctr7b, $rk2 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 2
aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 2
ldp $rk4q, $rk5q, [$cc, #64] @ load rk4, rk5
trn1 $acc_m.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid
pmull2 $acc_h.1q, $res0.2d, $h8.2d @ GHASH block 8k - high
rev64 $res6b, $res6b @ GHASH block 8k+6
aese $ctr4b, $rk3 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 3
pmull2 $t0.1q, $res1.2d, $h7.2d @ GHASH block 8k+1 - high
aese $ctr7b, $rk3 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 3
pmull $acc_l.1q, $res0.1d, $h8.1d @ GHASH block 8k - low
trn2 $res0.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid
pmull2 $t1.1q, $res2.2d, $h6.2d @ GHASH block 8k+2 - high
aese $ctr6b, $rk3 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 3
aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 3
aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 3
eor $acc_hb, $acc_hb, $t0.16b @ GHASH block 8k+1 - high
pmull $h7.1q, $res1.1d, $h7.1d @ GHASH block 8k+1 - low
pmull2 $t2.1q, $res3.2d, $h5.2d @ GHASH block 8k+3 - high
aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 3
aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 3
eor $res0.16b, $res0.16b, $acc_m.16b @ GHASH block 8k, 8k+1 - mid
aese $ctr5b, $rk3 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 3
pmull $h6.1q, $res2.1d, $h6.1d @ GHASH block 8k+2 - low
aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 4
aese $ctr6b, $rk4 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 4
aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 4
aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 4
aese $ctr4b, $rk4 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 4
aese $ctr6b, $rk5 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 5
pmull2 $acc_m.1q, $res0.2d, $h78k.2d @ GHASH block 8k - mid
eor3 $acc_hb, $acc_hb, $t1.16b, $t2.16b @ GHASH block 8k+2, 8k+3 - high
aese $ctr7b, $rk4 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 4
trn1 $t3.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid
trn2 $res2.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid
aese $ctr5b, $rk4 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 4
eor $acc_lb, $acc_lb, $h7.16b @ GHASH block 8k+1 - low
aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 4
pmull $h5.1q, $res3.1d, $h5.1d @ GHASH block 8k+3 - low
pmull $h78k.1q, $res0.1d, $h78k.1d @ GHASH block 8k+1 - mid
eor $res2.16b, $res2.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid
rev64 $res4b, $res4b @ GHASH block 8k+4
aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 5
aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 5
aese $ctr7b, $rk5 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 5
aese $ctr4b, $rk5 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 5
ldp $rk6q, $rk7q, [$cc, #96] @ load rk6, rk7
ldr $h3q, [$current_tag, #80] @ load h3l | h3h
ext $h3.16b, $h3.16b, $h3.16b, #8
ldr $h4q, [$current_tag, #112] @ load h4l | h4h
ext $h4.16b, $h4.16b, $h4.16b, #8
pmull2 $t3.1q, $res2.2d, $h56k.2d @ GHASH block 8k+2 - mid
pmull $h56k.1q, $res2.1d, $h56k.1d @ GHASH block 8k+3 - mid
eor3 $acc_lb, $acc_lb, $h6.16b, $h5.16b @ GHASH block 8k+2, 8k+3 - low
eor $acc_mb, $acc_mb, $h78k.16b @ GHASH block 8k+1 - mid
aese $ctr5b, $rk5 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 5
rev64 $res7b, $res7b @ GHASH block 8k+7
trn1 $t6.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid
aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 5
aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 5
eor3 $acc_mb, $acc_mb, $h56k.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid
aese $ctr7b, $rk6 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 6
aese $ctr4b, $rk6 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 6
aese $ctr6b, $rk6 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 6
ldr $h12kq, [$current_tag, #48] @ load h2k | h1k
ldr $h34kq, [$current_tag, #96] @ load h4k | h3k
aese $ctr5b, $rk6 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 6
aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 6
aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 6
aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 6
aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 6
pmull2 $t4.1q, $res4.2d, $h4.2d @ GHASH block 8k+4 - high
pmull $h4.1q, $res4.1d, $h4.1d @ GHASH block 8k+4 - low
ldr $h1q, [$current_tag, #32] @ load h1l | h1h
ext $h1.16b, $h1.16b, $h1.16b, #8
ldr $h2q, [$current_tag, #64] @ load h2l | h2h
ext $h2.16b, $h2.16b, $h2.16b, #8
ldp $rk8q, $rk9q, [$cc, #128] @ load rk8, rk9
aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 7
aese $ctr4b, $rk7 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 7
pmull2 $t5.1q, $res5.2d, $h3.2d @ GHASH block 8k+5 - high
trn2 $res4.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid
aese $ctr5b, $rk7 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 7
aese $ctr6b, $rk7 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 7
pmull $h3.1q, $res5.1d, $h3.1d @ GHASH block 8k+5 - low
aese $ctr7b, $rk7 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 7
aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 7
eor $res4.16b, $res4.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid
pmull2 $t7.1q, $res6.2d, $h2.2d @ GHASH block 8k+6 - high
pmull $h2.1q, $res6.1d, $h2.1d @ GHASH block 8k+6 - low
aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 7
trn1 $t9.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid
trn2 $res6.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid
aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 7
aese $ctr7b, $rk8 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 8
eor3 $acc_lb, $acc_lb, $h4.16b, $h3.16b @ GHASH block 8k+4, 8k+5 - low
aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 8
aese $ctr6b, $rk8 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 8
aese $ctr4b, $rk8 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 8
aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 8
aese $ctr5b, $rk8 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 8
eor $res6.16b, $res6.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid
aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 8
pmull2 $t6.1q, $res4.2d, $h34k.2d @ GHASH block 8k+4 - mid
pmull $h34k.1q, $res4.1d, $h34k.1d @ GHASH block 8k+5 - mid
aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 8
pmull2 $t8.1q, $res7.2d, $h1.2d @ GHASH block 8k+7 - high
pmull2 $t9.1q, $res6.2d, $h12k.2d @ GHASH block 8k+6 - mid
pmull $h12k.1q, $res6.1d, $h12k.1d @ GHASH block 8k+7 - mid
pmull $h1.1q, $res7.1d, $h1.1d @ GHASH block 8k+7 - low
eor3 $acc_mb, $acc_mb, $h34k.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid
eor3 $acc_hb, $acc_hb, $t4.16b, $t5.16b @ GHASH block 8k+4, 8k+5 - high
ldp $rk10q, $rk11q, [$cc, #160] @ load rk10, rk11
aese $ctr1b, $rk9 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 9
aese $ctr0b, $rk9 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 9
eor3 $acc_hb, $acc_hb, $t7.16b, $t8.16b @ GHASH block 8k+6, 8k+7 - high
eor3 $acc_mb, $acc_mb, $h12k.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid
ldr $mod_constantd, [$modulo_constant] @ MODULO - load modulo constant
eor3 $acc_lb, $acc_lb, $h2.16b, $h1.16b @ GHASH block 8k+6, 8k+7 - low
aese $ctr3b, $rk9 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 9
aese $ctr7b, $rk9 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 9
aese $ctr5b, $rk9 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 9
aese $ctr2b, $rk9 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 9
aese $ctr6b, $rk9 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 9
aese $ctr5b, $rk10 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 10
aese $ctr1b, $rk10 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 10
aese $ctr4b, $rk9 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 9
aese $ctr7b, $rk10 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 10
aese $ctr6b, $rk10 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 10
aese $ctr3b, $rk10 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 10
aese $ctr4b, $rk10 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 10
aese $ctr0b, $rk10 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 10
aese $ctr2b, $rk10 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 10
pmull $t12.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid
eor3 $acc_mb, $acc_mb, $acc_hb, $acc_lb @ MODULO - karatsuba tidy up
aese $ctr7b, $rk11 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 11
ldp $rk12q, $rk13q, [$cc, #192] @ load rk12, rk13
ext $t11.16b, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment
aese $ctr2b, $rk11 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 11
eor3 $acc_mb, $acc_mb, $t12.16b, $t11.16b @ MODULO - fold into mid
aese $ctr1b, $rk11 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 11
aese $ctr6b, $rk11 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 11
aese $ctr0b, $rk11 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 11
aese $ctr4b, $rk11 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 11
aese $ctr5b, $rk11 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 11
pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low
aese $ctr3b, $rk11 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 11
ldr $rk14q, [$cc, #224] @ load rk14
aese $ctr1b, $rk12 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 12
aese $ctr2b, $rk12 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 12
aese $ctr0b, $rk12 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 12
aese $ctr6b, $rk12 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 12
aese $ctr5b, $rk12 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 12
ext $t11.16b, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment
aese $ctr4b, $rk12 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 12
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+15
aese $ctr3b, $rk12 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 12
aese $ctr7b, $rk12 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 12
aese $ctr0b, $rk13 @ AES block 8k+8 - round 13
eor3 $acc_lb, $acc_lb, $t11.16b, $acc_hb @ MODULO - fold into low
aese $ctr5b, $rk13 @ AES block 8k+13 - round 13
aese $ctr1b, $rk13 @ AES block 8k+9 - round 13
aese $ctr3b, $rk13 @ AES block 8k+11 - round 13
aese $ctr4b, $rk13 @ AES block 8k+12 - round 13
aese $ctr7b, $rk13 @ AES block 8k+15 - round 13
aese $ctr2b, $rk13 @ AES block 8k+10 - round 13
aese $ctr6b, $rk13 @ AES block 8k+14 - round 13
.L256_enc_tail: @ TAIL
ldp $h78kq, $h8q, [$current_tag, #192] @ load h8l | h8h
ext $h8.16b, $h8.16b, $h8.16b, #8
sub $main_end_input_ptr, $end_input_ptr, $input_ptr @ main_end_input_ptr is number of bytes left to process
ldr $ctr_t0q, [$input_ptr], #16 @ AES block 8k+8 - load plaintext
ldp $h5q, $h56kq, [$current_tag, #128] @ load h5l | h5h
ext $h5.16b, $h5.16b, $h5.16b, #8
ext $t0.16b, $acc_lb, $acc_lb, #8 @ prepare final partial tag
ldp $h6q, $h7q, [$current_tag, #160] @ load h6l | h6h
ext $h6.16b, $h6.16b, $h6.16b, #8
ext $h7.16b, $h7.16b, $h7.16b, #8
mov $t1.16b, $rk14
cmp $main_end_input_ptr, #112
eor3 $res1b, $ctr_t0b, $ctr0b, $t1.16b @ AES block 8k+8 - result
b.gt .L256_enc_blocks_more_than_7
movi $acc_l.8b, #0
mov $ctr7b, $ctr6b
movi $acc_h.8b, #0
mov $ctr6b, $ctr5b
mov $ctr5b, $ctr4b
mov $ctr4b, $ctr3b
mov $ctr3b, $ctr2b
sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
mov $ctr2b, $ctr1b
movi $acc_m.8b, #0
cmp $main_end_input_ptr, #96
b.gt .L256_enc_blocks_more_than_6
mov $ctr7b, $ctr6b
mov $ctr6b, $ctr5b
cmp $main_end_input_ptr, #80
mov $ctr5b, $ctr4b
mov $ctr4b, $ctr3b
mov $ctr3b, $ctr1b
sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
b.gt .L256_enc_blocks_more_than_5
mov $ctr7b, $ctr6b
sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
mov $ctr6b, $ctr5b
mov $ctr5b, $ctr4b
cmp $main_end_input_ptr, #64
mov $ctr4b, $ctr1b
b.gt .L256_enc_blocks_more_than_4
cmp $main_end_input_ptr, #48
mov $ctr7b, $ctr6b
mov $ctr6b, $ctr5b
mov $ctr5b, $ctr1b
sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
b.gt .L256_enc_blocks_more_than_3
cmp $main_end_input_ptr, #32
mov $ctr7b, $ctr6b
ldr $h34kq, [$current_tag, #96] @ load h4k | h3k
mov $ctr6b, $ctr1b
sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
b.gt .L256_enc_blocks_more_than_2
mov $ctr7b, $ctr1b
sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
cmp $main_end_input_ptr, #16
b.gt .L256_enc_blocks_more_than_1
sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
ldr $h12kq, [$current_tag, #48] @ load h2k | h1k
b .L256_enc_blocks_less_than_1
.L256_enc_blocks_more_than_7: @ blocks left > 7
st1 { $res1b}, [$output_ptr], #16 @ AES final-7 block - store result
rev64 $res0b, $res1b @ GHASH final-7 block
eor $res0b, $res0b, $t0.16b @ feed in partial tag
ldr $ctr_t1q, [$input_ptr], #16 @ AES final-6 block - load plaintext
pmull2 $acc_h.1q, $res0.2d, $h8.2d @ GHASH final-7 block - high
ins $rk4v.d[0], $res0.d[1] @ GHASH final-7 block - mid
ins $acc_m.d[0], $h78k.d[1] @ GHASH final-7 block - mid
movi $t0.8b, #0 @ supress further partial tag feed in
eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-7 block - mid
eor3 $res1b, $ctr_t1b, $ctr1b, $t1.16b @ AES final-6 block - result
pmull $acc_m.1q, $rk4v.1d, $acc_m.1d @ GHASH final-7 block - mid
pmull $acc_l.1q, $res0.1d, $h8.1d @ GHASH final-7 block - low
.L256_enc_blocks_more_than_6: @ blocks left > 6
st1 { $res1b}, [$output_ptr], #16 @ AES final-6 block - store result
rev64 $res0b, $res1b @ GHASH final-6 block
eor $res0b, $res0b, $t0.16b @ feed in partial tag
pmull $rk3q1, $res0.1d, $h7.1d @ GHASH final-6 block - low
ins $rk4v.d[0], $res0.d[1] @ GHASH final-6 block - mid
pmull2 $rk2q1, $res0.2d, $h7.2d @ GHASH final-6 block - high
ldr $ctr_t1q, [$input_ptr], #16 @ AES final-5 block - load plaintext
eor $acc_lb, $acc_lb, $rk3 @ GHASH final-6 block - low
eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-6 block - mid
pmull $rk4v.1q, $rk4v.1d, $h78k.1d @ GHASH final-6 block - mid
eor3 $res1b, $ctr_t1b, $ctr2b, $t1.16b @ AES final-5 block - result
movi $t0.8b, #0 @ supress further partial tag feed in
eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-6 block - mid
eor $acc_hb, $acc_hb, $rk2 @ GHASH final-6 block - high
.L256_enc_blocks_more_than_5: @ blocks left > 5
st1 { $res1b}, [$output_ptr], #16 @ AES final-5 block - store result
rev64 $res0b, $res1b @ GHASH final-5 block
eor $res0b, $res0b, $t0.16b @ feed in partial tag
ins $rk4v.d[0], $res0.d[1] @ GHASH final-5 block - mid
pmull2 $rk2q1, $res0.2d, $h6.2d @ GHASH final-5 block - high
eor $acc_hb, $acc_hb, $rk2 @ GHASH final-5 block - high
eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-5 block - mid
ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-5 block - mid
ldr $ctr_t1q, [$input_ptr], #16 @ AES final-4 block - load plaintext
pmull $rk3q1, $res0.1d, $h6.1d @ GHASH final-5 block - low
pmull2 $rk4v.1q, $rk4v.2d, $h56k.2d @ GHASH final-5 block - mid
movi $t0.8b, #0 @ supress further partial tag feed in
eor $acc_lb, $acc_lb, $rk3 @ GHASH final-5 block - low
eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-5 block - mid
eor3 $res1b, $ctr_t1b, $ctr3b, $t1.16b @ AES final-4 block - result
.L256_enc_blocks_more_than_4: @ blocks left > 4
st1 { $res1b}, [$output_ptr], #16 @ AES final-4 block - store result
rev64 $res0b, $res1b @ GHASH final-4 block
ldr $ctr_t1q, [$input_ptr], #16 @ AES final-3 block - load plaintext
eor $res0b, $res0b, $t0.16b @ feed in partial tag
ins $rk4v.d[0], $res0.d[1] @ GHASH final-4 block - mid
pmull2 $rk2q1, $res0.2d, $h5.2d @ GHASH final-4 block - high
eor3 $res1b, $ctr_t1b, $ctr4b, $t1.16b @ AES final-3 block - result
pmull $rk3q1, $res0.1d, $h5.1d @ GHASH final-4 block - low
eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-4 block - mid
eor $acc_lb, $acc_lb, $rk3 @ GHASH final-4 block - low
pmull $rk4v.1q, $rk4v.1d, $h56k.1d @ GHASH final-4 block - mid
movi $t0.8b, #0 @ supress further partial tag feed in
eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-4 block - mid
eor $acc_hb, $acc_hb, $rk2 @ GHASH final-4 block - high
.L256_enc_blocks_more_than_3: @ blocks left > 3
st1 { $res1b}, [$output_ptr], #16 @ AES final-3 block - store result
ldr $h4q, [$current_tag, #112] @ load h4l | h4h
ext $h4.16b, $h4.16b, $h4.16b, #8
rev64 $res0b, $res1b @ GHASH final-3 block
eor $res0b, $res0b, $t0.16b @ feed in partial tag
ins $rk4v.d[0], $res0.d[1] @ GHASH final-3 block - mid
pmull2 $rk2q1, $res0.2d, $h4.2d @ GHASH final-3 block - high
eor $acc_hb, $acc_hb, $rk2 @ GHASH final-3 block - high
eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-3 block - mid
ldr $h34kq, [$current_tag, #96] @ load h4k | h3k
ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-3 block - mid
ldr $ctr_t1q, [$input_ptr], #16 @ AES final-2 block - load plaintext
pmull2 $rk4v.1q, $rk4v.2d, $h34k.2d @ GHASH final-3 block - mid
pmull $rk3q1, $res0.1d, $h4.1d @ GHASH final-3 block - low
eor3 $res1b, $ctr_t1b, $ctr5b, $t1.16b @ AES final-2 block - result
movi $t0.8b, #0 @ supress further partial tag feed in
eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-3 block - mid
eor $acc_lb, $acc_lb, $rk3 @ GHASH final-3 block - low
.L256_enc_blocks_more_than_2: @ blocks left > 2
ldr $h3q, [$current_tag, #80] @ load h3l | h3h
ext $h3.16b, $h3.16b, $h3.16b, #8
st1 { $res1b}, [$output_ptr], #16 @ AES final-2 block - store result
rev64 $res0b, $res1b @ GHASH final-2 block
ldr $ctr_t1q, [$input_ptr], #16 @ AES final-1 block - load plaintext
eor $res0b, $res0b, $t0.16b @ feed in partial tag
ins $rk4v.d[0], $res0.d[1] @ GHASH final-2 block - mid
movi $t0.8b, #0 @ supress further partial tag feed in
pmull2 $rk2q1, $res0.2d, $h3.2d @ GHASH final-2 block - high
eor3 $res1b, $ctr_t1b, $ctr6b, $t1.16b @ AES final-1 block - result
eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-2 block - mid
eor $acc_hb, $acc_hb, $rk2 @ GHASH final-2 block - high
pmull $rk4v.1q, $rk4v.1d, $h34k.1d @ GHASH final-2 block - mid
pmull $rk3q1, $res0.1d, $h3.1d @ GHASH final-2 block - low
eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-2 block - mid
eor $acc_lb, $acc_lb, $rk3 @ GHASH final-2 block - low
.L256_enc_blocks_more_than_1: @ blocks left > 1
st1 { $res1b}, [$output_ptr], #16 @ AES final-1 block - store result
ldr $h2q, [$current_tag, #64] @ load h2l | h2h
ext $h2.16b, $h2.16b, $h2.16b, #8
rev64 $res0b, $res1b @ GHASH final-1 block
ldr $ctr_t1q, [$input_ptr], #16 @ AES final block - load plaintext
eor $res0b, $res0b, $t0.16b @ feed in partial tag
movi $t0.8b, #0 @ supress further partial tag feed in
ins $rk4v.d[0], $res0.d[1] @ GHASH final-1 block - mid
pmull2 $rk2q1, $res0.2d, $h2.2d @ GHASH final-1 block - high
eor3 $res1b, $ctr_t1b, $ctr7b, $t1.16b @ AES final block - result
eor $acc_hb, $acc_hb, $rk2 @ GHASH final-1 block - high
pmull $rk3q1, $res0.1d, $h2.1d @ GHASH final-1 block - low
eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-1 block - mid
ldr $h12kq, [$current_tag, #48] @ load h2k | h1k
eor $acc_lb, $acc_lb, $rk3 @ GHASH final-1 block - low
ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-1 block - mid
pmull2 $rk4v.1q, $rk4v.2d, $h12k.2d @ GHASH final-1 block - mid
eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-1 block - mid
.L256_enc_blocks_less_than_1: @ blocks left <= 1
and $bit_length, $bit_length, #127 @ bit_length %= 128
sub $bit_length, $bit_length, #128 @ bit_length -= 128
neg $bit_length, $bit_length @ bit_length = 128 - #bits in input (in range [1,128])
mvn $temp0_x, xzr @ temp0_x = 0xffffffffffffffff
and $bit_length, $bit_length, #127 @ bit_length %= 128
lsr $temp0_x, $temp0_x, $bit_length @ temp0_x is mask for top 64b of last block
cmp $bit_length, #64
mvn $temp1_x, xzr @ temp1_x = 0xffffffffffffffff
csel $temp3_x, $temp0_x, xzr, lt
csel $temp2_x, $temp1_x, $temp0_x, lt
mov $ctr0.d[0], $temp2_x @ ctr0b is mask for last block
ldr $h1q, [$current_tag, #32] @ load h1l | h1h
ext $h1.16b, $h1.16b, $h1.16b, #8
ld1 { $rk0}, [$output_ptr] @ load existing bytes where the possibly partial last block is to be stored
mov $ctr0.d[1], $temp3_x
and $res1b, $res1b, $ctr0b @ possibly partial last block has zeroes in highest bits
rev64 $res0b, $res1b @ GHASH final block
rev32 $rtmp_ctr.16b, $rtmp_ctr.16b
bif $res1b, $rk0, $ctr0b @ insert existing bytes in top end of result before storing
str $rtmp_ctrq, [$counter] @ store the updated counter
eor $res0b, $res0b, $t0.16b @ feed in partial tag
st1 { $res1b}, [$output_ptr] @ store all 16B
ins $t0.d[0], $res0.d[1] @ GHASH final block - mid
pmull2 $rk2q1, $res0.2d, $h1.2d @ GHASH final block - high
pmull $rk3q1, $res0.1d, $h1.1d @ GHASH final block - low
eor $acc_hb, $acc_hb, $rk2 @ GHASH final block - high
eor $acc_lb, $acc_lb, $rk3 @ GHASH final block - low
eor $t0.8b, $t0.8b, $res0.8b @ GHASH final block - mid
pmull $t0.1q, $t0.1d, $h12k.1d @ GHASH final block - mid
eor $acc_mb, $acc_mb, $t0.16b @ GHASH final block - mid
ldr $mod_constantd, [$modulo_constant] @ MODULO - load modulo constant
ext $t11.16b, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment
eor3 $acc_mb, $acc_mb, $acc_hb, $acc_lb @ MODULO - karatsuba tidy up
pmull $t12.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid
eor3 $acc_mb, $acc_mb, $t12.16b, $t11.16b @ MODULO - fold into mid
pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low
ext $t11.16b, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment
eor3 $acc_lb, $acc_lb, $acc_hb, $t11.16b @ MODULO - fold into low
ext $acc_lb, $acc_lb, $acc_lb, #8
rev64 $acc_lb, $acc_lb
st1 { $acc_l.16b }, [$current_tag]
lsr x0, $bit_length, #3 @ return sizes
ldp d10, d11, [sp, #16]
ldp d12, d13, [sp, #32]
ldp d14, d15, [sp, #48]
ldp d8, d9, [sp], #80
ret
.L256_enc_ret:
mov w0, #0x0
ret
.size unroll8_eor3_aes_gcm_enc_256_kernel,.-unroll8_eor3_aes_gcm_enc_256_kernel
___
{
#########################################################################################
# size_t unroll8_eor3_aes_gcm_dec_256_kernel(const unsigned char *in,
# size_t len,
# unsigned char *out,
# const void *key,
# unsigned char ivec[16],
# u64 *Xi);
#
$code.=<<___;
.global unroll8_eor3_aes_gcm_dec_256_kernel
.type unroll8_eor3_aes_gcm_dec_256_kernel,%function
.align 4
unroll8_eor3_aes_gcm_dec_256_kernel:
AARCH64_VALID_CALL_TARGET
cbz x1, .L256_dec_ret
stp d8, d9, [sp, #-80]!
mov $counter, x4
mov $cc, x5
stp d10, d11, [sp, #16]
stp d12, d13, [sp, #32]
stp d14, d15, [sp, #48]
mov x5, #0xc200000000000000
stp x5, xzr, [sp, #64]
add $modulo_constant, sp, #64
ld1 { $ctr0b}, [$counter] @ CTR block 0
mov $constant_temp, #0x100000000 @ set up counter increment
movi $rctr_inc.16b, #0x0
mov $rctr_inc.d[1], $constant_temp
lsr $main_end_input_ptr, $bit_length, #3 @ byte_len
sub $main_end_input_ptr, $main_end_input_ptr, #1 @ byte_len - 1
rev32 $rtmp_ctr.16b, $ctr0.16b @ set up reversed counter
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 0
rev32 $ctr1.16b, $rtmp_ctr.16b @ CTR block 1
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 1
rev32 $ctr2.16b, $rtmp_ctr.16b @ CTR block 2
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 2
ldp $rk0q, $rk1q, [$cc, #0] @ load rk0, rk1
rev32 $ctr3.16b, $rtmp_ctr.16b @ CTR block 3
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 3
rev32 $ctr4.16b, $rtmp_ctr.16b @ CTR block 4
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 4
aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 0
rev32 $ctr5.16b, $rtmp_ctr.16b @ CTR block 5
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 5
aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 0
aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 0
rev32 $ctr6.16b, $rtmp_ctr.16b @ CTR block 6
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 6
rev32 $ctr7.16b, $rtmp_ctr.16b @ CTR block 7
aese $ctr4b, $rk0 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 0
aese $ctr6b, $rk0 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 0
aese $ctr5b, $rk0 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 0
aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 0
aese $ctr7b, $rk0 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 0
ldp $rk2q, $rk3q, [$cc, #32] @ load rk2, rk3
aese $ctr6b, $rk1 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 1
aese $ctr4b, $rk1 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 1
aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 1
aese $ctr5b, $rk1 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 1
aese $ctr7b, $rk1 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 1
aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 1
aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 1
aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 1
aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 2
aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 2
aese $ctr6b, $rk2 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 2
aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 2
aese $ctr7b, $rk2 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 2
aese $ctr5b, $rk2 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 2
aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 2
aese $ctr4b, $rk2 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 2
ldp $rk4q, $rk5q, [$cc, #64] @ load rk4, rk5
aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 3
aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 3
aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 3
aese $ctr4b, $rk3 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 3
aese $ctr5b, $rk3 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 3
aese $ctr7b, $rk3 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 3
aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 3
aese $ctr6b, $rk3 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 3
aese $ctr7b, $rk4 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 4
aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 4
aese $ctr6b, $rk4 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 4
aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 4
aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 4
aese $ctr4b, $rk4 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 4
aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 4
aese $ctr5b, $rk4 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 4
aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 5
aese $ctr6b, $rk5 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 5
ldp $rk6q, $rk7q, [$cc, #96] @ load rk6, rk7
aese $ctr4b, $rk5 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 5
aese $ctr7b, $rk5 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 5
aese $ctr5b, $rk5 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 5
aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 5
aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 5
aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 5
aese $ctr4b, $rk6 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 6
aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 6
aese $ctr7b, $rk6 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 6
aese $ctr6b, $rk6 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 6
aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 6
aese $ctr5b, $rk6 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 6
aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 6
aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 6
ldp $rk8q, $rk9q, [$cc, #128] @ load rk8, rk9
aese $ctr5b, $rk7 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 7
aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 7
aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 7
aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 7
aese $ctr7b, $rk7 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 7
aese $ctr4b, $rk7 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 7
aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 7
aese $ctr6b, $rk7 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 7
and $main_end_input_ptr, $main_end_input_ptr, #0xffffffffffffff80 @ number of bytes to be processed in main loop (at least 1 byte must be handled by tail)
aese $ctr7b, $rk8 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 8
aese $ctr5b, $rk8 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 8
aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 8
aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 8
aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 8
aese $ctr4b, $rk8 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 8
aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 8
aese $ctr6b, $rk8 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 8
aese $ctr2b, $rk9 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 9
ld1 { $acc_lb}, [$current_tag]
ext $acc_lb, $acc_lb, $acc_lb, #8
rev64 $acc_lb, $acc_lb
ldp $rk10q, $rk11q, [$cc, #160] @ load rk10, rk11
add $end_input_ptr, $input_ptr, $bit_length, lsr #3 @ end_input_ptr
add $main_end_input_ptr, $main_end_input_ptr, $input_ptr
aese $ctr3b, $rk9 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 9
aese $ctr6b, $rk9 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 9
aese $ctr4b, $rk9 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 9
aese $ctr5b, $rk9 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 9
aese $ctr7b, $rk9 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 9
aese $ctr0b, $rk9 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 9
aese $ctr1b, $rk9 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 9
aese $ctr4b, $rk10 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 10
aese $ctr7b, $rk10 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 10
aese $ctr5b, $rk10 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 10
aese $ctr1b, $rk10 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 10
aese $ctr2b, $rk10 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 10
aese $ctr0b, $rk10 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 10
aese $ctr6b, $rk10 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 10
aese $ctr3b, $rk10 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 10
ldp $rk12q, $rk13q, [$cc, #192] @ load rk12, rk13
aese $ctr0b, $rk11 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 11
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 7
aese $ctr7b, $rk11 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 11
aese $ctr3b, $rk11 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 11
aese $ctr1b, $rk11 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 11
aese $ctr5b, $rk11 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 11
aese $ctr4b, $rk11 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 11
aese $ctr2b, $rk11 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 11
aese $ctr6b, $rk11 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 11
ldr $rk14q, [$cc, #224] @ load rk14
aese $ctr1b, $rk12 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 12
aese $ctr4b, $rk12 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 12
aese $ctr5b, $rk12 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 12
cmp $input_ptr, $main_end_input_ptr @ check if we have <= 8 blocks
aese $ctr3b, $rk12 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 12
aese $ctr2b, $rk12 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 12
aese $ctr6b, $rk12 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 12
aese $ctr0b, $rk12 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 12
aese $ctr7b, $rk12 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 12
aese $ctr5b, $rk13 @ AES block 5 - round 13
aese $ctr1b, $rk13 @ AES block 1 - round 13
aese $ctr2b, $rk13 @ AES block 2 - round 13
aese $ctr0b, $rk13 @ AES block 0 - round 13
aese $ctr4b, $rk13 @ AES block 4 - round 13
aese $ctr6b, $rk13 @ AES block 6 - round 13
aese $ctr3b, $rk13 @ AES block 3 - round 13
aese $ctr7b, $rk13 @ AES block 7 - round 13
b.ge .L256_dec_tail @ handle tail
ldp $res0q, $res1q, [$input_ptr], #32 @ AES block 0, 1 - load ciphertext
ldp $res2q, $res3q, [$input_ptr], #32 @ AES block 2, 3 - load ciphertext
ldp $res4q, $res5q, [$input_ptr], #32 @ AES block 4, 5 - load ciphertext
ldp $res6q, $res7q, [$input_ptr], #32 @ AES block 6, 7 - load ciphertext
cmp $input_ptr, $main_end_input_ptr @ check if we have <= 8 blocks
eor3 $ctr1b, $res1b, $ctr1b, $rk14 @ AES block 1 - result
eor3 $ctr0b, $res0b, $ctr0b, $rk14 @ AES block 0 - result
stp $ctr0q, $ctr1q, [$output_ptr], #32 @ AES block 0, 1 - store result
rev32 $ctr0.16b, $rtmp_ctr.16b @ CTR block 8
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8
eor3 $ctr3b, $res3b, $ctr3b, $rk14 @ AES block 3 - result
eor3 $ctr5b, $res5b, $ctr5b, $rk14 @ AES block 5 - result
eor3 $ctr4b, $res4b, $ctr4b, $rk14 @ AES block 4 - result
rev32 $ctr1.16b, $rtmp_ctr.16b @ CTR block 9
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 9
eor3 $ctr2b, $res2b, $ctr2b, $rk14 @ AES block 2 - result
stp $ctr2q, $ctr3q, [$output_ptr], #32 @ AES block 2, 3 - store result
rev32 $ctr2.16b, $rtmp_ctr.16b @ CTR block 10
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 10
eor3 $ctr6b, $res6b, $ctr6b, $rk14 @ AES block 6 - result
rev32 $ctr3.16b, $rtmp_ctr.16b @ CTR block 11
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 11
stp $ctr4q, $ctr5q, [$output_ptr], #32 @ AES block 4, 5 - store result
eor3 $ctr7b, $res7b, $ctr7b, $rk14 @ AES block 7 - result
stp $ctr6q, $ctr7q, [$output_ptr], #32 @ AES block 6, 7 - store result
rev32 $ctr4.16b, $rtmp_ctr.16b @ CTR block 12
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 12
b.ge .L256_dec_prepretail @ do prepretail
.L256_dec_main_loop: @ main loop start
rev32 $ctr5.16b, $rtmp_ctr.16b @ CTR block 8k+13
ldp $rk0q, $rk1q, [$cc, #0] @ load rk0, rk1
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+13
rev64 $res1b, $res1b @ GHASH block 8k+1
ldr $h7q, [$current_tag, #176] @ load h7l | h7h
ext $h7.16b, $h7.16b, $h7.16b, #8
ldr $h8q, [$current_tag, #208] @ load h8l | h8h
ext $h8.16b, $h8.16b, $h8.16b, #8
rev32 $ctr6.16b, $rtmp_ctr.16b @ CTR block 8k+14
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+14
rev64 $res0b, $res0b @ GHASH block 8k
ext $acc_lb, $acc_lb, $acc_lb, #8 @ PRE 0
rev64 $res4b, $res4b @ GHASH block 8k+4
rev64 $res3b, $res3b @ GHASH block 8k+3
rev32 $ctr7.16b, $rtmp_ctr.16b @ CTR block 8k+15
rev64 $res7b, $res7b @ GHASH block 8k+7
aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 0
aese $ctr6b, $rk0 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 0
aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 0
aese $ctr7b, $rk0 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 0
aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 0
aese $ctr5b, $rk0 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 0
aese $ctr4b, $rk0 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 0
aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 0
ldp $rk2q, $rk3q, [$cc, #32] @ load rk2, rk3
eor $res0b, $res0b, $acc_lb @ PRE 1
ldr $h5q, [$current_tag, #128] @ load h5l | h5h
ext $h5.16b, $h5.16b, $h5.16b, #8
ldr $h6q, [$current_tag, #160] @ load h6l | h6h
ext $h6.16b, $h6.16b, $h6.16b, #8
aese $ctr6b, $rk1 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 1
aese $ctr4b, $rk1 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 1
rev64 $res2b, $res2b @ GHASH block 8k+2
aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 1
aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 1
aese $ctr5b, $rk1 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 1
aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 1
trn1 $acc_m.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid
aese $ctr7b, $rk1 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 1
aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 1
aese $ctr4b, $rk2 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 2
aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 2
aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 2
aese $ctr6b, $rk2 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 2
aese $ctr7b, $rk2 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 2
pmull $acc_l.1q, $res0.1d, $h8.1d @ GHASH block 8k - low
aese $ctr5b, $rk2 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 2
aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 2
aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 2
ldp $rk4q, $rk5q, [$cc, #64] @ load rk4, rk5
pmull2 $t1.1q, $res2.2d, $h6.2d @ GHASH block 8k+2 - high
aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 3
aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 3
pmull2 $t0.1q, $res1.2d, $h7.2d @ GHASH block 8k+1 - high
pmull $h7.1q, $res1.1d, $h7.1d @ GHASH block 8k+1 - low
aese $ctr5b, $rk3 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 3
aese $ctr6b, $rk3 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 3
pmull2 $acc_h.1q, $res0.2d, $h8.2d @ GHASH block 8k - high
aese $ctr4b, $rk3 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 3
aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 3
trn2 $res0.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid
pmull2 $t2.1q, $res3.2d, $h5.2d @ GHASH block 8k+3 - high
aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 3
eor $acc_hb, $acc_hb, $t0.16b @ GHASH block 8k+1 - high
aese $ctr5b, $rk4 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 4
aese $ctr7b, $rk3 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 3
aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 4
aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 4
aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 4
aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 4
aese $ctr6b, $rk4 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 4
aese $ctr7b, $rk4 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 4
aese $ctr4b, $rk4 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 4
ldr $h56kq, [$current_tag, #144] @ load h6k | h5k
ldr $h78kq, [$current_tag, #192] @ load h8k | h7k
eor $res0.16b, $res0.16b, $acc_m.16b @ GHASH block 8k, 8k+1 - mid
pmull $h6.1q, $res2.1d, $h6.1d @ GHASH block 8k+2 - low
ldp $rk6q, $rk7q, [$cc, #96] @ load rk6, rk7
aese $ctr5b, $rk5 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 5
eor $acc_lb, $acc_lb, $h7.16b @ GHASH block 8k+1 - low
aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 5
aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 5
aese $ctr7b, $rk5 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 5
aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 5
aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 5
aese $ctr6b, $rk5 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 5
eor3 $acc_hb, $acc_hb, $t1.16b, $t2.16b @ GHASH block 8k+2, 8k+3 - high
trn1 $t3.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid
rev64 $res5b, $res5b @ GHASH block 8k+5
pmull2 $acc_m.1q, $res0.2d, $h78k.2d @ GHASH block 8k - mid
pmull $h78k.1q, $res0.1d, $h78k.1d @ GHASH block 8k+1 - mid
trn2 $res2.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid
aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 6
aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 6
aese $ctr4b, $rk5 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 5
trn1 $t6.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid
aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 6
aese $ctr6b, $rk6 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 6
eor $res2.16b, $res2.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid
pmull $h5.1q, $res3.1d, $h5.1d @ GHASH block 8k+3 - low
aese $ctr4b, $rk6 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 6
aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 6
aese $ctr5b, $rk6 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 6
aese $ctr7b, $rk6 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 6
pmull2 $t3.1q, $res2.2d, $h56k.2d @ GHASH block 8k+2 - mid
pmull $h56k.1q, $res2.1d, $h56k.1d @ GHASH block 8k+3 - mid
eor3 $acc_lb, $acc_lb, $h6.16b, $h5.16b @ GHASH block 8k+2, 8k+3 - low
ldr $h3q, [$current_tag, #80] @ load h3l | h3h
ext $h3.16b, $h3.16b, $h3.16b, #8
ldr $h4q, [$current_tag, #112] @ load h4l | h4h
ext $h4.16b, $h4.16b, $h4.16b, #8
rev64 $res6b, $res6b @ GHASH block 8k+6
eor $acc_mb, $acc_mb, $h78k.16b @ GHASH block 8k+1 - mid
aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 7
aese $ctr5b, $rk7 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 7
ldp $rk8q, $rk9q, [$cc, #128] @ load rk8, rk9
ldr $h1q, [$current_tag, #32] @ load h1l | h1h
ext $h1.16b, $h1.16b, $h1.16b, #8
ldr $h2q, [$current_tag, #64] @ load h2l | h2h
ext $h2.16b, $h2.16b, $h2.16b, #8
eor3 $acc_mb, $acc_mb, $h56k.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid
aese $ctr7b, $rk7 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 7
aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 7
aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 7
aese $ctr6b, $rk7 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 7
ldr $h12kq, [$current_tag, #48] @ load h2k | h1k
ldr $h34kq, [$current_tag, #96] @ load h4k | h3k
aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 7
aese $ctr4b, $rk7 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 7
pmull2 $t4.1q, $res4.2d, $h4.2d @ GHASH block 8k+4 - high
pmull $h4.1q, $res4.1d, $h4.1d @ GHASH block 8k+4 - low
trn2 $res4.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid
aese $ctr5b, $rk8 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 8
pmull2 $t5.1q, $res5.2d, $h3.2d @ GHASH block 8k+5 - high
aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 8
aese $ctr6b, $rk8 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 8
pmull $h3.1q, $res5.1d, $h3.1d @ GHASH block 8k+5 - low
aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 8
aese $ctr4b, $rk8 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 8
aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 8
pmull2 $t7.1q, $res6.2d, $h2.2d @ GHASH block 8k+6 - high
trn1 $t9.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid
aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 8
aese $ctr7b, $rk8 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 8
ldp $rk10q, $rk11q, [$cc, #160] @ load rk10, rk11
pmull $h2.1q, $res6.1d, $h2.1d @ GHASH block 8k+6 - low
trn2 $res6.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+15
eor3 $acc_hb, $acc_hb, $t4.16b, $t5.16b @ GHASH block 8k+4, 8k+5 - high
aese $ctr3b, $rk9 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 9
aese $ctr6b, $rk9 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 9
eor $res6.16b, $res6.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid
aese $ctr5b, $rk9 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 9
ldp $res0q, $res1q, [$input_ptr], #32 @ AES block 8k+8, 8k+9 - load ciphertext
eor $res4.16b, $res4.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid
aese $ctr7b, $rk9 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 9
pmull2 $t9.1q, $res6.2d, $h12k.2d @ GHASH block 8k+6 - mid
aese $ctr2b, $rk9 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 9
aese $ctr1b, $rk9 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 9
pmull2 $t6.1q, $res4.2d, $h34k.2d @ GHASH block 8k+4 - mid
pmull $h34k.1q, $res4.1d, $h34k.1d @ GHASH block 8k+5 - mid
pmull2 $t8.1q, $res7.2d, $h1.2d @ GHASH block 8k+7 - high
pmull $h1.1q, $res7.1d, $h1.1d @ GHASH block 8k+7 - low
aese $ctr3b, $rk10 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 10
aese $ctr6b, $rk10 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 10
pmull $h12k.1q, $res6.1d, $h12k.1d @ GHASH block 8k+7 - mid
aese $ctr0b, $rk9 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 9
eor3 $acc_lb, $acc_lb, $h4.16b, $h3.16b @ GHASH block 8k+4, 8k+5 - low
aese $ctr4b, $rk9 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 9
eor3 $acc_mb, $acc_mb, $h34k.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid
eor3 $acc_hb, $acc_hb, $t7.16b, $t8.16b @ GHASH block 8k+6, 8k+7 - high
aese $ctr2b, $rk10 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 10
aese $ctr5b, $rk10 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 10
aese $ctr7b, $rk10 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 10
aese $ctr1b, $rk10 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 10
aese $ctr0b, $rk10 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 10
aese $ctr4b, $rk10 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 10
eor3 $acc_lb, $acc_lb, $h2.16b, $h1.16b @ GHASH block 8k+6, 8k+7 - low
rev32 $h1.16b, $rtmp_ctr.16b @ CTR block 8k+16
ldr $mod_constantd, [$modulo_constant] @ MODULO - load modulo constant
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+16
aese $ctr1b, $rk11 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 11
ldp $rk12q, $rk13q, [$cc, #192] @ load rk12, rk13
aese $ctr0b, $rk11 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 11
aese $ctr6b, $rk11 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 11
eor3 $acc_mb, $acc_mb, $h12k.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid
rev32 $h2.16b, $rtmp_ctr.16b @ CTR block 8k+17
aese $ctr2b, $rk11 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 11
ldp $res2q, $res3q, [$input_ptr], #32 @ AES block 8k+10, 8k+11 - load ciphertext
aese $ctr7b, $rk11 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 11
ext $t11.16b, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment
aese $ctr5b, $rk11 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 11
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+17
aese $ctr3b, $rk11 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 11
aese $ctr2b, $rk12 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 12
aese $ctr7b, $rk12 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 12
aese $ctr6b, $rk12 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 12
rev32 $h3.16b, $rtmp_ctr.16b @ CTR block 8k+18
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+18
pmull $t12.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid
eor3 $acc_mb, $acc_mb, $acc_hb, $acc_lb @ MODULO - karatsuba tidy up
aese $ctr1b, $rk12 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 12
aese $ctr4b, $rk11 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 11
ldr $rk14q, [$cc, #224] @ load rk14
aese $ctr5b, $rk12 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 12
aese $ctr3b, $rk12 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 12
eor3 $acc_mb, $acc_mb, $t12.16b, $t11.16b @ MODULO - fold into mid
aese $ctr0b, $rk12 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 12
aese $ctr4b, $rk12 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 12
ldp $res4q, $res5q, [$input_ptr], #32 @ AES block 8k+12, 8k+13 - load ciphertext
aese $ctr1b, $rk13 @ AES block 8k+9 - round 13
aese $ctr2b, $rk13 @ AES block 8k+10 - round 13
ldp $res6q, $res7q, [$input_ptr], #32 @ AES block 8k+14, 8k+15 - load ciphertext
aese $ctr0b, $rk13 @ AES block 8k+8 - round 13
aese $ctr5b, $rk13 @ AES block 8k+13 - round 13
rev32 $h4.16b, $rtmp_ctr.16b @ CTR block 8k+19
eor3 $ctr2b, $res2b, $ctr2b, $rk14 @ AES block 8k+10 - result
eor3 $ctr1b, $res1b, $ctr1b, $rk14 @ AES block 8k+9 - result
ext $t11.16b, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment
aese $ctr7b, $rk13 @ AES block 8k+15 - round 13
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+19
pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low
aese $ctr4b, $rk13 @ AES block 8k+12 - round 13
eor3 $ctr5b, $res5b, $ctr5b, $rk14 @ AES block 8k+13 - result
eor3 $ctr0b, $res0b, $ctr0b, $rk14 @ AES block 8k+8 - result
aese $ctr3b, $rk13 @ AES block 8k+11 - round 13
stp $ctr0q, $ctr1q, [$output_ptr], #32 @ AES block 8k+8, 8k+9 - store result
mov $ctr0.16b, $h1.16b @ CTR block 8k+16
eor3 $ctr4b, $res4b, $ctr4b, $rk14 @ AES block 8k+12 - result
eor3 $acc_lb, $acc_lb, $t11.16b, $acc_hb @ MODULO - fold into low
eor3 $ctr3b, $res3b, $ctr3b, $rk14 @ AES block 8k+11 - result
stp $ctr2q, $ctr3q, [$output_ptr], #32 @ AES block 8k+10, 8k+11 - store result
mov $ctr3.16b, $h4.16b @ CTR block 8k+19
mov $ctr2.16b, $h3.16b @ CTR block 8k+18
aese $ctr6b, $rk13 @ AES block 8k+14 - round 13
mov $ctr1.16b, $h2.16b @ CTR block 8k+17
stp $ctr4q, $ctr5q, [$output_ptr], #32 @ AES block 8k+12, 8k+13 - store result
eor3 $ctr7b, $res7b, $ctr7b, $rk14 @ AES block 8k+15 - result
eor3 $ctr6b, $res6b, $ctr6b, $rk14 @ AES block 8k+14 - result
rev32 $ctr4.16b, $rtmp_ctr.16b @ CTR block 8k+20
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+20
cmp $input_ptr, $main_end_input_ptr @ LOOP CONTROL
stp $ctr6q, $ctr7q, [$output_ptr], #32 @ AES block 8k+14, 8k+15 - store result
b.lt .L256_dec_main_loop
.L256_dec_prepretail: @ PREPRETAIL
ldp $rk0q, $rk1q, [$cc, #0] @ load rk0, rk1
rev32 $ctr5.16b, $rtmp_ctr.16b @ CTR block 8k+13
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+13
rev64 $res4b, $res4b @ GHASH block 8k+4
ldr $h56kq, [$current_tag, #144] @ load h6k | h5k
ldr $h78kq, [$current_tag, #192] @ load h8k | h7k
rev32 $ctr6.16b, $rtmp_ctr.16b @ CTR block 8k+14
rev64 $res0b, $res0b @ GHASH block 8k
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+14
ext $acc_lb, $acc_lb, $acc_lb, #8 @ PRE 0
ldr $h7q, [$current_tag, #176] @ load h7l | h7h
ext $h7.16b, $h7.16b, $h7.16b, #8
ldr $h8q, [$current_tag, #208] @ load h8l | h8h
ext $h8.16b, $h8.16b, $h8.16b, #8
rev64 $res1b, $res1b @ GHASH block 8k+1
rev32 $ctr7.16b, $rtmp_ctr.16b @ CTR block 8k+15
rev64 $res2b, $res2b @ GHASH block 8k+2
ldr $h5q, [$current_tag, #128] @ load h5l | h5h
ext $h5.16b, $h5.16b, $h5.16b, #8
ldr $h6q, [$current_tag, #160] @ load h6l | h6h
ext $h6.16b, $h6.16b, $h6.16b, #8
aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 0
aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 0
aese $ctr4b, $rk0 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 0
aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 0
aese $ctr5b, $rk0 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 0
aese $ctr6b, $rk0 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 0
aese $ctr4b, $rk1 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 1
aese $ctr7b, $rk0 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 0
aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 0
ldp $rk2q, $rk3q, [$cc, #32] @ load rk2, rk3
aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 1
eor $res0b, $res0b, $acc_lb @ PRE 1
aese $ctr7b, $rk1 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 1
aese $ctr6b, $rk1 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 1
aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 1
aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 1
aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 1
aese $ctr5b, $rk1 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 1
pmull2 $t0.1q, $res1.2d, $h7.2d @ GHASH block 8k+1 - high
trn1 $acc_m.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid
pmull $acc_l.1q, $res0.1d, $h8.1d @ GHASH block 8k - low
rev64 $res3b, $res3b @ GHASH block 8k+3
pmull $h7.1q, $res1.1d, $h7.1d @ GHASH block 8k+1 - low
aese $ctr5b, $rk2 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 2
aese $ctr7b, $rk2 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 2
aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 2
aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 2
aese $ctr6b, $rk2 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 2
pmull2 $acc_h.1q, $res0.2d, $h8.2d @ GHASH block 8k - high
aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 2
aese $ctr7b, $rk3 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 3
aese $ctr5b, $rk3 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 3
rev64 $res6b, $res6b @ GHASH block 8k+6
aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 3
aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 2
aese $ctr6b, $rk3 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 3
pmull2 $t1.1q, $res2.2d, $h6.2d @ GHASH block 8k+2 - high
trn2 $res0.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid
aese $ctr4b, $rk2 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 2
ldp $rk4q, $rk5q, [$cc, #64] @ load rk4, rk5
aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 3
pmull2 $t2.1q, $res3.2d, $h5.2d @ GHASH block 8k+3 - high
aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 3
eor $acc_hb, $acc_hb, $t0.16b @ GHASH block 8k+1 - high
eor $res0.16b, $res0.16b, $acc_m.16b @ GHASH block 8k, 8k+1 - mid
aese $ctr4b, $rk3 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 3
pmull $h6.1q, $res2.1d, $h6.1d @ GHASH block 8k+2 - low
aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 3
eor3 $acc_hb, $acc_hb, $t1.16b, $t2.16b @ GHASH block 8k+2, 8k+3 - high
trn1 $t3.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid
trn2 $res2.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid
pmull2 $acc_m.1q, $res0.2d, $h78k.2d @ GHASH block 8k - mid
pmull $h5.1q, $res3.1d, $h5.1d @ GHASH block 8k+3 - low
eor $acc_lb, $acc_lb, $h7.16b @ GHASH block 8k+1 - low
pmull $h78k.1q, $res0.1d, $h78k.1d @ GHASH block 8k+1 - mid
aese $ctr5b, $rk4 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 4
aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 4
eor3 $acc_lb, $acc_lb, $h6.16b, $h5.16b @ GHASH block 8k+2, 8k+3 - low
ldr $h1q, [$current_tag, #32] @ load h1l | h1h
ext $h1.16b, $h1.16b, $h1.16b, #8
ldr $h2q, [$current_tag, #64] @ load h2l | h2h
ext $h2.16b, $h2.16b, $h2.16b, #8
aese $ctr7b, $rk4 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 4
aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 4
aese $ctr6b, $rk4 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 4
eor $acc_mb, $acc_mb, $h78k.16b @ GHASH block 8k+1 - mid
eor $res2.16b, $res2.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid
aese $ctr7b, $rk5 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 5
aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 4
aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 5
aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 4
aese $ctr4b, $rk4 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 4
aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 5
pmull2 $t3.1q, $res2.2d, $h56k.2d @ GHASH block 8k+2 - mid
aese $ctr6b, $rk5 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 5
aese $ctr4b, $rk5 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 5
aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 5
pmull $h56k.1q, $res2.1d, $h56k.1d @ GHASH block 8k+3 - mid
aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 5
aese $ctr5b, $rk5 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 5
ldp $rk6q, $rk7q, [$cc, #96] @ load rk6, rk7
ldr $h3q, [$current_tag, #80] @ load h3l | h3h
ext $h3.16b, $h3.16b, $h3.16b, #8
ldr $h4q, [$current_tag, #112] @ load h4l | h4h
ext $h4.16b, $h4.16b, $h4.16b, #8
rev64 $res7b, $res7b @ GHASH block 8k+7
rev64 $res5b, $res5b @ GHASH block 8k+5
eor3 $acc_mb, $acc_mb, $h56k.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid
trn1 $t6.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid
aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 6
ldr $h12kq, [$current_tag, #48] @ load h2k | h1k
ldr $h34kq, [$current_tag, #96] @ load h4k | h3k
aese $ctr6b, $rk6 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 6
aese $ctr5b, $rk6 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 6
aese $ctr7b, $rk6 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 6
pmull2 $t4.1q, $res4.2d, $h4.2d @ GHASH block 8k+4 - high
pmull2 $t5.1q, $res5.2d, $h3.2d @ GHASH block 8k+5 - high
pmull $h4.1q, $res4.1d, $h4.1d @ GHASH block 8k+4 - low
trn2 $res4.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid
pmull $h3.1q, $res5.1d, $h3.1d @ GHASH block 8k+5 - low
trn1 $t9.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid
aese $ctr7b, $rk7 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 7
pmull2 $t7.1q, $res6.2d, $h2.2d @ GHASH block 8k+6 - high
aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 6
aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 6
aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 6
aese $ctr4b, $rk6 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 6
ldp $rk8q, $rk9q, [$cc, #128] @ load rk8, rk9
pmull $h2.1q, $res6.1d, $h2.1d @ GHASH block 8k+6 - low
aese $ctr5b, $rk7 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 7
aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 7
aese $ctr4b, $rk7 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 7
aese $ctr6b, $rk7 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 7
aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 7
eor3 $acc_hb, $acc_hb, $t4.16b, $t5.16b @ GHASH block 8k+4, 8k+5 - high
aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 7
trn2 $res6.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid
aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 7
aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 8
aese $ctr7b, $rk8 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 8
aese $ctr4b, $rk8 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 8
aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 8
aese $ctr5b, $rk8 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 8
aese $ctr6b, $rk8 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 8
aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 8
aese $ctr4b, $rk9 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 9
eor $res4.16b, $res4.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid
aese $ctr0b, $rk9 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 9
aese $ctr1b, $rk9 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 9
eor $res6.16b, $res6.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid
aese $ctr6b, $rk9 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 9
aese $ctr7b, $rk9 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 9
pmull2 $t6.1q, $res4.2d, $h34k.2d @ GHASH block 8k+4 - mid
aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 8
pmull $h34k.1q, $res4.1d, $h34k.1d @ GHASH block 8k+5 - mid
pmull2 $t8.1q, $res7.2d, $h1.2d @ GHASH block 8k+7 - high
pmull2 $t9.1q, $res6.2d, $h12k.2d @ GHASH block 8k+6 - mid
pmull $h12k.1q, $res6.1d, $h12k.1d @ GHASH block 8k+7 - mid
pmull $h1.1q, $res7.1d, $h1.1d @ GHASH block 8k+7 - low
ldp $rk10q, $rk11q, [$cc, #160] @ load rk10, rk11
eor3 $acc_lb, $acc_lb, $h4.16b, $h3.16b @ GHASH block 8k+4, 8k+5 - low
eor3 $acc_mb, $acc_mb, $h34k.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid
aese $ctr2b, $rk9 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 9
aese $ctr3b, $rk9 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 9
aese $ctr5b, $rk9 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 9
eor3 $acc_hb, $acc_hb, $t7.16b, $t8.16b @ GHASH block 8k+6, 8k+7 - high
eor3 $acc_lb, $acc_lb, $h2.16b, $h1.16b @ GHASH block 8k+6, 8k+7 - low
ldr $mod_constantd, [$modulo_constant] @ MODULO - load modulo constant
eor3 $acc_mb, $acc_mb, $h12k.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid
aese $ctr4b, $rk10 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 10
aese $ctr6b, $rk10 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 10
aese $ctr5b, $rk10 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 10
aese $ctr0b, $rk10 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 10
aese $ctr2b, $rk10 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 10
aese $ctr3b, $rk10 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 10
eor3 $acc_mb, $acc_mb, $acc_hb, $acc_lb @ MODULO - karatsuba tidy up
aese $ctr7b, $rk10 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 10
aese $ctr1b, $rk10 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 10
ldp $rk12q, $rk13q, [$cc, #192] @ load rk12, rk13
ext $t11.16b, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment
aese $ctr2b, $rk11 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 11
aese $ctr1b, $rk11 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 11
aese $ctr0b, $rk11 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 11
pmull $t12.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid
aese $ctr3b, $rk11 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 11
aese $ctr7b, $rk11 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 11
aese $ctr6b, $rk11 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 11
aese $ctr4b, $rk11 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 11
aese $ctr5b, $rk11 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 11
aese $ctr3b, $rk12 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 12
eor3 $acc_mb, $acc_mb, $t12.16b, $t11.16b @ MODULO - fold into mid
aese $ctr3b, $rk13 @ AES block 8k+11 - round 13
aese $ctr2b, $rk12 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 12
aese $ctr6b, $rk12 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 12
pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low
aese $ctr4b, $rk12 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 12
aese $ctr7b, $rk12 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 12
aese $ctr0b, $rk12 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 12
ldr $rk14q, [$cc, #224] @ load rk14
aese $ctr1b, $rk12 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 12
aese $ctr4b, $rk13 @ AES block 8k+12 - round 13
ext $t11.16b, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment
aese $ctr5b, $rk12 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 12
aese $ctr6b, $rk13 @ AES block 8k+14 - round 13
aese $ctr2b, $rk13 @ AES block 8k+10 - round 13
aese $ctr1b, $rk13 @ AES block 8k+9 - round 13
aese $ctr5b, $rk13 @ AES block 8k+13 - round 13
eor3 $acc_lb, $acc_lb, $t11.16b, $acc_hb @ MODULO - fold into low
add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+15
aese $ctr7b, $rk13 @ AES block 8k+15 - round 13
aese $ctr0b, $rk13 @ AES block 8k+8 - round 13
.L256_dec_tail: @ TAIL
ext $t0.16b, $acc_lb, $acc_lb, #8 @ prepare final partial tag
sub $main_end_input_ptr, $end_input_ptr, $input_ptr @ main_end_input_ptr is number of bytes left to process
cmp $main_end_input_ptr, #112
ldr $res1q, [$input_ptr], #16 @ AES block 8k+8 - load ciphertext
ldp $h78kq, $h8q, [$current_tag, #192] @ load h8k | h7k
ext $h8.16b, $h8.16b, $h8.16b, #8
mov $t1.16b, $rk14
ldp $h5q, $h56kq, [$current_tag, #128] @ load h5l | h5h
ext $h5.16b, $h5.16b, $h5.16b, #8
eor3 $res4b, $res1b, $ctr0b, $t1.16b @ AES block 8k+8 - result
ldp $h6q, $h7q, [$current_tag, #160] @ load h6l | h6h
ext $h6.16b, $h6.16b, $h6.16b, #8
ext $h7.16b, $h7.16b, $h7.16b, #8
b.gt .L256_dec_blocks_more_than_7
mov $ctr7b, $ctr6b
sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
mov $ctr6b, $ctr5b
mov $ctr5b, $ctr4b
mov $ctr4b, $ctr3b
movi $acc_l.8b, #0
movi $acc_h.8b, #0
movi $acc_m.8b, #0
mov $ctr3b, $ctr2b
cmp $main_end_input_ptr, #96
mov $ctr2b, $ctr1b
b.gt .L256_dec_blocks_more_than_6
mov $ctr7b, $ctr6b
mov $ctr6b, $ctr5b
mov $ctr5b, $ctr4b
cmp $main_end_input_ptr, #80
sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
mov $ctr4b, $ctr3b
mov $ctr3b, $ctr1b
b.gt .L256_dec_blocks_more_than_5
cmp $main_end_input_ptr, #64
mov $ctr7b, $ctr6b
sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
mov $ctr6b, $ctr5b
mov $ctr5b, $ctr4b
mov $ctr4b, $ctr1b
b.gt .L256_dec_blocks_more_than_4
sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
mov $ctr7b, $ctr6b
cmp $main_end_input_ptr, #48
mov $ctr6b, $ctr5b
mov $ctr5b, $ctr1b
b.gt .L256_dec_blocks_more_than_3
ldr $h34kq, [$current_tag, #96] @ load h4k | h3k
sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
mov $ctr7b, $ctr6b
cmp $main_end_input_ptr, #32
mov $ctr6b, $ctr1b
b.gt .L256_dec_blocks_more_than_2
sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
mov $ctr7b, $ctr1b
cmp $main_end_input_ptr, #16
b.gt .L256_dec_blocks_more_than_1
sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
ldr $h12kq, [$current_tag, #48] @ load h2k | h1k
b .L256_dec_blocks_less_than_1
.L256_dec_blocks_more_than_7: @ blocks left > 7
rev64 $res0b, $res1b @ GHASH final-7 block
ldr $res1q, [$input_ptr], #16 @ AES final-6 block - load ciphertext
st1 { $res4b}, [$output_ptr], #16 @ AES final-7 block - store result
ins $acc_m.d[0], $h78k.d[1] @ GHASH final-7 block - mid
eor $res0b, $res0b, $t0.16b @ feed in partial tag
ins $rk4v.d[0], $res0.d[1] @ GHASH final-7 block - mid
eor3 $res4b, $res1b, $ctr1b, $t1.16b @ AES final-6 block - result
pmull2 $acc_h.1q, $res0.2d, $h8.2d @ GHASH final-7 block - high
eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-7 block - mid
movi $t0.8b, #0 @ supress further partial tag feed in
pmull $acc_l.1q, $res0.1d, $h8.1d @ GHASH final-7 block - low
pmull $acc_m.1q, $rk4v.1d, $acc_m.1d @ GHASH final-7 block - mid
.L256_dec_blocks_more_than_6: @ blocks left > 6
rev64 $res0b, $res1b @ GHASH final-6 block
eor $res0b, $res0b, $t0.16b @ feed in partial tag
ldr $res1q, [$input_ptr], #16 @ AES final-5 block - load ciphertext
movi $t0.8b, #0 @ supress further partial tag feed in
ins $rk4v.d[0], $res0.d[1] @ GHASH final-6 block - mid
st1 { $res4b}, [$output_ptr], #16 @ AES final-6 block - store result
pmull2 $rk2q1, $res0.2d, $h7.2d @ GHASH final-6 block - high
pmull $rk3q1, $res0.1d, $h7.1d @ GHASH final-6 block - low
eor3 $res4b, $res1b, $ctr2b, $t1.16b @ AES final-5 block - result
eor $acc_lb, $acc_lb, $rk3 @ GHASH final-6 block - low
eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-6 block - mid
pmull $rk4v.1q, $rk4v.1d, $h78k.1d @ GHASH final-6 block - mid
eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-6 block - mid
eor $acc_hb, $acc_hb, $rk2 @ GHASH final-6 block - high
.L256_dec_blocks_more_than_5: @ blocks left > 5
rev64 $res0b, $res1b @ GHASH final-5 block
eor $res0b, $res0b, $t0.16b @ feed in partial tag
pmull2 $rk2q1, $res0.2d, $h6.2d @ GHASH final-5 block - high
ins $rk4v.d[0], $res0.d[1] @ GHASH final-5 block - mid
ldr $res1q, [$input_ptr], #16 @ AES final-4 block - load ciphertext
eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-5 block - mid
st1 { $res4b}, [$output_ptr], #16 @ AES final-5 block - store result
pmull $rk3q1, $res0.1d, $h6.1d @ GHASH final-5 block - low
ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-5 block - mid
pmull2 $rk4v.1q, $rk4v.2d, $h56k.2d @ GHASH final-5 block - mid
eor $acc_hb, $acc_hb, $rk2 @ GHASH final-5 block - high
eor3 $res4b, $res1b, $ctr3b, $t1.16b @ AES final-4 block - result
eor $acc_lb, $acc_lb, $rk3 @ GHASH final-5 block - low
eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-5 block - mid
movi $t0.8b, #0 @ supress further partial tag feed in
.L256_dec_blocks_more_than_4: @ blocks left > 4
rev64 $res0b, $res1b @ GHASH final-4 block
eor $res0b, $res0b, $t0.16b @ feed in partial tag
ins $rk4v.d[0], $res0.d[1] @ GHASH final-4 block - mid
ldr $res1q, [$input_ptr], #16 @ AES final-3 block - load ciphertext
movi $t0.8b, #0 @ supress further partial tag feed in
pmull $rk3q1, $res0.1d, $h5.1d @ GHASH final-4 block - low
pmull2 $rk2q1, $res0.2d, $h5.2d @ GHASH final-4 block - high
eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-4 block - mid
eor $acc_hb, $acc_hb, $rk2 @ GHASH final-4 block - high
pmull $rk4v.1q, $rk4v.1d, $h56k.1d @ GHASH final-4 block - mid
eor $acc_lb, $acc_lb, $rk3 @ GHASH final-4 block - low
st1 { $res4b}, [$output_ptr], #16 @ AES final-4 block - store result
eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-4 block - mid
eor3 $res4b, $res1b, $ctr4b, $t1.16b @ AES final-3 block - result
.L256_dec_blocks_more_than_3: @ blocks left > 3
ldr $h4q, [$current_tag, #112] @ load h4l | h4h
ext $h4.16b, $h4.16b, $h4.16b, #8
rev64 $res0b, $res1b @ GHASH final-3 block
eor $res0b, $res0b, $t0.16b @ feed in partial tag
ldr $res1q, [$input_ptr], #16 @ AES final-2 block - load ciphertext
ldr $h34kq, [$current_tag, #96] @ load h4k | h3k
ins $rk4v.d[0], $res0.d[1] @ GHASH final-3 block - mid
st1 { $res4b}, [$output_ptr], #16 @ AES final-3 block - store result
eor3 $res4b, $res1b, $ctr5b, $t1.16b @ AES final-2 block - result
eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-3 block - mid
ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-3 block - mid
pmull $rk3q1, $res0.1d, $h4.1d @ GHASH final-3 block - low
pmull2 $rk2q1, $res0.2d, $h4.2d @ GHASH final-3 block - high
movi $t0.8b, #0 @ supress further partial tag feed in
pmull2 $rk4v.1q, $rk4v.2d, $h34k.2d @ GHASH final-3 block - mid
eor $acc_lb, $acc_lb, $rk3 @ GHASH final-3 block - low
eor $acc_hb, $acc_hb, $rk2 @ GHASH final-3 block - high
eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-3 block - mid
.L256_dec_blocks_more_than_2: @ blocks left > 2
rev64 $res0b, $res1b @ GHASH final-2 block
ldr $h3q, [$current_tag, #80] @ load h3l | h3h
ext $h3.16b, $h3.16b, $h3.16b, #8
ldr $res1q, [$input_ptr], #16 @ AES final-1 block - load ciphertext
eor $res0b, $res0b, $t0.16b @ feed in partial tag
ins $rk4v.d[0], $res0.d[1] @ GHASH final-2 block - mid
pmull $rk3q1, $res0.1d, $h3.1d @ GHASH final-2 block - low
st1 { $res4b}, [$output_ptr], #16 @ AES final-2 block - store result
eor3 $res4b, $res1b, $ctr6b, $t1.16b @ AES final-1 block - result
eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-2 block - mid
eor $acc_lb, $acc_lb, $rk3 @ GHASH final-2 block - low
movi $t0.8b, #0 @ supress further partial tag feed in
pmull $rk4v.1q, $rk4v.1d, $h34k.1d @ GHASH final-2 block - mid
pmull2 $rk2q1, $res0.2d, $h3.2d @ GHASH final-2 block - high
eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-2 block - mid
eor $acc_hb, $acc_hb, $rk2 @ GHASH final-2 block - high
.L256_dec_blocks_more_than_1: @ blocks left > 1
rev64 $res0b, $res1b @ GHASH final-1 block
eor $res0b, $res0b, $t0.16b @ feed in partial tag
ins $rk4v.d[0], $res0.d[1] @ GHASH final-1 block - mid
ldr $h2q, [$current_tag, #64] @ load h2l | h2h
ext $h2.16b, $h2.16b, $h2.16b, #8
eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-1 block - mid
ldr $res1q, [$input_ptr], #16 @ AES final block - load ciphertext
st1 { $res4b}, [$output_ptr], #16 @ AES final-1 block - store result
ldr $h12kq, [$current_tag, #48] @ load h2k | h1k
pmull $rk3q1, $res0.1d, $h2.1d @ GHASH final-1 block - low
ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-1 block - mid
eor $acc_lb, $acc_lb, $rk3 @ GHASH final-1 block - low
eor3 $res4b, $res1b, $ctr7b, $t1.16b @ AES final block - result
pmull2 $rk2q1, $res0.2d, $h2.2d @ GHASH final-1 block - high
pmull2 $rk4v.1q, $rk4v.2d, $h12k.2d @ GHASH final-1 block - mid
movi $t0.8b, #0 @ supress further partial tag feed in
eor $acc_hb, $acc_hb, $rk2 @ GHASH final-1 block - high
eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-1 block - mid
.L256_dec_blocks_less_than_1: @ blocks left <= 1
ld1 { $rk0}, [$output_ptr] @ load existing bytes where the possibly partial last block is to be stored
mvn $temp0_x, xzr @ temp0_x = 0xffffffffffffffff
and $bit_length, $bit_length, #127 @ bit_length %= 128
sub $bit_length, $bit_length, #128 @ bit_length -= 128
rev32 $rtmp_ctr.16b, $rtmp_ctr.16b
str $rtmp_ctrq, [$counter] @ store the updated counter
neg $bit_length, $bit_length @ bit_length = 128 - #bits in input (in range [1,128])
and $bit_length, $bit_length, #127 @ bit_length %= 128
lsr $temp0_x, $temp0_x, $bit_length @ temp0_x is mask for top 64b of last block
cmp $bit_length, #64
mvn $temp1_x, xzr @ temp1_x = 0xffffffffffffffff
csel $temp3_x, $temp0_x, xzr, lt
csel $temp2_x, $temp1_x, $temp0_x, lt
mov $ctr0.d[0], $temp2_x @ ctr0b is mask for last block
mov $ctr0.d[1], $temp3_x
and $res1b, $res1b, $ctr0b @ possibly partial last block has zeroes in highest bits
ldr $h1q, [$current_tag, #32] @ load h1l | h1h
ext $h1.16b, $h1.16b, $h1.16b, #8
bif $res4b, $rk0, $ctr0b @ insert existing bytes in top end of result before storing
rev64 $res0b, $res1b @ GHASH final block
eor $res0b, $res0b, $t0.16b @ feed in partial tag
ins $t0.d[0], $res0.d[1] @ GHASH final block - mid
pmull2 $rk2q1, $res0.2d, $h1.2d @ GHASH final block - high
eor $t0.8b, $t0.8b, $res0.8b @ GHASH final block - mid
pmull $rk3q1, $res0.1d, $h1.1d @ GHASH final block - low
eor $acc_hb, $acc_hb, $rk2 @ GHASH final block - high
pmull $t0.1q, $t0.1d, $h12k.1d @ GHASH final block - mid
eor $acc_mb, $acc_mb, $t0.16b @ GHASH final block - mid
ldr $mod_constantd, [$modulo_constant] @ MODULO - load modulo constant
eor $acc_lb, $acc_lb, $rk3 @ GHASH final block - low
pmull $t11.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid
eor $t10.16b, $acc_hb, $acc_lb @ MODULO - karatsuba tidy up
ext $acc_hb, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment
st1 { $res4b}, [$output_ptr] @ store all 16B
eor $acc_mb, $acc_mb, $t10.16b @ MODULO - karatsuba tidy up
eor $t11.16b, $acc_hb, $t11.16b @ MODULO - fold into mid
eor $acc_mb, $acc_mb, $t11.16b @ MODULO - fold into mid
pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low
ext $acc_mb, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment
eor $acc_lb, $acc_lb, $acc_hb @ MODULO - fold into low
eor $acc_lb, $acc_lb, $acc_mb @ MODULO - fold into low
ext $acc_lb, $acc_lb, $acc_lb, #8
rev64 $acc_lb, $acc_lb
st1 { $acc_l.16b }, [$current_tag]
lsr x0, $bit_length, #3 @ return sizes
ldp d10, d11, [sp, #16]
ldp d12, d13, [sp, #32]
ldp d14, d15, [sp, #48]
ldp d8, d9, [sp], #80
ret
.L256_dec_ret:
mov w0, #0x0
ret
.size unroll8_eor3_aes_gcm_dec_256_kernel,.-unroll8_eor3_aes_gcm_dec_256_kernel
___
}
}
$code.=<<___;
.asciz "AES GCM module for ARMv8, SPDX BSD-3-Clause by <xiaokang.qian\@arm.com>"
.align 2
#endif
___
{
my %opcode = (
"rax1" => 0xce608c00, "eor3" => 0xce000000,
"bcax" => 0xce200000, "xar" => 0xce800000 );
sub unsha3 {
my ($mnemonic,$arg)=@_;
$arg =~ m/[qv]([0-9]+)[^,]*,\s*[qv]([0-9]+)[^,]*(?:,\s*[qv]([0-9]+)[^,]*(?:,\s*[qv#]([0-9\-]+))?)?/
&&
sprintf ".inst\t0x%08x\t//%s %s",
$opcode{$mnemonic}|$1|($2<<5)|($3<<16)|(eval($4)<<10),
$mnemonic,$arg;
}
sub unvmov {
my $arg=shift;
$arg =~ m/q([0-9]+)#(lo|hi),\s*q([0-9]+)#(lo|hi)/o &&
sprintf "ins v%d.d[%d],v%d.d[%d]",$1<8?$1:$1+8,($2 eq "lo")?0:1,
$3<8?$3:$3+8,($4 eq "lo")?0:1;
}
foreach(split("\n",$code)) {
s/@\s/\/\//o; # old->new style commentary
s/\`([^\`]*)\`/eval($1)/ge;
m/\bld1r\b/ and s/\.16b/.2d/g or
s/\b(eor3|rax1|xar|bcax)\s+(v.*)/unsha3($1,$2)/ge;
print $_,"\n";
}
}
close STDOUT or die "error closing STDOUT: $!"; # enforce flush
|