Viewing file:  aesv8-armx.pl (81.21 KB) -rwxr-xr-x
Select action/file-type:
 (+) |  (+) |  (+) | Code (+) | Session (+) |  (+) | SDB (+) |  (+) |  (+) |  (+) |  (+) |  (+) |
#! /usr/bin/env perl
# Copyright 2014-2022 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the Apache License 2.0 (the "License"). You may not use
# this file except in compliance with the License. You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html
#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
# This module implements support for ARMv8 AES instructions. The
# module is endian-agnostic in sense that it supports both big- and
# little-endian cases. As does it support both 32- and 64-bit modes
# of operation. Latter is achieved by limiting amount of utilized
# registers to 16, which implies additional NEON load and integer
# instructions. This has no effect on mighty Apple A7, where results
# are literally equal to the theoretical estimates based on AES
# instruction latencies and issue rates. On Cortex-A53, an in-order
# execution core, this costs up to 10-15%, which is partially
# compensated by implementing dedicated code path for 128-bit
# CBC encrypt case. On Cortex-A57 parallelizable mode performance
# seems to be limited by sheer amount of NEON instructions...
#
# April 2019
#
# Key to performance of parallelize-able modes is round instruction
# interleaving. But which factor to use? There is optimal one for
# each combination of instruction latency and issue rate, beyond
# which increasing interleave factor doesn't pay off. While on cons
# side we have code size increase and resource waste on platforms for
# which interleave factor is too high. In other words you want it to
# be just right. So far interleave factor of 3x was serving well all
# platforms. But for ThunderX2 optimal interleave factor was measured
# to be 5x...
#
# Performance in cycles per byte processed with 128-bit key:
#
# CBC enc CBC dec CTR
# Apple A7 2.39 1.20 1.20
# Cortex-A53 1.32 1.17/1.29(**) 1.36/1.46
# Cortex-A57(*) 1.95 0.82/0.85 0.89/0.93
# Cortex-A72 1.33 0.85/0.88 0.92/0.96
# Denver 1.96 0.65/0.86 0.76/0.80
# Mongoose 1.33 1.23/1.20 1.30/1.20
# Kryo 1.26 0.87/0.94 1.00/1.00
# ThunderX2 5.95 1.25 1.30
#
# (*) original 3.64/1.34/1.32 results were for r0p0 revision
# and are still same even for updated module;
# (**) numbers after slash are for 32-bit code, which is 3x-
# interleaved;
# $output is the last argument if it looks like a file (it has an extension)
# $flavour is the first argument if it doesn't look like a file
$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
$flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
die "can't locate arm-xlate.pl";
open OUT,"| \"$^X\" $xlate $flavour \"$output\""
or die "can't call $xlate: $!";
*STDOUT=*OUT;
$prefix="aes_v8";
$_byte = ($flavour =~ /win/ ? "DCB" : ".byte");
$code=<<___;
#include "arm_arch.h"
#if __ARM_MAX_ARCH__>=7
___
$code.=".arch armv8-a+crypto\n.text\n" if ($flavour =~ /64/);
$code.=<<___ if ($flavour !~ /64/);
.arch armv7-a // don't confuse not-so-latest binutils with argv8 :-)
.fpu neon
#ifdef __thumb2__
.syntax unified
.thumb
# define INST(a,b,c,d) $_byte c,d|0xc,a,b
#else
.code 32
# define INST(a,b,c,d) $_byte a,b,c,d
#endif
.text
___
# Assembler mnemonics are an eclectic mix of 32- and 64-bit syntax,
# NEON is mostly 32-bit mnemonics, integer - mostly 64. Goal is to
# maintain both 32- and 64-bit codes within single module and
# transliterate common code to either flavour with regex vodoo.
#
{{{
my ($inp,$bits,$out,$ptr,$rounds)=("x0","w1","x2","x3","w12");
my ($zero,$rcon,$mask,$in0,$in1,$tmp,$key)=
$flavour=~/64/? map("q$_",(0..6)) : map("q$_",(0..3,8..10));
$code.=<<___;
.align 5
.Lrcon:
.long 0x01,0x01,0x01,0x01
.long 0x0c0f0e0d,0x0c0f0e0d,0x0c0f0e0d,0x0c0f0e0d // rotate-n-splat
.long 0x1b,0x1b,0x1b,0x1b
.globl ${prefix}_set_encrypt_key
.type ${prefix}_set_encrypt_key,%function
.align 5
${prefix}_set_encrypt_key:
.Lenc_key:
___
$code.=<<___ if ($flavour =~ /64/);
AARCH64_VALID_CALL_TARGET
// Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later.
stp x29,x30,[sp,#-16]!
add x29,sp,#0
___
$code.=<<___;
mov $ptr,#-1
cmp $inp,#0
b.eq .Lenc_key_abort
cmp $out,#0
b.eq .Lenc_key_abort
mov $ptr,#-2
cmp $bits,#128
b.lt .Lenc_key_abort
cmp $bits,#256
b.gt .Lenc_key_abort
tst $bits,#0x3f
b.ne .Lenc_key_abort
adr $ptr,.Lrcon
cmp $bits,#192
veor $zero,$zero,$zero
vld1.8 {$in0},[$inp],#16
mov $bits,#8 // reuse $bits
vld1.32 {$rcon,$mask},[$ptr],#32
b.lt .Loop128
b.eq .L192
b .L256
.align 4
.Loop128:
vtbl.8 $key,{$in0},$mask
vext.8 $tmp,$zero,$in0,#12
vst1.32 {$in0},[$out],#16
aese $key,$zero
subs $bits,$bits,#1
veor $in0,$in0,$tmp
vext.8 $tmp,$zero,$tmp,#12
veor $in0,$in0,$tmp
vext.8 $tmp,$zero,$tmp,#12
veor $key,$key,$rcon
veor $in0,$in0,$tmp
vshl.u8 $rcon,$rcon,#1
veor $in0,$in0,$key
b.ne .Loop128
vld1.32 {$rcon},[$ptr]
vtbl.8 $key,{$in0},$mask
vext.8 $tmp,$zero,$in0,#12
vst1.32 {$in0},[$out],#16
aese $key,$zero
veor $in0,$in0,$tmp
vext.8 $tmp,$zero,$tmp,#12
veor $in0,$in0,$tmp
vext.8 $tmp,$zero,$tmp,#12
veor $key,$key,$rcon
veor $in0,$in0,$tmp
vshl.u8 $rcon,$rcon,#1
veor $in0,$in0,$key
vtbl.8 $key,{$in0},$mask
vext.8 $tmp,$zero,$in0,#12
vst1.32 {$in0},[$out],#16
aese $key,$zero
veor $in0,$in0,$tmp
vext.8 $tmp,$zero,$tmp,#12
veor $in0,$in0,$tmp
vext.8 $tmp,$zero,$tmp,#12
veor $key,$key,$rcon
veor $in0,$in0,$tmp
veor $in0,$in0,$key
vst1.32 {$in0},[$out]
add $out,$out,#0x50
mov $rounds,#10
b .Ldone
.align 4
.L192:
vld1.8 {$in1},[$inp],#8
vmov.i8 $key,#8 // borrow $key
vst1.32 {$in0},[$out],#16
vsub.i8 $mask,$mask,$key // adjust the mask
.Loop192:
vtbl.8 $key,{$in1},$mask
vext.8 $tmp,$zero,$in0,#12
#ifdef __ARMEB__
vst1.32 {$in1},[$out],#16
sub $out,$out,#8
#else
vst1.32 {$in1},[$out],#8
#endif
aese $key,$zero
subs $bits,$bits,#1
veor $in0,$in0,$tmp
vext.8 $tmp,$zero,$tmp,#12
veor $in0,$in0,$tmp
vext.8 $tmp,$zero,$tmp,#12
veor $in0,$in0,$tmp
vdup.32 $tmp,${in0}[3]
veor $tmp,$tmp,$in1
veor $key,$key,$rcon
vext.8 $in1,$zero,$in1,#12
vshl.u8 $rcon,$rcon,#1
veor $in1,$in1,$tmp
veor $in0,$in0,$key
veor $in1,$in1,$key
vst1.32 {$in0},[$out],#16
b.ne .Loop192
mov $rounds,#12
add $out,$out,#0x20
b .Ldone
.align 4
.L256:
vld1.8 {$in1},[$inp]
mov $bits,#7
mov $rounds,#14
vst1.32 {$in0},[$out],#16
.Loop256:
vtbl.8 $key,{$in1},$mask
vext.8 $tmp,$zero,$in0,#12
vst1.32 {$in1},[$out],#16
aese $key,$zero
subs $bits,$bits,#1
veor $in0,$in0,$tmp
vext.8 $tmp,$zero,$tmp,#12
veor $in0,$in0,$tmp
vext.8 $tmp,$zero,$tmp,#12
veor $key,$key,$rcon
veor $in0,$in0,$tmp
vshl.u8 $rcon,$rcon,#1
veor $in0,$in0,$key
vst1.32 {$in0},[$out],#16
b.eq .Ldone
vdup.32 $key,${in0}[3] // just splat
vext.8 $tmp,$zero,$in1,#12
aese $key,$zero
veor $in1,$in1,$tmp
vext.8 $tmp,$zero,$tmp,#12
veor $in1,$in1,$tmp
vext.8 $tmp,$zero,$tmp,#12
veor $in1,$in1,$tmp
veor $in1,$in1,$key
b .Loop256
.Ldone:
str $rounds,[$out]
mov $ptr,#0
.Lenc_key_abort:
mov x0,$ptr // return value
`"ldr x29,[sp],#16" if ($flavour =~ /64/)`
ret
.size ${prefix}_set_encrypt_key,.-${prefix}_set_encrypt_key
.globl ${prefix}_set_decrypt_key
.type ${prefix}_set_decrypt_key,%function
.align 5
${prefix}_set_decrypt_key:
___
$code.=<<___ if ($flavour =~ /64/);
AARCH64_SIGN_LINK_REGISTER
stp x29,x30,[sp,#-16]!
add x29,sp,#0
___
$code.=<<___ if ($flavour !~ /64/);
stmdb sp!,{r4,lr}
___
$code.=<<___;
bl .Lenc_key
cmp x0,#0
b.ne .Ldec_key_abort
sub $out,$out,#240 // restore original $out
mov x4,#-16
add $inp,$out,x12,lsl#4 // end of key schedule
vld1.32 {v0.16b},[$out]
vld1.32 {v1.16b},[$inp]
vst1.32 {v0.16b},[$inp],x4
vst1.32 {v1.16b},[$out],#16
.Loop_imc:
vld1.32 {v0.16b},[$out]
vld1.32 {v1.16b},[$inp]
aesimc v0.16b,v0.16b
aesimc v1.16b,v1.16b
vst1.32 {v0.16b},[$inp],x4
vst1.32 {v1.16b},[$out],#16
cmp $inp,$out
b.hi .Loop_imc
vld1.32 {v0.16b},[$out]
aesimc v0.16b,v0.16b
vst1.32 {v0.16b},[$inp]
eor x0,x0,x0 // return value
.Ldec_key_abort:
___
$code.=<<___ if ($flavour !~ /64/);
ldmia sp!,{r4,pc}
___
$code.=<<___ if ($flavour =~ /64/);
ldp x29,x30,[sp],#16
AARCH64_VALIDATE_LINK_REGISTER
ret
___
$code.=<<___;
.size ${prefix}_set_decrypt_key,.-${prefix}_set_decrypt_key
___
}}}
{{{
sub gen_block () {
my $dir = shift;
my ($e,$mc) = $dir eq "en" ? ("e","mc") : ("d","imc");
my ($inp,$out,$key)=map("x$_",(0..2));
my $rounds="w3";
my ($rndkey0,$rndkey1,$inout)=map("q$_",(0..3));
$code.=<<___;
.globl ${prefix}_${dir}crypt
.type ${prefix}_${dir}crypt,%function
.align 5
${prefix}_${dir}crypt:
___
$code.=<<___ if ($flavour =~ /64/);
AARCH64_VALID_CALL_TARGET
___
$code.=<<___;
ldr $rounds,[$key,#240]
vld1.32 {$rndkey0},[$key],#16
vld1.8 {$inout},[$inp]
sub $rounds,$rounds,#2
vld1.32 {$rndkey1},[$key],#16
.Loop_${dir}c:
aes$e $inout,$rndkey0
aes$mc $inout,$inout
vld1.32 {$rndkey0},[$key],#16
subs $rounds,$rounds,#2
aes$e $inout,$rndkey1
aes$mc $inout,$inout
vld1.32 {$rndkey1},[$key],#16
b.gt .Loop_${dir}c
aes$e $inout,$rndkey0
aes$mc $inout,$inout
vld1.32 {$rndkey0},[$key]
aes$e $inout,$rndkey1
veor $inout,$inout,$rndkey0
vst1.8 {$inout},[$out]
ret
.size ${prefix}_${dir}crypt,.-${prefix}_${dir}crypt
___
}
&gen_block("en");
&gen_block("de");
}}}
# Performance in cycles per byte.
# Processed with AES-ECB different key size.
# It shows the value before and after optimization as below:
# (before/after):
#
# AES-128-ECB AES-192-ECB AES-256-ECB
# Cortex-A57 1.85/0.82 2.16/0.96 2.47/1.10
# Cortex-A72 1.64/0.85 1.82/0.99 2.13/1.14
# Optimization is implemented by loop unrolling and interleaving.
# Commonly, we choose the unrolling factor as 5, if the input
# data size smaller than 5 blocks, but not smaller than 3 blocks,
# choose 3 as the unrolling factor.
# If the input data size dsize >= 5*16 bytes, then take 5 blocks
# as one iteration, every loop the left size lsize -= 5*16.
# If 5*16 > lsize >= 3*16 bytes, take 3 blocks as one iteration,
# every loop lsize -=3*16.
# If lsize < 3*16 bytes, treat them as the tail, interleave the
# two blocks AES instructions.
# There is one special case, if the original input data size dsize
# = 16 bytes, we will treat it separately to improve the
# performance: one independent code block without LR, FP load and
# store, just looks like what the original ECB implementation does.
{{{
my ($inp,$out,$len,$key)=map("x$_",(0..3));
my ($enc,$rounds,$cnt,$key_,$step)=("w4","w5","w6","x7","x8");
my ($dat0,$dat1,$in0,$in1,$tmp0,$tmp1,$tmp2,$rndlast)=map("q$_",(0..7));
my ($dat,$tmp,$rndzero_n_last)=($dat0,$tmp0,$tmp1);
### q7 last round key
### q10-q15 q7 Last 7 round keys
### q8-q9 preloaded round keys except last 7 keys for big size
### q5, q6, q8-q9 preloaded round keys except last 7 keys for only 16 byte
{
my ($dat2,$in2,$tmp2)=map("q$_",(10,11,9));
my ($dat3,$in3,$tmp3); # used only in 64-bit mode
my ($dat4,$in4,$tmp4);
if ($flavour =~ /64/) {
($dat2,$dat3,$dat4,$in2,$in3,$in4,$tmp3,$tmp4)=map("q$_",(16..23));
}
$code.=<<___;
.globl ${prefix}_ecb_encrypt
.type ${prefix}_ecb_encrypt,%function
.align 5
${prefix}_ecb_encrypt:
___
$code.=<<___ if ($flavour =~ /64/);
AARCH64_VALID_CALL_TARGET
subs $len,$len,#16
// Original input data size bigger than 16, jump to big size processing.
b.ne .Lecb_big_size
vld1.8 {$dat0},[$inp]
cmp $enc,#0 // en- or decrypting?
ldr $rounds,[$key,#240]
vld1.32 {q5-q6},[$key],#32 // load key schedule...
b.eq .Lecb_small_dec
aese $dat0,q5
aesmc $dat0,$dat0
vld1.32 {q8-q9},[$key],#32 // load key schedule...
aese $dat0,q6
aesmc $dat0,$dat0
subs $rounds,$rounds,#10 // if rounds==10, jump to aes-128-ecb processing
b.eq .Lecb_128_enc
.Lecb_round_loop:
aese $dat0,q8
aesmc $dat0,$dat0
vld1.32 {q8},[$key],#16 // load key schedule...
aese $dat0,q9
aesmc $dat0,$dat0
vld1.32 {q9},[$key],#16 // load key schedule...
subs $rounds,$rounds,#2 // bias
b.gt .Lecb_round_loop
.Lecb_128_enc:
vld1.32 {q10-q11},[$key],#32 // load key schedule...
aese $dat0,q8
aesmc $dat0,$dat0
aese $dat0,q9
aesmc $dat0,$dat0
vld1.32 {q12-q13},[$key],#32 // load key schedule...
aese $dat0,q10
aesmc $dat0,$dat0
aese $dat0,q11
aesmc $dat0,$dat0
vld1.32 {q14-q15},[$key],#32 // load key schedule...
aese $dat0,q12
aesmc $dat0,$dat0
aese $dat0,q13
aesmc $dat0,$dat0
vld1.32 {$rndlast},[$key]
aese $dat0,q14
aesmc $dat0,$dat0
aese $dat0,q15
veor $dat0,$dat0,$rndlast
vst1.8 {$dat0},[$out]
b .Lecb_Final_abort
.Lecb_small_dec:
aesd $dat0,q5
aesimc $dat0,$dat0
vld1.32 {q8-q9},[$key],#32 // load key schedule...
aesd $dat0,q6
aesimc $dat0,$dat0
subs $rounds,$rounds,#10 // bias
b.eq .Lecb_128_dec
.Lecb_dec_round_loop:
aesd $dat0,q8
aesimc $dat0,$dat0
vld1.32 {q8},[$key],#16 // load key schedule...
aesd $dat0,q9
aesimc $dat0,$dat0
vld1.32 {q9},[$key],#16 // load key schedule...
subs $rounds,$rounds,#2 // bias
b.gt .Lecb_dec_round_loop
.Lecb_128_dec:
vld1.32 {q10-q11},[$key],#32 // load key schedule...
aesd $dat0,q8
aesimc $dat0,$dat0
aesd $dat0,q9
aesimc $dat0,$dat0
vld1.32 {q12-q13},[$key],#32 // load key schedule...
aesd $dat0,q10
aesimc $dat0,$dat0
aesd $dat0,q11
aesimc $dat0,$dat0
vld1.32 {q14-q15},[$key],#32 // load key schedule...
aesd $dat0,q12
aesimc $dat0,$dat0
aesd $dat0,q13
aesimc $dat0,$dat0
vld1.32 {$rndlast},[$key]
aesd $dat0,q14
aesimc $dat0,$dat0
aesd $dat0,q15
veor $dat0,$dat0,$rndlast
vst1.8 {$dat0},[$out]
b .Lecb_Final_abort
.Lecb_big_size:
___
$code.=<<___ if ($flavour =~ /64/);
stp x29,x30,[sp,#-16]!
add x29,sp,#0
___
$code.=<<___ if ($flavour !~ /64/);
mov ip,sp
stmdb sp!,{r4-r8,lr}
vstmdb sp!,{d8-d15} @ ABI specification says so
ldmia ip,{r4-r5} @ load remaining args
subs $len,$len,#16
___
$code.=<<___;
mov $step,#16
b.lo .Lecb_done
cclr $step,eq
cmp $enc,#0 // en- or decrypting?
ldr $rounds,[$key,#240]
and $len,$len,#-16
vld1.8 {$dat},[$inp],$step
vld1.32 {q8-q9},[$key] // load key schedule...
sub $rounds,$rounds,#6
add $key_,$key,x5,lsl#4 // pointer to last 7 round keys
sub $rounds,$rounds,#2
vld1.32 {q10-q11},[$key_],#32
vld1.32 {q12-q13},[$key_],#32
vld1.32 {q14-q15},[$key_],#32
vld1.32 {$rndlast},[$key_]
add $key_,$key,#32
mov $cnt,$rounds
b.eq .Lecb_dec
vld1.8 {$dat1},[$inp],#16
subs $len,$len,#32 // bias
add $cnt,$rounds,#2
vorr $in1,$dat1,$dat1
vorr $dat2,$dat1,$dat1
vorr $dat1,$dat,$dat
b.lo .Lecb_enc_tail
vorr $dat1,$in1,$in1
vld1.8 {$dat2},[$inp],#16
___
$code.=<<___ if ($flavour =~ /64/);
cmp $len,#32
b.lo .Loop3x_ecb_enc
vld1.8 {$dat3},[$inp],#16
vld1.8 {$dat4},[$inp],#16
sub $len,$len,#32 // bias
mov $cnt,$rounds
.Loop5x_ecb_enc:
aese $dat0,q8
aesmc $dat0,$dat0
aese $dat1,q8
aesmc $dat1,$dat1
aese $dat2,q8
aesmc $dat2,$dat2
aese $dat3,q8
aesmc $dat3,$dat3
aese $dat4,q8
aesmc $dat4,$dat4
vld1.32 {q8},[$key_],#16
subs $cnt,$cnt,#2
aese $dat0,q9
aesmc $dat0,$dat0
aese $dat1,q9
aesmc $dat1,$dat1
aese $dat2,q9
aesmc $dat2,$dat2
aese $dat3,q9
aesmc $dat3,$dat3
aese $dat4,q9
aesmc $dat4,$dat4
vld1.32 {q9},[$key_],#16
b.gt .Loop5x_ecb_enc
aese $dat0,q8
aesmc $dat0,$dat0
aese $dat1,q8
aesmc $dat1,$dat1
aese $dat2,q8
aesmc $dat2,$dat2
aese $dat3,q8
aesmc $dat3,$dat3
aese $dat4,q8
aesmc $dat4,$dat4
cmp $len,#0x40 // because .Lecb_enc_tail4x
sub $len,$len,#0x50
aese $dat0,q9
aesmc $dat0,$dat0
aese $dat1,q9
aesmc $dat1,$dat1
aese $dat2,q9
aesmc $dat2,$dat2
aese $dat3,q9
aesmc $dat3,$dat3
aese $dat4,q9
aesmc $dat4,$dat4
csel x6,xzr,$len,gt // borrow x6, $cnt, "gt" is not typo
mov $key_,$key
aese $dat0,q10
aesmc $dat0,$dat0
aese $dat1,q10
aesmc $dat1,$dat1
aese $dat2,q10
aesmc $dat2,$dat2
aese $dat3,q10
aesmc $dat3,$dat3
aese $dat4,q10
aesmc $dat4,$dat4
add $inp,$inp,x6 // $inp is adjusted in such way that
// at exit from the loop $dat1-$dat4
// are loaded with last "words"
add x6,$len,#0x60 // because .Lecb_enc_tail4x
aese $dat0,q11
aesmc $dat0,$dat0
aese $dat1,q11
aesmc $dat1,$dat1
aese $dat2,q11
aesmc $dat2,$dat2
aese $dat3,q11
aesmc $dat3,$dat3
aese $dat4,q11
aesmc $dat4,$dat4
aese $dat0,q12
aesmc $dat0,$dat0
aese $dat1,q12
aesmc $dat1,$dat1
aese $dat2,q12
aesmc $dat2,$dat2
aese $dat3,q12
aesmc $dat3,$dat3
aese $dat4,q12
aesmc $dat4,$dat4
aese $dat0,q13
aesmc $dat0,$dat0
aese $dat1,q13
aesmc $dat1,$dat1
aese $dat2,q13
aesmc $dat2,$dat2
aese $dat3,q13
aesmc $dat3,$dat3
aese $dat4,q13
aesmc $dat4,$dat4
aese $dat0,q14
aesmc $dat0,$dat0
aese $dat1,q14
aesmc $dat1,$dat1
aese $dat2,q14
aesmc $dat2,$dat2
aese $dat3,q14
aesmc $dat3,$dat3
aese $dat4,q14
aesmc $dat4,$dat4
aese $dat0,q15
vld1.8 {$in0},[$inp],#16
aese $dat1,q15
vld1.8 {$in1},[$inp],#16
aese $dat2,q15
vld1.8 {$in2},[$inp],#16
aese $dat3,q15
vld1.8 {$in3},[$inp],#16
aese $dat4,q15
vld1.8 {$in4},[$inp],#16
cbz x6,.Lecb_enc_tail4x
vld1.32 {q8},[$key_],#16 // re-pre-load rndkey[0]
veor $tmp0,$rndlast,$dat0
vorr $dat0,$in0,$in0
veor $tmp1,$rndlast,$dat1
vorr $dat1,$in1,$in1
veor $tmp2,$rndlast,$dat2
vorr $dat2,$in2,$in2
veor $tmp3,$rndlast,$dat3
vorr $dat3,$in3,$in3
veor $tmp4,$rndlast,$dat4
vst1.8 {$tmp0},[$out],#16
vorr $dat4,$in4,$in4
vst1.8 {$tmp1},[$out],#16
mov $cnt,$rounds
vst1.8 {$tmp2},[$out],#16
vld1.32 {q9},[$key_],#16 // re-pre-load rndkey[1]
vst1.8 {$tmp3},[$out],#16
vst1.8 {$tmp4},[$out],#16
b.hs .Loop5x_ecb_enc
add $len,$len,#0x50
cbz $len,.Lecb_done
add $cnt,$rounds,#2
subs $len,$len,#0x30
vorr $dat0,$in2,$in2
vorr $dat1,$in3,$in3
vorr $dat2,$in4,$in4
b.lo .Lecb_enc_tail
b .Loop3x_ecb_enc
.align 4
.Lecb_enc_tail4x:
veor $tmp1,$rndlast,$dat1
veor $tmp2,$rndlast,$dat2
veor $tmp3,$rndlast,$dat3
veor $tmp4,$rndlast,$dat4
vst1.8 {$tmp1},[$out],#16
vst1.8 {$tmp2},[$out],#16
vst1.8 {$tmp3},[$out],#16
vst1.8 {$tmp4},[$out],#16
b .Lecb_done
.align 4
___
$code.=<<___;
.Loop3x_ecb_enc:
aese $dat0,q8
aesmc $dat0,$dat0
aese $dat1,q8
aesmc $dat1,$dat1
aese $dat2,q8
aesmc $dat2,$dat2
vld1.32 {q8},[$key_],#16
subs $cnt,$cnt,#2
aese $dat0,q9
aesmc $dat0,$dat0
aese $dat1,q9
aesmc $dat1,$dat1
aese $dat2,q9
aesmc $dat2,$dat2
vld1.32 {q9},[$key_],#16
b.gt .Loop3x_ecb_enc
aese $dat0,q8
aesmc $dat0,$dat0
aese $dat1,q8
aesmc $dat1,$dat1
aese $dat2,q8
aesmc $dat2,$dat2
subs $len,$len,#0x30
mov.lo x6,$len // x6, $cnt, is zero at this point
aese $dat0,q9
aesmc $dat0,$dat0
aese $dat1,q9
aesmc $dat1,$dat1
aese $dat2,q9
aesmc $dat2,$dat2
add $inp,$inp,x6 // $inp is adjusted in such way that
// at exit from the loop $dat1-$dat2
// are loaded with last "words"
mov $key_,$key
aese $dat0,q12
aesmc $dat0,$dat0
aese $dat1,q12
aesmc $dat1,$dat1
aese $dat2,q12
aesmc $dat2,$dat2
vld1.8 {$in0},[$inp],#16
aese $dat0,q13
aesmc $dat0,$dat0
aese $dat1,q13
aesmc $dat1,$dat1
aese $dat2,q13
aesmc $dat2,$dat2
vld1.8 {$in1},[$inp],#16
aese $dat0,q14
aesmc $dat0,$dat0
aese $dat1,q14
aesmc $dat1,$dat1
aese $dat2,q14
aesmc $dat2,$dat2
vld1.8 {$in2},[$inp],#16
aese $dat0,q15
aese $dat1,q15
aese $dat2,q15
vld1.32 {q8},[$key_],#16 // re-pre-load rndkey[0]
add $cnt,$rounds,#2
veor $tmp0,$rndlast,$dat0
veor $tmp1,$rndlast,$dat1
veor $dat2,$dat2,$rndlast
vld1.32 {q9},[$key_],#16 // re-pre-load rndkey[1]
vst1.8 {$tmp0},[$out],#16
vorr $dat0,$in0,$in0
vst1.8 {$tmp1},[$out],#16
vorr $dat1,$in1,$in1
vst1.8 {$dat2},[$out],#16
vorr $dat2,$in2,$in2
b.hs .Loop3x_ecb_enc
cmn $len,#0x30
b.eq .Lecb_done
nop
.Lecb_enc_tail:
aese $dat1,q8
aesmc $dat1,$dat1
aese $dat2,q8
aesmc $dat2,$dat2
vld1.32 {q8},[$key_],#16
subs $cnt,$cnt,#2
aese $dat1,q9
aesmc $dat1,$dat1
aese $dat2,q9
aesmc $dat2,$dat2
vld1.32 {q9},[$key_],#16
b.gt .Lecb_enc_tail
aese $dat1,q8
aesmc $dat1,$dat1
aese $dat2,q8
aesmc $dat2,$dat2
aese $dat1,q9
aesmc $dat1,$dat1
aese $dat2,q9
aesmc $dat2,$dat2
aese $dat1,q12
aesmc $dat1,$dat1
aese $dat2,q12
aesmc $dat2,$dat2
cmn $len,#0x20
aese $dat1,q13
aesmc $dat1,$dat1
aese $dat2,q13
aesmc $dat2,$dat2
aese $dat1,q14
aesmc $dat1,$dat1
aese $dat2,q14
aesmc $dat2,$dat2
aese $dat1,q15
aese $dat2,q15
b.eq .Lecb_enc_one
veor $tmp1,$rndlast,$dat1
veor $tmp2,$rndlast,$dat2
vst1.8 {$tmp1},[$out],#16
vst1.8 {$tmp2},[$out],#16
b .Lecb_done
.Lecb_enc_one:
veor $tmp1,$rndlast,$dat2
vst1.8 {$tmp1},[$out],#16
b .Lecb_done
___
$code.=<<___;
.align 5
.Lecb_dec:
vld1.8 {$dat1},[$inp],#16
subs $len,$len,#32 // bias
add $cnt,$rounds,#2
vorr $in1,$dat1,$dat1
vorr $dat2,$dat1,$dat1
vorr $dat1,$dat,$dat
b.lo .Lecb_dec_tail
vorr $dat1,$in1,$in1
vld1.8 {$dat2},[$inp],#16
___
$code.=<<___ if ($flavour =~ /64/);
cmp $len,#32
b.lo .Loop3x_ecb_dec
vld1.8 {$dat3},[$inp],#16
vld1.8 {$dat4},[$inp],#16
sub $len,$len,#32 // bias
mov $cnt,$rounds
.Loop5x_ecb_dec:
aesd $dat0,q8
aesimc $dat0,$dat0
aesd $dat1,q8
aesimc $dat1,$dat1
aesd $dat2,q8
aesimc $dat2,$dat2
aesd $dat3,q8
aesimc $dat3,$dat3
aesd $dat4,q8
aesimc $dat4,$dat4
vld1.32 {q8},[$key_],#16
subs $cnt,$cnt,#2
aesd $dat0,q9
aesimc $dat0,$dat0
aesd $dat1,q9
aesimc $dat1,$dat1
aesd $dat2,q9
aesimc $dat2,$dat2
aesd $dat3,q9
aesimc $dat3,$dat3
aesd $dat4,q9
aesimc $dat4,$dat4
vld1.32 {q9},[$key_],#16
b.gt .Loop5x_ecb_dec
aesd $dat0,q8
aesimc $dat0,$dat0
aesd $dat1,q8
aesimc $dat1,$dat1
aesd $dat2,q8
aesimc $dat2,$dat2
aesd $dat3,q8
aesimc $dat3,$dat3
aesd $dat4,q8
aesimc $dat4,$dat4
cmp $len,#0x40 // because .Lecb_tail4x
sub $len,$len,#0x50
aesd $dat0,q9
aesimc $dat0,$dat0
aesd $dat1,q9
aesimc $dat1,$dat1
aesd $dat2,q9
aesimc $dat2,$dat2
aesd $dat3,q9
aesimc $dat3,$dat3
aesd $dat4,q9
aesimc $dat4,$dat4
csel x6,xzr,$len,gt // borrow x6, $cnt, "gt" is not typo
mov $key_,$key
aesd $dat0,q10
aesimc $dat0,$dat0
aesd $dat1,q10
aesimc $dat1,$dat1
aesd $dat2,q10
aesimc $dat2,$dat2
aesd $dat3,q10
aesimc $dat3,$dat3
aesd $dat4,q10
aesimc $dat4,$dat4
add $inp,$inp,x6 // $inp is adjusted in such way that
// at exit from the loop $dat1-$dat4
// are loaded with last "words"
add x6,$len,#0x60 // because .Lecb_tail4x
aesd $dat0,q11
aesimc $dat0,$dat0
aesd $dat1,q11
aesimc $dat1,$dat1
aesd $dat2,q11
aesimc $dat2,$dat2
aesd $dat3,q11
aesimc $dat3,$dat3
aesd $dat4,q11
aesimc $dat4,$dat4
aesd $dat0,q12
aesimc $dat0,$dat0
aesd $dat1,q12
aesimc $dat1,$dat1
aesd $dat2,q12
aesimc $dat2,$dat2
aesd $dat3,q12
aesimc $dat3,$dat3
aesd $dat4,q12
aesimc $dat4,$dat4
aesd $dat0,q13
aesimc $dat0,$dat0
aesd $dat1,q13
aesimc $dat1,$dat1
aesd $dat2,q13
aesimc $dat2,$dat2
aesd $dat3,q13
aesimc $dat3,$dat3
aesd $dat4,q13
aesimc $dat4,$dat4
aesd $dat0,q14
aesimc $dat0,$dat0
aesd $dat1,q14
aesimc $dat1,$dat1
aesd $dat2,q14
aesimc $dat2,$dat2
aesd $dat3,q14
aesimc $dat3,$dat3
aesd $dat4,q14
aesimc $dat4,$dat4
aesd $dat0,q15
vld1.8 {$in0},[$inp],#16
aesd $dat1,q15
vld1.8 {$in1},[$inp],#16
aesd $dat2,q15
vld1.8 {$in2},[$inp],#16
aesd $dat3,q15
vld1.8 {$in3},[$inp],#16
aesd $dat4,q15
vld1.8 {$in4},[$inp],#16
cbz x6,.Lecb_tail4x
vld1.32 {q8},[$key_],#16 // re-pre-load rndkey[0]
veor $tmp0,$rndlast,$dat0
vorr $dat0,$in0,$in0
veor $tmp1,$rndlast,$dat1
vorr $dat1,$in1,$in1
veor $tmp2,$rndlast,$dat2
vorr $dat2,$in2,$in2
veor $tmp3,$rndlast,$dat3
vorr $dat3,$in3,$in3
veor $tmp4,$rndlast,$dat4
vst1.8 {$tmp0},[$out],#16
vorr $dat4,$in4,$in4
vst1.8 {$tmp1},[$out],#16
mov $cnt,$rounds
vst1.8 {$tmp2},[$out],#16
vld1.32 {q9},[$key_],#16 // re-pre-load rndkey[1]
vst1.8 {$tmp3},[$out],#16
vst1.8 {$tmp4},[$out],#16
b.hs .Loop5x_ecb_dec
add $len,$len,#0x50
cbz $len,.Lecb_done
add $cnt,$rounds,#2
subs $len,$len,#0x30
vorr $dat0,$in2,$in2
vorr $dat1,$in3,$in3
vorr $dat2,$in4,$in4
b.lo .Lecb_dec_tail
b .Loop3x_ecb_dec
.align 4
.Lecb_tail4x:
veor $tmp1,$rndlast,$dat1
veor $tmp2,$rndlast,$dat2
veor $tmp3,$rndlast,$dat3
veor $tmp4,$rndlast,$dat4
vst1.8 {$tmp1},[$out],#16
vst1.8 {$tmp2},[$out],#16
vst1.8 {$tmp3},[$out],#16
vst1.8 {$tmp4},[$out],#16
b .Lecb_done
.align 4
___
$code.=<<___;
.Loop3x_ecb_dec:
aesd $dat0,q8
aesimc $dat0,$dat0
aesd $dat1,q8
aesimc $dat1,$dat1
aesd $dat2,q8
aesimc $dat2,$dat2
vld1.32 {q8},[$key_],#16
subs $cnt,$cnt,#2
aesd $dat0,q9
aesimc $dat0,$dat0
aesd $dat1,q9
aesimc $dat1,$dat1
aesd $dat2,q9
aesimc $dat2,$dat2
vld1.32 {q9},[$key_],#16
b.gt .Loop3x_ecb_dec
aesd $dat0,q8
aesimc $dat0,$dat0
aesd $dat1,q8
aesimc $dat1,$dat1
aesd $dat2,q8
aesimc $dat2,$dat2
subs $len,$len,#0x30
mov.lo x6,$len // x6, $cnt, is zero at this point
aesd $dat0,q9
aesimc $dat0,$dat0
aesd $dat1,q9
aesimc $dat1,$dat1
aesd $dat2,q9
aesimc $dat2,$dat2
add $inp,$inp,x6 // $inp is adjusted in such way that
// at exit from the loop $dat1-$dat2
// are loaded with last "words"
mov $key_,$key
aesd $dat0,q12
aesimc $dat0,$dat0
aesd $dat1,q12
aesimc $dat1,$dat1
aesd $dat2,q12
aesimc $dat2,$dat2
vld1.8 {$in0},[$inp],#16
aesd $dat0,q13
aesimc $dat0,$dat0
aesd $dat1,q13
aesimc $dat1,$dat1
aesd $dat2,q13
aesimc $dat2,$dat2
vld1.8 {$in1},[$inp],#16
aesd $dat0,q14
aesimc $dat0,$dat0
aesd $dat1,q14
aesimc $dat1,$dat1
aesd $dat2,q14
aesimc $dat2,$dat2
vld1.8 {$in2},[$inp],#16
aesd $dat0,q15
aesd $dat1,q15
aesd $dat2,q15
vld1.32 {q8},[$key_],#16 // re-pre-load rndkey[0]
add $cnt,$rounds,#2
veor $tmp0,$rndlast,$dat0
veor $tmp1,$rndlast,$dat1
veor $dat2,$dat2,$rndlast
vld1.32 {q9},[$key_],#16 // re-pre-load rndkey[1]
vst1.8 {$tmp0},[$out],#16
vorr $dat0,$in0,$in0
vst1.8 {$tmp1},[$out],#16
vorr $dat1,$in1,$in1
vst1.8 {$dat2},[$out],#16
vorr $dat2,$in2,$in2
b.hs .Loop3x_ecb_dec
cmn $len,#0x30
b.eq .Lecb_done
nop
.Lecb_dec_tail:
aesd $dat1,q8
aesimc $dat1,$dat1
aesd $dat2,q8
aesimc $dat2,$dat2
vld1.32 {q8},[$key_],#16
subs $cnt,$cnt,#2
aesd $dat1,q9
aesimc $dat1,$dat1
aesd $dat2,q9
aesimc $dat2,$dat2
vld1.32 {q9},[$key_],#16
b.gt .Lecb_dec_tail
aesd $dat1,q8
aesimc $dat1,$dat1
aesd $dat2,q8
aesimc $dat2,$dat2
aesd $dat1,q9
aesimc $dat1,$dat1
aesd $dat2,q9
aesimc $dat2,$dat2
aesd $dat1,q12
aesimc $dat1,$dat1
aesd $dat2,q12
aesimc $dat2,$dat2
cmn $len,#0x20
aesd $dat1,q13
aesimc $dat1,$dat1
aesd $dat2,q13
aesimc $dat2,$dat2
aesd $dat1,q14
aesimc $dat1,$dat1
aesd $dat2,q14
aesimc $dat2,$dat2
aesd $dat1,q15
aesd $dat2,q15
b.eq .Lecb_dec_one
veor $tmp1,$rndlast,$dat1
veor $tmp2,$rndlast,$dat2
vst1.8 {$tmp1},[$out],#16
vst1.8 {$tmp2},[$out],#16
b .Lecb_done
.Lecb_dec_one:
veor $tmp1,$rndlast,$dat2
vst1.8 {$tmp1},[$out],#16
.Lecb_done:
___
}
$code.=<<___ if ($flavour !~ /64/);
vldmia sp!,{d8-d15}
ldmia sp!,{r4-r8,pc}
___
$code.=<<___ if ($flavour =~ /64/);
ldr x29,[sp],#16
___
$code.=<<___ if ($flavour =~ /64/);
.Lecb_Final_abort:
ret
___
$code.=<<___;
.size ${prefix}_ecb_encrypt,.-${prefix}_ecb_encrypt
___
}}}
{{{
my ($inp,$out,$len,$key,$ivp)=map("x$_",(0..4)); my $enc="w5";
my ($rounds,$cnt,$key_,$step,$step1)=($enc,"w6","x7","x8","x12");
my ($dat0,$dat1,$in0,$in1,$tmp0,$tmp1,$ivec,$rndlast)=map("q$_",(0..7));
my ($dat,$tmp,$rndzero_n_last)=($dat0,$tmp0,$tmp1);
my ($key4,$key5,$key6,$key7)=("x6","x12","x14",$key);
### q8-q15 preloaded key schedule
$code.=<<___;
.globl ${prefix}_cbc_encrypt
.type ${prefix}_cbc_encrypt,%function
.align 5
${prefix}_cbc_encrypt:
___
$code.=<<___ if ($flavour =~ /64/);
AARCH64_VALID_CALL_TARGET
// Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later.
stp x29,x30,[sp,#-16]!
add x29,sp,#0
___
$code.=<<___ if ($flavour !~ /64/);
mov ip,sp
stmdb sp!,{r4-r8,lr}
vstmdb sp!,{d8-d15} @ ABI specification says so
ldmia ip,{r4-r5} @ load remaining args
___
$code.=<<___;
subs $len,$len,#16
mov $step,#16
b.lo .Lcbc_abort
cclr $step,eq
cmp $enc,#0 // en- or decrypting?
ldr $rounds,[$key,#240]
and $len,$len,#-16
vld1.8 {$ivec},[$ivp]
vld1.8 {$dat},[$inp],$step
vld1.32 {q8-q9},[$key] // load key schedule...
sub $rounds,$rounds,#6
add $key_,$key,x5,lsl#4 // pointer to last 7 round keys
sub $rounds,$rounds,#2
vld1.32 {q10-q11},[$key_],#32
vld1.32 {q12-q13},[$key_],#32
vld1.32 {q14-q15},[$key_],#32
vld1.32 {$rndlast},[$key_]
add $key_,$key,#32
mov $cnt,$rounds
b.eq .Lcbc_dec
cmp $rounds,#2
veor $dat,$dat,$ivec
veor $rndzero_n_last,q8,$rndlast
b.eq .Lcbc_enc128
vld1.32 {$in0-$in1},[$key_]
add $key_,$key,#16
add $key4,$key,#16*4
add $key5,$key,#16*5
aese $dat,q8
aesmc $dat,$dat
add $key6,$key,#16*6
add $key7,$key,#16*7
b .Lenter_cbc_enc
.align 4
.Loop_cbc_enc:
aese $dat,q8
aesmc $dat,$dat
vst1.8 {$ivec},[$out],#16
.Lenter_cbc_enc:
aese $dat,q9
aesmc $dat,$dat
aese $dat,$in0
aesmc $dat,$dat
vld1.32 {q8},[$key4]
cmp $rounds,#4
aese $dat,$in1
aesmc $dat,$dat
vld1.32 {q9},[$key5]
b.eq .Lcbc_enc192
aese $dat,q8
aesmc $dat,$dat
vld1.32 {q8},[$key6]
aese $dat,q9
aesmc $dat,$dat
vld1.32 {q9},[$key7]
nop
.Lcbc_enc192:
aese $dat,q8
aesmc $dat,$dat
subs $len,$len,#16
aese $dat,q9
aesmc $dat,$dat
cclr $step,eq
aese $dat,q10
aesmc $dat,$dat
aese $dat,q11
aesmc $dat,$dat
vld1.8 {q8},[$inp],$step
aese $dat,q12
aesmc $dat,$dat
veor q8,q8,$rndzero_n_last
aese $dat,q13
aesmc $dat,$dat
vld1.32 {q9},[$key_] // re-pre-load rndkey[1]
aese $dat,q14
aesmc $dat,$dat
aese $dat,q15
veor $ivec,$dat,$rndlast
b.hs .Loop_cbc_enc
vst1.8 {$ivec},[$out],#16
b .Lcbc_done
.align 5
.Lcbc_enc128:
vld1.32 {$in0-$in1},[$key_]
aese $dat,q8
aesmc $dat,$dat
b .Lenter_cbc_enc128
.Loop_cbc_enc128:
aese $dat,q8
aesmc $dat,$dat
vst1.8 {$ivec},[$out],#16
.Lenter_cbc_enc128:
aese $dat,q9
aesmc $dat,$dat
subs $len,$len,#16
aese $dat,$in0
aesmc $dat,$dat
cclr $step,eq
aese $dat,$in1
aesmc $dat,$dat
aese $dat,q10
aesmc $dat,$dat
aese $dat,q11
aesmc $dat,$dat
vld1.8 {q8},[$inp],$step
aese $dat,q12
aesmc $dat,$dat
aese $dat,q13
aesmc $dat,$dat
aese $dat,q14
aesmc $dat,$dat
veor q8,q8,$rndzero_n_last
aese $dat,q15
veor $ivec,$dat,$rndlast
b.hs .Loop_cbc_enc128
vst1.8 {$ivec},[$out],#16
b .Lcbc_done
___
{
my ($dat2,$in2,$tmp2)=map("q$_",(10,11,9));
my ($dat3,$in3,$tmp3); # used only in 64-bit mode
my ($dat4,$in4,$tmp4);
if ($flavour =~ /64/) {
($dat2,$dat3,$dat4,$in2,$in3,$in4,$tmp3,$tmp4)=map("q$_",(16..23));
}
$code.=<<___;
.align 5
.Lcbc_dec:
vld1.8 {$dat2},[$inp],#16
subs $len,$len,#32 // bias
add $cnt,$rounds,#2
vorr $in1,$dat,$dat
vorr $dat1,$dat,$dat
vorr $in2,$dat2,$dat2
b.lo .Lcbc_dec_tail
vorr $dat1,$dat2,$dat2
vld1.8 {$dat2},[$inp],#16
vorr $in0,$dat,$dat
vorr $in1,$dat1,$dat1
vorr $in2,$dat2,$dat2
___
$code.=<<___ if ($flavour =~ /64/);
cmp $len,#32
b.lo .Loop3x_cbc_dec
vld1.8 {$dat3},[$inp],#16
vld1.8 {$dat4},[$inp],#16
sub $len,$len,#32 // bias
mov $cnt,$rounds
vorr $in3,$dat3,$dat3
vorr $in4,$dat4,$dat4
.Loop5x_cbc_dec:
aesd $dat0,q8
aesimc $dat0,$dat0
aesd $dat1,q8
aesimc $dat1,$dat1
aesd $dat2,q8
aesimc $dat2,$dat2
aesd $dat3,q8
aesimc $dat3,$dat3
aesd $dat4,q8
aesimc $dat4,$dat4
vld1.32 {q8},[$key_],#16
subs $cnt,$cnt,#2
aesd $dat0,q9
aesimc $dat0,$dat0
aesd $dat1,q9
aesimc $dat1,$dat1
aesd $dat2,q9
aesimc $dat2,$dat2
aesd $dat3,q9
aesimc $dat3,$dat3
aesd $dat4,q9
aesimc $dat4,$dat4
vld1.32 {q9},[$key_],#16
b.gt .Loop5x_cbc_dec
aesd $dat0,q8
aesimc $dat0,$dat0
aesd $dat1,q8
aesimc $dat1,$dat1
aesd $dat2,q8
aesimc $dat2,$dat2
aesd $dat3,q8
aesimc $dat3,$dat3
aesd $dat4,q8
aesimc $dat4,$dat4
cmp $len,#0x40 // because .Lcbc_tail4x
sub $len,$len,#0x50
aesd $dat0,q9
aesimc $dat0,$dat0
aesd $dat1,q9
aesimc $dat1,$dat1
aesd $dat2,q9
aesimc $dat2,$dat2
aesd $dat3,q9
aesimc $dat3,$dat3
aesd $dat4,q9
aesimc $dat4,$dat4
csel x6,xzr,$len,gt // borrow x6, $cnt, "gt" is not typo
mov $key_,$key
aesd $dat0,q10
aesimc $dat0,$dat0
aesd $dat1,q10
aesimc $dat1,$dat1
aesd $dat2,q10
aesimc $dat2,$dat2
aesd $dat3,q10
aesimc $dat3,$dat3
aesd $dat4,q10
aesimc $dat4,$dat4
add $inp,$inp,x6 // $inp is adjusted in such way that
// at exit from the loop $dat1-$dat4
// are loaded with last "words"
add x6,$len,#0x60 // because .Lcbc_tail4x
aesd $dat0,q11
aesimc $dat0,$dat0
aesd $dat1,q11
aesimc $dat1,$dat1
aesd $dat2,q11
aesimc $dat2,$dat2
aesd $dat3,q11
aesimc $dat3,$dat3
aesd $dat4,q11
aesimc $dat4,$dat4
aesd $dat0,q12
aesimc $dat0,$dat0
aesd $dat1,q12
aesimc $dat1,$dat1
aesd $dat2,q12
aesimc $dat2,$dat2
aesd $dat3,q12
aesimc $dat3,$dat3
aesd $dat4,q12
aesimc $dat4,$dat4
aesd $dat0,q13
aesimc $dat0,$dat0
aesd $dat1,q13
aesimc $dat1,$dat1
aesd $dat2,q13
aesimc $dat2,$dat2
aesd $dat3,q13
aesimc $dat3,$dat3
aesd $dat4,q13
aesimc $dat4,$dat4
aesd $dat0,q14
aesimc $dat0,$dat0
aesd $dat1,q14
aesimc $dat1,$dat1
aesd $dat2,q14
aesimc $dat2,$dat2
aesd $dat3,q14
aesimc $dat3,$dat3
aesd $dat4,q14
aesimc $dat4,$dat4
veor $tmp0,$ivec,$rndlast
aesd $dat0,q15
veor $tmp1,$in0,$rndlast
vld1.8 {$in0},[$inp],#16
aesd $dat1,q15
veor $tmp2,$in1,$rndlast
vld1.8 {$in1},[$inp],#16
aesd $dat2,q15
veor $tmp3,$in2,$rndlast
vld1.8 {$in2},[$inp],#16
aesd $dat3,q15
veor $tmp4,$in3,$rndlast
vld1.8 {$in3},[$inp],#16
aesd $dat4,q15
vorr $ivec,$in4,$in4
vld1.8 {$in4},[$inp],#16
cbz x6,.Lcbc_tail4x
vld1.32 {q8},[$key_],#16 // re-pre-load rndkey[0]
veor $tmp0,$tmp0,$dat0
vorr $dat0,$in0,$in0
veor $tmp1,$tmp1,$dat1
vorr $dat1,$in1,$in1
veor $tmp2,$tmp2,$dat2
vorr $dat2,$in2,$in2
veor $tmp3,$tmp3,$dat3
vorr $dat3,$in3,$in3
veor $tmp4,$tmp4,$dat4
vst1.8 {$tmp0},[$out],#16
vorr $dat4,$in4,$in4
vst1.8 {$tmp1},[$out],#16
mov $cnt,$rounds
vst1.8 {$tmp2},[$out],#16
vld1.32 {q9},[$key_],#16 // re-pre-load rndkey[1]
vst1.8 {$tmp3},[$out],#16
vst1.8 {$tmp4},[$out],#16
b.hs .Loop5x_cbc_dec
add $len,$len,#0x50
cbz $len,.Lcbc_done
add $cnt,$rounds,#2
subs $len,$len,#0x30
vorr $dat0,$in2,$in2
vorr $in0,$in2,$in2
vorr $dat1,$in3,$in3
vorr $in1,$in3,$in3
vorr $dat2,$in4,$in4
vorr $in2,$in4,$in4
b.lo .Lcbc_dec_tail
b .Loop3x_cbc_dec
.align 4
.Lcbc_tail4x:
veor $tmp1,$tmp0,$dat1
veor $tmp2,$tmp2,$dat2
veor $tmp3,$tmp3,$dat3
veor $tmp4,$tmp4,$dat4
vst1.8 {$tmp1},[$out],#16
vst1.8 {$tmp2},[$out],#16
vst1.8 {$tmp3},[$out],#16
vst1.8 {$tmp4},[$out],#16
b .Lcbc_done
.align 4
___
$code.=<<___;
.Loop3x_cbc_dec:
aesd $dat0,q8
aesimc $dat0,$dat0
aesd $dat1,q8
aesimc $dat1,$dat1
aesd $dat2,q8
aesimc $dat2,$dat2
vld1.32 {q8},[$key_],#16
subs $cnt,$cnt,#2
aesd $dat0,q9
aesimc $dat0,$dat0
aesd $dat1,q9
aesimc $dat1,$dat1
aesd $dat2,q9
aesimc $dat2,$dat2
vld1.32 {q9},[$key_],#16
b.gt .Loop3x_cbc_dec
aesd $dat0,q8
aesimc $dat0,$dat0
aesd $dat1,q8
aesimc $dat1,$dat1
aesd $dat2,q8
aesimc $dat2,$dat2
veor $tmp0,$ivec,$rndlast
subs $len,$len,#0x30
veor $tmp1,$in0,$rndlast
mov.lo x6,$len // x6, $cnt, is zero at this point
aesd $dat0,q9
aesimc $dat0,$dat0
aesd $dat1,q9
aesimc $dat1,$dat1
aesd $dat2,q9
aesimc $dat2,$dat2
veor $tmp2,$in1,$rndlast
add $inp,$inp,x6 // $inp is adjusted in such way that
// at exit from the loop $dat1-$dat2
// are loaded with last "words"
vorr $ivec,$in2,$in2
mov $key_,$key
aesd $dat0,q12
aesimc $dat0,$dat0
aesd $dat1,q12
aesimc $dat1,$dat1
aesd $dat2,q12
aesimc $dat2,$dat2
vld1.8 {$in0},[$inp],#16
aesd $dat0,q13
aesimc $dat0,$dat0
aesd $dat1,q13
aesimc $dat1,$dat1
aesd $dat2,q13
aesimc $dat2,$dat2
vld1.8 {$in1},[$inp],#16
aesd $dat0,q14
aesimc $dat0,$dat0
aesd $dat1,q14
aesimc $dat1,$dat1
aesd $dat2,q14
aesimc $dat2,$dat2
vld1.8 {$in2},[$inp],#16
aesd $dat0,q15
aesd $dat1,q15
aesd $dat2,q15
vld1.32 {q8},[$key_],#16 // re-pre-load rndkey[0]
add $cnt,$rounds,#2
veor $tmp0,$tmp0,$dat0
veor $tmp1,$tmp1,$dat1
veor $dat2,$dat2,$tmp2
vld1.32 {q9},[$key_],#16 // re-pre-load rndkey[1]
vst1.8 {$tmp0},[$out],#16
vorr $dat0,$in0,$in0
vst1.8 {$tmp1},[$out],#16
vorr $dat1,$in1,$in1
vst1.8 {$dat2},[$out],#16
vorr $dat2,$in2,$in2
b.hs .Loop3x_cbc_dec
cmn $len,#0x30
b.eq .Lcbc_done
nop
.Lcbc_dec_tail:
aesd $dat1,q8
aesimc $dat1,$dat1
aesd $dat2,q8
aesimc $dat2,$dat2
vld1.32 {q8},[$key_],#16
subs $cnt,$cnt,#2
aesd $dat1,q9
aesimc $dat1,$dat1
aesd $dat2,q9
aesimc $dat2,$dat2
vld1.32 {q9},[$key_],#16
b.gt .Lcbc_dec_tail
aesd $dat1,q8
aesimc $dat1,$dat1
aesd $dat2,q8
aesimc $dat2,$dat2
aesd $dat1,q9
aesimc $dat1,$dat1
aesd $dat2,q9
aesimc $dat2,$dat2
aesd $dat1,q12
aesimc $dat1,$dat1
aesd $dat2,q12
aesimc $dat2,$dat2
cmn $len,#0x20
aesd $dat1,q13
aesimc $dat1,$dat1
aesd $dat2,q13
aesimc $dat2,$dat2
veor $tmp1,$ivec,$rndlast
aesd $dat1,q14
aesimc $dat1,$dat1
aesd $dat2,q14
aesimc $dat2,$dat2
veor $tmp2,$in1,$rndlast
aesd $dat1,q15
aesd $dat2,q15
b.eq .Lcbc_dec_one
veor $tmp1,$tmp1,$dat1
veor $tmp2,$tmp2,$dat2
vorr $ivec,$in2,$in2
vst1.8 {$tmp1},[$out],#16
vst1.8 {$tmp2},[$out],#16
b .Lcbc_done
.Lcbc_dec_one:
veor $tmp1,$tmp1,$dat2
vorr $ivec,$in2,$in2
vst1.8 {$tmp1},[$out],#16
.Lcbc_done:
vst1.8 {$ivec},[$ivp]
.Lcbc_abort:
___
}
$code.=<<___ if ($flavour !~ /64/);
vldmia sp!,{d8-d15}
ldmia sp!,{r4-r8,pc}
___
$code.=<<___ if ($flavour =~ /64/);
ldr x29,[sp],#16
ret
___
$code.=<<___;
.size ${prefix}_cbc_encrypt,.-${prefix}_cbc_encrypt
___
}}}
{{{
my ($inp,$out,$len,$key,$ivp)=map("x$_",(0..4));
my ($rounds,$cnt,$key_)=("w5","w6","x7");
my ($ctr,$tctr0,$tctr1,$tctr2)=map("w$_",(8..10,12));
my $step="x12"; # aliases with $tctr2
my ($dat0,$dat1,$in0,$in1,$tmp0,$tmp1,$ivec,$rndlast)=map("q$_",(0..7));
my ($dat2,$in2,$tmp2)=map("q$_",(10,11,9));
# used only in 64-bit mode...
my ($dat3,$dat4,$in3,$in4)=map("q$_",(16..23));
my ($dat,$tmp)=($dat0,$tmp0);
### q8-q15 preloaded key schedule
$code.=<<___;
.globl ${prefix}_ctr32_encrypt_blocks
.type ${prefix}_ctr32_encrypt_blocks,%function
.align 5
${prefix}_ctr32_encrypt_blocks:
___
$code.=<<___ if ($flavour =~ /64/);
AARCH64_VALID_CALL_TARGET
// Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later.
stp x29,x30,[sp,#-16]!
add x29,sp,#0
___
$code.=<<___ if ($flavour !~ /64/);
mov ip,sp
stmdb sp!,{r4-r10,lr}
vstmdb sp!,{d8-d15} @ ABI specification says so
ldr r4, [ip] @ load remaining arg
___
$code.=<<___;
ldr $rounds,[$key,#240]
ldr $ctr, [$ivp, #12]
#ifdef __ARMEB__
vld1.8 {$dat0},[$ivp]
#else
vld1.32 {$dat0},[$ivp]
#endif
vld1.32 {q8-q9},[$key] // load key schedule...
sub $rounds,$rounds,#4
mov $step,#16
cmp $len,#2
add $key_,$key,x5,lsl#4 // pointer to last 5 round keys
sub $rounds,$rounds,#2
vld1.32 {q12-q13},[$key_],#32
vld1.32 {q14-q15},[$key_],#32
vld1.32 {$rndlast},[$key_]
add $key_,$key,#32
mov $cnt,$rounds
cclr $step,lo
#ifndef __ARMEB__
rev $ctr, $ctr
#endif
___
$code.=<<___ if ($flavour =~ /64/);
vorr $dat1,$dat0,$dat0
add $tctr1, $ctr, #1
vorr $dat2,$dat0,$dat0
add $ctr, $ctr, #2
vorr $ivec,$dat0,$dat0
rev $tctr1, $tctr1
vmov.32 ${dat1}[3],$tctr1
b.ls .Lctr32_tail
rev $tctr2, $ctr
sub $len,$len,#3 // bias
vmov.32 ${dat2}[3],$tctr2
___
$code.=<<___ if ($flavour !~ /64/);
add $tctr1, $ctr, #1
vorr $ivec,$dat0,$dat0
rev $tctr1, $tctr1
vmov.32 ${ivec}[3],$tctr1
add $ctr, $ctr, #2
vorr $dat1,$ivec,$ivec
b.ls .Lctr32_tail
rev $tctr2, $ctr
vmov.32 ${ivec}[3],$tctr2
sub $len,$len,#3 // bias
vorr $dat2,$ivec,$ivec
___
$code.=<<___ if ($flavour =~ /64/);
cmp $len,#32
b.lo .Loop3x_ctr32
add w13,$ctr,#1
add w14,$ctr,#2
vorr $dat3,$dat0,$dat0
rev w13,w13
vorr $dat4,$dat0,$dat0
rev w14,w14
vmov.32 ${dat3}[3],w13
sub $len,$len,#2 // bias
vmov.32 ${dat4}[3],w14
add $ctr,$ctr,#2
b .Loop5x_ctr32
.align 4
.Loop5x_ctr32:
aese $dat0,q8
aesmc $dat0,$dat0
aese $dat1,q8
aesmc $dat1,$dat1
aese $dat2,q8
aesmc $dat2,$dat2
aese $dat3,q8
aesmc $dat3,$dat3
aese $dat4,q8
aesmc $dat4,$dat4
vld1.32 {q8},[$key_],#16
subs $cnt,$cnt,#2
aese $dat0,q9
aesmc $dat0,$dat0
aese $dat1,q9
aesmc $dat1,$dat1
aese $dat2,q9
aesmc $dat2,$dat2
aese $dat3,q9
aesmc $dat3,$dat3
aese $dat4,q9
aesmc $dat4,$dat4
vld1.32 {q9},[$key_],#16
b.gt .Loop5x_ctr32
mov $key_,$key
aese $dat0,q8
aesmc $dat0,$dat0
aese $dat1,q8
aesmc $dat1,$dat1
aese $dat2,q8
aesmc $dat2,$dat2
aese $dat3,q8
aesmc $dat3,$dat3
aese $dat4,q8
aesmc $dat4,$dat4
vld1.32 {q8},[$key_],#16 // re-pre-load rndkey[0]
aese $dat0,q9
aesmc $dat0,$dat0
aese $dat1,q9
aesmc $dat1,$dat1
aese $dat2,q9
aesmc $dat2,$dat2
aese $dat3,q9
aesmc $dat3,$dat3
aese $dat4,q9
aesmc $dat4,$dat4
vld1.32 {q9},[$key_],#16 // re-pre-load rndkey[1]
aese $dat0,q12
aesmc $dat0,$dat0
add $tctr0,$ctr,#1
add $tctr1,$ctr,#2
aese $dat1,q12
aesmc $dat1,$dat1
add $tctr2,$ctr,#3
add w13,$ctr,#4
aese $dat2,q12
aesmc $dat2,$dat2
add w14,$ctr,#5
rev $tctr0,$tctr0
aese $dat3,q12
aesmc $dat3,$dat3
rev $tctr1,$tctr1
rev $tctr2,$tctr2
aese $dat4,q12
aesmc $dat4,$dat4
rev w13,w13
rev w14,w14
aese $dat0,q13
aesmc $dat0,$dat0
aese $dat1,q13
aesmc $dat1,$dat1
aese $dat2,q13
aesmc $dat2,$dat2
aese $dat3,q13
aesmc $dat3,$dat3
aese $dat4,q13
aesmc $dat4,$dat4
aese $dat0,q14
aesmc $dat0,$dat0
vld1.8 {$in0},[$inp],#16
aese $dat1,q14
aesmc $dat1,$dat1
vld1.8 {$in1},[$inp],#16
aese $dat2,q14
aesmc $dat2,$dat2
vld1.8 {$in2},[$inp],#16
aese $dat3,q14
aesmc $dat3,$dat3
vld1.8 {$in3},[$inp],#16
aese $dat4,q14
aesmc $dat4,$dat4
vld1.8 {$in4},[$inp],#16
aese $dat0,q15
veor $in0,$in0,$rndlast
aese $dat1,q15
veor $in1,$in1,$rndlast
aese $dat2,q15
veor $in2,$in2,$rndlast
aese $dat3,q15
veor $in3,$in3,$rndlast
aese $dat4,q15
veor $in4,$in4,$rndlast
veor $in0,$in0,$dat0
vorr $dat0,$ivec,$ivec
veor $in1,$in1,$dat1
vorr $dat1,$ivec,$ivec
veor $in2,$in2,$dat2
vorr $dat2,$ivec,$ivec
veor $in3,$in3,$dat3
vorr $dat3,$ivec,$ivec
veor $in4,$in4,$dat4
vorr $dat4,$ivec,$ivec
vst1.8 {$in0},[$out],#16
vmov.32 ${dat0}[3],$tctr0
vst1.8 {$in1},[$out],#16
vmov.32 ${dat1}[3],$tctr1
vst1.8 {$in2},[$out],#16
vmov.32 ${dat2}[3],$tctr2
vst1.8 {$in3},[$out],#16
vmov.32 ${dat3}[3],w13
vst1.8 {$in4},[$out],#16
vmov.32 ${dat4}[3],w14
mov $cnt,$rounds
cbz $len,.Lctr32_done
add $ctr,$ctr,#5
subs $len,$len,#5
b.hs .Loop5x_ctr32
add $len,$len,#5
sub $ctr,$ctr,#5
cmp $len,#2
mov $step,#16
cclr $step,lo
b.ls .Lctr32_tail
sub $len,$len,#3 // bias
add $ctr,$ctr,#3
___
$code.=<<___;
b .Loop3x_ctr32
.align 4
.Loop3x_ctr32:
aese $dat0,q8
aesmc $dat0,$dat0
aese $dat1,q8
aesmc $dat1,$dat1
aese $dat2,q8
aesmc $dat2,$dat2
vld1.32 {q8},[$key_],#16
subs $cnt,$cnt,#2
aese $dat0,q9
aesmc $dat0,$dat0
aese $dat1,q9
aesmc $dat1,$dat1
aese $dat2,q9
aesmc $dat2,$dat2
vld1.32 {q9},[$key_],#16
b.gt .Loop3x_ctr32
aese $dat0,q8
aesmc $tmp0,$dat0
aese $dat1,q8
aesmc $tmp1,$dat1
vld1.8 {$in0},[$inp],#16
___
$code.=<<___ if ($flavour =~ /64/);
vorr $dat0,$ivec,$ivec
___
$code.=<<___ if ($flavour !~ /64/);
add $tctr0,$ctr,#1
___
$code.=<<___;
aese $dat2,q8
aesmc $dat2,$dat2
vld1.8 {$in1},[$inp],#16
___
$code.=<<___ if ($flavour =~ /64/);
vorr $dat1,$ivec,$ivec
___
$code.=<<___ if ($flavour !~ /64/);
rev $tctr0,$tctr0
___
$code.=<<___;
aese $tmp0,q9
aesmc $tmp0,$tmp0
aese $tmp1,q9
aesmc $tmp1,$tmp1
vld1.8 {$in2},[$inp],#16
mov $key_,$key
aese $dat2,q9
aesmc $tmp2,$dat2
___
$code.=<<___ if ($flavour =~ /64/);
vorr $dat2,$ivec,$ivec
add $tctr0,$ctr,#1
___
$code.=<<___;
aese $tmp0,q12
aesmc $tmp0,$tmp0
aese $tmp1,q12
aesmc $tmp1,$tmp1
veor $in0,$in0,$rndlast
add $tctr1,$ctr,#2
aese $tmp2,q12
aesmc $tmp2,$tmp2
veor $in1,$in1,$rndlast
add $ctr,$ctr,#3
aese $tmp0,q13
aesmc $tmp0,$tmp0
aese $tmp1,q13
aesmc $tmp1,$tmp1
veor $in2,$in2,$rndlast
___
$code.=<<___ if ($flavour =~ /64/);
rev $tctr0,$tctr0
aese $tmp2,q13
aesmc $tmp2,$tmp2
vmov.32 ${dat0}[3], $tctr0
___
$code.=<<___ if ($flavour !~ /64/);
vmov.32 ${ivec}[3], $tctr0
aese $tmp2,q13
aesmc $tmp2,$tmp2
vorr $dat0,$ivec,$ivec
___
$code.=<<___;
rev $tctr1,$tctr1
aese $tmp0,q14
aesmc $tmp0,$tmp0
___
$code.=<<___ if ($flavour !~ /64/);
vmov.32 ${ivec}[3], $tctr1
rev $tctr2,$ctr
___
$code.=<<___;
aese $tmp1,q14
aesmc $tmp1,$tmp1
___
$code.=<<___ if ($flavour =~ /64/);
vmov.32 ${dat1}[3], $tctr1
rev $tctr2,$ctr
aese $tmp2,q14
aesmc $tmp2,$tmp2
vmov.32 ${dat2}[3], $tctr2
___
$code.=<<___ if ($flavour !~ /64/);
vorr $dat1,$ivec,$ivec
vmov.32 ${ivec}[3], $tctr2
aese $tmp2,q14
aesmc $tmp2,$tmp2
vorr $dat2,$ivec,$ivec
___
$code.=<<___;
subs $len,$len,#3
aese $tmp0,q15
aese $tmp1,q15
aese $tmp2,q15
veor $in0,$in0,$tmp0
vld1.32 {q8},[$key_],#16 // re-pre-load rndkey[0]
vst1.8 {$in0},[$out],#16
veor $in1,$in1,$tmp1
mov $cnt,$rounds
vst1.8 {$in1},[$out],#16
veor $in2,$in2,$tmp2
vld1.32 {q9},[$key_],#16 // re-pre-load rndkey[1]
vst1.8 {$in2},[$out],#16
b.hs .Loop3x_ctr32
adds $len,$len,#3
b.eq .Lctr32_done
cmp $len,#1
mov $step,#16
cclr $step,eq
.Lctr32_tail:
aese $dat0,q8
aesmc $dat0,$dat0
aese $dat1,q8
aesmc $dat1,$dat1
vld1.32 {q8},[$key_],#16
subs $cnt,$cnt,#2
aese $dat0,q9
aesmc $dat0,$dat0
aese $dat1,q9
aesmc $dat1,$dat1
vld1.32 {q9},[$key_],#16
b.gt .Lctr32_tail
aese $dat0,q8
aesmc $dat0,$dat0
aese $dat1,q8
aesmc $dat1,$dat1
aese $dat0,q9
aesmc $dat0,$dat0
aese $dat1,q9
aesmc $dat1,$dat1
vld1.8 {$in0},[$inp],$step
aese $dat0,q12
aesmc $dat0,$dat0
aese $dat1,q12
aesmc $dat1,$dat1
vld1.8 {$in1},[$inp]
aese $dat0,q13
aesmc $dat0,$dat0
aese $dat1,q13
aesmc $dat1,$dat1
veor $in0,$in0,$rndlast
aese $dat0,q14
aesmc $dat0,$dat0
aese $dat1,q14
aesmc $dat1,$dat1
veor $in1,$in1,$rndlast
aese $dat0,q15
aese $dat1,q15
cmp $len,#1
veor $in0,$in0,$dat0
veor $in1,$in1,$dat1
vst1.8 {$in0},[$out],#16
b.eq .Lctr32_done
vst1.8 {$in1},[$out]
.Lctr32_done:
___
$code.=<<___ if ($flavour !~ /64/);
vldmia sp!,{d8-d15}
ldmia sp!,{r4-r10,pc}
___
$code.=<<___ if ($flavour =~ /64/);
ldr x29,[sp],#16
ret
___
$code.=<<___;
.size ${prefix}_ctr32_encrypt_blocks,.-${prefix}_ctr32_encrypt_blocks
___
}}}
# Performance in cycles per byte.
# Processed with AES-XTS different key size.
# It shows the value before and after optimization as below:
# (before/after):
#
# AES-128-XTS AES-256-XTS
# Cortex-A57 3.36/1.09 4.02/1.37
# Cortex-A72 3.03/1.02 3.28/1.33
# Optimization is implemented by loop unrolling and interleaving.
# Commonly, we choose the unrolling factor as 5, if the input
# data size smaller than 5 blocks, but not smaller than 3 blocks,
# choose 3 as the unrolling factor.
# If the input data size dsize >= 5*16 bytes, then take 5 blocks
# as one iteration, every loop the left size lsize -= 5*16.
# If lsize < 5*16 bytes, treat them as the tail. Note: left 4*16 bytes
# will be processed specially, which be integrated into the 5*16 bytes
# loop to improve the efficiency.
# There is one special case, if the original input data size dsize
# = 16 bytes, we will treat it separately to improve the
# performance: one independent code block without LR, FP load and
# store.
# Encryption will process the (length -tailcnt) bytes as mentioned
# previously, then encrypt the composite block as last second
# cipher block.
# Decryption will process the (length -tailcnt -1) bytes as mentioned
# previously, then decrypt the last second cipher block to get the
# last plain block(tail), decrypt the composite block as last second
# plain text block.
{{{
my ($inp,$out,$len,$key1,$key2,$ivp)=map("x$_",(0..5));
my ($rounds0,$rounds,$key_,$step,$ivl,$ivh)=("w5","w6","x7","x8","x9","x10");
my ($tmpoutp,$loutp,$l2outp,$tmpinp)=("x13","w14","w15","x20");
my ($tailcnt,$midnum,$midnumx,$constnum,$constnumx)=("x21","w22","x22","w19","x19");
my ($xoffset,$tmpmx,$tmpmw)=("x6","x11","w11");
my ($dat0,$dat1,$in0,$in1,$tmp0,$tmp1,$tmp2,$rndlast)=map("q$_",(0..7));
my ($iv0,$iv1,$iv2,$iv3,$iv4)=("v6.16b","v8.16b","v9.16b","v10.16b","v11.16b");
my ($ivd00,$ivd01,$ivd20,$ivd21)=("d6","v6.d[1]","d9","v9.d[1]");
my ($ivd10,$ivd11,$ivd30,$ivd31,$ivd40,$ivd41)=("d8","v8.d[1]","d10","v10.d[1]","d11","v11.d[1]");
my ($tmpin)=("v26.16b");
my ($dat,$tmp,$rndzero_n_last)=($dat0,$tmp0,$tmp1);
# q7 last round key
# q10-q15, q7 Last 7 round keys
# q8-q9 preloaded round keys except last 7 keys for big size
# q20, q21, q8-q9 preloaded round keys except last 7 keys for only 16 byte
my ($dat2,$in2,$tmp2)=map("q$_",(10,11,9));
my ($dat3,$in3,$tmp3); # used only in 64-bit mode
my ($dat4,$in4,$tmp4);
if ($flavour =~ /64/) {
($dat2,$dat3,$dat4,$in2,$in3,$in4,$tmp3,$tmp4)=map("q$_",(16..23));
}
$code.=<<___ if ($flavour =~ /64/);
.globl ${prefix}_xts_encrypt
.type ${prefix}_xts_encrypt,%function
.align 5
${prefix}_xts_encrypt:
___
$code.=<<___ if ($flavour =~ /64/);
AARCH64_VALID_CALL_TARGET
cmp $len,#16
// Original input data size bigger than 16, jump to big size processing.
b.ne .Lxts_enc_big_size
// Encrypt the iv with key2, as the first XEX iv.
ldr $rounds,[$key2,#240]
vld1.8 {$dat},[$key2],#16
vld1.8 {$iv0},[$ivp]
sub $rounds,$rounds,#2
vld1.8 {$dat1},[$key2],#16
.Loop_enc_iv_enc:
aese $iv0,$dat
aesmc $iv0,$iv0
vld1.32 {$dat},[$key2],#16
subs $rounds,$rounds,#2
aese $iv0,$dat1
aesmc $iv0,$iv0
vld1.32 {$dat1},[$key2],#16
b.gt .Loop_enc_iv_enc
aese $iv0,$dat
aesmc $iv0,$iv0
vld1.32 {$dat},[$key2]
aese $iv0,$dat1
veor $iv0,$iv0,$dat
vld1.8 {$dat0},[$inp]
veor $dat0,$iv0,$dat0
ldr $rounds,[$key1,#240]
vld1.32 {q20-q21},[$key1],#32 // load key schedule...
aese $dat0,q20
aesmc $dat0,$dat0
vld1.32 {q8-q9},[$key1],#32 // load key schedule...
aese $dat0,q21
aesmc $dat0,$dat0
subs $rounds,$rounds,#10 // if rounds==10, jump to aes-128-xts processing
b.eq .Lxts_128_enc
.Lxts_enc_round_loop:
aese $dat0,q8
aesmc $dat0,$dat0
vld1.32 {q8},[$key1],#16 // load key schedule...
aese $dat0,q9
aesmc $dat0,$dat0
vld1.32 {q9},[$key1],#16 // load key schedule...
subs $rounds,$rounds,#2 // bias
b.gt .Lxts_enc_round_loop
.Lxts_128_enc:
vld1.32 {q10-q11},[$key1],#32 // load key schedule...
aese $dat0,q8
aesmc $dat0,$dat0
aese $dat0,q9
aesmc $dat0,$dat0
vld1.32 {q12-q13},[$key1],#32 // load key schedule...
aese $dat0,q10
aesmc $dat0,$dat0
aese $dat0,q11
aesmc $dat0,$dat0
vld1.32 {q14-q15},[$key1],#32 // load key schedule...
aese $dat0,q12
aesmc $dat0,$dat0
aese $dat0,q13
aesmc $dat0,$dat0
vld1.32 {$rndlast},[$key1]
aese $dat0,q14
aesmc $dat0,$dat0
aese $dat0,q15
veor $dat0,$dat0,$rndlast
veor $dat0,$dat0,$iv0
vst1.8 {$dat0},[$out]
b .Lxts_enc_final_abort
.align 4
.Lxts_enc_big_size:
___
$code.=<<___ if ($flavour =~ /64/);
stp $constnumx,$tmpinp,[sp,#-64]!
stp $tailcnt,$midnumx,[sp,#48]
stp $ivd10,$ivd20,[sp,#32]
stp $ivd30,$ivd40,[sp,#16]
// tailcnt store the tail value of length%16.
and $tailcnt,$len,#0xf
and $len,$len,#-16
subs $len,$len,#16
mov $step,#16
b.lo .Lxts_abort
csel $step,xzr,$step,eq
// Firstly, encrypt the iv with key2, as the first iv of XEX.
ldr $rounds,[$key2,#240]
vld1.32 {$dat},[$key2],#16
vld1.8 {$iv0},[$ivp]
sub $rounds,$rounds,#2
vld1.32 {$dat1},[$key2],#16
.Loop_iv_enc:
aese $iv0,$dat
aesmc $iv0,$iv0
vld1.32 {$dat},[$key2],#16
subs $rounds,$rounds,#2
aese $iv0,$dat1
aesmc $iv0,$iv0
vld1.32 {$dat1},[$key2],#16
b.gt .Loop_iv_enc
aese $iv0,$dat
aesmc $iv0,$iv0
vld1.32 {$dat},[$key2]
aese $iv0,$dat1
veor $iv0,$iv0,$dat
// The iv for second block
// $ivl- iv(low), $ivh - iv(high)
// the five ivs stored into, $iv0,$iv1,$iv2,$iv3,$iv4
fmov $ivl,$ivd00
fmov $ivh,$ivd01
mov $constnum,#0x87
extr $midnumx,$ivh,$ivh,#32
extr $ivh,$ivh,$ivl,#63
and $tmpmw,$constnum,$midnum,asr#31
eor $ivl,$tmpmx,$ivl,lsl#1
fmov $ivd10,$ivl
fmov $ivd11,$ivh
ldr $rounds0,[$key1,#240] // next starting point
vld1.8 {$dat},[$inp],$step
vld1.32 {q8-q9},[$key1] // load key schedule...
sub $rounds0,$rounds0,#6
add $key_,$key1,$ivp,lsl#4 // pointer to last 7 round keys
sub $rounds0,$rounds0,#2
vld1.32 {q10-q11},[$key_],#32
vld1.32 {q12-q13},[$key_],#32
vld1.32 {q14-q15},[$key_],#32
vld1.32 {$rndlast},[$key_]
add $key_,$key1,#32
mov $rounds,$rounds0
// Encryption
.Lxts_enc:
vld1.8 {$dat2},[$inp],#16
subs $len,$len,#32 // bias
add $rounds,$rounds0,#2
vorr $in1,$dat,$dat
vorr $dat1,$dat,$dat
vorr $in3,$dat,$dat
vorr $in2,$dat2,$dat2
vorr $in4,$dat2,$dat2
b.lo .Lxts_inner_enc_tail
veor $dat,$dat,$iv0 // before encryption, xor with iv
veor $dat2,$dat2,$iv1
// The iv for third block
extr $midnumx,$ivh,$ivh,#32
extr $ivh,$ivh,$ivl,#63
and $tmpmw,$constnum,$midnum,asr#31
eor $ivl,$tmpmx,$ivl,lsl#1
fmov $ivd20,$ivl
fmov $ivd21,$ivh
vorr $dat1,$dat2,$dat2
vld1.8 {$dat2},[$inp],#16
vorr $in0,$dat,$dat
vorr $in1,$dat1,$dat1
veor $in2,$dat2,$iv2 // the third block
veor $dat2,$dat2,$iv2
cmp $len,#32
b.lo .Lxts_outer_enc_tail
// The iv for fourth block
extr $midnumx,$ivh,$ivh,#32
extr $ivh,$ivh,$ivl,#63
and $tmpmw,$constnum,$midnum,asr#31
eor $ivl,$tmpmx,$ivl,lsl#1
fmov $ivd30,$ivl
fmov $ivd31,$ivh
vld1.8 {$dat3},[$inp],#16
// The iv for fifth block
extr $midnumx,$ivh,$ivh,#32
extr $ivh,$ivh,$ivl,#63
and $tmpmw,$constnum,$midnum,asr#31
eor $ivl,$tmpmx,$ivl,lsl#1
fmov $ivd40,$ivl
fmov $ivd41,$ivh
vld1.8 {$dat4},[$inp],#16
veor $dat3,$dat3,$iv3 // the fourth block
veor $dat4,$dat4,$iv4
sub $len,$len,#32 // bias
mov $rounds,$rounds0
b .Loop5x_xts_enc
.align 4
.Loop5x_xts_enc:
aese $dat0,q8
aesmc $dat0,$dat0
aese $dat1,q8
aesmc $dat1,$dat1
aese $dat2,q8
aesmc $dat2,$dat2
aese $dat3,q8
aesmc $dat3,$dat3
aese $dat4,q8
aesmc $dat4,$dat4
vld1.32 {q8},[$key_],#16
subs $rounds,$rounds,#2
aese $dat0,q9
aesmc $dat0,$dat0
aese $dat1,q9
aesmc $dat1,$dat1
aese $dat2,q9
aesmc $dat2,$dat2
aese $dat3,q9
aesmc $dat3,$dat3
aese $dat4,q9
aesmc $dat4,$dat4
vld1.32 {q9},[$key_],#16
b.gt .Loop5x_xts_enc
aese $dat0,q8
aesmc $dat0,$dat0
aese $dat1,q8
aesmc $dat1,$dat1
aese $dat2,q8
aesmc $dat2,$dat2
aese $dat3,q8
aesmc $dat3,$dat3
aese $dat4,q8
aesmc $dat4,$dat4
subs $len,$len,#0x50 // because .Lxts_enc_tail4x
aese $dat0,q9
aesmc $dat0,$dat0
aese $dat1,q9
aesmc $dat1,$dat1
aese $dat2,q9
aesmc $dat2,$dat2
aese $dat3,q9
aesmc $dat3,$dat3
aese $dat4,q9
aesmc $dat4,$dat4
csel $xoffset,xzr,$len,gt // borrow x6, w6, "gt" is not typo
mov $key_,$key1
aese $dat0,q10
aesmc $dat0,$dat0
aese $dat1,q10
aesmc $dat1,$dat1
aese $dat2,q10
aesmc $dat2,$dat2
aese $dat3,q10
aesmc $dat3,$dat3
aese $dat4,q10
aesmc $dat4,$dat4
add $inp,$inp,$xoffset // x0 is adjusted in such way that
// at exit from the loop v1.16b-v26.16b
// are loaded with last "words"
add $xoffset,$len,#0x60 // because .Lxts_enc_tail4x
aese $dat0,q11
aesmc $dat0,$dat0
aese $dat1,q11
aesmc $dat1,$dat1
aese $dat2,q11
aesmc $dat2,$dat2
aese $dat3,q11
aesmc $dat3,$dat3
aese $dat4,q11
aesmc $dat4,$dat4
aese $dat0,q12
aesmc $dat0,$dat0
aese $dat1,q12
aesmc $dat1,$dat1
aese $dat2,q12
aesmc $dat2,$dat2
aese $dat3,q12
aesmc $dat3,$dat3
aese $dat4,q12
aesmc $dat4,$dat4
aese $dat0,q13
aesmc $dat0,$dat0
aese $dat1,q13
aesmc $dat1,$dat1
aese $dat2,q13
aesmc $dat2,$dat2
aese $dat3,q13
aesmc $dat3,$dat3
aese $dat4,q13
aesmc $dat4,$dat4
aese $dat0,q14
aesmc $dat0,$dat0
aese $dat1,q14
aesmc $dat1,$dat1
aese $dat2,q14
aesmc $dat2,$dat2
aese $dat3,q14
aesmc $dat3,$dat3
aese $dat4,q14
aesmc $dat4,$dat4
veor $tmp0,$rndlast,$iv0
aese $dat0,q15
// The iv for first block of one iteration
extr $midnumx,$ivh,$ivh,#32
extr $ivh,$ivh,$ivl,#63
and $tmpmw,$constnum,$midnum,asr#31
eor $ivl,$tmpmx,$ivl,lsl#1
fmov $ivd00,$ivl
fmov $ivd01,$ivh
veor $tmp1,$rndlast,$iv1
vld1.8 {$in0},[$inp],#16
aese $dat1,q15
// The iv for second block
extr $midnumx,$ivh,$ivh,#32
extr $ivh,$ivh,$ivl,#63
and $tmpmw,$constnum,$midnum,asr#31
eor $ivl,$tmpmx,$ivl,lsl#1
fmov $ivd10,$ivl
fmov $ivd11,$ivh
veor $tmp2,$rndlast,$iv2
vld1.8 {$in1},[$inp],#16
aese $dat2,q15
// The iv for third block
extr $midnumx,$ivh,$ivh,#32
extr $ivh,$ivh,$ivl,#63
and $tmpmw,$constnum,$midnum,asr#31
eor $ivl,$tmpmx,$ivl,lsl#1
fmov $ivd20,$ivl
fmov $ivd21,$ivh
veor $tmp3,$rndlast,$iv3
vld1.8 {$in2},[$inp],#16
aese $dat3,q15
// The iv for fourth block
extr $midnumx,$ivh,$ivh,#32
extr $ivh,$ivh,$ivl,#63
and $tmpmw,$constnum,$midnum,asr#31
eor $ivl,$tmpmx,$ivl,lsl#1
fmov $ivd30,$ivl
fmov $ivd31,$ivh
veor $tmp4,$rndlast,$iv4
vld1.8 {$in3},[$inp],#16
aese $dat4,q15
// The iv for fifth block
extr $midnumx,$ivh,$ivh,#32
extr $ivh,$ivh,$ivl,#63
and $tmpmw,$constnum,$midnum,asr #31
eor $ivl,$tmpmx,$ivl,lsl #1
fmov $ivd40,$ivl
fmov $ivd41,$ivh
vld1.8 {$in4},[$inp],#16
cbz $xoffset,.Lxts_enc_tail4x
vld1.32 {q8},[$key_],#16 // re-pre-load rndkey[0]
veor $tmp0,$tmp0,$dat0
veor $dat0,$in0,$iv0
veor $tmp1,$tmp1,$dat1
veor $dat1,$in1,$iv1
veor $tmp2,$tmp2,$dat2
veor $dat2,$in2,$iv2
veor $tmp3,$tmp3,$dat3
veor $dat3,$in3,$iv3
veor $tmp4,$tmp4,$dat4
vst1.8 {$tmp0},[$out],#16
veor $dat4,$in4,$iv4
vst1.8 {$tmp1},[$out],#16
mov $rounds,$rounds0
vst1.8 {$tmp2},[$out],#16
vld1.32 {q9},[$key_],#16 // re-pre-load rndkey[1]
vst1.8 {$tmp3},[$out],#16
vst1.8 {$tmp4},[$out],#16
b.hs .Loop5x_xts_enc
// If left 4 blocks, borrow the five block's processing.
cmn $len,#0x10
b.ne .Loop5x_enc_after
vorr $iv4,$iv3,$iv3
vorr $iv3,$iv2,$iv2
vorr $iv2,$iv1,$iv1
vorr $iv1,$iv0,$iv0
fmov $ivl,$ivd40
fmov $ivh,$ivd41
veor $dat0,$iv0,$in0
veor $dat1,$iv1,$in1
veor $dat2,$in2,$iv2
veor $dat3,$in3,$iv3
veor $dat4,$in4,$iv4
b.eq .Loop5x_xts_enc
.Loop5x_enc_after:
add $len,$len,#0x50
cbz $len,.Lxts_enc_done
add $rounds,$rounds0,#2
subs $len,$len,#0x30
b.lo .Lxts_inner_enc_tail
veor $dat0,$iv0,$in2
veor $dat1,$iv1,$in3
veor $dat2,$in4,$iv2
b .Lxts_outer_enc_tail
.align 4
.Lxts_enc_tail4x:
add $inp,$inp,#16
veor $tmp1,$dat1,$tmp1
vst1.8 {$tmp1},[$out],#16
veor $tmp2,$dat2,$tmp2
vst1.8 {$tmp2},[$out],#16
veor $tmp3,$dat3,$tmp3
veor $tmp4,$dat4,$tmp4
vst1.8 {$tmp3-$tmp4},[$out],#32
b .Lxts_enc_done
.align 4
.Lxts_outer_enc_tail:
aese $dat0,q8
aesmc $dat0,$dat0
aese $dat1,q8
aesmc $dat1,$dat1
aese $dat2,q8
aesmc $dat2,$dat2
vld1.32 {q8},[$key_],#16
subs $rounds,$rounds,#2
aese $dat0,q9
aesmc $dat0,$dat0
aese $dat1,q9
aesmc $dat1,$dat1
aese $dat2,q9
aesmc $dat2,$dat2
vld1.32 {q9},[$key_],#16
b.gt .Lxts_outer_enc_tail
aese $dat0,q8
aesmc $dat0,$dat0
aese $dat1,q8
aesmc $dat1,$dat1
aese $dat2,q8
aesmc $dat2,$dat2
veor $tmp0,$iv0,$rndlast
subs $len,$len,#0x30
// The iv for first block
fmov $ivl,$ivd20
fmov $ivh,$ivd21
//mov $constnum,#0x87
extr $midnumx,$ivh,$ivh,#32
extr $ivh,$ivh,$ivl,#63
and $tmpmw,$constnum,$midnum,asr#31
eor $ivl,$tmpmx,$ivl,lsl#1
fmov $ivd00,$ivl
fmov $ivd01,$ivh
veor $tmp1,$iv1,$rndlast
csel $xoffset,$len,$xoffset,lo // x6, w6, is zero at this point
aese $dat0,q9
aesmc $dat0,$dat0
aese $dat1,q9
aesmc $dat1,$dat1
aese $dat2,q9
aesmc $dat2,$dat2
veor $tmp2,$iv2,$rndlast
add $xoffset,$xoffset,#0x20
add $inp,$inp,$xoffset
mov $key_,$key1
aese $dat0,q12
aesmc $dat0,$dat0
aese $dat1,q12
aesmc $dat1,$dat1
aese $dat2,q12
aesmc $dat2,$dat2
aese $dat0,q13
aesmc $dat0,$dat0
aese $dat1,q13
aesmc $dat1,$dat1
aese $dat2,q13
aesmc $dat2,$dat2
aese $dat0,q14
aesmc $dat0,$dat0
aese $dat1,q14
aesmc $dat1,$dat1
aese $dat2,q14
aesmc $dat2,$dat2
aese $dat0,q15
aese $dat1,q15
aese $dat2,q15
vld1.8 {$in2},[$inp],#16
add $rounds,$rounds0,#2
vld1.32 {q8},[$key_],#16 // re-pre-load rndkey[0]
veor $tmp0,$tmp0,$dat0
veor $tmp1,$tmp1,$dat1
veor $dat2,$dat2,$tmp2
vld1.32 {q9},[$key_],#16 // re-pre-load rndkey[1]
vst1.8 {$tmp0},[$out],#16
vst1.8 {$tmp1},[$out],#16
vst1.8 {$dat2},[$out],#16
cmn $len,#0x30
b.eq .Lxts_enc_done
.Lxts_encxor_one:
vorr $in3,$in1,$in1
vorr $in4,$in2,$in2
nop
.Lxts_inner_enc_tail:
cmn $len,#0x10
veor $dat1,$in3,$iv0
veor $dat2,$in4,$iv1
b.eq .Lxts_enc_tail_loop
veor $dat2,$in4,$iv0
.Lxts_enc_tail_loop:
aese $dat1,q8
aesmc $dat1,$dat1
aese $dat2,q8
aesmc $dat2,$dat2
vld1.32 {q8},[$key_],#16
subs $rounds,$rounds,#2
aese $dat1,q9
aesmc $dat1,$dat1
aese $dat2,q9
aesmc $dat2,$dat2
vld1.32 {q9},[$key_],#16
b.gt .Lxts_enc_tail_loop
aese $dat1,q8
aesmc $dat1,$dat1
aese $dat2,q8
aesmc $dat2,$dat2
aese $dat1,q9
aesmc $dat1,$dat1
aese $dat2,q9
aesmc $dat2,$dat2
aese $dat1,q12
aesmc $dat1,$dat1
aese $dat2,q12
aesmc $dat2,$dat2
cmn $len,#0x20
aese $dat1,q13
aesmc $dat1,$dat1
aese $dat2,q13
aesmc $dat2,$dat2
veor $tmp1,$iv0,$rndlast
aese $dat1,q14
aesmc $dat1,$dat1
aese $dat2,q14
aesmc $dat2,$dat2
veor $tmp2,$iv1,$rndlast
aese $dat1,q15
aese $dat2,q15
b.eq .Lxts_enc_one
veor $tmp1,$tmp1,$dat1
vst1.8 {$tmp1},[$out],#16
veor $tmp2,$tmp2,$dat2
vorr $iv0,$iv1,$iv1
vst1.8 {$tmp2},[$out],#16
fmov $ivl,$ivd10
fmov $ivh,$ivd11
mov $constnum,#0x87
extr $midnumx,$ivh,$ivh,#32
extr $ivh,$ivh,$ivl,#63
and $tmpmw,$constnum,$midnum,asr #31
eor $ivl,$tmpmx,$ivl,lsl #1
fmov $ivd00,$ivl
fmov $ivd01,$ivh
b .Lxts_enc_done
.Lxts_enc_one:
veor $tmp1,$tmp1,$dat2
vorr $iv0,$iv0,$iv0
vst1.8 {$tmp1},[$out],#16
fmov $ivl,$ivd00
fmov $ivh,$ivd01
mov $constnum,#0x87
extr $midnumx,$ivh,$ivh,#32
extr $ivh,$ivh,$ivl,#63
and $tmpmw,$constnum,$midnum,asr #31
eor $ivl,$tmpmx,$ivl,lsl #1
fmov $ivd00,$ivl
fmov $ivd01,$ivh
b .Lxts_enc_done
.align 5
.Lxts_enc_done:
// Process the tail block with cipher stealing.
tst $tailcnt,#0xf
b.eq .Lxts_abort
mov $tmpinp,$inp
mov $tmpoutp,$out
sub $out,$out,#16
.composite_enc_loop:
subs $tailcnt,$tailcnt,#1
ldrb $l2outp,[$out,$tailcnt]
ldrb $loutp,[$tmpinp,$tailcnt]
strb $l2outp,[$tmpoutp,$tailcnt]
strb $loutp,[$out,$tailcnt]
b.gt .composite_enc_loop
.Lxts_enc_load_done:
vld1.8 {$tmpin},[$out]
veor $tmpin,$tmpin,$iv0
// Encrypt the composite block to get the last second encrypted text block
ldr $rounds,[$key1,#240] // load key schedule...
vld1.8 {$dat},[$key1],#16
sub $rounds,$rounds,#2
vld1.8 {$dat1},[$key1],#16 // load key schedule...
.Loop_final_enc:
aese $tmpin,$dat0
aesmc $tmpin,$tmpin
vld1.32 {$dat0},[$key1],#16
subs $rounds,$rounds,#2
aese $tmpin,$dat1
aesmc $tmpin,$tmpin
vld1.32 {$dat1},[$key1],#16
b.gt .Loop_final_enc
aese $tmpin,$dat0
aesmc $tmpin,$tmpin
vld1.32 {$dat0},[$key1]
aese $tmpin,$dat1
veor $tmpin,$tmpin,$dat0
veor $tmpin,$tmpin,$iv0
vst1.8 {$tmpin},[$out]
.Lxts_abort:
ldp $tailcnt,$midnumx,[sp,#48]
ldp $ivd10,$ivd20,[sp,#32]
ldp $ivd30,$ivd40,[sp,#16]
ldp $constnumx,$tmpinp,[sp],#64
.Lxts_enc_final_abort:
ret
.size ${prefix}_xts_encrypt,.-${prefix}_xts_encrypt
___
}}}
{{{
my ($inp,$out,$len,$key1,$key2,$ivp)=map("x$_",(0..5));
my ($rounds0,$rounds,$key_,$step,$ivl,$ivh)=("w5","w6","x7","x8","x9","x10");
my ($tmpoutp,$loutp,$l2outp,$tmpinp)=("x13","w14","w15","x20");
my ($tailcnt,$midnum,$midnumx,$constnum,$constnumx)=("x21","w22","x22","w19","x19");
my ($xoffset,$tmpmx,$tmpmw)=("x6","x11","w11");
my ($dat0,$dat1,$in0,$in1,$tmp0,$tmp1,$tmp2,$rndlast)=map("q$_",(0..7));
my ($iv0,$iv1,$iv2,$iv3,$iv4,$tmpin)=("v6.16b","v8.16b","v9.16b","v10.16b","v11.16b","v26.16b");
my ($ivd00,$ivd01,$ivd20,$ivd21)=("d6","v6.d[1]","d9","v9.d[1]");
my ($ivd10,$ivd11,$ivd30,$ivd31,$ivd40,$ivd41)=("d8","v8.d[1]","d10","v10.d[1]","d11","v11.d[1]");
my ($dat,$tmp,$rndzero_n_last)=($dat0,$tmp0,$tmp1);
# q7 last round key
# q10-q15, q7 Last 7 round keys
# q8-q9 preloaded round keys except last 7 keys for big size
# q20, q21, q8-q9 preloaded round keys except last 7 keys for only 16 byte
{
my ($dat2,$in2,$tmp2)=map("q$_",(10,11,9));
my ($dat3,$in3,$tmp3); # used only in 64-bit mode
my ($dat4,$in4,$tmp4);
if ($flavour =~ /64/) {
($dat2,$dat3,$dat4,$in2,$in3,$in4,$tmp3,$tmp4)=map("q$_",(16..23));
}
$code.=<<___ if ($flavour =~ /64/);
.globl ${prefix}_xts_decrypt
.type ${prefix}_xts_decrypt,%function
.align 5
${prefix}_xts_decrypt:
AARCH64_VALID_CALL_TARGET
___
$code.=<<___ if ($flavour =~ /64/);
cmp $len,#16
// Original input data size bigger than 16, jump to big size processing.
b.ne .Lxts_dec_big_size
// Encrypt the iv with key2, as the first XEX iv.
ldr $rounds,[$key2,#240]
vld1.8 {$dat},[$key2],#16
vld1.8 {$iv0},[$ivp]
sub $rounds,$rounds,#2
vld1.8 {$dat1},[$key2],#16
.Loop_dec_small_iv_enc:
aese $iv0,$dat
aesmc $iv0,$iv0
vld1.32 {$dat},[$key2],#16
subs $rounds,$rounds,#2
aese $iv0,$dat1
aesmc $iv0,$iv0
vld1.32 {$dat1},[$key2],#16
b.gt .Loop_dec_small_iv_enc
aese $iv0,$dat
aesmc $iv0,$iv0
vld1.32 {$dat},[$key2]
aese $iv0,$dat1
veor $iv0,$iv0,$dat
vld1.8 {$dat0},[$inp]
veor $dat0,$iv0,$dat0
ldr $rounds,[$key1,#240]
vld1.32 {q20-q21},[$key1],#32 // load key schedule...
aesd $dat0,q20
aesimc $dat0,$dat0
vld1.32 {q8-q9},[$key1],#32 // load key schedule...
aesd $dat0,q21
aesimc $dat0,$dat0
subs $rounds,$rounds,#10 // bias
b.eq .Lxts_128_dec
.Lxts_dec_round_loop:
aesd $dat0,q8
aesimc $dat0,$dat0
vld1.32 {q8},[$key1],#16 // load key schedule...
aesd $dat0,q9
aesimc $dat0,$dat0
vld1.32 {q9},[$key1],#16 // load key schedule...
subs $rounds,$rounds,#2 // bias
b.gt .Lxts_dec_round_loop
.Lxts_128_dec:
vld1.32 {q10-q11},[$key1],#32 // load key schedule...
aesd $dat0,q8
aesimc $dat0,$dat0
aesd $dat0,q9
aesimc $dat0,$dat0
vld1.32 {q12-q13},[$key1],#32 // load key schedule...
aesd $dat0,q10
aesimc $dat0,$dat0
aesd $dat0,q11
aesimc $dat0,$dat0
vld1.32 {q14-q15},[$key1],#32 // load key schedule...
aesd $dat0,q12
aesimc $dat0,$dat0
aesd $dat0,q13
aesimc $dat0,$dat0
vld1.32 {$rndlast},[$key1]
aesd $dat0,q14
aesimc $dat0,$dat0
aesd $dat0,q15
veor $dat0,$dat0,$rndlast
veor $dat0,$iv0,$dat0
vst1.8 {$dat0},[$out]
b .Lxts_dec_final_abort
.Lxts_dec_big_size:
___
$code.=<<___ if ($flavour =~ /64/);
stp $constnumx,$tmpinp,[sp,#-64]!
stp $tailcnt,$midnumx,[sp,#48]
stp $ivd10,$ivd20,[sp,#32]
stp $ivd30,$ivd40,[sp,#16]
and $tailcnt,$len,#0xf
and $len,$len,#-16
subs $len,$len,#16
mov $step,#16
b.lo .Lxts_dec_abort
// Encrypt the iv with key2, as the first XEX iv
ldr $rounds,[$key2,#240]
vld1.8 {$dat},[$key2],#16
vld1.8 {$iv0},[$ivp]
sub $rounds,$rounds,#2
vld1.8 {$dat1},[$key2],#16
.Loop_dec_iv_enc:
aese $iv0,$dat
aesmc $iv0,$iv0
vld1.32 {$dat},[$key2],#16
subs $rounds,$rounds,#2
aese $iv0,$dat1
aesmc $iv0,$iv0
vld1.32 {$dat1},[$key2],#16
b.gt .Loop_dec_iv_enc
aese $iv0,$dat
aesmc $iv0,$iv0
vld1.32 {$dat},[$key2]
aese $iv0,$dat1
veor $iv0,$iv0,$dat
// The iv for second block
// $ivl- iv(low), $ivh - iv(high)
// the five ivs stored into, $iv0,$iv1,$iv2,$iv3,$iv4
fmov $ivl,$ivd00
fmov $ivh,$ivd01
mov $constnum,#0x87
extr $midnumx,$ivh,$ivh,#32
extr $ivh,$ivh,$ivl,#63
and $tmpmw,$constnum,$midnum,asr #31
eor $ivl,$tmpmx,$ivl,lsl #1
fmov $ivd10,$ivl
fmov $ivd11,$ivh
ldr $rounds0,[$key1,#240] // load rounds number
// The iv for third block
extr $midnumx,$ivh,$ivh,#32
extr $ivh,$ivh,$ivl,#63
and $tmpmw,$constnum,$midnum,asr #31
eor $ivl,$tmpmx,$ivl,lsl #1
fmov $ivd20,$ivl
fmov $ivd21,$ivh
vld1.32 {q8-q9},[$key1] // load key schedule...
sub $rounds0,$rounds0,#6
add $key_,$key1,$ivp,lsl#4 // pointer to last 7 round keys
sub $rounds0,$rounds0,#2
vld1.32 {q10-q11},[$key_],#32 // load key schedule...
vld1.32 {q12-q13},[$key_],#32
vld1.32 {q14-q15},[$key_],#32
vld1.32 {$rndlast},[$key_]
// The iv for fourth block
extr $midnumx,$ivh,$ivh,#32
extr $ivh,$ivh,$ivl,#63
and $tmpmw,$constnum,$midnum,asr #31
eor $ivl,$tmpmx,$ivl,lsl #1
fmov $ivd30,$ivl
fmov $ivd31,$ivh
add $key_,$key1,#32
mov $rounds,$rounds0
b .Lxts_dec
// Decryption
.align 5
.Lxts_dec:
tst $tailcnt,#0xf
b.eq .Lxts_dec_begin
subs $len,$len,#16
csel $step,xzr,$step,eq
vld1.8 {$dat},[$inp],#16
b.lo .Lxts_done
sub $inp,$inp,#16
.Lxts_dec_begin:
vld1.8 {$dat},[$inp],$step
subs $len,$len,#32 // bias
add $rounds,$rounds0,#2
vorr $in1,$dat,$dat
vorr $dat1,$dat,$dat
vorr $in3,$dat,$dat
vld1.8 {$dat2},[$inp],#16
vorr $in2,$dat2,$dat2
vorr $in4,$dat2,$dat2
b.lo .Lxts_inner_dec_tail
veor $dat,$dat,$iv0 // before decryt, xor with iv
veor $dat2,$dat2,$iv1
vorr $dat1,$dat2,$dat2
vld1.8 {$dat2},[$inp],#16
vorr $in0,$dat,$dat
vorr $in1,$dat1,$dat1
veor $in2,$dat2,$iv2 // third block xox with third iv
veor $dat2,$dat2,$iv2
cmp $len,#32
b.lo .Lxts_outer_dec_tail
vld1.8 {$dat3},[$inp],#16
// The iv for fifth block
extr $midnumx,$ivh,$ivh,#32
extr $ivh,$ivh,$ivl,#63
and $tmpmw,$constnum,$midnum,asr #31
eor $ivl,$tmpmx,$ivl,lsl #1
fmov $ivd40,$ivl
fmov $ivd41,$ivh
vld1.8 {$dat4},[$inp],#16
veor $dat3,$dat3,$iv3 // the fourth block
veor $dat4,$dat4,$iv4
sub $len,$len,#32 // bias
mov $rounds,$rounds0
b .Loop5x_xts_dec
.align 4
.Loop5x_xts_dec:
aesd $dat0,q8
aesimc $dat0,$dat0
aesd $dat1,q8
aesimc $dat1,$dat1
aesd $dat2,q8
aesimc $dat2,$dat2
aesd $dat3,q8
aesimc $dat3,$dat3
aesd $dat4,q8
aesimc $dat4,$dat4
vld1.32 {q8},[$key_],#16 // load key schedule...
subs $rounds,$rounds,#2
aesd $dat0,q9
aesimc $dat0,$dat0
aesd $dat1,q9
aesimc $dat1,$dat1
aesd $dat2,q9
aesimc $dat2,$dat2
aesd $dat3,q9
aesimc $dat3,$dat3
aesd $dat4,q9
aesimc $dat4,$dat4
vld1.32 {q9},[$key_],#16 // load key schedule...
b.gt .Loop5x_xts_dec
aesd $dat0,q8
aesimc $dat0,$dat0
aesd $dat1,q8
aesimc $dat1,$dat1
aesd $dat2,q8
aesimc $dat2,$dat2
aesd $dat3,q8
aesimc $dat3,$dat3
aesd $dat4,q8
aesimc $dat4,$dat4
subs $len,$len,#0x50 // because .Lxts_dec_tail4x
aesd $dat0,q9
aesimc $dat0,$dat
aesd $dat1,q9
aesimc $dat1,$dat1
aesd $dat2,q9
aesimc $dat2,$dat2
aesd $dat3,q9
aesimc $dat3,$dat3
aesd $dat4,q9
aesimc $dat4,$dat4
csel $xoffset,xzr,$len,gt // borrow x6, w6, "gt" is not typo
mov $key_,$key1
aesd $dat0,q10
aesimc $dat0,$dat0
aesd $dat1,q10
aesimc $dat1,$dat1
aesd $dat2,q10
aesimc $dat2,$dat2
aesd $dat3,q10
aesimc $dat3,$dat3
aesd $dat4,q10
aesimc $dat4,$dat4
add $inp,$inp,$xoffset // x0 is adjusted in such way that
// at exit from the loop v1.16b-v26.16b
// are loaded with last "words"
add $xoffset,$len,#0x60 // because .Lxts_dec_tail4x
aesd $dat0,q11
aesimc $dat0,$dat0
aesd $dat1,q11
aesimc $dat1,$dat1
aesd $dat2,q11
aesimc $dat2,$dat2
aesd $dat3,q11
aesimc $dat3,$dat3
aesd $dat4,q11
aesimc $dat4,$dat4
aesd $dat0,q12
aesimc $dat0,$dat0
aesd $dat1,q12
aesimc $dat1,$dat1
aesd $dat2,q12
aesimc $dat2,$dat2
aesd $dat3,q12
aesimc $dat3,$dat3
aesd $dat4,q12
aesimc $dat4,$dat4
aesd $dat0,q13
aesimc $dat0,$dat0
aesd $dat1,q13
aesimc $dat1,$dat1
aesd $dat2,q13
aesimc $dat2,$dat2
aesd $dat3,q13
aesimc $dat3,$dat3
aesd $dat4,q13
aesimc $dat4,$dat4
aesd $dat0,q14
aesimc $dat0,$dat0
aesd $dat1,q14
aesimc $dat1,$dat1
aesd $dat2,q14
aesimc $dat2,$dat2
aesd $dat3,q14
aesimc $dat3,$dat3
aesd $dat4,q14
aesimc $dat4,$dat4
veor $tmp0,$rndlast,$iv0
aesd $dat0,q15
// The iv for first block of next iteration.
extr $midnumx,$ivh,$ivh,#32
extr $ivh,$ivh,$ivl,#63
and $tmpmw,$constnum,$midnum,asr #31
eor $ivl,$tmpmx,$ivl,lsl #1
fmov $ivd00,$ivl
fmov $ivd01,$ivh
veor $tmp1,$rndlast,$iv1
vld1.8 {$in0},[$inp],#16
aesd $dat1,q15
// The iv for second block
extr $midnumx,$ivh,$ivh,#32
extr $ivh,$ivh,$ivl,#63
and $tmpmw,$constnum,$midnum,asr #31
eor $ivl,$tmpmx,$ivl,lsl #1
fmov $ivd10,$ivl
fmov $ivd11,$ivh
veor $tmp2,$rndlast,$iv2
vld1.8 {$in1},[$inp],#16
aesd $dat2,q15
// The iv for third block
extr $midnumx,$ivh,$ivh,#32
extr $ivh,$ivh,$ivl,#63
and $tmpmw,$constnum,$midnum,asr #31
eor $ivl,$tmpmx,$ivl,lsl #1
fmov $ivd20,$ivl
fmov $ivd21,$ivh
veor $tmp3,$rndlast,$iv3
vld1.8 {$in2},[$inp],#16
aesd $dat3,q15
// The iv for fourth block
extr $midnumx,$ivh,$ivh,#32
extr $ivh,$ivh,$ivl,#63
and $tmpmw,$constnum,$midnum,asr #31
eor $ivl,$tmpmx,$ivl,lsl #1
fmov $ivd30,$ivl
fmov $ivd31,$ivh
veor $tmp4,$rndlast,$iv4
vld1.8 {$in3},[$inp],#16
aesd $dat4,q15
// The iv for fifth block
extr $midnumx,$ivh,$ivh,#32
extr $ivh,$ivh,$ivl,#63
and $tmpmw,$constnum,$midnum,asr #31
eor $ivl,$tmpmx,$ivl,lsl #1
fmov $ivd40,$ivl
fmov $ivd41,$ivh
vld1.8 {$in4},[$inp],#16
cbz $xoffset,.Lxts_dec_tail4x
vld1.32 {q8},[$key_],#16 // re-pre-load rndkey[0]
veor $tmp0,$tmp0,$dat0
veor $dat0,$in0,$iv0
veor $tmp1,$tmp1,$dat1
veor $dat1,$in1,$iv1
veor $tmp2,$tmp2,$dat2
veor $dat2,$in2,$iv2
veor $tmp3,$tmp3,$dat3
veor $dat3,$in3,$iv3
veor $tmp4,$tmp4,$dat4
vst1.8 {$tmp0},[$out],#16
veor $dat4,$in4,$iv4
vst1.8 {$tmp1},[$out],#16
mov $rounds,$rounds0
vst1.8 {$tmp2},[$out],#16
vld1.32 {q9},[$key_],#16 // re-pre-load rndkey[1]
vst1.8 {$tmp3},[$out],#16
vst1.8 {$tmp4},[$out],#16
b.hs .Loop5x_xts_dec
cmn $len,#0x10
b.ne .Loop5x_dec_after
// If x2($len) equal to -0x10, the left blocks is 4.
// After specially processing, utilize the five blocks processing again.
// It will use the following IVs: $iv0,$iv0,$iv1,$iv2,$iv3.
vorr $iv4,$iv3,$iv3
vorr $iv3,$iv2,$iv2
vorr $iv2,$iv1,$iv1
vorr $iv1,$iv0,$iv0
fmov $ivl,$ivd40
fmov $ivh,$ivd41
veor $dat0,$iv0,$in0
veor $dat1,$iv1,$in1
veor $dat2,$in2,$iv2
veor $dat3,$in3,$iv3
veor $dat4,$in4,$iv4
b.eq .Loop5x_xts_dec
.Loop5x_dec_after:
add $len,$len,#0x50
cbz $len,.Lxts_done
add $rounds,$rounds0,#2
subs $len,$len,#0x30
b.lo .Lxts_inner_dec_tail
veor $dat0,$iv0,$in2
veor $dat1,$iv1,$in3
veor $dat2,$in4,$iv2
b .Lxts_outer_dec_tail
.align 4
.Lxts_dec_tail4x:
add $inp,$inp,#16
vld1.32 {$dat0},[$inp],#16
veor $tmp1,$dat1,$tmp0
vst1.8 {$tmp1},[$out],#16
veor $tmp2,$dat2,$tmp2
vst1.8 {$tmp2},[$out],#16
veor $tmp3,$dat3,$tmp3
veor $tmp4,$dat4,$tmp4
vst1.8 {$tmp3-$tmp4},[$out],#32
b .Lxts_done
.align 4
.Lxts_outer_dec_tail:
aesd $dat0,q8
aesimc $dat0,$dat0
aesd $dat1,q8
aesimc $dat1,$dat1
aesd $dat2,q8
aesimc $dat2,$dat2
vld1.32 {q8},[$key_],#16
subs $rounds,$rounds,#2
aesd $dat0,q9
aesimc $dat0,$dat0
aesd $dat1,q9
aesimc $dat1,$dat1
aesd $dat2,q9
aesimc $dat2,$dat2
vld1.32 {q9},[$key_],#16
b.gt .Lxts_outer_dec_tail
aesd $dat0,q8
aesimc $dat0,$dat0
aesd $dat1,q8
aesimc $dat1,$dat1
aesd $dat2,q8
aesimc $dat2,$dat2
veor $tmp0,$iv0,$rndlast
subs $len,$len,#0x30
// The iv for first block
fmov $ivl,$ivd20
fmov $ivh,$ivd21
mov $constnum,#0x87
extr $midnumx,$ivh,$ivh,#32
extr $ivh,$ivh,$ivl,#63
and $tmpmw,$constnum,$midnum,asr #31
eor $ivl,$tmpmx,$ivl,lsl #1
fmov $ivd00,$ivl
fmov $ivd01,$ivh
veor $tmp1,$iv1,$rndlast
csel $xoffset,$len,$xoffset,lo // x6, w6, is zero at this point
aesd $dat0,q9
aesimc $dat0,$dat0
aesd $dat1,q9
aesimc $dat1,$dat1
aesd $dat2,q9
aesimc $dat2,$dat2
veor $tmp2,$iv2,$rndlast
// The iv for second block
extr $midnumx,$ivh,$ivh,#32
extr $ivh,$ivh,$ivl,#63
and $tmpmw,$constnum,$midnum,asr #31
eor $ivl,$tmpmx,$ivl,lsl #1
fmov $ivd10,$ivl
fmov $ivd11,$ivh
add $xoffset,$xoffset,#0x20
add $inp,$inp,$xoffset // $inp is adjusted to the last data
mov $key_,$key1
// The iv for third block
extr $midnumx,$ivh,$ivh,#32
extr $ivh,$ivh,$ivl,#63
and $tmpmw,$constnum,$midnum,asr #31
eor $ivl,$tmpmx,$ivl,lsl #1
fmov $ivd20,$ivl
fmov $ivd21,$ivh
aesd $dat0,q12
aesimc $dat0,$dat0
aesd $dat1,q12
aesimc $dat1,$dat1
aesd $dat2,q12
aesimc $dat2,$dat2
aesd $dat0,q13
aesimc $dat0,$dat0
aesd $dat1,q13
aesimc $dat1,$dat1
aesd $dat2,q13
aesimc $dat2,$dat2
aesd $dat0,q14
aesimc $dat0,$dat0
aesd $dat1,q14
aesimc $dat1,$dat1
aesd $dat2,q14
aesimc $dat2,$dat2
vld1.8 {$in2},[$inp],#16
aesd $dat0,q15
aesd $dat1,q15
aesd $dat2,q15
vld1.32 {q8},[$key_],#16 // re-pre-load rndkey[0]
add $rounds,$rounds0,#2
veor $tmp0,$tmp0,$dat0
veor $tmp1,$tmp1,$dat1
veor $dat2,$dat2,$tmp2
vld1.32 {q9},[$key_],#16 // re-pre-load rndkey[1]
vst1.8 {$tmp0},[$out],#16
vst1.8 {$tmp1},[$out],#16
vst1.8 {$dat2},[$out],#16
cmn $len,#0x30
add $len,$len,#0x30
b.eq .Lxts_done
sub $len,$len,#0x30
vorr $in3,$in1,$in1
vorr $in4,$in2,$in2
nop
.Lxts_inner_dec_tail:
// $len == -0x10 means two blocks left.
cmn $len,#0x10
veor $dat1,$in3,$iv0
veor $dat2,$in4,$iv1
b.eq .Lxts_dec_tail_loop
veor $dat2,$in4,$iv0
.Lxts_dec_tail_loop:
aesd $dat1,q8
aesimc $dat1,$dat1
aesd $dat2,q8
aesimc $dat2,$dat2
vld1.32 {q8},[$key_],#16
subs $rounds,$rounds,#2
aesd $dat1,q9
aesimc $dat1,$dat1
aesd $dat2,q9
aesimc $dat2,$dat2
vld1.32 {q9},[$key_],#16
b.gt .Lxts_dec_tail_loop
aesd $dat1,q8
aesimc $dat1,$dat1
aesd $dat2,q8
aesimc $dat2,$dat2
aesd $dat1,q9
aesimc $dat1,$dat1
aesd $dat2,q9
aesimc $dat2,$dat2
aesd $dat1,q12
aesimc $dat1,$dat1
aesd $dat2,q12
aesimc $dat2,$dat2
cmn $len,#0x20
aesd $dat1,q13
aesimc $dat1,$dat1
aesd $dat2,q13
aesimc $dat2,$dat2
veor $tmp1,$iv0,$rndlast
aesd $dat1,q14
aesimc $dat1,$dat1
aesd $dat2,q14
aesimc $dat2,$dat2
veor $tmp2,$iv1,$rndlast
aesd $dat1,q15
aesd $dat2,q15
b.eq .Lxts_dec_one
veor $tmp1,$tmp1,$dat1
veor $tmp2,$tmp2,$dat2
vorr $iv0,$iv2,$iv2
vorr $iv1,$iv3,$iv3
vst1.8 {$tmp1},[$out],#16
vst1.8 {$tmp2},[$out],#16
add $len,$len,#16
b .Lxts_done
.Lxts_dec_one:
veor $tmp1,$tmp1,$dat2
vorr $iv0,$iv1,$iv1
vorr $iv1,$iv2,$iv2
vst1.8 {$tmp1},[$out],#16
add $len,$len,#32
.Lxts_done:
tst $tailcnt,#0xf
b.eq .Lxts_dec_abort
// Processing the last two blocks with cipher stealing.
mov x7,x3
cbnz x2,.Lxts_dec_1st_done
vld1.32 {$dat0},[$inp],#16
// Decrypt the last second block to get the last plain text block
.Lxts_dec_1st_done:
eor $tmpin,$dat0,$iv1
ldr $rounds,[$key1,#240]
vld1.32 {$dat0},[$key1],#16
sub $rounds,$rounds,#2
vld1.32 {$dat1},[$key1],#16
.Loop_final_2nd_dec:
aesd $tmpin,$dat0
aesimc $tmpin,$tmpin
vld1.32 {$dat0},[$key1],#16 // load key schedule...
subs $rounds,$rounds,#2
aesd $tmpin,$dat1
aesimc $tmpin,$tmpin
vld1.32 {$dat1},[$key1],#16 // load key schedule...
b.gt .Loop_final_2nd_dec
aesd $tmpin,$dat0
aesimc $tmpin,$tmpin
vld1.32 {$dat0},[$key1]
aesd $tmpin,$dat1
veor $tmpin,$tmpin,$dat0
veor $tmpin,$tmpin,$iv1
vst1.8 {$tmpin},[$out]
mov $tmpinp,$inp
add $tmpoutp,$out,#16
// Composite the tailcnt "16 byte not aligned block" into the last second plain blocks
// to get the last encrypted block.
.composite_dec_loop:
subs $tailcnt,$tailcnt,#1
ldrb $l2outp,[$out,$tailcnt]
ldrb $loutp,[$tmpinp,$tailcnt]
strb $l2outp,[$tmpoutp,$tailcnt]
strb $loutp,[$out,$tailcnt]
b.gt .composite_dec_loop
.Lxts_dec_load_done:
vld1.8 {$tmpin},[$out]
veor $tmpin,$tmpin,$iv0
// Decrypt the composite block to get the last second plain text block
ldr $rounds,[$key_,#240]
vld1.8 {$dat},[$key_],#16
sub $rounds,$rounds,#2
vld1.8 {$dat1},[$key_],#16
.Loop_final_dec:
aesd $tmpin,$dat0
aesimc $tmpin,$tmpin
vld1.32 {$dat0},[$key_],#16 // load key schedule...
subs $rounds,$rounds,#2
aesd $tmpin,$dat1
aesimc $tmpin,$tmpin
vld1.32 {$dat1},[$key_],#16 // load key schedule...
b.gt .Loop_final_dec
aesd $tmpin,$dat0
aesimc $tmpin,$tmpin
vld1.32 {$dat0},[$key_]
aesd $tmpin,$dat1
veor $tmpin,$tmpin,$dat0
veor $tmpin,$tmpin,$iv0
vst1.8 {$tmpin},[$out]
.Lxts_dec_abort:
ldp $tailcnt,$midnumx,[sp,#48]
ldp $ivd10,$ivd20,[sp,#32]
ldp $ivd30,$ivd40,[sp,#16]
ldp $constnumx,$tmpinp,[sp],#64
.Lxts_dec_final_abort:
ret
.size ${prefix}_xts_decrypt,.-${prefix}_xts_decrypt
___
}
}}}
$code.=<<___;
#endif
___
########################################
if ($flavour =~ /64/) { ######## 64-bit code
my %opcode = (
"aesd" => 0x4e285800, "aese" => 0x4e284800,
"aesimc"=> 0x4e287800, "aesmc" => 0x4e286800 );
local *unaes = sub {
my ($mnemonic,$arg)=@_;
$arg =~ m/[qv]([0-9]+)[^,]*,\s*[qv]([0-9]+)/o &&
sprintf ".inst\t0x%08x\t//%s %s",
$opcode{$mnemonic}|$1|($2<<5),
$mnemonic,$arg;
};
foreach(split("\n",$code)) {
s/\`([^\`]*)\`/eval($1)/geo;
s/\bq([0-9]+)\b/"v".($1<8?$1:$1+8).".16b"/geo; # old->new registers
s/@\s/\/\//o; # old->new style commentary
#s/[v]?(aes\w+)\s+([qv].*)/unaes($1,$2)/geo or
s/cclr\s+([wx])([^,]+),\s*([a-z]+)/csel $1$2,$1zr,$1$2,$3/o or
s/mov\.([a-z]+)\s+([wx][0-9]+),\s*([wx][0-9]+)/csel $2,$3,$2,$1/o or
s/vmov\.i8/movi/o or # fix up legacy mnemonics
s/vext\.8/ext/o or
s/vrev32\.8/rev32/o or
s/vtst\.8/cmtst/o or
s/vshr/ushr/o or
s/^(\s+)v/$1/o or # strip off v prefix
s/\bbx\s+lr\b/ret/o;
# fix up remaining legacy suffixes
s/\.[ui]?8//o;
m/\],#8/o and s/\.16b/\.8b/go;
s/\.[ui]?32//o and s/\.16b/\.4s/go;
s/\.[ui]?64//o and s/\.16b/\.2d/go;
s/\.[42]([sd])\[([0-3])\]/\.$1\[$2\]/o;
# Switch preprocessor checks to aarch64 versions.
s/__ARME([BL])__/__AARCH64E$1__/go;
print $_,"\n";
}
} else { ######## 32-bit code
my %opcode = (
"aesd" => 0xf3b00340, "aese" => 0xf3b00300,
"aesimc"=> 0xf3b003c0, "aesmc" => 0xf3b00380 );
local *unaes = sub {
my ($mnemonic,$arg)=@_;
if ($arg =~ m/[qv]([0-9]+)[^,]*,\s*[qv]([0-9]+)/o) {
my $word = $opcode{$mnemonic}|(($1&7)<<13)|(($1&8)<<19)
|(($2&7)<<1) |(($2&8)<<2);
# since ARMv7 instructions are always encoded little-endian.
# correct solution is to use .inst directive, but older
# assemblers don't implement it:-(
sprintf "INST(0x%02x,0x%02x,0x%02x,0x%02x)\t@ %s %s",
$word&0xff,($word>>8)&0xff,
($word>>16)&0xff,($word>>24)&0xff,
$mnemonic,$arg;
}
};
sub unvtbl {
my $arg=shift;
$arg =~ m/q([0-9]+),\s*\{q([0-9]+)\},\s*q([0-9]+)/o &&
sprintf "vtbl.8 d%d,{q%d},d%d\n\t".
"vtbl.8 d%d,{q%d},d%d", 2*$1,$2,2*$3, 2*$1+1,$2,2*$3+1;
}
sub unvdup32 {
my $arg=shift;
$arg =~ m/q([0-9]+),\s*q([0-9]+)\[([0-3])\]/o &&
sprintf "vdup.32 q%d,d%d[%d]",$1,2*$2+($3>>1),$3&1;
}
sub unvmov32 {
my $arg=shift;
$arg =~ m/q([0-9]+)\[([0-3])\],(.*)/o &&
sprintf "vmov.32 d%d[%d],%s",2*$1+($2>>1),$2&1,$3;
}
foreach(split("\n",$code)) {
s/\`([^\`]*)\`/eval($1)/geo;
s/\b[wx]([0-9]+)\b/r$1/go; # new->old registers
s/\bv([0-9])\.[12468]+[bsd]\b/q$1/go; # new->old registers
s/\/\/\s?/@ /o; # new->old style commentary
# fix up remaining new-style suffixes
s/\{q([0-9]+)\},\s*\[(.+)\],#8/sprintf "{d%d},[$2]!",2*$1/eo or
s/\],#[0-9]+/]!/o;
s/[v]?(aes\w+)\s+([qv].*)/unaes($1,$2)/geo or
s/cclr\s+([^,]+),\s*([a-z]+)/mov.$2 $1,#0/o or
s/vtbl\.8\s+(.*)/unvtbl($1)/geo or
s/vdup\.32\s+(.*)/unvdup32($1)/geo or
s/vmov\.32\s+(.*)/unvmov32($1)/geo or
s/^(\s+)b\./$1b/o or
s/^(\s+)ret/$1bx\tlr/o;
if (s/^(\s+)mov\.([a-z]+)/$1mov$2/) {
print " it $2\n";
}
print $_,"\n";
}
}
close STDOUT or die "error closing STDOUT: $!";
|