mirror of https://github.com/lukechilds/node.git
Browse Source
This just replaces all sources in deps/openssl/openssl to originals in https://www.openssl.org/source/openssl-1.0.2a.tar.gz Fixes: https://github.com/iojs/io.js/issues/589 PR-URL: https://github.com/iojs/io.js/pull/1389 Reviewed-By: Fedor Indutny <fedor@indutny.com> Reviewed-By: Ben Noordhuis <info@bnoordhuis.nl>v1.8.0-commit
Shigeki Ohtsu
10 years ago
678 changed files with 97139 additions and 14133 deletions
@ -0,0 +1,8 @@ |
|||
#!/bin/sh |
|||
|
|||
BRANCH=`git rev-parse --abbrev-ref HEAD` |
|||
|
|||
./Configure $@ no-symlinks |
|||
make files |
|||
util/mk1mf.pl OUT=out.$BRANCH TMP=tmp.$BRANCH INC=inc.$BRANCH copy > makefile.$BRANCH |
|||
make -f makefile.$BRANCH init |
@ -0,0 +1,5 @@ |
|||
#!/bin/sh |
|||
|
|||
BRANCH=`git rev-parse --abbrev-ref HEAD` |
|||
|
|||
make -f makefile.$BRANCH $@ |
File diff suppressed because it is too large
File diff suppressed because it is too large
File diff suppressed because it is too large
File diff suppressed because it is too large
File diff suppressed because it is too large
File diff suppressed because it is too large
File diff suppressed because it is too large
@ -0,0 +1,919 @@ |
|||
#!/usr/bin/env perl |
|||
|
|||
# ==================================================================== |
|||
# Written by David S. Miller <davem@devemloft.net> and Andy Polyakov |
|||
# <appro@openssl.org>. The module is licensed under 2-clause BSD |
|||
# license. October 2012. All rights reserved. |
|||
# ==================================================================== |
|||
|
|||
###################################################################### |
|||
# AES for SPARC T4. |
|||
# |
|||
# AES round instructions complete in 3 cycles and can be issued every |
|||
# cycle. It means that round calculations should take 4*rounds cycles, |
|||
# because any given round instruction depends on result of *both* |
|||
# previous instructions: |
|||
# |
|||
# |0 |1 |2 |3 |4 |
|||
# |01|01|01| |
|||
# |23|23|23| |
|||
# |01|01|... |
|||
# |23|... |
|||
# |
|||
# Provided that fxor [with IV] takes 3 cycles to complete, critical |
|||
# path length for CBC encrypt would be 3+4*rounds, or in other words |
|||
# it should process one byte in at least (3+4*rounds)/16 cycles. This |
|||
# estimate doesn't account for "collateral" instructions, such as |
|||
# fetching input from memory, xor-ing it with zero-round key and |
|||
# storing the result. Yet, *measured* performance [for data aligned |
|||
# at 64-bit boundary!] deviates from this equation by less than 0.5%: |
|||
# |
|||
# 128-bit key 192- 256- |
|||
# CBC encrypt 2.70/2.90(*) 3.20/3.40 3.70/3.90 |
|||
# (*) numbers after slash are for |
|||
# misaligned data. |
|||
# |
|||
# Out-of-order execution logic managed to fully overlap "collateral" |
|||
# instructions with those on critical path. Amazing! |
|||
# |
|||
# As with Intel AES-NI, question is if it's possible to improve |
|||
# performance of parallelizeable modes by interleaving round |
|||
# instructions. Provided round instruction latency and throughput |
|||
# optimal interleave factor is 2. But can we expect 2x performance |
|||
# improvement? Well, as round instructions can be issued one per |
|||
# cycle, they don't saturate the 2-way issue pipeline and therefore |
|||
# there is room for "collateral" calculations... Yet, 2x speed-up |
|||
# over CBC encrypt remains unattaintable: |
|||
# |
|||
# 128-bit key 192- 256- |
|||
# CBC decrypt 1.64/2.11 1.89/2.37 2.23/2.61 |
|||
# CTR 1.64/2.08(*) 1.89/2.33 2.23/2.61 |
|||
# (*) numbers after slash are for |
|||
# misaligned data. |
|||
# |
|||
# Estimates based on amount of instructions under assumption that |
|||
# round instructions are not pairable with any other instruction |
|||
# suggest that latter is the actual case and pipeline runs |
|||
# underutilized. It should be noted that T4 out-of-order execution |
|||
# logic is so capable that performance gain from 2x interleave is |
|||
# not even impressive, ~7-13% over non-interleaved code, largest |
|||
# for 256-bit keys. |
|||
|
|||
# To anchor to something else, software implementation processes |
|||
# one byte in 29 cycles with 128-bit key on same processor. Intel |
|||
# Sandy Bridge encrypts byte in 5.07 cycles in CBC mode and decrypts |
|||
# in 0.93, naturally with AES-NI. |
|||
|
|||
$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; |
|||
push(@INC,"${dir}","${dir}../../perlasm"); |
|||
require "sparcv9_modes.pl"; |
|||
|
|||
&asm_init(@ARGV); |
|||
|
|||
$::evp=1; # if $evp is set to 0, script generates module with |
|||
# AES_[en|de]crypt, AES_set_[en|de]crypt_key and AES_cbc_encrypt entry |
|||
# points. These however are not fully compatible with openssl/aes.h, |
|||
# because they expect AES_KEY to be aligned at 64-bit boundary. When |
|||
# used through EVP, alignment is arranged at EVP layer. Second thing |
|||
# that is arranged by EVP is at least 32-bit alignment of IV. |
|||
|
|||
###################################################################### |
|||
# single-round subroutines |
|||
# |
|||
{ |
|||
my ($inp,$out,$key,$rounds,$tmp,$mask)=map("%o$_",(0..5)); |
|||
|
|||
$code.=<<___ if ($::abibits==64); |
|||
.register %g2,#scratch |
|||
.register %g3,#scratch |
|||
|
|||
___ |
|||
$code.=<<___; |
|||
.text |
|||
|
|||
.globl aes_t4_encrypt |
|||
.align 32 |
|||
aes_t4_encrypt: |
|||
andcc $inp, 7, %g1 ! is input aligned? |
|||
andn $inp, 7, $inp |
|||
|
|||
ldx [$key + 0], %g4 |
|||
ldx [$key + 8], %g5 |
|||
|
|||
ldx [$inp + 0], %o4 |
|||
bz,pt %icc, 1f |
|||
ldx [$inp + 8], %o5 |
|||
ldx [$inp + 16], $inp |
|||
sll %g1, 3, %g1 |
|||
sub %g0, %g1, %o3 |
|||
sllx %o4, %g1, %o4 |
|||
sllx %o5, %g1, %g1 |
|||
srlx %o5, %o3, %o5 |
|||
srlx $inp, %o3, %o3 |
|||
or %o5, %o4, %o4 |
|||
or %o3, %g1, %o5 |
|||
1: |
|||
ld [$key + 240], $rounds |
|||
ldd [$key + 16], %f12 |
|||
ldd [$key + 24], %f14 |
|||
xor %g4, %o4, %o4 |
|||
xor %g5, %o5, %o5 |
|||
movxtod %o4, %f0 |
|||
movxtod %o5, %f2 |
|||
srl $rounds, 1, $rounds |
|||
ldd [$key + 32], %f16 |
|||
sub $rounds, 1, $rounds |
|||
ldd [$key + 40], %f18 |
|||
add $key, 48, $key |
|||
|
|||
.Lenc: |
|||
aes_eround01 %f12, %f0, %f2, %f4 |
|||
aes_eround23 %f14, %f0, %f2, %f2 |
|||
ldd [$key + 0], %f12 |
|||
ldd [$key + 8], %f14 |
|||
sub $rounds,1,$rounds |
|||
aes_eround01 %f16, %f4, %f2, %f0 |
|||
aes_eround23 %f18, %f4, %f2, %f2 |
|||
ldd [$key + 16], %f16 |
|||
ldd [$key + 24], %f18 |
|||
brnz,pt $rounds, .Lenc |
|||
add $key, 32, $key |
|||
|
|||
andcc $out, 7, $tmp ! is output aligned? |
|||
aes_eround01 %f12, %f0, %f2, %f4 |
|||
aes_eround23 %f14, %f0, %f2, %f2 |
|||
aes_eround01_l %f16, %f4, %f2, %f0 |
|||
aes_eround23_l %f18, %f4, %f2, %f2 |
|||
|
|||
bnz,pn %icc, 2f |
|||
nop |
|||
|
|||
std %f0, [$out + 0] |
|||
retl |
|||
std %f2, [$out + 8] |
|||
|
|||
2: alignaddrl $out, %g0, $out |
|||
mov 0xff, $mask |
|||
srl $mask, $tmp, $mask |
|||
|
|||
faligndata %f0, %f0, %f4 |
|||
faligndata %f0, %f2, %f6 |
|||
faligndata %f2, %f2, %f8 |
|||
|
|||
stda %f4, [$out + $mask]0xc0 ! partial store |
|||
std %f6, [$out + 8] |
|||
add $out, 16, $out |
|||
orn %g0, $mask, $mask |
|||
retl |
|||
stda %f8, [$out + $mask]0xc0 ! partial store |
|||
.type aes_t4_encrypt,#function |
|||
.size aes_t4_encrypt,.-aes_t4_encrypt |
|||
|
|||
.globl aes_t4_decrypt |
|||
.align 32 |
|||
aes_t4_decrypt: |
|||
andcc $inp, 7, %g1 ! is input aligned? |
|||
andn $inp, 7, $inp |
|||
|
|||
ldx [$key + 0], %g4 |
|||
ldx [$key + 8], %g5 |
|||
|
|||
ldx [$inp + 0], %o4 |
|||
bz,pt %icc, 1f |
|||
ldx [$inp + 8], %o5 |
|||
ldx [$inp + 16], $inp |
|||
sll %g1, 3, %g1 |
|||
sub %g0, %g1, %o3 |
|||
sllx %o4, %g1, %o4 |
|||
sllx %o5, %g1, %g1 |
|||
srlx %o5, %o3, %o5 |
|||
srlx $inp, %o3, %o3 |
|||
or %o5, %o4, %o4 |
|||
or %o3, %g1, %o5 |
|||
1: |
|||
ld [$key + 240], $rounds |
|||
ldd [$key + 16], %f12 |
|||
ldd [$key + 24], %f14 |
|||
xor %g4, %o4, %o4 |
|||
xor %g5, %o5, %o5 |
|||
movxtod %o4, %f0 |
|||
movxtod %o5, %f2 |
|||
srl $rounds, 1, $rounds |
|||
ldd [$key + 32], %f16 |
|||
sub $rounds, 1, $rounds |
|||
ldd [$key + 40], %f18 |
|||
add $key, 48, $key |
|||
|
|||
.Ldec: |
|||
aes_dround01 %f12, %f0, %f2, %f4 |
|||
aes_dround23 %f14, %f0, %f2, %f2 |
|||
ldd [$key + 0], %f12 |
|||
ldd [$key + 8], %f14 |
|||
sub $rounds,1,$rounds |
|||
aes_dround01 %f16, %f4, %f2, %f0 |
|||
aes_dround23 %f18, %f4, %f2, %f2 |
|||
ldd [$key + 16], %f16 |
|||
ldd [$key + 24], %f18 |
|||
brnz,pt $rounds, .Ldec |
|||
add $key, 32, $key |
|||
|
|||
andcc $out, 7, $tmp ! is output aligned? |
|||
aes_dround01 %f12, %f0, %f2, %f4 |
|||
aes_dround23 %f14, %f0, %f2, %f2 |
|||
aes_dround01_l %f16, %f4, %f2, %f0 |
|||
aes_dround23_l %f18, %f4, %f2, %f2 |
|||
|
|||
bnz,pn %icc, 2f |
|||
nop |
|||
|
|||
std %f0, [$out + 0] |
|||
retl |
|||
std %f2, [$out + 8] |
|||
|
|||
2: alignaddrl $out, %g0, $out |
|||
mov 0xff, $mask |
|||
srl $mask, $tmp, $mask |
|||
|
|||
faligndata %f0, %f0, %f4 |
|||
faligndata %f0, %f2, %f6 |
|||
faligndata %f2, %f2, %f8 |
|||
|
|||
stda %f4, [$out + $mask]0xc0 ! partial store |
|||
std %f6, [$out + 8] |
|||
add $out, 16, $out |
|||
orn %g0, $mask, $mask |
|||
retl |
|||
stda %f8, [$out + $mask]0xc0 ! partial store |
|||
.type aes_t4_decrypt,#function |
|||
.size aes_t4_decrypt,.-aes_t4_decrypt |
|||
___ |
|||
} |
|||
|
|||
###################################################################### |
|||
# key setup subroutines |
|||
# |
|||
{ |
|||
my ($inp,$bits,$out,$tmp)=map("%o$_",(0..5)); |
|||
$code.=<<___; |
|||
.globl aes_t4_set_encrypt_key |
|||
.align 32 |
|||
aes_t4_set_encrypt_key: |
|||
.Lset_encrypt_key: |
|||
and $inp, 7, $tmp |
|||
alignaddr $inp, %g0, $inp |
|||
cmp $bits, 192 |
|||
ldd [$inp + 0], %f0 |
|||
bl,pt %icc,.L128 |
|||
ldd [$inp + 8], %f2 |
|||
|
|||
be,pt %icc,.L192 |
|||
ldd [$inp + 16], %f4 |
|||
brz,pt $tmp, .L256aligned |
|||
ldd [$inp + 24], %f6 |
|||
|
|||
ldd [$inp + 32], %f8 |
|||
faligndata %f0, %f2, %f0 |
|||
faligndata %f2, %f4, %f2 |
|||
faligndata %f4, %f6, %f4 |
|||
faligndata %f6, %f8, %f6 |
|||
.L256aligned: |
|||
___ |
|||
for ($i=0; $i<6; $i++) { |
|||
$code.=<<___; |
|||
std %f0, [$out + `32*$i+0`] |
|||
aes_kexpand1 %f0, %f6, $i, %f0 |
|||
std %f2, [$out + `32*$i+8`] |
|||
aes_kexpand2 %f2, %f0, %f2 |
|||
std %f4, [$out + `32*$i+16`] |
|||
aes_kexpand0 %f4, %f2, %f4 |
|||
std %f6, [$out + `32*$i+24`] |
|||
aes_kexpand2 %f6, %f4, %f6 |
|||
___ |
|||
} |
|||
$code.=<<___; |
|||
std %f0, [$out + `32*$i+0`] |
|||
aes_kexpand1 %f0, %f6, $i, %f0 |
|||
std %f2, [$out + `32*$i+8`] |
|||
aes_kexpand2 %f2, %f0, %f2 |
|||
std %f4, [$out + `32*$i+16`] |
|||
std %f6, [$out + `32*$i+24`] |
|||
std %f0, [$out + `32*$i+32`] |
|||
std %f2, [$out + `32*$i+40`] |
|||
|
|||
mov 14, $tmp |
|||
st $tmp, [$out + 240] |
|||
retl |
|||
xor %o0, %o0, %o0 |
|||
|
|||
.align 16 |
|||
.L192: |
|||
brz,pt $tmp, .L192aligned |
|||
nop |
|||
|
|||
ldd [$inp + 24], %f6 |
|||
faligndata %f0, %f2, %f0 |
|||
faligndata %f2, %f4, %f2 |
|||
faligndata %f4, %f6, %f4 |
|||
.L192aligned: |
|||
___ |
|||
for ($i=0; $i<7; $i++) { |
|||
$code.=<<___; |
|||
std %f0, [$out + `24*$i+0`] |
|||
aes_kexpand1 %f0, %f4, $i, %f0 |
|||
std %f2, [$out + `24*$i+8`] |
|||
aes_kexpand2 %f2, %f0, %f2 |
|||
std %f4, [$out + `24*$i+16`] |
|||
aes_kexpand2 %f4, %f2, %f4 |
|||
___ |
|||
} |
|||
$code.=<<___; |
|||
std %f0, [$out + `24*$i+0`] |
|||
aes_kexpand1 %f0, %f4, $i, %f0 |
|||
std %f2, [$out + `24*$i+8`] |
|||
aes_kexpand2 %f2, %f0, %f2 |
|||
std %f4, [$out + `24*$i+16`] |
|||
std %f0, [$out + `24*$i+24`] |
|||
std %f2, [$out + `24*$i+32`] |
|||
|
|||
mov 12, $tmp |
|||
st $tmp, [$out + 240] |
|||
retl |
|||
xor %o0, %o0, %o0 |
|||
|
|||
.align 16 |
|||
.L128: |
|||
brz,pt $tmp, .L128aligned |
|||
nop |
|||
|
|||
ldd [$inp + 16], %f4 |
|||
faligndata %f0, %f2, %f0 |
|||
faligndata %f2, %f4, %f2 |
|||
.L128aligned: |
|||
___ |
|||
for ($i=0; $i<10; $i++) { |
|||
$code.=<<___; |
|||
std %f0, [$out + `16*$i+0`] |
|||
aes_kexpand1 %f0, %f2, $i, %f0 |
|||
std %f2, [$out + `16*$i+8`] |
|||
aes_kexpand2 %f2, %f0, %f2 |
|||
___ |
|||
} |
|||
$code.=<<___; |
|||
std %f0, [$out + `16*$i+0`] |
|||
std %f2, [$out + `16*$i+8`] |
|||
|
|||
mov 10, $tmp |
|||
st $tmp, [$out + 240] |
|||
retl |
|||
xor %o0, %o0, %o0 |
|||
.type aes_t4_set_encrypt_key,#function |
|||
.size aes_t4_set_encrypt_key,.-aes_t4_set_encrypt_key |
|||
|
|||
.globl aes_t4_set_decrypt_key |
|||
.align 32 |
|||
aes_t4_set_decrypt_key: |
|||
mov %o7, %o5 |
|||
call .Lset_encrypt_key |
|||
nop |
|||
|
|||
mov %o5, %o7 |
|||
sll $tmp, 4, $inp ! $tmp is number of rounds |
|||
add $tmp, 2, $tmp |
|||
add $out, $inp, $inp ! $inp=$out+16*rounds |
|||
srl $tmp, 2, $tmp ! $tmp=(rounds+2)/4 |
|||
|
|||
.Lkey_flip: |
|||
ldd [$out + 0], %f0 |
|||
ldd [$out + 8], %f2 |
|||
ldd [$out + 16], %f4 |
|||
ldd [$out + 24], %f6 |
|||
ldd [$inp + 0], %f8 |
|||
ldd [$inp + 8], %f10 |
|||
ldd [$inp - 16], %f12 |
|||
ldd [$inp - 8], %f14 |
|||
sub $tmp, 1, $tmp |
|||
std %f0, [$inp + 0] |
|||
std %f2, [$inp + 8] |
|||
std %f4, [$inp - 16] |
|||
std %f6, [$inp - 8] |
|||
std %f8, [$out + 0] |
|||
std %f10, [$out + 8] |
|||
std %f12, [$out + 16] |
|||
std %f14, [$out + 24] |
|||
add $out, 32, $out |
|||
brnz $tmp, .Lkey_flip |
|||
sub $inp, 32, $inp |
|||
|
|||
retl |
|||
xor %o0, %o0, %o0 |
|||
.type aes_t4_set_decrypt_key,#function |
|||
.size aes_t4_set_decrypt_key,.-aes_t4_set_decrypt_key |
|||
___ |
|||
} |
|||
|
|||
{{{ |
|||
my ($inp,$out,$len,$key,$ivec,$enc)=map("%i$_",(0..5)); |
|||
my ($ileft,$iright,$ooff,$omask,$ivoff)=map("%l$_",(1..7)); |
|||
|
|||
$code.=<<___; |
|||
.align 32 |
|||
_aes128_encrypt_1x: |
|||
___ |
|||
for ($i=0; $i<4; $i++) { |
|||
$code.=<<___; |
|||
aes_eround01 %f`16+8*$i+0`, %f0, %f2, %f4 |
|||
aes_eround23 %f`16+8*$i+2`, %f0, %f2, %f2 |
|||
aes_eround01 %f`16+8*$i+4`, %f4, %f2, %f0 |
|||
aes_eround23 %f`16+8*$i+6`, %f4, %f2, %f2 |
|||
___ |
|||
} |
|||
$code.=<<___; |
|||
aes_eround01 %f48, %f0, %f2, %f4 |
|||
aes_eround23 %f50, %f0, %f2, %f2 |
|||
aes_eround01_l %f52, %f4, %f2, %f0 |
|||
retl |
|||
aes_eround23_l %f54, %f4, %f2, %f2 |
|||
.type _aes128_encrypt_1x,#function |
|||
.size _aes128_encrypt_1x,.-_aes128_encrypt_1x |
|||
|
|||
.align 32 |
|||
_aes128_encrypt_2x: |
|||
___ |
|||
for ($i=0; $i<4; $i++) { |
|||
$code.=<<___; |
|||
aes_eround01 %f`16+8*$i+0`, %f0, %f2, %f8 |
|||
aes_eround23 %f`16+8*$i+2`, %f0, %f2, %f2 |
|||
aes_eround01 %f`16+8*$i+0`, %f4, %f6, %f10 |
|||
aes_eround23 %f`16+8*$i+2`, %f4, %f6, %f6 |
|||
aes_eround01 %f`16+8*$i+4`, %f8, %f2, %f0 |
|||
aes_eround23 %f`16+8*$i+6`, %f8, %f2, %f2 |
|||
aes_eround01 %f`16+8*$i+4`, %f10, %f6, %f4 |
|||
aes_eround23 %f`16+8*$i+6`, %f10, %f6, %f6 |
|||
___ |
|||
} |
|||
$code.=<<___; |
|||
aes_eround01 %f48, %f0, %f2, %f8 |
|||
aes_eround23 %f50, %f0, %f2, %f2 |
|||
aes_eround01 %f48, %f4, %f6, %f10 |
|||
aes_eround23 %f50, %f4, %f6, %f6 |
|||
aes_eround01_l %f52, %f8, %f2, %f0 |
|||
aes_eround23_l %f54, %f8, %f2, %f2 |
|||
aes_eround01_l %f52, %f10, %f6, %f4 |
|||
retl |
|||
aes_eround23_l %f54, %f10, %f6, %f6 |
|||
.type _aes128_encrypt_2x,#function |
|||
.size _aes128_encrypt_2x,.-_aes128_encrypt_2x |
|||
|
|||
.align 32 |
|||
_aes128_loadkey: |
|||
ldx [$key + 0], %g4 |
|||
ldx [$key + 8], %g5 |
|||
___ |
|||
for ($i=2; $i<22;$i++) { # load key schedule |
|||
$code.=<<___; |
|||
ldd [$key + `8*$i`], %f`12+2*$i` |
|||
___ |
|||
} |
|||
$code.=<<___; |
|||
retl |
|||
nop |
|||
.type _aes128_loadkey,#function |
|||
.size _aes128_loadkey,.-_aes128_loadkey |
|||
_aes128_load_enckey=_aes128_loadkey |
|||
_aes128_load_deckey=_aes128_loadkey |
|||
|
|||
___ |
|||
|
|||
&alg_cbc_encrypt_implement("aes",128); |
|||
if ($::evp) { |
|||
&alg_ctr32_implement("aes",128); |
|||
&alg_xts_implement("aes",128,"en"); |
|||
&alg_xts_implement("aes",128,"de"); |
|||
} |
|||
&alg_cbc_decrypt_implement("aes",128); |
|||
|
|||
$code.=<<___; |
|||
.align 32 |
|||
_aes128_decrypt_1x: |
|||
___ |
|||
for ($i=0; $i<4; $i++) { |
|||
$code.=<<___; |
|||
aes_dround01 %f`16+8*$i+0`, %f0, %f2, %f4 |
|||
aes_dround23 %f`16+8*$i+2`, %f0, %f2, %f2 |
|||
aes_dround01 %f`16+8*$i+4`, %f4, %f2, %f0 |
|||
aes_dround23 %f`16+8*$i+6`, %f4, %f2, %f2 |
|||
___ |
|||
} |
|||
$code.=<<___; |
|||
aes_dround01 %f48, %f0, %f2, %f4 |
|||
aes_dround23 %f50, %f0, %f2, %f2 |
|||
aes_dround01_l %f52, %f4, %f2, %f0 |
|||
retl |
|||
aes_dround23_l %f54, %f4, %f2, %f2 |
|||
.type _aes128_decrypt_1x,#function |
|||
.size _aes128_decrypt_1x,.-_aes128_decrypt_1x |
|||
|
|||
.align 32 |
|||
_aes128_decrypt_2x: |
|||
___ |
|||
for ($i=0; $i<4; $i++) { |
|||
$code.=<<___; |
|||
aes_dround01 %f`16+8*$i+0`, %f0, %f2, %f8 |
|||
aes_dround23 %f`16+8*$i+2`, %f0, %f2, %f2 |
|||
aes_dround01 %f`16+8*$i+0`, %f4, %f6, %f10 |
|||
aes_dround23 %f`16+8*$i+2`, %f4, %f6, %f6 |
|||
aes_dround01 %f`16+8*$i+4`, %f8, %f2, %f0 |
|||
aes_dround23 %f`16+8*$i+6`, %f8, %f2, %f2 |
|||
aes_dround01 %f`16+8*$i+4`, %f10, %f6, %f4 |
|||
aes_dround23 %f`16+8*$i+6`, %f10, %f6, %f6 |
|||
___ |
|||
} |
|||
$code.=<<___; |
|||
aes_dround01 %f48, %f0, %f2, %f8 |
|||
aes_dround23 %f50, %f0, %f2, %f2 |
|||
aes_dround01 %f48, %f4, %f6, %f10 |
|||
aes_dround23 %f50, %f4, %f6, %f6 |
|||
aes_dround01_l %f52, %f8, %f2, %f0 |
|||
aes_dround23_l %f54, %f8, %f2, %f2 |
|||
aes_dround01_l %f52, %f10, %f6, %f4 |
|||
retl |
|||
aes_dround23_l %f54, %f10, %f6, %f6 |
|||
.type _aes128_decrypt_2x,#function |
|||
.size _aes128_decrypt_2x,.-_aes128_decrypt_2x |
|||
___ |
|||
|
|||
$code.=<<___; |
|||
.align 32 |
|||
_aes192_encrypt_1x: |
|||
___ |
|||
for ($i=0; $i<5; $i++) { |
|||
$code.=<<___; |
|||
aes_eround01 %f`16+8*$i+0`, %f0, %f2, %f4 |
|||
aes_eround23 %f`16+8*$i+2`, %f0, %f2, %f2 |
|||
aes_eround01 %f`16+8*$i+4`, %f4, %f2, %f0 |
|||
aes_eround23 %f`16+8*$i+6`, %f4, %f2, %f2 |
|||
___ |
|||
} |
|||
$code.=<<___; |
|||
aes_eround01 %f56, %f0, %f2, %f4 |
|||
aes_eround23 %f58, %f0, %f2, %f2 |
|||
aes_eround01_l %f60, %f4, %f2, %f0 |
|||
retl |
|||
aes_eround23_l %f62, %f4, %f2, %f2 |
|||
.type _aes192_encrypt_1x,#function |
|||
.size _aes192_encrypt_1x,.-_aes192_encrypt_1x |
|||
|
|||
.align 32 |
|||
_aes192_encrypt_2x: |
|||
___ |
|||
for ($i=0; $i<5; $i++) { |
|||
$code.=<<___; |
|||
aes_eround01 %f`16+8*$i+0`, %f0, %f2, %f8 |
|||
aes_eround23 %f`16+8*$i+2`, %f0, %f2, %f2 |
|||
aes_eround01 %f`16+8*$i+0`, %f4, %f6, %f10 |
|||
aes_eround23 %f`16+8*$i+2`, %f4, %f6, %f6 |
|||
aes_eround01 %f`16+8*$i+4`, %f8, %f2, %f0 |
|||
aes_eround23 %f`16+8*$i+6`, %f8, %f2, %f2 |
|||
aes_eround01 %f`16+8*$i+4`, %f10, %f6, %f4 |
|||
aes_eround23 %f`16+8*$i+6`, %f10, %f6, %f6 |
|||
___ |
|||
} |
|||
$code.=<<___; |
|||
aes_eround01 %f56, %f0, %f2, %f8 |
|||
aes_eround23 %f58, %f0, %f2, %f2 |
|||
aes_eround01 %f56, %f4, %f6, %f10 |
|||
aes_eround23 %f58, %f4, %f6, %f6 |
|||
aes_eround01_l %f60, %f8, %f2, %f0 |
|||
aes_eround23_l %f62, %f8, %f2, %f2 |
|||
aes_eround01_l %f60, %f10, %f6, %f4 |
|||
retl |
|||
aes_eround23_l %f62, %f10, %f6, %f6 |
|||
.type _aes192_encrypt_2x,#function |
|||
.size _aes192_encrypt_2x,.-_aes192_encrypt_2x |
|||
|
|||
.align 32 |
|||
_aes256_encrypt_1x: |
|||
aes_eround01 %f16, %f0, %f2, %f4 |
|||
aes_eround23 %f18, %f0, %f2, %f2 |
|||
ldd [$key + 208], %f16 |
|||
ldd [$key + 216], %f18 |
|||
aes_eround01 %f20, %f4, %f2, %f0 |
|||
aes_eround23 %f22, %f4, %f2, %f2 |
|||
ldd [$key + 224], %f20 |
|||
ldd [$key + 232], %f22 |
|||
___ |
|||
for ($i=1; $i<6; $i++) { |
|||
$code.=<<___; |
|||
aes_eround01 %f`16+8*$i+0`, %f0, %f2, %f4 |
|||
aes_eround23 %f`16+8*$i+2`, %f0, %f2, %f2 |
|||
aes_eround01 %f`16+8*$i+4`, %f4, %f2, %f0 |
|||
aes_eround23 %f`16+8*$i+6`, %f4, %f2, %f2 |
|||
___ |
|||
} |
|||
$code.=<<___; |
|||
aes_eround01 %f16, %f0, %f2, %f4 |
|||
aes_eround23 %f18, %f0, %f2, %f2 |
|||
ldd [$key + 16], %f16 |
|||
ldd [$key + 24], %f18 |
|||
aes_eround01_l %f20, %f4, %f2, %f0 |
|||
aes_eround23_l %f22, %f4, %f2, %f2 |
|||
ldd [$key + 32], %f20 |
|||
retl |
|||
ldd [$key + 40], %f22 |
|||
.type _aes256_encrypt_1x,#function |
|||
.size _aes256_encrypt_1x,.-_aes256_encrypt_1x |
|||
|
|||
.align 32 |
|||
_aes256_encrypt_2x: |
|||
aes_eround01 %f16, %f0, %f2, %f8 |
|||
aes_eround23 %f18, %f0, %f2, %f2 |
|||
aes_eround01 %f16, %f4, %f6, %f10 |
|||
aes_eround23 %f18, %f4, %f6, %f6 |
|||
ldd [$key + 208], %f16 |
|||
ldd [$key + 216], %f18 |
|||
aes_eround01 %f20, %f8, %f2, %f0 |
|||
aes_eround23 %f22, %f8, %f2, %f2 |
|||
aes_eround01 %f20, %f10, %f6, %f4 |
|||
aes_eround23 %f22, %f10, %f6, %f6 |
|||
ldd [$key + 224], %f20 |
|||
ldd [$key + 232], %f22 |
|||
___ |
|||
for ($i=1; $i<6; $i++) { |
|||
$code.=<<___; |
|||
aes_eround01 %f`16+8*$i+0`, %f0, %f2, %f8 |
|||
aes_eround23 %f`16+8*$i+2`, %f0, %f2, %f2 |
|||
aes_eround01 %f`16+8*$i+0`, %f4, %f6, %f10 |
|||
aes_eround23 %f`16+8*$i+2`, %f4, %f6, %f6 |
|||
aes_eround01 %f`16+8*$i+4`, %f8, %f2, %f0 |
|||
aes_eround23 %f`16+8*$i+6`, %f8, %f2, %f2 |
|||
aes_eround01 %f`16+8*$i+4`, %f10, %f6, %f4 |
|||
aes_eround23 %f`16+8*$i+6`, %f10, %f6, %f6 |
|||
___ |
|||
} |
|||
$code.=<<___; |
|||
aes_eround01 %f16, %f0, %f2, %f8 |
|||
aes_eround23 %f18, %f0, %f2, %f2 |
|||
aes_eround01 %f16, %f4, %f6, %f10 |
|||
aes_eround23 %f18, %f4, %f6, %f6 |
|||
ldd [$key + 16], %f16 |
|||
ldd [$key + 24], %f18 |
|||
aes_eround01_l %f20, %f8, %f2, %f0 |
|||
aes_eround23_l %f22, %f8, %f2, %f2 |
|||
aes_eround01_l %f20, %f10, %f6, %f4 |
|||
aes_eround23_l %f22, %f10, %f6, %f6 |
|||
ldd [$key + 32], %f20 |
|||
retl |
|||
ldd [$key + 40], %f22 |
|||
.type _aes256_encrypt_2x,#function |
|||
.size _aes256_encrypt_2x,.-_aes256_encrypt_2x |
|||
|
|||
.align 32 |
|||
_aes192_loadkey: |
|||
ldx [$key + 0], %g4 |
|||
ldx [$key + 8], %g5 |
|||
___ |
|||
for ($i=2; $i<26;$i++) { # load key schedule |
|||
$code.=<<___; |
|||
ldd [$key + `8*$i`], %f`12+2*$i` |
|||
___ |
|||
} |
|||
$code.=<<___; |
|||
retl |
|||
nop |
|||
.type _aes192_loadkey,#function |
|||
.size _aes192_loadkey,.-_aes192_loadkey |
|||
_aes256_loadkey=_aes192_loadkey |
|||
_aes192_load_enckey=_aes192_loadkey |
|||
_aes192_load_deckey=_aes192_loadkey |
|||
_aes256_load_enckey=_aes192_loadkey |
|||
_aes256_load_deckey=_aes192_loadkey |
|||
___ |
|||
|
|||
&alg_cbc_encrypt_implement("aes",256); |
|||
&alg_cbc_encrypt_implement("aes",192); |
|||
if ($::evp) { |
|||
&alg_ctr32_implement("aes",256); |
|||
&alg_xts_implement("aes",256,"en"); |
|||
&alg_xts_implement("aes",256,"de"); |
|||
&alg_ctr32_implement("aes",192); |
|||
} |
|||
&alg_cbc_decrypt_implement("aes",192); |
|||
&alg_cbc_decrypt_implement("aes",256); |
|||
|
|||
$code.=<<___; |
|||
.align 32 |
|||
_aes256_decrypt_1x: |
|||
aes_dround01 %f16, %f0, %f2, %f4 |
|||
aes_dround23 %f18, %f0, %f2, %f2 |
|||
ldd [$key + 208], %f16 |
|||
ldd [$key + 216], %f18 |
|||
aes_dround01 %f20, %f4, %f2, %f0 |
|||
aes_dround23 %f22, %f4, %f2, %f2 |
|||
ldd [$key + 224], %f20 |
|||
ldd [$key + 232], %f22 |
|||
___ |
|||
for ($i=1; $i<6; $i++) { |
|||
$code.=<<___; |
|||
aes_dround01 %f`16+8*$i+0`, %f0, %f2, %f4 |
|||
aes_dround23 %f`16+8*$i+2`, %f0, %f2, %f2 |
|||
aes_dround01 %f`16+8*$i+4`, %f4, %f2, %f0 |
|||
aes_dround23 %f`16+8*$i+6`, %f4, %f2, %f2 |
|||
___ |
|||
} |
|||
$code.=<<___; |
|||
aes_dround01 %f16, %f0, %f2, %f4 |
|||
aes_dround23 %f18, %f0, %f2, %f2 |
|||
ldd [$key + 16], %f16 |
|||
ldd [$key + 24], %f18 |
|||
aes_dround01_l %f20, %f4, %f2, %f0 |
|||
aes_dround23_l %f22, %f4, %f2, %f2 |
|||
ldd [$key + 32], %f20 |
|||
retl |
|||
ldd [$key + 40], %f22 |
|||
.type _aes256_decrypt_1x,#function |
|||
.size _aes256_decrypt_1x,.-_aes256_decrypt_1x |
|||
|
|||
.align 32 |
|||
_aes256_decrypt_2x: |
|||
aes_dround01 %f16, %f0, %f2, %f8 |
|||
aes_dround23 %f18, %f0, %f2, %f2 |
|||
aes_dround01 %f16, %f4, %f6, %f10 |
|||
aes_dround23 %f18, %f4, %f6, %f6 |
|||
ldd [$key + 208], %f16 |
|||
ldd [$key + 216], %f18 |
|||
aes_dround01 %f20, %f8, %f2, %f0 |
|||
aes_dround23 %f22, %f8, %f2, %f2 |
|||
aes_dround01 %f20, %f10, %f6, %f4 |
|||
aes_dround23 %f22, %f10, %f6, %f6 |
|||
ldd [$key + 224], %f20 |
|||
ldd [$key + 232], %f22 |
|||
___ |
|||
for ($i=1; $i<6; $i++) { |
|||
$code.=<<___; |
|||
aes_dround01 %f`16+8*$i+0`, %f0, %f2, %f8 |
|||
aes_dround23 %f`16+8*$i+2`, %f0, %f2, %f2 |
|||
aes_dround01 %f`16+8*$i+0`, %f4, %f6, %f10 |
|||
aes_dround23 %f`16+8*$i+2`, %f4, %f6, %f6 |
|||
aes_dround01 %f`16+8*$i+4`, %f8, %f2, %f0 |
|||
aes_dround23 %f`16+8*$i+6`, %f8, %f2, %f2 |
|||
aes_dround01 %f`16+8*$i+4`, %f10, %f6, %f4 |
|||
aes_dround23 %f`16+8*$i+6`, %f10, %f6, %f6 |
|||
___ |
|||
} |
|||
$code.=<<___; |
|||
aes_dround01 %f16, %f0, %f2, %f8 |
|||
aes_dround23 %f18, %f0, %f2, %f2 |
|||
aes_dround01 %f16, %f4, %f6, %f10 |
|||
aes_dround23 %f18, %f4, %f6, %f6 |
|||
ldd [$key + 16], %f16 |
|||
ldd [$key + 24], %f18 |
|||
aes_dround01_l %f20, %f8, %f2, %f0 |
|||
aes_dround23_l %f22, %f8, %f2, %f2 |
|||
aes_dround01_l %f20, %f10, %f6, %f4 |
|||
aes_dround23_l %f22, %f10, %f6, %f6 |
|||
ldd [$key + 32], %f20 |
|||
retl |
|||
ldd [$key + 40], %f22 |
|||
.type _aes256_decrypt_2x,#function |
|||
.size _aes256_decrypt_2x,.-_aes256_decrypt_2x |
|||
|
|||
.align 32 |
|||
_aes192_decrypt_1x: |
|||
___ |
|||
for ($i=0; $i<5; $i++) { |
|||
$code.=<<___; |
|||
aes_dround01 %f`16+8*$i+0`, %f0, %f2, %f4 |
|||
aes_dround23 %f`16+8*$i+2`, %f0, %f2, %f2 |
|||
aes_dround01 %f`16+8*$i+4`, %f4, %f2, %f0 |
|||
aes_dround23 %f`16+8*$i+6`, %f4, %f2, %f2 |
|||
___ |
|||
} |
|||
$code.=<<___; |
|||
aes_dround01 %f56, %f0, %f2, %f4 |
|||
aes_dround23 %f58, %f0, %f2, %f2 |
|||
aes_dround01_l %f60, %f4, %f2, %f0 |
|||
retl |
|||
aes_dround23_l %f62, %f4, %f2, %f2 |
|||
.type _aes192_decrypt_1x,#function |
|||
.size _aes192_decrypt_1x,.-_aes192_decrypt_1x |
|||
|
|||
.align 32 |
|||
_aes192_decrypt_2x: |
|||
___ |
|||
for ($i=0; $i<5; $i++) { |
|||
$code.=<<___; |
|||
aes_dround01 %f`16+8*$i+0`, %f0, %f2, %f8 |
|||
aes_dround23 %f`16+8*$i+2`, %f0, %f2, %f2 |
|||
aes_dround01 %f`16+8*$i+0`, %f4, %f6, %f10 |
|||
aes_dround23 %f`16+8*$i+2`, %f4, %f6, %f6 |
|||
aes_dround01 %f`16+8*$i+4`, %f8, %f2, %f0 |
|||
aes_dround23 %f`16+8*$i+6`, %f8, %f2, %f2 |
|||
aes_dround01 %f`16+8*$i+4`, %f10, %f6, %f4 |
|||
aes_dround23 %f`16+8*$i+6`, %f10, %f6, %f6 |
|||
___ |
|||
} |
|||
$code.=<<___; |
|||
aes_dround01 %f56, %f0, %f2, %f8 |
|||
aes_dround23 %f58, %f0, %f2, %f2 |
|||
aes_dround01 %f56, %f4, %f6, %f10 |
|||
aes_dround23 %f58, %f4, %f6, %f6 |
|||
aes_dround01_l %f60, %f8, %f2, %f0 |
|||
aes_dround23_l %f62, %f8, %f2, %f2 |
|||
aes_dround01_l %f60, %f10, %f6, %f4 |
|||
retl |
|||
aes_dround23_l %f62, %f10, %f6, %f6 |
|||
.type _aes192_decrypt_2x,#function |
|||
.size _aes192_decrypt_2x,.-_aes192_decrypt_2x |
|||
___ |
|||
}}} |
|||
|
|||
if (!$::evp) { |
|||
$code.=<<___; |
|||
.global AES_encrypt |
|||
AES_encrypt=aes_t4_encrypt |
|||
.global AES_decrypt |
|||
AES_decrypt=aes_t4_decrypt |
|||
.global AES_set_encrypt_key |
|||
.align 32 |
|||
AES_set_encrypt_key: |
|||
andcc %o2, 7, %g0 ! check alignment |
|||
bnz,a,pn %icc, 1f |
|||
mov -1, %o0 |
|||
brz,a,pn %o0, 1f |
|||
mov -1, %o0 |
|||
brz,a,pn %o2, 1f |
|||
mov -1, %o0 |
|||
andncc %o1, 0x1c0, %g0 |
|||
bnz,a,pn %icc, 1f |
|||
mov -2, %o0 |
|||
cmp %o1, 128 |
|||
bl,a,pn %icc, 1f |
|||
mov -2, %o0 |
|||
b aes_t4_set_encrypt_key |
|||
nop |
|||
1: retl |
|||
nop |
|||
.type AES_set_encrypt_key,#function |
|||
.size AES_set_encrypt_key,.-AES_set_encrypt_key |
|||
|
|||
.global AES_set_decrypt_key |
|||
.align 32 |
|||
AES_set_decrypt_key: |
|||
andcc %o2, 7, %g0 ! check alignment |
|||
bnz,a,pn %icc, 1f |
|||
mov -1, %o0 |
|||
brz,a,pn %o0, 1f |
|||
mov -1, %o0 |
|||
brz,a,pn %o2, 1f |
|||
mov -1, %o0 |
|||
andncc %o1, 0x1c0, %g0 |
|||
bnz,a,pn %icc, 1f |
|||
mov -2, %o0 |
|||
cmp %o1, 128 |
|||
bl,a,pn %icc, 1f |
|||
mov -2, %o0 |
|||
b aes_t4_set_decrypt_key |
|||
nop |
|||
1: retl |
|||
nop |
|||
.type AES_set_decrypt_key,#function |
|||
.size AES_set_decrypt_key,.-AES_set_decrypt_key |
|||
___ |
|||
|
|||
my ($inp,$out,$len,$key,$ivec,$enc)=map("%o$_",(0..5)); |
|||
|
|||
$code.=<<___; |
|||
.globl AES_cbc_encrypt |
|||
.align 32 |
|||
AES_cbc_encrypt: |
|||
ld [$key + 240], %g1 |
|||
nop |
|||
brz $enc, .Lcbc_decrypt |
|||
cmp %g1, 12 |
|||
|
|||
bl,pt %icc, aes128_t4_cbc_encrypt |
|||
nop |
|||
be,pn %icc, aes192_t4_cbc_encrypt |
|||
nop |
|||
ba aes256_t4_cbc_encrypt |
|||
nop |
|||
|
|||
.Lcbc_decrypt: |
|||
bl,pt %icc, aes128_t4_cbc_decrypt |
|||
nop |
|||
be,pn %icc, aes192_t4_cbc_decrypt |
|||
nop |
|||
ba aes256_t4_cbc_decrypt |
|||
nop |
|||
.type AES_cbc_encrypt,#function |
|||
.size AES_cbc_encrypt,.-AES_cbc_encrypt |
|||
___ |
|||
} |
|||
$code.=<<___; |
|||
.asciz "AES for SPARC T4, David S. Miller, Andy Polyakov" |
|||
.align 4 |
|||
___ |
|||
|
|||
&emit_assembler(); |
|||
|
|||
close STDOUT; |
@ -0,0 +1,962 @@ |
|||
#!/usr/bin/env perl |
|||
# |
|||
# ==================================================================== |
|||
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL |
|||
# project. The module is, however, dual licensed under OpenSSL and |
|||
# CRYPTOGAMS licenses depending on where you obtain it. For further |
|||
# details see http://www.openssl.org/~appro/cryptogams/. |
|||
# ==================================================================== |
|||
# |
|||
# This module implements support for ARMv8 AES instructions. The |
|||
# module is endian-agnostic in sense that it supports both big- and |
|||
# little-endian cases. As does it support both 32- and 64-bit modes |
|||
# of operation. Latter is achieved by limiting amount of utilized |
|||
# registers to 16, which implies additional NEON load and integer |
|||
# instructions. This has no effect on mighty Apple A7, where results |
|||
# are literally equal to the theoretical estimates based on AES |
|||
# instruction latencies and issue rates. On Cortex-A53, an in-order |
|||
# execution core, this costs up to 10-15%, which is partially |
|||
# compensated by implementing dedicated code path for 128-bit |
|||
# CBC encrypt case. On Cortex-A57 parallelizable mode performance |
|||
# seems to be limited by sheer amount of NEON instructions... |
|||
# |
|||
# Performance in cycles per byte processed with 128-bit key: |
|||
# |
|||
# CBC enc CBC dec CTR |
|||
# Apple A7 2.39 1.20 1.20 |
|||
# Cortex-A53 2.45 1.87 1.94 |
|||
# Cortex-A57 3.64 1.34 1.32 |
|||
|
|||
$flavour = shift; |
|||
open STDOUT,">".shift; |
|||
|
|||
$prefix="aes_v8"; |
|||
|
|||
$code=<<___; |
|||
#include "arm_arch.h" |
|||
|
|||
#if __ARM_MAX_ARCH__>=7 |
|||
.text |
|||
___ |
|||
$code.=".arch armv8-a+crypto\n" if ($flavour =~ /64/); |
|||
$code.=".arch armv7-a\n.fpu neon\n.code 32\n" if ($flavour !~ /64/); |
|||
#^^^^^^ this is done to simplify adoption by not depending |
|||
# on latest binutils. |
|||
|
|||
# Assembler mnemonics are an eclectic mix of 32- and 64-bit syntax, |
|||
# NEON is mostly 32-bit mnemonics, integer - mostly 64. Goal is to |
|||
# maintain both 32- and 64-bit codes within single module and |
|||
# transliterate common code to either flavour with regex vodoo. |
|||
# |
|||
{{{ |
|||
my ($inp,$bits,$out,$ptr,$rounds)=("x0","w1","x2","x3","w12"); |
|||
my ($zero,$rcon,$mask,$in0,$in1,$tmp,$key)= |
|||
$flavour=~/64/? map("q$_",(0..6)) : map("q$_",(0..3,8..10)); |
|||
|
|||
|
|||
$code.=<<___; |
|||
.align 5 |
|||
rcon: |
|||
.long 0x01,0x01,0x01,0x01 |
|||
.long 0x0c0f0e0d,0x0c0f0e0d,0x0c0f0e0d,0x0c0f0e0d // rotate-n-splat |
|||
.long 0x1b,0x1b,0x1b,0x1b |
|||
|
|||
.globl ${prefix}_set_encrypt_key |
|||
.type ${prefix}_set_encrypt_key,%function |
|||
.align 5 |
|||
${prefix}_set_encrypt_key: |
|||
.Lenc_key: |
|||
___ |
|||
$code.=<<___ if ($flavour =~ /64/); |
|||
stp x29,x30,[sp,#-16]! |
|||
add x29,sp,#0 |
|||
___ |
|||
$code.=<<___; |
|||
mov $ptr,#-1 |
|||
cmp $inp,#0 |
|||
b.eq .Lenc_key_abort |
|||
cmp $out,#0 |
|||
b.eq .Lenc_key_abort |
|||
mov $ptr,#-2 |
|||
cmp $bits,#128 |
|||
b.lt .Lenc_key_abort |
|||
cmp $bits,#256 |
|||
b.gt .Lenc_key_abort |
|||
tst $bits,#0x3f |
|||
b.ne .Lenc_key_abort |
|||
|
|||
adr $ptr,rcon |
|||
cmp $bits,#192 |
|||
|
|||
veor $zero,$zero,$zero |
|||
vld1.8 {$in0},[$inp],#16 |
|||
mov $bits,#8 // reuse $bits |
|||
vld1.32 {$rcon,$mask},[$ptr],#32 |
|||
|
|||
b.lt .Loop128 |
|||
b.eq .L192 |
|||
b .L256 |
|||
|
|||
.align 4 |
|||
.Loop128: |
|||
vtbl.8 $key,{$in0},$mask |
|||
vext.8 $tmp,$zero,$in0,#12 |
|||
vst1.32 {$in0},[$out],#16 |
|||
aese $key,$zero |
|||
subs $bits,$bits,#1 |
|||
|
|||
veor $in0,$in0,$tmp |
|||
vext.8 $tmp,$zero,$tmp,#12 |
|||
veor $in0,$in0,$tmp |
|||
vext.8 $tmp,$zero,$tmp,#12 |
|||
veor $key,$key,$rcon |
|||
veor $in0,$in0,$tmp |
|||
vshl.u8 $rcon,$rcon,#1 |
|||
veor $in0,$in0,$key |
|||
b.ne .Loop128 |
|||
|
|||
vld1.32 {$rcon},[$ptr] |
|||
|
|||
vtbl.8 $key,{$in0},$mask |
|||
vext.8 $tmp,$zero,$in0,#12 |
|||
vst1.32 {$in0},[$out],#16 |
|||
aese $key,$zero |
|||
|
|||
veor $in0,$in0,$tmp |
|||
vext.8 $tmp,$zero,$tmp,#12 |
|||
veor $in0,$in0,$tmp |
|||
vext.8 $tmp,$zero,$tmp,#12 |
|||
veor $key,$key,$rcon |
|||
veor $in0,$in0,$tmp |
|||
vshl.u8 $rcon,$rcon,#1 |
|||
veor $in0,$in0,$key |
|||
|
|||
vtbl.8 $key,{$in0},$mask |
|||
vext.8 $tmp,$zero,$in0,#12 |
|||
vst1.32 {$in0},[$out],#16 |
|||
aese $key,$zero |
|||
|
|||
veor $in0,$in0,$tmp |
|||
vext.8 $tmp,$zero,$tmp,#12 |
|||
veor $in0,$in0,$tmp |
|||
vext.8 $tmp,$zero,$tmp,#12 |
|||
veor $key,$key,$rcon |
|||
veor $in0,$in0,$tmp |
|||
veor $in0,$in0,$key |
|||
vst1.32 {$in0},[$out] |
|||
add $out,$out,#0x50 |
|||
|
|||
mov $rounds,#10 |
|||
b .Ldone |
|||
|
|||
.align 4 |
|||
.L192: |
|||
vld1.8 {$in1},[$inp],#8 |
|||
vmov.i8 $key,#8 // borrow $key |
|||
vst1.32 {$in0},[$out],#16 |
|||
vsub.i8 $mask,$mask,$key // adjust the mask |
|||
|
|||
.Loop192: |
|||
vtbl.8 $key,{$in1},$mask |
|||
vext.8 $tmp,$zero,$in0,#12 |
|||
vst1.32 {$in1},[$out],#8 |
|||
aese $key,$zero |
|||
subs $bits,$bits,#1 |
|||
|
|||
veor $in0,$in0,$tmp |
|||
vext.8 $tmp,$zero,$tmp,#12 |
|||
veor $in0,$in0,$tmp |
|||
vext.8 $tmp,$zero,$tmp,#12 |
|||
veor $in0,$in0,$tmp |
|||
|
|||
vdup.32 $tmp,${in0}[3] |
|||
veor $tmp,$tmp,$in1 |
|||
veor $key,$key,$rcon |
|||
vext.8 $in1,$zero,$in1,#12 |
|||
vshl.u8 $rcon,$rcon,#1 |
|||
veor $in1,$in1,$tmp |
|||
veor $in0,$in0,$key |
|||
veor $in1,$in1,$key |
|||
vst1.32 {$in0},[$out],#16 |
|||
b.ne .Loop192 |
|||
|
|||
mov $rounds,#12 |
|||
add $out,$out,#0x20 |
|||
b .Ldone |
|||
|
|||
.align 4 |
|||
.L256: |
|||
vld1.8 {$in1},[$inp] |
|||
mov $bits,#7 |
|||
mov $rounds,#14 |
|||
vst1.32 {$in0},[$out],#16 |
|||
|
|||
.Loop256: |
|||
vtbl.8 $key,{$in1},$mask |
|||
vext.8 $tmp,$zero,$in0,#12 |
|||
vst1.32 {$in1},[$out],#16 |
|||
aese $key,$zero |
|||
subs $bits,$bits,#1 |
|||
|
|||
veor $in0,$in0,$tmp |
|||
vext.8 $tmp,$zero,$tmp,#12 |
|||
veor $in0,$in0,$tmp |
|||
vext.8 $tmp,$zero,$tmp,#12 |
|||
veor $key,$key,$rcon |
|||
veor $in0,$in0,$tmp |
|||
vshl.u8 $rcon,$rcon,#1 |
|||
veor $in0,$in0,$key |
|||
vst1.32 {$in0},[$out],#16 |
|||
b.eq .Ldone |
|||
|
|||
vdup.32 $key,${in0}[3] // just splat |
|||
vext.8 $tmp,$zero,$in1,#12 |
|||
aese $key,$zero |
|||
|
|||
veor $in1,$in1,$tmp |
|||
vext.8 $tmp,$zero,$tmp,#12 |
|||
veor $in1,$in1,$tmp |
|||
vext.8 $tmp,$zero,$tmp,#12 |
|||
veor $in1,$in1,$tmp |
|||
|
|||
veor $in1,$in1,$key |
|||
b .Loop256 |
|||
|
|||
.Ldone: |
|||
str $rounds,[$out] |
|||
mov $ptr,#0 |
|||
|
|||
.Lenc_key_abort: |
|||
mov x0,$ptr // return value |
|||
`"ldr x29,[sp],#16" if ($flavour =~ /64/)` |
|||
ret |
|||
.size ${prefix}_set_encrypt_key,.-${prefix}_set_encrypt_key |
|||
|
|||
.globl ${prefix}_set_decrypt_key |
|||
.type ${prefix}_set_decrypt_key,%function |
|||
.align 5 |
|||
${prefix}_set_decrypt_key: |
|||
___ |
|||
$code.=<<___ if ($flavour =~ /64/); |
|||
stp x29,x30,[sp,#-16]! |
|||
add x29,sp,#0 |
|||
___ |
|||
$code.=<<___ if ($flavour !~ /64/); |
|||
stmdb sp!,{r4,lr} |
|||
___ |
|||
$code.=<<___; |
|||
bl .Lenc_key |
|||
|
|||
cmp x0,#0 |
|||
b.ne .Ldec_key_abort |
|||
|
|||
sub $out,$out,#240 // restore original $out |
|||
mov x4,#-16 |
|||
add $inp,$out,x12,lsl#4 // end of key schedule |
|||
|
|||
vld1.32 {v0.16b},[$out] |
|||
vld1.32 {v1.16b},[$inp] |
|||
vst1.32 {v0.16b},[$inp],x4 |
|||
vst1.32 {v1.16b},[$out],#16 |
|||
|
|||
.Loop_imc: |
|||
vld1.32 {v0.16b},[$out] |
|||
vld1.32 {v1.16b},[$inp] |
|||
aesimc v0.16b,v0.16b |
|||
aesimc v1.16b,v1.16b |
|||
vst1.32 {v0.16b},[$inp],x4 |
|||
vst1.32 {v1.16b},[$out],#16 |
|||
cmp $inp,$out |
|||
b.hi .Loop_imc |
|||
|
|||
vld1.32 {v0.16b},[$out] |
|||
aesimc v0.16b,v0.16b |
|||
vst1.32 {v0.16b},[$inp] |
|||
|
|||
eor x0,x0,x0 // return value |
|||
.Ldec_key_abort: |
|||
___ |
|||
$code.=<<___ if ($flavour !~ /64/); |
|||
ldmia sp!,{r4,pc} |
|||
___ |
|||
$code.=<<___ if ($flavour =~ /64/); |
|||
ldp x29,x30,[sp],#16 |
|||
ret |
|||
___ |
|||
$code.=<<___; |
|||
.size ${prefix}_set_decrypt_key,.-${prefix}_set_decrypt_key |
|||
___ |
|||
}}} |
|||
{{{ |
|||
sub gen_block () { |
|||
my $dir = shift; |
|||
my ($e,$mc) = $dir eq "en" ? ("e","mc") : ("d","imc"); |
|||
my ($inp,$out,$key)=map("x$_",(0..2)); |
|||
my $rounds="w3"; |
|||
my ($rndkey0,$rndkey1,$inout)=map("q$_",(0..3)); |
|||
|
|||
$code.=<<___; |
|||
.globl ${prefix}_${dir}crypt |
|||
.type ${prefix}_${dir}crypt,%function |
|||
.align 5 |
|||
${prefix}_${dir}crypt: |
|||
ldr $rounds,[$key,#240] |
|||
vld1.32 {$rndkey0},[$key],#16 |
|||
vld1.8 {$inout},[$inp] |
|||
sub $rounds,$rounds,#2 |
|||
vld1.32 {$rndkey1},[$key],#16 |
|||
|
|||
.Loop_${dir}c: |
|||
aes$e $inout,$rndkey0 |
|||
vld1.32 {$rndkey0},[$key],#16 |
|||
aes$mc $inout,$inout |
|||
subs $rounds,$rounds,#2 |
|||
aes$e $inout,$rndkey1 |
|||
vld1.32 {$rndkey1},[$key],#16 |
|||
aes$mc $inout,$inout |
|||
b.gt .Loop_${dir}c |
|||
|
|||
aes$e $inout,$rndkey0 |
|||
vld1.32 {$rndkey0},[$key] |
|||
aes$mc $inout,$inout |
|||
aes$e $inout,$rndkey1 |
|||
veor $inout,$inout,$rndkey0 |
|||
|
|||
vst1.8 {$inout},[$out] |
|||
ret |
|||
.size ${prefix}_${dir}crypt,.-${prefix}_${dir}crypt |
|||
___ |
|||
} |
|||
&gen_block("en"); |
|||
&gen_block("de"); |
|||
}}} |
|||
{{{ |
|||
my ($inp,$out,$len,$key,$ivp)=map("x$_",(0..4)); my $enc="w5"; |
|||
my ($rounds,$cnt,$key_,$step,$step1)=($enc,"w6","x7","x8","x12"); |
|||
my ($dat0,$dat1,$in0,$in1,$tmp0,$tmp1,$ivec,$rndlast)=map("q$_",(0..7)); |
|||
|
|||
my ($dat,$tmp,$rndzero_n_last)=($dat0,$tmp0,$tmp1); |
|||
|
|||
### q8-q15 preloaded key schedule |
|||
|
|||
$code.=<<___; |
|||
.globl ${prefix}_cbc_encrypt |
|||
.type ${prefix}_cbc_encrypt,%function |
|||
.align 5 |
|||
${prefix}_cbc_encrypt: |
|||
___ |
|||
$code.=<<___ if ($flavour =~ /64/); |
|||
stp x29,x30,[sp,#-16]! |
|||
add x29,sp,#0 |
|||
___ |
|||
$code.=<<___ if ($flavour !~ /64/); |
|||
mov ip,sp |
|||
stmdb sp!,{r4-r8,lr} |
|||
vstmdb sp!,{d8-d15} @ ABI specification says so |
|||
ldmia ip,{r4-r5} @ load remaining args |
|||
___ |
|||
$code.=<<___; |
|||
subs $len,$len,#16 |
|||
mov $step,#16 |
|||
b.lo .Lcbc_abort |
|||
cclr $step,eq |
|||
|
|||
cmp $enc,#0 // en- or decrypting? |
|||
ldr $rounds,[$key,#240] |
|||
and $len,$len,#-16 |
|||
vld1.8 {$ivec},[$ivp] |
|||
vld1.8 {$dat},[$inp],$step |
|||
|
|||
vld1.32 {q8-q9},[$key] // load key schedule... |
|||
sub $rounds,$rounds,#6 |
|||
add $key_,$key,x5,lsl#4 // pointer to last 7 round keys |
|||
sub $rounds,$rounds,#2 |
|||
vld1.32 {q10-q11},[$key_],#32 |
|||
vld1.32 {q12-q13},[$key_],#32 |
|||
vld1.32 {q14-q15},[$key_],#32 |
|||
vld1.32 {$rndlast},[$key_] |
|||
|
|||
add $key_,$key,#32 |
|||
mov $cnt,$rounds |
|||
b.eq .Lcbc_dec |
|||
|
|||
cmp $rounds,#2 |
|||
veor $dat,$dat,$ivec |
|||
veor $rndzero_n_last,q8,$rndlast |
|||
b.eq .Lcbc_enc128 |
|||
|
|||
.Loop_cbc_enc: |
|||
aese $dat,q8 |
|||
vld1.32 {q8},[$key_],#16 |
|||
aesmc $dat,$dat |
|||
subs $cnt,$cnt,#2 |
|||
aese $dat,q9 |
|||
vld1.32 {q9},[$key_],#16 |
|||
aesmc $dat,$dat |
|||
b.gt .Loop_cbc_enc |
|||
|
|||
aese $dat,q8 |
|||
aesmc $dat,$dat |
|||
subs $len,$len,#16 |
|||
aese $dat,q9 |
|||
aesmc $dat,$dat |
|||
cclr $step,eq |
|||
aese $dat,q10 |
|||
aesmc $dat,$dat |
|||
add $key_,$key,#16 |
|||
aese $dat,q11 |
|||
aesmc $dat,$dat |
|||
vld1.8 {q8},[$inp],$step |
|||
aese $dat,q12 |
|||
aesmc $dat,$dat |
|||
veor q8,q8,$rndzero_n_last |
|||
aese $dat,q13 |
|||
aesmc $dat,$dat |
|||
vld1.32 {q9},[$key_],#16 // re-pre-load rndkey[1] |
|||
aese $dat,q14 |
|||
aesmc $dat,$dat |
|||
aese $dat,q15 |
|||
|
|||
mov $cnt,$rounds |
|||
veor $ivec,$dat,$rndlast |
|||
vst1.8 {$ivec},[$out],#16 |
|||
b.hs .Loop_cbc_enc |
|||
|
|||
b .Lcbc_done |
|||
|
|||
.align 5 |
|||
.Lcbc_enc128: |
|||
vld1.32 {$in0-$in1},[$key_] |
|||
aese $dat,q8 |
|||
aesmc $dat,$dat |
|||
b .Lenter_cbc_enc128 |
|||
.Loop_cbc_enc128: |
|||
aese $dat,q8 |
|||
aesmc $dat,$dat |
|||
vst1.8 {$ivec},[$out],#16 |
|||
.Lenter_cbc_enc128: |
|||
aese $dat,q9 |
|||
aesmc $dat,$dat |
|||
subs $len,$len,#16 |
|||
aese $dat,$in0 |
|||
aesmc $dat,$dat |
|||
cclr $step,eq |
|||
aese $dat,$in1 |
|||
aesmc $dat,$dat |
|||
aese $dat,q10 |
|||
aesmc $dat,$dat |
|||
aese $dat,q11 |
|||
aesmc $dat,$dat |
|||
vld1.8 {q8},[$inp],$step |
|||
aese $dat,q12 |
|||
aesmc $dat,$dat |
|||
aese $dat,q13 |
|||
aesmc $dat,$dat |
|||
aese $dat,q14 |
|||
aesmc $dat,$dat |
|||
veor q8,q8,$rndzero_n_last |
|||
aese $dat,q15 |
|||
veor $ivec,$dat,$rndlast |
|||
b.hs .Loop_cbc_enc128 |
|||
|
|||
vst1.8 {$ivec},[$out],#16 |
|||
b .Lcbc_done |
|||
___ |
|||
{ |
|||
my ($dat2,$in2,$tmp2)=map("q$_",(10,11,9)); |
|||
$code.=<<___; |
|||
.align 5 |
|||
.Lcbc_dec: |
|||
vld1.8 {$dat2},[$inp],#16 |
|||
subs $len,$len,#32 // bias |
|||
add $cnt,$rounds,#2 |
|||
vorr $in1,$dat,$dat |
|||
vorr $dat1,$dat,$dat |
|||
vorr $in2,$dat2,$dat2 |
|||
b.lo .Lcbc_dec_tail |
|||
|
|||
vorr $dat1,$dat2,$dat2 |
|||
vld1.8 {$dat2},[$inp],#16 |
|||
vorr $in0,$dat,$dat |
|||
vorr $in1,$dat1,$dat1 |
|||
vorr $in2,$dat2,$dat2 |
|||
|
|||
.Loop3x_cbc_dec: |
|||
aesd $dat0,q8 |
|||
aesd $dat1,q8 |
|||
aesd $dat2,q8 |
|||
vld1.32 {q8},[$key_],#16 |
|||
aesimc $dat0,$dat0 |
|||
aesimc $dat1,$dat1 |
|||
aesimc $dat2,$dat2 |
|||
subs $cnt,$cnt,#2 |
|||
aesd $dat0,q9 |
|||
aesd $dat1,q9 |
|||
aesd $dat2,q9 |
|||
vld1.32 {q9},[$key_],#16 |
|||
aesimc $dat0,$dat0 |
|||
aesimc $dat1,$dat1 |
|||
aesimc $dat2,$dat2 |
|||
b.gt .Loop3x_cbc_dec |
|||
|
|||
aesd $dat0,q8 |
|||
aesd $dat1,q8 |
|||
aesd $dat2,q8 |
|||
veor $tmp0,$ivec,$rndlast |
|||
aesimc $dat0,$dat0 |
|||
aesimc $dat1,$dat1 |
|||
aesimc $dat2,$dat2 |
|||
veor $tmp1,$in0,$rndlast |
|||
aesd $dat0,q9 |
|||
aesd $dat1,q9 |
|||
aesd $dat2,q9 |
|||
veor $tmp2,$in1,$rndlast |
|||
subs $len,$len,#0x30 |
|||
aesimc $dat0,$dat0 |
|||
aesimc $dat1,$dat1 |
|||
aesimc $dat2,$dat2 |
|||
vorr $ivec,$in2,$in2 |
|||
mov.lo x6,$len // x6, $cnt, is zero at this point |
|||
aesd $dat0,q12 |
|||
aesd $dat1,q12 |
|||
aesd $dat2,q12 |
|||
add $inp,$inp,x6 // $inp is adjusted in such way that |
|||
// at exit from the loop $dat1-$dat2 |
|||
// are loaded with last "words" |
|||
aesimc $dat0,$dat0 |
|||
aesimc $dat1,$dat1 |
|||
aesimc $dat2,$dat2 |
|||
mov $key_,$key |
|||
aesd $dat0,q13 |
|||
aesd $dat1,q13 |
|||
aesd $dat2,q13 |
|||
vld1.8 {$in0},[$inp],#16 |
|||
aesimc $dat0,$dat0 |
|||
aesimc $dat1,$dat1 |
|||
aesimc $dat2,$dat2 |
|||
vld1.8 {$in1},[$inp],#16 |
|||
aesd $dat0,q14 |
|||
aesd $dat1,q14 |
|||
aesd $dat2,q14 |
|||
vld1.8 {$in2},[$inp],#16 |
|||
aesimc $dat0,$dat0 |
|||
aesimc $dat1,$dat1 |
|||
aesimc $dat2,$dat2 |
|||
vld1.32 {q8},[$key_],#16 // re-pre-load rndkey[0] |
|||
aesd $dat0,q15 |
|||
aesd $dat1,q15 |
|||
aesd $dat2,q15 |
|||
|
|||
add $cnt,$rounds,#2 |
|||
veor $tmp0,$tmp0,$dat0 |
|||
veor $tmp1,$tmp1,$dat1 |
|||
veor $dat2,$dat2,$tmp2 |
|||
vld1.32 {q9},[$key_],#16 // re-pre-load rndkey[1] |
|||
vorr $dat0,$in0,$in0 |
|||
vst1.8 {$tmp0},[$out],#16 |
|||
vorr $dat1,$in1,$in1 |
|||
vst1.8 {$tmp1},[$out],#16 |
|||
vst1.8 {$dat2},[$out],#16 |
|||
vorr $dat2,$in2,$in2 |
|||
b.hs .Loop3x_cbc_dec |
|||
|
|||
cmn $len,#0x30 |
|||
b.eq .Lcbc_done |
|||
nop |
|||
|
|||
.Lcbc_dec_tail: |
|||
aesd $dat1,q8 |
|||
aesd $dat2,q8 |
|||
vld1.32 {q8},[$key_],#16 |
|||
aesimc $dat1,$dat1 |
|||
aesimc $dat2,$dat2 |
|||
subs $cnt,$cnt,#2 |
|||
aesd $dat1,q9 |
|||
aesd $dat2,q9 |
|||
vld1.32 {q9},[$key_],#16 |
|||
aesimc $dat1,$dat1 |
|||
aesimc $dat2,$dat2 |
|||
b.gt .Lcbc_dec_tail |
|||
|
|||
aesd $dat1,q8 |
|||
aesd $dat2,q8 |
|||
aesimc $dat1,$dat1 |
|||
aesimc $dat2,$dat2 |
|||
aesd $dat1,q9 |
|||
aesd $dat2,q9 |
|||
aesimc $dat1,$dat1 |
|||
aesimc $dat2,$dat2 |
|||
aesd $dat1,q12 |
|||
aesd $dat2,q12 |
|||
aesimc $dat1,$dat1 |
|||
aesimc $dat2,$dat2 |
|||
cmn $len,#0x20 |
|||
aesd $dat1,q13 |
|||
aesd $dat2,q13 |
|||
aesimc $dat1,$dat1 |
|||
aesimc $dat2,$dat2 |
|||
veor $tmp1,$ivec,$rndlast |
|||
aesd $dat1,q14 |
|||
aesd $dat2,q14 |
|||
aesimc $dat1,$dat1 |
|||
aesimc $dat2,$dat2 |
|||
veor $tmp2,$in1,$rndlast |
|||
aesd $dat1,q15 |
|||
aesd $dat2,q15 |
|||
b.eq .Lcbc_dec_one |
|||
veor $tmp1,$tmp1,$dat1 |
|||
veor $tmp2,$tmp2,$dat2 |
|||
vorr $ivec,$in2,$in2 |
|||
vst1.8 {$tmp1},[$out],#16 |
|||
vst1.8 {$tmp2},[$out],#16 |
|||
b .Lcbc_done |
|||
|
|||
.Lcbc_dec_one: |
|||
veor $tmp1,$tmp1,$dat2 |
|||
vorr $ivec,$in2,$in2 |
|||
vst1.8 {$tmp1},[$out],#16 |
|||
|
|||
.Lcbc_done: |
|||
vst1.8 {$ivec},[$ivp] |
|||
.Lcbc_abort: |
|||
___ |
|||
} |
|||
$code.=<<___ if ($flavour !~ /64/); |
|||
vldmia sp!,{d8-d15} |
|||
ldmia sp!,{r4-r8,pc} |
|||
___ |
|||
$code.=<<___ if ($flavour =~ /64/); |
|||
ldr x29,[sp],#16 |
|||
ret |
|||
___ |
|||
$code.=<<___; |
|||
.size ${prefix}_cbc_encrypt,.-${prefix}_cbc_encrypt |
|||
___ |
|||
}}} |
|||
{{{ |
|||
my ($inp,$out,$len,$key,$ivp)=map("x$_",(0..4)); |
|||
my ($rounds,$cnt,$key_)=("w5","w6","x7"); |
|||
my ($ctr,$tctr0,$tctr1,$tctr2)=map("w$_",(8..10,12)); |
|||
my $step="x12"; # aliases with $tctr2 |
|||
|
|||
my ($dat0,$dat1,$in0,$in1,$tmp0,$tmp1,$ivec,$rndlast)=map("q$_",(0..7)); |
|||
my ($dat2,$in2,$tmp2)=map("q$_",(10,11,9)); |
|||
|
|||
my ($dat,$tmp)=($dat0,$tmp0); |
|||
|
|||
### q8-q15 preloaded key schedule |
|||
|
|||
$code.=<<___; |
|||
.globl ${prefix}_ctr32_encrypt_blocks |
|||
.type ${prefix}_ctr32_encrypt_blocks,%function |
|||
.align 5 |
|||
${prefix}_ctr32_encrypt_blocks: |
|||
___ |
|||
$code.=<<___ if ($flavour =~ /64/); |
|||
stp x29,x30,[sp,#-16]! |
|||
add x29,sp,#0 |
|||
___ |
|||
$code.=<<___ if ($flavour !~ /64/); |
|||
mov ip,sp |
|||
stmdb sp!,{r4-r10,lr} |
|||
vstmdb sp!,{d8-d15} @ ABI specification says so |
|||
ldr r4, [ip] @ load remaining arg |
|||
___ |
|||
$code.=<<___; |
|||
ldr $rounds,[$key,#240] |
|||
|
|||
ldr $ctr, [$ivp, #12] |
|||
vld1.32 {$dat0},[$ivp] |
|||
|
|||
vld1.32 {q8-q9},[$key] // load key schedule... |
|||
sub $rounds,$rounds,#4 |
|||
mov $step,#16 |
|||
cmp $len,#2 |
|||
add $key_,$key,x5,lsl#4 // pointer to last 5 round keys |
|||
sub $rounds,$rounds,#2 |
|||
vld1.32 {q12-q13},[$key_],#32 |
|||
vld1.32 {q14-q15},[$key_],#32 |
|||
vld1.32 {$rndlast},[$key_] |
|||
add $key_,$key,#32 |
|||
mov $cnt,$rounds |
|||
cclr $step,lo |
|||
#ifndef __ARMEB__ |
|||
rev $ctr, $ctr |
|||
#endif |
|||
vorr $dat1,$dat0,$dat0 |
|||
add $tctr1, $ctr, #1 |
|||
vorr $dat2,$dat0,$dat0 |
|||
add $ctr, $ctr, #2 |
|||
vorr $ivec,$dat0,$dat0 |
|||
rev $tctr1, $tctr1 |
|||
vmov.32 ${dat1}[3],$tctr1 |
|||
b.ls .Lctr32_tail |
|||
rev $tctr2, $ctr |
|||
sub $len,$len,#3 // bias |
|||
vmov.32 ${dat2}[3],$tctr2 |
|||
b .Loop3x_ctr32 |
|||
|
|||
.align 4 |
|||
.Loop3x_ctr32: |
|||
aese $dat0,q8 |
|||
aese $dat1,q8 |
|||
aese $dat2,q8 |
|||
vld1.32 {q8},[$key_],#16 |
|||
aesmc $dat0,$dat0 |
|||
aesmc $dat1,$dat1 |
|||
aesmc $dat2,$dat2 |
|||
subs $cnt,$cnt,#2 |
|||
aese $dat0,q9 |
|||
aese $dat1,q9 |
|||
aese $dat2,q9 |
|||
vld1.32 {q9},[$key_],#16 |
|||
aesmc $dat0,$dat0 |
|||
aesmc $dat1,$dat1 |
|||
aesmc $dat2,$dat2 |
|||
b.gt .Loop3x_ctr32 |
|||
|
|||
aese $dat0,q8 |
|||
aese $dat1,q8 |
|||
aese $dat2,q8 |
|||
mov $key_,$key |
|||
aesmc $tmp0,$dat0 |
|||
vld1.8 {$in0},[$inp],#16 |
|||
aesmc $tmp1,$dat1 |
|||
aesmc $dat2,$dat2 |
|||
vorr $dat0,$ivec,$ivec |
|||
aese $tmp0,q9 |
|||
vld1.8 {$in1},[$inp],#16 |
|||
aese $tmp1,q9 |
|||
aese $dat2,q9 |
|||
vorr $dat1,$ivec,$ivec |
|||
aesmc $tmp0,$tmp0 |
|||
vld1.8 {$in2},[$inp],#16 |
|||
aesmc $tmp1,$tmp1 |
|||
aesmc $tmp2,$dat2 |
|||
vorr $dat2,$ivec,$ivec |
|||
add $tctr0,$ctr,#1 |
|||
aese $tmp0,q12 |
|||
aese $tmp1,q12 |
|||
aese $tmp2,q12 |
|||
veor $in0,$in0,$rndlast |
|||
add $tctr1,$ctr,#2 |
|||
aesmc $tmp0,$tmp0 |
|||
aesmc $tmp1,$tmp1 |
|||
aesmc $tmp2,$tmp2 |
|||
veor $in1,$in1,$rndlast |
|||
add $ctr,$ctr,#3 |
|||
aese $tmp0,q13 |
|||
aese $tmp1,q13 |
|||
aese $tmp2,q13 |
|||
veor $in2,$in2,$rndlast |
|||
rev $tctr0,$tctr0 |
|||
aesmc $tmp0,$tmp0 |
|||
vld1.32 {q8},[$key_],#16 // re-pre-load rndkey[0] |
|||
aesmc $tmp1,$tmp1 |
|||
aesmc $tmp2,$tmp2 |
|||
vmov.32 ${dat0}[3], $tctr0 |
|||
rev $tctr1,$tctr1 |
|||
aese $tmp0,q14 |
|||
aese $tmp1,q14 |
|||
aese $tmp2,q14 |
|||
vmov.32 ${dat1}[3], $tctr1 |
|||
rev $tctr2,$ctr |
|||
aesmc $tmp0,$tmp0 |
|||
aesmc $tmp1,$tmp1 |
|||
aesmc $tmp2,$tmp2 |
|||
vmov.32 ${dat2}[3], $tctr2 |
|||
subs $len,$len,#3 |
|||
aese $tmp0,q15 |
|||
aese $tmp1,q15 |
|||
aese $tmp2,q15 |
|||
|
|||
mov $cnt,$rounds |
|||
veor $in0,$in0,$tmp0 |
|||
veor $in1,$in1,$tmp1 |
|||
veor $in2,$in2,$tmp2 |
|||
vld1.32 {q9},[$key_],#16 // re-pre-load rndkey[1] |
|||
vst1.8 {$in0},[$out],#16 |
|||
vst1.8 {$in1},[$out],#16 |
|||
vst1.8 {$in2},[$out],#16 |
|||
b.hs .Loop3x_ctr32 |
|||
|
|||
adds $len,$len,#3 |
|||
b.eq .Lctr32_done |
|||
cmp $len,#1 |
|||
mov $step,#16 |
|||
cclr $step,eq |
|||
|
|||
.Lctr32_tail: |
|||
aese $dat0,q8 |
|||
aese $dat1,q8 |
|||
vld1.32 {q8},[$key_],#16 |
|||
aesmc $dat0,$dat0 |
|||
aesmc $dat1,$dat1 |
|||
subs $cnt,$cnt,#2 |
|||
aese $dat0,q9 |
|||
aese $dat1,q9 |
|||
vld1.32 {q9},[$key_],#16 |
|||
aesmc $dat0,$dat0 |
|||
aesmc $dat1,$dat1 |
|||
b.gt .Lctr32_tail |
|||
|
|||
aese $dat0,q8 |
|||
aese $dat1,q8 |
|||
aesmc $dat0,$dat0 |
|||
aesmc $dat1,$dat1 |
|||
aese $dat0,q9 |
|||
aese $dat1,q9 |
|||
aesmc $dat0,$dat0 |
|||
aesmc $dat1,$dat1 |
|||
vld1.8 {$in0},[$inp],$step |
|||
aese $dat0,q12 |
|||
aese $dat1,q12 |
|||
vld1.8 {$in1},[$inp] |
|||
aesmc $dat0,$dat0 |
|||
aesmc $dat1,$dat1 |
|||
aese $dat0,q13 |
|||
aese $dat1,q13 |
|||
aesmc $dat0,$dat0 |
|||
aesmc $dat1,$dat1 |
|||
aese $dat0,q14 |
|||
aese $dat1,q14 |
|||
veor $in0,$in0,$rndlast |
|||
aesmc $dat0,$dat0 |
|||
aesmc $dat1,$dat1 |
|||
veor $in1,$in1,$rndlast |
|||
aese $dat0,q15 |
|||
aese $dat1,q15 |
|||
|
|||
cmp $len,#1 |
|||
veor $in0,$in0,$dat0 |
|||
veor $in1,$in1,$dat1 |
|||
vst1.8 {$in0},[$out],#16 |
|||
b.eq .Lctr32_done |
|||
vst1.8 {$in1},[$out] |
|||
|
|||
.Lctr32_done: |
|||
___ |
|||
$code.=<<___ if ($flavour !~ /64/); |
|||
vldmia sp!,{d8-d15} |
|||
ldmia sp!,{r4-r10,pc} |
|||
___ |
|||
$code.=<<___ if ($flavour =~ /64/); |
|||
ldr x29,[sp],#16 |
|||
ret |
|||
___ |
|||
$code.=<<___; |
|||
.size ${prefix}_ctr32_encrypt_blocks,.-${prefix}_ctr32_encrypt_blocks |
|||
___ |
|||
}}} |
|||
$code.=<<___; |
|||
#endif |
|||
___ |
|||
######################################## |
|||
if ($flavour =~ /64/) { ######## 64-bit code |
|||
my %opcode = ( |
|||
"aesd" => 0x4e285800, "aese" => 0x4e284800, |
|||
"aesimc"=> 0x4e287800, "aesmc" => 0x4e286800 ); |
|||
|
|||
local *unaes = sub { |
|||
my ($mnemonic,$arg)=@_; |
|||
|
|||
$arg =~ m/[qv]([0-9]+)[^,]*,\s*[qv]([0-9]+)/o && |
|||
sprintf ".inst\t0x%08x\t//%s %s", |
|||
$opcode{$mnemonic}|$1|($2<<5), |
|||
$mnemonic,$arg; |
|||
}; |
|||
|
|||
foreach(split("\n",$code)) { |
|||
s/\`([^\`]*)\`/eval($1)/geo; |
|||
|
|||
s/\bq([0-9]+)\b/"v".($1<8?$1:$1+8).".16b"/geo; # old->new registers |
|||
s/@\s/\/\//o; # old->new style commentary |
|||
|
|||
#s/[v]?(aes\w+)\s+([qv].*)/unaes($1,$2)/geo or |
|||
s/cclr\s+([wx])([^,]+),\s*([a-z]+)/csel $1$2,$1zr,$1$2,$3/o or |
|||
s/mov\.([a-z]+)\s+([wx][0-9]+),\s*([wx][0-9]+)/csel $2,$3,$2,$1/o or |
|||
s/vmov\.i8/movi/o or # fix up legacy mnemonics |
|||
s/vext\.8/ext/o or |
|||
s/vrev32\.8/rev32/o or |
|||
s/vtst\.8/cmtst/o or |
|||
s/vshr/ushr/o or |
|||
s/^(\s+)v/$1/o or # strip off v prefix |
|||
s/\bbx\s+lr\b/ret/o; |
|||
|
|||
# fix up remainig legacy suffixes |
|||
s/\.[ui]?8//o; |
|||
m/\],#8/o and s/\.16b/\.8b/go; |
|||
s/\.[ui]?32//o and s/\.16b/\.4s/go; |
|||
s/\.[ui]?64//o and s/\.16b/\.2d/go; |
|||
s/\.[42]([sd])\[([0-3])\]/\.$1\[$2\]/o; |
|||
|
|||
print $_,"\n"; |
|||
} |
|||
} else { ######## 32-bit code |
|||
my %opcode = ( |
|||
"aesd" => 0xf3b00340, "aese" => 0xf3b00300, |
|||
"aesimc"=> 0xf3b003c0, "aesmc" => 0xf3b00380 ); |
|||
|
|||
local *unaes = sub { |
|||
my ($mnemonic,$arg)=@_; |
|||
|
|||
if ($arg =~ m/[qv]([0-9]+)[^,]*,\s*[qv]([0-9]+)/o) { |
|||
my $word = $opcode{$mnemonic}|(($1&7)<<13)|(($1&8)<<19) |
|||
|(($2&7)<<1) |(($2&8)<<2); |
|||
# since ARMv7 instructions are always encoded little-endian. |
|||
# correct solution is to use .inst directive, but older |
|||
# assemblers don't implement it:-( |
|||
sprintf ".byte\t0x%02x,0x%02x,0x%02x,0x%02x\t@ %s %s", |
|||
$word&0xff,($word>>8)&0xff, |
|||
($word>>16)&0xff,($word>>24)&0xff, |
|||
$mnemonic,$arg; |
|||
} |
|||
}; |
|||
|
|||
sub unvtbl { |
|||
my $arg=shift; |
|||
|
|||
$arg =~ m/q([0-9]+),\s*\{q([0-9]+)\},\s*q([0-9]+)/o && |
|||
sprintf "vtbl.8 d%d,{q%d},d%d\n\t". |
|||
"vtbl.8 d%d,{q%d},d%d", 2*$1,$2,2*$3, 2*$1+1,$2,2*$3+1; |
|||
} |
|||
|
|||
sub unvdup32 { |
|||
my $arg=shift; |
|||
|
|||
$arg =~ m/q([0-9]+),\s*q([0-9]+)\[([0-3])\]/o && |
|||
sprintf "vdup.32 q%d,d%d[%d]",$1,2*$2+($3>>1),$3&1; |
|||
} |
|||
|
|||
sub unvmov32 { |
|||
my $arg=shift; |
|||
|
|||
$arg =~ m/q([0-9]+)\[([0-3])\],(.*)/o && |
|||
sprintf "vmov.32 d%d[%d],%s",2*$1+($2>>1),$2&1,$3; |
|||
} |
|||
|
|||
foreach(split("\n",$code)) { |
|||
s/\`([^\`]*)\`/eval($1)/geo; |
|||
|
|||
s/\b[wx]([0-9]+)\b/r$1/go; # new->old registers |
|||
s/\bv([0-9])\.[12468]+[bsd]\b/q$1/go; # new->old registers |
|||
s/\/\/\s?/@ /o; # new->old style commentary |
|||
|
|||
# fix up remainig new-style suffixes |
|||
s/\{q([0-9]+)\},\s*\[(.+)\],#8/sprintf "{d%d},[$2]!",2*$1/eo or |
|||
s/\],#[0-9]+/]!/o; |
|||
|
|||
s/[v]?(aes\w+)\s+([qv].*)/unaes($1,$2)/geo or |
|||
s/cclr\s+([^,]+),\s*([a-z]+)/mov$2 $1,#0/o or |
|||
s/vtbl\.8\s+(.*)/unvtbl($1)/geo or |
|||
s/vdup\.32\s+(.*)/unvdup32($1)/geo or |
|||
s/vmov\.32\s+(.*)/unvmov32($1)/geo or |
|||
s/^(\s+)b\./$1b/o or |
|||
s/^(\s+)mov\./$1mov/o or |
|||
s/^(\s+)ret/$1bx\tlr/o; |
|||
|
|||
print $_,"\n"; |
|||
} |
|||
} |
|||
|
|||
close STDOUT; |
File diff suppressed because it is too large
File diff suppressed because it is too large
@ -0,0 +1,46 @@ |
|||
#include "arm_arch.h" |
|||
|
|||
.text |
|||
.arch armv8-a+crypto |
|||
|
|||
.align 5 |
|||
.global _armv7_neon_probe |
|||
.type _armv7_neon_probe,%function |
|||
_armv7_neon_probe: |
|||
orr v15.16b, v15.16b, v15.16b |
|||
ret |
|||
.size _armv7_neon_probe,.-_armv7_neon_probe |
|||
|
|||
.global _armv7_tick |
|||
.type _armv7_tick,%function |
|||
_armv7_tick: |
|||
mrs x0, CNTVCT_EL0 |
|||
ret |
|||
.size _armv7_tick,.-_armv7_tick |
|||
|
|||
.global _armv8_aes_probe |
|||
.type _armv8_aes_probe,%function |
|||
_armv8_aes_probe: |
|||
aese v0.16b, v0.16b |
|||
ret |
|||
.size _armv8_aes_probe,.-_armv8_aes_probe |
|||
|
|||
.global _armv8_sha1_probe |
|||
.type _armv8_sha1_probe,%function |
|||
_armv8_sha1_probe: |
|||
sha1h s0, s0 |
|||
ret |
|||
.size _armv8_sha1_probe,.-_armv8_sha1_probe |
|||
|
|||
.global _armv8_sha256_probe |
|||
.type _armv8_sha256_probe,%function |
|||
_armv8_sha256_probe: |
|||
sha256su0 v0.4s, v0.4s |
|||
ret |
|||
.size _armv8_sha256_probe,.-_armv8_sha256_probe |
|||
.global _armv8_pmull_probe |
|||
.type _armv8_pmull_probe,%function |
|||
_armv8_pmull_probe: |
|||
pmull v0.1q, v0.1d, v0.1d |
|||
ret |
|||
.size _armv8_pmull_probe,.-_armv8_pmull_probe |
File diff suppressed because it is too large
File diff suppressed because it is too large
File diff suppressed because it is too large
File diff suppressed because it is too large
File diff suppressed because it is too large
@ -0,0 +1,190 @@ |
|||
#!/usr/bin/env perl |
|||
# |
|||
# ==================================================================== |
|||
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL |
|||
# project. The module is, however, dual licensed under OpenSSL and |
|||
# CRYPTOGAMS licenses depending on where you obtain it. For further |
|||
# details see http://www.openssl.org/~appro/cryptogams/. |
|||
# ==================================================================== |
|||
# |
|||
# October 2012 |
|||
# |
|||
# The module implements bn_GF2m_mul_2x2 polynomial multiplication used |
|||
# in bn_gf2m.c. It's kind of low-hanging mechanical port from C for |
|||
# the time being... Except that it has two code paths: one suitable |
|||
# for all SPARCv9 processors and one for VIS3-capable ones. Former |
|||
# delivers ~25-45% more, more for longer keys, heaviest DH and DSA |
|||
# verify operations on venerable UltraSPARC II. On T4 VIS3 code is |
|||
# ~100-230% faster than gcc-generated code and ~35-90% faster than |
|||
# the pure SPARCv9 code path. |
|||
|
|||
$locals=16*8; |
|||
|
|||
$tab="%l0"; |
|||
|
|||
@T=("%g2","%g3"); |
|||
@i=("%g4","%g5"); |
|||
|
|||
($a1,$a2,$a4,$a8,$a12,$a48)=map("%o$_",(0..5)); |
|||
($lo,$hi,$b)=("%g1",$a8,"%o7"); $a=$lo; |
|||
|
|||
$code.=<<___; |
|||
#include <sparc_arch.h> |
|||
|
|||
#ifdef __arch64__ |
|||
.register %g2,#scratch |
|||
.register %g3,#scratch |
|||
#endif |
|||
|
|||
#ifdef __PIC__ |
|||
SPARC_PIC_THUNK(%g1) |
|||
#endif |
|||
|
|||
.globl bn_GF2m_mul_2x2 |
|||
.align 16 |
|||
bn_GF2m_mul_2x2: |
|||
SPARC_LOAD_ADDRESS_LEAF(OPENSSL_sparcv9cap_P,%g1,%g5) |
|||
ld [%g1+0],%g1 ! OPENSSL_sparcv9cap_P[0] |
|||
|
|||
andcc %g1, SPARCV9_VIS3, %g0 |
|||
bz,pn %icc,.Lsoftware |
|||
nop |
|||
|
|||
sllx %o1, 32, %o1 |
|||
sllx %o3, 32, %o3 |
|||
or %o2, %o1, %o1 |
|||
or %o4, %o3, %o3 |
|||
.word 0x95b262ab ! xmulx %o1, %o3, %o2 |
|||
.word 0x99b262cb ! xmulxhi %o1, %o3, %o4 |
|||
srlx %o2, 32, %o1 ! 13 cycles later |
|||
st %o2, [%o0+0] |
|||
st %o1, [%o0+4] |
|||
srlx %o4, 32, %o3 |
|||
st %o4, [%o0+8] |
|||
retl |
|||
st %o3, [%o0+12] |
|||
|
|||
.align 16 |
|||
.Lsoftware: |
|||
save %sp,-STACK_FRAME-$locals,%sp |
|||
|
|||
sllx %i1,32,$a |
|||
mov -1,$a12 |
|||
sllx %i3,32,$b |
|||
or %i2,$a,$a |
|||
srlx $a12,1,$a48 ! 0x7fff... |
|||
or %i4,$b,$b |
|||
srlx $a12,2,$a12 ! 0x3fff... |
|||
add %sp,STACK_BIAS+STACK_FRAME,$tab |
|||
|
|||
sllx $a,2,$a4 |
|||
mov $a,$a1 |
|||
sllx $a,1,$a2 |
|||
|
|||
srax $a4,63,@i[1] ! broadcast 61st bit |
|||
and $a48,$a4,$a4 ! (a<<2)&0x7fff... |
|||
srlx $a48,2,$a48 |
|||
srax $a2,63,@i[0] ! broadcast 62nd bit |
|||
and $a12,$a2,$a2 ! (a<<1)&0x3fff... |
|||
srax $a1,63,$lo ! broadcast 63rd bit |
|||
and $a48,$a1,$a1 ! (a<<0)&0x1fff... |
|||
|
|||
sllx $a1,3,$a8 |
|||
and $b,$lo,$lo |
|||
and $b,@i[0],@i[0] |
|||
and $b,@i[1],@i[1] |
|||
|
|||
stx %g0,[$tab+0*8] ! tab[0]=0 |
|||
xor $a1,$a2,$a12 |
|||
stx $a1,[$tab+1*8] ! tab[1]=a1 |
|||
stx $a2,[$tab+2*8] ! tab[2]=a2 |
|||
xor $a4,$a8,$a48 |
|||
stx $a12,[$tab+3*8] ! tab[3]=a1^a2 |
|||
xor $a4,$a1,$a1 |
|||
|
|||
stx $a4,[$tab+4*8] ! tab[4]=a4 |
|||
xor $a4,$a2,$a2 |
|||
stx $a1,[$tab+5*8] ! tab[5]=a1^a4 |
|||
xor $a4,$a12,$a12 |
|||
stx $a2,[$tab+6*8] ! tab[6]=a2^a4 |
|||
xor $a48,$a1,$a1 |
|||
stx $a12,[$tab+7*8] ! tab[7]=a1^a2^a4 |
|||
xor $a48,$a2,$a2 |
|||
|
|||
stx $a8,[$tab+8*8] ! tab[8]=a8 |
|||
xor $a48,$a12,$a12 |
|||
stx $a1,[$tab+9*8] ! tab[9]=a1^a8 |
|||
xor $a4,$a1,$a1 |
|||
stx $a2,[$tab+10*8] ! tab[10]=a2^a8 |
|||
xor $a4,$a2,$a2 |
|||
stx $a12,[$tab+11*8] ! tab[11]=a1^a2^a8 |
|||
|
|||
xor $a4,$a12,$a12 |
|||
stx $a48,[$tab+12*8] ! tab[12]=a4^a8 |
|||
srlx $lo,1,$hi |
|||
stx $a1,[$tab+13*8] ! tab[13]=a1^a4^a8 |
|||
sllx $lo,63,$lo |
|||
stx $a2,[$tab+14*8] ! tab[14]=a2^a4^a8 |
|||
srlx @i[0],2,@T[0] |
|||
stx $a12,[$tab+15*8] ! tab[15]=a1^a2^a4^a8 |
|||
|
|||
sllx @i[0],62,$a1 |
|||
sllx $b,3,@i[0] |
|||
srlx @i[1],3,@T[1] |
|||
and @i[0],`0xf<<3`,@i[0] |
|||
sllx @i[1],61,$a2 |
|||
ldx [$tab+@i[0]],@i[0] |
|||
srlx $b,4-3,@i[1] |
|||
xor @T[0],$hi,$hi |
|||
and @i[1],`0xf<<3`,@i[1] |
|||
xor $a1,$lo,$lo |
|||
ldx [$tab+@i[1]],@i[1] |
|||
xor @T[1],$hi,$hi |
|||
|
|||
xor @i[0],$lo,$lo |
|||
srlx $b,8-3,@i[0] |
|||
xor $a2,$lo,$lo |
|||
and @i[0],`0xf<<3`,@i[0] |
|||
___ |
|||
for($n=1;$n<14;$n++) { |
|||
$code.=<<___; |
|||
sllx @i[1],`$n*4`,@T[0] |
|||
ldx [$tab+@i[0]],@i[0] |
|||
srlx @i[1],`64-$n*4`,@T[1] |
|||
xor @T[0],$lo,$lo |
|||
srlx $b,`($n+2)*4`-3,@i[1] |
|||
xor @T[1],$hi,$hi |
|||
and @i[1],`0xf<<3`,@i[1] |
|||
___ |
|||
push(@i,shift(@i)); push(@T,shift(@T)); |
|||
} |
|||
$code.=<<___; |
|||
sllx @i[1],`$n*4`,@T[0] |
|||
ldx [$tab+@i[0]],@i[0] |
|||
srlx @i[1],`64-$n*4`,@T[1] |
|||
xor @T[0],$lo,$lo |
|||
|
|||
sllx @i[0],`($n+1)*4`,@T[0] |
|||
xor @T[1],$hi,$hi |
|||
srlx @i[0],`64-($n+1)*4`,@T[1] |
|||
xor @T[0],$lo,$lo |
|||
xor @T[1],$hi,$hi |
|||
|
|||
srlx $lo,32,%i1 |
|||
st $lo,[%i0+0] |
|||
st %i1,[%i0+4] |
|||
srlx $hi,32,%i2 |
|||
st $hi,[%i0+8] |
|||
st %i2,[%i0+12] |
|||
|
|||
ret |
|||
restore |
|||
.type bn_GF2m_mul_2x2,#function |
|||
.size bn_GF2m_mul_2x2,.-bn_GF2m_mul_2x2 |
|||
.asciz "GF(2^m) Multiplication for SPARCv9, CRYPTOGAMS by <appro\@openssl.org>" |
|||
.align 4 |
|||
___ |
|||
|
|||
$code =~ s/\`([^\`]*)\`/eval($1)/gem; |
|||
print $code; |
|||
close STDOUT; |
@ -0,0 +1,373 @@ |
|||
#!/usr/bin/env perl |
|||
|
|||
# ==================================================================== |
|||
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL |
|||
# project. The module is, however, dual licensed under OpenSSL and |
|||
# CRYPTOGAMS licenses depending on where you obtain it. For further |
|||
# details see http://www.openssl.org/~appro/cryptogams/. |
|||
# ==================================================================== |
|||
|
|||
# October 2012. |
|||
# |
|||
# SPARCv9 VIS3 Montgomery multiplicaion procedure suitable for T3 and |
|||
# onward. There are three new instructions used here: umulxhi, |
|||
# addxc[cc] and initializing store. On T3 RSA private key operations |
|||
# are 1.54/1.87/2.11/2.26 times faster for 512/1024/2048/4096-bit key |
|||
# lengths. This is without dedicated squaring procedure. On T4 |
|||
# corresponding coefficients are 1.47/2.10/2.80/2.90x, which is mostly |
|||
# for reference purposes, because T4 has dedicated Montgomery |
|||
# multiplication and squaring *instructions* that deliver even more. |
|||
|
|||
$bits=32; |
|||
for (@ARGV) { $bits=64 if (/\-m64/ || /\-xarch\=v9/); } |
|||
if ($bits==64) { $bias=2047; $frame=192; } |
|||
else { $bias=0; $frame=112; } |
|||
|
|||
$code.=<<___ if ($bits==64); |
|||
.register %g2,#scratch |
|||
.register %g3,#scratch |
|||
___ |
|||
$code.=<<___; |
|||
.section ".text",#alloc,#execinstr |
|||
___ |
|||
|
|||
($n0,$m0,$m1,$lo0,$hi0, $lo1,$hi1,$aj,$alo,$nj,$nlo,$tj)= |
|||
(map("%g$_",(1..5)),map("%o$_",(0..5,7))); |
|||
|
|||
# int bn_mul_mont( |
|||
$rp="%o0"; # BN_ULONG *rp, |
|||
$ap="%o1"; # const BN_ULONG *ap, |
|||
$bp="%o2"; # const BN_ULONG *bp, |
|||
$np="%o3"; # const BN_ULONG *np, |
|||
$n0p="%o4"; # const BN_ULONG *n0, |
|||
$num="%o5"; # int num); # caller ensures that num is even |
|||
# and >=6 |
|||
$code.=<<___; |
|||
.globl bn_mul_mont_vis3 |
|||
.align 32 |
|||
bn_mul_mont_vis3: |
|||
add %sp, $bias, %g4 ! real top of stack |
|||
sll $num, 2, $num ! size in bytes |
|||
add $num, 63, %g5 |
|||
andn %g5, 63, %g5 ! buffer size rounded up to 64 bytes |
|||
add %g5, %g5, %g1 |
|||
add %g5, %g1, %g1 ! 3*buffer size |
|||
sub %g4, %g1, %g1 |
|||
andn %g1, 63, %g1 ! align at 64 byte |
|||
sub %g1, $frame, %g1 ! new top of stack |
|||
sub %g1, %g4, %g1 |
|||
|
|||
save %sp, %g1, %sp |
|||
___ |
|||
|
|||
# +-------------------------------+<----- %sp |
|||
# . . |
|||
# +-------------------------------+<----- aligned at 64 bytes |
|||
# | __int64 tmp[0] | |
|||
# +-------------------------------+ |
|||
# . . |
|||
# . . |
|||
# +-------------------------------+<----- aligned at 64 bytes |
|||
# | __int64 ap[1..0] | converted ap[] |
|||
# +-------------------------------+ |
|||
# | __int64 np[1..0] | converted np[] |
|||
# +-------------------------------+ |
|||
# | __int64 ap[3..2] | |
|||
# . . |
|||
# . . |
|||
# +-------------------------------+ |
|||
($rp,$ap,$bp,$np,$n0p,$num)=map("%i$_",(0..5)); |
|||
($t0,$t1,$t2,$t3,$cnt,$tp,$bufsz,$anp)=map("%l$_",(0..7)); |
|||
($ovf,$i)=($t0,$t1); |
|||
$code.=<<___; |
|||
ld [$n0p+0], $t0 ! pull n0[0..1] value |
|||
add %sp, $bias+$frame, $tp |
|||
ld [$n0p+4], $t1 |
|||
add $tp, %g5, $anp |
|||
ld [$bp+0], $t2 ! m0=bp[0] |
|||
sllx $t1, 32, $n0 |
|||
ld [$bp+4], $t3 |
|||
or $t0, $n0, $n0 |
|||
add $bp, 8, $bp |
|||
|
|||
ld [$ap+0], $t0 ! ap[0] |
|||
sllx $t3, 32, $m0 |
|||
ld [$ap+4], $t1 |
|||
or $t2, $m0, $m0 |
|||
|
|||
ld [$ap+8], $t2 ! ap[1] |
|||
sllx $t1, 32, $aj |
|||
ld [$ap+12], $t3 |
|||
or $t0, $aj, $aj |
|||
add $ap, 16, $ap |
|||
stxa $aj, [$anp]0xe2 ! converted ap[0] |
|||
|
|||
mulx $aj, $m0, $lo0 ! ap[0]*bp[0] |
|||
umulxhi $aj, $m0, $hi0 |
|||
|
|||
ld [$np+0], $t0 ! np[0] |
|||
sllx $t3, 32, $aj |
|||
ld [$np+4], $t1 |
|||
or $t2, $aj, $aj |
|||
|
|||
ld [$np+8], $t2 ! np[1] |
|||
sllx $t1, 32, $nj |
|||
ld [$np+12], $t3 |
|||
or $t0, $nj, $nj |
|||
add $np, 16, $np |
|||
stx $nj, [$anp+8] ! converted np[0] |
|||
|
|||
mulx $lo0, $n0, $m1 ! "tp[0]"*n0 |
|||
stx $aj, [$anp+16] ! converted ap[1] |
|||
|
|||
mulx $aj, $m0, $alo ! ap[1]*bp[0] |
|||
umulxhi $aj, $m0, $aj ! ahi=aj |
|||
|
|||
mulx $nj, $m1, $lo1 ! np[0]*m1 |
|||
umulxhi $nj, $m1, $hi1 |
|||
|
|||
sllx $t3, 32, $nj |
|||
or $t2, $nj, $nj |
|||
stx $nj, [$anp+24] ! converted np[1] |
|||
add $anp, 32, $anp |
|||
|
|||
addcc $lo0, $lo1, $lo1 |
|||
addxc %g0, $hi1, $hi1 |
|||
|
|||
mulx $nj, $m1, $nlo ! np[1]*m1 |
|||
umulxhi $nj, $m1, $nj ! nhi=nj |
|||
|
|||
ba .L1st |
|||
sub $num, 24, $cnt ! cnt=num-3 |
|||
|
|||
.align 16 |
|||
.L1st: |
|||
ld [$ap+0], $t0 ! ap[j] |
|||
addcc $alo, $hi0, $lo0 |
|||
ld [$ap+4], $t1 |
|||
addxc $aj, %g0, $hi0 |
|||
|
|||
sllx $t1, 32, $aj |
|||
add $ap, 8, $ap |
|||
or $t0, $aj, $aj |
|||
stxa $aj, [$anp]0xe2 ! converted ap[j] |
|||
|
|||
ld [$np+0], $t2 ! np[j] |
|||
addcc $nlo, $hi1, $lo1 |
|||
ld [$np+4], $t3 |
|||
addxc $nj, %g0, $hi1 ! nhi=nj |
|||
|
|||
sllx $t3, 32, $nj |
|||
add $np, 8, $np |
|||
mulx $aj, $m0, $alo ! ap[j]*bp[0] |
|||
or $t2, $nj, $nj |
|||
umulxhi $aj, $m0, $aj ! ahi=aj |
|||
stx $nj, [$anp+8] ! converted np[j] |
|||
add $anp, 16, $anp ! anp++ |
|||
|
|||
mulx $nj, $m1, $nlo ! np[j]*m1 |
|||
addcc $lo0, $lo1, $lo1 ! np[j]*m1+ap[j]*bp[0] |
|||
umulxhi $nj, $m1, $nj ! nhi=nj |
|||
addxc %g0, $hi1, $hi1 |
|||
stxa $lo1, [$tp]0xe2 ! tp[j-1] |
|||
add $tp, 8, $tp ! tp++ |
|||
|
|||
brnz,pt $cnt, .L1st |
|||
sub $cnt, 8, $cnt ! j-- |
|||
!.L1st |
|||
addcc $alo, $hi0, $lo0 |
|||
addxc $aj, %g0, $hi0 ! ahi=aj |
|||
|
|||
addcc $nlo, $hi1, $lo1 |
|||
addxc $nj, %g0, $hi1 |
|||
addcc $lo0, $lo1, $lo1 ! np[j]*m1+ap[j]*bp[0] |
|||
addxc %g0, $hi1, $hi1 |
|||
stxa $lo1, [$tp]0xe2 ! tp[j-1] |
|||
add $tp, 8, $tp |
|||
|
|||
addcc $hi0, $hi1, $hi1 |
|||
addxc %g0, %g0, $ovf ! upmost overflow bit |
|||
stxa $hi1, [$tp]0xe2 |
|||
add $tp, 8, $tp |
|||
|
|||
ba .Louter |
|||
sub $num, 16, $i ! i=num-2 |
|||
|
|||
.align 16 |
|||
.Louter: |
|||
ld [$bp+0], $t2 ! m0=bp[i] |
|||
ld [$bp+4], $t3 |
|||
|
|||
sub $anp, $num, $anp ! rewind |
|||
sub $tp, $num, $tp |
|||
sub $anp, $num, $anp |
|||
|
|||
add $bp, 8, $bp |
|||
sllx $t3, 32, $m0 |
|||
ldx [$anp+0], $aj ! ap[0] |
|||
or $t2, $m0, $m0 |
|||
ldx [$anp+8], $nj ! np[0] |
|||
|
|||
mulx $aj, $m0, $lo0 ! ap[0]*bp[i] |
|||
ldx [$tp], $tj ! tp[0] |
|||
umulxhi $aj, $m0, $hi0 |
|||
ldx [$anp+16], $aj ! ap[1] |
|||
addcc $lo0, $tj, $lo0 ! ap[0]*bp[i]+tp[0] |
|||
mulx $aj, $m0, $alo ! ap[1]*bp[i] |
|||
addxc %g0, $hi0, $hi0 |
|||
mulx $lo0, $n0, $m1 ! tp[0]*n0 |
|||
umulxhi $aj, $m0, $aj ! ahi=aj |
|||
mulx $nj, $m1, $lo1 ! np[0]*m1 |
|||
umulxhi $nj, $m1, $hi1 |
|||
ldx [$anp+24], $nj ! np[1] |
|||
add $anp, 32, $anp |
|||
addcc $lo1, $lo0, $lo1 |
|||
mulx $nj, $m1, $nlo ! np[1]*m1 |
|||
addxc %g0, $hi1, $hi1 |
|||
umulxhi $nj, $m1, $nj ! nhi=nj |
|||
|
|||
ba .Linner |
|||
sub $num, 24, $cnt ! cnt=num-3 |
|||
.align 16 |
|||
.Linner: |
|||
addcc $alo, $hi0, $lo0 |
|||
ldx [$tp+8], $tj ! tp[j] |
|||
addxc $aj, %g0, $hi0 ! ahi=aj |
|||
ldx [$anp+0], $aj ! ap[j] |
|||
addcc $nlo, $hi1, $lo1 |
|||
mulx $aj, $m0, $alo ! ap[j]*bp[i] |
|||
addxc $nj, %g0, $hi1 ! nhi=nj |
|||
ldx [$anp+8], $nj ! np[j] |
|||
add $anp, 16, $anp |
|||
umulxhi $aj, $m0, $aj ! ahi=aj |
|||
addcc $lo0, $tj, $lo0 ! ap[j]*bp[i]+tp[j] |
|||
mulx $nj, $m1, $nlo ! np[j]*m1 |
|||
addxc %g0, $hi0, $hi0 |
|||
umulxhi $nj, $m1, $nj ! nhi=nj |
|||
addcc $lo1, $lo0, $lo1 ! np[j]*m1+ap[j]*bp[i]+tp[j] |
|||
addxc %g0, $hi1, $hi1 |
|||
stx $lo1, [$tp] ! tp[j-1] |
|||
add $tp, 8, $tp |
|||
brnz,pt $cnt, .Linner |
|||
sub $cnt, 8, $cnt |
|||
!.Linner |
|||
ldx [$tp+8], $tj ! tp[j] |
|||
addcc $alo, $hi0, $lo0 |
|||
addxc $aj, %g0, $hi0 ! ahi=aj |
|||
addcc $lo0, $tj, $lo0 ! ap[j]*bp[i]+tp[j] |
|||
addxc %g0, $hi0, $hi0 |
|||
|
|||
addcc $nlo, $hi1, $lo1 |
|||
addxc $nj, %g0, $hi1 ! nhi=nj |
|||
addcc $lo1, $lo0, $lo1 ! np[j]*m1+ap[j]*bp[i]+tp[j] |
|||
addxc %g0, $hi1, $hi1 |
|||
stx $lo1, [$tp] ! tp[j-1] |
|||
|
|||
subcc %g0, $ovf, %g0 ! move upmost overflow to CCR.xcc |
|||
addxccc $hi1, $hi0, $hi1 |
|||
addxc %g0, %g0, $ovf |
|||
stx $hi1, [$tp+8] |
|||
add $tp, 16, $tp |
|||
|
|||
brnz,pt $i, .Louter |
|||
sub $i, 8, $i |
|||
|
|||
sub $anp, $num, $anp ! rewind |
|||
sub $tp, $num, $tp |
|||
sub $anp, $num, $anp |
|||
ba .Lsub |
|||
subcc $num, 8, $cnt ! cnt=num-1 and clear CCR.xcc |
|||
|
|||
.align 16 |
|||
.Lsub: |
|||
ldx [$tp], $tj |
|||
add $tp, 8, $tp |
|||
ldx [$anp+8], $nj |
|||
add $anp, 16, $anp |
|||
subccc $tj, $nj, $t2 ! tp[j]-np[j] |
|||
srlx $tj, 32, $tj |
|||
srlx $nj, 32, $nj |
|||
subccc $tj, $nj, $t3 |
|||
add $rp, 8, $rp |
|||
st $t2, [$rp-4] ! reverse order |
|||
st $t3, [$rp-8] |
|||
brnz,pt $cnt, .Lsub |
|||
sub $cnt, 8, $cnt |
|||
|
|||
sub $anp, $num, $anp ! rewind |
|||
sub $tp, $num, $tp |
|||
sub $anp, $num, $anp |
|||
sub $rp, $num, $rp |
|||
|
|||
subc $ovf, %g0, $ovf ! handle upmost overflow bit |
|||
and $tp, $ovf, $ap |
|||
andn $rp, $ovf, $np |
|||
or $np, $ap, $ap ! ap=borrow?tp:rp |
|||
ba .Lcopy |
|||
sub $num, 8, $cnt |
|||
|
|||
.align 16 |
|||
.Lcopy: ! copy or in-place refresh |
|||
ld [$ap+0], $t2 |
|||
ld [$ap+4], $t3 |
|||
add $ap, 8, $ap |
|||
stx %g0, [$tp] ! zap |
|||
add $tp, 8, $tp |
|||
stx %g0, [$anp] ! zap |
|||
stx %g0, [$anp+8] |
|||
add $anp, 16, $anp |
|||
st $t3, [$rp+0] ! flip order |
|||
st $t2, [$rp+4] |
|||
add $rp, 8, $rp |
|||
brnz $cnt, .Lcopy |
|||
sub $cnt, 8, $cnt |
|||
|
|||
mov 1, %o0 |
|||
ret |
|||
restore |
|||
.type bn_mul_mont_vis3, #function |
|||
.size bn_mul_mont_vis3, .-bn_mul_mont_vis3 |
|||
.asciz "Montgomery Multiplication for SPARCv9 VIS3, CRYPTOGAMS by <appro\@openssl.org>" |
|||
.align 4 |
|||
___ |
|||
|
|||
# Purpose of these subroutines is to explicitly encode VIS instructions, |
|||
# so that one can compile the module without having to specify VIS |
|||
# extentions on compiler command line, e.g. -xarch=v9 vs. -xarch=v9a. |
|||
# Idea is to reserve for option to produce "universal" binary and let |
|||
# programmer detect if current CPU is VIS capable at run-time. |
|||
sub unvis3 { |
|||
my ($mnemonic,$rs1,$rs2,$rd)=@_; |
|||
my %bias = ( "g" => 0, "o" => 8, "l" => 16, "i" => 24 ); |
|||
my ($ref,$opf); |
|||
my %visopf = ( "addxc" => 0x011, |
|||
"addxccc" => 0x013, |
|||
"umulxhi" => 0x016 ); |
|||
|
|||
$ref = "$mnemonic\t$rs1,$rs2,$rd"; |
|||
|
|||
if ($opf=$visopf{$mnemonic}) { |
|||
foreach ($rs1,$rs2,$rd) { |
|||
return $ref if (!/%([goli])([0-9])/); |
|||
$_=$bias{$1}+$2; |
|||
} |
|||
|
|||
return sprintf ".word\t0x%08x !%s", |
|||
0x81b00000|$rd<<25|$rs1<<14|$opf<<5|$rs2, |
|||
$ref; |
|||
} else { |
|||
return $ref; |
|||
} |
|||
} |
|||
|
|||
foreach (split("\n",$code)) { |
|||
s/\`([^\`]*)\`/eval $1/ge; |
|||
|
|||
s/\b(umulxhi|addxc[c]{0,2})\s+(%[goli][0-7]),\s*(%[goli][0-7]),\s*(%[goli][0-7])/ |
|||
&unvis3($1,$2,$3,$4) |
|||
/ge; |
|||
|
|||
print $_,"\n"; |
|||
} |
|||
|
|||
close STDOUT; |
File diff suppressed because it is too large
File diff suppressed because it is too large
Some files were not shown because too many files changed in this diff
Loading…
Reference in new issue