// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
#include "textflag.h"
#define B0 V0
#define B1 V1
#define B2 V2
#define B3 V3
#define B4 V4
#define B5 V5
#define B6 V6
#define B7 V7
#define ACC0 V8
#define ACC1 V9
#define ACCM V10
#define T0 V11
#define T1 V12
#define T2 V13
#define T3 V14
#define POLY V15
#define ZERO V16
#define INC V17
#define CTR V18
#define K0 V19
#define K1 V20
#define K2 V21
#define K3 V22
#define K4 V23
#define K5 V24
#define K6 V25
#define K7 V26
#define K8 V27
#define K9 V28
#define K10 V29
#define K11 V30
#define KLAST V31
#define reduce() \
VEOR ACC0.B16, ACCM.B16, ACCM.B16 \
VEOR ACC1.B16, ACCM.B16, ACCM.B16 \
VEXT $8, ZERO.B16, ACCM.B16, T0.B16 \
VEXT $8, ACCM.B16, ZERO.B16, ACCM.B16 \
VEOR ACCM.B16, ACC0.B16, ACC0.B16 \
VEOR T0.B16, ACC1.B16, ACC1.B16 \
VPMULL POLY.D1, ACC0.D1, T0.Q1 \
VEXT $8, ACC0.B16, ACC0.B16, ACC0.B16 \
VEOR T0.B16, ACC0.B16, ACC0.B16 \
VPMULL POLY.D1, ACC0.D1, T0.Q1 \
VEOR T0.B16, ACC1.B16, ACC1.B16 \
VEXT $8, ACC1.B16, ACC1.B16, ACC1.B16 \
VEOR ACC1.B16, ACC0.B16, ACC0.B16 \
// func gcmAesFinish(productTable *[256]byte, tagMask, T *[16]byte, pLen, dLen uint64)
TEXT ·gcmAesFinish(SB),NOSPLIT,$0
#define pTbl R0
#define tMsk R1
#define tPtr R2
#define plen R3
#define dlen R4
MOVD $0xC2, R1
LSL $56, R1
MOVD $1, R0
VMOV R1, POLY.D[0]
VMOV R0, POLY.D[1]
VEOR ZERO.B16, ZERO.B16, ZERO.B16
MOVD productTable+0(FP), pTbl
MOVD tagMask+8(FP), tMsk
MOVD T+16(FP), tPtr
MOVD pLen+24(FP), plen
MOVD dLen+32(FP), dlen
VLD1 (tPtr), [ACC0.B16]
VLD1 (tMsk), [B1.B16]
LSL $3, plen
LSL $3, dlen
VMOV dlen, B0.D[0]
VMOV plen, B0.D[1]
ADD $14*16, pTbl
VLD1.P (pTbl), [T1.B16, T2.B16]
VEOR ACC0.B16, B0.B16, B0.B16
VEXT $8, B0.B16, B0.B16, T0.B16
VEOR B0.B16, T0.B16, T0.B16
VPMULL B0.D1, T1.D1, ACC1.Q1
VPMULL2 B0.D2, T1.D2, ACC0.Q1
VPMULL T0.D1, T2.D1, ACCM.Q1
reduce()
VREV64 ACC0.B16, ACC0.B16
VEOR B1.B16, ACC0.B16, ACC0.B16
VST1 [ACC0.B16], (tPtr)
RET
#undef pTbl
#undef tMsk
#undef tPtr
#undef plen
#undef dlen
// func gcmAesInit(productTable *[256]byte, ks []uint32)
TEXT ·gcmAesInit(SB),NOSPLIT,$0
#define pTbl R0
#define KS R1
#define NR R2
#define I R3
MOVD productTable+0(FP), pTbl
MOVD ks_base+8(FP), KS
MOVD ks_len+16(FP), NR
MOVD $0xC2, I
LSL $56, I
VMOV I, POLY.D[0]
MOVD $1, I
VMOV I, POLY.D[1]
VEOR ZERO.B16, ZERO.B16, ZERO.B16
// Encrypt block 0 with the AES key to generate the hash key H
VLD1.P 64(KS), [T0.B16, T1.B16, T2.B16, T3.B16]
VEOR B0.B16, B0.B16, B0.B16
AESE T0.B16, B0.B16
AESMC B0.B16, B0.B16
AESE T1.B16, B0.B16
AESMC B0.B16, B0.B16
AESE T2.B16, B0.B16
AESMC B0.B16, B0.B16
AESE T3.B16, B0.B16
AESMC B0.B16, B0.B16
VLD1.P 64(KS), [T0.B16, T1.B16, T2.B16, T3.B16]
AESE T0.B16, B0.B16
AESMC B0.B16, B0.B16
AESE T1.B16, B0.B16
AESMC B0.B16, B0.B16
AESE T2.B16, B0.B16
AESMC B0.B16, B0.B16
AESE T3.B16, B0.B16
AESMC B0.B16, B0.B16
TBZ $4, NR, initEncFinish
VLD1.P 32(KS), [T0.B16, T1.B16]
AESE T0.B16, B0.B16
AESMC B0.B16, B0.B16
AESE T1.B16, B0.B16
AESMC B0.B16, B0.B16
TBZ $3, NR, initEncFinish
VLD1.P 32(KS), [T0.B16, T1.B16]
AESE T0.B16, B0.B16
AESMC B0.B16, B0.B16
AESE T1.B16, B0.B16
AESMC B0.B16, B0.B16
initEncFinish:
VLD1 (KS), [T0.B16, T1.B16, T2.B16]
AESE T0.B16, B0.B16
AESMC B0.B16, B0.B16
AESE T1.B16, B0.B16
VEOR T2.B16, B0.B16, B0.B16
VREV64 B0.B16, B0.B16
// Multiply by 2 modulo P
VMOV B0.D[0], I
ASR $63, I
VMOV I, T1.D[0]
VMOV I, T1.D[1]
VAND POLY.B16, T1.B16, T1.B16
VUSHR $63, B0.D2, T2.D2
VEXT $8, ZERO.B16, T2.B16, T2.B16
VSHL $1, B0.D2, B0.D2
VEOR T1.B16, B0.B16, B0.B16
VEOR T2.B16, B0.B16, B0.B16 // Can avoid this when VSLI is available
// Karatsuba pre-computation
VEXT $8, B0.B16, B0.B16, B1.B16
VEOR B0.B16, B1.B16, B1.B16
ADD $14*16, pTbl
VST1 [B0.B16, B1.B16], (pTbl)
SUB $2*16, pTbl
VMOV B0.B16, B2.B16
VMOV B1.B16, B3.B16
MOVD $7, I
initLoop:
// Compute powers of H
SUBS $1, I
VPMULL B0.D1, B2.D1, T1.Q1
VPMULL2 B0.D2, B2.D2, T0.Q1
VPMULL B1.D1, B3.D1, T2.Q1
VEOR T0.B16, T2.B16, T2.B16
VEOR T1.B16, T2.B16, T2.B16
VEXT $8, ZERO.B16, T2.B16, T3.B16
VEXT $8, T2.B16, ZERO.B16, T2.B16
VEOR T2.B16, T0.B16, T0.B16
VEOR T3.B16, T1.B16, T1.B16
VPMULL POLY.D1, T0.D1, T2.Q1
VEXT $8, T0.B16, T0.B16, T0.B16
VEOR T2.B16, T0.B16, T0.B16
VPMULL POLY.D1, T0.D1, T2.Q1
VEXT $8, T0.B16, T0.B16, T0.B16
VEOR T2.B16, T0.B16, T0.B16
VEOR T1.B16, T0.B16, B2.B16
VMOV B2.B16, B3.B16
VEXT $8, B2.B16, B2.B16, B2.B16
VEOR B2.B16, B3.B16, B3.B16
VST1 [B2.B16, B3.B16], (pTbl)
SUB $2*16, pTbl
BNE initLoop
RET
#undef I
#undef NR
#undef KS
#undef pTbl
// func gcmAesData(productTable *[256]byte, data []byte, T *[16]byte)
TEXT ·gcmAesData(SB),NOSPLIT,$0
#define pTbl R0
#define aut R1
#define tPtr R2
#define autLen R3
#define H0 R4
#define pTblSave R5
#define mulRound(X) \
VLD1.P 32(pTbl), [T1.B16, T2.B16] \
VREV64 X.B16, X.B16 \
VEXT $8, X.B16, X.B16, T0.B16 \
VEOR X.B16, T0.B16, T0.B16 \
VPMULL X.D1, T1.D1, T3.Q1 \
VEOR T3.B16, ACC1.B16, ACC1.B16 \
VPMULL2 X.D2, T1.D2, T3.Q1 \
VEOR T3.B16, ACC0.B16, ACC0.B16 \
VPMULL T0.D1, T2.D1, T3.Q1 \
VEOR T3.B16, ACCM.B16, ACCM.B16
MOVD productTable+0(FP), pTbl
MOVD data_base+8(FP), aut
MOVD data_len+16(FP), autLen
MOVD T+32(FP), tPtr
VEOR ACC0.B16, ACC0.B16, ACC0.B16
CBZ autLen, dataBail
MOVD $0xC2, H0
LSL $56, H0
VMOV H0, POLY.D[0]
MOVD $1, H0
VMOV H0, POLY.D[1]
VEOR ZERO.B16, ZERO.B16, ZERO.B16
MOVD pTbl, pTblSave
CMP $13, autLen
BEQ dataTLS
CMP $128, autLen
BLT startSinglesLoop
B octetsLoop
dataTLS:
ADD $14*16, pTbl
VLD1.P (pTbl), [T1.B16, T2.B16]
VEOR B0.B16, B0.B16, B0.B16
MOVD (aut), H0
VMOV H0, B0.D[0]
MOVW 8(aut), H0
VMOV H0, B0.S[2]
MOVB 12(aut), H0
VMOV H0, B0.B[12]
MOVD $0, autLen
B dataMul
octetsLoop:
CMP $128, autLen
BLT startSinglesLoop
SUB $128, autLen
VLD1.P 32(aut), [B0.B16, B1.B16]
VLD1.P 32(pTbl), [T1.B16, T2.B16]
VREV64 B0.B16, B0.B16
VEOR ACC0.B16, B0.B16, B0.B16
VEXT $8, B0.B16, B0.B16, T0.B16
VEOR B0.B16, T0.B16, T0.B16
VPMULL B0.D1, T1.D1, ACC1.Q1
VPMULL2 B0.D2, T1.D2, ACC0.Q1
VPMULL T0.D1, T2.D1, ACCM.Q1
mulRound(B1)
VLD1.P 32(aut), [B2.B16, B3.B16]
mulRound(B2)
mulRound(B3)
VLD1.P 32(aut), [B4.B16, B5.B16]
mulRound(B4)
mulRound(B5)
VLD1.P 32(aut), [B6.B16, B7.B16]
mulRound(B6)
mulRound(B7)
MOVD pTblSave, pTbl
reduce()
B octetsLoop
startSinglesLoop:
ADD $14*16, pTbl
VLD1.P (pTbl), [T1.B16, T2.B16]
singlesLoop:
CMP $16, autLen
BLT dataEnd
SUB $16, autLen
VLD1.P 16(aut), [B0.B16]
dataMul:
VREV64 B0.B16, B0.B16
VEOR ACC0.B16, B0.B16, B0.B16
VEXT $8, B0.B16, B0.B16, T0.B16
VEOR B0.B16, T0.B16, T0.B16
VPMULL B0.D1, T1.D1, ACC1.Q1
VPMULL2 B0.D2, T1.D2, ACC0.Q1
VPMULL T0.D1, T2.D1, ACCM.Q1
reduce()
B singlesLoop
dataEnd:
CBZ autLen, dataBail
VEOR B0.B16, B0.B16, B0.B16
ADD autLen, aut
dataLoadLoop:
MOVB.W -1(aut), H0
VEXT $15, B0.B16, ZERO.B16, B0.B16
VMOV H0, B0.B[0]
SUBS $1, autLen
BNE dataLoadLoop
B dataMul
dataBail:
VST1 [ACC0.B16], (tPtr)
RET
#undef pTbl
#undef aut
#undef tPtr
#undef autLen
#undef H0
#undef pTblSave
// func gcmAesEnc(productTable *[256]byte, dst, src []byte, ctr, T *[16]byte, ks []uint32)
TEXT ·gcmAesEnc(SB),NOSPLIT,$0
#define pTbl R0
#define dstPtr R1
#define ctrPtr R2
#define srcPtr R3
#define ks R4
#define tPtr R5
#define srcPtrLen R6
#define aluCTR R7
#define aluTMP R8
#define aluK R9
#define NR R10
#define H0 R11
#define H1 R12
#define curK R13
#define pTblSave R14
#define aesrndx8(K) \
AESE K.B16, B0.B16 \
AESMC B0.B16, B0.B16 \
AESE K.B16, B1.B16 \
AESMC B1.B16, B1.B16 \
AESE K.B16, B2.B16 \
AESMC B2.B16, B2.B16 \
AESE K.B16, B3.B16 \
AESMC B3.B16, B3.B16 \
AESE K.B16, B4.B16 \
AESMC B4.B16, B4.B16 \
AESE K.B16, B5.B16 \
AESMC B5.B16, B5.B16 \
AESE K.B16, B6.B16 \
AESMC B6.B16, B6.B16 \
AESE K.B16, B7.B16 \
AESMC B7.B16, B7.B16
#define aesrndlastx8(K) \
AESE K.B16, B0.B16 \
AESE K.B16, B1.B16 \
AESE K.B16, B2.B16 \
AESE K.B16, B3.B16 \
AESE K.B16, B4.B16 \
AESE K.B16, B5.B16 \
AESE K.B16, B6.B16 \
AESE K.B16, B7.B16
MOVD productTable+0(FP), pTbl
MOVD dst+8(FP), dstPtr
MOVD src_base+32(FP), srcPtr
MOVD src_len+40(FP), srcPtrLen
MOVD ctr+56(FP), ctrPtr
MOVD T+64(FP), tPtr
MOVD ks_base+72(FP), ks
MOVD ks_len+80(FP), NR
MOVD $0xC2, H1
LSL $56, H1
MOVD $1, H0
VMOV H1, POLY.D[0]
VMOV H0, POLY.D[1]
VEOR ZERO.B16, ZERO.B16, ZERO.B16
// Compute NR from len(ks)
MOVD pTbl, pTblSave
// Current tag, after AAD
VLD1 (tPtr), [ACC0.B16]
VEOR ACC1.B16, ACC1.B16, ACC1.B16
VEOR ACCM.B16, ACCM.B16, ACCM.B16
// Prepare initial counter, and the increment vector
VLD1 (ctrPtr), [CTR.B16]
VEOR INC.B16, INC.B16, INC.B16
MOVD $1, H0
VMOV H0, INC.S[3]
VREV32 CTR.B16, CTR.B16
VADD CTR.S4, INC.S4, CTR.S4
// Skip to <8 blocks loop
CMP $128, srcPtrLen
MOVD ks, H0
// For AES-128 round keys are stored in: K0 .. K10, KLAST
VLD1.P 64(H0), [K0.B16, K1.B16, K2.B16, K3.B16]
VLD1.P 64(H0), [K4.B16, K5.B16, K6.B16, K7.B16]
VLD1.P 48(H0), [K8.B16, K9.B16, K10.B16]
VMOV K10.B16, KLAST.B16
BLT startSingles
// There are at least 8 blocks to encrypt
TBZ $4, NR, octetsLoop
// For AES-192 round keys occupy: K0 .. K7, K10, K11, K8, K9, KLAST
VMOV K8.B16, K10.B16
VMOV K9.B16, K11.B16
VMOV KLAST.B16, K8.B16
VLD1.P 16(H0), [K9.B16]
VLD1.P 16(H0), [KLAST.B16]
TBZ $3, NR, octetsLoop
// For AES-256 round keys occupy: K0 .. K7, K10, K11, mem, mem, K8, K9, KLAST
VMOV KLAST.B16, K8.B16
VLD1.P 16(H0), [K9.B16]
VLD1.P 16(H0), [KLAST.B16]
ADD $10*16, ks, H0
MOVD H0, curK
octetsLoop:
SUB $128, srcPtrLen
VMOV CTR.B16, B0.B16
VADD B0.S4, INC.S4, B1.S4
VREV32 B0.B16, B0.B16
VADD B1.S4, INC.S4, B2.S4
VREV32 B1.B16, B1.B16
VADD B2.S4, INC.S4, B3.S4
VREV32 B2.B16, B2.B16
VADD B3.S4, INC.S4, B4.S4
VREV32 B3.B16, B3.B16
VADD B4.S4, INC.S4, B5.S4
VREV32 B4.B16, B4.B16
VADD B5.S4, INC.S4, B6.S4
VREV32 B5.B16, B5.B16
VADD B6.S4, INC.S4, B7.S4
VREV32 B6.B16, B6.B16
VADD B7.S4, INC.S4, CTR.S4
VREV32 B7.B16, B7.B16
aesrndx8(K0)
aesrndx8(K1)
aesrndx8(K2)
aesrndx8(K3)
aesrndx8(K4)
aesrndx8(K5)
aesrndx8(K6)
aesrndx8(K7)
TBZ $4, NR, octetsFinish
aesrndx8(K10)
aesrndx8(K11)
TBZ $3, NR, octetsFinish
VLD1.P 32(curK), [T1.B16, T2.B16]
aesrndx8(T1)
aesrndx8(T2)
MOVD H0, curK
octetsFinish:
aesrndx8(K8)
aesrndlastx8(K9)
VEOR KLAST.B16, B0.B16, B0.B16
VEOR KLAST.B16, B1.B16, B1.B16
VEOR KLAST.B16, B2.B16, B2.B16
VEOR KLAST.B16, B3.B16, B3.B16
VEOR KLAST.B16, B4.B16, B4.B16
VEOR KLAST.B16, B5.B16, B5.B16
VEOR KLAST.B16, B6.B16, B6.B16
VEOR KLAST.B16, B7.B16, B7.B16
VLD1.P 32(srcPtr), [T1.B16, T2.B16]
VEOR B0.B16, T1.B16, B0.B16
VEOR B1.B16, T2.B16, B1.B16
VST1.P [B0.B16, B1.B16], 32(dstPtr)
VLD1.P 32(srcPtr), [T1.B16, T2.B16]
VEOR B2.B16, T1.B16, B2.B16
VEOR B3.B16, T2.B16, B3.B16
VST1.P [B2.B16, B3.B16], 32(dstPtr)
VLD1.P 32(srcPtr), [T1.B16, T2.B16]
VEOR B4.B16, T1.B16, B4.B16
VEOR B5.B16, T2.B16, B5.B16
VST1.P [B4.B16, B5.B16], 32(dstPtr)
VLD1.P 32(srcPtr), [T1.B16, T2.B16]
VEOR B6.B16, T1.B16, B6.B16
VEOR B7.B16, T2.B16, B7.B16
VST1.P [B6.B16, B7.B16], 32(dstPtr)
VLD1.P 32(pTbl), [T1.B16, T2.B16]
VREV64 B0.B16, B0.B16
VEOR ACC0.B16, B0.B16, B0.B16
VEXT $8, B0.B16, B0.B16, T0.B16
VEOR B0.B16, T0.B16, T0.B16
VPMULL B0.D1, T1.D1, ACC1.Q1
VPMULL2 B0.D2, T1.D2, ACC0.Q1
VPMULL T0.D1, T2.D1, ACCM.Q1
mulRound(B1)
mulRound(B2)
mulRound(B3)
mulRound(B4)
mulRound(B5)
mulRound(B6)
mulRound(B7)
MOVD pTblSave, pTbl
reduce()
CMP $128, srcPtrLen
BGE octetsLoop
startSingles:
CBZ srcPtrLen, done
ADD $14*16, pTbl
// Preload H and its Karatsuba precomp
VLD1.P (pTbl), [T1.B16, T2.B16]
// Preload AES round keys
ADD $128, ks
VLD1.P 48(ks), [K8.B16, K9.B16, K10.B16]
VMOV K10.B16, KLAST.B16
TBZ $4, NR, singlesLoop
VLD1.P 32(ks), [B1.B16, B2.B16]
VMOV B2.B16, KLAST.B16
TBZ $3, NR, singlesLoop
VLD1.P 32(ks), [B3.B16, B4.B16]
VMOV B4.B16, KLAST.B16
singlesLoop:
CMP $16, srcPtrLen
BLT tail
SUB $16, srcPtrLen
VLD1.P 16(srcPtr), [T0.B16]
VEOR KLAST.B16, T0.B16, T0.B16
VREV32 CTR.B16, B0.B16
VADD CTR.S4, INC.S4, CTR.S4
AESE K0.B16, B0.B16
AESMC B0.B16, B0.B16
AESE K1.B16, B0.B16
AESMC B0.B16, B0.B16
AESE K2.B16, B0.B16
AESMC B0.B16, B0.B16
AESE K3.B16, B0.B16
AESMC B0.B16, B0.B16
AESE K4.B16, B0.B16
AESMC B0.B16, B0.B16
AESE K5.B16, B0.B16
AESMC B0.B16, B0.B16
AESE K6.B16, B0.B16
AESMC B0.B16, B0.B16
AESE K7.B16, B0.B16
AESMC B0.B16, B0.B16
AESE K8.B16, B0.B16
AESMC B0.B16, B0.B16
AESE K9.B16, B0.B16
TBZ $4, NR, singlesLast
AESMC B0.B16, B0.B16
AESE K10.B16, B0.B16
AESMC B0.B16, B0.B16
AESE B1.B16, B0.B16
TBZ $3, NR, singlesLast
AESMC B0.B16, B0.B16
AESE B2.B16, B0.B16
AESMC B0.B16, B0.B16
AESE B3.B16, B0.B16
singlesLast:
VEOR T0.B16, B0.B16, B0.B16
encReduce:
VST1.P [B0.B16], 16(dstPtr)
VREV64 B0.B16, B0.B16
VEOR ACC0.B16, B0.B16, B0.B16
VEXT $8, B0.B16, B0.B16, T0.B16
VEOR B0.B16, T0.B16, T0.B16
VPMULL B0.D1, T1.D1, ACC1.Q1
VPMULL2 B0.D2, T1.D2, ACC0.Q1
VPMULL T0.D1, T2.D1, ACCM.Q1
reduce()
B singlesLoop
tail:
CBZ srcPtrLen, done
VEOR T0.B16, T0.B16, T0.B16
VEOR T3.B16, T3.B16, T3.B16
MOVD $0, H1
SUB $1, H1
ADD srcPtrLen, srcPtr
TBZ $3, srcPtrLen, ld4
MOVD.W -8(srcPtr), H0
VMOV H0, T0.D[0]
VMOV H1, T3.D[0]
ld4:
TBZ $2, srcPtrLen, ld2
MOVW.W -4(srcPtr), H0
VEXT $12, T0.B16, ZERO.B16, T0.B16
VEXT $12, T3.B16, ZERO.B16, T3.B16
VMOV H0, T0.S[0]
VMOV H1, T3.S[0]
ld2:
TBZ $1, srcPtrLen, ld1
MOVH.W -2(srcPtr), H0
VEXT $14, T0.B16, ZERO.B16, T0.B16
VEXT $14, T3.B16, ZERO.B16, T3.B16
VMOV H0, T0.H[0]
VMOV H1, T3.H[0]
ld1:
TBZ $0, srcPtrLen, ld0
MOVB.W -1(srcPtr), H0
VEXT $15, T0.B16, ZERO.B16, T0.B16
VEXT $15, T3.B16, ZERO.B16, T3.B16
VMOV H0, T0.B[0]
VMOV H1, T3.B[0]
ld0:
MOVD ZR, srcPtrLen
VEOR KLAST.B16, T0.B16, T0.B16
VREV32 CTR.B16, B0.B16
AESE K0.B16, B0.B16
AESMC B0.B16, B0.B16
AESE K1.B16, B0.B16
AESMC B0.B16, B0.B16
AESE K2.B16, B0.B16
AESMC B0.B16, B0.B16
AESE K3.B16, B0.B16
AESMC B0.B16, B0.B16
AESE K4.B16, B0.B16
AESMC B0.B16, B0.B16
AESE K5.B16, B0.B16
AESMC B0.B16, B0.B16
AESE K6.B16, B0.B16
AESMC B0.B16, B0.B16
AESE K7.B16, B0.B16
AESMC B0.B16, B0.B16
AESE K8.B16, B0.B16
AESMC B0.B16, B0.B16
AESE K9.B16, B0.B16
TBZ $4, NR, tailLast
AESMC B0.B16, B0.B16
AESE K10.B16, B0.B16
AESMC B0.B16, B0.B16
AESE B1.B16, B0.B16
TBZ $3, NR, tailLast
AESMC B0.B16, B0.B16
AESE B2.B16, B0.B16
AESMC B0.B16, B0.B16
AESE B3.B16, B0.B16
tailLast:
VEOR T0.B16, B0.B16, B0.B16
VAND T3.B16, B0.B16, B0.B16
B encReduce
done:
VST1 [ACC0.B16], (tPtr)
RET
// func gcmAesDec(productTable *[256]byte, dst, src []byte, ctr, T *[16]byte, ks []uint32)
TEXT ·gcmAesDec(SB),NOSPLIT,$0
MOVD productTable+0(FP), pTbl
MOVD dst+8(FP), dstPtr
MOVD src_base+32(FP), srcPtr
MOVD src_len+40(FP), srcPtrLen
MOVD ctr+56(FP), ctrPtr
MOVD T+64(FP), tPtr
MOVD ks_base+72(FP), ks
MOVD ks_len+80(FP), NR
MOVD $0xC2, H1
LSL $56, H1
MOVD $1, H0
VMOV H1, POLY.D[0]
VMOV H0, POLY.D[1]
VEOR ZERO.B16, ZERO.B16, ZERO.B16
// Compute NR from len(ks)
MOVD pTbl, pTblSave
// Current tag, after AAD
VLD1 (tPtr), [ACC0.B16]
VEOR ACC1.B16, ACC1.B16, ACC1.B16
VEOR ACCM.B16, ACCM.B16, ACCM.B16
// Prepare initial counter, and the increment vector
VLD1 (ctrPtr), [CTR.B16]
VEOR INC.B16, INC.B16, INC.B16
MOVD $1, H0
VMOV H0, INC.S[3]
VREV32 CTR.B16, CTR.B16
VADD CTR.S4, INC.S4, CTR.S4
MOVD ks, H0
// For AES-128 round keys are stored in: K0 .. K10, KLAST
VLD1.P 64(H0), [K0.B16, K1.B16, K2.B16, K3.B16]
VLD1.P 64(H0), [K4.B16, K5.B16, K6.B16, K7.B16]
VLD1.P 48(H0), [K8.B16, K9.B16, K10.B16]
VMOV K10.B16, KLAST.B16
// Skip to <8 blocks loop
CMP $128, srcPtrLen
BLT startSingles
// There are at least 8 blocks to encrypt
TBZ $4, NR, octetsLoop
// For AES-192 round keys occupy: K0 .. K7, K10, K11, K8, K9, KLAST
VMOV K8.B16, K10.B16
VMOV K9.B16, K11.B16
VMOV KLAST.B16, K8.B16
VLD1.P 16(H0), [K9.B16]
VLD1.P 16(H0), [KLAST.B16]
TBZ $3, NR, octetsLoop
// For AES-256 round keys occupy: K0 .. K7, K10, K11, mem, mem, K8, K9, KLAST
VMOV KLAST.B16, K8.B16
VLD1.P 16(H0), [K9.B16]
VLD1.P 16(H0), [KLAST.B16]
ADD $10*16, ks, H0
MOVD H0, curK
octetsLoop:
SUB $128, srcPtrLen
VMOV CTR.B16, B0.B16
VADD B0.S4, INC.S4, B1.S4
VREV32 B0.B16, B0.B16
VADD B1.S4, INC.S4, B2.S4
VREV32 B1.B16, B1.B16
VADD B2.S4, INC.S4, B3.S4
VREV32 B2.B16, B2.B16
VADD B3.S4, INC.S4, B4.S4
VREV32 B3.B16, B3.B16
VADD B4.S4, INC.S4, B5.S4
VREV32 B4.B16, B4.B16
VADD B5.S4, INC.S4, B6.S4
VREV32 B5.B16, B5.B16
VADD B6.S4, INC.S4, B7.S4
VREV32 B6.B16, B6.B16
VADD B7.S4, INC.S4, CTR.S4
VREV32 B7.B16, B7.B16
aesrndx8(K0)
aesrndx8(K1)
aesrndx8(K2)
aesrndx8(K3)
aesrndx8(K4)
aesrndx8(K5)
aesrndx8(K6)
aesrndx8(K7)
TBZ $4, NR, octetsFinish
aesrndx8(K10)
aesrndx8(K11)
TBZ $3, NR, octetsFinish
VLD1.P 32(curK), [T1.B16, T2.B16]
aesrndx8(T1)
aesrndx8(T2)
MOVD H0, curK
octetsFinish:
aesrndx8(K8)
aesrndlastx8(K9)
VEOR KLAST.B16, B0.B16, T1.B16
VEOR KLAST.B16, B1.B16, T2.B16
VEOR KLAST.B16, B2.B16, B2.B16
VEOR KLAST.B16, B3.B16, B3.B16
VEOR KLAST.B16, B4.B16, B4.B16
VEOR KLAST.B16, B5.B16, B5.B16
VEOR KLAST.B16, B6.B16, B6.B16
VEOR KLAST.B16, B7.B16, B7.B16
VLD1.P 32(srcPtr), [B0.B16, B1.B16]
VEOR B0.B16, T1.B16, T1.B16
VEOR B1.B16, T2.B16, T2.B16
VST1.P [T1.B16, T2.B16], 32(dstPtr)
VLD1.P 32(pTbl), [T1.B16, T2.B16]
VREV64 B0.B16, B0.B16
VEOR ACC0.B16, B0.B16, B0.B16
VEXT $8, B0.B16, B0.B16, T0.B16
VEOR B0.B16, T0.B16, T0.B16
VPMULL B0.D1, T1.D1, ACC1.Q1
VPMULL2 B0.D2, T1.D2, ACC0.Q1
VPMULL T0.D1, T2.D1, ACCM.Q1
mulRound(B1)
VLD1.P 32(srcPtr), [B0.B16, B1.B16]
VEOR B2.B16, B0.B16, T1.B16
VEOR B3.B16, B1.B16, T2.B16
VST1.P [T1.B16, T2.B16], 32(dstPtr)
mulRound(B0)
mulRound(B1)
VLD1.P 32(srcPtr), [B0.B16, B1.B16]
VEOR B4.B16, B0.B16, T1.B16
VEOR B5.B16, B1.B16, T2.B16
VST1.P [T1.B16, T2.B16], 32(dstPtr)
mulRound(B0)
mulRound(B1)
VLD1.P 32(srcPtr), [B0.B16, B1.B16]
VEOR B6.B16, B0.B16, T1.B16
VEOR B7.B16, B1.B16, T2.B16
VST1.P [T1.B16, T2.B16], 32(dstPtr)
mulRound(B0)
mulRound(B1)
MOVD pTblSave, pTbl
reduce()
CMP $128, srcPtrLen
BGE octetsLoop
startSingles:
CBZ srcPtrLen, done
ADD $14*16, pTbl
// Preload H and its Karatsuba precomp
VLD1.P (pTbl), [T1.B16, T2.B16]
// Preload AES round keys
ADD $128, ks
VLD1.P 48(ks), [K8.B16, K9.B16, K10.B16]
VMOV K10.B16, KLAST.B16
TBZ $4, NR, singlesLoop
VLD1.P 32(ks), [B1.B16, B2.B16]
VMOV B2.B16, KLAST.B16
TBZ $3, NR, singlesLoop
VLD1.P 32(ks), [B3.B16, B4.B16]
VMOV B4.B16, KLAST.B16
singlesLoop:
CMP $16, srcPtrLen
BLT tail
SUB $16, srcPtrLen
VLD1.P 16(srcPtr), [T0.B16]
VREV64 T0.B16, B5.B16
VEOR KLAST.B16, T0.B16, T0.B16
VREV32 CTR.B16, B0.B16
VADD CTR.S4, INC.S4, CTR.S4
AESE K0.B16, B0.B16
AESMC B0.B16, B0.B16
AESE K1.B16, B0.B16
AESMC B0.B16, B0.B16
AESE K2.B16, B0.B16
AESMC B0.B16, B0.B16
AESE K3.B16, B0.B16
AESMC B0.B16, B0.B16
AESE K4.B16, B0.B16
AESMC B0.B16, B0.B16
AESE K5.B16, B0.B16
AESMC B0.B16, B0.B16
AESE K6.B16, B0.B16
AESMC B0.B16, B0.B16
AESE K7.B16, B0.B16
AESMC B0.B16, B0.B16
AESE K8.B16, B0.B16
AESMC B0.B16, B0.B16
AESE K9.B16, B0.B16
TBZ $4, NR, singlesLast
AESMC B0.B16, B0.B16
AESE K10.B16, B0.B16
AESMC B0.B16, B0.B16
AESE B1.B16, B0.B16
TBZ $3, NR, singlesLast
AESMC B0.B16, B0.B16
AESE B2.B16, B0.B16
AESMC B0.B16, B0.B16
AESE B3.B16, B0.B16
singlesLast:
VEOR T0.B16, B0.B16, B0.B16
VST1.P [B0.B16], 16(dstPtr)
VEOR ACC0.B16, B5.B16, B5.B16
VEXT $8, B5.B16, B5.B16, T0.B16
VEOR B5.B16, T0.B16, T0.B16
VPMULL B5.D1, T1.D1, ACC1.Q1
VPMULL2 B5.D2, T1.D2, ACC0.Q1
VPMULL T0.D1, T2.D1, ACCM.Q1
reduce()
B singlesLoop
tail:
CBZ srcPtrLen, done
VREV32 CTR.B16, B0.B16
VADD CTR.S4, INC.S4, CTR.S4
AESE K0.B16, B0.B16
AESMC B0.B16, B0.B16
AESE K1.B16, B0.B16
AESMC B0.B16, B0.B16
AESE K2.B16, B0.B16
AESMC B0.B16, B0.B16
AESE K3.B16, B0.B16
AESMC B0.B16, B0.B16
AESE K4.B16, B0.B16
AESMC B0.B16, B0.B16
AESE K5.B16, B0.B16
AESMC B0.B16, B0.B16
AESE K6.B16, B0.B16
AESMC B0.B16, B0.B16
AESE K7.B16, B0.B16
AESMC B0.B16, B0.B16
AESE K8.B16, B0.B16
AESMC B0.B16, B0.B16
AESE K9.B16, B0.B16
TBZ $4, NR, tailLast
AESMC B0.B16, B0.B16
AESE K10.B16, B0.B16
AESMC B0.B16, B0.B16
AESE B1.B16, B0.B16
TBZ $3, NR, tailLast
AESMC B0.B16, B0.B16
AESE B2.B16, B0.B16
AESMC B0.B16, B0.B16
AESE B3.B16, B0.B16
tailLast:
VEOR KLAST.B16, B0.B16, B0.B16
// Assuming it is safe to load past dstPtr due to the presence of the tag
VLD1 (srcPtr), [B5.B16]
VEOR B5.B16, B0.B16, B0.B16
VEOR T3.B16, T3.B16, T3.B16
MOVD $0, H1
SUB $1, H1
TBZ $3, srcPtrLen, ld4
VMOV B0.D[0], H0
MOVD.P H0, 8(dstPtr)
VMOV H1, T3.D[0]
VEXT $8, ZERO.B16, B0.B16, B0.B16
ld4:
TBZ $2, srcPtrLen, ld2
VMOV B0.S[0], H0
MOVW.P H0, 4(dstPtr)
VEXT $12, T3.B16, ZERO.B16, T3.B16
VMOV H1, T3.S[0]
VEXT $4, ZERO.B16, B0.B16, B0.B16
ld2:
TBZ $1, srcPtrLen, ld1
VMOV B0.H[0], H0
MOVH.P H0, 2(dstPtr)
VEXT $14, T3.B16, ZERO.B16, T3.B16
VMOV H1, T3.H[0]
VEXT $2, ZERO.B16, B0.B16, B0.B16
ld1:
TBZ $0, srcPtrLen, ld0
VMOV B0.B[0], H0
MOVB.P H0, 1(dstPtr)
VEXT $15, T3.B16, ZERO.B16, T3.B16
VMOV H1, T3.B[0]
ld0:
VAND T3.B16, B5.B16, B5.B16
VREV64 B5.B16, B5.B16
VEOR ACC0.B16, B5.B16, B5.B16
VEXT $8, B5.B16, B5.B16, T0.B16
VEOR B5.B16, T0.B16, T0.B16
VPMULL B5.D1, T1.D1, ACC1.Q1
VPMULL2 B5.D2, T1.D2, ACC0.Q1
VPMULL T0.D1, T2.D1, ACCM.Q1
reduce()
done:
VST1 [ACC0.B16], (tPtr)
RET
|
