| /* SPDX-License-Identifier: GPL-2.0-or-later */ |
| /* |
| * Implement AES algorithm using Intel AES Key Locker instructions. |
| * |
| * Most code is based from the AES-NI implementation, aesni-intel_asm.S |
| * |
| */ |
| |
| #include <linux/linkage.h> |
| #include <asm/inst.h> |
| #include <asm/frame.h> |
| #include "aes-intel_asm.S" |
| |
| .text |
| |
| #define STATE1 %xmm0 |
| #define STATE2 %xmm1 |
| #define STATE3 %xmm2 |
| #define STATE4 %xmm3 |
| #define STATE5 %xmm4 |
| #define STATE6 %xmm5 |
| #define STATE7 %xmm6 |
| #define STATE8 %xmm7 |
| #define STATE STATE1 |
| |
| #define IV %xmm9 |
| #define KEY %xmm10 |
| #define BSWAP_MASK %xmm11 |
| #define CTR %xmm12 |
| #define INC %xmm13 |
| |
| #ifdef __x86_64__ |
| #define IN1 %xmm8 |
| #define IN2 %xmm9 |
| #define IN3 %xmm10 |
| #define IN4 %xmm11 |
| #define IN5 %xmm12 |
| #define IN6 %xmm13 |
| #define IN7 %xmm14 |
| #define IN8 %xmm15 |
| #define IN IN1 |
| #define TCTR_LOW %r11 |
| #else |
| #define IN %xmm1 |
| #endif |
| |
| #ifdef __x86_64__ |
| #define AREG %rax |
| #define HANDLEP %rdi |
| #define OUTP %rsi |
| #define KLEN %r9d |
| #define INP %rdx |
| #define T1 %r10 |
| #define LEN %rcx |
| #define IVP %r8 |
| #else |
| #define AREG %eax |
| #define HANDLEP %edi |
| #define OUTP AREG |
| #define KLEN %ebx |
| #define INP %edx |
| #define T1 %ecx |
| #define LEN %esi |
| #define IVP %ebp |
| #endif |
| |
| #define UKEYP OUTP |
| #define GF128MUL_MASK %xmm11 |
| |
| /* |
| * int aeskl_setkey(struct crypto_aes_ctx *ctx, const u8 *in_key, unsigned int key_len) |
| */ |
| SYM_FUNC_START(aeskl_setkey) |
| FRAME_BEGIN |
| #ifndef __x86_64__ |
| push HANDLEP |
| movl (FRAME_OFFSET+8)(%esp), HANDLEP # ctx |
| movl (FRAME_OFFSET+12)(%esp), UKEYP # in_key |
| movl (FRAME_OFFSET+16)(%esp), %edx # key_len |
| #endif |
| movl %edx, 480(HANDLEP) |
| movdqu (UKEYP), STATE1 |
| mov $1, %eax |
| cmp $16, %dl |
| je .Lsetkey_128 |
| |
| movdqu 0x10(UKEYP), STATE2 |
| encodekey256 %eax, %eax |
| movdqu STATE4, 0x30(HANDLEP) |
| jmp .Lsetkey_end |
| .Lsetkey_128: |
| encodekey128 %eax, %eax |
| |
| .Lsetkey_end: |
| movdqu STATE1, (HANDLEP) |
| movdqu STATE2, 0x10(HANDLEP) |
| movdqu STATE3, 0x20(HANDLEP) |
| |
| xor AREG, AREG |
| #ifndef __x86_64__ |
| popl HANDLEP |
| #endif |
| FRAME_END |
| ret |
| SYM_FUNC_END(aeskl_setkey) |
| |
| /* |
| * int _aeskl_enc(const void *ctx, u8 *dst, const u8 *src) |
| */ |
| SYM_FUNC_START(_aeskl_enc) |
| FRAME_BEGIN |
| #ifndef __x86_64__ |
| pushl HANDLEP |
| pushl KLEN |
| movl (FRAME_OFFSET+12)(%esp), HANDLEP # ctx |
| movl (FRAME_OFFSET+16)(%esp), OUTP # dst |
| movl (FRAME_OFFSET+20)(%esp), INP # src |
| #endif |
| movdqu (INP), STATE |
| movl 480(HANDLEP), KLEN |
| |
| cmp $16, KLEN |
| je .Lenc_128 |
| aesenc256kl (HANDLEP), STATE |
| jz .Lenc_err |
| jmp .Lenc_noerr |
| .Lenc_128: |
| aesenc128kl (HANDLEP), STATE |
| jz .Lenc_err |
| |
| .Lenc_noerr: |
| xor AREG, AREG |
| jmp .Lenc_end |
| .Lenc_err: |
| mov $1, AREG |
| .Lenc_end: |
| movdqu STATE, (OUTP) |
| #ifndef __x86_64__ |
| popl KLEN |
| popl HANDLEP |
| #endif |
| FRAME_END |
| ret |
| SYM_FUNC_END(_aeskl_enc) |
| |
| /* |
| * int _aeskl_dec(const void *ctx, u8 *dst, const u8 *src) |
| */ |
| SYM_FUNC_START(_aeskl_dec) |
| FRAME_BEGIN |
| #ifndef __x86_64__ |
| pushl HANDLEP |
| pushl KLEN |
| movl (FRAME_OFFSET+12)(%esp), HANDLEP # ctx |
| movl (FRAME_OFFSET+16)(%esp), OUTP # dst |
| movl (FRAME_OFFSET+20)(%esp), INP # src |
| #endif |
| movdqu (INP), STATE |
| mov 480(HANDLEP), KLEN |
| |
| cmp $16, KLEN |
| je .Ldec_128 |
| aesdec256kl (HANDLEP), STATE |
| jz .Ldec_err |
| jmp .Ldec_noerr |
| .Ldec_128: |
| aesdec128kl (HANDLEP), STATE |
| jz .Ldec_err |
| |
| .Ldec_noerr: |
| xor AREG, AREG |
| jmp .Ldec_end |
| .Ldec_err: |
| mov $1, AREG |
| .Ldec_end: |
| movdqu STATE, (OUTP) |
| #ifndef __x86_64__ |
| popl KLEN |
| popl HANDLEP |
| #endif |
| FRAME_END |
| ret |
| SYM_FUNC_END(_aeskl_dec) |
| |
| #ifdef __x86_64__ |
| |
| /* |
| * XTS implementation |
| */ |
| |
| /* |
| * _aeskl_gf128mul_x_ble: internal ABI |
| * Multiply in GF(2^128) for XTS IVs |
| * input: |
| * IV: current IV |
| * GF128MUL_MASK == mask with 0x87 and 0x01 |
| * output: |
| * IV: next IV |
| * changed: |
| * CTR: == temporary value |
| */ |
| #define _aeskl_gf128mul_x_ble() \ |
| pshufd $0x13, IV, KEY; \ |
| paddq IV, IV; \ |
| psrad $31, KEY; \ |
| pand GF128MUL_MASK, KEY; \ |
| pxor KEY, IV; |
| |
| /* |
| * int _aeskl_xts_encrypt(const struct crypto_aes_ctx *ctx, u8 *dst, |
| * const u8 *src, unsigned int len, le128 *iv) |
| */ |
| SYM_FUNC_START(_aeskl_xts_encrypt) |
| FRAME_BEGIN |
| movdqa .Lgf128mul_x_ble_mask(%rip), GF128MUL_MASK |
| movups (IVP), IV |
| |
| mov 480(HANDLEP), KLEN |
| |
| .Lxts_enc8: |
| sub $128, LEN |
| jl .Lxts_enc1_pre |
| |
| movdqa IV, STATE1 |
| movdqu (INP), INC |
| pxor INC, STATE1 |
| movdqu IV, (OUTP) |
| |
| _aeskl_gf128mul_x_ble() |
| movdqa IV, STATE2 |
| movdqu 0x10(INP), INC |
| pxor INC, STATE2 |
| movdqu IV, 0x10(OUTP) |
| |
| _aeskl_gf128mul_x_ble() |
| movdqa IV, STATE3 |
| movdqu 0x20(INP), INC |
| pxor INC, STATE3 |
| movdqu IV, 0x20(OUTP) |
| |
| _aeskl_gf128mul_x_ble() |
| movdqa IV, STATE4 |
| movdqu 0x30(INP), INC |
| pxor INC, STATE4 |
| movdqu IV, 0x30(OUTP) |
| |
| _aeskl_gf128mul_x_ble() |
| movdqa IV, STATE5 |
| movdqu 0x40(INP), INC |
| pxor INC, STATE5 |
| movdqu IV, 0x40(OUTP) |
| |
| _aeskl_gf128mul_x_ble() |
| movdqa IV, STATE6 |
| movdqu 0x50(INP), INC |
| pxor INC, STATE6 |
| movdqu IV, 0x50(OUTP) |
| |
| _aeskl_gf128mul_x_ble() |
| movdqa IV, STATE7 |
| movdqu 0x60(INP), INC |
| pxor INC, STATE7 |
| movdqu IV, 0x60(OUTP) |
| |
| _aeskl_gf128mul_x_ble() |
| movdqa IV, STATE8 |
| movdqu 0x70(INP), INC |
| pxor INC, STATE8 |
| movdqu IV, 0x70(OUTP) |
| |
| cmp $16, KLEN |
| je .Lxts_enc8_128 |
| aesencwide256kl (%rdi) |
| jz .Lxts_enc_ret_err |
| jmp .Lxts_enc8_end |
| .Lxts_enc8_128: |
| aesencwide128kl (%rdi) |
| jz .Lxts_enc_ret_err |
| |
| .Lxts_enc8_end: |
| movdqu 0x00(OUTP), INC |
| pxor INC, STATE1 |
| movdqu STATE1, 0x00(OUTP) |
| |
| movdqu 0x10(OUTP), INC |
| pxor INC, STATE2 |
| movdqu STATE2, 0x10(OUTP) |
| |
| movdqu 0x20(OUTP), INC |
| pxor INC, STATE3 |
| movdqu STATE3, 0x20(OUTP) |
| |
| movdqu 0x30(OUTP), INC |
| pxor INC, STATE4 |
| movdqu STATE4, 0x30(OUTP) |
| |
| movdqu 0x40(OUTP), INC |
| pxor INC, STATE5 |
| movdqu STATE5, 0x40(OUTP) |
| |
| movdqu 0x50(OUTP), INC |
| pxor INC, STATE6 |
| movdqu STATE6, 0x50(OUTP) |
| |
| movdqu 0x60(OUTP), INC |
| pxor INC, STATE7 |
| movdqu STATE7, 0x60(OUTP) |
| |
| movdqu 0x70(OUTP), INC |
| pxor INC, STATE8 |
| movdqu STATE8, 0x70(OUTP) |
| |
| _aeskl_gf128mul_x_ble() |
| |
| add $128, INP |
| add $128, OUTP |
| test LEN, LEN |
| jnz .Lxts_enc8 |
| |
| .Lxts_enc_ret_iv: |
| movups IV, (IVP) |
| .Lxts_enc_ret_noerr: |
| xor AREG, AREG |
| jmp .Lxts_enc_ret |
| .Lxts_enc_ret_err: |
| mov $1, AREG |
| .Lxts_enc_ret: |
| FRAME_END |
| ret |
| |
| .Lxts_enc1_pre: |
| add $128, LEN |
| jz .Lxts_enc_ret_iv |
| sub $16, LEN |
| jl .Lxts_enc_cts4 |
| |
| .Lxts_enc1: |
| movdqu (INP), STATE1 |
| pxor IV, STATE1 |
| |
| cmp $16, KLEN |
| je .Lxts_enc1_128 |
| aesenc256kl (HANDLEP), STATE1 |
| jz .Lxts_enc_ret_err |
| jmp .Lxts_enc1_end |
| .Lxts_enc1_128: |
| aesenc128kl (HANDLEP), STATE1 |
| jz .Lxts_enc_ret_err |
| |
| .Lxts_enc1_end: |
| pxor IV, STATE1 |
| _aeskl_gf128mul_x_ble() |
| |
| test LEN, LEN |
| jz .Lxts_enc1_out |
| |
| add $16, INP |
| sub $16, LEN |
| jl .Lxts_enc_cts1 |
| |
| movdqu STATE1, (OUTP) |
| add $16, OUTP |
| jmp .Lxts_enc1 |
| |
| .Lxts_enc1_out: |
| movdqu STATE1, (OUTP) |
| jmp .Lxts_enc_ret_iv |
| |
| .Lxts_enc_cts4: |
| movdqu STATE8, STATE1 |
| sub $16, OUTP |
| |
| .Lxts_enc_cts1: |
| lea .Lcts_permute_table(%rip), T1 |
| add LEN, INP /* rewind input pointer */ |
| add $16, LEN /* # bytes in final block */ |
| movups (INP), IN1 |
| |
| mov T1, IVP |
| add $32, IVP |
| add LEN, T1 |
| sub LEN, IVP |
| add OUTP, LEN |
| |
| movups (T1), STATE2 |
| movaps STATE1, STATE3 |
| pshufb STATE2, STATE1 |
| movups STATE1, (LEN) |
| |
| movups (IVP), STATE1 |
| pshufb STATE1, IN1 |
| pblendvb STATE3, IN1 |
| movaps IN1, STATE1 |
| |
| pxor IV, STATE1 |
| |
| cmp $16, KLEN |
| je .Lxts_enc1_cts_128 |
| aesenc256kl (HANDLEP), STATE1 |
| jz .Lxts_enc_ret_err |
| jmp .Lxts_enc1_cts_end |
| .Lxts_enc1_cts_128: |
| aesenc128kl (HANDLEP), STATE1 |
| jz .Lxts_enc_ret_err |
| |
| .Lxts_enc1_cts_end: |
| pxor IV, STATE1 |
| movups STATE1, (OUTP) |
| jmp .Lxts_enc_ret_noerr |
| SYM_FUNC_END(_aeskl_xts_encrypt) |
| |
| /* |
| * int _aeskl_xts_decrypt(const struct crypto_aes_ctx *ctx, u8 *dst, |
| * const u8 *src, unsigned int len, le128 *iv) |
| */ |
| SYM_FUNC_START(_aeskl_xts_decrypt) |
| FRAME_BEGIN |
| movdqa .Lgf128mul_x_ble_mask(%rip), GF128MUL_MASK |
| movups (IVP), IV |
| |
| mov 480(HANDLEP), KLEN |
| |
| test $15, LEN |
| jz .Lxts_dec8 |
| sub $16, LEN |
| |
| .Lxts_dec8: |
| sub $128, LEN |
| jl .Lxts_dec1_pre |
| |
| movdqa IV, STATE1 |
| movdqu (INP), INC |
| pxor INC, STATE1 |
| movdqu IV, (OUTP) |
| |
| _aeskl_gf128mul_x_ble() |
| movdqa IV, STATE2 |
| movdqu 0x10(INP), INC |
| pxor INC, STATE2 |
| movdqu IV, 0x10(OUTP) |
| |
| _aeskl_gf128mul_x_ble() |
| movdqa IV, STATE3 |
| movdqu 0x20(INP), INC |
| pxor INC, STATE3 |
| movdqu IV, 0x20(OUTP) |
| |
| _aeskl_gf128mul_x_ble() |
| movdqa IV, STATE4 |
| movdqu 0x30(INP), INC |
| pxor INC, STATE4 |
| movdqu IV, 0x30(OUTP) |
| |
| _aeskl_gf128mul_x_ble() |
| movdqa IV, STATE5 |
| movdqu 0x40(INP), INC |
| pxor INC, STATE5 |
| movdqu IV, 0x40(OUTP) |
| |
| _aeskl_gf128mul_x_ble() |
| movdqa IV, STATE6 |
| movdqu 0x50(INP), INC |
| pxor INC, STATE6 |
| movdqu IV, 0x50(OUTP) |
| |
| _aeskl_gf128mul_x_ble() |
| movdqa IV, STATE7 |
| movdqu 0x60(INP), INC |
| pxor INC, STATE7 |
| movdqu IV, 0x60(OUTP) |
| |
| _aeskl_gf128mul_x_ble() |
| movdqa IV, STATE8 |
| movdqu 0x70(INP), INC |
| pxor INC, STATE8 |
| movdqu IV, 0x70(OUTP) |
| |
| cmp $16, KLEN |
| je .Lxts_dec8_128 |
| aesdecwide256kl (%rdi) |
| jz .Lxts_dec_ret_err |
| jmp .Lxts_dec8_end |
| .Lxts_dec8_128: |
| aesdecwide128kl (%rdi) |
| jz .Lxts_dec_ret_err |
| |
| .Lxts_dec8_end: |
| movdqu 0x00(OUTP), INC |
| pxor INC, STATE1 |
| movdqu STATE1, 0x00(OUTP) |
| |
| movdqu 0x10(OUTP), INC |
| pxor INC, STATE2 |
| movdqu STATE2, 0x10(OUTP) |
| |
| movdqu 0x20(OUTP), INC |
| pxor INC, STATE3 |
| movdqu STATE3, 0x20(OUTP) |
| |
| movdqu 0x30(OUTP), INC |
| pxor INC, STATE4 |
| movdqu STATE4, 0x30(OUTP) |
| |
| movdqu 0x40(OUTP), INC |
| pxor INC, STATE5 |
| movdqu STATE5, 0x40(OUTP) |
| |
| movdqu 0x50(OUTP), INC |
| pxor INC, STATE6 |
| movdqu STATE6, 0x50(OUTP) |
| |
| movdqu 0x60(OUTP), INC |
| pxor INC, STATE7 |
| movdqu STATE7, 0x60(OUTP) |
| |
| movdqu 0x70(OUTP), INC |
| pxor INC, STATE8 |
| movdqu STATE8, 0x70(OUTP) |
| |
| _aeskl_gf128mul_x_ble() |
| |
| add $128, INP |
| add $128, OUTP |
| test LEN, LEN |
| jnz .Lxts_dec8 |
| |
| .Lxts_dec_ret_iv: |
| movups IV, (IVP) |
| .Lxts_dec_ret_noerr: |
| xor AREG, AREG |
| jmp .Lxts_dec_ret |
| .Lxts_dec_ret_err: |
| mov $1, AREG |
| .Lxts_dec_ret: |
| FRAME_END |
| ret |
| |
| .Lxts_dec1_pre: |
| add $128, LEN |
| jz .Lxts_dec_ret_iv |
| |
| .Lxts_dec1: |
| movdqu (INP), STATE1 |
| |
| add $16, INP |
| sub $16, LEN |
| jl .Lxts_dec_cts1 |
| |
| pxor IV, STATE1 |
| |
| cmp $16, KLEN |
| je .Lxts_dec1_128 |
| aesdec256kl (HANDLEP), STATE1 |
| jz .Lxts_dec_ret_err |
| jmp .Lxts_dec1_end |
| .Lxts_dec1_128: |
| aesdec128kl (HANDLEP), STATE1 |
| jz .Lxts_dec_ret_err |
| |
| .Lxts_dec1_end: |
| pxor IV, STATE1 |
| _aeskl_gf128mul_x_ble() |
| |
| test LEN, LEN |
| jz .Lxts_dec1_out |
| |
| movdqu STATE1, (OUTP) |
| add $16, OUTP |
| jmp .Lxts_dec1 |
| |
| .Lxts_dec1_out: |
| movdqu STATE1, (OUTP) |
| jmp .Lxts_dec_ret_iv |
| |
| .Lxts_dec_cts1: |
| movdqa IV, STATE5 |
| _aeskl_gf128mul_x_ble() |
| |
| pxor IV, STATE1 |
| |
| cmp $16, KLEN |
| je .Lxts_dec1_cts_pre_128 |
| aesdec256kl (HANDLEP), STATE1 |
| jz .Lxts_dec_ret_err |
| jmp .Lxts_dec1_cts_pre_end |
| .Lxts_dec1_cts_pre_128: |
| aesdec128kl (HANDLEP), STATE1 |
| jz .Lxts_dec_ret_err |
| |
| .Lxts_dec1_cts_pre_end: |
| pxor IV, STATE1 |
| |
| lea .Lcts_permute_table(%rip), T1 |
| add LEN, INP /* rewind input pointer */ |
| add $16, LEN /* # bytes in final block */ |
| movups (INP), IN1 |
| |
| mov T1, IVP |
| add $32, IVP |
| add LEN, T1 |
| sub LEN, IVP |
| add OUTP, LEN |
| |
| movups (T1), STATE2 |
| movaps STATE1, STATE3 |
| pshufb STATE2, STATE1 |
| movups STATE1, (LEN) |
| |
| movups (IVP), STATE1 |
| pshufb STATE1, IN1 |
| pblendvb STATE3, IN1 |
| movaps IN1, STATE1 |
| |
| pxor STATE5, STATE1 |
| |
| cmp $16, KLEN |
| je .Lxts_dec1_cts_128 |
| aesdec256kl (HANDLEP), STATE1 |
| jz .Lxts_dec_ret_err |
| jmp .Lxts_dec1_cts_end |
| .Lxts_dec1_cts_128: |
| aesdec128kl (HANDLEP), STATE1 |
| jz .Lxts_dec_ret_err |
| |
| .Lxts_dec1_cts_end: |
| pxor STATE5, STATE1 |
| |
| movups STATE1, (OUTP) |
| jmp .Lxts_dec_ret_noerr |
| |
| SYM_FUNC_END(_aeskl_xts_decrypt) |
| |
| #endif |
