| /* SPDX-License-Identifier: GPL-2.0-only */ | |
| /* | |
| * Accelerated GHASH implementation with Intel PCLMULQDQ-NI | |
| * instructions. This file contains accelerated part of ghash | |
| * implementation. More information about PCLMULQDQ can be found at: | |
| * | |
| * https://www.intel.com/content/dam/develop/external/us/en/documents/clmul-wp-rev-2-02-2014-04-20.pdf | |
| * | |
| * Copyright (c) 2009 Intel Corp. | |
| * Author: Huang Ying <ying.huang@intel.com> | |
| * Vinodh Gopal | |
| * Erdinc Ozturk | |
| * Deniz Karakoyunlu | |
| */ | |
| #include <linux/linkage.h> | |
| #include <asm/frame.h> | |
| .section .rodata.cst16.bswap_mask, "aM", @progbits, 16 | |
| .align 16 | |
| .Lbswap_mask: | |
| .octa 0x000102030405060708090a0b0c0d0e0f | |
| #define ACC %xmm0 | |
| #define KEY %xmm1 | |
| #define T1 %xmm2 | |
| #define T2 %xmm3 | |
| #define T3 %xmm4 | |
| #define BSWAP %xmm5 | |
| #define IN1 %xmm6 | |
| .text | |
| /* | |
| * __clmul_gf128mul_ble: internal ABI | |
| * input: | |
| * ACC: operand1 | |
| * KEY: operand2, hash_key << 1 mod poly | |
| * output: | |
| * ACC: operand1 * operand2 mod poly | |
| * changed: | |
| * T1 | |
| * T2 | |
| * T3 | |
| */ | |
| SYM_FUNC_START_LOCAL(__clmul_gf128mul_ble) | |
| movaps ACC, T1 | |
| pshufd $0b01001110, ACC, T2 | |
| pshufd $0b01001110, KEY, T3 | |
| pxor ACC, T2 | |
| pxor KEY, T3 | |
| pclmulqdq $0x00, KEY, ACC # ACC = a0 * b0 | |
| pclmulqdq $0x11, KEY, T1 # T1 = a1 * b1 | |
| pclmulqdq $0x00, T3, T2 # T2 = (a1 + a0) * (b1 + b0) | |
| pxor ACC, T2 | |
| pxor T1, T2 # T2 = a0 * b1 + a1 * b0 | |
| movaps T2, T3 | |
| pslldq $8, T3 | |
| psrldq $8, T2 | |
| pxor T3, ACC | |
| pxor T2, T1 # <T1:ACC> is result of | |
| # carry-less multiplication | |
| # first phase of the reduction | |
| movaps ACC, T3 | |
| psllq $1, T3 | |
| pxor ACC, T3 | |
| psllq $5, T3 | |
| pxor ACC, T3 | |
| psllq $57, T3 | |
| movaps T3, T2 | |
| pslldq $8, T2 | |
| psrldq $8, T3 | |
| pxor T2, ACC | |
| pxor T3, T1 | |
| # second phase of the reduction | |
| movaps ACC, T2 | |
| psrlq $5, T2 | |
| pxor ACC, T2 | |
| psrlq $1, T2 | |
| pxor ACC, T2 | |
| psrlq $1, T2 | |
| pxor T2, T1 | |
| pxor T1, ACC | |
| RET | |
| SYM_FUNC_END(__clmul_gf128mul_ble) | |
| /* | |
| * void polyval_mul_pclmul(struct polyval_elem *a, | |
| * const struct polyval_elem *b) | |
| */ | |
| SYM_FUNC_START(polyval_mul_pclmul) | |
| FRAME_BEGIN | |
| movups (%rdi), ACC | |
| movups (%rsi), KEY | |
| call __clmul_gf128mul_ble | |
| movups ACC, (%rdi) | |
| FRAME_END | |
| RET | |
| SYM_FUNC_END(polyval_mul_pclmul) | |
| /* | |
| * void ghash_blocks_pclmul(struct polyval_elem *acc, | |
| * const struct polyval_elem *key, | |
| * const u8 *data, size_t nblocks) | |
| */ | |
| SYM_FUNC_START(ghash_blocks_pclmul) | |
| FRAME_BEGIN | |
| movaps .Lbswap_mask(%rip), BSWAP | |
| movups (%rdi), ACC | |
| movups (%rsi), KEY | |
| .align 4 | |
| .Lnext_block: | |
| movups (%rdx), IN1 | |
| pshufb BSWAP, IN1 | |
| pxor IN1, ACC | |
| call __clmul_gf128mul_ble | |
| add $16, %rdx | |
| dec %rcx | |
| jnz .Lnext_block | |
| movups ACC, (%rdi) | |
| FRAME_END | |
| RET | |
| SYM_FUNC_END(ghash_blocks_pclmul) |