| /* SPDX-License-Identifier: GPL-2.0 */ | |
| /* | |
| * Copyright (C) 2024 Christophe Leroy <christophe.leroy@csgroup.eu>, CS GROUP France | |
| */ | |
| #include <linux/linkage.h> | |
| #include <asm/ppc_asm.h> | |
| #define dst_bytes r3 | |
| #define key r4 | |
| #define counter r5 | |
| #define nblocks r6 | |
| #define idx_r0 r0 | |
| #define val4 r4 | |
| #define const0 0x61707865 | |
| #define const1 0x3320646e | |
| #define const2 0x79622d32 | |
| #define const3 0x6b206574 | |
| #define key0 r5 | |
| #define key1 r6 | |
| #define key2 r7 | |
| #define key3 r8 | |
| #define key4 r9 | |
| #define key5 r10 | |
| #define key6 r11 | |
| #define key7 r12 | |
| #define counter0 r14 | |
| #define counter1 r15 | |
| #define state0 r16 | |
| #define state1 r17 | |
| #define state2 r18 | |
| #define state3 r19 | |
| #define state4 r20 | |
| #define state5 r21 | |
| #define state6 r22 | |
| #define state7 r23 | |
| #define state8 r24 | |
| #define state9 r25 | |
| #define state10 r26 | |
| #define state11 r27 | |
| #define state12 r28 | |
| #define state13 r29 | |
| #define state14 r30 | |
| #define state15 r31 | |
| .macro quarterround4 a1 b1 c1 d1 a2 b2 c2 d2 a3 b3 c3 d3 a4 b4 c4 d4 | |
| add \a1, \a1, \b1 | |
| add \a2, \a2, \b2 | |
| add \a3, \a3, \b3 | |
| add \a4, \a4, \b4 | |
| xor \d1, \d1, \a1 | |
| xor \d2, \d2, \a2 | |
| xor \d3, \d3, \a3 | |
| xor \d4, \d4, \a4 | |
| rotlwi \d1, \d1, 16 | |
| rotlwi \d2, \d2, 16 | |
| rotlwi \d3, \d3, 16 | |
| rotlwi \d4, \d4, 16 | |
| add \c1, \c1, \d1 | |
| add \c2, \c2, \d2 | |
| add \c3, \c3, \d3 | |
| add \c4, \c4, \d4 | |
| xor \b1, \b1, \c1 | |
| xor \b2, \b2, \c2 | |
| xor \b3, \b3, \c3 | |
| xor \b4, \b4, \c4 | |
| rotlwi \b1, \b1, 12 | |
| rotlwi \b2, \b2, 12 | |
| rotlwi \b3, \b3, 12 | |
| rotlwi \b4, \b4, 12 | |
| add \a1, \a1, \b1 | |
| add \a2, \a2, \b2 | |
| add \a3, \a3, \b3 | |
| add \a4, \a4, \b4 | |
| xor \d1, \d1, \a1 | |
| xor \d2, \d2, \a2 | |
| xor \d3, \d3, \a3 | |
| xor \d4, \d4, \a4 | |
| rotlwi \d1, \d1, 8 | |
| rotlwi \d2, \d2, 8 | |
| rotlwi \d3, \d3, 8 | |
| rotlwi \d4, \d4, 8 | |
| add \c1, \c1, \d1 | |
| add \c2, \c2, \d2 | |
| add \c3, \c3, \d3 | |
| add \c4, \c4, \d4 | |
| xor \b1, \b1, \c1 | |
| xor \b2, \b2, \c2 | |
| xor \b3, \b3, \c3 | |
| xor \b4, \b4, \c4 | |
| rotlwi \b1, \b1, 7 | |
| rotlwi \b2, \b2, 7 | |
| rotlwi \b3, \b3, 7 | |
| rotlwi \b4, \b4, 7 | |
| .endm | |
| #define QUARTERROUND4(a1,b1,c1,d1,a2,b2,c2,d2,a3,b3,c3,d3,a4,b4,c4,d4) \ | |
| quarterround4 state##a1 state##b1 state##c1 state##d1 \ | |
| state##a2 state##b2 state##c2 state##d2 \ | |
| state##a3 state##b3 state##c3 state##d3 \ | |
| state##a4 state##b4 state##c4 state##d4 | |
| /* | |
| * Very basic 32 bits implementation of ChaCha20. Produces a given positive number | |
| * of blocks of output with a nonce of 0, taking an input key and 8-byte | |
| * counter. Importantly does not spill to the stack. Its arguments are: | |
| * | |
| * r3: output bytes | |
| * r4: 32-byte key input | |
| * r5: 8-byte counter input/output (saved on stack) | |
| * r6: number of 64-byte blocks to write to output | |
| * | |
| * r0: counter of blocks (initialised with r6) | |
| * r4: Value '4' after key has been read. | |
| * r5-r12: key | |
| * r14-r15: counter | |
| * r16-r31: state | |
| */ | |
| SYM_FUNC_START(__arch_chacha20_blocks_nostack) | |
| #ifdef __powerpc64__ | |
| std counter, -216(r1) | |
| std r14, -144(r1) | |
| std r15, -136(r1) | |
| std r16, -128(r1) | |
| std r17, -120(r1) | |
| std r18, -112(r1) | |
| std r19, -104(r1) | |
| std r20, -96(r1) | |
| std r21, -88(r1) | |
| std r22, -80(r1) | |
| std r23, -72(r1) | |
| std r24, -64(r1) | |
| std r25, -56(r1) | |
| std r26, -48(r1) | |
| std r27, -40(r1) | |
| std r28, -32(r1) | |
| std r29, -24(r1) | |
| std r30, -16(r1) | |
| std r31, -8(r1) | |
| #else | |
| stwu r1, -96(r1) | |
| stw counter, 20(r1) | |
| #ifdef __BIG_ENDIAN__ | |
| stmw r14, 24(r1) | |
| #else | |
| stw r14, 24(r1) | |
| stw r15, 28(r1) | |
| stw r16, 32(r1) | |
| stw r17, 36(r1) | |
| stw r18, 40(r1) | |
| stw r19, 44(r1) | |
| stw r20, 48(r1) | |
| stw r21, 52(r1) | |
| stw r22, 56(r1) | |
| stw r23, 60(r1) | |
| stw r24, 64(r1) | |
| stw r25, 68(r1) | |
| stw r26, 72(r1) | |
| stw r27, 76(r1) | |
| stw r28, 80(r1) | |
| stw r29, 84(r1) | |
| stw r30, 88(r1) | |
| stw r31, 92(r1) | |
| #endif | |
| #endif /* __powerpc64__ */ | |
| lwz counter0, 0(counter) | |
| lwz counter1, 4(counter) | |
| #ifdef __powerpc64__ | |
| rldimi counter0, counter1, 32, 0 | |
| #endif | |
| mr idx_r0, nblocks | |
| subi dst_bytes, dst_bytes, 4 | |
| lwz key0, 0(key) | |
| lwz key1, 4(key) | |
| lwz key2, 8(key) | |
| lwz key3, 12(key) | |
| lwz key4, 16(key) | |
| lwz key5, 20(key) | |
| lwz key6, 24(key) | |
| lwz key7, 28(key) | |
| li val4, 4 | |
| .Lblock: | |
| li r31, 10 | |
| lis state0, const0@ha | |
| lis state1, const1@ha | |
| lis state2, const2@ha | |
| lis state3, const3@ha | |
| addi state0, state0, const0@l | |
| addi state1, state1, const1@l | |
| addi state2, state2, const2@l | |
| addi state3, state3, const3@l | |
| mtctr r31 | |
| mr state4, key0 | |
| mr state5, key1 | |
| mr state6, key2 | |
| mr state7, key3 | |
| mr state8, key4 | |
| mr state9, key5 | |
| mr state10, key6 | |
| mr state11, key7 | |
| mr state12, counter0 | |
| mr state13, counter1 | |
| li state14, 0 | |
| li state15, 0 | |
| .Lpermute: | |
| QUARTERROUND4( 0, 4, 8,12, 1, 5, 9,13, 2, 6,10,14, 3, 7,11,15) | |
| QUARTERROUND4( 0, 5,10,15, 1, 6,11,12, 2, 7, 8,13, 3, 4, 9,14) | |
| bdnz .Lpermute | |
| addis state0, state0, const0@ha | |
| addis state1, state1, const1@ha | |
| addis state2, state2, const2@ha | |
| addis state3, state3, const3@ha | |
| addi state0, state0, const0@l | |
| addi state1, state1, const1@l | |
| addi state2, state2, const2@l | |
| addi state3, state3, const3@l | |
| add state4, state4, key0 | |
| add state5, state5, key1 | |
| add state6, state6, key2 | |
| add state7, state7, key3 | |
| add state8, state8, key4 | |
| add state9, state9, key5 | |
| add state10, state10, key6 | |
| add state11, state11, key7 | |
| add state12, state12, counter0 | |
| add state13, state13, counter1 | |
| #ifdef __BIG_ENDIAN__ | |
| stwbrx state0, val4, dst_bytes | |
| addi dst_bytes, dst_bytes, 8 | |
| stwbrx state1, 0, dst_bytes | |
| stwbrx state2, val4, dst_bytes | |
| addi dst_bytes, dst_bytes, 8 | |
| stwbrx state3, 0, dst_bytes | |
| stwbrx state4, val4, dst_bytes | |
| addi dst_bytes, dst_bytes, 8 | |
| stwbrx state5, 0, dst_bytes | |
| stwbrx state6, val4, dst_bytes | |
| addi dst_bytes, dst_bytes, 8 | |
| stwbrx state7, 0, dst_bytes | |
| stwbrx state8, val4, dst_bytes | |
| addi dst_bytes, dst_bytes, 8 | |
| stwbrx state9, 0, dst_bytes | |
| stwbrx state10, val4, dst_bytes | |
| addi dst_bytes, dst_bytes, 8 | |
| stwbrx state11, 0, dst_bytes | |
| stwbrx state12, val4, dst_bytes | |
| addi dst_bytes, dst_bytes, 8 | |
| stwbrx state13, 0, dst_bytes | |
| stwbrx state14, val4, dst_bytes | |
| addi dst_bytes, dst_bytes, 8 | |
| stwbrx state15, 0, dst_bytes | |
| #else | |
| stw state0, 4(dst_bytes) | |
| stw state1, 8(dst_bytes) | |
| stw state2, 12(dst_bytes) | |
| stw state3, 16(dst_bytes) | |
| stw state4, 20(dst_bytes) | |
| stw state5, 24(dst_bytes) | |
| stw state6, 28(dst_bytes) | |
| stw state7, 32(dst_bytes) | |
| stw state8, 36(dst_bytes) | |
| stw state9, 40(dst_bytes) | |
| stw state10, 44(dst_bytes) | |
| stw state11, 48(dst_bytes) | |
| stw state12, 52(dst_bytes) | |
| stw state13, 56(dst_bytes) | |
| stw state14, 60(dst_bytes) | |
| stwu state15, 64(dst_bytes) | |
| #endif | |
| subic. idx_r0, idx_r0, 1 /* subi. can't use r0 as source */ | |
| #ifdef __powerpc64__ | |
| addi counter0, counter0, 1 | |
| srdi counter1, counter0, 32 | |
| #else | |
| addic counter0, counter0, 1 | |
| addze counter1, counter1 | |
| #endif | |
| bne .Lblock | |
| #ifdef __powerpc64__ | |
| ld counter, -216(r1) | |
| #else | |
| lwz counter, 20(r1) | |
| #endif | |
| stw counter0, 0(counter) | |
| stw counter1, 4(counter) | |
| li r6, 0 | |
| li r7, 0 | |
| li r8, 0 | |
| li r9, 0 | |
| li r10, 0 | |
| li r11, 0 | |
| li r12, 0 | |
| #ifdef __powerpc64__ | |
| ld r14, -144(r1) | |
| ld r15, -136(r1) | |
| ld r16, -128(r1) | |
| ld r17, -120(r1) | |
| ld r18, -112(r1) | |
| ld r19, -104(r1) | |
| ld r20, -96(r1) | |
| ld r21, -88(r1) | |
| ld r22, -80(r1) | |
| ld r23, -72(r1) | |
| ld r24, -64(r1) | |
| ld r25, -56(r1) | |
| ld r26, -48(r1) | |
| ld r27, -40(r1) | |
| ld r28, -32(r1) | |
| ld r29, -24(r1) | |
| ld r30, -16(r1) | |
| ld r31, -8(r1) | |
| #else | |
| #ifdef __BIG_ENDIAN__ | |
| lmw r14, 24(r1) | |
| #else | |
| lwz r14, 24(r1) | |
| lwz r15, 28(r1) | |
| lwz r16, 32(r1) | |
| lwz r17, 36(r1) | |
| lwz r18, 40(r1) | |
| lwz r19, 44(r1) | |
| lwz r20, 48(r1) | |
| lwz r21, 52(r1) | |
| lwz r22, 56(r1) | |
| lwz r23, 60(r1) | |
| lwz r24, 64(r1) | |
| lwz r25, 68(r1) | |
| lwz r26, 72(r1) | |
| lwz r27, 76(r1) | |
| lwz r28, 80(r1) | |
| lwz r29, 84(r1) | |
| lwz r30, 88(r1) | |
| lwz r31, 92(r1) | |
| #endif | |
| addi r1, r1, 96 | |
| #endif /* __powerpc64__ */ | |
| blr | |
| SYM_FUNC_END(__arch_chacha20_blocks_nostack) |