| /* SPDX-License-Identifier: GPL-2.0-or-later */ | |
| /* | |
| * This file contains assembly-language implementations | |
| * of IP-style 1's complement checksum routines. | |
| * | |
| * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org) | |
| * | |
| * Severely hacked about by Paul Mackerras (paulus@cs.anu.edu.au). | |
| */ | |
| #include <linux/export.h> | |
| #include <linux/sys.h> | |
| #include <asm/processor.h> | |
| #include <asm/cache.h> | |
| #include <asm/errno.h> | |
| #include <asm/ppc_asm.h> | |
| .text | |
| /* | |
| * computes the checksum of a memory block at buff, length len, | |
| * and adds in "sum" (32-bit) | |
| * | |
| * __csum_partial(buff, len, sum) | |
| */ | |
| _GLOBAL(__csum_partial) | |
| subi r3,r3,4 | |
| srawi. r6,r4,2 /* Divide len by 4 and also clear carry */ | |
| beq 3f /* if we're doing < 4 bytes */ | |
| andi. r0,r3,2 /* Align buffer to longword boundary */ | |
| beq+ 1f | |
| lhz r0,4(r3) /* do 2 bytes to get aligned */ | |
| subi r4,r4,2 | |
| addi r3,r3,2 | |
| srwi. r6,r4,2 /* # words to do */ | |
| adde r5,r5,r0 | |
| beq 3f | |
| 1: andi. r6,r6,3 /* Prepare to handle words 4 by 4 */ | |
| beq 21f | |
| mtctr r6 | |
| 2: lwzu r0,4(r3) | |
| adde r5,r5,r0 | |
| bdnz 2b | |
| 21: srwi. r6,r4,4 /* # blocks of 4 words to do */ | |
| beq 3f | |
| lwz r0,4(r3) | |
| mtctr r6 | |
| lwz r6,8(r3) | |
| adde r5,r5,r0 | |
| lwz r7,12(r3) | |
| adde r5,r5,r6 | |
| lwzu r8,16(r3) | |
| adde r5,r5,r7 | |
| bdz 23f | |
| 22: lwz r0,4(r3) | |
| adde r5,r5,r8 | |
| lwz r6,8(r3) | |
| adde r5,r5,r0 | |
| lwz r7,12(r3) | |
| adde r5,r5,r6 | |
| lwzu r8,16(r3) | |
| adde r5,r5,r7 | |
| bdnz 22b | |
| 23: adde r5,r5,r8 | |
| 3: andi. r0,r4,2 | |
| beq+ 4f | |
| lhz r0,4(r3) | |
| addi r3,r3,2 | |
| adde r5,r5,r0 | |
| 4: andi. r0,r4,1 | |
| beq+ 5f | |
| lbz r0,4(r3) | |
| slwi r0,r0,8 /* Upper byte of word */ | |
| adde r5,r5,r0 | |
| 5: addze r3,r5 /* add in final carry */ | |
| blr | |
| EXPORT_SYMBOL(__csum_partial) | |
| /* | |
| * Computes the checksum of a memory block at src, length len, | |
| * and adds in 0xffffffff, while copying the block to dst. | |
| * If an access exception occurs it returns zero. | |
| * | |
| * csum_partial_copy_generic(src, dst, len) | |
| */ | |
| #define CSUM_COPY_16_BYTES_WITHEX(n) \ | |
| 8 ## n ## 0: \ | |
| lwz r7,4(r4); \ | |
| 8 ## n ## 1: \ | |
| lwz r8,8(r4); \ | |
| 8 ## n ## 2: \ | |
| lwz r9,12(r4); \ | |
| 8 ## n ## 3: \ | |
| lwzu r10,16(r4); \ | |
| 8 ## n ## 4: \ | |
| stw r7,4(r6); \ | |
| adde r12,r12,r7; \ | |
| 8 ## n ## 5: \ | |
| stw r8,8(r6); \ | |
| adde r12,r12,r8; \ | |
| 8 ## n ## 6: \ | |
| stw r9,12(r6); \ | |
| adde r12,r12,r9; \ | |
| 8 ## n ## 7: \ | |
| stwu r10,16(r6); \ | |
| adde r12,r12,r10 | |
| #define CSUM_COPY_16_BYTES_EXCODE(n) \ | |
| EX_TABLE(8 ## n ## 0b, fault); \ | |
| EX_TABLE(8 ## n ## 1b, fault); \ | |
| EX_TABLE(8 ## n ## 2b, fault); \ | |
| EX_TABLE(8 ## n ## 3b, fault); \ | |
| EX_TABLE(8 ## n ## 4b, fault); \ | |
| EX_TABLE(8 ## n ## 5b, fault); \ | |
| EX_TABLE(8 ## n ## 6b, fault); \ | |
| EX_TABLE(8 ## n ## 7b, fault); | |
| .text | |
| CACHELINE_BYTES = L1_CACHE_BYTES | |
| LG_CACHELINE_BYTES = L1_CACHE_SHIFT | |
| CACHELINE_MASK = (L1_CACHE_BYTES-1) | |
| _GLOBAL(csum_partial_copy_generic) | |
| li r12,-1 | |
| addic r0,r0,0 /* clear carry */ | |
| addi r6,r4,-4 | |
| neg r0,r4 | |
| addi r4,r3,-4 | |
| andi. r0,r0,CACHELINE_MASK /* # bytes to start of cache line */ | |
| crset 4*cr7+eq | |
| beq 58f | |
| cmplw 0,r5,r0 /* is this more than total to do? */ | |
| blt 63f /* if not much to do */ | |
| rlwinm r7,r6,3,0x8 | |
| rlwnm r12,r12,r7,0,31 /* odd destination address: rotate one byte */ | |
| cmplwi cr7,r7,0 /* is destination address even ? */ | |
| andi. r8,r0,3 /* get it word-aligned first */ | |
| mtctr r8 | |
| beq+ 61f | |
| li r3,0 | |
| 70: lbz r9,4(r4) /* do some bytes */ | |
| addi r4,r4,1 | |
| slwi r3,r3,8 | |
| rlwimi r3,r9,0,24,31 | |
| 71: stb r9,4(r6) | |
| addi r6,r6,1 | |
| bdnz 70b | |
| adde r12,r12,r3 | |
| 61: subf r5,r0,r5 | |
| srwi. r0,r0,2 | |
| mtctr r0 | |
| beq 58f | |
| 72: lwzu r9,4(r4) /* do some words */ | |
| adde r12,r12,r9 | |
| 73: stwu r9,4(r6) | |
| bdnz 72b | |
| 58: srwi. r0,r5,LG_CACHELINE_BYTES /* # complete cachelines */ | |
| clrlwi r5,r5,32-LG_CACHELINE_BYTES | |
| li r11,4 | |
| beq 63f | |
| /* Here we decide how far ahead to prefetch the source */ | |
| li r3,4 | |
| cmpwi r0,1 | |
| li r7,0 | |
| ble 114f | |
| li r7,1 | |
| #if MAX_COPY_PREFETCH > 1 | |
| /* Heuristically, for large transfers we prefetch | |
| MAX_COPY_PREFETCH cachelines ahead. For small transfers | |
| we prefetch 1 cacheline ahead. */ | |
| cmpwi r0,MAX_COPY_PREFETCH | |
| ble 112f | |
| li r7,MAX_COPY_PREFETCH | |
| 112: mtctr r7 | |
| 111: dcbt r3,r4 | |
| addi r3,r3,CACHELINE_BYTES | |
| bdnz 111b | |
| #else | |
| dcbt r3,r4 | |
| addi r3,r3,CACHELINE_BYTES | |
| #endif /* MAX_COPY_PREFETCH > 1 */ | |
| 114: subf r8,r7,r0 | |
| mr r0,r7 | |
| mtctr r8 | |
| 53: dcbt r3,r4 | |
| 54: dcbz r11,r6 | |
| /* the main body of the cacheline loop */ | |
| CSUM_COPY_16_BYTES_WITHEX(0) | |
| #if L1_CACHE_BYTES >= 32 | |
| CSUM_COPY_16_BYTES_WITHEX(1) | |
| #if L1_CACHE_BYTES >= 64 | |
| CSUM_COPY_16_BYTES_WITHEX(2) | |
| CSUM_COPY_16_BYTES_WITHEX(3) | |
| #if L1_CACHE_BYTES >= 128 | |
| CSUM_COPY_16_BYTES_WITHEX(4) | |
| CSUM_COPY_16_BYTES_WITHEX(5) | |
| CSUM_COPY_16_BYTES_WITHEX(6) | |
| CSUM_COPY_16_BYTES_WITHEX(7) | |
| #endif | |
| #endif | |
| #endif | |
| bdnz 53b | |
| cmpwi r0,0 | |
| li r3,4 | |
| li r7,0 | |
| bne 114b | |
| 63: srwi. r0,r5,2 | |
| mtctr r0 | |
| beq 64f | |
| 30: lwzu r0,4(r4) | |
| adde r12,r12,r0 | |
| 31: stwu r0,4(r6) | |
| bdnz 30b | |
| 64: andi. r0,r5,2 | |
| beq+ 65f | |
| 40: lhz r0,4(r4) | |
| addi r4,r4,2 | |
| 41: sth r0,4(r6) | |
| adde r12,r12,r0 | |
| addi r6,r6,2 | |
| 65: andi. r0,r5,1 | |
| beq+ 66f | |
| 50: lbz r0,4(r4) | |
| 51: stb r0,4(r6) | |
| slwi r0,r0,8 | |
| adde r12,r12,r0 | |
| 66: addze r3,r12 | |
| beqlr+ cr7 | |
| rlwinm r3,r3,8,0,31 /* odd destination address: rotate one byte */ | |
| blr | |
| fault: | |
| li r3,0 | |
| blr | |
| EX_TABLE(70b, fault); | |
| EX_TABLE(71b, fault); | |
| EX_TABLE(72b, fault); | |
| EX_TABLE(73b, fault); | |
| EX_TABLE(54b, fault); | |
| /* | |
| * this stuff handles faults in the cacheline loop and branches to either | |
| * fault (if in read part) or fault (if in write part) | |
| */ | |
| CSUM_COPY_16_BYTES_EXCODE(0) | |
| #if L1_CACHE_BYTES >= 32 | |
| CSUM_COPY_16_BYTES_EXCODE(1) | |
| #if L1_CACHE_BYTES >= 64 | |
| CSUM_COPY_16_BYTES_EXCODE(2) | |
| CSUM_COPY_16_BYTES_EXCODE(3) | |
| #if L1_CACHE_BYTES >= 128 | |
| CSUM_COPY_16_BYTES_EXCODE(4) | |
| CSUM_COPY_16_BYTES_EXCODE(5) | |
| CSUM_COPY_16_BYTES_EXCODE(6) | |
| CSUM_COPY_16_BYTES_EXCODE(7) | |
| #endif | |
| #endif | |
| #endif | |
| EX_TABLE(30b, fault); | |
| EX_TABLE(31b, fault); | |
| EX_TABLE(40b, fault); | |
| EX_TABLE(41b, fault); | |
| EX_TABLE(50b, fault); | |
| EX_TABLE(51b, fault); | |
| EXPORT_SYMBOL(csum_partial_copy_generic) | |
| /* | |
| * __sum16 csum_ipv6_magic(const struct in6_addr *saddr, | |
| * const struct in6_addr *daddr, | |
| * __u32 len, __u8 proto, __wsum sum) | |
| */ | |
| _GLOBAL(csum_ipv6_magic) | |
| lwz r8, 0(r3) | |
| lwz r9, 4(r3) | |
| addc r0, r7, r8 | |
| lwz r10, 8(r3) | |
| adde r0, r0, r9 | |
| lwz r11, 12(r3) | |
| adde r0, r0, r10 | |
| lwz r8, 0(r4) | |
| adde r0, r0, r11 | |
| lwz r9, 4(r4) | |
| adde r0, r0, r8 | |
| lwz r10, 8(r4) | |
| adde r0, r0, r9 | |
| lwz r11, 12(r4) | |
| adde r0, r0, r10 | |
| add r5, r5, r6 /* assumption: len + proto doesn't carry */ | |
| adde r0, r0, r11 | |
| adde r0, r0, r5 | |
| addze r0, r0 | |
| rotlwi r3, r0, 16 | |
| add r3, r0, r3 | |
| not r3, r3 | |
| rlwinm r3, r3, 16, 16, 31 | |
| blr | |
| EXPORT_SYMBOL(csum_ipv6_magic) |