| /* SPDX-License-Identifier: GPL-2.0 */ | |
| #include <linux/export.h> | |
| #include <linux/linkage.h> | |
| SYM_FUNC_START(memmove) | |
| /* | |
| * void *memmove(void *dest_in, const void *src_in, size_t n) | |
| * -mregparm=3 passes these in registers: | |
| * dest_in: %eax | |
| * src_in: %edx | |
| * n: %ecx | |
| * See also: arch/x86/entry/calling.h for description of the calling convention. | |
| * | |
| * n can remain in %ecx, but for `rep movsl`, we'll need dest in %edi and src | |
| * in %esi. | |
| */ | |
| .set dest_in, %eax | |
| .set dest, %edi | |
| .set src_in, %edx | |
| .set src, %esi | |
| .set n, %ecx | |
| .set tmp0, %edx | |
| .set tmp0w, %dx | |
| .set tmp1, %ebx | |
| .set tmp1w, %bx | |
| .set tmp2, %eax | |
| .set tmp3b, %cl | |
| /* | |
| * Save all callee-saved registers, because this function is going to clobber | |
| * all of them: | |
| */ | |
| pushl %ebp | |
| movl %esp, %ebp // set standard frame pointer | |
| pushl %ebx | |
| pushl %edi | |
| pushl %esi | |
| pushl %eax // save 'dest_in' parameter [eax] as the return value | |
| movl src_in, src | |
| movl dest_in, dest | |
| /* Handle more 16 bytes in loop */ | |
| cmpl $0x10, n | |
| jb .Lmove_16B | |
| /* Decide forward/backward copy mode */ | |
| cmpl dest, src | |
| jb .Lbackwards_header | |
| /* | |
| * movs instruction have many startup latency | |
| * so we handle small size by general register. | |
| */ | |
| cmpl $680, n | |
| jb .Ltoo_small_forwards | |
| /* movs instruction is only good for aligned case. */ | |
| movl src, tmp0 | |
| xorl dest, tmp0 | |
| andl $0xff, tmp0 | |
| jz .Lforward_movs | |
| .Ltoo_small_forwards: | |
| subl $0x10, n | |
| /* We gobble 16 bytes forward in each loop. */ | |
| .Lmove_16B_forwards_loop: | |
| subl $0x10, n | |
| movl 0*4(src), tmp0 | |
| movl 1*4(src), tmp1 | |
| movl tmp0, 0*4(dest) | |
| movl tmp1, 1*4(dest) | |
| movl 2*4(src), tmp0 | |
| movl 3*4(src), tmp1 | |
| movl tmp0, 2*4(dest) | |
| movl tmp1, 3*4(dest) | |
| leal 0x10(src), src | |
| leal 0x10(dest), dest | |
| jae .Lmove_16B_forwards_loop | |
| addl $0x10, n | |
| jmp .Lmove_16B | |
| /* Handle data forward by movs. */ | |
| .p2align 4 | |
| .Lforward_movs: | |
| movl -4(src, n), tmp0 | |
| leal -4(dest, n), tmp1 | |
| shrl $2, n | |
| rep movsl | |
| movl tmp0, (tmp1) | |
| jmp .Ldone | |
| /* Handle data backward by movs. */ | |
| .p2align 4 | |
| .Lbackwards_movs: | |
| movl (src), tmp0 | |
| movl dest, tmp1 | |
| leal -4(src, n), src | |
| leal -4(dest, n), dest | |
| shrl $2, n | |
| std | |
| rep movsl | |
| movl tmp0,(tmp1) | |
| cld | |
| jmp .Ldone | |
| /* Start to prepare for backward copy. */ | |
| .p2align 4 | |
| .Lbackwards_header: | |
| cmpl $680, n | |
| jb .Ltoo_small_backwards | |
| movl src, tmp0 | |
| xorl dest, tmp0 | |
| andl $0xff, tmp0 | |
| jz .Lbackwards_movs | |
| /* Calculate copy position to tail. */ | |
| .Ltoo_small_backwards: | |
| addl n, src | |
| addl n, dest | |
| subl $0x10, n | |
| /* We gobble 16 bytes backward in each loop. */ | |
| .Lmove_16B_backwards_loop: | |
| subl $0x10, n | |
| movl -1*4(src), tmp0 | |
| movl -2*4(src), tmp1 | |
| movl tmp0, -1*4(dest) | |
| movl tmp1, -2*4(dest) | |
| movl -3*4(src), tmp0 | |
| movl -4*4(src), tmp1 | |
| movl tmp0, -3*4(dest) | |
| movl tmp1, -4*4(dest) | |
| leal -0x10(src), src | |
| leal -0x10(dest), dest | |
| jae .Lmove_16B_backwards_loop | |
| /* Calculate copy position to head. */ | |
| addl $0x10, n | |
| subl n, src | |
| subl n, dest | |
| /* Move data from 8 bytes to 15 bytes. */ | |
| .p2align 4 | |
| .Lmove_16B: | |
| cmpl $8, n | |
| jb .Lmove_8B | |
| movl 0*4(src), tmp0 | |
| movl 1*4(src), tmp1 | |
| movl -2*4(src, n), tmp2 | |
| movl -1*4(src, n), src | |
| movl tmp0, 0*4(dest) | |
| movl tmp1, 1*4(dest) | |
| movl tmp2, -2*4(dest, n) | |
| movl src, -1*4(dest, n) | |
| jmp .Ldone | |
| /* Move data from 4 bytes to 7 bytes. */ | |
| .p2align 4 | |
| .Lmove_8B: | |
| cmpl $4, n | |
| jb .Lmove_4B | |
| movl 0*4(src), tmp0 | |
| movl -1*4(src, n), tmp1 | |
| movl tmp0, 0*4(dest) | |
| movl tmp1, -1*4(dest, n) | |
| jmp .Ldone | |
| /* Move data from 2 bytes to 3 bytes. */ | |
| .p2align 4 | |
| .Lmove_4B: | |
| cmpl $2, n | |
| jb .Lmove_1B | |
| movw 0*2(src), tmp0w | |
| movw -1*2(src, n), tmp1w | |
| movw tmp0w, 0*2(dest) | |
| movw tmp1w, -1*2(dest, n) | |
| jmp .Ldone | |
| /* Move data for 1 byte. */ | |
| .p2align 4 | |
| .Lmove_1B: | |
| cmpl $1, n | |
| jb .Ldone | |
| movb (src), tmp3b | |
| movb tmp3b, (dest) | |
| .p2align 4 | |
| .Ldone: | |
| popl dest_in // restore 'dest_in' [eax] as the return value | |
| /* Restore all callee-saved registers: */ | |
| popl %esi | |
| popl %edi | |
| popl %ebx | |
| popl %ebp | |
| RET | |
| SYM_FUNC_END(memmove) | |
| EXPORT_SYMBOL(memmove) |