| /* SPDX-License-Identifier: GPL-2.0-only */ | |
| /* | |
| * relocate_kernel.S - put the kernel image in place to boot | |
| * Copyright (C) 2002-2005 Eric Biederman <ebiederm@xmission.com> | |
| */ | |
| #include <linux/linkage.h> | |
| #include <linux/stringify.h> | |
| #include <asm/alternative.h> | |
| #include <asm/page_types.h> | |
| #include <asm/kexec.h> | |
| #include <asm/processor-flags.h> | |
| #include <asm/pgtable_types.h> | |
| #include <asm/nospec-branch.h> | |
| #include <asm/unwind_hints.h> | |
| #include <asm/asm-offsets.h> | |
| /* | |
| * Must be relocatable PIC code callable as a C function, in particular | |
| * there must be a plain RET and not jump to return thunk. | |
| */ | |
| #define PTR(x) (x << 3) | |
| #define PAGE_ATTR (_PAGE_PRESENT | _PAGE_RW | _PAGE_ACCESSED | _PAGE_DIRTY) | |
| /* | |
| * The .text..relocate_kernel and .data..relocate_kernel sections are copied | |
| * into the control page, and the remainder of the page is used as the stack. | |
| */ | |
| .section .data..relocate_kernel,"a"; | |
| /* Minimal CPU state */ | |
| SYM_DATA_LOCAL(saved_rsp, .quad 0) | |
| SYM_DATA_LOCAL(saved_cr0, .quad 0) | |
| SYM_DATA_LOCAL(saved_cr3, .quad 0) | |
| SYM_DATA_LOCAL(saved_cr4, .quad 0) | |
| /* other data */ | |
| SYM_DATA(kexec_va_control_page, .quad 0) | |
| SYM_DATA(kexec_pa_table_page, .quad 0) | |
| SYM_DATA(kexec_pa_swap_page, .quad 0) | |
| SYM_DATA_LOCAL(pa_backup_pages_map, .quad 0) | |
| SYM_DATA(kexec_debug_8250_mmio32, .quad 0) | |
| SYM_DATA(kexec_debug_8250_port, .word 0) | |
| .balign 16 | |
| SYM_DATA_START_LOCAL(kexec_debug_gdt) | |
| .word kexec_debug_gdt_end - kexec_debug_gdt - 1 | |
| .long 0 | |
| .word 0 | |
| .quad 0x00cf9a000000ffff /* __KERNEL32_CS */ | |
| .quad 0x00af9a000000ffff /* __KERNEL_CS */ | |
| .quad 0x00cf92000000ffff /* __KERNEL_DS */ | |
| SYM_DATA_END_LABEL(kexec_debug_gdt, SYM_L_LOCAL, kexec_debug_gdt_end) | |
| .balign 8 | |
| SYM_DATA_START(kexec_debug_idt) | |
| .skip 0x100, 0x00 | |
| SYM_DATA_END(kexec_debug_idt) | |
| .section .text..relocate_kernel,"ax"; | |
| .code64 | |
| SYM_CODE_START_NOALIGN(relocate_kernel) | |
| UNWIND_HINT_END_OF_STACK | |
| ANNOTATE_NOENDBR | |
| /* | |
| * %rdi indirection_page | |
| * %rsi pa_control_page | |
| * %rdx start address | |
| * %rcx flags: RELOC_KERNEL_* | |
| */ | |
| /* Save the CPU context, used for jumping back */ | |
| pushq %rbx | |
| pushq %rbp | |
| pushq %r12 | |
| pushq %r13 | |
| pushq %r14 | |
| pushq %r15 | |
| pushf | |
| /* Invalidate GDT/IDT, zero out flags */ | |
| pushq $0 | |
| pushq $0 | |
| lidt (%rsp) | |
| lgdt (%rsp) | |
| addq $8, %rsp | |
| popfq | |
| /* Switch to the identity mapped page tables */ | |
| movq %cr3, %rax | |
| movq kexec_pa_table_page(%rip), %r9 | |
| movq %r9, %cr3 | |
| /* Leave CR4 in %r13 to enable the right paging mode later. */ | |
| movq %cr4, %r13 | |
| /* | |
| * Disable global pages immediately to ensure this mapping is RWX. | |
| * Disable LASS before jumping to the identity mapped page. | |
| */ | |
| movq %r13, %r12 | |
| andq $~(X86_CR4_PGE | X86_CR4_LASS), %r12 | |
| movq %r12, %cr4 | |
| /* Save %rsp and CRs. */ | |
| movq %r13, saved_cr4(%rip) | |
| movq %rsp, saved_rsp(%rip) | |
| movq %rax, saved_cr3(%rip) | |
| movq %cr0, %rax | |
| movq %rax, saved_cr0(%rip) | |
| /* save indirection list for jumping back */ | |
| movq %rdi, pa_backup_pages_map(%rip) | |
| /* Save the flags to %r11 as swap_pages clobbers %rcx. */ | |
| movq %rcx, %r11 | |
| /* setup a new stack at the end of the physical control page */ | |
| lea PAGE_SIZE(%rsi), %rsp | |
| /* jump to identity mapped page */ | |
| 0: addq $identity_mapped - 0b, %rsi | |
| subq $__relocate_kernel_start - 0b, %rsi | |
| ANNOTATE_RETPOLINE_SAFE | |
| jmp *%rsi | |
| SYM_CODE_END(relocate_kernel) | |
| SYM_CODE_START_LOCAL_NOALIGN(identity_mapped) | |
| UNWIND_HINT_END_OF_STACK | |
| /* | |
| * %rdi indirection page | |
| * %rdx start address | |
| * %r9 page table page | |
| * %r11 flags: RELOC_KERNEL_* | |
| * %r13 original CR4 when relocate_kernel() was invoked | |
| */ | |
| /* | |
| * Set return address to 0 if not preserving context. The purgatory | |
| * shipped in kexec-tools will unconditionally look for the return | |
| * address on the stack and set a kexec_jump_back_entry= command | |
| * line option if it's non-zero. There's no other way that it can | |
| * tell a preserve-context (kjump) kexec from a normal one. | |
| */ | |
| pushq $0 | |
| /* store the start address on the stack */ | |
| pushq %rdx | |
| /* Create a GDTR (16 bits limit, 64 bits addr) on stack */ | |
| leaq kexec_debug_gdt(%rip), %rax | |
| pushq %rax | |
| pushw (%rax) | |
| /* Load the GDT, put the stack back */ | |
| lgdt (%rsp) | |
| addq $10, %rsp | |
| /* Test that we can load segments */ | |
| movq %ds, %rax | |
| movq %rax, %ds | |
| /* Now an IDTR on the stack to load the IDT the kernel created */ | |
| leaq kexec_debug_idt(%rip), %rsi | |
| pushq %rsi | |
| pushw $0xff | |
| lidt (%rsp) | |
| addq $10, %rsp | |
| //int3 | |
| /* | |
| * Clear X86_CR4_CET (if it was set) such that we can clear CR0_WP | |
| * below. | |
| */ | |
| movq %cr4, %rax | |
| andq $~(X86_CR4_CET), %rax | |
| movq %rax, %cr4 | |
| /* | |
| * Set cr0 to a known state: | |
| * - Paging enabled | |
| * - Alignment check disabled | |
| * - Write protect disabled | |
| * - No task switch | |
| * - Don't do FP software emulation. | |
| * - Protected mode enabled | |
| */ | |
| movq %cr0, %rax | |
| andq $~(X86_CR0_AM | X86_CR0_WP | X86_CR0_TS | X86_CR0_EM), %rax | |
| orl $(X86_CR0_PG | X86_CR0_PE), %eax | |
| movq %rax, %cr0 | |
| /* | |
| * Set cr4 to a known state: | |
| * - physical address extension enabled | |
| * - 5-level paging, if it was enabled before | |
| * - Machine check exception on TDX guest, if it was enabled before. | |
| * Clearing MCE might not be allowed in TDX guests, depending on setup. | |
| * | |
| * Use R13 that contains the original CR4 value, read in relocate_kernel(). | |
| * PAE is always set in the original CR4. | |
| */ | |
| andl $(X86_CR4_PAE | X86_CR4_LA57), %r13d | |
| ALTERNATIVE "", __stringify(orl $X86_CR4_MCE, %r13d), X86_FEATURE_TDX_GUEST | |
| movq %r13, %cr4 | |
| /* Flush the TLB (needed?) */ | |
| movq %r9, %cr3 | |
| /* | |
| * If the memory cache is in incoherent state, e.g., due to | |
| * memory encryption, do WBINVD to flush cache. | |
| * | |
| * If SME is active, there could be old encrypted cache line | |
| * entries that will conflict with the now unencrypted memory | |
| * used by kexec. Flush the caches before copying the kernel. | |
| * | |
| * Note SME sets this flag to true when the platform supports | |
| * SME, so the WBINVD is performed even SME is not activated | |
| * by the kernel. But this has no harm. | |
| */ | |
| testb $RELOC_KERNEL_CACHE_INCOHERENT, %r11b | |
| jz .Lnowbinvd | |
| wbinvd | |
| .Lnowbinvd: | |
| call swap_pages | |
| /* | |
| * To be certain of avoiding problems with self-modifying code | |
| * I need to execute a serializing instruction here. | |
| * So I flush the TLB by reloading %cr3 here, it's handy, | |
| * and not processor dependent. | |
| */ | |
| movq %cr3, %rax | |
| movq %rax, %cr3 | |
| testb $RELOC_KERNEL_PRESERVE_CONTEXT, %r11b | |
| jnz .Lrelocate | |
| /* | |
| * set all of the registers to known values | |
| * leave %rsp alone | |
| */ | |
| xorl %eax, %eax | |
| xorl %ebx, %ebx | |
| xorl %ecx, %ecx | |
| xorl %edx, %edx | |
| xorl %esi, %esi | |
| xorl %edi, %edi | |
| xorl %ebp, %ebp | |
| xorl %r8d, %r8d | |
| xorl %r9d, %r9d | |
| xorl %r10d, %r10d | |
| xorl %r11d, %r11d | |
| xorl %r12d, %r12d | |
| xorl %r13d, %r13d | |
| xorl %r14d, %r14d | |
| xorl %r15d, %r15d | |
| ANNOTATE_UNRET_SAFE | |
| ret | |
| int3 | |
| .Lrelocate: | |
| popq %rdx | |
| /* Use the swap page for the callee's stack */ | |
| movq kexec_pa_swap_page(%rip), %r10 | |
| leaq PAGE_SIZE(%r10), %rsp | |
| /* push the existing entry point onto the callee's stack */ | |
| pushq %rdx | |
| ANNOTATE_RETPOLINE_SAFE | |
| call *%rdx | |
| /* get the re-entry point of the peer system */ | |
| popq %rbp | |
| movq kexec_pa_swap_page(%rip), %r10 | |
| movq pa_backup_pages_map(%rip), %rdi | |
| movq kexec_pa_table_page(%rip), %rax | |
| movq %rax, %cr3 | |
| /* Find start (and end) of this physical mapping of control page */ | |
| leaq (%rip), %r8 | |
| ANNOTATE_NOENDBR | |
| andq $PAGE_MASK, %r8 | |
| lea PAGE_SIZE(%r8), %rsp | |
| /* | |
| * Ensure RELOC_KERNEL_PRESERVE_CONTEXT flag is set so that | |
| * swap_pages() can swap pages correctly. Note all other | |
| * RELOC_KERNEL_* flags passed to relocate_kernel() are not | |
| * restored. | |
| */ | |
| movl $RELOC_KERNEL_PRESERVE_CONTEXT, %r11d | |
| call swap_pages | |
| movq kexec_va_control_page(%rip), %rax | |
| 0: addq $virtual_mapped - 0b, %rax | |
| subq $__relocate_kernel_start - 0b, %rax | |
| pushq %rax | |
| ANNOTATE_UNRET_SAFE | |
| ret | |
| int3 | |
| SYM_CODE_END(identity_mapped) | |
| SYM_CODE_START_LOCAL_NOALIGN(virtual_mapped) | |
| UNWIND_HINT_END_OF_STACK | |
| ANNOTATE_NOENDBR // RET target, above | |
| movq saved_rsp(%rip), %rsp | |
| movq saved_cr4(%rip), %rax | |
| movq %rax, %cr4 | |
| movq saved_cr3(%rip), %rax | |
| movq saved_cr0(%rip), %r8 | |
| movq %rax, %cr3 | |
| movq %r8, %cr0 | |
| #ifdef CONFIG_KEXEC_JUMP | |
| /* Saved in save_processor_state. */ | |
| movq $saved_context, %rax | |
| lgdt saved_context_gdt_desc(%rax) | |
| #endif | |
| /* relocate_kernel() returns the re-entry point for next time */ | |
| movq %rbp, %rax | |
| popf | |
| popq %r15 | |
| popq %r14 | |
| popq %r13 | |
| popq %r12 | |
| popq %rbp | |
| popq %rbx | |
| ANNOTATE_UNRET_SAFE | |
| ret | |
| int3 | |
| SYM_CODE_END(virtual_mapped) | |
| /* Do the copies */ | |
| SYM_CODE_START_LOCAL_NOALIGN(swap_pages) | |
| UNWIND_HINT_END_OF_STACK | |
| /* | |
| * %rdi indirection page | |
| * %r11 flags: RELOC_KERNEL_* | |
| */ | |
| movq %rdi, %rcx /* Put the indirection_page in %rcx */ | |
| xorl %edi, %edi | |
| xorl %esi, %esi | |
| jmp .Lstart /* Should start with an indirection record */ | |
| .Lloop: /* top, read another word for the indirection page */ | |
| movq (%rbx), %rcx | |
| addq $8, %rbx | |
| .Lstart: | |
| testb $0x1, %cl /* is it a destination page? */ | |
| jz .Lnotdest | |
| movq %rcx, %rdi | |
| andq $0xfffffffffffff000, %rdi | |
| jmp .Lloop | |
| .Lnotdest: | |
| testb $0x2, %cl /* is it an indirection page? */ | |
| jz .Lnotind | |
| movq %rcx, %rbx | |
| andq $0xfffffffffffff000, %rbx | |
| jmp .Lloop | |
| .Lnotind: | |
| testb $0x4, %cl /* is it the done indicator? */ | |
| jz .Lnotdone | |
| jmp .Ldone | |
| .Lnotdone: | |
| testb $0x8, %cl /* is it the source indicator? */ | |
| jz .Lloop /* Ignore it otherwise */ | |
| movq %rcx, %rsi /* For ever source page do a copy */ | |
| andq $0xfffffffffffff000, %rsi | |
| movq %rdi, %rdx /* Save destination page to %rdx */ | |
| movq %rsi, %rax /* Save source page to %rax */ | |
| /* Only actually swap for ::preserve_context */ | |
| testb $RELOC_KERNEL_PRESERVE_CONTEXT, %r11b | |
| jz .Lnoswap | |
| /* copy source page to swap page */ | |
| movq kexec_pa_swap_page(%rip), %rdi | |
| movl $512, %ecx | |
| rep movsq | |
| /* copy destination page to source page */ | |
| movq %rax, %rdi | |
| movq %rdx, %rsi | |
| movl $512, %ecx | |
| rep movsq | |
| /* copy swap page to destination page */ | |
| movq %rdx, %rdi | |
| movq kexec_pa_swap_page(%rip), %rsi | |
| .Lnoswap: | |
| movl $512, %ecx | |
| rep movsq | |
| lea PAGE_SIZE(%rax), %rsi | |
| jmp .Lloop | |
| .Ldone: | |
| ANNOTATE_UNRET_SAFE | |
| ret | |
| int3 | |
| SYM_CODE_END(swap_pages) | |
| /* | |
| * Generic 'print character' routine | |
| * - %al: Character to be printed (may clobber %rax) | |
| * - %rdx: MMIO address or port. | |
| */ | |
| #define XMTRDY 0x20 | |
| #define TXR 0 /* Transmit register (WRITE) */ | |
| #define LSR 5 /* Line Status */ | |
| SYM_CODE_START_LOCAL_NOALIGN(pr_char_8250) | |
| UNWIND_HINT_FUNC | |
| ANNOTATE_NOENDBR | |
| addw $LSR, %dx | |
| xchg %al, %ah | |
| .Lxmtrdy_loop: | |
| inb %dx, %al | |
| testb $XMTRDY, %al | |
| jnz .Lready | |
| pause | |
| jmp .Lxmtrdy_loop | |
| .Lready: | |
| subw $LSR, %dx | |
| xchg %al, %ah | |
| outb %al, %dx | |
| pr_char_null: | |
| ANNOTATE_NOENDBR | |
| ANNOTATE_UNRET_SAFE | |
| ret | |
| SYM_CODE_END(pr_char_8250) | |
| SYM_CODE_START_LOCAL_NOALIGN(pr_char_8250_mmio32) | |
| UNWIND_HINT_FUNC | |
| ANNOTATE_NOENDBR | |
| .Lxmtrdy_loop_mmio: | |
| movb (LSR*4)(%rdx), %ah | |
| testb $XMTRDY, %ah | |
| jnz .Lready_mmio | |
| pause | |
| jmp .Lxmtrdy_loop_mmio | |
| .Lready_mmio: | |
| movb %al, (%rdx) | |
| ANNOTATE_UNRET_SAFE | |
| ret | |
| SYM_CODE_END(pr_char_8250_mmio32) | |
| /* | |
| * Load pr_char function pointer into %rsi and load %rdx with whatever | |
| * that function wants to see there (typically port/MMIO address). | |
| */ | |
| .macro pr_setup | |
| leaq pr_char_8250(%rip), %rsi | |
| movw kexec_debug_8250_port(%rip), %dx | |
| testw %dx, %dx | |
| jnz 1f | |
| leaq pr_char_8250_mmio32(%rip), %rsi | |
| movq kexec_debug_8250_mmio32(%rip), %rdx | |
| testq %rdx, %rdx | |
| jnz 1f | |
| leaq pr_char_null(%rip), %rsi | |
| 1: | |
| .endm | |
| /* Print the nybble in %bl, clobber %rax */ | |
| SYM_CODE_START_LOCAL_NOALIGN(pr_nybble) | |
| UNWIND_HINT_FUNC | |
| movb %bl, %al | |
| nop | |
| andb $0x0f, %al | |
| addb $0x30, %al | |
| cmpb $0x3a, %al | |
| jb 1f | |
| addb $('a' - '0' - 10), %al | |
| ANNOTATE_RETPOLINE_SAFE | |
| 1: jmp *%rsi | |
| SYM_CODE_END(pr_nybble) | |
| SYM_CODE_START_LOCAL_NOALIGN(pr_qword) | |
| UNWIND_HINT_FUNC | |
| movq $16, %rcx | |
| 1: rolq $4, %rbx | |
| call pr_nybble | |
| loop 1b | |
| movb $'\n', %al | |
| ANNOTATE_RETPOLINE_SAFE | |
| jmp *%rsi | |
| SYM_CODE_END(pr_qword) | |
| .macro print_reg a, b, c, d, r | |
| movb $\a, %al | |
| ANNOTATE_RETPOLINE_SAFE | |
| call *%rsi | |
| movb $\b, %al | |
| ANNOTATE_RETPOLINE_SAFE | |
| call *%rsi | |
| movb $\c, %al | |
| ANNOTATE_RETPOLINE_SAFE | |
| call *%rsi | |
| movb $\d, %al | |
| ANNOTATE_RETPOLINE_SAFE | |
| call *%rsi | |
| movq \r, %rbx | |
| call pr_qword | |
| .endm | |
| SYM_CODE_START_NOALIGN(kexec_debug_exc_vectors) | |
| /* Each of these is 6 bytes. */ | |
| .macro vec_err exc | |
| UNWIND_HINT_ENTRY | |
| . = kexec_debug_exc_vectors + (\exc * KEXEC_DEBUG_EXC_HANDLER_SIZE) | |
| nop | |
| nop | |
| pushq $\exc | |
| jmp exc_handler | |
| .endm | |
| .macro vec_noerr exc | |
| UNWIND_HINT_ENTRY | |
| . = kexec_debug_exc_vectors + (\exc * KEXEC_DEBUG_EXC_HANDLER_SIZE) | |
| pushq $0 | |
| pushq $\exc | |
| jmp exc_handler | |
| .endm | |
| ANNOTATE_NOENDBR | |
| vec_noerr 0 // #DE | |
| vec_noerr 1 // #DB | |
| vec_noerr 2 // #NMI | |
| vec_noerr 3 // #BP | |
| vec_noerr 4 // #OF | |
| vec_noerr 5 // #BR | |
| vec_noerr 6 // #UD | |
| vec_noerr 7 // #NM | |
| vec_err 8 // #DF | |
| vec_noerr 9 | |
| vec_err 10 // #TS | |
| vec_err 11 // #NP | |
| vec_err 12 // #SS | |
| vec_err 13 // #GP | |
| vec_err 14 // #PF | |
| vec_noerr 15 | |
| SYM_CODE_END(kexec_debug_exc_vectors) | |
| SYM_CODE_START_LOCAL_NOALIGN(exc_handler) | |
| /* No need for RET mitigations during kexec */ | |
| VALIDATE_UNRET_END | |
| pushq %rax | |
| pushq %rbx | |
| pushq %rcx | |
| pushq %rdx | |
| pushq %rsi | |
| /* Stack frame */ | |
| #define EXC_SS 0x58 /* Architectural... */ | |
| #define EXC_RSP 0x50 | |
| #define EXC_EFLAGS 0x48 | |
| #define EXC_CS 0x40 | |
| #define EXC_RIP 0x38 | |
| #define EXC_ERRORCODE 0x30 /* Either architectural or zero pushed by handler */ | |
| #define EXC_EXCEPTION 0x28 /* Pushed by handler entry point */ | |
| #define EXC_RAX 0x20 /* Pushed just above in exc_handler */ | |
| #define EXC_RBX 0x18 | |
| #define EXC_RCX 0x10 | |
| #define EXC_RDX 0x08 | |
| #define EXC_RSI 0x00 | |
| /* Set up %rdx/%rsi for debug output */ | |
| pr_setup | |
| /* rip and exception info */ | |
| print_reg 'E', 'x', 'c', ':', EXC_EXCEPTION(%rsp) | |
| print_reg 'E', 'r', 'r', ':', EXC_ERRORCODE(%rsp) | |
| print_reg 'r', 'i', 'p', ':', EXC_RIP(%rsp) | |
| print_reg 'r', 's', 'p', ':', EXC_RSP(%rsp) | |
| /* We spilled these to the stack */ | |
| print_reg 'r', 'a', 'x', ':', EXC_RAX(%rsp) | |
| print_reg 'r', 'b', 'x', ':', EXC_RBX(%rsp) | |
| print_reg 'r', 'c', 'x', ':', EXC_RCX(%rsp) | |
| print_reg 'r', 'd', 'x', ':', EXC_RDX(%rsp) | |
| print_reg 'r', 's', 'i', ':', EXC_RSI(%rsp) | |
| /* Other registers untouched */ | |
| print_reg 'r', 'd', 'i', ':', %rdi | |
| print_reg 'r', '8', ' ', ':', %r8 | |
| print_reg 'r', '9', ' ', ':', %r9 | |
| print_reg 'r', '1', '0', ':', %r10 | |
| print_reg 'r', '1', '1', ':', %r11 | |
| print_reg 'r', '1', '2', ':', %r12 | |
| print_reg 'r', '1', '3', ':', %r13 | |
| print_reg 'r', '1', '4', ':', %r14 | |
| print_reg 'r', '1', '5', ':', %r15 | |
| print_reg 'c', 'r', '2', ':', %cr2 | |
| /* Only return from INT3 */ | |
| cmpq $3, EXC_EXCEPTION(%rsp) | |
| jne .Ldie | |
| popq %rsi | |
| popq %rdx | |
| popq %rcx | |
| popq %rbx | |
| popq %rax | |
| addq $16, %rsp | |
| iretq | |
| .Ldie: | |
| hlt | |
| jmp .Ldie | |
| SYM_CODE_END(exc_handler) |