| /* SPDX-License-Identifier: GPL-2.0-only */ | |
| /* | |
| * Copyright (C) 2020 - Google Inc | |
| * Author: Andrew Scull <ascull@google.com> | |
| */ | |
| #include <linux/linkage.h> | |
| #include <asm/assembler.h> | |
| #include <asm/kvm_arm.h> | |
| #include <asm/kvm_asm.h> | |
| #include <asm/kvm_mmu.h> | |
| .text | |
| SYM_FUNC_START(__host_exit) | |
| get_host_ctxt x0, x1 | |
| /* Store the host regs x2 and x3 */ | |
| stp x2, x3, [x0, #CPU_XREG_OFFSET(2)] | |
| /* Retrieve the host regs x0-x1 from the stack */ | |
| ldp x2, x3, [sp], #16 // x0, x1 | |
| /* Store the host regs x0-x1 and x4-x17 */ | |
| stp x2, x3, [x0, #CPU_XREG_OFFSET(0)] | |
| stp x4, x5, [x0, #CPU_XREG_OFFSET(4)] | |
| stp x6, x7, [x0, #CPU_XREG_OFFSET(6)] | |
| stp x8, x9, [x0, #CPU_XREG_OFFSET(8)] | |
| stp x10, x11, [x0, #CPU_XREG_OFFSET(10)] | |
| stp x12, x13, [x0, #CPU_XREG_OFFSET(12)] | |
| stp x14, x15, [x0, #CPU_XREG_OFFSET(14)] | |
| stp x16, x17, [x0, #CPU_XREG_OFFSET(16)] | |
| /* Store the host regs x18-x29, lr */ | |
| save_callee_saved_regs x0 | |
| /* Save the host context pointer in x29 across the function call */ | |
| mov x29, x0 | |
| bl handle_trap | |
| /* Restore host regs x0-x17 */ | |
| __host_enter_restore_full: | |
| ldp x0, x1, [x29, #CPU_XREG_OFFSET(0)] | |
| ldp x2, x3, [x29, #CPU_XREG_OFFSET(2)] | |
| ldp x4, x5, [x29, #CPU_XREG_OFFSET(4)] | |
| ldp x6, x7, [x29, #CPU_XREG_OFFSET(6)] | |
| /* x0-7 are use for panic arguments */ | |
| __host_enter_for_panic: | |
| ldp x8, x9, [x29, #CPU_XREG_OFFSET(8)] | |
| ldp x10, x11, [x29, #CPU_XREG_OFFSET(10)] | |
| ldp x12, x13, [x29, #CPU_XREG_OFFSET(12)] | |
| ldp x14, x15, [x29, #CPU_XREG_OFFSET(14)] | |
| ldp x16, x17, [x29, #CPU_XREG_OFFSET(16)] | |
| /* Restore host regs x18-x29, lr */ | |
| restore_callee_saved_regs x29 | |
| /* Do not touch any register after this! */ | |
| __host_enter_without_restoring: | |
| eret | |
| sb | |
| SYM_FUNC_END(__host_exit) | |
| /* | |
| * void __noreturn __host_enter(struct kvm_cpu_context *host_ctxt); | |
| */ | |
| SYM_FUNC_START(__host_enter) | |
| mov x29, x0 | |
| b __host_enter_restore_full | |
| SYM_FUNC_END(__host_enter) | |
| /* | |
| * void __noreturn __hyp_do_panic(struct kvm_cpu_context *host_ctxt, u64 spsr, | |
| * u64 elr, u64 par); | |
| */ | |
| SYM_FUNC_START(__hyp_do_panic) | |
| /* Prepare and exit to the host's panic funciton. */ | |
| mov lr, #(PSR_F_BIT | PSR_I_BIT | PSR_A_BIT | PSR_D_BIT |\ | |
| PSR_MODE_EL1h) | |
| msr spsr_el2, lr | |
| adr_l lr, nvhe_hyp_panic_handler | |
| hyp_kimg_va lr, x6 | |
| msr elr_el2, lr | |
| mov x29, x0 | |
| #ifdef CONFIG_NVHE_EL2_DEBUG | |
| /* Ensure host stage-2 is disabled */ | |
| mrs x0, hcr_el2 | |
| bic x0, x0, #HCR_VM | |
| msr hcr_el2, x0 | |
| isb | |
| tlbi vmalls12e1 | |
| dsb nsh | |
| #endif | |
| /* Load the panic arguments into x0-7 */ | |
| mrs x0, esr_el2 | |
| mov x4, x3 | |
| mov x3, x2 | |
| hyp_pa x3, x6 | |
| get_vcpu_ptr x5, x6 | |
| mrs x6, far_el2 | |
| mrs x7, hpfar_el2 | |
| /* Enter the host, conditionally restoring the host context. */ | |
| cbz x29, __host_enter_without_restoring | |
| b __host_enter_for_panic | |
| SYM_FUNC_END(__hyp_do_panic) | |
| SYM_FUNC_START(__host_hvc) | |
| ldp x0, x1, [sp] // Don't fixup the stack yet | |
| /* No stub for you, sonny Jim */ | |
| alternative_if ARM64_KVM_PROTECTED_MODE | |
| b __host_exit | |
| alternative_else_nop_endif | |
| /* Check for a stub HVC call */ | |
| cmp x0, #HVC_STUB_HCALL_NR | |
| b.hs __host_exit | |
| add sp, sp, #16 | |
| /* | |
| * Compute the idmap address of __kvm_handle_stub_hvc and | |
| * jump there. | |
| * | |
| * Preserve x0-x4, which may contain stub parameters. | |
| */ | |
| adr_l x5, __kvm_handle_stub_hvc | |
| hyp_pa x5, x6 | |
| br x5 | |
| SYM_FUNC_END(__host_hvc) | |
| .macro host_el1_sync_vect | |
| .align 7 | |
| .L__vect_start\@: | |
| stp x0, x1, [sp, #-16]! | |
| mrs x0, esr_el2 | |
| ubfx x0, x0, #ESR_ELx_EC_SHIFT, #ESR_ELx_EC_WIDTH | |
| cmp x0, #ESR_ELx_EC_HVC64 | |
| b.eq __host_hvc | |
| b __host_exit | |
| .L__vect_end\@: | |
| .if ((.L__vect_end\@ - .L__vect_start\@) > 0x80) | |
| .error "host_el1_sync_vect larger than vector entry" | |
| .endif | |
| .endm | |
| .macro invalid_host_el2_vect | |
| .align 7 | |
| /* | |
| * Test whether the SP has overflowed, without corrupting a GPR. | |
| * nVHE hypervisor stacks are aligned so that the PAGE_SHIFT bit | |
| * of SP should always be 1. | |
| */ | |
| add sp, sp, x0 // sp' = sp + x0 | |
| sub x0, sp, x0 // x0' = sp' - x0 = (sp + x0) - x0 = sp | |
| tbz x0, #PAGE_SHIFT, .L__hyp_sp_overflow\@ | |
| sub x0, sp, x0 // x0'' = sp' - x0' = (sp + x0) - sp = x0 | |
| sub sp, sp, x0 // sp'' = sp' - x0 = (sp + x0) - x0 = sp | |
| /* If a guest is loaded, panic out of it. */ | |
| stp x0, x1, [sp, #-16]! | |
| get_loaded_vcpu x0, x1 | |
| cbnz x0, __guest_exit_panic | |
| add sp, sp, #16 | |
| /* | |
| * The panic may not be clean if the exception is taken before the host | |
| * context has been saved by __host_exit or after the hyp context has | |
| * been partially clobbered by __host_enter. | |
| */ | |
| b hyp_panic | |
| .L__hyp_sp_overflow\@: | |
| /* Switch to the overflow stack */ | |
| adr_this_cpu sp, overflow_stack + OVERFLOW_STACK_SIZE, x0 | |
| b hyp_panic_bad_stack | |
| ASM_BUG() | |
| .endm | |
| .macro invalid_host_el1_vect | |
| .align 7 | |
| mov x0, xzr /* restore_host = false */ | |
| mrs x1, spsr_el2 | |
| mrs x2, elr_el2 | |
| mrs x3, par_el1 | |
| b __hyp_do_panic | |
| .endm | |
| /* | |
| * The host vector does not use an ESB instruction in order to avoid consuming | |
| * SErrors that should only be consumed by the host. Guest entry is deferred by | |
| * __guest_enter if there are any pending asynchronous exceptions so hyp will | |
| * always return to the host without having consumerd host SErrors. | |
| * | |
| * CONFIG_KVM_INDIRECT_VECTORS is not applied to the host vectors because the | |
| * host knows about the EL2 vectors already, and there is no point in hiding | |
| * them. | |
| */ | |
| .align 11 | |
| SYM_CODE_START(__kvm_hyp_host_vector) | |
| invalid_host_el2_vect // Synchronous EL2t | |
| invalid_host_el2_vect // IRQ EL2t | |
| invalid_host_el2_vect // FIQ EL2t | |
| invalid_host_el2_vect // Error EL2t | |
| invalid_host_el2_vect // Synchronous EL2h | |
| invalid_host_el2_vect // IRQ EL2h | |
| invalid_host_el2_vect // FIQ EL2h | |
| invalid_host_el2_vect // Error EL2h | |
| host_el1_sync_vect // Synchronous 64-bit EL1/EL0 | |
| invalid_host_el1_vect // IRQ 64-bit EL1/EL0 | |
| invalid_host_el1_vect // FIQ 64-bit EL1/EL0 | |
| invalid_host_el1_vect // Error 64-bit EL1/EL0 | |
| host_el1_sync_vect // Synchronous 32-bit EL1/EL0 | |
| invalid_host_el1_vect // IRQ 32-bit EL1/EL0 | |
| invalid_host_el1_vect // FIQ 32-bit EL1/EL0 | |
| invalid_host_el1_vect // Error 32-bit EL1/EL0 | |
| SYM_CODE_END(__kvm_hyp_host_vector) | |
| /* | |
| * Forward SMC with arguments in struct kvm_cpu_context, and | |
| * store the result into the same struct. Assumes SMCCC 1.2 or older. | |
| * | |
| * x0: struct kvm_cpu_context* | |
| */ | |
| SYM_CODE_START(__kvm_hyp_host_forward_smc) | |
| /* | |
| * Use x18 to keep the pointer to the host context because | |
| * x18 is callee-saved in SMCCC but not in AAPCS64. | |
| */ | |
| mov x18, x0 | |
| ldp x0, x1, [x18, #CPU_XREG_OFFSET(0)] | |
| ldp x2, x3, [x18, #CPU_XREG_OFFSET(2)] | |
| ldp x4, x5, [x18, #CPU_XREG_OFFSET(4)] | |
| ldp x6, x7, [x18, #CPU_XREG_OFFSET(6)] | |
| ldp x8, x9, [x18, #CPU_XREG_OFFSET(8)] | |
| ldp x10, x11, [x18, #CPU_XREG_OFFSET(10)] | |
| ldp x12, x13, [x18, #CPU_XREG_OFFSET(12)] | |
| ldp x14, x15, [x18, #CPU_XREG_OFFSET(14)] | |
| ldp x16, x17, [x18, #CPU_XREG_OFFSET(16)] | |
| smc #0 | |
| stp x0, x1, [x18, #CPU_XREG_OFFSET(0)] | |
| stp x2, x3, [x18, #CPU_XREG_OFFSET(2)] | |
| stp x4, x5, [x18, #CPU_XREG_OFFSET(4)] | |
| stp x6, x7, [x18, #CPU_XREG_OFFSET(6)] | |
| stp x8, x9, [x18, #CPU_XREG_OFFSET(8)] | |
| stp x10, x11, [x18, #CPU_XREG_OFFSET(10)] | |
| stp x12, x13, [x18, #CPU_XREG_OFFSET(12)] | |
| stp x14, x15, [x18, #CPU_XREG_OFFSET(14)] | |
| stp x16, x17, [x18, #CPU_XREG_OFFSET(16)] | |
| ret | |
| SYM_CODE_END(__kvm_hyp_host_forward_smc) |