2 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
3 * Author: Christoffer Dall <c.dall@virtualopensystems.com>
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License, version 2, as
7 * published by the Free Software Foundation.
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
19 #include <linux/linkage.h>
20 #include <linux/const.h>
21 #include <asm/unified.h>
23 #include <asm/ptrace.h>
24 #include <asm/asm-offsets.h>
25 #include <asm/kvm_asm.h>
26 #include <asm/kvm_arm.h>
27 #include <asm/vfpmacros.h>
28 #include "interrupts_head.S"
33 .globl __kvm_hyp_code_start
35 /********************************************************************
38 * void __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa);
40 * We rely on the hardware to broadcast the TLB invalidation to all CPUs
41 * inside the inner-shareable domain (which is the case for all v7
42 * implementations). If we come across a non-IS SMP implementation, we'll
43 * have to use an IPI based mechanism. Until then, we stick to the simple
44 * hardware assisted version.
46 * As v7 does not support flushing per IPA, just nuke the whole TLB
47 * instead, ignoring the ipa value.
49 ENTRY(__kvm_tlb_flush_vmid_ipa)
53 add r0, r0, #KVM_VTTBR
55 mcrr p15, 6, rr_lo_hi(r2, r3), c2 @ Write VTTBR
57 mcr p15, 0, r0, c8, c3, 0 @ TLBIALLIS (rt ignored)
62 mcrr p15, 6, r2, r3, c2 @ Back to VMID #0
63 isb @ Not necessary if followed by eret
67 ENDPROC(__kvm_tlb_flush_vmid_ipa)
70 * void __kvm_tlb_flush_vmid(struct kvm *kvm) - Flush per-VMID TLBs
72 * Reuses __kvm_tlb_flush_vmid_ipa() for ARMv7, without passing address
76 ENTRY(__kvm_tlb_flush_vmid)
77 b __kvm_tlb_flush_vmid_ipa
78 ENDPROC(__kvm_tlb_flush_vmid)
80 /********************************************************************
81 * Flush TLBs and instruction caches of all CPUs inside the inner-shareable
82 * domain, for all VMIDs
84 * void __kvm_flush_vm_context(void);
86 ENTRY(__kvm_flush_vm_context)
87 mov r0, #0 @ rn parameter for c15 flushes is SBZ
89 /* Invalidate NS Non-Hyp TLB Inner Shareable (TLBIALLNSNHIS) */
90 mcr p15, 4, r0, c8, c3, 4
91 /* Invalidate instruction caches Inner Shareable (ICIALLUIS) */
92 mcr p15, 0, r0, c7, c1, 0
94 isb @ Not necessary if followed by eret
97 ENDPROC(__kvm_flush_vm_context)
100 /********************************************************************
101 * Hypervisor world-switch code
104 * int __kvm_vcpu_run(struct kvm_vcpu *vcpu)
106 ENTRY(__kvm_vcpu_run)
107 @ Save the vcpu pointer
108 mcr p15, 4, vcpu, c13, c0, 2 @ HTPIDR
115 @ Store hardware CP15 state and load guest state
116 read_cp15_state store_to_vcpu = 0
117 write_cp15_state read_from_vcpu = 1
119 @ If the host kernel has not been configured with VFPv3 support,
120 @ then it is safer if we deny guests from using it as well.
122 @ Set FPEXC_EN so the guest doesn't trap floating point instructions
123 VFPFMRX r2, FPEXC @ VMRS
125 orr r2, r2, #FPEXC_EN
126 VFPFMXR FPEXC, r2 @ VMSR
130 configure_hyp_role vmentry
132 @ Trap coprocessor CRx accesses
134 set_hcptr vmentry, (HCPTR_TTA | HCPTR_TCP(10) | HCPTR_TCP(11))
137 @ Write configured ID register into MIDR alias
138 ldr r1, [vcpu, #VCPU_MIDR]
139 mcr p15, 4, r1, c0, c0, 0
141 @ Write guest view of MPIDR into VMPIDR
142 ldr r1, [vcpu, #CP15_OFFSET(c0_MPIDR)]
143 mcr p15, 4, r1, c0, c0, 5
145 @ Set up guest memory translation
146 ldr r1, [vcpu, #VCPU_KVM]
147 add r1, r1, #KVM_VTTBR
149 mcrr p15, 6, rr_lo_hi(r2, r3), c2 @ Write VTTBR
151 @ We're all done, just restore the GPRs and go to the guest
153 clrex @ Clear exclusive monitor
159 * guest r0, r1, r2 saved on the stack
168 mcrr p15, 6, r2, r3, c2 @ Write VTTBR
170 @ Don't trap coprocessor accesses for host kernel
173 set_hcptr vmexit, (HCPTR_TTA | HCPTR_TCP(10) | HCPTR_TCP(11))
176 @ Save floating point registers we if let guest use them.
177 tst r2, #(HCPTR_TCP(10) | HCPTR_TCP(11))
178 bne after_vfp_restore
180 @ Switch VFP/NEON hardware state to the host's
181 add r7, vcpu, #VCPU_VFP_GUEST
183 add r7, vcpu, #VCPU_VFP_HOST
188 @ Restore FPEXC_EN which we clobbered on entry
194 configure_hyp_role vmexit
196 @ Let host read hardware MIDR
197 mrc p15, 0, r2, c0, c0, 0
198 mcr p15, 4, r2, c0, c0, 0
200 @ Back to hardware MPIDR
201 mrc p15, 0, r2, c0, c0, 5
202 mcr p15, 4, r2, c0, c0, 5
204 @ Store guest CP15 state and restore host state
205 read_cp15_state store_to_vcpu = 1
206 write_cp15_state read_from_vcpu = 0
212 clrex @ Clear exclusive monitor
213 #ifndef CONFIG_CPU_ENDIAN_BE8
214 mov r0, r1 @ Return the return code
215 mov r1, #0 @ Clear upper bits in return value
217 @ r1 already has return code
218 mov r0, #0 @ Clear upper bits in return value
219 #endif /* CONFIG_CPU_ENDIAN_BE8 */
220 bx lr @ return to IOCTL
222 /********************************************************************
223 * Call function in Hyp mode
226 * u64 kvm_call_hyp(void *hypfn, ...);
228 * This is not really a variadic function in the classic C-way and care must
229 * be taken when calling this to ensure parameters are passed in registers
230 * only, since the stack will change between the caller and the callee.
232 * Call the function with the first argument containing a pointer to the
233 * function you wish to call in Hyp mode, and subsequent arguments will be
234 * passed as r0, r1, and r2 (a maximum of 3 arguments in addition to the
235 * function pointer can be passed). The function being called must be mapped
236 * in Hyp mode (see init_hyp_mode in arch/arm/kvm/arm.c). Return values are
237 * passed in r0 and r1.
239 * A function pointer with a value of 0xffffffff has a special meaning,
240 * and is used to implement __hyp_get_vectors in the same way as in
241 * arch/arm/kernel/hyp_stub.S.
243 * The calling convention follows the standard AAPCS:
244 * r0 - r3: caller save
252 /********************************************************************
253 * Hypervisor exception vector and handlers
256 * The KVM/ARM Hypervisor ABI is defined as follows:
258 * Entry to Hyp mode from the host kernel will happen _only_ when an HVC
259 * instruction is issued since all traps are disabled when running the host
260 * kernel as per the Hyp-mode initialization at boot time.
262 * HVC instructions cause a trap to the vector page + offset 0x14 (see hyp_hvc
263 * below) when the HVC instruction is called from SVC mode (i.e. a guest or the
264 * host kernel) and they cause a trap to the vector page + offset 0x8 when HVC
265 * instructions are called from within Hyp-mode.
267 * Hyp-ABI: Calling HYP-mode functions from host (in SVC mode):
268 * Switching to Hyp mode is done through a simple HVC #0 instruction. The
269 * exception vector code will check that the HVC comes from VMID==0 and if
270 * so will push the necessary state (SPSR, lr_usr) on the Hyp stack.
271 * - r0 contains a pointer to a HYP function
272 * - r1, r2, and r3 contain arguments to the above function.
273 * - The HYP function will be called with its arguments in r0, r1 and r2.
274 * On HYP function return, we return directly to SVC.
276 * Note that the above is used to execute code in Hyp-mode from a host-kernel
277 * point of view, and is a different concept from performing a world-switch and
278 * executing guest code SVC mode (with a VMID != 0).
281 /* Handle undef, svc, pabt, or dabt by crashing with a user notice */
282 .macro bad_exception exception_code, panic_str
284 mrrc p15, 6, r0, r1, c2 @ Read VTTBR
289 load_vcpu @ Load VCPU pointer
290 .if \exception_code == ARM_EXCEPTION_DATA_ABORT
291 mrc p15, 4, r2, c5, c2, 0 @ HSR
292 mrc p15, 4, r1, c6, c0, 0 @ HDFAR
293 str r2, [vcpu, #VCPU_HSR]
294 str r1, [vcpu, #VCPU_HxFAR]
296 .if \exception_code == ARM_EXCEPTION_PREF_ABORT
297 mrc p15, 4, r2, c5, c2, 0 @ HSR
298 mrc p15, 4, r1, c6, c0, 2 @ HIFAR
299 str r2, [vcpu, #VCPU_HSR]
300 str r1, [vcpu, #VCPU_HxFAR]
302 mov r1, #\exception_code
305 @ We were in the host already. Let's craft a panic-ing return to SVC.
307 bic r2, r2, #MODE_MASK
308 orr r2, r2, #SVC_MODE
309 THUMB( orr r2, r2, #PSR_T_BIT )
315 clrex @ Clear exclusive monitor
323 .globl __kvm_hyp_vector
325 @ Hyp-mode exception vector
341 bad_exception ARM_EXCEPTION_UNDEFINED, und_die_str
345 bad_exception ARM_EXCEPTION_HVC, svc_die_str
349 bad_exception ARM_EXCEPTION_PREF_ABORT, pabt_die_str
353 bad_exception ARM_EXCEPTION_DATA_ABORT, dabt_die_str
358 * Getting here is either becuase of a trap from a guest or from calling
359 * HVC from the host kernel, which means "switch to Hyp mode".
363 @ Check syndrome register
364 mrc p15, 4, r1, c5, c2, 0 @ HSR
365 lsr r0, r1, #HSR_EC_SHIFT
367 cmp r0, #HSR_EC_CP_0_13
368 beq switch_to_guest_vfp
371 bne guest_trap @ Not HVC instr.
374 * Let's check if the HVC came from VMID 0 and allow simple
377 mrrc p15, 6, r0, r2, c2
381 bne guest_trap @ Guest called HVC
386 /* Check for __hyp_get_vectors */
388 mrceq p15, 4, r0, c12, c0, 0 @ get HVBAR
401 blx lr @ Call the HYP function
409 load_vcpu @ Load VCPU pointer to r0
410 str r1, [vcpu, #VCPU_HSR]
412 @ Check if we need the fault information
413 lsr r1, r1, #HSR_EC_SHIFT
415 mrceq p15, 4, r2, c6, c0, 2 @ HIFAR
419 mrc p15, 4, r2, c6, c0, 0 @ HDFAR
421 2: str r2, [vcpu, #VCPU_HxFAR]
424 * B3.13.5 Reporting exceptions taken to the Non-secure PL2 mode:
426 * Abort on the stage 2 translation for a memory access from a
427 * Non-secure PL1 or PL0 mode:
429 * For any Access flag fault or Translation fault, and also for any
430 * Permission fault on the stage 2 translation of a memory access
431 * made as part of a translation table walk for a stage 1 translation,
432 * the HPFAR holds the IPA that caused the fault. Otherwise, the HPFAR
436 /* Check for permission fault, and S1PTW */
437 mrc p15, 4, r1, c5, c2, 0 @ HSR
438 and r0, r1, #HSR_FSC_TYPE
440 tsteq r1, #(1 << 7) @ S1PTW
441 mrcne p15, 4, r2, c6, c0, 4 @ HPFAR
445 mrrc p15, 0, r0, r1, c7 @ PAR
448 /* Resolve IPA using the xFAR */
449 mcr p15, 0, r2, c7, c8, 0 @ ATS1CPR
451 mrrc p15, 0, r0, r1, c7 @ PAR
453 bne 4f @ Failed translation
454 ubfx r2, r0, #12, #20
456 orr r2, r2, r1, lsl #24
460 mcrr p15, 0, r0, r1, c7 @ PAR
462 3: load_vcpu @ Load VCPU pointer to r0
463 str r2, [r0, #VCPU_HPFAR]
465 1: mov r1, #ARM_EXCEPTION_HVC
468 4: pop {r0, r1} @ Failed translation, return to guest
469 mcrr p15, 0, r0, r1, c7 @ PAR
475 * If VFPv3 support is not available, then we will not switch the VFP
476 * registers; however cp10 and cp11 accesses will still trap and fallback
477 * to the regular coprocessor emulation code, which currently will
478 * inject an undefined exception to the guest.
482 load_vcpu @ Load VCPU pointer to r0
485 @ NEON/VFP used. Turn on VFP access.
486 set_hcptr vmexit, (HCPTR_TCP(10) | HCPTR_TCP(11))
488 @ Switch VFP/NEON hardware state to the guest's
489 add r7, r0, #VCPU_VFP_HOST
492 add r7, r0, #VCPU_VFP_GUEST
504 mov r1, #ARM_EXCEPTION_IRQ
505 load_vcpu @ Load VCPU pointer to r0
515 .globl __kvm_hyp_code_end
520 .ascii "unexpected undefined exception in Hyp mode at: %#08x\n"
522 .ascii "unexpected prefetch abort in Hyp mode at: %#08x\n"
524 .ascii "unexpected data abort in Hyp mode at: %#08x\n"
526 .ascii "unexpected HVC/SVC trap in Hyp mode at: %#08x\n"