2 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
3 * Author: Christoffer Dall <c.dall@virtualopensystems.com>
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License, version 2, as
7 * published by the Free Software Foundation.
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
19 #include <linux/errno.h>
20 #include <linux/err.h>
21 #include <linux/kvm_host.h>
22 #include <linux/module.h>
23 #include <linux/vmalloc.h>
25 #include <linux/mman.h>
26 #include <linux/sched.h>
27 #include <linux/kvm.h>
28 #include <trace/events/kvm.h>
30 #define CREATE_TRACE_POINTS
33 #include <asm/unified.h>
34 #include <asm/uaccess.h>
35 #include <asm/ptrace.h>
37 #include <asm/cputype.h>
38 #include <asm/tlbflush.h>
39 #include <asm/cacheflush.h>
41 #include <asm/kvm_arm.h>
42 #include <asm/kvm_asm.h>
43 #include <asm/kvm_mmu.h>
44 #include <asm/kvm_emulate.h>
45 #include <asm/kvm_coproc.h>
46 #include <asm/kvm_psci.h>
47 #include <asm/opcodes.h>
50 __asm__(".arch_extension virt");
53 static DEFINE_PER_CPU(unsigned long, kvm_arm_hyp_stack_page
);
54 static struct vfp_hard_struct __percpu
*kvm_host_vfp_state
;
55 static unsigned long hyp_default_vectors
;
57 /* Per-CPU variable containing the currently running vcpu. */
58 static DEFINE_PER_CPU(struct kvm_vcpu
*, kvm_arm_running_vcpu
);
60 /* The VMID used in the VTTBR */
61 static atomic64_t kvm_vmid_gen
= ATOMIC64_INIT(1);
62 static u8 kvm_next_vmid
;
63 static DEFINE_SPINLOCK(kvm_vmid_lock
);
65 static bool vgic_present
;
67 static void kvm_arm_set_running_vcpu(struct kvm_vcpu
*vcpu
)
69 BUG_ON(preemptible());
70 __get_cpu_var(kvm_arm_running_vcpu
) = vcpu
;
74 * kvm_arm_get_running_vcpu - get the vcpu running on the current CPU.
75 * Must be called from non-preemptible context
77 struct kvm_vcpu
*kvm_arm_get_running_vcpu(void)
79 BUG_ON(preemptible());
80 return __get_cpu_var(kvm_arm_running_vcpu
);
84 * kvm_arm_get_running_vcpus - get the per-CPU array of currently running vcpus.
86 struct kvm_vcpu __percpu
**kvm_get_running_vcpus(void)
88 return &kvm_arm_running_vcpu
;
91 int kvm_arch_hardware_enable(void *garbage
)
96 int kvm_arch_vcpu_should_kick(struct kvm_vcpu
*vcpu
)
98 return kvm_vcpu_exiting_guest_mode(vcpu
) == IN_GUEST_MODE
;
101 void kvm_arch_hardware_disable(void *garbage
)
105 int kvm_arch_hardware_setup(void)
110 void kvm_arch_hardware_unsetup(void)
114 void kvm_arch_check_processor_compat(void *rtn
)
119 void kvm_arch_sync_events(struct kvm
*kvm
)
124 * kvm_arch_init_vm - initializes a VM data structure
125 * @kvm: pointer to the KVM struct
127 int kvm_arch_init_vm(struct kvm
*kvm
, unsigned long type
)
134 ret
= kvm_alloc_stage2_pgd(kvm
);
138 ret
= create_hyp_mappings(kvm
, kvm
+ 1);
140 goto out_free_stage2_pgd
;
142 /* Mark the initial VMID generation invalid */
143 kvm
->arch
.vmid_gen
= 0;
147 kvm_free_stage2_pgd(kvm
);
152 int kvm_arch_vcpu_fault(struct kvm_vcpu
*vcpu
, struct vm_fault
*vmf
)
154 return VM_FAULT_SIGBUS
;
157 void kvm_arch_free_memslot(struct kvm_memory_slot
*free
,
158 struct kvm_memory_slot
*dont
)
162 int kvm_arch_create_memslot(struct kvm_memory_slot
*slot
, unsigned long npages
)
168 * kvm_arch_destroy_vm - destroy the VM data structure
169 * @kvm: pointer to the KVM struct
171 void kvm_arch_destroy_vm(struct kvm
*kvm
)
175 kvm_free_stage2_pgd(kvm
);
177 for (i
= 0; i
< KVM_MAX_VCPUS
; ++i
) {
179 kvm_arch_vcpu_free(kvm
->vcpus
[i
]);
180 kvm
->vcpus
[i
] = NULL
;
185 int kvm_dev_ioctl_check_extension(long ext
)
189 case KVM_CAP_IRQCHIP
:
192 case KVM_CAP_USER_MEMORY
:
193 case KVM_CAP_SYNC_MMU
:
194 case KVM_CAP_DESTROY_MEMORY_REGION_WORKS
:
195 case KVM_CAP_ONE_REG
:
196 case KVM_CAP_ARM_PSCI
:
199 case KVM_CAP_COALESCED_MMIO
:
200 r
= KVM_COALESCED_MMIO_PAGE_OFFSET
;
202 case KVM_CAP_ARM_SET_DEVICE_ADDR
:
204 case KVM_CAP_NR_VCPUS
:
205 r
= num_online_cpus();
207 case KVM_CAP_MAX_VCPUS
:
217 long kvm_arch_dev_ioctl(struct file
*filp
,
218 unsigned int ioctl
, unsigned long arg
)
223 int kvm_arch_set_memory_region(struct kvm
*kvm
,
224 struct kvm_userspace_memory_region
*mem
,
225 struct kvm_memory_slot old
,
231 int kvm_arch_prepare_memory_region(struct kvm
*kvm
,
232 struct kvm_memory_slot
*memslot
,
233 struct kvm_memory_slot old
,
234 struct kvm_userspace_memory_region
*mem
,
240 void kvm_arch_commit_memory_region(struct kvm
*kvm
,
241 struct kvm_userspace_memory_region
*mem
,
242 struct kvm_memory_slot old
,
247 void kvm_arch_flush_shadow_all(struct kvm
*kvm
)
251 void kvm_arch_flush_shadow_memslot(struct kvm
*kvm
,
252 struct kvm_memory_slot
*slot
)
256 struct kvm_vcpu
*kvm_arch_vcpu_create(struct kvm
*kvm
, unsigned int id
)
259 struct kvm_vcpu
*vcpu
;
261 vcpu
= kmem_cache_zalloc(kvm_vcpu_cache
, GFP_KERNEL
);
267 err
= kvm_vcpu_init(vcpu
, kvm
, id
);
271 err
= create_hyp_mappings(vcpu
, vcpu
+ 1);
277 kvm_vcpu_uninit(vcpu
);
279 kmem_cache_free(kvm_vcpu_cache
, vcpu
);
284 int kvm_arch_vcpu_postcreate(struct kvm_vcpu
*vcpu
)
289 void kvm_arch_vcpu_free(struct kvm_vcpu
*vcpu
)
291 kvm_mmu_free_memory_caches(vcpu
);
292 kvm_timer_vcpu_terminate(vcpu
);
293 kmem_cache_free(kvm_vcpu_cache
, vcpu
);
296 void kvm_arch_vcpu_destroy(struct kvm_vcpu
*vcpu
)
298 kvm_arch_vcpu_free(vcpu
);
301 int kvm_cpu_has_pending_timer(struct kvm_vcpu
*vcpu
)
306 int __attribute_const__
kvm_target_cpu(void)
308 unsigned long implementor
= read_cpuid_implementor();
309 unsigned long part_number
= read_cpuid_part_number();
311 if (implementor
!= ARM_CPU_IMP_ARM
)
314 switch (part_number
) {
315 case ARM_CPU_PART_CORTEX_A15
:
316 return KVM_ARM_TARGET_CORTEX_A15
;
322 int kvm_arch_vcpu_init(struct kvm_vcpu
*vcpu
)
326 /* Force users to call KVM_ARM_VCPU_INIT */
327 vcpu
->arch
.target
= -1;
330 ret
= kvm_vgic_vcpu_init(vcpu
);
334 /* Set up the timer */
335 kvm_timer_vcpu_init(vcpu
);
340 void kvm_arch_vcpu_uninit(struct kvm_vcpu
*vcpu
)
344 void kvm_arch_vcpu_load(struct kvm_vcpu
*vcpu
, int cpu
)
347 vcpu
->arch
.vfp_host
= this_cpu_ptr(kvm_host_vfp_state
);
350 * Check whether this vcpu requires the cache to be flushed on
351 * this physical CPU. This is a consequence of doing dcache
352 * operations by set/way on this vcpu. We do it here to be in
353 * a non-preemptible section.
355 if (cpumask_test_and_clear_cpu(cpu
, &vcpu
->arch
.require_dcache_flush
))
356 flush_cache_all(); /* We'd really want v7_flush_dcache_all() */
358 kvm_arm_set_running_vcpu(vcpu
);
361 void kvm_arch_vcpu_put(struct kvm_vcpu
*vcpu
)
363 kvm_arm_set_running_vcpu(NULL
);
366 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu
*vcpu
,
367 struct kvm_guest_debug
*dbg
)
373 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu
*vcpu
,
374 struct kvm_mp_state
*mp_state
)
379 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu
*vcpu
,
380 struct kvm_mp_state
*mp_state
)
386 * kvm_arch_vcpu_runnable - determine if the vcpu can be scheduled
387 * @v: The VCPU pointer
389 * If the guest CPU is not waiting for interrupts or an interrupt line is
390 * asserted, the CPU is by definition runnable.
392 int kvm_arch_vcpu_runnable(struct kvm_vcpu
*v
)
394 return !!v
->arch
.irq_lines
|| kvm_vgic_vcpu_pending_irq(v
);
397 /* Just ensure a guest exit from a particular CPU */
398 static void exit_vm_noop(void *info
)
402 void force_vm_exit(const cpumask_t
*mask
)
404 smp_call_function_many(mask
, exit_vm_noop
, NULL
, true);
408 * need_new_vmid_gen - check that the VMID is still valid
409 * @kvm: The VM's VMID to checkt
411 * return true if there is a new generation of VMIDs being used
413 * The hardware supports only 256 values with the value zero reserved for the
414 * host, so we check if an assigned value belongs to a previous generation,
415 * which which requires us to assign a new value. If we're the first to use a
416 * VMID for the new generation, we must flush necessary caches and TLBs on all
419 static bool need_new_vmid_gen(struct kvm
*kvm
)
421 return unlikely(kvm
->arch
.vmid_gen
!= atomic64_read(&kvm_vmid_gen
));
425 * update_vttbr - Update the VTTBR with a valid VMID before the guest runs
426 * @kvm The guest that we are about to run
428 * Called from kvm_arch_vcpu_ioctl_run before entering the guest to ensure the
429 * VM has a valid VMID, otherwise assigns a new one and flushes corresponding
432 static void update_vttbr(struct kvm
*kvm
)
434 phys_addr_t pgd_phys
;
437 if (!need_new_vmid_gen(kvm
))
440 spin_lock(&kvm_vmid_lock
);
443 * We need to re-check the vmid_gen here to ensure that if another vcpu
444 * already allocated a valid vmid for this vm, then this vcpu should
447 if (!need_new_vmid_gen(kvm
)) {
448 spin_unlock(&kvm_vmid_lock
);
452 /* First user of a new VMID generation? */
453 if (unlikely(kvm_next_vmid
== 0)) {
454 atomic64_inc(&kvm_vmid_gen
);
458 * On SMP we know no other CPUs can use this CPU's or each
459 * other's VMID after force_vm_exit returns since the
460 * kvm_vmid_lock blocks them from reentry to the guest.
462 force_vm_exit(cpu_all_mask
);
464 * Now broadcast TLB + ICACHE invalidation over the inner
465 * shareable domain to make sure all data structures are
468 kvm_call_hyp(__kvm_flush_vm_context
);
471 kvm
->arch
.vmid_gen
= atomic64_read(&kvm_vmid_gen
);
472 kvm
->arch
.vmid
= kvm_next_vmid
;
475 /* update vttbr to be used with the new vmid */
476 pgd_phys
= virt_to_phys(kvm
->arch
.pgd
);
477 vmid
= ((u64
)(kvm
->arch
.vmid
) << VTTBR_VMID_SHIFT
) & VTTBR_VMID_MASK
;
478 kvm
->arch
.vttbr
= pgd_phys
& VTTBR_BADDR_MASK
;
479 kvm
->arch
.vttbr
|= vmid
;
481 spin_unlock(&kvm_vmid_lock
);
484 static int handle_svc_hyp(struct kvm_vcpu
*vcpu
, struct kvm_run
*run
)
486 /* SVC called from Hyp mode should never get here */
487 kvm_debug("SVC called from Hyp mode shouldn't go here\n");
489 return -EINVAL
; /* Squash warning */
492 static int handle_hvc(struct kvm_vcpu
*vcpu
, struct kvm_run
*run
)
494 trace_kvm_hvc(*vcpu_pc(vcpu
), *vcpu_reg(vcpu
, 0),
495 kvm_vcpu_get_hsr(vcpu
) & HSR_HVC_IMM_MASK
);
497 if (kvm_psci_call(vcpu
))
500 kvm_inject_undefined(vcpu
);
504 static int handle_smc(struct kvm_vcpu
*vcpu
, struct kvm_run
*run
)
506 if (kvm_psci_call(vcpu
))
509 kvm_inject_undefined(vcpu
);
513 static int handle_pabt_hyp(struct kvm_vcpu
*vcpu
, struct kvm_run
*run
)
515 /* The hypervisor should never cause aborts */
516 kvm_err("Prefetch Abort taken from Hyp mode at %#08lx (HSR: %#08x)\n",
517 kvm_vcpu_get_hfar(vcpu
), kvm_vcpu_get_hsr(vcpu
));
521 static int handle_dabt_hyp(struct kvm_vcpu
*vcpu
, struct kvm_run
*run
)
523 /* This is either an error in the ws. code or an external abort */
524 kvm_err("Data Abort taken from Hyp mode at %#08lx (HSR: %#08x)\n",
525 kvm_vcpu_get_hfar(vcpu
), kvm_vcpu_get_hsr(vcpu
));
529 typedef int (*exit_handle_fn
)(struct kvm_vcpu
*, struct kvm_run
*);
530 static exit_handle_fn arm_exit_handlers
[] = {
531 [HSR_EC_WFI
] = kvm_handle_wfi
,
532 [HSR_EC_CP15_32
] = kvm_handle_cp15_32
,
533 [HSR_EC_CP15_64
] = kvm_handle_cp15_64
,
534 [HSR_EC_CP14_MR
] = kvm_handle_cp14_access
,
535 [HSR_EC_CP14_LS
] = kvm_handle_cp14_load_store
,
536 [HSR_EC_CP14_64
] = kvm_handle_cp14_access
,
537 [HSR_EC_CP_0_13
] = kvm_handle_cp_0_13_access
,
538 [HSR_EC_CP10_ID
] = kvm_handle_cp10_id
,
539 [HSR_EC_SVC_HYP
] = handle_svc_hyp
,
540 [HSR_EC_HVC
] = handle_hvc
,
541 [HSR_EC_SMC
] = handle_smc
,
542 [HSR_EC_IABT
] = kvm_handle_guest_abort
,
543 [HSR_EC_IABT_HYP
] = handle_pabt_hyp
,
544 [HSR_EC_DABT
] = kvm_handle_guest_abort
,
545 [HSR_EC_DABT_HYP
] = handle_dabt_hyp
,
549 * A conditional instruction is allowed to trap, even though it
550 * wouldn't be executed. So let's re-implement the hardware, in
553 static bool kvm_condition_valid(struct kvm_vcpu
*vcpu
)
555 unsigned long cpsr
, cond
, insn
;
558 * Exception Code 0 can only happen if we set HCR.TGE to 1, to
559 * catch undefined instructions, and then we won't get past
560 * the arm_exit_handlers test anyway.
562 BUG_ON(((kvm_vcpu_get_hsr(vcpu
) & HSR_EC
) >> HSR_EC_SHIFT
) == 0);
564 /* Top two bits non-zero? Unconditional. */
565 if (kvm_vcpu_get_hsr(vcpu
) >> 30)
568 cpsr
= *vcpu_cpsr(vcpu
);
570 /* Is condition field valid? */
571 if ((kvm_vcpu_get_hsr(vcpu
) & HSR_CV
) >> HSR_CV_SHIFT
)
572 cond
= (kvm_vcpu_get_hsr(vcpu
) & HSR_COND
) >> HSR_COND_SHIFT
;
574 /* This can happen in Thumb mode: examine IT state. */
577 it
= ((cpsr
>> 8) & 0xFC) | ((cpsr
>> 25) & 0x3);
579 /* it == 0 => unconditional. */
583 /* The cond for this insn works out as the top 4 bits. */
587 /* Shift makes it look like an ARM-mode instruction */
589 return arm_check_condition(insn
, cpsr
) != ARM_OPCODE_CONDTEST_FAIL
;
593 * Return > 0 to return to guest, < 0 on error, 0 (and set exit_reason) on
594 * proper exit to QEMU.
596 static int handle_exit(struct kvm_vcpu
*vcpu
, struct kvm_run
*run
,
599 unsigned long hsr_ec
;
601 switch (exception_index
) {
602 case ARM_EXCEPTION_IRQ
:
604 case ARM_EXCEPTION_UNDEFINED
:
605 kvm_err("Undefined exception in Hyp mode at: %#08lx\n",
606 kvm_vcpu_get_hyp_pc(vcpu
));
608 panic("KVM: Hypervisor undefined exception!\n");
609 case ARM_EXCEPTION_DATA_ABORT
:
610 case ARM_EXCEPTION_PREF_ABORT
:
611 case ARM_EXCEPTION_HVC
:
612 hsr_ec
= (kvm_vcpu_get_hsr(vcpu
) & HSR_EC
) >> HSR_EC_SHIFT
;
614 if (hsr_ec
>= ARRAY_SIZE(arm_exit_handlers
)
615 || !arm_exit_handlers
[hsr_ec
]) {
616 kvm_err("Unkown exception class: %#08lx, "
617 "hsr: %#08x\n", hsr_ec
,
618 (unsigned int)kvm_vcpu_get_hsr(vcpu
));
623 * See ARM ARM B1.14.1: "Hyp traps on instructions
624 * that fail their condition code check"
626 if (!kvm_condition_valid(vcpu
)) {
627 kvm_skip_instr(vcpu
, kvm_vcpu_trap_il_is32bit(vcpu
));
631 return arm_exit_handlers
[hsr_ec
](vcpu
, run
);
633 kvm_pr_unimpl("Unsupported exception type: %d",
635 run
->exit_reason
= KVM_EXIT_INTERNAL_ERROR
;
640 static int kvm_vcpu_first_run_init(struct kvm_vcpu
*vcpu
)
642 if (likely(vcpu
->arch
.has_run_once
))
645 vcpu
->arch
.has_run_once
= true;
648 * Initialize the VGIC before running a vcpu the first time on
651 if (irqchip_in_kernel(vcpu
->kvm
) &&
652 unlikely(!vgic_initialized(vcpu
->kvm
))) {
653 int ret
= kvm_vgic_init(vcpu
->kvm
);
659 * Handle the "start in power-off" case by calling into the
662 if (test_and_clear_bit(KVM_ARM_VCPU_POWER_OFF
, vcpu
->arch
.features
)) {
663 *vcpu_reg(vcpu
, 0) = KVM_PSCI_FN_CPU_OFF
;
670 static void vcpu_pause(struct kvm_vcpu
*vcpu
)
672 wait_queue_head_t
*wq
= kvm_arch_vcpu_wq(vcpu
);
674 wait_event_interruptible(*wq
, !vcpu
->arch
.pause
);
678 * kvm_arch_vcpu_ioctl_run - the main VCPU run function to execute guest code
679 * @vcpu: The VCPU pointer
680 * @run: The kvm_run structure pointer used for userspace state exchange
682 * This function is called through the VCPU_RUN ioctl called from user space. It
683 * will execute VM code in a loop until the time slice for the process is used
684 * or some emulation is needed from user space in which case the function will
685 * return with return value 0 and with the kvm_run structure filled in with the
686 * required data for the requested emulation.
688 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu
*vcpu
, struct kvm_run
*run
)
693 /* Make sure they initialize the vcpu with KVM_ARM_VCPU_INIT */
694 if (unlikely(vcpu
->arch
.target
< 0))
697 ret
= kvm_vcpu_first_run_init(vcpu
);
701 if (run
->exit_reason
== KVM_EXIT_MMIO
) {
702 ret
= kvm_handle_mmio_return(vcpu
, vcpu
->run
);
707 if (vcpu
->sigset_active
)
708 sigprocmask(SIG_SETMASK
, &vcpu
->sigset
, &sigsaved
);
711 run
->exit_reason
= KVM_EXIT_UNKNOWN
;
714 * Check conditions before entering the guest
718 update_vttbr(vcpu
->kvm
);
720 if (vcpu
->arch
.pause
)
723 kvm_vgic_flush_hwstate(vcpu
);
724 kvm_timer_flush_hwstate(vcpu
);
729 * Re-check atomic conditions
731 if (signal_pending(current
)) {
733 run
->exit_reason
= KVM_EXIT_INTR
;
736 if (ret
<= 0 || need_new_vmid_gen(vcpu
->kvm
)) {
738 kvm_timer_sync_hwstate(vcpu
);
739 kvm_vgic_sync_hwstate(vcpu
);
743 /**************************************************************
746 trace_kvm_entry(*vcpu_pc(vcpu
));
748 vcpu
->mode
= IN_GUEST_MODE
;
750 ret
= kvm_call_hyp(__kvm_vcpu_run
, vcpu
);
752 vcpu
->mode
= OUTSIDE_GUEST_MODE
;
753 vcpu
->arch
.last_pcpu
= smp_processor_id();
755 trace_kvm_exit(*vcpu_pc(vcpu
));
757 * We may have taken a host interrupt in HYP mode (ie
758 * while executing the guest). This interrupt is still
759 * pending, as we haven't serviced it yet!
761 * We're now back in SVC mode, with interrupts
762 * disabled. Enabling the interrupts now will have
763 * the effect of taking the interrupt again, in SVC
770 *************************************************************/
772 kvm_timer_sync_hwstate(vcpu
);
773 kvm_vgic_sync_hwstate(vcpu
);
775 ret
= handle_exit(vcpu
, run
, ret
);
778 if (vcpu
->sigset_active
)
779 sigprocmask(SIG_SETMASK
, &sigsaved
, NULL
);
783 static int vcpu_interrupt_line(struct kvm_vcpu
*vcpu
, int number
, bool level
)
789 if (number
== KVM_ARM_IRQ_CPU_IRQ
)
790 bit_index
= __ffs(HCR_VI
);
791 else /* KVM_ARM_IRQ_CPU_FIQ */
792 bit_index
= __ffs(HCR_VF
);
794 ptr
= (unsigned long *)&vcpu
->arch
.irq_lines
;
796 set
= test_and_set_bit(bit_index
, ptr
);
798 set
= test_and_clear_bit(bit_index
, ptr
);
801 * If we didn't change anything, no need to wake up or kick other CPUs
807 * The vcpu irq_lines field was updated, wake up sleeping VCPUs and
808 * trigger a world-switch round on the running physical CPU to set the
809 * virtual IRQ/FIQ fields in the HCR appropriately.
816 int kvm_vm_ioctl_irq_line(struct kvm
*kvm
, struct kvm_irq_level
*irq_level
)
818 u32 irq
= irq_level
->irq
;
819 unsigned int irq_type
, vcpu_idx
, irq_num
;
820 int nrcpus
= atomic_read(&kvm
->online_vcpus
);
821 struct kvm_vcpu
*vcpu
= NULL
;
822 bool level
= irq_level
->level
;
824 irq_type
= (irq
>> KVM_ARM_IRQ_TYPE_SHIFT
) & KVM_ARM_IRQ_TYPE_MASK
;
825 vcpu_idx
= (irq
>> KVM_ARM_IRQ_VCPU_SHIFT
) & KVM_ARM_IRQ_VCPU_MASK
;
826 irq_num
= (irq
>> KVM_ARM_IRQ_NUM_SHIFT
) & KVM_ARM_IRQ_NUM_MASK
;
828 trace_kvm_irq_line(irq_type
, vcpu_idx
, irq_num
, irq_level
->level
);
831 case KVM_ARM_IRQ_TYPE_CPU
:
832 if (irqchip_in_kernel(kvm
))
835 if (vcpu_idx
>= nrcpus
)
838 vcpu
= kvm_get_vcpu(kvm
, vcpu_idx
);
842 if (irq_num
> KVM_ARM_IRQ_CPU_FIQ
)
845 return vcpu_interrupt_line(vcpu
, irq_num
, level
);
846 case KVM_ARM_IRQ_TYPE_PPI
:
847 if (!irqchip_in_kernel(kvm
))
850 if (vcpu_idx
>= nrcpus
)
853 vcpu
= kvm_get_vcpu(kvm
, vcpu_idx
);
857 if (irq_num
< VGIC_NR_SGIS
|| irq_num
>= VGIC_NR_PRIVATE_IRQS
)
860 return kvm_vgic_inject_irq(kvm
, vcpu
->vcpu_id
, irq_num
, level
);
861 case KVM_ARM_IRQ_TYPE_SPI
:
862 if (!irqchip_in_kernel(kvm
))
865 if (irq_num
< VGIC_NR_PRIVATE_IRQS
||
866 irq_num
> KVM_ARM_IRQ_GIC_MAX
)
869 return kvm_vgic_inject_irq(kvm
, 0, irq_num
, level
);
875 long kvm_arch_vcpu_ioctl(struct file
*filp
,
876 unsigned int ioctl
, unsigned long arg
)
878 struct kvm_vcpu
*vcpu
= filp
->private_data
;
879 void __user
*argp
= (void __user
*)arg
;
882 case KVM_ARM_VCPU_INIT
: {
883 struct kvm_vcpu_init init
;
885 if (copy_from_user(&init
, argp
, sizeof(init
)))
888 return kvm_vcpu_set_target(vcpu
, &init
);
891 case KVM_SET_ONE_REG
:
892 case KVM_GET_ONE_REG
: {
893 struct kvm_one_reg reg
;
894 if (copy_from_user(®
, argp
, sizeof(reg
)))
896 if (ioctl
== KVM_SET_ONE_REG
)
897 return kvm_arm_set_reg(vcpu
, ®
);
899 return kvm_arm_get_reg(vcpu
, ®
);
901 case KVM_GET_REG_LIST
: {
902 struct kvm_reg_list __user
*user_list
= argp
;
903 struct kvm_reg_list reg_list
;
906 if (copy_from_user(®_list
, user_list
, sizeof(reg_list
)))
909 reg_list
.n
= kvm_arm_num_regs(vcpu
);
910 if (copy_to_user(user_list
, ®_list
, sizeof(reg_list
)))
914 return kvm_arm_copy_reg_indices(vcpu
, user_list
->reg
);
921 int kvm_vm_ioctl_get_dirty_log(struct kvm
*kvm
, struct kvm_dirty_log
*log
)
926 static int kvm_vm_ioctl_set_device_addr(struct kvm
*kvm
,
927 struct kvm_arm_device_addr
*dev_addr
)
929 unsigned long dev_id
, type
;
931 dev_id
= (dev_addr
->id
& KVM_ARM_DEVICE_ID_MASK
) >>
932 KVM_ARM_DEVICE_ID_SHIFT
;
933 type
= (dev_addr
->id
& KVM_ARM_DEVICE_TYPE_MASK
) >>
934 KVM_ARM_DEVICE_TYPE_SHIFT
;
937 case KVM_ARM_DEVICE_VGIC_V2
:
940 return kvm_vgic_set_addr(kvm
, type
, dev_addr
->addr
);
946 long kvm_arch_vm_ioctl(struct file
*filp
,
947 unsigned int ioctl
, unsigned long arg
)
949 struct kvm
*kvm
= filp
->private_data
;
950 void __user
*argp
= (void __user
*)arg
;
953 case KVM_CREATE_IRQCHIP
: {
955 return kvm_vgic_create(kvm
);
959 case KVM_ARM_SET_DEVICE_ADDR
: {
960 struct kvm_arm_device_addr dev_addr
;
962 if (copy_from_user(&dev_addr
, argp
, sizeof(dev_addr
)))
964 return kvm_vm_ioctl_set_device_addr(kvm
, &dev_addr
);
971 static void cpu_init_hyp_mode(void *vector
)
973 unsigned long long pgd_ptr
;
974 unsigned long pgd_low
, pgd_high
;
975 unsigned long hyp_stack_ptr
;
976 unsigned long stack_page
;
977 unsigned long vector_ptr
;
979 /* Switch from the HYP stub to our own HYP init vector */
980 __hyp_set_vectors((unsigned long)vector
);
982 pgd_ptr
= (unsigned long long)kvm_mmu_get_httbr();
983 pgd_low
= (pgd_ptr
& ((1ULL << 32) - 1));
984 pgd_high
= (pgd_ptr
>> 32ULL);
985 stack_page
= __get_cpu_var(kvm_arm_hyp_stack_page
);
986 hyp_stack_ptr
= stack_page
+ PAGE_SIZE
;
987 vector_ptr
= (unsigned long)__kvm_hyp_vector
;
990 * Call initialization code, and switch to the full blown
991 * HYP code. The init code doesn't need to preserve these registers as
992 * r1-r3 and r12 are already callee save according to the AAPCS.
993 * Note that we slightly misuse the prototype by casing the pgd_low to
996 kvm_call_hyp((void *)pgd_low
, pgd_high
, hyp_stack_ptr
, vector_ptr
);
1000 * Inits Hyp-mode on all online CPUs
1002 static int init_hyp_mode(void)
1004 phys_addr_t init_phys_addr
;
1009 * Allocate Hyp PGD and setup Hyp identity mapping
1011 err
= kvm_mmu_init();
1016 * It is probably enough to obtain the default on one
1017 * CPU. It's unlikely to be different on the others.
1019 hyp_default_vectors
= __hyp_get_vectors();
1022 * Allocate stack pages for Hypervisor-mode
1024 for_each_possible_cpu(cpu
) {
1025 unsigned long stack_page
;
1027 stack_page
= __get_free_page(GFP_KERNEL
);
1030 goto out_free_stack_pages
;
1033 per_cpu(kvm_arm_hyp_stack_page
, cpu
) = stack_page
;
1037 * Execute the init code on each CPU.
1039 * Note: The stack is not mapped yet, so don't do anything else than
1040 * initializing the hypervisor mode on each CPU using a local stack
1041 * space for temporary storage.
1043 init_phys_addr
= virt_to_phys(__kvm_hyp_init
);
1044 for_each_online_cpu(cpu
) {
1045 smp_call_function_single(cpu
, cpu_init_hyp_mode
,
1046 (void *)(long)init_phys_addr
, 1);
1050 * Unmap the identity mapping
1052 kvm_clear_hyp_idmap();
1055 * Map the Hyp-code called directly from the host
1057 err
= create_hyp_mappings(__kvm_hyp_code_start
, __kvm_hyp_code_end
);
1059 kvm_err("Cannot map world-switch code\n");
1060 goto out_free_mappings
;
1064 * Map the Hyp stack pages
1066 for_each_possible_cpu(cpu
) {
1067 char *stack_page
= (char *)per_cpu(kvm_arm_hyp_stack_page
, cpu
);
1068 err
= create_hyp_mappings(stack_page
, stack_page
+ PAGE_SIZE
);
1071 kvm_err("Cannot map hyp stack\n");
1072 goto out_free_mappings
;
1077 * Map the host VFP structures
1079 kvm_host_vfp_state
= alloc_percpu(struct vfp_hard_struct
);
1080 if (!kvm_host_vfp_state
) {
1082 kvm_err("Cannot allocate host VFP state\n");
1083 goto out_free_mappings
;
1086 for_each_possible_cpu(cpu
) {
1087 struct vfp_hard_struct
*vfp
;
1089 vfp
= per_cpu_ptr(kvm_host_vfp_state
, cpu
);
1090 err
= create_hyp_mappings(vfp
, vfp
+ 1);
1093 kvm_err("Cannot map host VFP state: %d\n", err
);
1099 * Init HYP view of VGIC
1101 err
= kvm_vgic_hyp_init();
1105 #ifdef CONFIG_KVM_ARM_VGIC
1106 vgic_present
= true;
1110 * Init HYP architected timer support
1112 err
= kvm_timer_hyp_init();
1114 goto out_free_mappings
;
1116 kvm_info("Hyp mode initialized successfully\n");
1119 free_percpu(kvm_host_vfp_state
);
1122 out_free_stack_pages
:
1123 for_each_possible_cpu(cpu
)
1124 free_page(per_cpu(kvm_arm_hyp_stack_page
, cpu
));
1126 kvm_err("error initializing Hyp mode: %d\n", err
);
1131 * Initialize Hyp-mode and memory mappings on all CPUs.
1133 int kvm_arch_init(void *opaque
)
1137 if (!is_hyp_mode_available()) {
1138 kvm_err("HYP mode not available\n");
1142 if (kvm_target_cpu() < 0) {
1143 kvm_err("Target CPU not supported!\n");
1147 err
= init_hyp_mode();
1151 kvm_coproc_table_init();
1157 /* NOP: Compiling as a module not supported */
1158 void kvm_arch_exit(void)
1162 static int arm_init(void)
1164 int rc
= kvm_init(NULL
, sizeof(struct kvm_vcpu
), 0, THIS_MODULE
);
1168 module_init(arm_init
);