2 * ARM implementation of KVM hooks
4 * Copyright Christoffer Dall 2009-2010
6 * This work is licensed under the terms of the GNU GPL, version 2 or later.
7 * See the COPYING file in the top-level directory.
11 #include "qemu/osdep.h"
12 #include <sys/ioctl.h>
14 #include <linux/kvm.h>
16 #include "qemu/timer.h"
17 #include "qemu/error-report.h"
18 #include "qemu/main-loop.h"
19 #include "qom/object.h"
20 #include "qapi/error.h"
21 #include "sysemu/sysemu.h"
22 #include "sysemu/kvm.h"
23 #include "sysemu/kvm_int.h"
27 #include "internals.h"
28 #include "hw/pci/pci.h"
29 #include "exec/memattrs.h"
30 #include "exec/address-spaces.h"
31 #include "hw/boards.h"
33 #include "qapi/visitor.h"
36 const KVMCapabilityInfo kvm_arch_required_capabilities
[] = {
40 static bool cap_has_mp_state
;
41 static bool cap_has_inject_serror_esr
;
42 static bool cap_has_inject_ext_dabt
;
44 static ARMHostCPUFeatures arm_host_cpu_features
;
46 int kvm_arm_vcpu_init(CPUState
*cs
)
48 ARMCPU
*cpu
= ARM_CPU(cs
);
49 struct kvm_vcpu_init init
;
51 init
.target
= cpu
->kvm_target
;
52 memcpy(init
.features
, cpu
->kvm_init_features
, sizeof(init
.features
));
54 return kvm_vcpu_ioctl(cs
, KVM_ARM_VCPU_INIT
, &init
);
57 int kvm_arm_vcpu_finalize(CPUState
*cs
, int feature
)
59 return kvm_vcpu_ioctl(cs
, KVM_ARM_VCPU_FINALIZE
, &feature
);
62 void kvm_arm_init_serror_injection(CPUState
*cs
)
64 cap_has_inject_serror_esr
= kvm_check_extension(cs
->kvm_state
,
65 KVM_CAP_ARM_INJECT_SERROR_ESR
);
68 bool kvm_arm_create_scratch_host_vcpu(const uint32_t *cpus_to_try
,
70 struct kvm_vcpu_init
*init
)
72 int ret
= 0, kvmfd
= -1, vmfd
= -1, cpufd
= -1;
75 kvmfd
= qemu_open_old("/dev/kvm", O_RDWR
);
79 max_vm_pa_size
= ioctl(kvmfd
, KVM_CHECK_EXTENSION
, KVM_CAP_ARM_VM_IPA_SIZE
);
80 if (max_vm_pa_size
< 0) {
84 vmfd
= ioctl(kvmfd
, KVM_CREATE_VM
, max_vm_pa_size
);
85 } while (vmfd
== -1 && errno
== EINTR
);
89 cpufd
= ioctl(vmfd
, KVM_CREATE_VCPU
, 0);
95 /* Caller doesn't want the VCPU to be initialized, so skip it */
99 if (init
->target
== -1) {
100 struct kvm_vcpu_init preferred
;
102 ret
= ioctl(vmfd
, KVM_ARM_PREFERRED_TARGET
, &preferred
);
104 init
->target
= preferred
.target
;
108 ret
= ioctl(cpufd
, KVM_ARM_VCPU_INIT
, init
);
112 } else if (cpus_to_try
) {
113 /* Old kernel which doesn't know about the
114 * PREFERRED_TARGET ioctl: we know it will only support
115 * creating one kind of guest CPU which is its preferred
118 struct kvm_vcpu_init
try;
120 while (*cpus_to_try
!= QEMU_KVM_ARM_TARGET_NONE
) {
121 try.target
= *cpus_to_try
++;
122 memcpy(try.features
, init
->features
, sizeof(init
->features
));
123 ret
= ioctl(cpufd
, KVM_ARM_VCPU_INIT
, &try);
131 init
->target
= try.target
;
133 /* Treat a NULL cpus_to_try argument the same as an empty
134 * list, which means we will fail the call since this must
135 * be an old kernel which doesn't support PREFERRED_TARGET.
161 void kvm_arm_destroy_scratch_host_vcpu(int *fdarray
)
165 for (i
= 2; i
>= 0; i
--) {
170 void kvm_arm_set_cpu_features_from_host(ARMCPU
*cpu
)
172 CPUARMState
*env
= &cpu
->env
;
174 if (!arm_host_cpu_features
.dtb_compatible
) {
175 if (!kvm_enabled() ||
176 !kvm_arm_get_host_cpu_features(&arm_host_cpu_features
)) {
177 /* We can't report this error yet, so flag that we need to
178 * in arm_cpu_realizefn().
180 cpu
->kvm_target
= QEMU_KVM_ARM_TARGET_NONE
;
181 cpu
->host_cpu_probe_failed
= true;
186 cpu
->kvm_target
= arm_host_cpu_features
.target
;
187 cpu
->dtb_compatible
= arm_host_cpu_features
.dtb_compatible
;
188 cpu
->isar
= arm_host_cpu_features
.isar
;
189 env
->features
= arm_host_cpu_features
.features
;
192 static bool kvm_no_adjvtime_get(Object
*obj
, Error
**errp
)
194 return !ARM_CPU(obj
)->kvm_adjvtime
;
197 static void kvm_no_adjvtime_set(Object
*obj
, bool value
, Error
**errp
)
199 ARM_CPU(obj
)->kvm_adjvtime
= !value
;
202 static bool kvm_steal_time_get(Object
*obj
, Error
**errp
)
204 return ARM_CPU(obj
)->kvm_steal_time
!= ON_OFF_AUTO_OFF
;
207 static void kvm_steal_time_set(Object
*obj
, bool value
, Error
**errp
)
209 ARM_CPU(obj
)->kvm_steal_time
= value
? ON_OFF_AUTO_ON
: ON_OFF_AUTO_OFF
;
212 /* KVM VCPU properties should be prefixed with "kvm-". */
213 void kvm_arm_add_vcpu_properties(Object
*obj
)
215 ARMCPU
*cpu
= ARM_CPU(obj
);
216 CPUARMState
*env
= &cpu
->env
;
218 if (arm_feature(env
, ARM_FEATURE_GENERIC_TIMER
)) {
219 cpu
->kvm_adjvtime
= true;
220 object_property_add_bool(obj
, "kvm-no-adjvtime", kvm_no_adjvtime_get
,
221 kvm_no_adjvtime_set
);
222 object_property_set_description(obj
, "kvm-no-adjvtime",
223 "Set on to disable the adjustment of "
224 "the virtual counter. VM stopped time "
228 cpu
->kvm_steal_time
= ON_OFF_AUTO_AUTO
;
229 object_property_add_bool(obj
, "kvm-steal-time", kvm_steal_time_get
,
231 object_property_set_description(obj
, "kvm-steal-time",
232 "Set off to disable KVM steal time.");
235 bool kvm_arm_pmu_supported(void)
237 return kvm_check_extension(kvm_state
, KVM_CAP_ARM_PMU_V3
);
240 int kvm_arm_get_max_vm_ipa_size(MachineState
*ms
, bool *fixed_ipa
)
242 KVMState
*s
= KVM_STATE(ms
->accelerator
);
245 ret
= kvm_check_extension(s
, KVM_CAP_ARM_VM_IPA_SIZE
);
246 *fixed_ipa
= ret
<= 0;
248 return ret
> 0 ? ret
: 40;
251 int kvm_arch_get_default_type(MachineState
*ms
)
254 int size
= kvm_arm_get_max_vm_ipa_size(ms
, &fixed_ipa
);
255 return fixed_ipa
? 0 : size
;
258 int kvm_arch_init(MachineState
*ms
, KVMState
*s
)
261 /* For ARM interrupt delivery is always asynchronous,
262 * whether we are using an in-kernel VGIC or not.
264 kvm_async_interrupts_allowed
= true;
267 * PSCI wakes up secondary cores, so we always need to
268 * have vCPUs waiting in kernel space
270 kvm_halt_in_kernel_allowed
= true;
272 cap_has_mp_state
= kvm_check_extension(s
, KVM_CAP_MP_STATE
);
274 if (ms
->smp
.cpus
> 256 &&
275 !kvm_check_extension(s
, KVM_CAP_ARM_IRQ_LINE_LAYOUT_2
)) {
276 error_report("Using more than 256 vcpus requires a host kernel "
277 "with KVM_CAP_ARM_IRQ_LINE_LAYOUT_2");
281 if (kvm_check_extension(s
, KVM_CAP_ARM_NISV_TO_USER
)) {
282 if (kvm_vm_enable_cap(s
, KVM_CAP_ARM_NISV_TO_USER
, 0)) {
283 error_report("Failed to enable KVM_CAP_ARM_NISV_TO_USER cap");
285 /* Set status for supporting the external dabt injection */
286 cap_has_inject_ext_dabt
= kvm_check_extension(s
,
287 KVM_CAP_ARM_INJECT_EXT_DABT
);
291 if (s
->kvm_eager_split_size
) {
294 sizes
= kvm_vm_check_extension(s
, KVM_CAP_ARM_SUPPORTED_BLOCK_SIZES
);
296 s
->kvm_eager_split_size
= 0;
297 warn_report("Eager Page Split support not available");
298 } else if (!(s
->kvm_eager_split_size
& sizes
)) {
299 error_report("Eager Page Split requested chunk size not valid");
302 ret
= kvm_vm_enable_cap(s
, KVM_CAP_ARM_EAGER_SPLIT_CHUNK_SIZE
, 0,
303 s
->kvm_eager_split_size
);
305 error_report("Enabling of Eager Page Split failed: %s",
311 max_hw_wps
= kvm_check_extension(s
, KVM_CAP_GUEST_DEBUG_HW_WPS
);
312 hw_watchpoints
= g_array_sized_new(true, true,
313 sizeof(HWWatchpoint
), max_hw_wps
);
315 max_hw_bps
= kvm_check_extension(s
, KVM_CAP_GUEST_DEBUG_HW_BPS
);
316 hw_breakpoints
= g_array_sized_new(true, true,
317 sizeof(HWBreakpoint
), max_hw_bps
);
322 unsigned long kvm_arch_vcpu_id(CPUState
*cpu
)
324 return cpu
->cpu_index
;
327 /* We track all the KVM devices which need their memory addresses
328 * passing to the kernel in a list of these structures.
329 * When board init is complete we run through the list and
330 * tell the kernel the base addresses of the memory regions.
331 * We use a MemoryListener to track mapping and unmapping of
332 * the regions during board creation, so the board models don't
333 * need to do anything special for the KVM case.
335 * Sometimes the address must be OR'ed with some other fields
336 * (for example for KVM_VGIC_V3_ADDR_TYPE_REDIST_REGION).
337 * @kda_addr_ormask aims at storing the value of those fields.
339 typedef struct KVMDevice
{
340 struct kvm_arm_device_addr kda
;
341 struct kvm_device_attr kdattr
;
342 uint64_t kda_addr_ormask
;
344 QSLIST_ENTRY(KVMDevice
) entries
;
348 static QSLIST_HEAD(, KVMDevice
) kvm_devices_head
;
350 static void kvm_arm_devlistener_add(MemoryListener
*listener
,
351 MemoryRegionSection
*section
)
355 QSLIST_FOREACH(kd
, &kvm_devices_head
, entries
) {
356 if (section
->mr
== kd
->mr
) {
357 kd
->kda
.addr
= section
->offset_within_address_space
;
362 static void kvm_arm_devlistener_del(MemoryListener
*listener
,
363 MemoryRegionSection
*section
)
367 QSLIST_FOREACH(kd
, &kvm_devices_head
, entries
) {
368 if (section
->mr
== kd
->mr
) {
374 static MemoryListener devlistener
= {
376 .region_add
= kvm_arm_devlistener_add
,
377 .region_del
= kvm_arm_devlistener_del
,
378 .priority
= MEMORY_LISTENER_PRIORITY_MIN
,
381 static void kvm_arm_set_device_addr(KVMDevice
*kd
)
383 struct kvm_device_attr
*attr
= &kd
->kdattr
;
386 /* If the device control API is available and we have a device fd on the
387 * KVMDevice struct, let's use the newer API
389 if (kd
->dev_fd
>= 0) {
390 uint64_t addr
= kd
->kda
.addr
;
392 addr
|= kd
->kda_addr_ormask
;
393 attr
->addr
= (uintptr_t)&addr
;
394 ret
= kvm_device_ioctl(kd
->dev_fd
, KVM_SET_DEVICE_ATTR
, attr
);
396 ret
= kvm_vm_ioctl(kvm_state
, KVM_ARM_SET_DEVICE_ADDR
, &kd
->kda
);
400 fprintf(stderr
, "Failed to set device address: %s\n",
406 static void kvm_arm_machine_init_done(Notifier
*notifier
, void *data
)
410 QSLIST_FOREACH_SAFE(kd
, &kvm_devices_head
, entries
, tkd
) {
411 if (kd
->kda
.addr
!= -1) {
412 kvm_arm_set_device_addr(kd
);
414 memory_region_unref(kd
->mr
);
415 QSLIST_REMOVE_HEAD(&kvm_devices_head
, entries
);
418 memory_listener_unregister(&devlistener
);
421 static Notifier notify
= {
422 .notify
= kvm_arm_machine_init_done
,
425 void kvm_arm_register_device(MemoryRegion
*mr
, uint64_t devid
, uint64_t group
,
426 uint64_t attr
, int dev_fd
, uint64_t addr_ormask
)
430 if (!kvm_irqchip_in_kernel()) {
434 if (QSLIST_EMPTY(&kvm_devices_head
)) {
435 memory_listener_register(&devlistener
, &address_space_memory
);
436 qemu_add_machine_init_done_notifier(¬ify
);
438 kd
= g_new0(KVMDevice
, 1);
442 kd
->kdattr
.flags
= 0;
443 kd
->kdattr
.group
= group
;
444 kd
->kdattr
.attr
= attr
;
446 kd
->kda_addr_ormask
= addr_ormask
;
447 QSLIST_INSERT_HEAD(&kvm_devices_head
, kd
, entries
);
448 memory_region_ref(kd
->mr
);
451 static int compare_u64(const void *a
, const void *b
)
453 if (*(uint64_t *)a
> *(uint64_t *)b
) {
456 if (*(uint64_t *)a
< *(uint64_t *)b
) {
463 * cpreg_values are sorted in ascending order by KVM register ID
464 * (see kvm_arm_init_cpreg_list). This allows us to cheaply find
465 * the storage for a KVM register by ID with a binary search.
467 static uint64_t *kvm_arm_get_cpreg_ptr(ARMCPU
*cpu
, uint64_t regidx
)
471 res
= bsearch(®idx
, cpu
->cpreg_indexes
, cpu
->cpreg_array_len
,
472 sizeof(uint64_t), compare_u64
);
475 return &cpu
->cpreg_values
[res
- cpu
->cpreg_indexes
];
478 /* Initialize the ARMCPU cpreg list according to the kernel's
479 * definition of what CPU registers it knows about (and throw away
480 * the previous TCG-created cpreg list).
482 int kvm_arm_init_cpreg_list(ARMCPU
*cpu
)
484 struct kvm_reg_list rl
;
485 struct kvm_reg_list
*rlp
;
486 int i
, ret
, arraylen
;
487 CPUState
*cs
= CPU(cpu
);
490 ret
= kvm_vcpu_ioctl(cs
, KVM_GET_REG_LIST
, &rl
);
494 rlp
= g_malloc(sizeof(struct kvm_reg_list
) + rl
.n
* sizeof(uint64_t));
496 ret
= kvm_vcpu_ioctl(cs
, KVM_GET_REG_LIST
, rlp
);
500 /* Sort the list we get back from the kernel, since cpreg_tuples
501 * must be in strictly ascending order.
503 qsort(&rlp
->reg
, rlp
->n
, sizeof(rlp
->reg
[0]), compare_u64
);
505 for (i
= 0, arraylen
= 0; i
< rlp
->n
; i
++) {
506 if (!kvm_arm_reg_syncs_via_cpreg_list(rlp
->reg
[i
])) {
509 switch (rlp
->reg
[i
] & KVM_REG_SIZE_MASK
) {
510 case KVM_REG_SIZE_U32
:
511 case KVM_REG_SIZE_U64
:
514 fprintf(stderr
, "Can't handle size of register in kernel list\n");
522 cpu
->cpreg_indexes
= g_renew(uint64_t, cpu
->cpreg_indexes
, arraylen
);
523 cpu
->cpreg_values
= g_renew(uint64_t, cpu
->cpreg_values
, arraylen
);
524 cpu
->cpreg_vmstate_indexes
= g_renew(uint64_t, cpu
->cpreg_vmstate_indexes
,
526 cpu
->cpreg_vmstate_values
= g_renew(uint64_t, cpu
->cpreg_vmstate_values
,
528 cpu
->cpreg_array_len
= arraylen
;
529 cpu
->cpreg_vmstate_array_len
= arraylen
;
531 for (i
= 0, arraylen
= 0; i
< rlp
->n
; i
++) {
532 uint64_t regidx
= rlp
->reg
[i
];
533 if (!kvm_arm_reg_syncs_via_cpreg_list(regidx
)) {
536 cpu
->cpreg_indexes
[arraylen
] = regidx
;
539 assert(cpu
->cpreg_array_len
== arraylen
);
541 if (!write_kvmstate_to_list(cpu
)) {
542 /* Shouldn't happen unless kernel is inconsistent about
543 * what registers exist.
545 fprintf(stderr
, "Initial read of kernel register state failed\n");
555 bool write_kvmstate_to_list(ARMCPU
*cpu
)
557 CPUState
*cs
= CPU(cpu
);
561 for (i
= 0; i
< cpu
->cpreg_array_len
; i
++) {
562 uint64_t regidx
= cpu
->cpreg_indexes
[i
];
566 switch (regidx
& KVM_REG_SIZE_MASK
) {
567 case KVM_REG_SIZE_U32
:
568 ret
= kvm_get_one_reg(cs
, regidx
, &v32
);
570 cpu
->cpreg_values
[i
] = v32
;
573 case KVM_REG_SIZE_U64
:
574 ret
= kvm_get_one_reg(cs
, regidx
, cpu
->cpreg_values
+ i
);
577 g_assert_not_reached();
586 bool write_list_to_kvmstate(ARMCPU
*cpu
, int level
)
588 CPUState
*cs
= CPU(cpu
);
592 for (i
= 0; i
< cpu
->cpreg_array_len
; i
++) {
593 uint64_t regidx
= cpu
->cpreg_indexes
[i
];
597 if (kvm_arm_cpreg_level(regidx
) > level
) {
601 switch (regidx
& KVM_REG_SIZE_MASK
) {
602 case KVM_REG_SIZE_U32
:
603 v32
= cpu
->cpreg_values
[i
];
604 ret
= kvm_set_one_reg(cs
, regidx
, &v32
);
606 case KVM_REG_SIZE_U64
:
607 ret
= kvm_set_one_reg(cs
, regidx
, cpu
->cpreg_values
+ i
);
610 g_assert_not_reached();
613 /* We might fail for "unknown register" and also for
614 * "you tried to set a register which is constant with
615 * a different value from what it actually contains".
623 void kvm_arm_cpu_pre_save(ARMCPU
*cpu
)
625 /* KVM virtual time adjustment */
626 if (cpu
->kvm_vtime_dirty
) {
627 *kvm_arm_get_cpreg_ptr(cpu
, KVM_REG_ARM_TIMER_CNT
) = cpu
->kvm_vtime
;
631 void kvm_arm_cpu_post_load(ARMCPU
*cpu
)
633 /* KVM virtual time adjustment */
634 if (cpu
->kvm_adjvtime
) {
635 cpu
->kvm_vtime
= *kvm_arm_get_cpreg_ptr(cpu
, KVM_REG_ARM_TIMER_CNT
);
636 cpu
->kvm_vtime_dirty
= true;
640 void kvm_arm_reset_vcpu(ARMCPU
*cpu
)
644 /* Re-init VCPU so that all registers are set to
645 * their respective reset values.
647 ret
= kvm_arm_vcpu_init(CPU(cpu
));
649 fprintf(stderr
, "kvm_arm_vcpu_init failed: %s\n", strerror(-ret
));
652 if (!write_kvmstate_to_list(cpu
)) {
653 fprintf(stderr
, "write_kvmstate_to_list failed\n");
657 * Sync the reset values also into the CPUState. This is necessary
658 * because the next thing we do will be a kvm_arch_put_registers()
659 * which will update the list values from the CPUState before copying
660 * the list values back to KVM. It's OK to ignore failure returns here
661 * for the same reason we do so in kvm_arch_get_registers().
663 write_list_to_cpustate(cpu
);
667 * Update KVM's MP_STATE based on what QEMU thinks it is
669 int kvm_arm_sync_mpstate_to_kvm(ARMCPU
*cpu
)
671 if (cap_has_mp_state
) {
672 struct kvm_mp_state mp_state
= {
673 .mp_state
= (cpu
->power_state
== PSCI_OFF
) ?
674 KVM_MP_STATE_STOPPED
: KVM_MP_STATE_RUNNABLE
676 int ret
= kvm_vcpu_ioctl(CPU(cpu
), KVM_SET_MP_STATE
, &mp_state
);
678 fprintf(stderr
, "%s: failed to set MP_STATE %d/%s\n",
679 __func__
, ret
, strerror(-ret
));
688 * Sync the KVM MP_STATE into QEMU
690 int kvm_arm_sync_mpstate_to_qemu(ARMCPU
*cpu
)
692 if (cap_has_mp_state
) {
693 struct kvm_mp_state mp_state
;
694 int ret
= kvm_vcpu_ioctl(CPU(cpu
), KVM_GET_MP_STATE
, &mp_state
);
696 fprintf(stderr
, "%s: failed to get MP_STATE %d/%s\n",
697 __func__
, ret
, strerror(-ret
));
700 cpu
->power_state
= (mp_state
.mp_state
== KVM_MP_STATE_STOPPED
) ?
707 void kvm_arm_get_virtual_time(CPUState
*cs
)
709 ARMCPU
*cpu
= ARM_CPU(cs
);
712 if (cpu
->kvm_vtime_dirty
) {
716 ret
= kvm_get_one_reg(cs
, KVM_REG_ARM_TIMER_CNT
, &cpu
->kvm_vtime
);
718 error_report("Failed to get KVM_REG_ARM_TIMER_CNT");
722 cpu
->kvm_vtime_dirty
= true;
725 void kvm_arm_put_virtual_time(CPUState
*cs
)
727 ARMCPU
*cpu
= ARM_CPU(cs
);
730 if (!cpu
->kvm_vtime_dirty
) {
734 ret
= kvm_set_one_reg(cs
, KVM_REG_ARM_TIMER_CNT
, &cpu
->kvm_vtime
);
736 error_report("Failed to set KVM_REG_ARM_TIMER_CNT");
740 cpu
->kvm_vtime_dirty
= false;
743 int kvm_put_vcpu_events(ARMCPU
*cpu
)
745 CPUARMState
*env
= &cpu
->env
;
746 struct kvm_vcpu_events events
;
749 if (!kvm_has_vcpu_events()) {
753 memset(&events
, 0, sizeof(events
));
754 events
.exception
.serror_pending
= env
->serror
.pending
;
756 /* Inject SError to guest with specified syndrome if host kernel
757 * supports it, otherwise inject SError without syndrome.
759 if (cap_has_inject_serror_esr
) {
760 events
.exception
.serror_has_esr
= env
->serror
.has_esr
;
761 events
.exception
.serror_esr
= env
->serror
.esr
;
764 ret
= kvm_vcpu_ioctl(CPU(cpu
), KVM_SET_VCPU_EVENTS
, &events
);
766 error_report("failed to put vcpu events");
772 int kvm_get_vcpu_events(ARMCPU
*cpu
)
774 CPUARMState
*env
= &cpu
->env
;
775 struct kvm_vcpu_events events
;
778 if (!kvm_has_vcpu_events()) {
782 memset(&events
, 0, sizeof(events
));
783 ret
= kvm_vcpu_ioctl(CPU(cpu
), KVM_GET_VCPU_EVENTS
, &events
);
785 error_report("failed to get vcpu events");
789 env
->serror
.pending
= events
.exception
.serror_pending
;
790 env
->serror
.has_esr
= events
.exception
.serror_has_esr
;
791 env
->serror
.esr
= events
.exception
.serror_esr
;
796 #define ARM64_REG_ESR_EL1 ARM64_SYS_REG(3, 0, 5, 2, 0)
797 #define ARM64_REG_TCR_EL1 ARM64_SYS_REG(3, 0, 2, 0, 2)
802 * AARCH64: DFSC, bits [5:0]
806 * FS[3:0] - DFSR[3:0]
810 #define ESR_DFSC(aarch64, lpae, v) \
811 ((aarch64 || (lpae)) ? ((v) & 0x3F) \
812 : (((v) >> 6) | ((v) & 0x1F)))
814 #define ESR_DFSC_EXTABT(aarch64, lpae) \
815 ((aarch64) ? 0x10 : (lpae) ? 0x10 : 0x8)
818 * kvm_arm_verify_ext_dabt_pending:
821 * Verify the fault status code wrt the Ext DABT injection
823 * Returns: true if the fault status code is as expected, false otherwise
825 static bool kvm_arm_verify_ext_dabt_pending(CPUState
*cs
)
829 if (!kvm_get_one_reg(cs
, ARM64_REG_ESR_EL1
, &dfsr_val
)) {
830 ARMCPU
*cpu
= ARM_CPU(cs
);
831 CPUARMState
*env
= &cpu
->env
;
832 int aarch64_mode
= arm_feature(env
, ARM_FEATURE_AARCH64
);
838 if (!kvm_get_one_reg(cs
, ARM64_REG_TCR_EL1
, &ttbcr
)) {
839 lpae
= arm_feature(env
, ARM_FEATURE_LPAE
)
840 && (ttbcr
& TTBCR_EAE
);
844 * The verification here is based on the DFSC bits
845 * of the ESR_EL1 reg only
847 return (ESR_DFSC(aarch64_mode
, lpae
, dfsr_val
) ==
848 ESR_DFSC_EXTABT(aarch64_mode
, lpae
));
853 void kvm_arch_pre_run(CPUState
*cs
, struct kvm_run
*run
)
855 ARMCPU
*cpu
= ARM_CPU(cs
);
856 CPUARMState
*env
= &cpu
->env
;
858 if (unlikely(env
->ext_dabt_raised
)) {
860 * Verifying that the ext DABT has been properly injected,
861 * otherwise risking indefinitely re-running the faulting instruction
862 * Covering a very narrow case for kernels 5.5..5.5.4
863 * when injected abort was misconfigured to be
864 * an IMPLEMENTATION DEFINED exception (for 32-bit EL1)
866 if (!arm_feature(env
, ARM_FEATURE_AARCH64
) &&
867 unlikely(!kvm_arm_verify_ext_dabt_pending(cs
))) {
869 error_report("Data abort exception with no valid ISS generated by "
870 "guest memory access. KVM unable to emulate faulting "
871 "instruction. Failed to inject an external data abort "
875 /* Clear the status */
876 env
->ext_dabt_raised
= 0;
880 MemTxAttrs
kvm_arch_post_run(CPUState
*cs
, struct kvm_run
*run
)
883 uint32_t switched_level
;
885 if (kvm_irqchip_in_kernel()) {
887 * We only need to sync timer states with user-space interrupt
888 * controllers, so return early and save cycles if we don't.
890 return MEMTXATTRS_UNSPECIFIED
;
895 /* Synchronize our shadowed in-kernel device irq lines with the kvm ones */
896 if (run
->s
.regs
.device_irq_level
!= cpu
->device_irq_level
) {
897 switched_level
= cpu
->device_irq_level
^ run
->s
.regs
.device_irq_level
;
899 qemu_mutex_lock_iothread();
901 if (switched_level
& KVM_ARM_DEV_EL1_VTIMER
) {
902 qemu_set_irq(cpu
->gt_timer_outputs
[GTIMER_VIRT
],
903 !!(run
->s
.regs
.device_irq_level
&
904 KVM_ARM_DEV_EL1_VTIMER
));
905 switched_level
&= ~KVM_ARM_DEV_EL1_VTIMER
;
908 if (switched_level
& KVM_ARM_DEV_EL1_PTIMER
) {
909 qemu_set_irq(cpu
->gt_timer_outputs
[GTIMER_PHYS
],
910 !!(run
->s
.regs
.device_irq_level
&
911 KVM_ARM_DEV_EL1_PTIMER
));
912 switched_level
&= ~KVM_ARM_DEV_EL1_PTIMER
;
915 if (switched_level
& KVM_ARM_DEV_PMU
) {
916 qemu_set_irq(cpu
->pmu_interrupt
,
917 !!(run
->s
.regs
.device_irq_level
& KVM_ARM_DEV_PMU
));
918 switched_level
&= ~KVM_ARM_DEV_PMU
;
921 if (switched_level
) {
922 qemu_log_mask(LOG_UNIMP
, "%s: unhandled in-kernel device IRQ %x\n",
923 __func__
, switched_level
);
926 /* We also mark unknown levels as processed to not waste cycles */
927 cpu
->device_irq_level
= run
->s
.regs
.device_irq_level
;
928 qemu_mutex_unlock_iothread();
931 return MEMTXATTRS_UNSPECIFIED
;
934 void kvm_arm_vm_state_change(void *opaque
, bool running
, RunState state
)
936 CPUState
*cs
= opaque
;
937 ARMCPU
*cpu
= ARM_CPU(cs
);
940 if (cpu
->kvm_adjvtime
) {
941 kvm_arm_put_virtual_time(cs
);
944 if (cpu
->kvm_adjvtime
) {
945 kvm_arm_get_virtual_time(cs
);
951 * kvm_arm_handle_dabt_nisv:
953 * @esr_iss: ISS encoding (limited) for the exception from Data Abort
954 * ISV bit set to '0b0' -> no valid instruction syndrome
955 * @fault_ipa: faulting address for the synchronous data abort
957 * Returns: 0 if the exception has been handled, < 0 otherwise
959 static int kvm_arm_handle_dabt_nisv(CPUState
*cs
, uint64_t esr_iss
,
962 ARMCPU
*cpu
= ARM_CPU(cs
);
963 CPUARMState
*env
= &cpu
->env
;
965 * Request KVM to inject the external data abort into the guest
967 if (cap_has_inject_ext_dabt
) {
968 struct kvm_vcpu_events events
= { };
970 * The external data abort event will be handled immediately by KVM
971 * using the address fault that triggered the exit on given VCPU.
972 * Requesting injection of the external data abort does not rely
973 * on any other VCPU state. Therefore, in this particular case, the VCPU
974 * synchronization can be exceptionally skipped.
976 events
.exception
.ext_dabt_pending
= 1;
977 /* KVM_CAP_ARM_INJECT_EXT_DABT implies KVM_CAP_VCPU_EVENTS */
978 if (!kvm_vcpu_ioctl(cs
, KVM_SET_VCPU_EVENTS
, &events
)) {
979 env
->ext_dabt_raised
= 1;
983 error_report("Data abort exception triggered by guest memory access "
984 "at physical address: 0x" TARGET_FMT_lx
,
985 (target_ulong
)fault_ipa
);
986 error_printf("KVM unable to emulate faulting instruction.\n");
992 * kvm_arm_handle_debug:
994 * @debug_exit: debug part of the KVM exit structure
996 * Returns: TRUE if the debug exception was handled.
998 * See v8 ARM ARM D7.2.27 ESR_ELx, Exception Syndrome Register
1000 * To minimise translating between kernel and user-space the kernel
1001 * ABI just provides user-space with the full exception syndrome
1002 * register value to be decoded in QEMU.
1004 static bool kvm_arm_handle_debug(CPUState
*cs
,
1005 struct kvm_debug_exit_arch
*debug_exit
)
1007 int hsr_ec
= syn_get_ec(debug_exit
->hsr
);
1008 ARMCPU
*cpu
= ARM_CPU(cs
);
1009 CPUARMState
*env
= &cpu
->env
;
1011 /* Ensure PC is synchronised */
1012 kvm_cpu_synchronize_state(cs
);
1015 case EC_SOFTWARESTEP
:
1016 if (cs
->singlestep_enabled
) {
1020 * The kernel should have suppressed the guest's ability to
1021 * single step at this point so something has gone wrong.
1023 error_report("%s: guest single-step while debugging unsupported"
1024 " (%"PRIx64
", %"PRIx32
")",
1025 __func__
, env
->pc
, debug_exit
->hsr
);
1030 if (kvm_find_sw_breakpoint(cs
, env
->pc
)) {
1035 if (find_hw_breakpoint(cs
, env
->pc
)) {
1041 CPUWatchpoint
*wp
= find_hw_watchpoint(cs
, debug_exit
->far
);
1043 cs
->watchpoint_hit
= wp
;
1049 error_report("%s: unhandled debug exit (%"PRIx32
", %"PRIx64
")",
1050 __func__
, debug_exit
->hsr
, env
->pc
);
1053 /* If we are not handling the debug exception it must belong to
1054 * the guest. Let's re-use the existing TCG interrupt code to set
1055 * everything up properly.
1057 cs
->exception_index
= EXCP_BKPT
;
1058 env
->exception
.syndrome
= debug_exit
->hsr
;
1059 env
->exception
.vaddress
= debug_exit
->far
;
1060 env
->exception
.target_el
= 1;
1061 qemu_mutex_lock_iothread();
1062 arm_cpu_do_interrupt(cs
);
1063 qemu_mutex_unlock_iothread();
1068 int kvm_arch_handle_exit(CPUState
*cs
, struct kvm_run
*run
)
1072 switch (run
->exit_reason
) {
1073 case KVM_EXIT_DEBUG
:
1074 if (kvm_arm_handle_debug(cs
, &run
->debug
.arch
)) {
1076 } /* otherwise return to guest */
1078 case KVM_EXIT_ARM_NISV
:
1079 /* External DABT with no valid iss to decode */
1080 ret
= kvm_arm_handle_dabt_nisv(cs
, run
->arm_nisv
.esr_iss
,
1081 run
->arm_nisv
.fault_ipa
);
1084 qemu_log_mask(LOG_UNIMP
, "%s: un-handled exit reason %d\n",
1085 __func__
, run
->exit_reason
);
1091 bool kvm_arch_stop_on_emulation_error(CPUState
*cs
)
1096 int kvm_arch_process_async_events(CPUState
*cs
)
1102 * kvm_arm_hw_debug_active:
1105 * Return: TRUE if any hardware breakpoints in use.
1107 static bool kvm_arm_hw_debug_active(CPUState
*cs
)
1109 return ((cur_hw_wps
> 0) || (cur_hw_bps
> 0));
1113 * kvm_arm_copy_hw_debug_data:
1114 * @ptr: kvm_guest_debug_arch structure
1116 * Copy the architecture specific debug registers into the
1117 * kvm_guest_debug ioctl structure.
1119 static void kvm_arm_copy_hw_debug_data(struct kvm_guest_debug_arch
*ptr
)
1122 memset(ptr
, 0, sizeof(struct kvm_guest_debug_arch
));
1124 for (i
= 0; i
< max_hw_wps
; i
++) {
1125 HWWatchpoint
*wp
= get_hw_wp(i
);
1126 ptr
->dbg_wcr
[i
] = wp
->wcr
;
1127 ptr
->dbg_wvr
[i
] = wp
->wvr
;
1129 for (i
= 0; i
< max_hw_bps
; i
++) {
1130 HWBreakpoint
*bp
= get_hw_bp(i
);
1131 ptr
->dbg_bcr
[i
] = bp
->bcr
;
1132 ptr
->dbg_bvr
[i
] = bp
->bvr
;
1136 void kvm_arch_update_guest_debug(CPUState
*cs
, struct kvm_guest_debug
*dbg
)
1138 if (kvm_sw_breakpoints_active(cs
)) {
1139 dbg
->control
|= KVM_GUESTDBG_ENABLE
| KVM_GUESTDBG_USE_SW_BP
;
1141 if (kvm_arm_hw_debug_active(cs
)) {
1142 dbg
->control
|= KVM_GUESTDBG_ENABLE
| KVM_GUESTDBG_USE_HW
;
1143 kvm_arm_copy_hw_debug_data(&dbg
->arch
);
1147 void kvm_arch_init_irq_routing(KVMState
*s
)
1151 int kvm_arch_irqchip_create(KVMState
*s
)
1153 if (kvm_kernel_irqchip_split()) {
1154 error_report("-machine kernel_irqchip=split is not supported on ARM.");
1158 /* If we can create the VGIC using the newer device control API, we
1159 * let the device do this when it initializes itself, otherwise we
1160 * fall back to the old API */
1161 return kvm_check_extension(s
, KVM_CAP_DEVICE_CTRL
);
1164 int kvm_arm_vgic_probe(void)
1168 if (kvm_create_device(kvm_state
,
1169 KVM_DEV_TYPE_ARM_VGIC_V3
, true) == 0) {
1170 val
|= KVM_ARM_VGIC_V3
;
1172 if (kvm_create_device(kvm_state
,
1173 KVM_DEV_TYPE_ARM_VGIC_V2
, true) == 0) {
1174 val
|= KVM_ARM_VGIC_V2
;
1179 int kvm_arm_set_irq(int cpu
, int irqtype
, int irq
, int level
)
1181 int kvm_irq
= (irqtype
<< KVM_ARM_IRQ_TYPE_SHIFT
) | irq
;
1182 int cpu_idx1
= cpu
% 256;
1183 int cpu_idx2
= cpu
/ 256;
1185 kvm_irq
|= (cpu_idx1
<< KVM_ARM_IRQ_VCPU_SHIFT
) |
1186 (cpu_idx2
<< KVM_ARM_IRQ_VCPU2_SHIFT
);
1188 return kvm_set_irq(kvm_state
, kvm_irq
, !!level
);
1191 int kvm_arch_fixup_msi_route(struct kvm_irq_routing_entry
*route
,
1192 uint64_t address
, uint32_t data
, PCIDevice
*dev
)
1194 AddressSpace
*as
= pci_device_iommu_address_space(dev
);
1195 hwaddr xlat
, len
, doorbell_gpa
;
1196 MemoryRegionSection mrs
;
1199 if (as
== &address_space_memory
) {
1203 /* MSI doorbell address is translated by an IOMMU */
1205 RCU_READ_LOCK_GUARD();
1207 mr
= address_space_translate(as
, address
, &xlat
, &len
, true,
1208 MEMTXATTRS_UNSPECIFIED
);
1214 mrs
= memory_region_find(mr
, xlat
, 1);
1220 doorbell_gpa
= mrs
.offset_within_address_space
;
1221 memory_region_unref(mrs
.mr
);
1223 route
->u
.msi
.address_lo
= doorbell_gpa
;
1224 route
->u
.msi
.address_hi
= doorbell_gpa
>> 32;
1226 trace_kvm_arm_fixup_msi_route(address
, doorbell_gpa
);
1231 int kvm_arch_add_msi_route_post(struct kvm_irq_routing_entry
*route
,
1232 int vector
, PCIDevice
*dev
)
1237 int kvm_arch_release_virq_post(int virq
)
1242 int kvm_arch_msi_data_to_gsi(uint32_t data
)
1244 return (data
- 32) & 0xffff;
1247 bool kvm_arch_cpu_check_are_resettable(void)
1252 static void kvm_arch_get_eager_split_size(Object
*obj
, Visitor
*v
,
1253 const char *name
, void *opaque
,
1256 KVMState
*s
= KVM_STATE(obj
);
1257 uint64_t value
= s
->kvm_eager_split_size
;
1259 visit_type_size(v
, name
, &value
, errp
);
1262 static void kvm_arch_set_eager_split_size(Object
*obj
, Visitor
*v
,
1263 const char *name
, void *opaque
,
1266 KVMState
*s
= KVM_STATE(obj
);
1270 error_setg(errp
, "Unable to set early-split-size after KVM has been initialized");
1274 if (!visit_type_size(v
, name
, &value
, errp
)) {
1278 if (value
&& !is_power_of_2(value
)) {
1279 error_setg(errp
, "early-split-size must be a power of two");
1283 s
->kvm_eager_split_size
= value
;
1286 void kvm_arch_accel_class_init(ObjectClass
*oc
)
1288 object_class_property_add(oc
, "eager-split-size", "size",
1289 kvm_arch_get_eager_split_size
,
1290 kvm_arch_set_eager_split_size
, NULL
, NULL
);
1292 object_class_property_set_description(oc
, "eager-split-size",
1293 "Eager Page Split chunk size for hugepages. (default: 0, disabled)");