4 * Copyright IBM, Corp. 2008
8 * Anthony Liguori <aliguori@us.ibm.com>
9 * Glauber Costa <gcosta@redhat.com>
11 * This work is licensed under the terms of the GNU GPL, version 2 or later.
12 * See the COPYING file in the top-level directory.
16 #include <sys/types.h>
17 #include <sys/ioctl.h>
21 #include <linux/kvm.h>
23 #include "qemu-common.h"
28 /* KVM uses PAGE_SIZE in it's definition of COALESCED_MMIO_MAX */
29 #define PAGE_SIZE TARGET_PAGE_SIZE
34 #define dprintf(fmt, ...) \
35 do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
37 #define dprintf(fmt, ...) \
41 typedef struct KVMSlot
43 target_phys_addr_t start_addr
;
44 ram_addr_t memory_size
;
45 ram_addr_t phys_offset
;
50 typedef struct kvm_dirty_log KVMDirtyLog
;
60 #ifdef KVM_CAP_SET_GUEST_DEBUG
61 struct kvm_sw_breakpoint_head kvm_sw_breakpoints
;
65 static KVMState
*kvm_state
;
67 static KVMSlot
*kvm_alloc_slot(KVMState
*s
)
71 for (i
= 0; i
< ARRAY_SIZE(s
->slots
); i
++) {
72 /* KVM private memory slots */
75 if (s
->slots
[i
].memory_size
== 0)
79 fprintf(stderr
, "%s: no free slot available\n", __func__
);
83 static KVMSlot
*kvm_lookup_matching_slot(KVMState
*s
,
84 target_phys_addr_t start_addr
,
85 target_phys_addr_t end_addr
)
89 for (i
= 0; i
< ARRAY_SIZE(s
->slots
); i
++) {
90 KVMSlot
*mem
= &s
->slots
[i
];
92 if (start_addr
== mem
->start_addr
&&
93 end_addr
== mem
->start_addr
+ mem
->memory_size
) {
101 static KVMSlot
*kvm_lookup_slot(KVMState
*s
, target_phys_addr_t start_addr
)
105 for (i
= 0; i
< ARRAY_SIZE(s
->slots
); i
++) {
106 KVMSlot
*mem
= &s
->slots
[i
];
108 if (start_addr
>= mem
->start_addr
&&
109 start_addr
< (mem
->start_addr
+ mem
->memory_size
))
116 static int kvm_set_user_memory_region(KVMState
*s
, KVMSlot
*slot
)
118 struct kvm_userspace_memory_region mem
;
120 mem
.slot
= slot
->slot
;
121 mem
.guest_phys_addr
= slot
->start_addr
;
122 mem
.memory_size
= slot
->memory_size
;
123 mem
.userspace_addr
= (unsigned long)qemu_get_ram_ptr(slot
->phys_offset
);
124 mem
.flags
= slot
->flags
;
126 return kvm_vm_ioctl(s
, KVM_SET_USER_MEMORY_REGION
, &mem
);
130 int kvm_init_vcpu(CPUState
*env
)
132 KVMState
*s
= kvm_state
;
136 dprintf("kvm_init_vcpu\n");
138 ret
= kvm_vm_ioctl(s
, KVM_CREATE_VCPU
, env
->cpu_index
);
140 dprintf("kvm_create_vcpu failed\n");
147 mmap_size
= kvm_ioctl(s
, KVM_GET_VCPU_MMAP_SIZE
, 0);
149 dprintf("KVM_GET_VCPU_MMAP_SIZE failed\n");
153 env
->kvm_run
= mmap(NULL
, mmap_size
, PROT_READ
| PROT_WRITE
, MAP_SHARED
,
155 if (env
->kvm_run
== MAP_FAILED
) {
157 dprintf("mmap'ing vcpu state failed\n");
161 ret
= kvm_arch_init_vcpu(env
);
167 int kvm_sync_vcpus(void)
171 for (env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
174 ret
= kvm_arch_put_registers(env
);
183 * dirty pages logging control
185 static int kvm_dirty_pages_log_change(target_phys_addr_t phys_addr
,
186 ram_addr_t size
, unsigned flags
,
189 KVMState
*s
= kvm_state
;
190 KVMSlot
*mem
= kvm_lookup_matching_slot(s
, phys_addr
, phys_addr
+ size
);
192 fprintf(stderr
, "BUG: %s: invalid parameters " TARGET_FMT_plx
"-"
193 TARGET_FMT_plx
"\n", __func__
, phys_addr
,
194 phys_addr
+ size
- 1);
198 flags
= (mem
->flags
& ~mask
) | flags
;
199 /* Nothing changed, no need to issue ioctl */
200 if (flags
== mem
->flags
)
205 return kvm_set_user_memory_region(s
, mem
);
208 int kvm_log_start(target_phys_addr_t phys_addr
, ram_addr_t size
)
210 return kvm_dirty_pages_log_change(phys_addr
, size
,
211 KVM_MEM_LOG_DIRTY_PAGES
,
212 KVM_MEM_LOG_DIRTY_PAGES
);
215 int kvm_log_stop(target_phys_addr_t phys_addr
, ram_addr_t size
)
217 return kvm_dirty_pages_log_change(phys_addr
, size
,
219 KVM_MEM_LOG_DIRTY_PAGES
);
223 * kvm_physical_sync_dirty_bitmap - Grab dirty bitmap from kernel space
224 * This function updates qemu's dirty bitmap using cpu_physical_memory_set_dirty().
225 * This means all bits are set to dirty.
227 * @start_add: start of logged region.
228 * @end_addr: end of logged region.
230 void kvm_physical_sync_dirty_bitmap(target_phys_addr_t start_addr
,
231 target_phys_addr_t end_addr
)
233 KVMState
*s
= kvm_state
;
235 KVMSlot
*mem
= kvm_lookup_matching_slot(s
, start_addr
, end_addr
);
236 unsigned long alloc_size
;
238 target_phys_addr_t phys_addr
= start_addr
;
240 dprintf("sync addr: " TARGET_FMT_lx
" into %lx\n", start_addr
,
243 fprintf(stderr
, "BUG: %s: invalid parameters " TARGET_FMT_plx
"-"
244 TARGET_FMT_plx
"\n", __func__
, phys_addr
, end_addr
- 1);
248 alloc_size
= mem
->memory_size
>> TARGET_PAGE_BITS
/ sizeof(d
.dirty_bitmap
);
249 d
.dirty_bitmap
= qemu_mallocz(alloc_size
);
252 dprintf("slot %d, phys_addr %llx, uaddr: %llx\n",
253 d
.slot
, mem
->start_addr
, mem
->phys_offset
);
255 if (kvm_vm_ioctl(s
, KVM_GET_DIRTY_LOG
, &d
) == -1) {
256 dprintf("ioctl failed %d\n", errno
);
260 phys_addr
= start_addr
;
261 for (addr
= mem
->phys_offset
; phys_addr
< end_addr
; phys_addr
+= TARGET_PAGE_SIZE
, addr
+= TARGET_PAGE_SIZE
) {
262 unsigned long *bitmap
= (unsigned long *)d
.dirty_bitmap
;
263 unsigned nr
= (phys_addr
- start_addr
) >> TARGET_PAGE_BITS
;
264 unsigned word
= nr
/ (sizeof(*bitmap
) * 8);
265 unsigned bit
= nr
% (sizeof(*bitmap
) * 8);
266 if ((bitmap
[word
] >> bit
) & 1)
267 cpu_physical_memory_set_dirty(addr
);
270 qemu_free(d
.dirty_bitmap
);
273 int kvm_coalesce_mmio_region(target_phys_addr_t start
, ram_addr_t size
)
276 #ifdef KVM_CAP_COALESCED_MMIO
277 KVMState
*s
= kvm_state
;
279 if (s
->coalesced_mmio
) {
280 struct kvm_coalesced_mmio_zone zone
;
285 ret
= kvm_vm_ioctl(s
, KVM_REGISTER_COALESCED_MMIO
, &zone
);
292 int kvm_uncoalesce_mmio_region(target_phys_addr_t start
, ram_addr_t size
)
295 #ifdef KVM_CAP_COALESCED_MMIO
296 KVMState
*s
= kvm_state
;
298 if (s
->coalesced_mmio
) {
299 struct kvm_coalesced_mmio_zone zone
;
304 ret
= kvm_vm_ioctl(s
, KVM_UNREGISTER_COALESCED_MMIO
, &zone
);
311 int kvm_init(int smp_cpus
)
320 s
= qemu_mallocz(sizeof(KVMState
));
322 #ifdef KVM_CAP_SET_GUEST_DEBUG
323 TAILQ_INIT(&s
->kvm_sw_breakpoints
);
325 for (i
= 0; i
< ARRAY_SIZE(s
->slots
); i
++)
326 s
->slots
[i
].slot
= i
;
329 s
->fd
= open("/dev/kvm", O_RDWR
);
331 fprintf(stderr
, "Could not access KVM kernel module: %m\n");
336 ret
= kvm_ioctl(s
, KVM_GET_API_VERSION
, 0);
337 if (ret
< KVM_API_VERSION
) {
340 fprintf(stderr
, "kvm version too old\n");
344 if (ret
> KVM_API_VERSION
) {
346 fprintf(stderr
, "kvm version not supported\n");
350 s
->vmfd
= kvm_ioctl(s
, KVM_CREATE_VM
, 0);
354 /* initially, KVM allocated its own memory and we had to jump through
355 * hooks to make phys_ram_base point to this. Modern versions of KVM
356 * just use a user allocated buffer so we can use regular pages
357 * unmodified. Make sure we have a sufficiently modern version of KVM.
359 ret
= kvm_ioctl(s
, KVM_CHECK_EXTENSION
, KVM_CAP_USER_MEMORY
);
363 fprintf(stderr
, "kvm does not support KVM_CAP_USER_MEMORY\n");
367 /* There was a nasty bug in < kvm-80 that prevents memory slots from being
368 * destroyed properly. Since we rely on this capability, refuse to work
369 * with any kernel without this capability. */
370 ret
= kvm_ioctl(s
, KVM_CHECK_EXTENSION
,
371 KVM_CAP_DESTROY_MEMORY_REGION_WORKS
);
377 "KVM kernel module broken (DESTROY_MEMORY_REGION)\n"
378 "Please upgrade to at least kvm-81.\n");
382 s
->coalesced_mmio
= 0;
383 #ifdef KVM_CAP_COALESCED_MMIO
384 ret
= kvm_ioctl(s
, KVM_CHECK_EXTENSION
, KVM_CAP_COALESCED_MMIO
);
386 s
->coalesced_mmio
= ret
;
389 ret
= kvm_arch_init(s
, smp_cpus
);
409 static int kvm_handle_io(CPUState
*env
, uint16_t port
, void *data
,
410 int direction
, int size
, uint32_t count
)
415 for (i
= 0; i
< count
; i
++) {
416 if (direction
== KVM_EXIT_IO_IN
) {
419 stb_p(ptr
, cpu_inb(env
, port
));
422 stw_p(ptr
, cpu_inw(env
, port
));
425 stl_p(ptr
, cpu_inl(env
, port
));
431 cpu_outb(env
, port
, ldub_p(ptr
));
434 cpu_outw(env
, port
, lduw_p(ptr
));
437 cpu_outl(env
, port
, ldl_p(ptr
));
448 static void kvm_run_coalesced_mmio(CPUState
*env
, struct kvm_run
*run
)
450 #ifdef KVM_CAP_COALESCED_MMIO
451 KVMState
*s
= kvm_state
;
452 if (s
->coalesced_mmio
) {
453 struct kvm_coalesced_mmio_ring
*ring
;
455 ring
= (void *)run
+ (s
->coalesced_mmio
* TARGET_PAGE_SIZE
);
456 while (ring
->first
!= ring
->last
) {
457 struct kvm_coalesced_mmio
*ent
;
459 ent
= &ring
->coalesced_mmio
[ring
->first
];
461 cpu_physical_memory_write(ent
->phys_addr
, ent
->data
, ent
->len
);
462 /* FIXME smp_wmb() */
463 ring
->first
= (ring
->first
+ 1) % KVM_COALESCED_MMIO_MAX
;
469 int kvm_cpu_exec(CPUState
*env
)
471 struct kvm_run
*run
= env
->kvm_run
;
474 dprintf("kvm_cpu_exec()\n");
477 kvm_arch_pre_run(env
, run
);
479 if (env
->exit_request
) {
480 dprintf("interrupt exit requested\n");
485 ret
= kvm_vcpu_ioctl(env
, KVM_RUN
, 0);
486 kvm_arch_post_run(env
, run
);
488 if (ret
== -EINTR
|| ret
== -EAGAIN
) {
489 dprintf("io window exit\n");
495 dprintf("kvm run failed %s\n", strerror(-ret
));
499 kvm_run_coalesced_mmio(env
, run
);
501 ret
= 0; /* exit loop */
502 switch (run
->exit_reason
) {
504 dprintf("handle_io\n");
505 ret
= kvm_handle_io(env
, run
->io
.port
,
506 (uint8_t *)run
+ run
->io
.data_offset
,
512 dprintf("handle_mmio\n");
513 cpu_physical_memory_rw(run
->mmio
.phys_addr
,
519 case KVM_EXIT_IRQ_WINDOW_OPEN
:
520 dprintf("irq_window_open\n");
522 case KVM_EXIT_SHUTDOWN
:
523 dprintf("shutdown\n");
524 qemu_system_reset_request();
527 case KVM_EXIT_UNKNOWN
:
528 dprintf("kvm_exit_unknown\n");
530 case KVM_EXIT_FAIL_ENTRY
:
531 dprintf("kvm_exit_fail_entry\n");
533 case KVM_EXIT_EXCEPTION
:
534 dprintf("kvm_exit_exception\n");
537 dprintf("kvm_exit_debug\n");
538 #ifdef KVM_CAP_SET_GUEST_DEBUG
539 if (kvm_arch_debug(&run
->debug
.arch
)) {
540 gdb_set_stop_cpu(env
);
542 env
->exception_index
= EXCP_DEBUG
;
545 /* re-enter, this exception was guest-internal */
547 #endif /* KVM_CAP_SET_GUEST_DEBUG */
550 dprintf("kvm_arch_handle_exit\n");
551 ret
= kvm_arch_handle_exit(env
, run
);
556 if (env
->exit_request
) {
557 env
->exit_request
= 0;
558 env
->exception_index
= EXCP_INTERRUPT
;
564 void kvm_set_phys_mem(target_phys_addr_t start_addr
,
566 ram_addr_t phys_offset
)
568 KVMState
*s
= kvm_state
;
569 ram_addr_t flags
= phys_offset
& ~TARGET_PAGE_MASK
;
572 if (start_addr
& ~TARGET_PAGE_MASK
) {
573 fprintf(stderr
, "Only page-aligned memory slots supported\n");
577 /* KVM does not support read-only slots */
578 phys_offset
&= ~IO_MEM_ROM
;
580 mem
= kvm_lookup_slot(s
, start_addr
);
582 if (flags
>= IO_MEM_UNASSIGNED
) {
583 mem
->memory_size
= 0;
584 mem
->start_addr
= start_addr
;
585 mem
->phys_offset
= 0;
588 kvm_set_user_memory_region(s
, mem
);
589 } else if (start_addr
>= mem
->start_addr
&&
590 (start_addr
+ size
) <= (mem
->start_addr
+
593 target_phys_addr_t mem_start
;
594 ram_addr_t mem_size
, mem_offset
;
597 if ((phys_offset
- (start_addr
- mem
->start_addr
)) ==
601 /* unregister whole slot */
602 memcpy(&slot
, mem
, sizeof(slot
));
603 mem
->memory_size
= 0;
604 kvm_set_user_memory_region(s
, mem
);
606 /* register prefix slot */
607 mem_start
= slot
.start_addr
;
608 mem_size
= start_addr
- slot
.start_addr
;
609 mem_offset
= slot
.phys_offset
;
611 kvm_set_phys_mem(mem_start
, mem_size
, mem_offset
);
613 /* register new slot */
614 kvm_set_phys_mem(start_addr
, size
, phys_offset
);
616 /* register suffix slot */
617 mem_start
= start_addr
+ size
;
618 mem_offset
+= mem_size
+ size
;
619 mem_size
= slot
.memory_size
- mem_size
- size
;
621 kvm_set_phys_mem(mem_start
, mem_size
, mem_offset
);
625 printf("Registering overlapping slot\n");
629 /* KVM does not need to know about this memory */
630 if (flags
>= IO_MEM_UNASSIGNED
)
633 mem
= kvm_alloc_slot(s
);
634 mem
->memory_size
= size
;
635 mem
->start_addr
= start_addr
;
636 mem
->phys_offset
= phys_offset
;
639 kvm_set_user_memory_region(s
, mem
);
640 /* FIXME deal with errors */
643 int kvm_ioctl(KVMState
*s
, int type
, ...)
650 arg
= va_arg(ap
, void *);
653 ret
= ioctl(s
->fd
, type
, arg
);
660 int kvm_vm_ioctl(KVMState
*s
, int type
, ...)
667 arg
= va_arg(ap
, void *);
670 ret
= ioctl(s
->vmfd
, type
, arg
);
677 int kvm_vcpu_ioctl(CPUState
*env
, int type
, ...)
684 arg
= va_arg(ap
, void *);
687 ret
= ioctl(env
->kvm_fd
, type
, arg
);
694 int kvm_has_sync_mmu(void)
696 #ifdef KVM_CAP_SYNC_MMU
697 KVMState
*s
= kvm_state
;
699 if (kvm_ioctl(s
, KVM_CHECK_EXTENSION
, KVM_CAP_SYNC_MMU
) > 0)
706 #ifdef KVM_CAP_SET_GUEST_DEBUG
707 struct kvm_sw_breakpoint
*kvm_find_sw_breakpoint(CPUState
*env
,
710 struct kvm_sw_breakpoint
*bp
;
712 TAILQ_FOREACH(bp
, &env
->kvm_state
->kvm_sw_breakpoints
, entry
) {
719 int kvm_sw_breakpoints_active(CPUState
*env
)
721 return !TAILQ_EMPTY(&env
->kvm_state
->kvm_sw_breakpoints
);
724 int kvm_update_guest_debug(CPUState
*env
, unsigned long reinject_trap
)
726 struct kvm_guest_debug dbg
;
729 if (env
->singlestep_enabled
)
730 dbg
.control
= KVM_GUESTDBG_ENABLE
| KVM_GUESTDBG_SINGLESTEP
;
732 kvm_arch_update_guest_debug(env
, &dbg
);
733 dbg
.control
|= reinject_trap
;
735 return kvm_vcpu_ioctl(env
, KVM_SET_GUEST_DEBUG
, &dbg
);
738 int kvm_insert_breakpoint(CPUState
*current_env
, target_ulong addr
,
739 target_ulong len
, int type
)
741 struct kvm_sw_breakpoint
*bp
;
745 if (type
== GDB_BREAKPOINT_SW
) {
746 bp
= kvm_find_sw_breakpoint(current_env
, addr
);
752 bp
= qemu_malloc(sizeof(struct kvm_sw_breakpoint
));
758 err
= kvm_arch_insert_sw_breakpoint(current_env
, bp
);
764 TAILQ_INSERT_HEAD(¤t_env
->kvm_state
->kvm_sw_breakpoints
,
767 err
= kvm_arch_insert_hw_breakpoint(addr
, len
, type
);
772 for (env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
773 err
= kvm_update_guest_debug(env
, 0);
780 int kvm_remove_breakpoint(CPUState
*current_env
, target_ulong addr
,
781 target_ulong len
, int type
)
783 struct kvm_sw_breakpoint
*bp
;
787 if (type
== GDB_BREAKPOINT_SW
) {
788 bp
= kvm_find_sw_breakpoint(current_env
, addr
);
792 if (bp
->use_count
> 1) {
797 err
= kvm_arch_remove_sw_breakpoint(current_env
, bp
);
801 TAILQ_REMOVE(¤t_env
->kvm_state
->kvm_sw_breakpoints
, bp
, entry
);
804 err
= kvm_arch_remove_hw_breakpoint(addr
, len
, type
);
809 for (env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
810 err
= kvm_update_guest_debug(env
, 0);
817 void kvm_remove_all_breakpoints(CPUState
*current_env
)
819 struct kvm_sw_breakpoint
*bp
, *next
;
820 KVMState
*s
= current_env
->kvm_state
;
823 TAILQ_FOREACH_SAFE(bp
, &s
->kvm_sw_breakpoints
, entry
, next
) {
824 if (kvm_arch_remove_sw_breakpoint(current_env
, bp
) != 0) {
825 /* Try harder to find a CPU that currently sees the breakpoint. */
826 for (env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
827 if (kvm_arch_remove_sw_breakpoint(env
, bp
) == 0)
832 kvm_arch_remove_all_hw_breakpoints();
834 for (env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
)
835 kvm_update_guest_debug(env
, 0);
838 #else /* !KVM_CAP_SET_GUEST_DEBUG */
840 int kvm_update_guest_debug(CPUState
*env
, unsigned long reinject_trap
)
845 int kvm_insert_breakpoint(CPUState
*current_env
, target_ulong addr
,
846 target_ulong len
, int type
)
851 int kvm_remove_breakpoint(CPUState
*current_env
, target_ulong addr
,
852 target_ulong len
, int type
)
857 void kvm_remove_all_breakpoints(CPUState
*current_env
)
860 #endif /* !KVM_CAP_SET_GUEST_DEBUG */