]> git.proxmox.com Git - mirror_qemu.git/blame - kvm-all.c
kvm-all: add support for multiple address spaces
[mirror_qemu.git] / kvm-all.c
CommitLineData
05330448
AL
1/*
2 * QEMU KVM support
3 *
4 * Copyright IBM, Corp. 2008
5832d1f2 5 * Red Hat, Inc. 2008
05330448
AL
6 *
7 * Authors:
8 * Anthony Liguori <aliguori@us.ibm.com>
5832d1f2 9 * Glauber Costa <gcosta@redhat.com>
05330448
AL
10 *
11 * This work is licensed under the terms of the GNU GPL, version 2 or later.
12 * See the COPYING file in the top-level directory.
13 *
14 */
15
16#include <sys/types.h>
17#include <sys/ioctl.h>
18#include <sys/mman.h>
984b5181 19#include <stdarg.h>
05330448
AL
20
21#include <linux/kvm.h>
22
23#include "qemu-common.h"
1de7afc9
PB
24#include "qemu/atomic.h"
25#include "qemu/option.h"
26#include "qemu/config-file.h"
d33a1810 27#include "hw/hw.h"
a2cb15b0 28#include "hw/pci/msi.h"
d426d9fb 29#include "hw/s390x/adapter.h"
022c62cb 30#include "exec/gdbstub.h"
8571ed35 31#include "sysemu/kvm_int.h"
1de7afc9 32#include "qemu/bswap.h"
022c62cb 33#include "exec/memory.h"
747afd5b 34#include "exec/ram_addr.h"
022c62cb 35#include "exec/address-spaces.h"
1de7afc9 36#include "qemu/event_notifier.h"
9c775729 37#include "trace.h"
05330448 38
135a129a
AK
39#include "hw/boards.h"
40
d2f2b8a7
SH
41/* This check must be after config-host.h is included */
42#ifdef CONFIG_EVENTFD
43#include <sys/eventfd.h>
44#endif
45
93148aa5 46/* KVM uses PAGE_SIZE in its definition of COALESCED_MMIO_MAX */
f65ed4c1
AL
47#define PAGE_SIZE TARGET_PAGE_SIZE
48
05330448
AL
49//#define DEBUG_KVM
50
51#ifdef DEBUG_KVM
8c0d577e 52#define DPRINTF(fmt, ...) \
05330448
AL
53 do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
54#else
8c0d577e 55#define DPRINTF(fmt, ...) \
05330448
AL
56 do { } while (0)
57#endif
58
04fa27f5
JK
59#define KVM_MSI_HASHTAB_SIZE 256
60
9d1c35df 61struct KVMState
05330448 62{
fc02086b
EH
63 AccelState parent_obj;
64
fb541ca5 65 int nr_slots;
05330448
AL
66 int fd;
67 int vmfd;
f65ed4c1 68 int coalesced_mmio;
62a2744c 69 struct kvm_coalesced_mmio_ring *coalesced_mmio_ring;
1cae88b9 70 bool coalesced_flush_in_progress;
e69917e2 71 int broken_set_mem_region;
a0fb002c 72 int vcpu_events;
b0b1d690 73 int robust_singlestep;
ff44f1a3 74 int debugregs;
e22a25c9
AL
75#ifdef KVM_CAP_SET_GUEST_DEBUG
76 struct kvm_sw_breakpoint_head kvm_sw_breakpoints;
77#endif
8a7c7393 78 int pit_state2;
f1665b21 79 int xsave, xcrs;
d2f2b8a7 80 int many_ioeventfds;
3ab73842 81 int intx_set_mask;
92e4b519
DG
82 /* The man page (and posix) say ioctl numbers are signed int, but
83 * they're not. Linux, glibc and *BSD all treat ioctl numbers as
84 * unsigned, and treating them as signed here can break things */
e333cd69 85 unsigned irq_set_ioctl;
aed6efb9 86 unsigned int sigmask_len;
84b058d7
JK
87#ifdef KVM_CAP_IRQ_ROUTING
88 struct kvm_irq_routing *irq_routes;
89 int nr_allocated_irq_routes;
90 uint32_t *used_gsi_bitmap;
4e2e4e63 91 unsigned int gsi_count;
04fa27f5 92 QTAILQ_HEAD(msi_hashtab, KVMMSIRoute) msi_hashtab[KVM_MSI_HASHTAB_SIZE];
4a3adebb 93 bool direct_msi;
84b058d7 94#endif
7bbda04c 95 KVMMemoryListener memory_listener;
9d1c35df 96};
05330448 97
6a7af8cb 98KVMState *kvm_state;
3d4b2649 99bool kvm_kernel_irqchip;
7ae26bd4 100bool kvm_async_interrupts_allowed;
215e79c0 101bool kvm_halt_in_kernel_allowed;
69e03ae6 102bool kvm_eventfds_allowed;
cc7e0ddf 103bool kvm_irqfds_allowed;
f41389ae 104bool kvm_resamplefds_allowed;
614e41bc 105bool kvm_msi_via_irqfd_allowed;
f3e1bed8 106bool kvm_gsi_routing_allowed;
76fe21de 107bool kvm_gsi_direct_mapping;
13eed94e 108bool kvm_allowed;
df9c8b75 109bool kvm_readonly_mem_allowed;
d0a073a1 110bool kvm_vm_attributes_allowed;
05330448 111
94a8d39a
JK
112static const KVMCapabilityInfo kvm_required_capabilites[] = {
113 KVM_CAP_INFO(USER_MEMORY),
114 KVM_CAP_INFO(DESTROY_MEMORY_REGION_WORKS),
115 KVM_CAP_LAST_INFO
116};
117
7bbda04c 118static KVMSlot *kvm_get_free_slot(KVMMemoryListener *kml)
05330448 119{
7bbda04c 120 KVMState *s = kvm_state;
05330448
AL
121 int i;
122
fb541ca5 123 for (i = 0; i < s->nr_slots; i++) {
7bbda04c
PB
124 if (kml->slots[i].memory_size == 0) {
125 return &kml->slots[i];
a426e122 126 }
05330448
AL
127 }
128
b8865591
IM
129 return NULL;
130}
131
132bool kvm_has_free_slot(MachineState *ms)
133{
7bbda04c
PB
134 KVMState *s = KVM_STATE(ms->accelerator);
135
136 return kvm_get_free_slot(&s->memory_listener);
b8865591
IM
137}
138
7bbda04c 139static KVMSlot *kvm_alloc_slot(KVMMemoryListener *kml)
b8865591 140{
7bbda04c 141 KVMSlot *slot = kvm_get_free_slot(kml);
b8865591
IM
142
143 if (slot) {
144 return slot;
145 }
146
d3f8d37f
AL
147 fprintf(stderr, "%s: no free slot available\n", __func__);
148 abort();
149}
150
7bbda04c 151static KVMSlot *kvm_lookup_matching_slot(KVMMemoryListener *kml,
a8170e5e
AK
152 hwaddr start_addr,
153 hwaddr end_addr)
d3f8d37f 154{
7bbda04c 155 KVMState *s = kvm_state;
d3f8d37f
AL
156 int i;
157
fb541ca5 158 for (i = 0; i < s->nr_slots; i++) {
7bbda04c 159 KVMSlot *mem = &kml->slots[i];
d3f8d37f
AL
160
161 if (start_addr == mem->start_addr &&
162 end_addr == mem->start_addr + mem->memory_size) {
163 return mem;
164 }
165 }
166
05330448
AL
167 return NULL;
168}
169
6152e2ae
AL
170/*
171 * Find overlapping slot with lowest start address
172 */
7bbda04c 173static KVMSlot *kvm_lookup_overlapping_slot(KVMMemoryListener *kml,
a8170e5e
AK
174 hwaddr start_addr,
175 hwaddr end_addr)
05330448 176{
7bbda04c 177 KVMState *s = kvm_state;
6152e2ae 178 KVMSlot *found = NULL;
05330448
AL
179 int i;
180
fb541ca5 181 for (i = 0; i < s->nr_slots; i++) {
7bbda04c 182 KVMSlot *mem = &kml->slots[i];
05330448 183
6152e2ae
AL
184 if (mem->memory_size == 0 ||
185 (found && found->start_addr < mem->start_addr)) {
186 continue;
187 }
188
189 if (end_addr > mem->start_addr &&
190 start_addr < mem->start_addr + mem->memory_size) {
191 found = mem;
192 }
05330448
AL
193 }
194
6152e2ae 195 return found;
05330448
AL
196}
197
9f213ed9 198int kvm_physical_memory_addr_from_host(KVMState *s, void *ram,
a8170e5e 199 hwaddr *phys_addr)
983dfc3b 200{
7bbda04c 201 KVMMemoryListener *kml = &s->memory_listener;
983dfc3b
HY
202 int i;
203
fb541ca5 204 for (i = 0; i < s->nr_slots; i++) {
7bbda04c 205 KVMSlot *mem = &kml->slots[i];
983dfc3b 206
9f213ed9
AK
207 if (ram >= mem->ram && ram < mem->ram + mem->memory_size) {
208 *phys_addr = mem->start_addr + (ram - mem->ram);
983dfc3b
HY
209 return 1;
210 }
211 }
212
213 return 0;
214}
215
7bbda04c 216static int kvm_set_user_memory_region(KVMMemoryListener *kml, KVMSlot *slot)
5832d1f2 217{
7bbda04c 218 KVMState *s = kvm_state;
5832d1f2
AL
219 struct kvm_userspace_memory_region mem;
220
38bfe691 221 mem.slot = slot->slot | (kml->as_id << 16);
5832d1f2 222 mem.guest_phys_addr = slot->start_addr;
9f213ed9 223 mem.userspace_addr = (unsigned long)slot->ram;
5832d1f2 224 mem.flags = slot->flags;
651eb0f4
XG
225
226 if (slot->memory_size && mem.flags & KVM_MEM_READONLY) {
235e8982
JJ
227 /* Set the slot size to 0 before setting the slot to the desired
228 * value. This is needed based on KVM commit 75d61fbc. */
229 mem.memory_size = 0;
230 kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
231 }
232 mem.memory_size = slot->memory_size;
5832d1f2
AL
233 return kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
234}
235
504134d2 236int kvm_init_vcpu(CPUState *cpu)
05330448
AL
237{
238 KVMState *s = kvm_state;
239 long mmap_size;
240 int ret;
241
8c0d577e 242 DPRINTF("kvm_init_vcpu\n");
05330448 243
b164e48e 244 ret = kvm_vm_ioctl(s, KVM_CREATE_VCPU, (void *)kvm_arch_vcpu_id(cpu));
05330448 245 if (ret < 0) {
8c0d577e 246 DPRINTF("kvm_create_vcpu failed\n");
05330448
AL
247 goto err;
248 }
249
8737c51c 250 cpu->kvm_fd = ret;
a60f24b5 251 cpu->kvm_state = s;
20d695a9 252 cpu->kvm_vcpu_dirty = true;
05330448
AL
253
254 mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0);
255 if (mmap_size < 0) {
748a680b 256 ret = mmap_size;
8c0d577e 257 DPRINTF("KVM_GET_VCPU_MMAP_SIZE failed\n");
05330448
AL
258 goto err;
259 }
260
f7575c96 261 cpu->kvm_run = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED,
8737c51c 262 cpu->kvm_fd, 0);
f7575c96 263 if (cpu->kvm_run == MAP_FAILED) {
05330448 264 ret = -errno;
8c0d577e 265 DPRINTF("mmap'ing vcpu state failed\n");
05330448
AL
266 goto err;
267 }
268
a426e122
JK
269 if (s->coalesced_mmio && !s->coalesced_mmio_ring) {
270 s->coalesced_mmio_ring =
f7575c96 271 (void *)cpu->kvm_run + s->coalesced_mmio * PAGE_SIZE;
a426e122 272 }
62a2744c 273
20d695a9 274 ret = kvm_arch_init_vcpu(cpu);
05330448
AL
275err:
276 return ret;
277}
278
5832d1f2
AL
279/*
280 * dirty pages logging control
281 */
25254bbc 282
d6ff5cbc 283static int kvm_mem_flags(MemoryRegion *mr)
25254bbc 284{
d6ff5cbc 285 bool readonly = mr->readonly || memory_region_is_romd(mr);
235e8982 286 int flags = 0;
d6ff5cbc
AJ
287
288 if (memory_region_get_dirty_log_mask(mr) != 0) {
289 flags |= KVM_MEM_LOG_DIRTY_PAGES;
290 }
235e8982
JJ
291 if (readonly && kvm_readonly_mem_allowed) {
292 flags |= KVM_MEM_READONLY;
293 }
294 return flags;
25254bbc
MT
295}
296
7bbda04c
PB
297static int kvm_slot_update_flags(KVMMemoryListener *kml, KVMSlot *mem,
298 MemoryRegion *mr)
5832d1f2 299{
4495d6a7
JK
300 int old_flags;
301
4495d6a7 302 old_flags = mem->flags;
d6ff5cbc 303 mem->flags = kvm_mem_flags(mr);
5832d1f2 304
4495d6a7 305 /* If nothing changed effectively, no need to issue ioctl */
d6ff5cbc 306 if (mem->flags == old_flags) {
25254bbc 307 return 0;
4495d6a7
JK
308 }
309
7bbda04c 310 return kvm_set_user_memory_region(kml, mem);
5832d1f2
AL
311}
312
7bbda04c
PB
313static int kvm_section_update_flags(KVMMemoryListener *kml,
314 MemoryRegionSection *section)
25254bbc 315{
d6ff5cbc
AJ
316 hwaddr phys_addr = section->offset_within_address_space;
317 ram_addr_t size = int128_get64(section->size);
7bbda04c 318 KVMSlot *mem = kvm_lookup_matching_slot(kml, phys_addr, phys_addr + size);
25254bbc
MT
319
320 if (mem == NULL) {
ea8cb1a8
PB
321 return 0;
322 } else {
7bbda04c 323 return kvm_slot_update_flags(kml, mem, section->mr);
25254bbc 324 }
25254bbc
MT
325}
326
a01672d3 327static void kvm_log_start(MemoryListener *listener,
b2dfd71c
PB
328 MemoryRegionSection *section,
329 int old, int new)
5832d1f2 330{
7bbda04c 331 KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener);
a01672d3
AK
332 int r;
333
b2dfd71c
PB
334 if (old != 0) {
335 return;
336 }
337
7bbda04c 338 r = kvm_section_update_flags(kml, section);
a01672d3
AK
339 if (r < 0) {
340 abort();
341 }
5832d1f2
AL
342}
343
a01672d3 344static void kvm_log_stop(MemoryListener *listener,
b2dfd71c
PB
345 MemoryRegionSection *section,
346 int old, int new)
5832d1f2 347{
7bbda04c 348 KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener);
a01672d3
AK
349 int r;
350
b2dfd71c
PB
351 if (new != 0) {
352 return;
353 }
354
7bbda04c 355 r = kvm_section_update_flags(kml, section);
a01672d3
AK
356 if (r < 0) {
357 abort();
358 }
5832d1f2
AL
359}
360
8369e01c 361/* get kvm's dirty pages bitmap and update qemu's */
ffcde12f
AK
362static int kvm_get_dirty_pages_log_range(MemoryRegionSection *section,
363 unsigned long *bitmap)
96c1606b 364{
c9dd46fc 365 ram_addr_t start = section->offset_within_region + section->mr->ram_addr;
5ff7fb77
JQ
366 ram_addr_t pages = int128_get64(section->size) / getpagesize();
367
368 cpu_physical_memory_set_dirty_lebitmap(bitmap, start, pages);
8369e01c 369 return 0;
96c1606b
AG
370}
371
8369e01c
MT
372#define ALIGN(x, y) (((x)+(y)-1) & ~((y)-1))
373
5832d1f2
AL
374/**
375 * kvm_physical_sync_dirty_bitmap - Grab dirty bitmap from kernel space
fd4aa979
BS
376 * This function updates qemu's dirty bitmap using
377 * memory_region_set_dirty(). This means all bits are set
378 * to dirty.
5832d1f2 379 *
d3f8d37f 380 * @start_add: start of logged region.
5832d1f2
AL
381 * @end_addr: end of logged region.
382 */
7bbda04c
PB
383static int kvm_physical_sync_dirty_bitmap(KVMMemoryListener *kml,
384 MemoryRegionSection *section)
5832d1f2
AL
385{
386 KVMState *s = kvm_state;
151f7749 387 unsigned long size, allocated_size = 0;
714f78c5 388 struct kvm_dirty_log d = {};
151f7749
JK
389 KVMSlot *mem;
390 int ret = 0;
a8170e5e 391 hwaddr start_addr = section->offset_within_address_space;
052e87b0 392 hwaddr end_addr = start_addr + int128_get64(section->size);
5832d1f2 393
151f7749
JK
394 d.dirty_bitmap = NULL;
395 while (start_addr < end_addr) {
7bbda04c 396 mem = kvm_lookup_overlapping_slot(kml, start_addr, end_addr);
151f7749
JK
397 if (mem == NULL) {
398 break;
399 }
5832d1f2 400
51b0c606
MT
401 /* XXX bad kernel interface alert
402 * For dirty bitmap, kernel allocates array of size aligned to
403 * bits-per-long. But for case when the kernel is 64bits and
404 * the userspace is 32bits, userspace can't align to the same
405 * bits-per-long, since sizeof(long) is different between kernel
406 * and user space. This way, userspace will provide buffer which
407 * may be 4 bytes less than the kernel will use, resulting in
408 * userspace memory corruption (which is not detectable by valgrind
409 * too, in most cases).
410 * So for now, let's align to 64 instead of HOST_LONG_BITS here, in
411 * a hope that sizeof(long) wont become >8 any time soon.
412 */
413 size = ALIGN(((mem->memory_size) >> TARGET_PAGE_BITS),
414 /*HOST_LONG_BITS*/ 64) / 8;
151f7749 415 if (!d.dirty_bitmap) {
7267c094 416 d.dirty_bitmap = g_malloc(size);
151f7749 417 } else if (size > allocated_size) {
7267c094 418 d.dirty_bitmap = g_realloc(d.dirty_bitmap, size);
151f7749
JK
419 }
420 allocated_size = size;
421 memset(d.dirty_bitmap, 0, allocated_size);
5832d1f2 422
38bfe691 423 d.slot = mem->slot | (kml->as_id << 16);
50212d63 424 if (kvm_vm_ioctl(s, KVM_GET_DIRTY_LOG, &d) == -1) {
8c0d577e 425 DPRINTF("ioctl failed %d\n", errno);
151f7749
JK
426 ret = -1;
427 break;
428 }
5832d1f2 429
ffcde12f 430 kvm_get_dirty_pages_log_range(section, d.dirty_bitmap);
8369e01c 431 start_addr = mem->start_addr + mem->memory_size;
5832d1f2 432 }
7267c094 433 g_free(d.dirty_bitmap);
151f7749
JK
434
435 return ret;
5832d1f2
AL
436}
437
95d2994a
AK
438static void kvm_coalesce_mmio_region(MemoryListener *listener,
439 MemoryRegionSection *secion,
a8170e5e 440 hwaddr start, hwaddr size)
f65ed4c1 441{
f65ed4c1
AL
442 KVMState *s = kvm_state;
443
444 if (s->coalesced_mmio) {
445 struct kvm_coalesced_mmio_zone zone;
446
447 zone.addr = start;
448 zone.size = size;
7e680753 449 zone.pad = 0;
f65ed4c1 450
95d2994a 451 (void)kvm_vm_ioctl(s, KVM_REGISTER_COALESCED_MMIO, &zone);
f65ed4c1 452 }
f65ed4c1
AL
453}
454
95d2994a
AK
455static void kvm_uncoalesce_mmio_region(MemoryListener *listener,
456 MemoryRegionSection *secion,
a8170e5e 457 hwaddr start, hwaddr size)
f65ed4c1 458{
f65ed4c1
AL
459 KVMState *s = kvm_state;
460
461 if (s->coalesced_mmio) {
462 struct kvm_coalesced_mmio_zone zone;
463
464 zone.addr = start;
465 zone.size = size;
7e680753 466 zone.pad = 0;
f65ed4c1 467
95d2994a 468 (void)kvm_vm_ioctl(s, KVM_UNREGISTER_COALESCED_MMIO, &zone);
f65ed4c1 469 }
f65ed4c1
AL
470}
471
ad7b8b33
AL
472int kvm_check_extension(KVMState *s, unsigned int extension)
473{
474 int ret;
475
476 ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, extension);
477 if (ret < 0) {
478 ret = 0;
479 }
480
481 return ret;
482}
483
7d0a07fa
AG
484int kvm_vm_check_extension(KVMState *s, unsigned int extension)
485{
486 int ret;
487
488 ret = kvm_vm_ioctl(s, KVM_CHECK_EXTENSION, extension);
489 if (ret < 0) {
490 /* VM wide version not implemented, use global one instead */
491 ret = kvm_check_extension(s, extension);
492 }
493
494 return ret;
495}
496
b680c5ba
GK
497static uint32_t adjust_ioeventfd_endianness(uint32_t val, uint32_t size)
498{
499#if defined(HOST_WORDS_BIGENDIAN) != defined(TARGET_WORDS_BIGENDIAN)
500 /* The kernel expects ioeventfd values in HOST_WORDS_BIGENDIAN
501 * endianness, but the memory core hands them in target endianness.
502 * For example, PPC is always treated as big-endian even if running
503 * on KVM and on PPC64LE. Correct here.
504 */
505 switch (size) {
506 case 2:
507 val = bswap16(val);
508 break;
509 case 4:
510 val = bswap32(val);
511 break;
512 }
513#endif
514 return val;
515}
516
584f2be7 517static int kvm_set_ioeventfd_mmio(int fd, hwaddr addr, uint32_t val,
41cb62c2 518 bool assign, uint32_t size, bool datamatch)
500ffd4a
MT
519{
520 int ret;
03a96b83
TH
521 struct kvm_ioeventfd iofd = {
522 .datamatch = datamatch ? adjust_ioeventfd_endianness(val, size) : 0,
523 .addr = addr,
524 .len = size,
525 .flags = 0,
526 .fd = fd,
527 };
500ffd4a
MT
528
529 if (!kvm_enabled()) {
530 return -ENOSYS;
531 }
532
41cb62c2
MT
533 if (datamatch) {
534 iofd.flags |= KVM_IOEVENTFD_FLAG_DATAMATCH;
535 }
500ffd4a
MT
536 if (!assign) {
537 iofd.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN;
538 }
539
540 ret = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &iofd);
541
542 if (ret < 0) {
543 return -errno;
544 }
545
546 return 0;
547}
548
44c3f8f7 549static int kvm_set_ioeventfd_pio(int fd, uint16_t addr, uint16_t val,
41cb62c2 550 bool assign, uint32_t size, bool datamatch)
500ffd4a
MT
551{
552 struct kvm_ioeventfd kick = {
b680c5ba 553 .datamatch = datamatch ? adjust_ioeventfd_endianness(val, size) : 0,
500ffd4a 554 .addr = addr,
41cb62c2 555 .flags = KVM_IOEVENTFD_FLAG_PIO,
44c3f8f7 556 .len = size,
500ffd4a
MT
557 .fd = fd,
558 };
559 int r;
560 if (!kvm_enabled()) {
561 return -ENOSYS;
562 }
41cb62c2
MT
563 if (datamatch) {
564 kick.flags |= KVM_IOEVENTFD_FLAG_DATAMATCH;
565 }
500ffd4a
MT
566 if (!assign) {
567 kick.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN;
568 }
569 r = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &kick);
570 if (r < 0) {
571 return r;
572 }
573 return 0;
574}
575
576
d2f2b8a7
SH
577static int kvm_check_many_ioeventfds(void)
578{
d0dcac83
SH
579 /* Userspace can use ioeventfd for io notification. This requires a host
580 * that supports eventfd(2) and an I/O thread; since eventfd does not
581 * support SIGIO it cannot interrupt the vcpu.
582 *
583 * Older kernels have a 6 device limit on the KVM io bus. Find out so we
d2f2b8a7
SH
584 * can avoid creating too many ioeventfds.
585 */
12d4536f 586#if defined(CONFIG_EVENTFD)
d2f2b8a7
SH
587 int ioeventfds[7];
588 int i, ret = 0;
589 for (i = 0; i < ARRAY_SIZE(ioeventfds); i++) {
590 ioeventfds[i] = eventfd(0, EFD_CLOEXEC);
591 if (ioeventfds[i] < 0) {
592 break;
593 }
41cb62c2 594 ret = kvm_set_ioeventfd_pio(ioeventfds[i], 0, i, true, 2, true);
d2f2b8a7
SH
595 if (ret < 0) {
596 close(ioeventfds[i]);
597 break;
598 }
599 }
600
601 /* Decide whether many devices are supported or not */
602 ret = i == ARRAY_SIZE(ioeventfds);
603
604 while (i-- > 0) {
41cb62c2 605 kvm_set_ioeventfd_pio(ioeventfds[i], 0, i, false, 2, true);
d2f2b8a7
SH
606 close(ioeventfds[i]);
607 }
608 return ret;
609#else
610 return 0;
611#endif
612}
613
94a8d39a
JK
614static const KVMCapabilityInfo *
615kvm_check_extension_list(KVMState *s, const KVMCapabilityInfo *list)
616{
617 while (list->name) {
618 if (!kvm_check_extension(s, list->value)) {
619 return list;
620 }
621 list++;
622 }
623 return NULL;
624}
625
7bbda04c
PB
626static void kvm_set_phys_mem(KVMMemoryListener *kml,
627 MemoryRegionSection *section, bool add)
46dbef6a
MT
628{
629 KVMState *s = kvm_state;
46dbef6a
MT
630 KVMSlot *mem, old;
631 int err;
a01672d3 632 MemoryRegion *mr = section->mr;
235e8982 633 bool writeable = !mr->readonly && !mr->rom_device;
a8170e5e 634 hwaddr start_addr = section->offset_within_address_space;
052e87b0 635 ram_addr_t size = int128_get64(section->size);
9f213ed9 636 void *ram = NULL;
8f6f962b 637 unsigned delta;
46dbef6a 638
14542fea 639 /* kvm works in page size chunks, but the function may be called
f2a64032
AG
640 with sub-page size and unaligned start address. Pad the start
641 address to next and truncate size to previous page boundary. */
642 delta = (TARGET_PAGE_SIZE - (start_addr & ~TARGET_PAGE_MASK));
643 delta &= ~TARGET_PAGE_MASK;
8f6f962b
AK
644 if (delta > size) {
645 return;
646 }
647 start_addr += delta;
648 size -= delta;
649 size &= TARGET_PAGE_MASK;
650 if (!size || (start_addr & ~TARGET_PAGE_MASK)) {
651 return;
652 }
46dbef6a 653
a01672d3 654 if (!memory_region_is_ram(mr)) {
235e8982
JJ
655 if (writeable || !kvm_readonly_mem_allowed) {
656 return;
657 } else if (!mr->romd_mode) {
658 /* If the memory device is not in romd_mode, then we actually want
659 * to remove the kvm memory slot so all accesses will trap. */
660 add = false;
661 }
9f213ed9
AK
662 }
663
8f6f962b 664 ram = memory_region_get_ram_ptr(mr) + section->offset_within_region + delta;
a01672d3 665
46dbef6a 666 while (1) {
7bbda04c 667 mem = kvm_lookup_overlapping_slot(kml, start_addr, start_addr + size);
46dbef6a
MT
668 if (!mem) {
669 break;
670 }
671
a01672d3 672 if (add && start_addr >= mem->start_addr &&
46dbef6a 673 (start_addr + size <= mem->start_addr + mem->memory_size) &&
9f213ed9 674 (ram - start_addr == mem->ram - mem->start_addr)) {
46dbef6a 675 /* The new slot fits into the existing one and comes with
25254bbc 676 * identical parameters - update flags and done. */
7bbda04c 677 kvm_slot_update_flags(kml, mem, mr);
46dbef6a
MT
678 return;
679 }
680
681 old = *mem;
682
1bfbac4e 683 if (mem->flags & KVM_MEM_LOG_DIRTY_PAGES) {
7bbda04c 684 kvm_physical_sync_dirty_bitmap(kml, section);
3fbffb62
AK
685 }
686
46dbef6a
MT
687 /* unregister the overlapping slot */
688 mem->memory_size = 0;
7bbda04c 689 err = kvm_set_user_memory_region(kml, mem);
46dbef6a
MT
690 if (err) {
691 fprintf(stderr, "%s: error unregistering overlapping slot: %s\n",
692 __func__, strerror(-err));
693 abort();
694 }
695
696 /* Workaround for older KVM versions: we can't join slots, even not by
697 * unregistering the previous ones and then registering the larger
698 * slot. We have to maintain the existing fragmentation. Sigh.
699 *
700 * This workaround assumes that the new slot starts at the same
701 * address as the first existing one. If not or if some overlapping
702 * slot comes around later, we will fail (not seen in practice so far)
703 * - and actually require a recent KVM version. */
704 if (s->broken_set_mem_region &&
a01672d3 705 old.start_addr == start_addr && old.memory_size < size && add) {
7bbda04c 706 mem = kvm_alloc_slot(kml);
46dbef6a
MT
707 mem->memory_size = old.memory_size;
708 mem->start_addr = old.start_addr;
9f213ed9 709 mem->ram = old.ram;
d6ff5cbc 710 mem->flags = kvm_mem_flags(mr);
46dbef6a 711
7bbda04c 712 err = kvm_set_user_memory_region(kml, mem);
46dbef6a
MT
713 if (err) {
714 fprintf(stderr, "%s: error updating slot: %s\n", __func__,
715 strerror(-err));
716 abort();
717 }
718
719 start_addr += old.memory_size;
9f213ed9 720 ram += old.memory_size;
46dbef6a
MT
721 size -= old.memory_size;
722 continue;
723 }
724
725 /* register prefix slot */
726 if (old.start_addr < start_addr) {
7bbda04c 727 mem = kvm_alloc_slot(kml);
46dbef6a
MT
728 mem->memory_size = start_addr - old.start_addr;
729 mem->start_addr = old.start_addr;
9f213ed9 730 mem->ram = old.ram;
d6ff5cbc 731 mem->flags = kvm_mem_flags(mr);
46dbef6a 732
7bbda04c 733 err = kvm_set_user_memory_region(kml, mem);
46dbef6a
MT
734 if (err) {
735 fprintf(stderr, "%s: error registering prefix slot: %s\n",
736 __func__, strerror(-err));
d4d6868f
AG
737#ifdef TARGET_PPC
738 fprintf(stderr, "%s: This is probably because your kernel's " \
739 "PAGE_SIZE is too big. Please try to use 4k " \
740 "PAGE_SIZE!\n", __func__);
741#endif
46dbef6a
MT
742 abort();
743 }
744 }
745
746 /* register suffix slot */
747 if (old.start_addr + old.memory_size > start_addr + size) {
748 ram_addr_t size_delta;
749
7bbda04c 750 mem = kvm_alloc_slot(kml);
46dbef6a
MT
751 mem->start_addr = start_addr + size;
752 size_delta = mem->start_addr - old.start_addr;
753 mem->memory_size = old.memory_size - size_delta;
9f213ed9 754 mem->ram = old.ram + size_delta;
d6ff5cbc 755 mem->flags = kvm_mem_flags(mr);
46dbef6a 756
7bbda04c 757 err = kvm_set_user_memory_region(kml, mem);
46dbef6a
MT
758 if (err) {
759 fprintf(stderr, "%s: error registering suffix slot: %s\n",
760 __func__, strerror(-err));
761 abort();
762 }
763 }
764 }
765
766 /* in case the KVM bug workaround already "consumed" the new slot */
a426e122 767 if (!size) {
46dbef6a 768 return;
a426e122 769 }
a01672d3 770 if (!add) {
46dbef6a 771 return;
a426e122 772 }
7bbda04c 773 mem = kvm_alloc_slot(kml);
46dbef6a
MT
774 mem->memory_size = size;
775 mem->start_addr = start_addr;
9f213ed9 776 mem->ram = ram;
d6ff5cbc 777 mem->flags = kvm_mem_flags(mr);
46dbef6a 778
7bbda04c 779 err = kvm_set_user_memory_region(kml, mem);
46dbef6a
MT
780 if (err) {
781 fprintf(stderr, "%s: error registering slot: %s\n", __func__,
782 strerror(-err));
783 abort();
784 }
785}
786
a01672d3
AK
787static void kvm_region_add(MemoryListener *listener,
788 MemoryRegionSection *section)
789{
7bbda04c
PB
790 KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener);
791
dfde4e6e 792 memory_region_ref(section->mr);
7bbda04c 793 kvm_set_phys_mem(kml, section, true);
a01672d3
AK
794}
795
796static void kvm_region_del(MemoryListener *listener,
797 MemoryRegionSection *section)
798{
7bbda04c
PB
799 KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener);
800
801 kvm_set_phys_mem(kml, section, false);
dfde4e6e 802 memory_region_unref(section->mr);
a01672d3
AK
803}
804
805static void kvm_log_sync(MemoryListener *listener,
806 MemoryRegionSection *section)
7b8f3b78 807{
7bbda04c 808 KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener);
a01672d3
AK
809 int r;
810
7bbda04c 811 r = kvm_physical_sync_dirty_bitmap(kml, section);
a01672d3
AK
812 if (r < 0) {
813 abort();
814 }
7b8f3b78
MT
815}
816
d22b096e
AK
817static void kvm_mem_ioeventfd_add(MemoryListener *listener,
818 MemoryRegionSection *section,
819 bool match_data, uint64_t data,
820 EventNotifier *e)
821{
822 int fd = event_notifier_get_fd(e);
80a1ea37
AK
823 int r;
824
4b8f1c88 825 r = kvm_set_ioeventfd_mmio(fd, section->offset_within_address_space,
052e87b0
PB
826 data, true, int128_get64(section->size),
827 match_data);
80a1ea37 828 if (r < 0) {
fa4ba923
AK
829 fprintf(stderr, "%s: error adding ioeventfd: %s\n",
830 __func__, strerror(-r));
80a1ea37
AK
831 abort();
832 }
833}
834
d22b096e
AK
835static void kvm_mem_ioeventfd_del(MemoryListener *listener,
836 MemoryRegionSection *section,
837 bool match_data, uint64_t data,
838 EventNotifier *e)
80a1ea37 839{
d22b096e 840 int fd = event_notifier_get_fd(e);
80a1ea37
AK
841 int r;
842
4b8f1c88 843 r = kvm_set_ioeventfd_mmio(fd, section->offset_within_address_space,
052e87b0
PB
844 data, false, int128_get64(section->size),
845 match_data);
80a1ea37
AK
846 if (r < 0) {
847 abort();
848 }
849}
850
d22b096e
AK
851static void kvm_io_ioeventfd_add(MemoryListener *listener,
852 MemoryRegionSection *section,
853 bool match_data, uint64_t data,
854 EventNotifier *e)
80a1ea37 855{
d22b096e 856 int fd = event_notifier_get_fd(e);
80a1ea37
AK
857 int r;
858
44c3f8f7 859 r = kvm_set_ioeventfd_pio(fd, section->offset_within_address_space,
052e87b0
PB
860 data, true, int128_get64(section->size),
861 match_data);
80a1ea37 862 if (r < 0) {
fa4ba923
AK
863 fprintf(stderr, "%s: error adding ioeventfd: %s\n",
864 __func__, strerror(-r));
80a1ea37
AK
865 abort();
866 }
867}
868
d22b096e
AK
869static void kvm_io_ioeventfd_del(MemoryListener *listener,
870 MemoryRegionSection *section,
871 bool match_data, uint64_t data,
872 EventNotifier *e)
80a1ea37
AK
873
874{
d22b096e 875 int fd = event_notifier_get_fd(e);
80a1ea37
AK
876 int r;
877
44c3f8f7 878 r = kvm_set_ioeventfd_pio(fd, section->offset_within_address_space,
052e87b0
PB
879 data, false, int128_get64(section->size),
880 match_data);
80a1ea37
AK
881 if (r < 0) {
882 abort();
883 }
884}
885
38bfe691
PB
886void kvm_memory_listener_register(KVMState *s, KVMMemoryListener *kml,
887 AddressSpace *as, int as_id)
7bbda04c
PB
888{
889 int i;
890
891 kml->slots = g_malloc0(s->nr_slots * sizeof(KVMSlot));
38bfe691 892 kml->as_id = as_id;
7bbda04c
PB
893
894 for (i = 0; i < s->nr_slots; i++) {
895 kml->slots[i].slot = i;
896 }
897
898 kml->listener.region_add = kvm_region_add;
899 kml->listener.region_del = kvm_region_del;
900 kml->listener.log_start = kvm_log_start;
901 kml->listener.log_stop = kvm_log_stop;
902 kml->listener.log_sync = kvm_log_sync;
903 kml->listener.priority = 10;
904
905 memory_listener_register(&kml->listener, as);
906}
d22b096e
AK
907
908static MemoryListener kvm_io_listener = {
d22b096e
AK
909 .eventfd_add = kvm_io_ioeventfd_add,
910 .eventfd_del = kvm_io_ioeventfd_del,
72e22d2f 911 .priority = 10,
7b8f3b78
MT
912};
913
c3affe56 914static void kvm_handle_interrupt(CPUState *cpu, int mask)
aa7f74d1 915{
259186a7 916 cpu->interrupt_request |= mask;
aa7f74d1 917
60e82579 918 if (!qemu_cpu_is_self(cpu)) {
c08d7424 919 qemu_cpu_kick(cpu);
aa7f74d1
JK
920 }
921}
922
3889c3fa 923int kvm_set_irq(KVMState *s, int irq, int level)
84b058d7
JK
924{
925 struct kvm_irq_level event;
926 int ret;
927
7ae26bd4 928 assert(kvm_async_interrupts_enabled());
84b058d7
JK
929
930 event.level = level;
931 event.irq = irq;
e333cd69 932 ret = kvm_vm_ioctl(s, s->irq_set_ioctl, &event);
84b058d7 933 if (ret < 0) {
3889c3fa 934 perror("kvm_set_irq");
84b058d7
JK
935 abort();
936 }
937
e333cd69 938 return (s->irq_set_ioctl == KVM_IRQ_LINE) ? 1 : event.status;
84b058d7
JK
939}
940
941#ifdef KVM_CAP_IRQ_ROUTING
d3d3bef0
JK
942typedef struct KVMMSIRoute {
943 struct kvm_irq_routing_entry kroute;
944 QTAILQ_ENTRY(KVMMSIRoute) entry;
945} KVMMSIRoute;
946
84b058d7
JK
947static void set_gsi(KVMState *s, unsigned int gsi)
948{
84b058d7
JK
949 s->used_gsi_bitmap[gsi / 32] |= 1U << (gsi % 32);
950}
951
04fa27f5
JK
952static void clear_gsi(KVMState *s, unsigned int gsi)
953{
954 s->used_gsi_bitmap[gsi / 32] &= ~(1U << (gsi % 32));
955}
956
7b774593 957void kvm_init_irq_routing(KVMState *s)
84b058d7 958{
04fa27f5 959 int gsi_count, i;
84b058d7 960
00008418 961 gsi_count = kvm_check_extension(s, KVM_CAP_IRQ_ROUTING) - 1;
84b058d7
JK
962 if (gsi_count > 0) {
963 unsigned int gsi_bits, i;
964
965 /* Round up so we can search ints using ffs */
bc8c6788 966 gsi_bits = ALIGN(gsi_count, 32);
84b058d7 967 s->used_gsi_bitmap = g_malloc0(gsi_bits / 8);
4e2e4e63 968 s->gsi_count = gsi_count;
84b058d7
JK
969
970 /* Mark any over-allocated bits as already in use */
971 for (i = gsi_count; i < gsi_bits; i++) {
972 set_gsi(s, i);
973 }
974 }
975
976 s->irq_routes = g_malloc0(sizeof(*s->irq_routes));
977 s->nr_allocated_irq_routes = 0;
978
4a3adebb
JK
979 if (!s->direct_msi) {
980 for (i = 0; i < KVM_MSI_HASHTAB_SIZE; i++) {
981 QTAILQ_INIT(&s->msi_hashtab[i]);
982 }
04fa27f5
JK
983 }
984
84b058d7
JK
985 kvm_arch_init_irq_routing(s);
986}
987
cb925cf9 988void kvm_irqchip_commit_routes(KVMState *s)
e7b20308
JK
989{
990 int ret;
991
992 s->irq_routes->flags = 0;
993 ret = kvm_vm_ioctl(s, KVM_SET_GSI_ROUTING, s->irq_routes);
994 assert(ret == 0);
995}
996
84b058d7
JK
997static void kvm_add_routing_entry(KVMState *s,
998 struct kvm_irq_routing_entry *entry)
999{
1000 struct kvm_irq_routing_entry *new;
1001 int n, size;
1002
1003 if (s->irq_routes->nr == s->nr_allocated_irq_routes) {
1004 n = s->nr_allocated_irq_routes * 2;
1005 if (n < 64) {
1006 n = 64;
1007 }
1008 size = sizeof(struct kvm_irq_routing);
1009 size += n * sizeof(*new);
1010 s->irq_routes = g_realloc(s->irq_routes, size);
1011 s->nr_allocated_irq_routes = n;
1012 }
1013 n = s->irq_routes->nr++;
1014 new = &s->irq_routes->entries[n];
0fbc2074
MT
1015
1016 *new = *entry;
84b058d7
JK
1017
1018 set_gsi(s, entry->gsi);
1019}
1020
cc57407e
JK
1021static int kvm_update_routing_entry(KVMState *s,
1022 struct kvm_irq_routing_entry *new_entry)
1023{
1024 struct kvm_irq_routing_entry *entry;
1025 int n;
1026
1027 for (n = 0; n < s->irq_routes->nr; n++) {
1028 entry = &s->irq_routes->entries[n];
1029 if (entry->gsi != new_entry->gsi) {
1030 continue;
1031 }
1032
40509f7f
MT
1033 if(!memcmp(entry, new_entry, sizeof *entry)) {
1034 return 0;
1035 }
1036
0fbc2074 1037 *entry = *new_entry;
cc57407e
JK
1038
1039 kvm_irqchip_commit_routes(s);
1040
1041 return 0;
1042 }
1043
1044 return -ESRCH;
1045}
1046
1df186df 1047void kvm_irqchip_add_irq_route(KVMState *s, int irq, int irqchip, int pin)
84b058d7 1048{
0fbc2074 1049 struct kvm_irq_routing_entry e = {};
84b058d7 1050
4e2e4e63
JK
1051 assert(pin < s->gsi_count);
1052
84b058d7
JK
1053 e.gsi = irq;
1054 e.type = KVM_IRQ_ROUTING_IRQCHIP;
1055 e.flags = 0;
1056 e.u.irqchip.irqchip = irqchip;
1057 e.u.irqchip.pin = pin;
1058 kvm_add_routing_entry(s, &e);
1059}
1060
1e2aa8be 1061void kvm_irqchip_release_virq(KVMState *s, int virq)
04fa27f5
JK
1062{
1063 struct kvm_irq_routing_entry *e;
1064 int i;
1065
76fe21de
AK
1066 if (kvm_gsi_direct_mapping()) {
1067 return;
1068 }
1069
04fa27f5
JK
1070 for (i = 0; i < s->irq_routes->nr; i++) {
1071 e = &s->irq_routes->entries[i];
1072 if (e->gsi == virq) {
1073 s->irq_routes->nr--;
1074 *e = s->irq_routes->entries[s->irq_routes->nr];
1075 }
1076 }
1077 clear_gsi(s, virq);
1078}
1079
1080static unsigned int kvm_hash_msi(uint32_t data)
1081{
1082 /* This is optimized for IA32 MSI layout. However, no other arch shall
1083 * repeat the mistake of not providing a direct MSI injection API. */
1084 return data & 0xff;
1085}
1086
1087static void kvm_flush_dynamic_msi_routes(KVMState *s)
1088{
1089 KVMMSIRoute *route, *next;
1090 unsigned int hash;
1091
1092 for (hash = 0; hash < KVM_MSI_HASHTAB_SIZE; hash++) {
1093 QTAILQ_FOREACH_SAFE(route, &s->msi_hashtab[hash], entry, next) {
1094 kvm_irqchip_release_virq(s, route->kroute.gsi);
1095 QTAILQ_REMOVE(&s->msi_hashtab[hash], route, entry);
1096 g_free(route);
1097 }
1098 }
1099}
1100
1101static int kvm_irqchip_get_virq(KVMState *s)
1102{
1103 uint32_t *word = s->used_gsi_bitmap;
1104 int max_words = ALIGN(s->gsi_count, 32) / 32;
bd2a8884 1105 int i, zeroes;
04fa27f5 1106
bdf02631
WM
1107 /*
1108 * PIC and IOAPIC share the first 16 GSI numbers, thus the available
1109 * GSI numbers are more than the number of IRQ route. Allocating a GSI
1110 * number can succeed even though a new route entry cannot be added.
1111 * When this happens, flush dynamic MSI entries to free IRQ route entries.
1112 */
1113 if (!s->direct_msi && s->irq_routes->nr == s->gsi_count) {
1114 kvm_flush_dynamic_msi_routes(s);
1115 }
1116
04fa27f5
JK
1117 /* Return the lowest unused GSI in the bitmap */
1118 for (i = 0; i < max_words; i++) {
bd2a8884
SH
1119 zeroes = ctz32(~word[i]);
1120 if (zeroes == 32) {
04fa27f5
JK
1121 continue;
1122 }
1123
bd2a8884 1124 return zeroes + i * 32;
04fa27f5 1125 }
04fa27f5
JK
1126 return -ENOSPC;
1127
1128}
1129
1130static KVMMSIRoute *kvm_lookup_msi_route(KVMState *s, MSIMessage msg)
1131{
1132 unsigned int hash = kvm_hash_msi(msg.data);
1133 KVMMSIRoute *route;
1134
1135 QTAILQ_FOREACH(route, &s->msi_hashtab[hash], entry) {
1136 if (route->kroute.u.msi.address_lo == (uint32_t)msg.address &&
1137 route->kroute.u.msi.address_hi == (msg.address >> 32) &&
d07cc1f1 1138 route->kroute.u.msi.data == le32_to_cpu(msg.data)) {
04fa27f5
JK
1139 return route;
1140 }
1141 }
1142 return NULL;
1143}
1144
1145int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg)
1146{
4a3adebb 1147 struct kvm_msi msi;
04fa27f5
JK
1148 KVMMSIRoute *route;
1149
4a3adebb
JK
1150 if (s->direct_msi) {
1151 msi.address_lo = (uint32_t)msg.address;
1152 msi.address_hi = msg.address >> 32;
d07cc1f1 1153 msi.data = le32_to_cpu(msg.data);
4a3adebb
JK
1154 msi.flags = 0;
1155 memset(msi.pad, 0, sizeof(msi.pad));
1156
1157 return kvm_vm_ioctl(s, KVM_SIGNAL_MSI, &msi);
1158 }
1159
04fa27f5
JK
1160 route = kvm_lookup_msi_route(s, msg);
1161 if (!route) {
e7b20308 1162 int virq;
04fa27f5
JK
1163
1164 virq = kvm_irqchip_get_virq(s);
1165 if (virq < 0) {
1166 return virq;
1167 }
1168
0fbc2074 1169 route = g_malloc0(sizeof(KVMMSIRoute));
04fa27f5
JK
1170 route->kroute.gsi = virq;
1171 route->kroute.type = KVM_IRQ_ROUTING_MSI;
1172 route->kroute.flags = 0;
1173 route->kroute.u.msi.address_lo = (uint32_t)msg.address;
1174 route->kroute.u.msi.address_hi = msg.address >> 32;
d07cc1f1 1175 route->kroute.u.msi.data = le32_to_cpu(msg.data);
04fa27f5
JK
1176
1177 kvm_add_routing_entry(s, &route->kroute);
cb925cf9 1178 kvm_irqchip_commit_routes(s);
04fa27f5
JK
1179
1180 QTAILQ_INSERT_TAIL(&s->msi_hashtab[kvm_hash_msi(msg.data)], route,
1181 entry);
04fa27f5
JK
1182 }
1183
1184 assert(route->kroute.type == KVM_IRQ_ROUTING_MSI);
1185
3889c3fa 1186 return kvm_set_irq(s, route->kroute.gsi, 1);
04fa27f5
JK
1187}
1188
92b4e489
JK
1189int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg)
1190{
0fbc2074 1191 struct kvm_irq_routing_entry kroute = {};
92b4e489
JK
1192 int virq;
1193
76fe21de 1194 if (kvm_gsi_direct_mapping()) {
1850b6b7 1195 return kvm_arch_msi_data_to_gsi(msg.data);
76fe21de
AK
1196 }
1197
f3e1bed8 1198 if (!kvm_gsi_routing_enabled()) {
92b4e489
JK
1199 return -ENOSYS;
1200 }
1201
1202 virq = kvm_irqchip_get_virq(s);
1203 if (virq < 0) {
1204 return virq;
1205 }
1206
1207 kroute.gsi = virq;
1208 kroute.type = KVM_IRQ_ROUTING_MSI;
1209 kroute.flags = 0;
1210 kroute.u.msi.address_lo = (uint32_t)msg.address;
1211 kroute.u.msi.address_hi = msg.address >> 32;
d07cc1f1 1212 kroute.u.msi.data = le32_to_cpu(msg.data);
9e03a040
FB
1213 if (kvm_arch_fixup_msi_route(&kroute, msg.address, msg.data)) {
1214 kvm_irqchip_release_virq(s, virq);
1215 return -EINVAL;
1216 }
92b4e489
JK
1217
1218 kvm_add_routing_entry(s, &kroute);
cb925cf9 1219 kvm_irqchip_commit_routes(s);
92b4e489
JK
1220
1221 return virq;
1222}
1223
cc57407e
JK
1224int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg)
1225{
0fbc2074 1226 struct kvm_irq_routing_entry kroute = {};
cc57407e 1227
76fe21de
AK
1228 if (kvm_gsi_direct_mapping()) {
1229 return 0;
1230 }
1231
cc57407e
JK
1232 if (!kvm_irqchip_in_kernel()) {
1233 return -ENOSYS;
1234 }
1235
1236 kroute.gsi = virq;
1237 kroute.type = KVM_IRQ_ROUTING_MSI;
1238 kroute.flags = 0;
1239 kroute.u.msi.address_lo = (uint32_t)msg.address;
1240 kroute.u.msi.address_hi = msg.address >> 32;
d07cc1f1 1241 kroute.u.msi.data = le32_to_cpu(msg.data);
9e03a040
FB
1242 if (kvm_arch_fixup_msi_route(&kroute, msg.address, msg.data)) {
1243 return -EINVAL;
1244 }
cc57407e
JK
1245
1246 return kvm_update_routing_entry(s, &kroute);
1247}
1248
ca916d37
VM
1249static int kvm_irqchip_assign_irqfd(KVMState *s, int fd, int rfd, int virq,
1250 bool assign)
39853bbc
JK
1251{
1252 struct kvm_irqfd irqfd = {
1253 .fd = fd,
1254 .gsi = virq,
1255 .flags = assign ? 0 : KVM_IRQFD_FLAG_DEASSIGN,
1256 };
1257
ca916d37
VM
1258 if (rfd != -1) {
1259 irqfd.flags |= KVM_IRQFD_FLAG_RESAMPLE;
1260 irqfd.resamplefd = rfd;
1261 }
1262
cc7e0ddf 1263 if (!kvm_irqfds_enabled()) {
39853bbc
JK
1264 return -ENOSYS;
1265 }
1266
1267 return kvm_vm_ioctl(s, KVM_IRQFD, &irqfd);
1268}
1269
d426d9fb
CH
1270int kvm_irqchip_add_adapter_route(KVMState *s, AdapterInfo *adapter)
1271{
e9af2fef 1272 struct kvm_irq_routing_entry kroute = {};
d426d9fb
CH
1273 int virq;
1274
1275 if (!kvm_gsi_routing_enabled()) {
1276 return -ENOSYS;
1277 }
1278
1279 virq = kvm_irqchip_get_virq(s);
1280 if (virq < 0) {
1281 return virq;
1282 }
1283
1284 kroute.gsi = virq;
1285 kroute.type = KVM_IRQ_ROUTING_S390_ADAPTER;
1286 kroute.flags = 0;
1287 kroute.u.adapter.summary_addr = adapter->summary_addr;
1288 kroute.u.adapter.ind_addr = adapter->ind_addr;
1289 kroute.u.adapter.summary_offset = adapter->summary_offset;
1290 kroute.u.adapter.ind_offset = adapter->ind_offset;
1291 kroute.u.adapter.adapter_id = adapter->adapter_id;
1292
1293 kvm_add_routing_entry(s, &kroute);
1294 kvm_irqchip_commit_routes(s);
1295
1296 return virq;
1297}
1298
84b058d7
JK
1299#else /* !KVM_CAP_IRQ_ROUTING */
1300
7b774593 1301void kvm_init_irq_routing(KVMState *s)
84b058d7
JK
1302{
1303}
04fa27f5 1304
d3d3bef0
JK
1305void kvm_irqchip_release_virq(KVMState *s, int virq)
1306{
1307}
1308
04fa27f5
JK
1309int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg)
1310{
1311 abort();
1312}
92b4e489
JK
1313
1314int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg)
1315{
df410675 1316 return -ENOSYS;
92b4e489 1317}
39853bbc 1318
d426d9fb
CH
1319int kvm_irqchip_add_adapter_route(KVMState *s, AdapterInfo *adapter)
1320{
1321 return -ENOSYS;
1322}
1323
39853bbc
JK
1324static int kvm_irqchip_assign_irqfd(KVMState *s, int fd, int virq, bool assign)
1325{
1326 abort();
1327}
dabe3143
MT
1328
1329int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg)
1330{
1331 return -ENOSYS;
1332}
84b058d7
JK
1333#endif /* !KVM_CAP_IRQ_ROUTING */
1334
ca916d37
VM
1335int kvm_irqchip_add_irqfd_notifier(KVMState *s, EventNotifier *n,
1336 EventNotifier *rn, int virq)
39853bbc 1337{
ca916d37
VM
1338 return kvm_irqchip_assign_irqfd(s, event_notifier_get_fd(n),
1339 rn ? event_notifier_get_fd(rn) : -1, virq, true);
39853bbc
JK
1340}
1341
b131c74a 1342int kvm_irqchip_remove_irqfd_notifier(KVMState *s, EventNotifier *n, int virq)
15b2bd18 1343{
ca916d37
VM
1344 return kvm_irqchip_assign_irqfd(s, event_notifier_get_fd(n), -1, virq,
1345 false);
15b2bd18
PB
1346}
1347
446f16a6 1348static int kvm_irqchip_create(MachineState *machine, KVMState *s)
84b058d7 1349{
84b058d7
JK
1350 int ret;
1351
446f16a6 1352 if (!machine_kernel_irqchip_allowed(machine) ||
d426d9fb
CH
1353 (!kvm_check_extension(s, KVM_CAP_IRQCHIP) &&
1354 (kvm_vm_enable_cap(s, KVM_CAP_S390_IRQCHIP, 0) < 0))) {
84b058d7
JK
1355 return 0;
1356 }
1357
d6032e06
CD
1358 /* First probe and see if there's a arch-specific hook to create the
1359 * in-kernel irqchip for us */
1360 ret = kvm_arch_irqchip_create(s);
84b058d7 1361 if (ret < 0) {
84b058d7 1362 return ret;
d6032e06
CD
1363 } else if (ret == 0) {
1364 ret = kvm_vm_ioctl(s, KVM_CREATE_IRQCHIP);
1365 if (ret < 0) {
1366 fprintf(stderr, "Create kernel irqchip failed\n");
1367 return ret;
1368 }
84b058d7
JK
1369 }
1370
3d4b2649 1371 kvm_kernel_irqchip = true;
7ae26bd4
PM
1372 /* If we have an in-kernel IRQ chip then we must have asynchronous
1373 * interrupt delivery (though the reverse is not necessarily true)
1374 */
1375 kvm_async_interrupts_allowed = true;
215e79c0 1376 kvm_halt_in_kernel_allowed = true;
84b058d7
JK
1377
1378 kvm_init_irq_routing(s);
1379
1380 return 0;
1381}
1382
670436ce
AJ
1383/* Find number of supported CPUs using the recommended
1384 * procedure from the kernel API documentation to cope with
1385 * older kernels that may be missing capabilities.
1386 */
1387static int kvm_recommended_vcpus(KVMState *s)
3ed444e9 1388{
670436ce
AJ
1389 int ret = kvm_check_extension(s, KVM_CAP_NR_VCPUS);
1390 return (ret) ? ret : 4;
1391}
3ed444e9 1392
670436ce
AJ
1393static int kvm_max_vcpus(KVMState *s)
1394{
1395 int ret = kvm_check_extension(s, KVM_CAP_MAX_VCPUS);
1396 return (ret) ? ret : kvm_recommended_vcpus(s);
3ed444e9
DH
1397}
1398
f6a1ef64 1399static int kvm_init(MachineState *ms)
05330448 1400{
f6a1ef64 1401 MachineClass *mc = MACHINE_GET_CLASS(ms);
168ccc11
JK
1402 static const char upgrade_note[] =
1403 "Please upgrade to at least kernel 2.6.29 or recent kvm-kmod\n"
1404 "(see http://sourceforge.net/projects/kvm).\n";
670436ce
AJ
1405 struct {
1406 const char *name;
1407 int num;
1408 } num_cpus[] = {
1409 { "SMP", smp_cpus },
1410 { "hotpluggable", max_cpus },
1411 { NULL, }
1412 }, *nc = num_cpus;
1413 int soft_vcpus_limit, hard_vcpus_limit;
05330448 1414 KVMState *s;
94a8d39a 1415 const KVMCapabilityInfo *missing_cap;
05330448 1416 int ret;
7bbda04c 1417 int type = 0;
135a129a 1418 const char *kvm_type;
05330448 1419
fc02086b 1420 s = KVM_STATE(ms->accelerator);
05330448 1421
3145fcb6
DG
1422 /*
1423 * On systems where the kernel can support different base page
1424 * sizes, host page size may be different from TARGET_PAGE_SIZE,
1425 * even with KVM. TARGET_PAGE_SIZE is assumed to be the minimum
1426 * page size for the system though.
1427 */
1428 assert(TARGET_PAGE_SIZE <= getpagesize());
47c16ed5 1429 page_size_init();
3145fcb6 1430
aed6efb9
JH
1431 s->sigmask_len = 8;
1432
e22a25c9 1433#ifdef KVM_CAP_SET_GUEST_DEBUG
72cf2d4f 1434 QTAILQ_INIT(&s->kvm_sw_breakpoints);
e22a25c9 1435#endif
05330448 1436 s->vmfd = -1;
40ff6d7e 1437 s->fd = qemu_open("/dev/kvm", O_RDWR);
05330448
AL
1438 if (s->fd == -1) {
1439 fprintf(stderr, "Could not access KVM kernel module: %m\n");
1440 ret = -errno;
1441 goto err;
1442 }
1443
1444 ret = kvm_ioctl(s, KVM_GET_API_VERSION, 0);
1445 if (ret < KVM_API_VERSION) {
0e1dac6c 1446 if (ret >= 0) {
05330448 1447 ret = -EINVAL;
a426e122 1448 }
05330448
AL
1449 fprintf(stderr, "kvm version too old\n");
1450 goto err;
1451 }
1452
1453 if (ret > KVM_API_VERSION) {
1454 ret = -EINVAL;
1455 fprintf(stderr, "kvm version not supported\n");
1456 goto err;
1457 }
1458
fb541ca5
AW
1459 s->nr_slots = kvm_check_extension(s, KVM_CAP_NR_MEMSLOTS);
1460
1461 /* If unspecified, use the default value */
1462 if (!s->nr_slots) {
1463 s->nr_slots = 32;
1464 }
1465
670436ce
AJ
1466 /* check the vcpu limits */
1467 soft_vcpus_limit = kvm_recommended_vcpus(s);
1468 hard_vcpus_limit = kvm_max_vcpus(s);
3ed444e9 1469
670436ce
AJ
1470 while (nc->name) {
1471 if (nc->num > soft_vcpus_limit) {
1472 fprintf(stderr,
1473 "Warning: Number of %s cpus requested (%d) exceeds "
1474 "the recommended cpus supported by KVM (%d)\n",
1475 nc->name, nc->num, soft_vcpus_limit);
1476
1477 if (nc->num > hard_vcpus_limit) {
670436ce
AJ
1478 fprintf(stderr, "Number of %s cpus requested (%d) exceeds "
1479 "the maximum cpus supported by KVM (%d)\n",
1480 nc->name, nc->num, hard_vcpus_limit);
9ba3cf54 1481 exit(1);
670436ce
AJ
1482 }
1483 }
1484 nc++;
7dc52526
MT
1485 }
1486
135a129a 1487 kvm_type = qemu_opt_get(qemu_get_machine_opts(), "kvm-type");
f1e29879
MA
1488 if (mc->kvm_type) {
1489 type = mc->kvm_type(kvm_type);
135a129a 1490 } else if (kvm_type) {
0e1dac6c 1491 ret = -EINVAL;
135a129a
AK
1492 fprintf(stderr, "Invalid argument kvm-type=%s\n", kvm_type);
1493 goto err;
1494 }
1495
94ccff13 1496 do {
135a129a 1497 ret = kvm_ioctl(s, KVM_CREATE_VM, type);
94ccff13
TK
1498 } while (ret == -EINTR);
1499
1500 if (ret < 0) {
521f438e 1501 fprintf(stderr, "ioctl(KVM_CREATE_VM) failed: %d %s\n", -ret,
94ccff13
TK
1502 strerror(-ret));
1503
0104dcac 1504#ifdef TARGET_S390X
2c80e996
CH
1505 if (ret == -EINVAL) {
1506 fprintf(stderr,
1507 "Host kernel setup problem detected. Please verify:\n");
1508 fprintf(stderr, "- for kernels supporting the switch_amode or"
1509 " user_mode parameters, whether\n");
1510 fprintf(stderr,
1511 " user space is running in primary address space\n");
1512 fprintf(stderr,
1513 "- for kernels supporting the vm.allocate_pgste sysctl, "
1514 "whether it is enabled\n");
1515 }
0104dcac 1516#endif
05330448 1517 goto err;
0104dcac 1518 }
05330448 1519
94ccff13 1520 s->vmfd = ret;
94a8d39a
JK
1521 missing_cap = kvm_check_extension_list(s, kvm_required_capabilites);
1522 if (!missing_cap) {
1523 missing_cap =
1524 kvm_check_extension_list(s, kvm_arch_required_capabilities);
05330448 1525 }
94a8d39a 1526 if (missing_cap) {
ad7b8b33 1527 ret = -EINVAL;
94a8d39a
JK
1528 fprintf(stderr, "kvm does not support %s\n%s",
1529 missing_cap->name, upgrade_note);
d85dc283
AL
1530 goto err;
1531 }
1532
ad7b8b33 1533 s->coalesced_mmio = kvm_check_extension(s, KVM_CAP_COALESCED_MMIO);
f65ed4c1 1534
e69917e2 1535 s->broken_set_mem_region = 1;
14a09518 1536 ret = kvm_check_extension(s, KVM_CAP_JOIN_MEMORY_REGIONS_WORKS);
e69917e2
JK
1537 if (ret > 0) {
1538 s->broken_set_mem_region = 0;
1539 }
e69917e2 1540
a0fb002c
JK
1541#ifdef KVM_CAP_VCPU_EVENTS
1542 s->vcpu_events = kvm_check_extension(s, KVM_CAP_VCPU_EVENTS);
1543#endif
1544
b0b1d690
JK
1545 s->robust_singlestep =
1546 kvm_check_extension(s, KVM_CAP_X86_ROBUST_SINGLESTEP);
b0b1d690 1547
ff44f1a3
JK
1548#ifdef KVM_CAP_DEBUGREGS
1549 s->debugregs = kvm_check_extension(s, KVM_CAP_DEBUGREGS);
1550#endif
1551
f1665b21
SY
1552#ifdef KVM_CAP_XSAVE
1553 s->xsave = kvm_check_extension(s, KVM_CAP_XSAVE);
1554#endif
1555
f1665b21
SY
1556#ifdef KVM_CAP_XCRS
1557 s->xcrs = kvm_check_extension(s, KVM_CAP_XCRS);
1558#endif
1559
8a7c7393
JK
1560#ifdef KVM_CAP_PIT_STATE2
1561 s->pit_state2 = kvm_check_extension(s, KVM_CAP_PIT_STATE2);
1562#endif
1563
d3d3bef0 1564#ifdef KVM_CAP_IRQ_ROUTING
4a3adebb 1565 s->direct_msi = (kvm_check_extension(s, KVM_CAP_SIGNAL_MSI) > 0);
d3d3bef0 1566#endif
4a3adebb 1567
3ab73842
JK
1568 s->intx_set_mask = kvm_check_extension(s, KVM_CAP_PCI_2_3);
1569
e333cd69 1570 s->irq_set_ioctl = KVM_IRQ_LINE;
8732fbd2 1571 if (kvm_check_extension(s, KVM_CAP_IRQ_INJECT_STATUS)) {
e333cd69 1572 s->irq_set_ioctl = KVM_IRQ_LINE_STATUS;
8732fbd2
PM
1573 }
1574
df9c8b75
JJ
1575#ifdef KVM_CAP_READONLY_MEM
1576 kvm_readonly_mem_allowed =
1577 (kvm_check_extension(s, KVM_CAP_READONLY_MEM) > 0);
1578#endif
1579
69e03ae6
NN
1580 kvm_eventfds_allowed =
1581 (kvm_check_extension(s, KVM_CAP_IOEVENTFD) > 0);
1582
f41389ae
EA
1583 kvm_irqfds_allowed =
1584 (kvm_check_extension(s, KVM_CAP_IRQFD) > 0);
1585
1586 kvm_resamplefds_allowed =
1587 (kvm_check_extension(s, KVM_CAP_IRQFD_RESAMPLE) > 0);
1588
d0a073a1
DD
1589 kvm_vm_attributes_allowed =
1590 (kvm_check_extension(s, KVM_CAP_VM_ATTRIBUTES) > 0);
1591
b16565b3 1592 ret = kvm_arch_init(ms, s);
a426e122 1593 if (ret < 0) {
05330448 1594 goto err;
a426e122 1595 }
05330448 1596
446f16a6 1597 ret = kvm_irqchip_create(ms, s);
84b058d7
JK
1598 if (ret < 0) {
1599 goto err;
1600 }
1601
05330448 1602 kvm_state = s;
7bbda04c
PB
1603
1604 s->memory_listener.listener.eventfd_add = kvm_mem_ioeventfd_add;
1605 s->memory_listener.listener.eventfd_del = kvm_mem_ioeventfd_del;
1606 s->memory_listener.listener.coalesced_mmio_add = kvm_coalesce_mmio_region;
1607 s->memory_listener.listener.coalesced_mmio_del = kvm_uncoalesce_mmio_region;
1608
1609 kvm_memory_listener_register(s, &s->memory_listener,
38bfe691 1610 &address_space_memory, 0);
7bbda04c
PB
1611 memory_listener_register(&kvm_io_listener,
1612 &address_space_io);
05330448 1613
d2f2b8a7
SH
1614 s->many_ioeventfds = kvm_check_many_ioeventfds();
1615
aa7f74d1
JK
1616 cpu_interrupt_handler = kvm_handle_interrupt;
1617
05330448
AL
1618 return 0;
1619
1620err:
0e1dac6c 1621 assert(ret < 0);
6d1cc321
SW
1622 if (s->vmfd >= 0) {
1623 close(s->vmfd);
1624 }
1625 if (s->fd != -1) {
1626 close(s->fd);
05330448 1627 }
7bbda04c 1628 g_free(s->memory_listener.slots);
05330448
AL
1629
1630 return ret;
1631}
1632
aed6efb9
JH
1633void kvm_set_sigmask_len(KVMState *s, unsigned int sigmask_len)
1634{
1635 s->sigmask_len = sigmask_len;
1636}
1637
4c663752
PB
1638static void kvm_handle_io(uint16_t port, MemTxAttrs attrs, void *data, int direction,
1639 int size, uint32_t count)
05330448
AL
1640{
1641 int i;
1642 uint8_t *ptr = data;
1643
1644 for (i = 0; i < count; i++) {
4c663752 1645 address_space_rw(&address_space_io, port, attrs,
5c9eb028 1646 ptr, size,
354678c5 1647 direction == KVM_EXIT_IO_OUT);
05330448
AL
1648 ptr += size;
1649 }
05330448
AL
1650}
1651
5326ab55 1652static int kvm_handle_internal_error(CPUState *cpu, struct kvm_run *run)
7c80eef8 1653{
977c7b6d
RK
1654 fprintf(stderr, "KVM internal error. Suberror: %d\n",
1655 run->internal.suberror);
1656
7c80eef8
MT
1657 if (kvm_check_extension(kvm_state, KVM_CAP_INTERNAL_ERROR_DATA)) {
1658 int i;
1659
7c80eef8
MT
1660 for (i = 0; i < run->internal.ndata; ++i) {
1661 fprintf(stderr, "extra data[%d]: %"PRIx64"\n",
1662 i, (uint64_t)run->internal.data[i]);
1663 }
1664 }
7c80eef8
MT
1665 if (run->internal.suberror == KVM_INTERNAL_ERROR_EMULATION) {
1666 fprintf(stderr, "emulation failure\n");
20d695a9 1667 if (!kvm_arch_stop_on_emulation_error(cpu)) {
878096ee 1668 cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_CODE);
d73cd8f4 1669 return EXCP_INTERRUPT;
a426e122 1670 }
7c80eef8
MT
1671 }
1672 /* FIXME: Should trigger a qmp message to let management know
1673 * something went wrong.
1674 */
73aaec4a 1675 return -1;
7c80eef8 1676}
7c80eef8 1677
62a2744c 1678void kvm_flush_coalesced_mmio_buffer(void)
f65ed4c1 1679{
f65ed4c1 1680 KVMState *s = kvm_state;
1cae88b9
AK
1681
1682 if (s->coalesced_flush_in_progress) {
1683 return;
1684 }
1685
1686 s->coalesced_flush_in_progress = true;
1687
62a2744c
SY
1688 if (s->coalesced_mmio_ring) {
1689 struct kvm_coalesced_mmio_ring *ring = s->coalesced_mmio_ring;
f65ed4c1
AL
1690 while (ring->first != ring->last) {
1691 struct kvm_coalesced_mmio *ent;
1692
1693 ent = &ring->coalesced_mmio[ring->first];
1694
1695 cpu_physical_memory_write(ent->phys_addr, ent->data, ent->len);
85199474 1696 smp_wmb();
f65ed4c1
AL
1697 ring->first = (ring->first + 1) % KVM_COALESCED_MMIO_MAX;
1698 }
1699 }
1cae88b9
AK
1700
1701 s->coalesced_flush_in_progress = false;
f65ed4c1
AL
1702}
1703
20d695a9 1704static void do_kvm_cpu_synchronize_state(void *arg)
4c0960c0 1705{
20d695a9 1706 CPUState *cpu = arg;
2705d56a 1707
20d695a9
AF
1708 if (!cpu->kvm_vcpu_dirty) {
1709 kvm_arch_get_registers(cpu);
1710 cpu->kvm_vcpu_dirty = true;
4c0960c0
AK
1711 }
1712}
1713
dd1750d7 1714void kvm_cpu_synchronize_state(CPUState *cpu)
2705d56a 1715{
20d695a9
AF
1716 if (!cpu->kvm_vcpu_dirty) {
1717 run_on_cpu(cpu, do_kvm_cpu_synchronize_state, cpu);
a426e122 1718 }
2705d56a
JK
1719}
1720
c8e2085d 1721static void do_kvm_cpu_synchronize_post_reset(void *arg)
ea375f9a 1722{
c8e2085d
DH
1723 CPUState *cpu = arg;
1724
20d695a9
AF
1725 kvm_arch_put_registers(cpu, KVM_PUT_RESET_STATE);
1726 cpu->kvm_vcpu_dirty = false;
ea375f9a
JK
1727}
1728
c8e2085d
DH
1729void kvm_cpu_synchronize_post_reset(CPUState *cpu)
1730{
1731 run_on_cpu(cpu, do_kvm_cpu_synchronize_post_reset, cpu);
1732}
1733
1734static void do_kvm_cpu_synchronize_post_init(void *arg)
ea375f9a 1735{
c8e2085d
DH
1736 CPUState *cpu = arg;
1737
20d695a9
AF
1738 kvm_arch_put_registers(cpu, KVM_PUT_FULL_STATE);
1739 cpu->kvm_vcpu_dirty = false;
ea375f9a
JK
1740}
1741
c8e2085d
DH
1742void kvm_cpu_synchronize_post_init(CPUState *cpu)
1743{
1744 run_on_cpu(cpu, do_kvm_cpu_synchronize_post_init, cpu);
1745}
1746
de9d61e8
MT
1747void kvm_cpu_clean_state(CPUState *cpu)
1748{
1749 cpu->kvm_vcpu_dirty = false;
1750}
1751
1458c363 1752int kvm_cpu_exec(CPUState *cpu)
05330448 1753{
f7575c96 1754 struct kvm_run *run = cpu->kvm_run;
7cbb533f 1755 int ret, run_ret;
05330448 1756
8c0d577e 1757 DPRINTF("kvm_cpu_exec()\n");
05330448 1758
20d695a9 1759 if (kvm_arch_process_async_events(cpu)) {
fcd7d003 1760 cpu->exit_request = 0;
6792a57b 1761 return EXCP_HLT;
9ccfac9e 1762 }
0af691d7 1763
4b8523ee
JK
1764 qemu_mutex_unlock_iothread();
1765
9ccfac9e 1766 do {
4c663752
PB
1767 MemTxAttrs attrs;
1768
20d695a9
AF
1769 if (cpu->kvm_vcpu_dirty) {
1770 kvm_arch_put_registers(cpu, KVM_PUT_RUNTIME_STATE);
1771 cpu->kvm_vcpu_dirty = false;
4c0960c0
AK
1772 }
1773
20d695a9 1774 kvm_arch_pre_run(cpu, run);
fcd7d003 1775 if (cpu->exit_request) {
9ccfac9e
JK
1776 DPRINTF("interrupt exit requested\n");
1777 /*
1778 * KVM requires us to reenter the kernel after IO exits to complete
1779 * instruction emulation. This self-signal will ensure that we
1780 * leave ASAP again.
1781 */
1782 qemu_cpu_kick_self();
1783 }
9ccfac9e 1784
1bc22652 1785 run_ret = kvm_vcpu_ioctl(cpu, KVM_RUN, 0);
9ccfac9e 1786
4c663752 1787 attrs = kvm_arch_post_run(cpu, run);
05330448 1788
7cbb533f 1789 if (run_ret < 0) {
dc77d341
JK
1790 if (run_ret == -EINTR || run_ret == -EAGAIN) {
1791 DPRINTF("io window exit\n");
d73cd8f4 1792 ret = EXCP_INTERRUPT;
dc77d341
JK
1793 break;
1794 }
7b011fbc
ME
1795 fprintf(stderr, "error: kvm run failed %s\n",
1796 strerror(-run_ret));
dae02ba5
LV
1797#ifdef TARGET_PPC
1798 if (run_ret == -EBUSY) {
1799 fprintf(stderr,
1800 "This is probably because your SMT is enabled.\n"
1801 "VCPU can only run on primary threads with all "
1802 "secondary threads offline.\n");
1803 }
1804#endif
a85e130e
PB
1805 ret = -1;
1806 break;
05330448
AL
1807 }
1808
b76ac80a 1809 trace_kvm_run_exit(cpu->cpu_index, run->exit_reason);
05330448
AL
1810 switch (run->exit_reason) {
1811 case KVM_EXIT_IO:
8c0d577e 1812 DPRINTF("handle_io\n");
80b7d2ef 1813 /* Called outside BQL */
4c663752 1814 kvm_handle_io(run->io.port, attrs,
b30e93e9
JK
1815 (uint8_t *)run + run->io.data_offset,
1816 run->io.direction,
1817 run->io.size,
1818 run->io.count);
d73cd8f4 1819 ret = 0;
05330448
AL
1820 break;
1821 case KVM_EXIT_MMIO:
8c0d577e 1822 DPRINTF("handle_mmio\n");
de7ea885 1823 /* Called outside BQL */
4c663752
PB
1824 address_space_rw(&address_space_memory,
1825 run->mmio.phys_addr, attrs,
1826 run->mmio.data,
1827 run->mmio.len,
1828 run->mmio.is_write);
d73cd8f4 1829 ret = 0;
05330448
AL
1830 break;
1831 case KVM_EXIT_IRQ_WINDOW_OPEN:
8c0d577e 1832 DPRINTF("irq_window_open\n");
d73cd8f4 1833 ret = EXCP_INTERRUPT;
05330448
AL
1834 break;
1835 case KVM_EXIT_SHUTDOWN:
8c0d577e 1836 DPRINTF("shutdown\n");
05330448 1837 qemu_system_reset_request();
d73cd8f4 1838 ret = EXCP_INTERRUPT;
05330448
AL
1839 break;
1840 case KVM_EXIT_UNKNOWN:
bb44e0d1
JK
1841 fprintf(stderr, "KVM: unknown exit, hardware reason %" PRIx64 "\n",
1842 (uint64_t)run->hw.hardware_exit_reason);
73aaec4a 1843 ret = -1;
05330448 1844 break;
7c80eef8 1845 case KVM_EXIT_INTERNAL_ERROR:
5326ab55 1846 ret = kvm_handle_internal_error(cpu, run);
7c80eef8 1847 break;
99040447
PS
1848 case KVM_EXIT_SYSTEM_EVENT:
1849 switch (run->system_event.type) {
1850 case KVM_SYSTEM_EVENT_SHUTDOWN:
1851 qemu_system_shutdown_request();
1852 ret = EXCP_INTERRUPT;
1853 break;
1854 case KVM_SYSTEM_EVENT_RESET:
1855 qemu_system_reset_request();
1856 ret = EXCP_INTERRUPT;
1857 break;
1858 default:
1859 DPRINTF("kvm_arch_handle_exit\n");
1860 ret = kvm_arch_handle_exit(cpu, run);
1861 break;
1862 }
1863 break;
05330448 1864 default:
8c0d577e 1865 DPRINTF("kvm_arch_handle_exit\n");
20d695a9 1866 ret = kvm_arch_handle_exit(cpu, run);
05330448
AL
1867 break;
1868 }
d73cd8f4 1869 } while (ret == 0);
05330448 1870
4b8523ee
JK
1871 qemu_mutex_lock_iothread();
1872
73aaec4a 1873 if (ret < 0) {
878096ee 1874 cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_CODE);
0461d5a6 1875 vm_stop(RUN_STATE_INTERNAL_ERROR);
becfc390
AL
1876 }
1877
fcd7d003 1878 cpu->exit_request = 0;
05330448
AL
1879 return ret;
1880}
1881
984b5181 1882int kvm_ioctl(KVMState *s, int type, ...)
05330448
AL
1883{
1884 int ret;
984b5181
AL
1885 void *arg;
1886 va_list ap;
05330448 1887
984b5181
AL
1888 va_start(ap, type);
1889 arg = va_arg(ap, void *);
1890 va_end(ap);
1891
9c775729 1892 trace_kvm_ioctl(type, arg);
984b5181 1893 ret = ioctl(s->fd, type, arg);
a426e122 1894 if (ret == -1) {
05330448 1895 ret = -errno;
a426e122 1896 }
05330448
AL
1897 return ret;
1898}
1899
984b5181 1900int kvm_vm_ioctl(KVMState *s, int type, ...)
05330448
AL
1901{
1902 int ret;
984b5181
AL
1903 void *arg;
1904 va_list ap;
1905
1906 va_start(ap, type);
1907 arg = va_arg(ap, void *);
1908 va_end(ap);
05330448 1909
9c775729 1910 trace_kvm_vm_ioctl(type, arg);
984b5181 1911 ret = ioctl(s->vmfd, type, arg);
a426e122 1912 if (ret == -1) {
05330448 1913 ret = -errno;
a426e122 1914 }
05330448
AL
1915 return ret;
1916}
1917
1bc22652 1918int kvm_vcpu_ioctl(CPUState *cpu, int type, ...)
05330448
AL
1919{
1920 int ret;
984b5181
AL
1921 void *arg;
1922 va_list ap;
1923
1924 va_start(ap, type);
1925 arg = va_arg(ap, void *);
1926 va_end(ap);
05330448 1927
9c775729 1928 trace_kvm_vcpu_ioctl(cpu->cpu_index, type, arg);
8737c51c 1929 ret = ioctl(cpu->kvm_fd, type, arg);
a426e122 1930 if (ret == -1) {
05330448 1931 ret = -errno;
a426e122 1932 }
05330448
AL
1933 return ret;
1934}
bd322087 1935
0a6a7cca
CD
1936int kvm_device_ioctl(int fd, int type, ...)
1937{
1938 int ret;
1939 void *arg;
1940 va_list ap;
1941
1942 va_start(ap, type);
1943 arg = va_arg(ap, void *);
1944 va_end(ap);
1945
1946 trace_kvm_device_ioctl(fd, type, arg);
1947 ret = ioctl(fd, type, arg);
1948 if (ret == -1) {
1949 ret = -errno;
1950 }
1951 return ret;
1952}
1953
d0a073a1
DD
1954int kvm_vm_check_attr(KVMState *s, uint32_t group, uint64_t attr)
1955{
1956 int ret;
1957 struct kvm_device_attr attribute = {
1958 .group = group,
1959 .attr = attr,
1960 };
1961
1962 if (!kvm_vm_attributes_allowed) {
1963 return 0;
1964 }
1965
1966 ret = kvm_vm_ioctl(s, KVM_HAS_DEVICE_ATTR, &attribute);
1967 /* kvm returns 0 on success for HAS_DEVICE_ATTR */
1968 return ret ? 0 : 1;
1969}
1970
bd322087
AL
1971int kvm_has_sync_mmu(void)
1972{
94a8d39a 1973 return kvm_check_extension(kvm_state, KVM_CAP_SYNC_MMU);
bd322087 1974}
e22a25c9 1975
a0fb002c
JK
1976int kvm_has_vcpu_events(void)
1977{
1978 return kvm_state->vcpu_events;
1979}
1980
b0b1d690
JK
1981int kvm_has_robust_singlestep(void)
1982{
1983 return kvm_state->robust_singlestep;
1984}
1985
ff44f1a3
JK
1986int kvm_has_debugregs(void)
1987{
1988 return kvm_state->debugregs;
1989}
1990
f1665b21
SY
1991int kvm_has_xsave(void)
1992{
1993 return kvm_state->xsave;
1994}
1995
1996int kvm_has_xcrs(void)
1997{
1998 return kvm_state->xcrs;
1999}
2000
8a7c7393
JK
2001int kvm_has_pit_state2(void)
2002{
2003 return kvm_state->pit_state2;
2004}
2005
d2f2b8a7
SH
2006int kvm_has_many_ioeventfds(void)
2007{
2008 if (!kvm_enabled()) {
2009 return 0;
2010 }
2011 return kvm_state->many_ioeventfds;
2012}
2013
84b058d7
JK
2014int kvm_has_gsi_routing(void)
2015{
a9c5eb0d 2016#ifdef KVM_CAP_IRQ_ROUTING
84b058d7 2017 return kvm_check_extension(kvm_state, KVM_CAP_IRQ_ROUTING);
a9c5eb0d
AG
2018#else
2019 return false;
2020#endif
84b058d7
JK
2021}
2022
3ab73842
JK
2023int kvm_has_intx_set_mask(void)
2024{
2025 return kvm_state->intx_set_mask;
2026}
2027
6f0437e8
JK
2028void kvm_setup_guest_memory(void *start, size_t size)
2029{
2030 if (!kvm_has_sync_mmu()) {
e78815a5 2031 int ret = qemu_madvise(start, size, QEMU_MADV_DONTFORK);
6f0437e8
JK
2032
2033 if (ret) {
e78815a5
AF
2034 perror("qemu_madvise");
2035 fprintf(stderr,
2036 "Need MADV_DONTFORK in absence of synchronous KVM MMU\n");
6f0437e8
JK
2037 exit(1);
2038 }
6f0437e8
JK
2039 }
2040}
2041
e22a25c9 2042#ifdef KVM_CAP_SET_GUEST_DEBUG
a60f24b5 2043struct kvm_sw_breakpoint *kvm_find_sw_breakpoint(CPUState *cpu,
e22a25c9
AL
2044 target_ulong pc)
2045{
2046 struct kvm_sw_breakpoint *bp;
2047
a60f24b5 2048 QTAILQ_FOREACH(bp, &cpu->kvm_state->kvm_sw_breakpoints, entry) {
a426e122 2049 if (bp->pc == pc) {
e22a25c9 2050 return bp;
a426e122 2051 }
e22a25c9
AL
2052 }
2053 return NULL;
2054}
2055
a60f24b5 2056int kvm_sw_breakpoints_active(CPUState *cpu)
e22a25c9 2057{
a60f24b5 2058 return !QTAILQ_EMPTY(&cpu->kvm_state->kvm_sw_breakpoints);
e22a25c9
AL
2059}
2060
452e4751
GC
2061struct kvm_set_guest_debug_data {
2062 struct kvm_guest_debug dbg;
a60f24b5 2063 CPUState *cpu;
452e4751
GC
2064 int err;
2065};
2066
2067static void kvm_invoke_set_guest_debug(void *data)
2068{
2069 struct kvm_set_guest_debug_data *dbg_data = data;
b3807725 2070
a60f24b5
AF
2071 dbg_data->err = kvm_vcpu_ioctl(dbg_data->cpu, KVM_SET_GUEST_DEBUG,
2072 &dbg_data->dbg);
452e4751
GC
2073}
2074
38e478ec 2075int kvm_update_guest_debug(CPUState *cpu, unsigned long reinject_trap)
e22a25c9 2076{
452e4751 2077 struct kvm_set_guest_debug_data data;
e22a25c9 2078
b0b1d690 2079 data.dbg.control = reinject_trap;
e22a25c9 2080
ed2803da 2081 if (cpu->singlestep_enabled) {
b0b1d690
JK
2082 data.dbg.control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_SINGLESTEP;
2083 }
20d695a9 2084 kvm_arch_update_guest_debug(cpu, &data.dbg);
a60f24b5 2085 data.cpu = cpu;
e22a25c9 2086
f100f0b3 2087 run_on_cpu(cpu, kvm_invoke_set_guest_debug, &data);
452e4751 2088 return data.err;
e22a25c9
AL
2089}
2090
62278814 2091int kvm_insert_breakpoint(CPUState *cpu, target_ulong addr,
e22a25c9
AL
2092 target_ulong len, int type)
2093{
2094 struct kvm_sw_breakpoint *bp;
e22a25c9
AL
2095 int err;
2096
2097 if (type == GDB_BREAKPOINT_SW) {
80b7cd73 2098 bp = kvm_find_sw_breakpoint(cpu, addr);
e22a25c9
AL
2099 if (bp) {
2100 bp->use_count++;
2101 return 0;
2102 }
2103
7267c094 2104 bp = g_malloc(sizeof(struct kvm_sw_breakpoint));
e22a25c9
AL
2105 bp->pc = addr;
2106 bp->use_count = 1;
80b7cd73 2107 err = kvm_arch_insert_sw_breakpoint(cpu, bp);
e22a25c9 2108 if (err) {
7267c094 2109 g_free(bp);
e22a25c9
AL
2110 return err;
2111 }
2112
80b7cd73 2113 QTAILQ_INSERT_HEAD(&cpu->kvm_state->kvm_sw_breakpoints, bp, entry);
e22a25c9
AL
2114 } else {
2115 err = kvm_arch_insert_hw_breakpoint(addr, len, type);
a426e122 2116 if (err) {
e22a25c9 2117 return err;
a426e122 2118 }
e22a25c9
AL
2119 }
2120
bdc44640 2121 CPU_FOREACH(cpu) {
38e478ec 2122 err = kvm_update_guest_debug(cpu, 0);
a426e122 2123 if (err) {
e22a25c9 2124 return err;
a426e122 2125 }
e22a25c9
AL
2126 }
2127 return 0;
2128}
2129
62278814 2130int kvm_remove_breakpoint(CPUState *cpu, target_ulong addr,
e22a25c9
AL
2131 target_ulong len, int type)
2132{
2133 struct kvm_sw_breakpoint *bp;
e22a25c9
AL
2134 int err;
2135
2136 if (type == GDB_BREAKPOINT_SW) {
80b7cd73 2137 bp = kvm_find_sw_breakpoint(cpu, addr);
a426e122 2138 if (!bp) {
e22a25c9 2139 return -ENOENT;
a426e122 2140 }
e22a25c9
AL
2141
2142 if (bp->use_count > 1) {
2143 bp->use_count--;
2144 return 0;
2145 }
2146
80b7cd73 2147 err = kvm_arch_remove_sw_breakpoint(cpu, bp);
a426e122 2148 if (err) {
e22a25c9 2149 return err;
a426e122 2150 }
e22a25c9 2151
80b7cd73 2152 QTAILQ_REMOVE(&cpu->kvm_state->kvm_sw_breakpoints, bp, entry);
7267c094 2153 g_free(bp);
e22a25c9
AL
2154 } else {
2155 err = kvm_arch_remove_hw_breakpoint(addr, len, type);
a426e122 2156 if (err) {
e22a25c9 2157 return err;
a426e122 2158 }
e22a25c9
AL
2159 }
2160
bdc44640 2161 CPU_FOREACH(cpu) {
38e478ec 2162 err = kvm_update_guest_debug(cpu, 0);
a426e122 2163 if (err) {
e22a25c9 2164 return err;
a426e122 2165 }
e22a25c9
AL
2166 }
2167 return 0;
2168}
2169
1d5791f4 2170void kvm_remove_all_breakpoints(CPUState *cpu)
e22a25c9
AL
2171{
2172 struct kvm_sw_breakpoint *bp, *next;
80b7cd73 2173 KVMState *s = cpu->kvm_state;
dc54e252 2174 CPUState *tmpcpu;
e22a25c9 2175
72cf2d4f 2176 QTAILQ_FOREACH_SAFE(bp, &s->kvm_sw_breakpoints, entry, next) {
80b7cd73 2177 if (kvm_arch_remove_sw_breakpoint(cpu, bp) != 0) {
e22a25c9 2178 /* Try harder to find a CPU that currently sees the breakpoint. */
dc54e252
CG
2179 CPU_FOREACH(tmpcpu) {
2180 if (kvm_arch_remove_sw_breakpoint(tmpcpu, bp) == 0) {
e22a25c9 2181 break;
a426e122 2182 }
e22a25c9
AL
2183 }
2184 }
78021d6d
JK
2185 QTAILQ_REMOVE(&s->kvm_sw_breakpoints, bp, entry);
2186 g_free(bp);
e22a25c9
AL
2187 }
2188 kvm_arch_remove_all_hw_breakpoints();
2189
bdc44640 2190 CPU_FOREACH(cpu) {
38e478ec 2191 kvm_update_guest_debug(cpu, 0);
a426e122 2192 }
e22a25c9
AL
2193}
2194
2195#else /* !KVM_CAP_SET_GUEST_DEBUG */
2196
38e478ec 2197int kvm_update_guest_debug(CPUState *cpu, unsigned long reinject_trap)
e22a25c9
AL
2198{
2199 return -EINVAL;
2200}
2201
62278814 2202int kvm_insert_breakpoint(CPUState *cpu, target_ulong addr,
e22a25c9
AL
2203 target_ulong len, int type)
2204{
2205 return -EINVAL;
2206}
2207
62278814 2208int kvm_remove_breakpoint(CPUState *cpu, target_ulong addr,
e22a25c9
AL
2209 target_ulong len, int type)
2210{
2211 return -EINVAL;
2212}
2213
1d5791f4 2214void kvm_remove_all_breakpoints(CPUState *cpu)
e22a25c9
AL
2215{
2216}
2217#endif /* !KVM_CAP_SET_GUEST_DEBUG */
cc84de95 2218
491d6e80 2219int kvm_set_signal_mask(CPUState *cpu, const sigset_t *sigset)
cc84de95 2220{
aed6efb9 2221 KVMState *s = kvm_state;
cc84de95
MT
2222 struct kvm_signal_mask *sigmask;
2223 int r;
2224
a426e122 2225 if (!sigset) {
1bc22652 2226 return kvm_vcpu_ioctl(cpu, KVM_SET_SIGNAL_MASK, NULL);
a426e122 2227 }
cc84de95 2228
7267c094 2229 sigmask = g_malloc(sizeof(*sigmask) + sizeof(*sigset));
cc84de95 2230
aed6efb9 2231 sigmask->len = s->sigmask_len;
cc84de95 2232 memcpy(sigmask->sigset, sigset, sizeof(*sigset));
1bc22652 2233 r = kvm_vcpu_ioctl(cpu, KVM_SET_SIGNAL_MASK, sigmask);
7267c094 2234 g_free(sigmask);
cc84de95
MT
2235
2236 return r;
2237}
290adf38 2238int kvm_on_sigbus_vcpu(CPUState *cpu, int code, void *addr)
a1b87fe0 2239{
20d695a9 2240 return kvm_arch_on_sigbus_vcpu(cpu, code, addr);
a1b87fe0
JK
2241}
2242
2243int kvm_on_sigbus(int code, void *addr)
2244{
2245 return kvm_arch_on_sigbus(code, addr);
2246}
0a6a7cca
CD
2247
2248int kvm_create_device(KVMState *s, uint64_t type, bool test)
2249{
2250 int ret;
2251 struct kvm_create_device create_dev;
2252
2253 create_dev.type = type;
2254 create_dev.fd = -1;
2255 create_dev.flags = test ? KVM_CREATE_DEVICE_TEST : 0;
2256
2257 if (!kvm_check_extension(s, KVM_CAP_DEVICE_CTRL)) {
2258 return -ENOTSUP;
2259 }
2260
2261 ret = kvm_vm_ioctl(s, KVM_CREATE_DEVICE, &create_dev);
2262 if (ret) {
2263 return ret;
2264 }
2265
2266 return test ? 0 : create_dev.fd;
2267}
ada4135f
CH
2268
2269int kvm_set_one_reg(CPUState *cs, uint64_t id, void *source)
2270{
2271 struct kvm_one_reg reg;
2272 int r;
2273
2274 reg.id = id;
2275 reg.addr = (uintptr_t) source;
2276 r = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
2277 if (r) {
2278 trace_kvm_failed_reg_set(id, strerror(r));
2279 }
2280 return r;
2281}
2282
2283int kvm_get_one_reg(CPUState *cs, uint64_t id, void *target)
2284{
2285 struct kvm_one_reg reg;
2286 int r;
2287
2288 reg.id = id;
2289 reg.addr = (uintptr_t) target;
2290 r = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
2291 if (r) {
2292 trace_kvm_failed_reg_get(id, strerror(r));
2293 }
2294 return r;
2295}
782c3f29
EH
2296
2297static void kvm_accel_class_init(ObjectClass *oc, void *data)
2298{
2299 AccelClass *ac = ACCEL_CLASS(oc);
2300 ac->name = "KVM";
0d15da8e 2301 ac->init_machine = kvm_init;
782c3f29
EH
2302 ac->allowed = &kvm_allowed;
2303}
2304
2305static const TypeInfo kvm_accel_type = {
2306 .name = TYPE_KVM_ACCEL,
2307 .parent = TYPE_ACCEL,
2308 .class_init = kvm_accel_class_init,
fc02086b 2309 .instance_size = sizeof(KVMState),
782c3f29
EH
2310};
2311
2312static void kvm_type_init(void)
2313{
2314 type_register_static(&kvm_accel_type);
2315}
2316
2317type_init(kvm_type_init);