accel/kvm/kvm-all.c

   1 /*
   2  * QEMU KVM support
   3  *
   4  * Copyright IBM, Corp. 2008
   5  *           Red Hat, Inc. 2008
   6  *
   7  * Authors:
   8  *  Anthony Liguori   <aliguori@us.ibm.com>
   9  *  Glauber Costa     <gcosta@redhat.com>
  10  *
  11  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  12  * See the COPYING file in the top-level directory.
  13  *
  14  */
  15
  16 #include "qemu/osdep.h"
  17 #include <sys/ioctl.h>
  18
  19 #include <linux/kvm.h>
  20
  21 #include "qemu/atomic.h"
  22 #include "qemu/option.h"
  23 #include "qemu/config-file.h"
  24 #include "qemu/error-report.h"
  25 #include "qapi/error.h"
  26 #include "hw/hw.h"
  27 #include "hw/pci/msi.h"
  28 #include "hw/pci/msix.h"
  29 #include "hw/s390x/adapter.h"
  30 #include "exec/gdbstub.h"
  31 #include "sysemu/kvm_int.h"
  32 #include "sysemu/cpus.h"
  33 #include "qemu/bswap.h"
  34 #include "exec/memory.h"
  35 #include "exec/ram_addr.h"
  36 #include "exec/address-spaces.h"
  37 #include "qemu/event_notifier.h"
  38 #include "trace.h"
  39 #include "hw/irq.h"
  40 #include "sysemu/sev.h"
  41 #include "sysemu/balloon.h"
  42
  43 #include "hw/boards.h"
  44
  45 /* This check must be after config-host.h is included */
  46 #ifdef CONFIG_EVENTFD
  47 #include <sys/eventfd.h>
  48 #endif
  49
  50 /* KVM uses PAGE_SIZE in its definition of KVM_COALESCED_MMIO_MAX. We
  51  * need to use the real host PAGE_SIZE, as that's what KVM will use.
  52  */
  53 #define PAGE_SIZE getpagesize()
  54
  55 //#define DEBUG_KVM
  56
  57 #ifdef DEBUG_KVM
  58 #define DPRINTF(fmt, ...) \
  59     do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
  60 #else
  61 #define DPRINTF(fmt, ...) \
  62     do { } while (0)
  63 #endif
  64
  65 #define KVM_MSI_HASHTAB_SIZE    256
  66
  67 struct KVMParkedVcpu {
  68     unsigned long vcpu_id;
  69     int kvm_fd;
  70     QLIST_ENTRY(KVMParkedVcpu) node;
  71 };
  72
  73 struct KVMState
  74 {
  75     AccelState parent_obj;
  76
  77     int nr_slots;
  78     int fd;
  79     int vmfd;
  80     int coalesced_mmio;
  81     int coalesced_pio;
  82     struct kvm_coalesced_mmio_ring *coalesced_mmio_ring;
  83     bool coalesced_flush_in_progress;
  84     int vcpu_events;
  85     int robust_singlestep;
  86     int debugregs;
  87 #ifdef KVM_CAP_SET_GUEST_DEBUG
  88     QTAILQ_HEAD(, kvm_sw_breakpoint) kvm_sw_breakpoints;
  89 #endif
  90     int many_ioeventfds;
  91     int intx_set_mask;
  92     bool sync_mmu;
  93     /* The man page (and posix) say ioctl numbers are signed int, but
  94      * they're not.  Linux, glibc and *BSD all treat ioctl numbers as
  95      * unsigned, and treating them as signed here can break things */
  96     unsigned irq_set_ioctl;
  97     unsigned int sigmask_len;
  98     GHashTable *gsimap;
  99 #ifdef KVM_CAP_IRQ_ROUTING
 100     struct kvm_irq_routing *irq_routes;
 101     int nr_allocated_irq_routes;
 102     unsigned long *used_gsi_bitmap;
 103     unsigned int gsi_count;
 104     QTAILQ_HEAD(, KVMMSIRoute) msi_hashtab[KVM_MSI_HASHTAB_SIZE];
 105 #endif
 106     KVMMemoryListener memory_listener;
 107     QLIST_HEAD(, KVMParkedVcpu) kvm_parked_vcpus;
 108
 109     /* memory encryption */
 110     void *memcrypt_handle;
 111     int (*memcrypt_encrypt_data)(void *handle, uint8_t *ptr, uint64_t len);
 112 };
 113
 114 KVMState *kvm_state;
 115 bool kvm_kernel_irqchip;
 116 bool kvm_split_irqchip;
 117 bool kvm_async_interrupts_allowed;
 118 bool kvm_halt_in_kernel_allowed;
 119 bool kvm_eventfds_allowed;
 120 bool kvm_irqfds_allowed;
 121 bool kvm_resamplefds_allowed;
 122 bool kvm_msi_via_irqfd_allowed;
 123 bool kvm_gsi_routing_allowed;
 124 bool kvm_gsi_direct_mapping;
 125 bool kvm_allowed;
 126 bool kvm_readonly_mem_allowed;
 127 bool kvm_vm_attributes_allowed;
 128 bool kvm_direct_msi_allowed;
 129 bool kvm_ioeventfd_any_length_allowed;
 130 bool kvm_msi_use_devid;
 131 static bool kvm_immediate_exit;
 132
 133 static const KVMCapabilityInfo kvm_required_capabilites[] = {
 134     KVM_CAP_INFO(USER_MEMORY),
 135     KVM_CAP_INFO(DESTROY_MEMORY_REGION_WORKS),
 136     KVM_CAP_INFO(JOIN_MEMORY_REGIONS_WORKS),
 137     KVM_CAP_LAST_INFO
 138 };
 139
 140 int kvm_get_max_memslots(void)
 141 {
 142     KVMState *s = KVM_STATE(current_machine->accelerator);
 143
 144     return s->nr_slots;
 145 }
 146
 147 bool kvm_memcrypt_enabled(void)
 148 {
 149     if (kvm_state && kvm_state->memcrypt_handle) {
 150         return true;
 151     }
 152
 153     return false;
 154 }
 155
 156 int kvm_memcrypt_encrypt_data(uint8_t *ptr, uint64_t len)
 157 {
 158     if (kvm_state->memcrypt_handle &&
 159         kvm_state->memcrypt_encrypt_data) {
 160         return kvm_state->memcrypt_encrypt_data(kvm_state->memcrypt_handle,
 161                                               ptr, len);
 162     }
 163
 164     return 1;
 165 }
 166
 167 static KVMSlot *kvm_get_free_slot(KVMMemoryListener *kml)
 168 {
 169     KVMState *s = kvm_state;
 170     int i;
 171
 172     for (i = 0; i < s->nr_slots; i++) {
 173         if (kml->slots[i].memory_size == 0) {
 174             return &kml->slots[i];
 175         }
 176     }
 177
 178     return NULL;
 179 }
 180
 181 bool kvm_has_free_slot(MachineState *ms)
 182 {
 183     KVMState *s = KVM_STATE(ms->accelerator);
 184
 185     return kvm_get_free_slot(&s->memory_listener);
 186 }
 187
 188 static KVMSlot *kvm_alloc_slot(KVMMemoryListener *kml)
 189 {
 190     KVMSlot *slot = kvm_get_free_slot(kml);
 191
 192     if (slot) {
 193         return slot;
 194     }
 195
 196     fprintf(stderr, "%s: no free slot available\n", __func__);
 197     abort();
 198 }
 199
 200 static KVMSlot *kvm_lookup_matching_slot(KVMMemoryListener *kml,
 201                                          hwaddr start_addr,
 202                                          hwaddr size)
 203 {
 204     KVMState *s = kvm_state;
 205     int i;
 206
 207     for (i = 0; i < s->nr_slots; i++) {
 208         KVMSlot *mem = &kml->slots[i];
 209
 210         if (start_addr == mem->start_addr && size == mem->memory_size) {
 211             return mem;
 212         }
 213     }
 214
 215     return NULL;
 216 }
 217
 218 /*
 219  * Calculate and align the start address and the size of the section.
 220  * Return the size. If the size is 0, the aligned section is empty.
 221  */
 222 static hwaddr kvm_align_section(MemoryRegionSection *section,
 223                                 hwaddr *start)
 224 {
 225     hwaddr size = int128_get64(section->size);
 226     hwaddr delta, aligned;
 227
 228     /* kvm works in page size chunks, but the function may be called
 229        with sub-page size and unaligned start address. Pad the start
 230        address to next and truncate size to previous page boundary. */
 231     aligned = ROUND_UP(section->offset_within_address_space,
 232                        qemu_real_host_page_size);
 233     delta = aligned - section->offset_within_address_space;
 234     *start = aligned;
 235     if (delta > size) {
 236         return 0;
 237     }
 238
 239     return (size - delta) & qemu_real_host_page_mask;
 240 }
 241
 242 int kvm_physical_memory_addr_from_host(KVMState *s, void *ram,
 243                                        hwaddr *phys_addr)
 244 {
 245     KVMMemoryListener *kml = &s->memory_listener;
 246     int i;
 247
 248     for (i = 0; i < s->nr_slots; i++) {
 249         KVMSlot *mem = &kml->slots[i];
 250
 251         if (ram >= mem->ram && ram < mem->ram + mem->memory_size) {
 252             *phys_addr = mem->start_addr + (ram - mem->ram);
 253             return 1;
 254         }
 255     }
 256
 257     return 0;
 258 }
 259
 260 static int kvm_set_user_memory_region(KVMMemoryListener *kml, KVMSlot *slot, bool new)
 261 {
 262     KVMState *s = kvm_state;
 263     struct kvm_userspace_memory_region mem;
 264     int ret;
 265
 266     mem.slot = slot->slot | (kml->as_id << 16);
 267     mem.guest_phys_addr = slot->start_addr;
 268     mem.userspace_addr = (unsigned long)slot->ram;
 269     mem.flags = slot->flags;
 270
 271     if (slot->memory_size && !new && (mem.flags ^ slot->old_flags) & KVM_MEM_READONLY) {
 272         /* Set the slot size to 0 before setting the slot to the desired
 273          * value. This is needed based on KVM commit 75d61fbc. */
 274         mem.memory_size = 0;
 275         kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
 276     }
 277     mem.memory_size = slot->memory_size;
 278     ret = kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
 279     slot->old_flags = mem.flags;
 280     trace_kvm_set_user_memory(mem.slot, mem.flags, mem.guest_phys_addr,
 281                               mem.memory_size, mem.userspace_addr, ret);
 282     return ret;
 283 }
 284
 285 int kvm_destroy_vcpu(CPUState *cpu)
 286 {
 287     KVMState *s = kvm_state;
 288     long mmap_size;
 289     struct KVMParkedVcpu *vcpu = NULL;
 290     int ret = 0;
 291
 292     DPRINTF("kvm_destroy_vcpu\n");
 293
 294     mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0);
 295     if (mmap_size < 0) {
 296         ret = mmap_size;
 297         DPRINTF("KVM_GET_VCPU_MMAP_SIZE failed\n");
 298         goto err;
 299     }
 300
 301     ret = munmap(cpu->kvm_run, mmap_size);
 302     if (ret < 0) {
 303         goto err;
 304     }
 305
 306     vcpu = g_malloc0(sizeof(*vcpu));
 307     vcpu->vcpu_id = kvm_arch_vcpu_id(cpu);
 308     vcpu->kvm_fd = cpu->kvm_fd;
 309     QLIST_INSERT_HEAD(&kvm_state->kvm_parked_vcpus, vcpu, node);
 310 err:
 311     return ret;
 312 }
 313
 314 static int kvm_get_vcpu(KVMState *s, unsigned long vcpu_id)
 315 {
 316     struct KVMParkedVcpu *cpu;
 317
 318     QLIST_FOREACH(cpu, &s->kvm_parked_vcpus, node) {
 319         if (cpu->vcpu_id == vcpu_id) {
 320             int kvm_fd;
 321
 322             QLIST_REMOVE(cpu, node);
 323             kvm_fd = cpu->kvm_fd;
 324             g_free(cpu);
 325             return kvm_fd;
 326         }
 327     }
 328
 329     return kvm_vm_ioctl(s, KVM_CREATE_VCPU, (void *)vcpu_id);
 330 }
 331
 332 int kvm_init_vcpu(CPUState *cpu)
 333 {
 334     KVMState *s = kvm_state;
 335     long mmap_size;
 336     int ret;
 337
 338     DPRINTF("kvm_init_vcpu\n");
 339
 340     ret = kvm_get_vcpu(s, kvm_arch_vcpu_id(cpu));
 341     if (ret < 0) {
 342         DPRINTF("kvm_create_vcpu failed\n");
 343         goto err;
 344     }
 345
 346     cpu->kvm_fd = ret;
 347     cpu->kvm_state = s;
 348     cpu->vcpu_dirty = true;
 349
 350     mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0);
 351     if (mmap_size < 0) {
 352         ret = mmap_size;
 353         DPRINTF("KVM_GET_VCPU_MMAP_SIZE failed\n");
 354         goto err;
 355     }
 356
 357     cpu->kvm_run = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED,
 358                         cpu->kvm_fd, 0);
 359     if (cpu->kvm_run == MAP_FAILED) {
 360         ret = -errno;
 361         DPRINTF("mmap'ing vcpu state failed\n");
 362         goto err;
 363     }
 364
 365     if (s->coalesced_mmio && !s->coalesced_mmio_ring) {
 366         s->coalesced_mmio_ring =
 367             (void *)cpu->kvm_run + s->coalesced_mmio * PAGE_SIZE;
 368     }
 369
 370     ret = kvm_arch_init_vcpu(cpu);
 371 err:
 372     return ret;
 373 }
 374
 375 /*
 376  * dirty pages logging control
 377  */
 378
 379 static int kvm_mem_flags(MemoryRegion *mr)
 380 {
 381     bool readonly = mr->readonly || memory_region_is_romd(mr);
 382     int flags = 0;
 383
 384     if (memory_region_get_dirty_log_mask(mr) != 0) {
 385         flags |= KVM_MEM_LOG_DIRTY_PAGES;
 386     }
 387     if (readonly && kvm_readonly_mem_allowed) {
 388         flags |= KVM_MEM_READONLY;
 389     }
 390     return flags;
 391 }
 392
 393 static int kvm_slot_update_flags(KVMMemoryListener *kml, KVMSlot *mem,
 394                                  MemoryRegion *mr)
 395 {
 396     mem->flags = kvm_mem_flags(mr);
 397
 398     /* If nothing changed effectively, no need to issue ioctl */
 399     if (mem->flags == mem->old_flags) {
 400         return 0;
 401     }
 402
 403     return kvm_set_user_memory_region(kml, mem, false);
 404 }
 405
 406 static int kvm_section_update_flags(KVMMemoryListener *kml,
 407                                     MemoryRegionSection *section)
 408 {
 409     hwaddr start_addr, size;
 410     KVMSlot *mem;
 411
 412     size = kvm_align_section(section, &start_addr);
 413     if (!size) {
 414         return 0;
 415     }
 416
 417     mem = kvm_lookup_matching_slot(kml, start_addr, size);
 418     if (!mem) {
 419         /* We don't have a slot if we want to trap every access. */
 420         return 0;
 421     }
 422
 423     return kvm_slot_update_flags(kml, mem, section->mr);
 424 }
 425
 426 static void kvm_log_start(MemoryListener *listener,
 427                           MemoryRegionSection *section,
 428                           int old, int new)
 429 {
 430     KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener);
 431     int r;
 432
 433     if (old != 0) {
 434         return;
 435     }
 436
 437     r = kvm_section_update_flags(kml, section);
 438     if (r < 0) {
 439         abort();
 440     }
 441 }
 442
 443 static void kvm_log_stop(MemoryListener *listener,
 444                           MemoryRegionSection *section,
 445                           int old, int new)
 446 {
 447     KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener);
 448     int r;
 449
 450     if (new != 0) {
 451         return;
 452     }
 453
 454     r = kvm_section_update_flags(kml, section);
 455     if (r < 0) {
 456         abort();
 457     }
 458 }
 459
 460 /* get kvm's dirty pages bitmap and update qemu's */
 461 static int kvm_get_dirty_pages_log_range(MemoryRegionSection *section,
 462                                          unsigned long *bitmap)
 463 {
 464     ram_addr_t start = section->offset_within_region +
 465                        memory_region_get_ram_addr(section->mr);
 466     ram_addr_t pages = int128_get64(section->size) / getpagesize();
 467
 468     cpu_physical_memory_set_dirty_lebitmap(bitmap, start, pages);
 469     return 0;
 470 }
 471
 472 #define ALIGN(x, y)  (((x)+(y)-1) & ~((y)-1))
 473
 474 /**
 475  * kvm_physical_sync_dirty_bitmap - Grab dirty bitmap from kernel space
 476  * This function updates qemu's dirty bitmap using
 477  * memory_region_set_dirty().  This means all bits are set
 478  * to dirty.
 479  *
 480  * @start_add: start of logged region.
 481  * @end_addr: end of logged region.
 482  */
 483 static int kvm_physical_sync_dirty_bitmap(KVMMemoryListener *kml,
 484                                           MemoryRegionSection *section)
 485 {
 486     KVMState *s = kvm_state;
 487     struct kvm_dirty_log d = {};
 488     KVMSlot *mem;
 489     hwaddr start_addr, size;
 490
 491     size = kvm_align_section(section, &start_addr);
 492     if (size) {
 493         mem = kvm_lookup_matching_slot(kml, start_addr, size);
 494         if (!mem) {
 495             /* We don't have a slot if we want to trap every access. */
 496             return 0;
 497         }
 498
 499         /* XXX bad kernel interface alert
 500          * For dirty bitmap, kernel allocates array of size aligned to
 501          * bits-per-long.  But for case when the kernel is 64bits and
 502          * the userspace is 32bits, userspace can't align to the same
 503          * bits-per-long, since sizeof(long) is different between kernel
 504          * and user space.  This way, userspace will provide buffer which
 505          * may be 4 bytes less than the kernel will use, resulting in
 506          * userspace memory corruption (which is not detectable by valgrind
 507          * too, in most cases).
 508          * So for now, let's align to 64 instead of HOST_LONG_BITS here, in
 509          * a hope that sizeof(long) won't become >8 any time soon.
 510          */
 511         size = ALIGN(((mem->memory_size) >> TARGET_PAGE_BITS),
 512                      /*HOST_LONG_BITS*/ 64) / 8;
 513         d.dirty_bitmap = g_malloc0(size);
 514
 515         d.slot = mem->slot | (kml->as_id << 16);
 516         if (kvm_vm_ioctl(s, KVM_GET_DIRTY_LOG, &d) == -1) {
 517             DPRINTF("ioctl failed %d\n", errno);
 518             g_free(d.dirty_bitmap);
 519             return -1;
 520         }
 521
 522         kvm_get_dirty_pages_log_range(section, d.dirty_bitmap);
 523         g_free(d.dirty_bitmap);
 524     }
 525
 526     return 0;
 527 }
 528
 529 static void kvm_coalesce_mmio_region(MemoryListener *listener,
 530                                      MemoryRegionSection *secion,
 531                                      hwaddr start, hwaddr size)
 532 {
 533     KVMState *s = kvm_state;
 534
 535     if (s->coalesced_mmio) {
 536         struct kvm_coalesced_mmio_zone zone;
 537
 538         zone.addr = start;
 539         zone.size = size;
 540         zone.pad = 0;
 541
 542         (void)kvm_vm_ioctl(s, KVM_REGISTER_COALESCED_MMIO, &zone);
 543     }
 544 }
 545
 546 static void kvm_uncoalesce_mmio_region(MemoryListener *listener,
 547                                        MemoryRegionSection *secion,
 548                                        hwaddr start, hwaddr size)
 549 {
 550     KVMState *s = kvm_state;
 551
 552     if (s->coalesced_mmio) {
 553         struct kvm_coalesced_mmio_zone zone;
 554
 555         zone.addr = start;
 556         zone.size = size;
 557         zone.pad = 0;
 558
 559         (void)kvm_vm_ioctl(s, KVM_UNREGISTER_COALESCED_MMIO, &zone);
 560     }
 561 }
 562
 563 static void kvm_coalesce_pio_add(MemoryListener *listener,
 564                                 MemoryRegionSection *section,
 565                                 hwaddr start, hwaddr size)
 566 {
 567     KVMState *s = kvm_state;
 568
 569     if (s->coalesced_pio) {
 570         struct kvm_coalesced_mmio_zone zone;
 571
 572         zone.addr = start;
 573         zone.size = size;
 574         zone.pio = 1;
 575
 576         (void)kvm_vm_ioctl(s, KVM_REGISTER_COALESCED_MMIO, &zone);
 577     }
 578 }
 579
 580 static void kvm_coalesce_pio_del(MemoryListener *listener,
 581                                 MemoryRegionSection *section,
 582                                 hwaddr start, hwaddr size)
 583 {
 584     KVMState *s = kvm_state;
 585
 586     if (s->coalesced_pio) {
 587         struct kvm_coalesced_mmio_zone zone;
 588
 589         zone.addr = start;
 590         zone.size = size;
 591         zone.pio = 1;
 592
 593         (void)kvm_vm_ioctl(s, KVM_UNREGISTER_COALESCED_MMIO, &zone);
 594      }
 595 }
 596
 597 static MemoryListener kvm_coalesced_pio_listener = {
 598     .coalesced_io_add = kvm_coalesce_pio_add,
 599     .coalesced_io_del = kvm_coalesce_pio_del,
 600 };
 601
 602 int kvm_check_extension(KVMState *s, unsigned int extension)
 603 {
 604     int ret;
 605
 606     ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, extension);
 607     if (ret < 0) {
 608         ret = 0;
 609     }
 610
 611     return ret;
 612 }
 613
 614 int kvm_vm_check_extension(KVMState *s, unsigned int extension)
 615 {
 616     int ret;
 617
 618     ret = kvm_vm_ioctl(s, KVM_CHECK_EXTENSION, extension);
 619     if (ret < 0) {
 620         /* VM wide version not implemented, use global one instead */
 621         ret = kvm_check_extension(s, extension);
 622     }
 623
 624     return ret;
 625 }
 626
 627 static uint32_t adjust_ioeventfd_endianness(uint32_t val, uint32_t size)
 628 {
 629 #if defined(HOST_WORDS_BIGENDIAN) != defined(TARGET_WORDS_BIGENDIAN)
 630     /* The kernel expects ioeventfd values in HOST_WORDS_BIGENDIAN
 631      * endianness, but the memory core hands them in target endianness.
 632      * For example, PPC is always treated as big-endian even if running
 633      * on KVM and on PPC64LE.  Correct here.
 634      */
 635     switch (size) {
 636     case 2:
 637         val = bswap16(val);
 638         break;
 639     case 4:
 640         val = bswap32(val);
 641         break;
 642     }
 643 #endif
 644     return val;
 645 }
 646
 647 static int kvm_set_ioeventfd_mmio(int fd, hwaddr addr, uint32_t val,
 648                                   bool assign, uint32_t size, bool datamatch)
 649 {
 650     int ret;
 651     struct kvm_ioeventfd iofd = {
 652         .datamatch = datamatch ? adjust_ioeventfd_endianness(val, size) : 0,
 653         .addr = addr,
 654         .len = size,
 655         .flags = 0,
 656         .fd = fd,
 657     };
 658
 659     trace_kvm_set_ioeventfd_mmio(fd, (uint64_t)addr, val, assign, size,
 660                                  datamatch);
 661     if (!kvm_enabled()) {
 662         return -ENOSYS;
 663     }
 664
 665     if (datamatch) {
 666         iofd.flags |= KVM_IOEVENTFD_FLAG_DATAMATCH;
 667     }
 668     if (!assign) {
 669         iofd.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN;
 670     }
 671
 672     ret = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &iofd);
 673
 674     if (ret < 0) {
 675         return -errno;
 676     }
 677
 678     return 0;
 679 }
 680
 681 static int kvm_set_ioeventfd_pio(int fd, uint16_t addr, uint16_t val,
 682                                  bool assign, uint32_t size, bool datamatch)
 683 {
 684     struct kvm_ioeventfd kick = {
 685         .datamatch = datamatch ? adjust_ioeventfd_endianness(val, size) : 0,
 686         .addr = addr,
 687         .flags = KVM_IOEVENTFD_FLAG_PIO,
 688         .len = size,
 689         .fd = fd,
 690     };
 691     int r;
 692     trace_kvm_set_ioeventfd_pio(fd, addr, val, assign, size, datamatch);
 693     if (!kvm_enabled()) {
 694         return -ENOSYS;
 695     }
 696     if (datamatch) {
 697         kick.flags |= KVM_IOEVENTFD_FLAG_DATAMATCH;
 698     }
 699     if (!assign) {
 700         kick.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN;
 701     }
 702     r = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &kick);
 703     if (r < 0) {
 704         return r;
 705     }
 706     return 0;
 707 }
 708
 709
 710 static int kvm_check_many_ioeventfds(void)
 711 {
 712     /* Userspace can use ioeventfd for io notification.  This requires a host
 713      * that supports eventfd(2) and an I/O thread; since eventfd does not
 714      * support SIGIO it cannot interrupt the vcpu.
 715      *
 716      * Older kernels have a 6 device limit on the KVM io bus.  Find out so we
 717      * can avoid creating too many ioeventfds.
 718      */
 719 #if defined(CONFIG_EVENTFD)
 720     int ioeventfds[7];
 721     int i, ret = 0;
 722     for (i = 0; i < ARRAY_SIZE(ioeventfds); i++) {
 723         ioeventfds[i] = eventfd(0, EFD_CLOEXEC);
 724         if (ioeventfds[i] < 0) {
 725             break;
 726         }
 727         ret = kvm_set_ioeventfd_pio(ioeventfds[i], 0, i, true, 2, true);
 728         if (ret < 0) {
 729             close(ioeventfds[i]);
 730             break;
 731         }
 732     }
 733
 734     /* Decide whether many devices are supported or not */
 735     ret = i == ARRAY_SIZE(ioeventfds);
 736
 737     while (i-- > 0) {
 738         kvm_set_ioeventfd_pio(ioeventfds[i], 0, i, false, 2, true);
 739         close(ioeventfds[i]);
 740     }
 741     return ret;
 742 #else
 743     return 0;
 744 #endif
 745 }
 746
 747 static const KVMCapabilityInfo *
 748 kvm_check_extension_list(KVMState *s, const KVMCapabilityInfo *list)
 749 {
 750     while (list->name) {
 751         if (!kvm_check_extension(s, list->value)) {
 752             return list;
 753         }
 754         list++;
 755     }
 756     return NULL;
 757 }
 758
 759 static void kvm_set_phys_mem(KVMMemoryListener *kml,
 760                              MemoryRegionSection *section, bool add)
 761 {
 762     KVMSlot *mem;
 763     int err;
 764     MemoryRegion *mr = section->mr;
 765     bool writeable = !mr->readonly && !mr->rom_device;
 766     hwaddr start_addr, size;
 767     void *ram;
 768
 769     if (!memory_region_is_ram(mr)) {
 770         if (writeable || !kvm_readonly_mem_allowed) {
 771             return;
 772         } else if (!mr->romd_mode) {
 773             /* If the memory device is not in romd_mode, then we actually want
 774              * to remove the kvm memory slot so all accesses will trap. */
 775             add = false;
 776         }
 777     }
 778
 779     size = kvm_align_section(section, &start_addr);
 780     if (!size) {
 781         return;
 782     }
 783
 784     /* use aligned delta to align the ram address */
 785     ram = memory_region_get_ram_ptr(mr) + section->offset_within_region +
 786           (start_addr - section->offset_within_address_space);
 787
 788     if (!add) {
 789         mem = kvm_lookup_matching_slot(kml, start_addr, size);
 790         if (!mem) {
 791             return;
 792         }
 793         if (mem->flags & KVM_MEM_LOG_DIRTY_PAGES) {
 794             kvm_physical_sync_dirty_bitmap(kml, section);
 795         }
 796
 797         /* unregister the slot */
 798         mem->memory_size = 0;
 799         mem->flags = 0;
 800         err = kvm_set_user_memory_region(kml, mem, false);
 801         if (err) {
 802             fprintf(stderr, "%s: error unregistering slot: %s\n",
 803                     __func__, strerror(-err));
 804             abort();
 805         }
 806         return;
 807     }
 808
 809     /* register the new slot */
 810     mem = kvm_alloc_slot(kml);
 811     mem->memory_size = size;
 812     mem->start_addr = start_addr;
 813     mem->ram = ram;
 814     mem->flags = kvm_mem_flags(mr);
 815
 816     err = kvm_set_user_memory_region(kml, mem, true);
 817     if (err) {
 818         fprintf(stderr, "%s: error registering slot: %s\n", __func__,
 819                 strerror(-err));
 820         abort();
 821     }
 822 }
 823
 824 static void kvm_region_add(MemoryListener *listener,
 825                            MemoryRegionSection *section)
 826 {
 827     KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener);
 828
 829     memory_region_ref(section->mr);
 830     kvm_set_phys_mem(kml, section, true);
 831 }
 832
 833 static void kvm_region_del(MemoryListener *listener,
 834                            MemoryRegionSection *section)
 835 {
 836     KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener);
 837
 838     kvm_set_phys_mem(kml, section, false);
 839     memory_region_unref(section->mr);
 840 }
 841
 842 static void kvm_log_sync(MemoryListener *listener,
 843                          MemoryRegionSection *section)
 844 {
 845     KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener);
 846     int r;
 847
 848     r = kvm_physical_sync_dirty_bitmap(kml, section);
 849     if (r < 0) {
 850         abort();
 851     }
 852 }
 853
 854 static void kvm_mem_ioeventfd_add(MemoryListener *listener,
 855                                   MemoryRegionSection *section,
 856                                   bool match_data, uint64_t data,
 857                                   EventNotifier *e)
 858 {
 859     int fd = event_notifier_get_fd(e);
 860     int r;
 861
 862     r = kvm_set_ioeventfd_mmio(fd, section->offset_within_address_space,
 863                                data, true, int128_get64(section->size),
 864                                match_data);
 865     if (r < 0) {
 866         fprintf(stderr, "%s: error adding ioeventfd: %s\n",
 867                 __func__, strerror(-r));
 868         abort();
 869     }
 870 }
 871
 872 static void kvm_mem_ioeventfd_del(MemoryListener *listener,
 873                                   MemoryRegionSection *section,
 874                                   bool match_data, uint64_t data,
 875                                   EventNotifier *e)
 876 {
 877     int fd = event_notifier_get_fd(e);
 878     int r;
 879
 880     r = kvm_set_ioeventfd_mmio(fd, section->offset_within_address_space,
 881                                data, false, int128_get64(section->size),
 882                                match_data);
 883     if (r < 0) {
 884         abort();
 885     }
 886 }
 887
 888 static void kvm_io_ioeventfd_add(MemoryListener *listener,
 889                                  MemoryRegionSection *section,
 890                                  bool match_data, uint64_t data,
 891                                  EventNotifier *e)
 892 {
 893     int fd = event_notifier_get_fd(e);
 894     int r;
 895
 896     r = kvm_set_ioeventfd_pio(fd, section->offset_within_address_space,
 897                               data, true, int128_get64(section->size),
 898                               match_data);
 899     if (r < 0) {
 900         fprintf(stderr, "%s: error adding ioeventfd: %s\n",
 901                 __func__, strerror(-r));
 902         abort();
 903     }
 904 }
 905
 906 static void kvm_io_ioeventfd_del(MemoryListener *listener,
 907                                  MemoryRegionSection *section,
 908                                  bool match_data, uint64_t data,
 909                                  EventNotifier *e)
 910
 911 {
 912     int fd = event_notifier_get_fd(e);
 913     int r;
 914
 915     r = kvm_set_ioeventfd_pio(fd, section->offset_within_address_space,
 916                               data, false, int128_get64(section->size),
 917                               match_data);
 918     if (r < 0) {
 919         abort();
 920     }
 921 }
 922
 923 void kvm_memory_listener_register(KVMState *s, KVMMemoryListener *kml,
 924                                   AddressSpace *as, int as_id)
 925 {
 926     int i;
 927
 928     kml->slots = g_malloc0(s->nr_slots * sizeof(KVMSlot));
 929     kml->as_id = as_id;
 930
 931     for (i = 0; i < s->nr_slots; i++) {
 932         kml->slots[i].slot = i;
 933     }
 934
 935     kml->listener.region_add = kvm_region_add;
 936     kml->listener.region_del = kvm_region_del;
 937     kml->listener.log_start = kvm_log_start;
 938     kml->listener.log_stop = kvm_log_stop;
 939     kml->listener.log_sync = kvm_log_sync;
 940     kml->listener.priority = 10;
 941
 942     memory_listener_register(&kml->listener, as);
 943 }
 944
 945 static MemoryListener kvm_io_listener = {
 946     .eventfd_add = kvm_io_ioeventfd_add,
 947     .eventfd_del = kvm_io_ioeventfd_del,
 948     .priority = 10,
 949 };
 950
 951 int kvm_set_irq(KVMState *s, int irq, int level)
 952 {
 953     struct kvm_irq_level event;
 954     int ret;
 955
 956     assert(kvm_async_interrupts_enabled());
 957
 958     event.level = level;
 959     event.irq = irq;
 960     ret = kvm_vm_ioctl(s, s->irq_set_ioctl, &event);
 961     if (ret < 0) {
 962         perror("kvm_set_irq");
 963         abort();
 964     }
 965
 966     return (s->irq_set_ioctl == KVM_IRQ_LINE) ? 1 : event.status;
 967 }
 968
 969 #ifdef KVM_CAP_IRQ_ROUTING
 970 typedef struct KVMMSIRoute {
 971     struct kvm_irq_routing_entry kroute;
 972     QTAILQ_ENTRY(KVMMSIRoute) entry;
 973 } KVMMSIRoute;
 974
 975 static void set_gsi(KVMState *s, unsigned int gsi)
 976 {
 977     set_bit(gsi, s->used_gsi_bitmap);
 978 }
 979
 980 static void clear_gsi(KVMState *s, unsigned int gsi)
 981 {
 982     clear_bit(gsi, s->used_gsi_bitmap);
 983 }
 984
 985 void kvm_init_irq_routing(KVMState *s)
 986 {
 987     int gsi_count, i;
 988
 989     gsi_count = kvm_check_extension(s, KVM_CAP_IRQ_ROUTING) - 1;
 990     if (gsi_count > 0) {
 991         /* Round up so we can search ints using ffs */
 992         s->used_gsi_bitmap = bitmap_new(gsi_count);
 993         s->gsi_count = gsi_count;
 994     }
 995
 996     s->irq_routes = g_malloc0(sizeof(*s->irq_routes));
 997     s->nr_allocated_irq_routes = 0;
 998
 999     if (!kvm_direct_msi_allowed) {
1000         for (i = 0; i < KVM_MSI_HASHTAB_SIZE; i++) {
1001             QTAILQ_INIT(&s->msi_hashtab[i]);
1002         }
1003     }
1004
1005     kvm_arch_init_irq_routing(s);
1006 }
1007
1008 void kvm_irqchip_commit_routes(KVMState *s)
1009 {
1010     int ret;
1011
1012     if (kvm_gsi_direct_mapping()) {
1013         return;
1014     }
1015
1016     if (!kvm_gsi_routing_enabled()) {
1017         return;
1018     }
1019
1020     s->irq_routes->flags = 0;
1021     trace_kvm_irqchip_commit_routes();
1022     ret = kvm_vm_ioctl(s, KVM_SET_GSI_ROUTING, s->irq_routes);
1023     assert(ret == 0);
1024 }
1025
1026 static void kvm_add_routing_entry(KVMState *s,
1027                                   struct kvm_irq_routing_entry *entry)
1028 {
1029     struct kvm_irq_routing_entry *new;
1030     int n, size;
1031
1032     if (s->irq_routes->nr == s->nr_allocated_irq_routes) {
1033         n = s->nr_allocated_irq_routes * 2;
1034         if (n < 64) {
1035             n = 64;
1036         }
1037         size = sizeof(struct kvm_irq_routing);
1038         size += n * sizeof(*new);
1039         s->irq_routes = g_realloc(s->irq_routes, size);
1040         s->nr_allocated_irq_routes = n;
1041     }
1042     n = s->irq_routes->nr++;
1043     new = &s->irq_routes->entries[n];
1044
1045     *new = *entry;
1046
1047     set_gsi(s, entry->gsi);
1048 }
1049
1050 static int kvm_update_routing_entry(KVMState *s,
1051                                     struct kvm_irq_routing_entry *new_entry)
1052 {
1053     struct kvm_irq_routing_entry *entry;
1054     int n;
1055
1056     for (n = 0; n < s->irq_routes->nr; n++) {
1057         entry = &s->irq_routes->entries[n];
1058         if (entry->gsi != new_entry->gsi) {
1059             continue;
1060         }
1061
1062         if(!memcmp(entry, new_entry, sizeof *entry)) {
1063             return 0;
1064         }
1065
1066         *entry = *new_entry;
1067
1068         return 0;
1069     }
1070
1071     return -ESRCH;
1072 }
1073
1074 void kvm_irqchip_add_irq_route(KVMState *s, int irq, int irqchip, int pin)
1075 {
1076     struct kvm_irq_routing_entry e = {};
1077
1078     assert(pin < s->gsi_count);
1079
1080     e.gsi = irq;
1081     e.type = KVM_IRQ_ROUTING_IRQCHIP;
1082     e.flags = 0;
1083     e.u.irqchip.irqchip = irqchip;
1084     e.u.irqchip.pin = pin;
1085     kvm_add_routing_entry(s, &e);
1086 }
1087
1088 void kvm_irqchip_release_virq(KVMState *s, int virq)
1089 {
1090     struct kvm_irq_routing_entry *e;
1091     int i;
1092
1093     if (kvm_gsi_direct_mapping()) {
1094         return;
1095     }
1096
1097     for (i = 0; i < s->irq_routes->nr; i++) {
1098         e = &s->irq_routes->entries[i];
1099         if (e->gsi == virq) {
1100             s->irq_routes->nr--;
1101             *e = s->irq_routes->entries[s->irq_routes->nr];
1102         }
1103     }
1104     clear_gsi(s, virq);
1105     kvm_arch_release_virq_post(virq);
1106     trace_kvm_irqchip_release_virq(virq);
1107 }
1108
1109 static unsigned int kvm_hash_msi(uint32_t data)
1110 {
1111     /* This is optimized for IA32 MSI layout. However, no other arch shall
1112      * repeat the mistake of not providing a direct MSI injection API. */
1113     return data & 0xff;
1114 }
1115
1116 static void kvm_flush_dynamic_msi_routes(KVMState *s)
1117 {
1118     KVMMSIRoute *route, *next;
1119     unsigned int hash;
1120
1121     for (hash = 0; hash < KVM_MSI_HASHTAB_SIZE; hash++) {
1122         QTAILQ_FOREACH_SAFE(route, &s->msi_hashtab[hash], entry, next) {
1123             kvm_irqchip_release_virq(s, route->kroute.gsi);
1124             QTAILQ_REMOVE(&s->msi_hashtab[hash], route, entry);
1125             g_free(route);
1126         }
1127     }
1128 }
1129
1130 static int kvm_irqchip_get_virq(KVMState *s)
1131 {
1132     int next_virq;
1133
1134     /*
1135      * PIC and IOAPIC share the first 16 GSI numbers, thus the available
1136      * GSI numbers are more than the number of IRQ route. Allocating a GSI
1137      * number can succeed even though a new route entry cannot be added.
1138      * When this happens, flush dynamic MSI entries to free IRQ route entries.
1139      */
1140     if (!kvm_direct_msi_allowed && s->irq_routes->nr == s->gsi_count) {
1141         kvm_flush_dynamic_msi_routes(s);
1142     }
1143
1144     /* Return the lowest unused GSI in the bitmap */
1145     next_virq = find_first_zero_bit(s->used_gsi_bitmap, s->gsi_count);
1146     if (next_virq >= s->gsi_count) {
1147         return -ENOSPC;
1148     } else {
1149         return next_virq;
1150     }
1151 }
1152
1153 static KVMMSIRoute *kvm_lookup_msi_route(KVMState *s, MSIMessage msg)
1154 {
1155     unsigned int hash = kvm_hash_msi(msg.data);
1156     KVMMSIRoute *route;
1157
1158     QTAILQ_FOREACH(route, &s->msi_hashtab[hash], entry) {
1159         if (route->kroute.u.msi.address_lo == (uint32_t)msg.address &&
1160             route->kroute.u.msi.address_hi == (msg.address >> 32) &&
1161             route->kroute.u.msi.data == le32_to_cpu(msg.data)) {
1162             return route;
1163         }
1164     }
1165     return NULL;
1166 }
1167
1168 int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg)
1169 {
1170     struct kvm_msi msi;
1171     KVMMSIRoute *route;
1172
1173     if (kvm_direct_msi_allowed) {
1174         msi.address_lo = (uint32_t)msg.address;
1175         msi.address_hi = msg.address >> 32;
1176         msi.data = le32_to_cpu(msg.data);
1177         msi.flags = 0;
1178         memset(msi.pad, 0, sizeof(msi.pad));
1179
1180         return kvm_vm_ioctl(s, KVM_SIGNAL_MSI, &msi);
1181     }
1182
1183     route = kvm_lookup_msi_route(s, msg);
1184     if (!route) {
1185         int virq;
1186
1187         virq = kvm_irqchip_get_virq(s);
1188         if (virq < 0) {
1189             return virq;
1190         }
1191
1192         route = g_malloc0(sizeof(KVMMSIRoute));
1193         route->kroute.gsi = virq;
1194         route->kroute.type = KVM_IRQ_ROUTING_MSI;
1195         route->kroute.flags = 0;
1196         route->kroute.u.msi.address_lo = (uint32_t)msg.address;
1197         route->kroute.u.msi.address_hi = msg.address >> 32;
1198         route->kroute.u.msi.data = le32_to_cpu(msg.data);
1199
1200         kvm_add_routing_entry(s, &route->kroute);
1201         kvm_irqchip_commit_routes(s);
1202
1203         QTAILQ_INSERT_TAIL(&s->msi_hashtab[kvm_hash_msi(msg.data)], route,
1204                            entry);
1205     }
1206
1207     assert(route->kroute.type == KVM_IRQ_ROUTING_MSI);
1208
1209     return kvm_set_irq(s, route->kroute.gsi, 1);
1210 }
1211
1212 int kvm_irqchip_add_msi_route(KVMState *s, int vector, PCIDevice *dev)
1213 {
1214     struct kvm_irq_routing_entry kroute = {};
1215     int virq;
1216     MSIMessage msg = {0, 0};
1217
1218     if (pci_available && dev) {
1219         msg = pci_get_msi_message(dev, vector);
1220     }
1221
1222     if (kvm_gsi_direct_mapping()) {
1223         return kvm_arch_msi_data_to_gsi(msg.data);
1224     }
1225
1226     if (!kvm_gsi_routing_enabled()) {
1227         return -ENOSYS;
1228     }
1229
1230     virq = kvm_irqchip_get_virq(s);
1231     if (virq < 0) {
1232         return virq;
1233     }
1234
1235     kroute.gsi = virq;
1236     kroute.type = KVM_IRQ_ROUTING_MSI;
1237     kroute.flags = 0;
1238     kroute.u.msi.address_lo = (uint32_t)msg.address;
1239     kroute.u.msi.address_hi = msg.address >> 32;
1240     kroute.u.msi.data = le32_to_cpu(msg.data);
1241     if (pci_available && kvm_msi_devid_required()) {
1242         kroute.flags = KVM_MSI_VALID_DEVID;
1243         kroute.u.msi.devid = pci_requester_id(dev);
1244     }
1245     if (kvm_arch_fixup_msi_route(&kroute, msg.address, msg.data, dev)) {
1246         kvm_irqchip_release_virq(s, virq);
1247         return -EINVAL;
1248     }
1249
1250     trace_kvm_irqchip_add_msi_route(dev ? dev->name : (char *)"N/A",
1251                                     vector, virq);
1252
1253     kvm_add_routing_entry(s, &kroute);
1254     kvm_arch_add_msi_route_post(&kroute, vector, dev);
1255     kvm_irqchip_commit_routes(s);
1256
1257     return virq;
1258 }
1259
1260 int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg,
1261                                  PCIDevice *dev)
1262 {
1263     struct kvm_irq_routing_entry kroute = {};
1264
1265     if (kvm_gsi_direct_mapping()) {
1266         return 0;
1267     }
1268
1269     if (!kvm_irqchip_in_kernel()) {
1270         return -ENOSYS;
1271     }
1272
1273     kroute.gsi = virq;
1274     kroute.type = KVM_IRQ_ROUTING_MSI;
1275     kroute.flags = 0;
1276     kroute.u.msi.address_lo = (uint32_t)msg.address;
1277     kroute.u.msi.address_hi = msg.address >> 32;
1278     kroute.u.msi.data = le32_to_cpu(msg.data);
1279     if (pci_available && kvm_msi_devid_required()) {
1280         kroute.flags = KVM_MSI_VALID_DEVID;
1281         kroute.u.msi.devid = pci_requester_id(dev);
1282     }
1283     if (kvm_arch_fixup_msi_route(&kroute, msg.address, msg.data, dev)) {
1284         return -EINVAL;
1285     }
1286
1287     trace_kvm_irqchip_update_msi_route(virq);
1288
1289     return kvm_update_routing_entry(s, &kroute);
1290 }
1291
1292 static int kvm_irqchip_assign_irqfd(KVMState *s, int fd, int rfd, int virq,
1293                                     bool assign)
1294 {
1295     struct kvm_irqfd irqfd = {
1296         .fd = fd,
1297         .gsi = virq,
1298         .flags = assign ? 0 : KVM_IRQFD_FLAG_DEASSIGN,
1299     };
1300
1301     if (rfd != -1) {
1302         irqfd.flags |= KVM_IRQFD_FLAG_RESAMPLE;
1303         irqfd.resamplefd = rfd;
1304     }
1305
1306     if (!kvm_irqfds_enabled()) {
1307         return -ENOSYS;
1308     }
1309
1310     return kvm_vm_ioctl(s, KVM_IRQFD, &irqfd);
1311 }
1312
1313 int kvm_irqchip_add_adapter_route(KVMState *s, AdapterInfo *adapter)
1314 {
1315     struct kvm_irq_routing_entry kroute = {};
1316     int virq;
1317
1318     if (!kvm_gsi_routing_enabled()) {
1319         return -ENOSYS;
1320     }
1321
1322     virq = kvm_irqchip_get_virq(s);
1323     if (virq < 0) {
1324         return virq;
1325     }
1326
1327     kroute.gsi = virq;
1328     kroute.type = KVM_IRQ_ROUTING_S390_ADAPTER;
1329     kroute.flags = 0;
1330     kroute.u.adapter.summary_addr = adapter->summary_addr;
1331     kroute.u.adapter.ind_addr = adapter->ind_addr;
1332     kroute.u.adapter.summary_offset = adapter->summary_offset;
1333     kroute.u.adapter.ind_offset = adapter->ind_offset;
1334     kroute.u.adapter.adapter_id = adapter->adapter_id;
1335
1336     kvm_add_routing_entry(s, &kroute);
1337
1338     return virq;
1339 }
1340
1341 int kvm_irqchip_add_hv_sint_route(KVMState *s, uint32_t vcpu, uint32_t sint)
1342 {
1343     struct kvm_irq_routing_entry kroute = {};
1344     int virq;
1345
1346     if (!kvm_gsi_routing_enabled()) {
1347         return -ENOSYS;
1348     }
1349     if (!kvm_check_extension(s, KVM_CAP_HYPERV_SYNIC)) {
1350         return -ENOSYS;
1351     }
1352     virq = kvm_irqchip_get_virq(s);
1353     if (virq < 0) {
1354         return virq;
1355     }
1356
1357     kroute.gsi = virq;
1358     kroute.type = KVM_IRQ_ROUTING_HV_SINT;
1359     kroute.flags = 0;
1360     kroute.u.hv_sint.vcpu = vcpu;
1361     kroute.u.hv_sint.sint = sint;
1362
1363     kvm_add_routing_entry(s, &kroute);
1364     kvm_irqchip_commit_routes(s);
1365
1366     return virq;
1367 }
1368
1369 #else /* !KVM_CAP_IRQ_ROUTING */
1370
1371 void kvm_init_irq_routing(KVMState *s)
1372 {
1373 }
1374
1375 void kvm_irqchip_release_virq(KVMState *s, int virq)
1376 {
1377 }
1378
1379 int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg)
1380 {
1381     abort();
1382 }
1383
1384 int kvm_irqchip_add_msi_route(KVMState *s, int vector, PCIDevice *dev)
1385 {
1386     return -ENOSYS;
1387 }
1388
1389 int kvm_irqchip_add_adapter_route(KVMState *s, AdapterInfo *adapter)
1390 {
1391     return -ENOSYS;
1392 }
1393
1394 int kvm_irqchip_add_hv_sint_route(KVMState *s, uint32_t vcpu, uint32_t sint)
1395 {
1396     return -ENOSYS;
1397 }
1398
1399 static int kvm_irqchip_assign_irqfd(KVMState *s, int fd, int virq, bool assign)
1400 {
1401     abort();
1402 }
1403
1404 int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg)
1405 {
1406     return -ENOSYS;
1407 }
1408 #endif /* !KVM_CAP_IRQ_ROUTING */
1409
1410 int kvm_irqchip_add_irqfd_notifier_gsi(KVMState *s, EventNotifier *n,
1411                                        EventNotifier *rn, int virq)
1412 {
1413     return kvm_irqchip_assign_irqfd(s, event_notifier_get_fd(n),
1414            rn ? event_notifier_get_fd(rn) : -1, virq, true);
1415 }
1416
1417 int kvm_irqchip_remove_irqfd_notifier_gsi(KVMState *s, EventNotifier *n,
1418                                           int virq)
1419 {
1420     return kvm_irqchip_assign_irqfd(s, event_notifier_get_fd(n), -1, virq,
1421            false);
1422 }
1423
1424 int kvm_irqchip_add_irqfd_notifier(KVMState *s, EventNotifier *n,
1425                                    EventNotifier *rn, qemu_irq irq)
1426 {
1427     gpointer key, gsi;
1428     gboolean found = g_hash_table_lookup_extended(s->gsimap, irq, &key, &gsi);
1429
1430     if (!found) {
1431         return -ENXIO;
1432     }
1433     return kvm_irqchip_add_irqfd_notifier_gsi(s, n, rn, GPOINTER_TO_INT(gsi));
1434 }
1435
1436 int kvm_irqchip_remove_irqfd_notifier(KVMState *s, EventNotifier *n,
1437                                       qemu_irq irq)
1438 {
1439     gpointer key, gsi;
1440     gboolean found = g_hash_table_lookup_extended(s->gsimap, irq, &key, &gsi);
1441
1442     if (!found) {
1443         return -ENXIO;
1444     }
1445     return kvm_irqchip_remove_irqfd_notifier_gsi(s, n, GPOINTER_TO_INT(gsi));
1446 }
1447
1448 void kvm_irqchip_set_qemuirq_gsi(KVMState *s, qemu_irq irq, int gsi)
1449 {
1450     g_hash_table_insert(s->gsimap, irq, GINT_TO_POINTER(gsi));
1451 }
1452
1453 static void kvm_irqchip_create(MachineState *machine, KVMState *s)
1454 {
1455     int ret;
1456
1457     if (kvm_check_extension(s, KVM_CAP_IRQCHIP)) {
1458         ;
1459     } else if (kvm_check_extension(s, KVM_CAP_S390_IRQCHIP)) {
1460         ret = kvm_vm_enable_cap(s, KVM_CAP_S390_IRQCHIP, 0);
1461         if (ret < 0) {
1462             fprintf(stderr, "Enable kernel irqchip failed: %s\n", strerror(-ret));
1463             exit(1);
1464         }
1465     } else {
1466         return;
1467     }
1468
1469     /* First probe and see if there's a arch-specific hook to create the
1470      * in-kernel irqchip for us */
1471     ret = kvm_arch_irqchip_create(machine, s);
1472     if (ret == 0) {
1473         if (machine_kernel_irqchip_split(machine)) {
1474             perror("Split IRQ chip mode not supported.");
1475             exit(1);
1476         } else {
1477             ret = kvm_vm_ioctl(s, KVM_CREATE_IRQCHIP);
1478         }
1479     }
1480     if (ret < 0) {
1481         fprintf(stderr, "Create kernel irqchip failed: %s\n", strerror(-ret));
1482         exit(1);
1483     }
1484
1485     kvm_kernel_irqchip = true;
1486     /* If we have an in-kernel IRQ chip then we must have asynchronous
1487      * interrupt delivery (though the reverse is not necessarily true)
1488      */
1489     kvm_async_interrupts_allowed = true;
1490     kvm_halt_in_kernel_allowed = true;
1491
1492     kvm_init_irq_routing(s);
1493
1494     s->gsimap = g_hash_table_new(g_direct_hash, g_direct_equal);
1495 }
1496
1497 /* Find number of supported CPUs using the recommended
1498  * procedure from the kernel API documentation to cope with
1499  * older kernels that may be missing capabilities.
1500  */
1501 static int kvm_recommended_vcpus(KVMState *s)
1502 {
1503     int ret = kvm_vm_check_extension(s, KVM_CAP_NR_VCPUS);
1504     return (ret) ? ret : 4;
1505 }
1506
1507 static int kvm_max_vcpus(KVMState *s)
1508 {
1509     int ret = kvm_check_extension(s, KVM_CAP_MAX_VCPUS);
1510     return (ret) ? ret : kvm_recommended_vcpus(s);
1511 }
1512
1513 static int kvm_max_vcpu_id(KVMState *s)
1514 {
1515     int ret = kvm_check_extension(s, KVM_CAP_MAX_VCPU_ID);
1516     return (ret) ? ret : kvm_max_vcpus(s);
1517 }
1518
1519 bool kvm_vcpu_id_is_valid(int vcpu_id)
1520 {
1521     KVMState *s = KVM_STATE(current_machine->accelerator);
1522     return vcpu_id >= 0 && vcpu_id < kvm_max_vcpu_id(s);
1523 }
1524
1525 static int kvm_init(MachineState *ms)
1526 {
1527     MachineClass *mc = MACHINE_GET_CLASS(ms);
1528     static const char upgrade_note[] =
1529         "Please upgrade to at least kernel 2.6.29 or recent kvm-kmod\n"
1530         "(see http://sourceforge.net/projects/kvm).\n";
1531     struct {
1532         const char *name;
1533         int num;
1534     } num_cpus[] = {
1535         { "SMP",          smp_cpus },
1536         { "hotpluggable", max_cpus },
1537         { NULL, }
1538     }, *nc = num_cpus;
1539     int soft_vcpus_limit, hard_vcpus_limit;
1540     KVMState *s;
1541     const KVMCapabilityInfo *missing_cap;
1542     int ret;
1543     int type = 0;
1544     const char *kvm_type;
1545
1546     s = KVM_STATE(ms->accelerator);
1547
1548     /*
1549      * On systems where the kernel can support different base page
1550      * sizes, host page size may be different from TARGET_PAGE_SIZE,
1551      * even with KVM.  TARGET_PAGE_SIZE is assumed to be the minimum
1552      * page size for the system though.
1553      */
1554     assert(TARGET_PAGE_SIZE <= getpagesize());
1555
1556     s->sigmask_len = 8;
1557
1558 #ifdef KVM_CAP_SET_GUEST_DEBUG
1559     QTAILQ_INIT(&s->kvm_sw_breakpoints);
1560 #endif
1561     QLIST_INIT(&s->kvm_parked_vcpus);
1562     s->vmfd = -1;
1563     s->fd = qemu_open("/dev/kvm", O_RDWR);
1564     if (s->fd == -1) {
1565         fprintf(stderr, "Could not access KVM kernel module: %m\n");
1566         ret = -errno;
1567         goto err;
1568     }
1569
1570     ret = kvm_ioctl(s, KVM_GET_API_VERSION, 0);
1571     if (ret < KVM_API_VERSION) {
1572         if (ret >= 0) {
1573             ret = -EINVAL;
1574         }
1575         fprintf(stderr, "kvm version too old\n");
1576         goto err;
1577     }
1578
1579     if (ret > KVM_API_VERSION) {
1580         ret = -EINVAL;
1581         fprintf(stderr, "kvm version not supported\n");
1582         goto err;
1583     }
1584
1585     kvm_immediate_exit = kvm_check_extension(s, KVM_CAP_IMMEDIATE_EXIT);
1586     s->nr_slots = kvm_check_extension(s, KVM_CAP_NR_MEMSLOTS);
1587
1588     /* If unspecified, use the default value */
1589     if (!s->nr_slots) {
1590         s->nr_slots = 32;
1591     }
1592
1593     kvm_type = qemu_opt_get(qemu_get_machine_opts(), "kvm-type");
1594     if (mc->kvm_type) {
1595         type = mc->kvm_type(ms, kvm_type);
1596     } else if (kvm_type) {
1597         ret = -EINVAL;
1598         fprintf(stderr, "Invalid argument kvm-type=%s\n", kvm_type);
1599         goto err;
1600     }
1601
1602     do {
1603         ret = kvm_ioctl(s, KVM_CREATE_VM, type);
1604     } while (ret == -EINTR);
1605
1606     if (ret < 0) {
1607         fprintf(stderr, "ioctl(KVM_CREATE_VM) failed: %d %s\n", -ret,
1608                 strerror(-ret));
1609
1610 #ifdef TARGET_S390X
1611         if (ret == -EINVAL) {
1612             fprintf(stderr,
1613                     "Host kernel setup problem detected. Please verify:\n");
1614             fprintf(stderr, "- for kernels supporting the switch_amode or"
1615                     " user_mode parameters, whether\n");
1616             fprintf(stderr,
1617                     "  user space is running in primary address space\n");
1618             fprintf(stderr,
1619                     "- for kernels supporting the vm.allocate_pgste sysctl, "
1620                     "whether it is enabled\n");
1621         }
1622 #endif
1623         goto err;
1624     }
1625
1626     s->vmfd = ret;
1627
1628     /* check the vcpu limits */
1629     soft_vcpus_limit = kvm_recommended_vcpus(s);
1630     hard_vcpus_limit = kvm_max_vcpus(s);
1631
1632     while (nc->name) {
1633         if (nc->num > soft_vcpus_limit) {
1634             warn_report("Number of %s cpus requested (%d) exceeds "
1635                         "the recommended cpus supported by KVM (%d)",
1636                         nc->name, nc->num, soft_vcpus_limit);
1637
1638             if (nc->num > hard_vcpus_limit) {
1639                 fprintf(stderr, "Number of %s cpus requested (%d) exceeds "
1640                         "the maximum cpus supported by KVM (%d)\n",
1641                         nc->name, nc->num, hard_vcpus_limit);
1642                 exit(1);
1643             }
1644         }
1645         nc++;
1646     }
1647
1648     missing_cap = kvm_check_extension_list(s, kvm_required_capabilites);
1649     if (!missing_cap) {
1650         missing_cap =
1651             kvm_check_extension_list(s, kvm_arch_required_capabilities);
1652     }
1653     if (missing_cap) {
1654         ret = -EINVAL;
1655         fprintf(stderr, "kvm does not support %s\n%s",
1656                 missing_cap->name, upgrade_note);
1657         goto err;
1658     }
1659
1660     s->coalesced_mmio = kvm_check_extension(s, KVM_CAP_COALESCED_MMIO);
1661     s->coalesced_pio = s->coalesced_mmio &&
1662                        kvm_check_extension(s, KVM_CAP_COALESCED_PIO);
1663
1664 #ifdef KVM_CAP_VCPU_EVENTS
1665     s->vcpu_events = kvm_check_extension(s, KVM_CAP_VCPU_EVENTS);
1666 #endif
1667
1668     s->robust_singlestep =
1669         kvm_check_extension(s, KVM_CAP_X86_ROBUST_SINGLESTEP);
1670
1671 #ifdef KVM_CAP_DEBUGREGS
1672     s->debugregs = kvm_check_extension(s, KVM_CAP_DEBUGREGS);
1673 #endif
1674
1675 #ifdef KVM_CAP_IRQ_ROUTING
1676     kvm_direct_msi_allowed = (kvm_check_extension(s, KVM_CAP_SIGNAL_MSI) > 0);
1677 #endif
1678
1679     s->intx_set_mask = kvm_check_extension(s, KVM_CAP_PCI_2_3);
1680
1681     s->irq_set_ioctl = KVM_IRQ_LINE;
1682     if (kvm_check_extension(s, KVM_CAP_IRQ_INJECT_STATUS)) {
1683         s->irq_set_ioctl = KVM_IRQ_LINE_STATUS;
1684     }
1685
1686     kvm_readonly_mem_allowed =
1687         (kvm_check_extension(s, KVM_CAP_READONLY_MEM) > 0);
1688
1689     kvm_eventfds_allowed =
1690         (kvm_check_extension(s, KVM_CAP_IOEVENTFD) > 0);
1691
1692     kvm_irqfds_allowed =
1693         (kvm_check_extension(s, KVM_CAP_IRQFD) > 0);
1694
1695     kvm_resamplefds_allowed =
1696         (kvm_check_extension(s, KVM_CAP_IRQFD_RESAMPLE) > 0);
1697
1698     kvm_vm_attributes_allowed =
1699         (kvm_check_extension(s, KVM_CAP_VM_ATTRIBUTES) > 0);
1700
1701     kvm_ioeventfd_any_length_allowed =
1702         (kvm_check_extension(s, KVM_CAP_IOEVENTFD_ANY_LENGTH) > 0);
1703
1704     kvm_state = s;
1705
1706     /*
1707      * if memory encryption object is specified then initialize the memory
1708      * encryption context.
1709      */
1710     if (ms->memory_encryption) {
1711         kvm_state->memcrypt_handle = sev_guest_init(ms->memory_encryption);
1712         if (!kvm_state->memcrypt_handle) {
1713             ret = -1;
1714             goto err;
1715         }
1716
1717         kvm_state->memcrypt_encrypt_data = sev_encrypt_data;
1718     }
1719
1720     ret = kvm_arch_init(ms, s);
1721     if (ret < 0) {
1722         goto err;
1723     }
1724
1725     if (machine_kernel_irqchip_allowed(ms)) {
1726         kvm_irqchip_create(ms, s);
1727     }
1728
1729     if (kvm_eventfds_allowed) {
1730         s->memory_listener.listener.eventfd_add = kvm_mem_ioeventfd_add;
1731         s->memory_listener.listener.eventfd_del = kvm_mem_ioeventfd_del;
1732     }
1733     s->memory_listener.listener.coalesced_io_add = kvm_coalesce_mmio_region;
1734     s->memory_listener.listener.coalesced_io_del = kvm_uncoalesce_mmio_region;
1735
1736     kvm_memory_listener_register(s, &s->memory_listener,
1737                                  &address_space_memory, 0);
1738     memory_listener_register(&kvm_io_listener,
1739                              &address_space_io);
1740     memory_listener_register(&kvm_coalesced_pio_listener,
1741                              &address_space_io);
1742
1743     s->many_ioeventfds = kvm_check_many_ioeventfds();
1744
1745     s->sync_mmu = !!kvm_vm_check_extension(kvm_state, KVM_CAP_SYNC_MMU);
1746     if (!s->sync_mmu) {
1747         qemu_balloon_inhibit(true);
1748     }
1749
1750     return 0;
1751
1752 err:
1753     assert(ret < 0);
1754     if (s->vmfd >= 0) {
1755         close(s->vmfd);
1756     }
1757     if (s->fd != -1) {
1758         close(s->fd);
1759     }
1760     g_free(s->memory_listener.slots);
1761
1762     return ret;
1763 }
1764
1765 void kvm_set_sigmask_len(KVMState *s, unsigned int sigmask_len)
1766 {
1767     s->sigmask_len = sigmask_len;
1768 }
1769
1770 static void kvm_handle_io(uint16_t port, MemTxAttrs attrs, void *data, int direction,
1771                           int size, uint32_t count)
1772 {
1773     int i;
1774     uint8_t *ptr = data;
1775
1776     for (i = 0; i < count; i++) {
1777         address_space_rw(&address_space_io, port, attrs,
1778                          ptr, size,
1779                          direction == KVM_EXIT_IO_OUT);
1780         ptr += size;
1781     }
1782 }
1783
1784 static int kvm_handle_internal_error(CPUState *cpu, struct kvm_run *run)
1785 {
1786     fprintf(stderr, "KVM internal error. Suberror: %d\n",
1787             run->internal.suberror);
1788
1789     if (kvm_check_extension(kvm_state, KVM_CAP_INTERNAL_ERROR_DATA)) {
1790         int i;
1791
1792         for (i = 0; i < run->internal.ndata; ++i) {
1793             fprintf(stderr, "extra data[%d]: %"PRIx64"\n",
1794                     i, (uint64_t)run->internal.data[i]);
1795         }
1796     }
1797     if (run->internal.suberror == KVM_INTERNAL_ERROR_EMULATION) {
1798         fprintf(stderr, "emulation failure\n");
1799         if (!kvm_arch_stop_on_emulation_error(cpu)) {
1800             cpu_dump_state(cpu, stderr, CPU_DUMP_CODE);
1801             return EXCP_INTERRUPT;
1802         }
1803     }
1804     /* FIXME: Should trigger a qmp message to let management know
1805      * something went wrong.
1806      */
1807     return -1;
1808 }
1809
1810 void kvm_flush_coalesced_mmio_buffer(void)
1811 {
1812     KVMState *s = kvm_state;
1813
1814     if (s->coalesced_flush_in_progress) {
1815         return;
1816     }
1817
1818     s->coalesced_flush_in_progress = true;
1819
1820     if (s->coalesced_mmio_ring) {
1821         struct kvm_coalesced_mmio_ring *ring = s->coalesced_mmio_ring;
1822         while (ring->first != ring->last) {
1823             struct kvm_coalesced_mmio *ent;
1824
1825             ent = &ring->coalesced_mmio[ring->first];
1826
1827             if (ent->pio == 1) {
1828                 address_space_rw(&address_space_io, ent->phys_addr,
1829                                  MEMTXATTRS_UNSPECIFIED, ent->data,
1830                                  ent->len, true);
1831             } else {
1832                 cpu_physical_memory_write(ent->phys_addr, ent->data, ent->len);
1833             }
1834             smp_wmb();
1835             ring->first = (ring->first + 1) % KVM_COALESCED_MMIO_MAX;
1836         }
1837     }
1838
1839     s->coalesced_flush_in_progress = false;
1840 }
1841
1842 static void do_kvm_cpu_synchronize_state(CPUState *cpu, run_on_cpu_data arg)
1843 {
1844     if (!cpu->vcpu_dirty) {
1845         kvm_arch_get_registers(cpu);
1846         cpu->vcpu_dirty = true;
1847     }
1848 }
1849
1850 void kvm_cpu_synchronize_state(CPUState *cpu)
1851 {
1852     if (!cpu->vcpu_dirty) {
1853         run_on_cpu(cpu, do_kvm_cpu_synchronize_state, RUN_ON_CPU_NULL);
1854     }
1855 }
1856
1857 static void do_kvm_cpu_synchronize_post_reset(CPUState *cpu, run_on_cpu_data arg)
1858 {
1859     kvm_arch_put_registers(cpu, KVM_PUT_RESET_STATE);
1860     cpu->vcpu_dirty = false;
1861 }
1862
1863 void kvm_cpu_synchronize_post_reset(CPUState *cpu)
1864 {
1865     run_on_cpu(cpu, do_kvm_cpu_synchronize_post_reset, RUN_ON_CPU_NULL);
1866 }
1867
1868 static void do_kvm_cpu_synchronize_post_init(CPUState *cpu, run_on_cpu_data arg)
1869 {
1870     kvm_arch_put_registers(cpu, KVM_PUT_FULL_STATE);
1871     cpu->vcpu_dirty = false;
1872 }
1873
1874 void kvm_cpu_synchronize_post_init(CPUState *cpu)
1875 {
1876     run_on_cpu(cpu, do_kvm_cpu_synchronize_post_init, RUN_ON_CPU_NULL);
1877 }
1878
1879 static void do_kvm_cpu_synchronize_pre_loadvm(CPUState *cpu, run_on_cpu_data arg)
1880 {
1881     cpu->vcpu_dirty = true;
1882 }
1883
1884 void kvm_cpu_synchronize_pre_loadvm(CPUState *cpu)
1885 {
1886     run_on_cpu(cpu, do_kvm_cpu_synchronize_pre_loadvm, RUN_ON_CPU_NULL);
1887 }
1888
1889 #ifdef KVM_HAVE_MCE_INJECTION
1890 static __thread void *pending_sigbus_addr;
1891 static __thread int pending_sigbus_code;
1892 static __thread bool have_sigbus_pending;
1893 #endif
1894
1895 static void kvm_cpu_kick(CPUState *cpu)
1896 {
1897     atomic_set(&cpu->kvm_run->immediate_exit, 1);
1898 }
1899
1900 static void kvm_cpu_kick_self(void)
1901 {
1902     if (kvm_immediate_exit) {
1903         kvm_cpu_kick(current_cpu);
1904     } else {
1905         qemu_cpu_kick_self();
1906     }
1907 }
1908
1909 static void kvm_eat_signals(CPUState *cpu)
1910 {
1911     struct timespec ts = { 0, 0 };
1912     siginfo_t siginfo;
1913     sigset_t waitset;
1914     sigset_t chkset;
1915     int r;
1916
1917     if (kvm_immediate_exit) {
1918         atomic_set(&cpu->kvm_run->immediate_exit, 0);
1919         /* Write kvm_run->immediate_exit before the cpu->exit_request
1920          * write in kvm_cpu_exec.
1921          */
1922         smp_wmb();
1923         return;
1924     }
1925
1926     sigemptyset(&waitset);
1927     sigaddset(&waitset, SIG_IPI);
1928
1929     do {
1930         r = sigtimedwait(&waitset, &siginfo, &ts);
1931         if (r == -1 && !(errno == EAGAIN || errno == EINTR)) {
1932             perror("sigtimedwait");
1933             exit(1);
1934         }
1935
1936         r = sigpending(&chkset);
1937         if (r == -1) {
1938             perror("sigpending");
1939             exit(1);
1940         }
1941     } while (sigismember(&chkset, SIG_IPI));
1942 }
1943
1944 int kvm_cpu_exec(CPUState *cpu)
1945 {
1946     struct kvm_run *run = cpu->kvm_run;
1947     int ret, run_ret;
1948
1949     DPRINTF("kvm_cpu_exec()\n");
1950
1951     if (kvm_arch_process_async_events(cpu)) {
1952         atomic_set(&cpu->exit_request, 0);
1953         return EXCP_HLT;
1954     }
1955
1956     qemu_mutex_unlock_iothread();
1957     cpu_exec_start(cpu);
1958
1959     do {
1960         MemTxAttrs attrs;
1961
1962         if (cpu->vcpu_dirty) {
1963             kvm_arch_put_registers(cpu, KVM_PUT_RUNTIME_STATE);
1964             cpu->vcpu_dirty = false;
1965         }
1966
1967         kvm_arch_pre_run(cpu, run);
1968         if (atomic_read(&cpu->exit_request)) {
1969             DPRINTF("interrupt exit requested\n");
1970             /*
1971              * KVM requires us to reenter the kernel after IO exits to complete
1972              * instruction emulation. This self-signal will ensure that we
1973              * leave ASAP again.
1974              */
1975             kvm_cpu_kick_self();
1976         }
1977
1978         /* Read cpu->exit_request before KVM_RUN reads run->immediate_exit.
1979          * Matching barrier in kvm_eat_signals.
1980          */
1981         smp_rmb();
1982
1983         run_ret = kvm_vcpu_ioctl(cpu, KVM_RUN, 0);
1984
1985         attrs = kvm_arch_post_run(cpu, run);
1986
1987 #ifdef KVM_HAVE_MCE_INJECTION
1988         if (unlikely(have_sigbus_pending)) {
1989             qemu_mutex_lock_iothread();
1990             kvm_arch_on_sigbus_vcpu(cpu, pending_sigbus_code,
1991                                     pending_sigbus_addr);
1992             have_sigbus_pending = false;
1993             qemu_mutex_unlock_iothread();
1994         }
1995 #endif
1996
1997         if (run_ret < 0) {
1998             if (run_ret == -EINTR || run_ret == -EAGAIN) {
1999                 DPRINTF("io window exit\n");
2000                 kvm_eat_signals(cpu);
2001                 ret = EXCP_INTERRUPT;
2002                 break;
2003             }
2004             fprintf(stderr, "error: kvm run failed %s\n",
2005                     strerror(-run_ret));
2006 #ifdef TARGET_PPC
2007             if (run_ret == -EBUSY) {
2008                 fprintf(stderr,
2009                         "This is probably because your SMT is enabled.\n"
2010                         "VCPU can only run on primary threads with all "
2011                         "secondary threads offline.\n");
2012             }
2013 #endif
2014             ret = -1;
2015             break;
2016         }
2017
2018         trace_kvm_run_exit(cpu->cpu_index, run->exit_reason);
2019         switch (run->exit_reason) {
2020         case KVM_EXIT_IO:
2021             DPRINTF("handle_io\n");
2022             /* Called outside BQL */
2023             kvm_handle_io(run->io.port, attrs,
2024                           (uint8_t *)run + run->io.data_offset,
2025                           run->io.direction,
2026                           run->io.size,
2027                           run->io.count);
2028             ret = 0;
2029             break;
2030         case KVM_EXIT_MMIO:
2031             DPRINTF("handle_mmio\n");
2032             /* Called outside BQL */
2033             address_space_rw(&address_space_memory,
2034                              run->mmio.phys_addr, attrs,
2035                              run->mmio.data,
2036                              run->mmio.len,
2037                              run->mmio.is_write);
2038             ret = 0;
2039             break;
2040         case KVM_EXIT_IRQ_WINDOW_OPEN:
2041             DPRINTF("irq_window_open\n");
2042             ret = EXCP_INTERRUPT;
2043             break;
2044         case KVM_EXIT_SHUTDOWN:
2045             DPRINTF("shutdown\n");
2046             qemu_system_reset_request(SHUTDOWN_CAUSE_GUEST_RESET);
2047             ret = EXCP_INTERRUPT;
2048             break;
2049         case KVM_EXIT_UNKNOWN:
2050             fprintf(stderr, "KVM: unknown exit, hardware reason %" PRIx64 "\n",
2051                     (uint64_t)run->hw.hardware_exit_reason);
2052             ret = -1;
2053             break;
2054         case KVM_EXIT_INTERNAL_ERROR:
2055             ret = kvm_handle_internal_error(cpu, run);
2056             break;
2057         case KVM_EXIT_SYSTEM_EVENT:
2058             switch (run->system_event.type) {
2059             case KVM_SYSTEM_EVENT_SHUTDOWN:
2060                 qemu_system_shutdown_request(SHUTDOWN_CAUSE_GUEST_SHUTDOWN);
2061                 ret = EXCP_INTERRUPT;
2062                 break;
2063             case KVM_SYSTEM_EVENT_RESET:
2064                 qemu_system_reset_request(SHUTDOWN_CAUSE_GUEST_RESET);
2065                 ret = EXCP_INTERRUPT;
2066                 break;
2067             case KVM_SYSTEM_EVENT_CRASH:
2068                 kvm_cpu_synchronize_state(cpu);
2069                 qemu_mutex_lock_iothread();
2070                 qemu_system_guest_panicked(cpu_get_crash_info(cpu));
2071                 qemu_mutex_unlock_iothread();
2072                 ret = 0;
2073                 break;
2074             default:
2075                 DPRINTF("kvm_arch_handle_exit\n");
2076                 ret = kvm_arch_handle_exit(cpu, run);
2077                 break;
2078             }
2079             break;
2080         default:
2081             DPRINTF("kvm_arch_handle_exit\n");
2082             ret = kvm_arch_handle_exit(cpu, run);
2083             break;
2084         }
2085     } while (ret == 0);
2086
2087     cpu_exec_end(cpu);
2088     qemu_mutex_lock_iothread();
2089
2090     if (ret < 0) {
2091         cpu_dump_state(cpu, stderr, CPU_DUMP_CODE);
2092         vm_stop(RUN_STATE_INTERNAL_ERROR);
2093     }
2094
2095     atomic_set(&cpu->exit_request, 0);
2096     return ret;
2097 }
2098
2099 int kvm_ioctl(KVMState *s, int type, ...)
2100 {
2101     int ret;
2102     void *arg;
2103     va_list ap;
2104
2105     va_start(ap, type);
2106     arg = va_arg(ap, void *);
2107     va_end(ap);
2108
2109     trace_kvm_ioctl(type, arg);
2110     ret = ioctl(s->fd, type, arg);
2111     if (ret == -1) {
2112         ret = -errno;
2113     }
2114     return ret;
2115 }
2116
2117 int kvm_vm_ioctl(KVMState *s, int type, ...)
2118 {
2119     int ret;
2120     void *arg;
2121     va_list ap;
2122
2123     va_start(ap, type);
2124     arg = va_arg(ap, void *);
2125     va_end(ap);
2126
2127     trace_kvm_vm_ioctl(type, arg);
2128     ret = ioctl(s->vmfd, type, arg);
2129     if (ret == -1) {
2130         ret = -errno;
2131     }
2132     return ret;
2133 }
2134
2135 int kvm_vcpu_ioctl(CPUState *cpu, int type, ...)
2136 {
2137     int ret;
2138     void *arg;
2139     va_list ap;
2140
2141     va_start(ap, type);
2142     arg = va_arg(ap, void *);
2143     va_end(ap);
2144
2145     trace_kvm_vcpu_ioctl(cpu->cpu_index, type, arg);
2146     ret = ioctl(cpu->kvm_fd, type, arg);
2147     if (ret == -1) {
2148         ret = -errno;
2149     }
2150     return ret;
2151 }
2152
2153 int kvm_device_ioctl(int fd, int type, ...)
2154 {
2155     int ret;
2156     void *arg;
2157     va_list ap;
2158
2159     va_start(ap, type);
2160     arg = va_arg(ap, void *);
2161     va_end(ap);
2162
2163     trace_kvm_device_ioctl(fd, type, arg);
2164     ret = ioctl(fd, type, arg);
2165     if (ret == -1) {
2166         ret = -errno;
2167     }
2168     return ret;
2169 }
2170
2171 int kvm_vm_check_attr(KVMState *s, uint32_t group, uint64_t attr)
2172 {
2173     int ret;
2174     struct kvm_device_attr attribute = {
2175         .group = group,
2176         .attr = attr,
2177     };
2178
2179     if (!kvm_vm_attributes_allowed) {
2180         return 0;
2181     }
2182
2183     ret = kvm_vm_ioctl(s, KVM_HAS_DEVICE_ATTR, &attribute);
2184     /* kvm returns 0 on success for HAS_DEVICE_ATTR */
2185     return ret ? 0 : 1;
2186 }
2187
2188 int kvm_device_check_attr(int dev_fd, uint32_t group, uint64_t attr)
2189 {
2190     struct kvm_device_attr attribute = {
2191         .group = group,
2192         .attr = attr,
2193         .flags = 0,
2194     };
2195
2196     return kvm_device_ioctl(dev_fd, KVM_HAS_DEVICE_ATTR, &attribute) ? 0 : 1;
2197 }
2198
2199 int kvm_device_access(int fd, int group, uint64_t attr,
2200                       void *val, bool write, Error **errp)
2201 {
2202     struct kvm_device_attr kvmattr;
2203     int err;
2204
2205     kvmattr.flags = 0;
2206     kvmattr.group = group;
2207     kvmattr.attr = attr;
2208     kvmattr.addr = (uintptr_t)val;
2209
2210     err = kvm_device_ioctl(fd,
2211                            write ? KVM_SET_DEVICE_ATTR : KVM_GET_DEVICE_ATTR,
2212                            &kvmattr);
2213     if (err < 0) {
2214         error_setg_errno(errp, -err,
2215                          "KVM_%s_DEVICE_ATTR failed: Group %d "
2216                          "attr 0x%016" PRIx64,
2217                          write ? "SET" : "GET", group, attr);
2218     }
2219     return err;
2220 }
2221
2222 bool kvm_has_sync_mmu(void)
2223 {
2224     return kvm_state->sync_mmu;
2225 }
2226
2227 int kvm_has_vcpu_events(void)
2228 {
2229     return kvm_state->vcpu_events;
2230 }
2231
2232 int kvm_has_robust_singlestep(void)
2233 {
2234     return kvm_state->robust_singlestep;
2235 }
2236
2237 int kvm_has_debugregs(void)
2238 {
2239     return kvm_state->debugregs;
2240 }
2241
2242 int kvm_has_many_ioeventfds(void)
2243 {
2244     if (!kvm_enabled()) {
2245         return 0;
2246     }
2247     return kvm_state->many_ioeventfds;
2248 }
2249
2250 int kvm_has_gsi_routing(void)
2251 {
2252 #ifdef KVM_CAP_IRQ_ROUTING
2253     return kvm_check_extension(kvm_state, KVM_CAP_IRQ_ROUTING);
2254 #else
2255     return false;
2256 #endif
2257 }
2258
2259 int kvm_has_intx_set_mask(void)
2260 {
2261     return kvm_state->intx_set_mask;
2262 }
2263
2264 bool kvm_arm_supports_user_irq(void)
2265 {
2266     return kvm_check_extension(kvm_state, KVM_CAP_ARM_USER_IRQ);
2267 }
2268
2269 #ifdef KVM_CAP_SET_GUEST_DEBUG
2270 struct kvm_sw_breakpoint *kvm_find_sw_breakpoint(CPUState *cpu,
2271                                                  target_ulong pc)
2272 {
2273     struct kvm_sw_breakpoint *bp;
2274
2275     QTAILQ_FOREACH(bp, &cpu->kvm_state->kvm_sw_breakpoints, entry) {
2276         if (bp->pc == pc) {
2277             return bp;
2278         }
2279     }
2280     return NULL;
2281 }
2282
2283 int kvm_sw_breakpoints_active(CPUState *cpu)
2284 {
2285     return !QTAILQ_EMPTY(&cpu->kvm_state->kvm_sw_breakpoints);
2286 }
2287
2288 struct kvm_set_guest_debug_data {
2289     struct kvm_guest_debug dbg;
2290     int err;
2291 };
2292
2293 static void kvm_invoke_set_guest_debug(CPUState *cpu, run_on_cpu_data data)
2294 {
2295     struct kvm_set_guest_debug_data *dbg_data =
2296         (struct kvm_set_guest_debug_data *) data.host_ptr;
2297
2298     dbg_data->err = kvm_vcpu_ioctl(cpu, KVM_SET_GUEST_DEBUG,
2299                                    &dbg_data->dbg);
2300 }
2301
2302 int kvm_update_guest_debug(CPUState *cpu, unsigned long reinject_trap)
2303 {
2304     struct kvm_set_guest_debug_data data;
2305
2306     data.dbg.control = reinject_trap;
2307
2308     if (cpu->singlestep_enabled) {
2309         data.dbg.control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_SINGLESTEP;
2310     }
2311     kvm_arch_update_guest_debug(cpu, &data.dbg);
2312
2313     run_on_cpu(cpu, kvm_invoke_set_guest_debug,
2314                RUN_ON_CPU_HOST_PTR(&data));
2315     return data.err;
2316 }
2317
2318 int kvm_insert_breakpoint(CPUState *cpu, target_ulong addr,
2319                           target_ulong len, int type)
2320 {
2321     struct kvm_sw_breakpoint *bp;
2322     int err;
2323
2324     if (type == GDB_BREAKPOINT_SW) {
2325         bp = kvm_find_sw_breakpoint(cpu, addr);
2326         if (bp) {
2327             bp->use_count++;
2328             return 0;
2329         }
2330
2331         bp = g_malloc(sizeof(struct kvm_sw_breakpoint));
2332         bp->pc = addr;
2333         bp->use_count = 1;
2334         err = kvm_arch_insert_sw_breakpoint(cpu, bp);
2335         if (err) {
2336             g_free(bp);
2337             return err;
2338         }
2339
2340         QTAILQ_INSERT_HEAD(&cpu->kvm_state->kvm_sw_breakpoints, bp, entry);
2341     } else {
2342         err = kvm_arch_insert_hw_breakpoint(addr, len, type);
2343         if (err) {
2344             return err;
2345         }
2346     }
2347
2348     CPU_FOREACH(cpu) {
2349         err = kvm_update_guest_debug(cpu, 0);
2350         if (err) {
2351             return err;
2352         }
2353     }
2354     return 0;
2355 }
2356
2357 int kvm_remove_breakpoint(CPUState *cpu, target_ulong addr,
2358                           target_ulong len, int type)
2359 {
2360     struct kvm_sw_breakpoint *bp;
2361     int err;
2362
2363     if (type == GDB_BREAKPOINT_SW) {
2364         bp = kvm_find_sw_breakpoint(cpu, addr);
2365         if (!bp) {
2366             return -ENOENT;
2367         }
2368
2369         if (bp->use_count > 1) {
2370             bp->use_count--;
2371             return 0;
2372         }
2373
2374         err = kvm_arch_remove_sw_breakpoint(cpu, bp);
2375         if (err) {
2376             return err;
2377         }
2378
2379         QTAILQ_REMOVE(&cpu->kvm_state->kvm_sw_breakpoints, bp, entry);
2380         g_free(bp);
2381     } else {
2382         err = kvm_arch_remove_hw_breakpoint(addr, len, type);
2383         if (err) {
2384             return err;
2385         }
2386     }
2387
2388     CPU_FOREACH(cpu) {
2389         err = kvm_update_guest_debug(cpu, 0);
2390         if (err) {
2391             return err;
2392         }
2393     }
2394     return 0;
2395 }
2396
2397 void kvm_remove_all_breakpoints(CPUState *cpu)
2398 {
2399     struct kvm_sw_breakpoint *bp, *next;
2400     KVMState *s = cpu->kvm_state;
2401     CPUState *tmpcpu;
2402
2403     QTAILQ_FOREACH_SAFE(bp, &s->kvm_sw_breakpoints, entry, next) {
2404         if (kvm_arch_remove_sw_breakpoint(cpu, bp) != 0) {
2405             /* Try harder to find a CPU that currently sees the breakpoint. */
2406             CPU_FOREACH(tmpcpu) {
2407                 if (kvm_arch_remove_sw_breakpoint(tmpcpu, bp) == 0) {
2408                     break;
2409                 }
2410             }
2411         }
2412         QTAILQ_REMOVE(&s->kvm_sw_breakpoints, bp, entry);
2413         g_free(bp);
2414     }
2415     kvm_arch_remove_all_hw_breakpoints();
2416
2417     CPU_FOREACH(cpu) {
2418         kvm_update_guest_debug(cpu, 0);
2419     }
2420 }
2421
2422 #else /* !KVM_CAP_SET_GUEST_DEBUG */
2423
2424 int kvm_update_guest_debug(CPUState *cpu, unsigned long reinject_trap)
2425 {
2426     return -EINVAL;
2427 }
2428
2429 int kvm_insert_breakpoint(CPUState *cpu, target_ulong addr,
2430                           target_ulong len, int type)
2431 {
2432     return -EINVAL;
2433 }
2434
2435 int kvm_remove_breakpoint(CPUState *cpu, target_ulong addr,
2436                           target_ulong len, int type)
2437 {
2438     return -EINVAL;
2439 }
2440
2441 void kvm_remove_all_breakpoints(CPUState *cpu)
2442 {
2443 }
2444 #endif /* !KVM_CAP_SET_GUEST_DEBUG */
2445
2446 static int kvm_set_signal_mask(CPUState *cpu, const sigset_t *sigset)
2447 {
2448     KVMState *s = kvm_state;
2449     struct kvm_signal_mask *sigmask;
2450     int r;
2451
2452     sigmask = g_malloc(sizeof(*sigmask) + sizeof(*sigset));
2453
2454     sigmask->len = s->sigmask_len;
2455     memcpy(sigmask->sigset, sigset, sizeof(*sigset));
2456     r = kvm_vcpu_ioctl(cpu, KVM_SET_SIGNAL_MASK, sigmask);
2457     g_free(sigmask);
2458
2459     return r;
2460 }
2461
2462 static void kvm_ipi_signal(int sig)
2463 {
2464     if (current_cpu) {
2465         assert(kvm_immediate_exit);
2466         kvm_cpu_kick(current_cpu);
2467     }
2468 }
2469
2470 void kvm_init_cpu_signals(CPUState *cpu)
2471 {
2472     int r;
2473     sigset_t set;
2474     struct sigaction sigact;
2475
2476     memset(&sigact, 0, sizeof(sigact));
2477     sigact.sa_handler = kvm_ipi_signal;
2478     sigaction(SIG_IPI, &sigact, NULL);
2479
2480     pthread_sigmask(SIG_BLOCK, NULL, &set);
2481 #if defined KVM_HAVE_MCE_INJECTION
2482     sigdelset(&set, SIGBUS);
2483     pthread_sigmask(SIG_SETMASK, &set, NULL);
2484 #endif
2485     sigdelset(&set, SIG_IPI);
2486     if (kvm_immediate_exit) {
2487         r = pthread_sigmask(SIG_SETMASK, &set, NULL);
2488     } else {
2489         r = kvm_set_signal_mask(cpu, &set);
2490     }
2491     if (r) {
2492         fprintf(stderr, "kvm_set_signal_mask: %s\n", strerror(-r));
2493         exit(1);
2494     }
2495 }
2496
2497 /* Called asynchronously in VCPU thread.  */
2498 int kvm_on_sigbus_vcpu(CPUState *cpu, int code, void *addr)
2499 {
2500 #ifdef KVM_HAVE_MCE_INJECTION
2501     if (have_sigbus_pending) {
2502         return 1;
2503     }
2504     have_sigbus_pending = true;
2505     pending_sigbus_addr = addr;
2506     pending_sigbus_code = code;
2507     atomic_set(&cpu->exit_request, 1);
2508     return 0;
2509 #else
2510     return 1;
2511 #endif
2512 }
2513
2514 /* Called synchronously (via signalfd) in main thread.  */
2515 int kvm_on_sigbus(int code, void *addr)
2516 {
2517 #ifdef KVM_HAVE_MCE_INJECTION
2518     /* Action required MCE kills the process if SIGBUS is blocked.  Because
2519      * that's what happens in the I/O thread, where we handle MCE via signalfd,
2520      * we can only get action optional here.
2521      */
2522     assert(code != BUS_MCEERR_AR);
2523     kvm_arch_on_sigbus_vcpu(first_cpu, code, addr);
2524     return 0;
2525 #else
2526     return 1;
2527 #endif
2528 }
2529
2530 int kvm_create_device(KVMState *s, uint64_t type, bool test)
2531 {
2532     int ret;
2533     struct kvm_create_device create_dev;
2534
2535     create_dev.type = type;
2536     create_dev.fd = -1;
2537     create_dev.flags = test ? KVM_CREATE_DEVICE_TEST : 0;
2538
2539     if (!kvm_check_extension(s, KVM_CAP_DEVICE_CTRL)) {
2540         return -ENOTSUP;
2541     }
2542
2543     ret = kvm_vm_ioctl(s, KVM_CREATE_DEVICE, &create_dev);
2544     if (ret) {
2545         return ret;
2546     }
2547
2548     return test ? 0 : create_dev.fd;
2549 }
2550
2551 bool kvm_device_supported(int vmfd, uint64_t type)
2552 {
2553     struct kvm_create_device create_dev = {
2554         .type = type,
2555         .fd = -1,
2556         .flags = KVM_CREATE_DEVICE_TEST,
2557     };
2558
2559     if (ioctl(vmfd, KVM_CHECK_EXTENSION, KVM_CAP_DEVICE_CTRL) <= 0) {
2560         return false;
2561     }
2562
2563     return (ioctl(vmfd, KVM_CREATE_DEVICE, &create_dev) >= 0);
2564 }
2565
2566 int kvm_set_one_reg(CPUState *cs, uint64_t id, void *source)
2567 {
2568     struct kvm_one_reg reg;
2569     int r;
2570
2571     reg.id = id;
2572     reg.addr = (uintptr_t) source;
2573     r = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
2574     if (r) {
2575         trace_kvm_failed_reg_set(id, strerror(-r));
2576     }
2577     return r;
2578 }
2579
2580 int kvm_get_one_reg(CPUState *cs, uint64_t id, void *target)
2581 {
2582     struct kvm_one_reg reg;
2583     int r;
2584
2585     reg.id = id;
2586     reg.addr = (uintptr_t) target;
2587     r = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
2588     if (r) {
2589         trace_kvm_failed_reg_get(id, strerror(-r));
2590     }
2591     return r;
2592 }
2593
2594 static void kvm_accel_class_init(ObjectClass *oc, void *data)
2595 {
2596     AccelClass *ac = ACCEL_CLASS(oc);
2597     ac->name = "KVM";
2598     ac->init_machine = kvm_init;
2599     ac->allowed = &kvm_allowed;
2600 }
2601
2602 static const TypeInfo kvm_accel_type = {
2603     .name = TYPE_KVM_ACCEL,
2604     .parent = TYPE_ACCEL,
2605     .class_init = kvm_accel_class_init,
2606     .instance_size = sizeof(KVMState),
2607 };
2608
2609 static void kvm_type_init(void)
2610 {
2611     type_register_static(&kvm_accel_type);
2612 }
2613
2614 type_init(kvm_type_init);