target/arm/kvm.c

   1 /*
   2  * ARM implementation of KVM hooks
   3  *
   4  * Copyright Christoffer Dall 2009-2010
   5  *
   6  * This work is licensed under the terms of the GNU GPL, version 2 or later.
   7  * See the COPYING file in the top-level directory.
   8  *
   9  */
  10
  11 #include "qemu/osdep.h"
  12 #include <sys/ioctl.h>
  13
  14 #include <linux/kvm.h>
  15
  16 #include "qemu-common.h"
  17 #include "qemu/timer.h"
  18 #include "qemu/error-report.h"
  19 #include "sysemu/sysemu.h"
  20 #include "sysemu/kvm.h"
  21 #include "kvm_arm.h"
  22 #include "cpu.h"
  23 #include "internals.h"
  24 #include "hw/arm/arm.h"
  25 #include "exec/memattrs.h"
  26 #include "exec/address-spaces.h"
  27 #include "hw/boards.h"
  28 #include "qemu/log.h"
  29
  30 const KVMCapabilityInfo kvm_arch_required_capabilities[] = {
  31     KVM_CAP_LAST_INFO
  32 };
  33
  34 static bool cap_has_mp_state;
  35
  36 static ARMHostCPUFeatures arm_host_cpu_features;
  37
  38 int kvm_arm_vcpu_init(CPUState *cs)
  39 {
  40     ARMCPU *cpu = ARM_CPU(cs);
  41     struct kvm_vcpu_init init;
  42
  43     init.target = cpu->kvm_target;
  44     memcpy(init.features, cpu->kvm_init_features, sizeof(init.features));
  45
  46     return kvm_vcpu_ioctl(cs, KVM_ARM_VCPU_INIT, &init);
  47 }
  48
  49 bool kvm_arm_create_scratch_host_vcpu(const uint32_t *cpus_to_try,
  50                                       int *fdarray,
  51                                       struct kvm_vcpu_init *init)
  52 {
  53     int ret, kvmfd = -1, vmfd = -1, cpufd = -1;
  54
  55     kvmfd = qemu_open("/dev/kvm", O_RDWR);
  56     if (kvmfd < 0) {
  57         goto err;
  58     }
  59     vmfd = ioctl(kvmfd, KVM_CREATE_VM, 0);
  60     if (vmfd < 0) {
  61         goto err;
  62     }
  63     cpufd = ioctl(vmfd, KVM_CREATE_VCPU, 0);
  64     if (cpufd < 0) {
  65         goto err;
  66     }
  67
  68     if (!init) {
  69         /* Caller doesn't want the VCPU to be initialized, so skip it */
  70         goto finish;
  71     }
  72
  73     ret = ioctl(vmfd, KVM_ARM_PREFERRED_TARGET, init);
  74     if (ret >= 0) {
  75         ret = ioctl(cpufd, KVM_ARM_VCPU_INIT, init);
  76         if (ret < 0) {
  77             goto err;
  78         }
  79     } else if (cpus_to_try) {
  80         /* Old kernel which doesn't know about the
  81          * PREFERRED_TARGET ioctl: we know it will only support
  82          * creating one kind of guest CPU which is its preferred
  83          * CPU type.
  84          */
  85         while (*cpus_to_try != QEMU_KVM_ARM_TARGET_NONE) {
  86             init->target = *cpus_to_try++;
  87             memset(init->features, 0, sizeof(init->features));
  88             ret = ioctl(cpufd, KVM_ARM_VCPU_INIT, init);
  89             if (ret >= 0) {
  90                 break;
  91             }
  92         }
  93         if (ret < 0) {
  94             goto err;
  95         }
  96     } else {
  97         /* Treat a NULL cpus_to_try argument the same as an empty
  98          * list, which means we will fail the call since this must
  99          * be an old kernel which doesn't support PREFERRED_TARGET.
 100          */
 101         goto err;
 102     }
 103
 104 finish:
 105     fdarray[0] = kvmfd;
 106     fdarray[1] = vmfd;
 107     fdarray[2] = cpufd;
 108
 109     return true;
 110
 111 err:
 112     if (cpufd >= 0) {
 113         close(cpufd);
 114     }
 115     if (vmfd >= 0) {
 116         close(vmfd);
 117     }
 118     if (kvmfd >= 0) {
 119         close(kvmfd);
 120     }
 121
 122     return false;
 123 }
 124
 125 void kvm_arm_destroy_scratch_host_vcpu(int *fdarray)
 126 {
 127     int i;
 128
 129     for (i = 2; i >= 0; i--) {
 130         close(fdarray[i]);
 131     }
 132 }
 133
 134 void kvm_arm_set_cpu_features_from_host(ARMCPU *cpu)
 135 {
 136     CPUARMState *env = &cpu->env;
 137
 138     if (!arm_host_cpu_features.dtb_compatible) {
 139         if (!kvm_enabled() ||
 140             !kvm_arm_get_host_cpu_features(&arm_host_cpu_features)) {
 141             /* We can't report this error yet, so flag that we need to
 142              * in arm_cpu_realizefn().
 143              */
 144             cpu->kvm_target = QEMU_KVM_ARM_TARGET_NONE;
 145             cpu->host_cpu_probe_failed = true;
 146             return;
 147         }
 148     }
 149
 150     cpu->kvm_target = arm_host_cpu_features.target;
 151     cpu->dtb_compatible = arm_host_cpu_features.dtb_compatible;
 152     env->features = arm_host_cpu_features.features;
 153 }
 154
 155 int kvm_arch_init(MachineState *ms, KVMState *s)
 156 {
 157     /* For ARM interrupt delivery is always asynchronous,
 158      * whether we are using an in-kernel VGIC or not.
 159      */
 160     kvm_async_interrupts_allowed = true;
 161
 162     /*
 163      * PSCI wakes up secondary cores, so we always need to
 164      * have vCPUs waiting in kernel space
 165      */
 166     kvm_halt_in_kernel_allowed = true;
 167
 168     cap_has_mp_state = kvm_check_extension(s, KVM_CAP_MP_STATE);
 169
 170     return 0;
 171 }
 172
 173 unsigned long kvm_arch_vcpu_id(CPUState *cpu)
 174 {
 175     return cpu->cpu_index;
 176 }
 177
 178 /* We track all the KVM devices which need their memory addresses
 179  * passing to the kernel in a list of these structures.
 180  * When board init is complete we run through the list and
 181  * tell the kernel the base addresses of the memory regions.
 182  * We use a MemoryListener to track mapping and unmapping of
 183  * the regions during board creation, so the board models don't
 184  * need to do anything special for the KVM case.
 185  */
 186 typedef struct KVMDevice {
 187     struct kvm_arm_device_addr kda;
 188     struct kvm_device_attr kdattr;
 189     MemoryRegion *mr;
 190     QSLIST_ENTRY(KVMDevice) entries;
 191     int dev_fd;
 192 } KVMDevice;
 193
 194 static QSLIST_HEAD(kvm_devices_head, KVMDevice) kvm_devices_head;
 195
 196 static void kvm_arm_devlistener_add(MemoryListener *listener,
 197                                     MemoryRegionSection *section)
 198 {
 199     KVMDevice *kd;
 200
 201     QSLIST_FOREACH(kd, &kvm_devices_head, entries) {
 202         if (section->mr == kd->mr) {
 203             kd->kda.addr = section->offset_within_address_space;
 204         }
 205     }
 206 }
 207
 208 static void kvm_arm_devlistener_del(MemoryListener *listener,
 209                                     MemoryRegionSection *section)
 210 {
 211     KVMDevice *kd;
 212
 213     QSLIST_FOREACH(kd, &kvm_devices_head, entries) {
 214         if (section->mr == kd->mr) {
 215             kd->kda.addr = -1;
 216         }
 217     }
 218 }
 219
 220 static MemoryListener devlistener = {
 221     .region_add = kvm_arm_devlistener_add,
 222     .region_del = kvm_arm_devlistener_del,
 223 };
 224
 225 static void kvm_arm_set_device_addr(KVMDevice *kd)
 226 {
 227     struct kvm_device_attr *attr = &kd->kdattr;
 228     int ret;
 229
 230     /* If the device control API is available and we have a device fd on the
 231      * KVMDevice struct, let's use the newer API
 232      */
 233     if (kd->dev_fd >= 0) {
 234         uint64_t addr = kd->kda.addr;
 235         attr->addr = (uintptr_t)&addr;
 236         ret = kvm_device_ioctl(kd->dev_fd, KVM_SET_DEVICE_ATTR, attr);
 237     } else {
 238         ret = kvm_vm_ioctl(kvm_state, KVM_ARM_SET_DEVICE_ADDR, &kd->kda);
 239     }
 240
 241     if (ret < 0) {
 242         fprintf(stderr, "Failed to set device address: %s\n",
 243                 strerror(-ret));
 244         abort();
 245     }
 246 }
 247
 248 static void kvm_arm_machine_init_done(Notifier *notifier, void *data)
 249 {
 250     KVMDevice *kd, *tkd;
 251
 252     QSLIST_FOREACH_SAFE(kd, &kvm_devices_head, entries, tkd) {
 253         if (kd->kda.addr != -1) {
 254             kvm_arm_set_device_addr(kd);
 255         }
 256         memory_region_unref(kd->mr);
 257         g_free(kd);
 258     }
 259     memory_listener_unregister(&devlistener);
 260 }
 261
 262 static Notifier notify = {
 263     .notify = kvm_arm_machine_init_done,
 264 };
 265
 266 void kvm_arm_register_device(MemoryRegion *mr, uint64_t devid, uint64_t group,
 267                              uint64_t attr, int dev_fd)
 268 {
 269     KVMDevice *kd;
 270
 271     if (!kvm_irqchip_in_kernel()) {
 272         return;
 273     }
 274
 275     if (QSLIST_EMPTY(&kvm_devices_head)) {
 276         memory_listener_register(&devlistener, &address_space_memory);
 277         qemu_add_machine_init_done_notifier(&notify);
 278     }
 279     kd = g_new0(KVMDevice, 1);
 280     kd->mr = mr;
 281     kd->kda.id = devid;
 282     kd->kda.addr = -1;
 283     kd->kdattr.flags = 0;
 284     kd->kdattr.group = group;
 285     kd->kdattr.attr = attr;
 286     kd->dev_fd = dev_fd;
 287     QSLIST_INSERT_HEAD(&kvm_devices_head, kd, entries);
 288     memory_region_ref(kd->mr);
 289 }
 290
 291 static int compare_u64(const void *a, const void *b)
 292 {
 293     if (*(uint64_t *)a > *(uint64_t *)b) {
 294         return 1;
 295     }
 296     if (*(uint64_t *)a < *(uint64_t *)b) {
 297         return -1;
 298     }
 299     return 0;
 300 }
 301
 302 /* Initialize the CPUState's cpreg list according to the kernel's
 303  * definition of what CPU registers it knows about (and throw away
 304  * the previous TCG-created cpreg list).
 305  */
 306 int kvm_arm_init_cpreg_list(ARMCPU *cpu)
 307 {
 308     struct kvm_reg_list rl;
 309     struct kvm_reg_list *rlp;
 310     int i, ret, arraylen;
 311     CPUState *cs = CPU(cpu);
 312
 313     rl.n = 0;
 314     ret = kvm_vcpu_ioctl(cs, KVM_GET_REG_LIST, &rl);
 315     if (ret != -E2BIG) {
 316         return ret;
 317     }
 318     rlp = g_malloc(sizeof(struct kvm_reg_list) + rl.n * sizeof(uint64_t));
 319     rlp->n = rl.n;
 320     ret = kvm_vcpu_ioctl(cs, KVM_GET_REG_LIST, rlp);
 321     if (ret) {
 322         goto out;
 323     }
 324     /* Sort the list we get back from the kernel, since cpreg_tuples
 325      * must be in strictly ascending order.
 326      */
 327     qsort(&rlp->reg, rlp->n, sizeof(rlp->reg[0]), compare_u64);
 328
 329     for (i = 0, arraylen = 0; i < rlp->n; i++) {
 330         if (!kvm_arm_reg_syncs_via_cpreg_list(rlp->reg[i])) {
 331             continue;
 332         }
 333         switch (rlp->reg[i] & KVM_REG_SIZE_MASK) {
 334         case KVM_REG_SIZE_U32:
 335         case KVM_REG_SIZE_U64:
 336             break;
 337         default:
 338             fprintf(stderr, "Can't handle size of register in kernel list\n");
 339             ret = -EINVAL;
 340             goto out;
 341         }
 342
 343         arraylen++;
 344     }
 345
 346     cpu->cpreg_indexes = g_renew(uint64_t, cpu->cpreg_indexes, arraylen);
 347     cpu->cpreg_values = g_renew(uint64_t, cpu->cpreg_values, arraylen);
 348     cpu->cpreg_vmstate_indexes = g_renew(uint64_t, cpu->cpreg_vmstate_indexes,
 349                                          arraylen);
 350     cpu->cpreg_vmstate_values = g_renew(uint64_t, cpu->cpreg_vmstate_values,
 351                                         arraylen);
 352     cpu->cpreg_array_len = arraylen;
 353     cpu->cpreg_vmstate_array_len = arraylen;
 354
 355     for (i = 0, arraylen = 0; i < rlp->n; i++) {
 356         uint64_t regidx = rlp->reg[i];
 357         if (!kvm_arm_reg_syncs_via_cpreg_list(regidx)) {
 358             continue;
 359         }
 360         cpu->cpreg_indexes[arraylen] = regidx;
 361         arraylen++;
 362     }
 363     assert(cpu->cpreg_array_len == arraylen);
 364
 365     if (!write_kvmstate_to_list(cpu)) {
 366         /* Shouldn't happen unless kernel is inconsistent about
 367          * what registers exist.
 368          */
 369         fprintf(stderr, "Initial read of kernel register state failed\n");
 370         ret = -EINVAL;
 371         goto out;
 372     }
 373
 374 out:
 375     g_free(rlp);
 376     return ret;
 377 }
 378
 379 bool write_kvmstate_to_list(ARMCPU *cpu)
 380 {
 381     CPUState *cs = CPU(cpu);
 382     int i;
 383     bool ok = true;
 384
 385     for (i = 0; i < cpu->cpreg_array_len; i++) {
 386         struct kvm_one_reg r;
 387         uint64_t regidx = cpu->cpreg_indexes[i];
 388         uint32_t v32;
 389         int ret;
 390
 391         r.id = regidx;
 392
 393         switch (regidx & KVM_REG_SIZE_MASK) {
 394         case KVM_REG_SIZE_U32:
 395             r.addr = (uintptr_t)&v32;
 396             ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &r);
 397             if (!ret) {
 398                 cpu->cpreg_values[i] = v32;
 399             }
 400             break;
 401         case KVM_REG_SIZE_U64:
 402             r.addr = (uintptr_t)(cpu->cpreg_values + i);
 403             ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &r);
 404             break;
 405         default:
 406             abort();
 407         }
 408         if (ret) {
 409             ok = false;
 410         }
 411     }
 412     return ok;
 413 }
 414
 415 bool write_list_to_kvmstate(ARMCPU *cpu, int level)
 416 {
 417     CPUState *cs = CPU(cpu);
 418     int i;
 419     bool ok = true;
 420
 421     for (i = 0; i < cpu->cpreg_array_len; i++) {
 422         struct kvm_one_reg r;
 423         uint64_t regidx = cpu->cpreg_indexes[i];
 424         uint32_t v32;
 425         int ret;
 426
 427         if (kvm_arm_cpreg_level(regidx) > level) {
 428             continue;
 429         }
 430
 431         r.id = regidx;
 432         switch (regidx & KVM_REG_SIZE_MASK) {
 433         case KVM_REG_SIZE_U32:
 434             v32 = cpu->cpreg_values[i];
 435             r.addr = (uintptr_t)&v32;
 436             break;
 437         case KVM_REG_SIZE_U64:
 438             r.addr = (uintptr_t)(cpu->cpreg_values + i);
 439             break;
 440         default:
 441             abort();
 442         }
 443         ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &r);
 444         if (ret) {
 445             /* We might fail for "unknown register" and also for
 446              * "you tried to set a register which is constant with
 447              * a different value from what it actually contains".
 448              */
 449             ok = false;
 450         }
 451     }
 452     return ok;
 453 }
 454
 455 void kvm_arm_reset_vcpu(ARMCPU *cpu)
 456 {
 457     int ret;
 458
 459     /* Re-init VCPU so that all registers are set to
 460      * their respective reset values.
 461      */
 462     ret = kvm_arm_vcpu_init(CPU(cpu));
 463     if (ret < 0) {
 464         fprintf(stderr, "kvm_arm_vcpu_init failed: %s\n", strerror(-ret));
 465         abort();
 466     }
 467     if (!write_kvmstate_to_list(cpu)) {
 468         fprintf(stderr, "write_kvmstate_to_list failed\n");
 469         abort();
 470     }
 471 }
 472
 473 /*
 474  * Update KVM's MP_STATE based on what QEMU thinks it is
 475  */
 476 int kvm_arm_sync_mpstate_to_kvm(ARMCPU *cpu)
 477 {
 478     if (cap_has_mp_state) {
 479         struct kvm_mp_state mp_state = {
 480             .mp_state = (cpu->power_state == PSCI_OFF) ?
 481             KVM_MP_STATE_STOPPED : KVM_MP_STATE_RUNNABLE
 482         };
 483         int ret = kvm_vcpu_ioctl(CPU(cpu), KVM_SET_MP_STATE, &mp_state);
 484         if (ret) {
 485             fprintf(stderr, "%s: failed to set MP_STATE %d/%s\n",
 486                     __func__, ret, strerror(-ret));
 487             return -1;
 488         }
 489     }
 490
 491     return 0;
 492 }
 493
 494 /*
 495  * Sync the KVM MP_STATE into QEMU
 496  */
 497 int kvm_arm_sync_mpstate_to_qemu(ARMCPU *cpu)
 498 {
 499     if (cap_has_mp_state) {
 500         struct kvm_mp_state mp_state;
 501         int ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_MP_STATE, &mp_state);
 502         if (ret) {
 503             fprintf(stderr, "%s: failed to get MP_STATE %d/%s\n",
 504                     __func__, ret, strerror(-ret));
 505             abort();
 506         }
 507         cpu->power_state = (mp_state.mp_state == KVM_MP_STATE_STOPPED) ?
 508             PSCI_OFF : PSCI_ON;
 509     }
 510
 511     return 0;
 512 }
 513
 514 void kvm_arch_pre_run(CPUState *cs, struct kvm_run *run)
 515 {
 516 }
 517
 518 MemTxAttrs kvm_arch_post_run(CPUState *cs, struct kvm_run *run)
 519 {
 520     ARMCPU *cpu;
 521     uint32_t switched_level;
 522
 523     if (kvm_irqchip_in_kernel()) {
 524         /*
 525          * We only need to sync timer states with user-space interrupt
 526          * controllers, so return early and save cycles if we don't.
 527          */
 528         return MEMTXATTRS_UNSPECIFIED;
 529     }
 530
 531     cpu = ARM_CPU(cs);
 532
 533     /* Synchronize our shadowed in-kernel device irq lines with the kvm ones */
 534     if (run->s.regs.device_irq_level != cpu->device_irq_level) {
 535         switched_level = cpu->device_irq_level ^ run->s.regs.device_irq_level;
 536
 537         qemu_mutex_lock_iothread();
 538
 539         if (switched_level & KVM_ARM_DEV_EL1_VTIMER) {
 540             qemu_set_irq(cpu->gt_timer_outputs[GTIMER_VIRT],
 541                          !!(run->s.regs.device_irq_level &
 542                             KVM_ARM_DEV_EL1_VTIMER));
 543             switched_level &= ~KVM_ARM_DEV_EL1_VTIMER;
 544         }
 545
 546         if (switched_level & KVM_ARM_DEV_EL1_PTIMER) {
 547             qemu_set_irq(cpu->gt_timer_outputs[GTIMER_PHYS],
 548                          !!(run->s.regs.device_irq_level &
 549                             KVM_ARM_DEV_EL1_PTIMER));
 550             switched_level &= ~KVM_ARM_DEV_EL1_PTIMER;
 551         }
 552
 553         if (switched_level & KVM_ARM_DEV_PMU) {
 554             qemu_set_irq(cpu->pmu_interrupt,
 555                          !!(run->s.regs.device_irq_level & KVM_ARM_DEV_PMU));
 556             switched_level &= ~KVM_ARM_DEV_PMU;
 557         }
 558
 559         if (switched_level) {
 560             qemu_log_mask(LOG_UNIMP, "%s: unhandled in-kernel device IRQ %x\n",
 561                           __func__, switched_level);
 562         }
 563
 564         /* We also mark unknown levels as processed to not waste cycles */
 565         cpu->device_irq_level = run->s.regs.device_irq_level;
 566         qemu_mutex_unlock_iothread();
 567     }
 568
 569     return MEMTXATTRS_UNSPECIFIED;
 570 }
 571
 572
 573 int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run)
 574 {
 575     int ret = 0;
 576
 577     switch (run->exit_reason) {
 578     case KVM_EXIT_DEBUG:
 579         if (kvm_arm_handle_debug(cs, &run->debug.arch)) {
 580             ret = EXCP_DEBUG;
 581         } /* otherwise return to guest */
 582         break;
 583     default:
 584         qemu_log_mask(LOG_UNIMP, "%s: un-handled exit reason %d\n",
 585                       __func__, run->exit_reason);
 586         break;
 587     }
 588     return ret;
 589 }
 590
 591 bool kvm_arch_stop_on_emulation_error(CPUState *cs)
 592 {
 593     return true;
 594 }
 595
 596 int kvm_arch_process_async_events(CPUState *cs)
 597 {
 598     return 0;
 599 }
 600
 601 /* The #ifdef protections are until 32bit headers are imported and can
 602  * be removed once both 32 and 64 bit reach feature parity.
 603  */
 604 void kvm_arch_update_guest_debug(CPUState *cs, struct kvm_guest_debug *dbg)
 605 {
 606 #ifdef KVM_GUESTDBG_USE_SW_BP
 607     if (kvm_sw_breakpoints_active(cs)) {
 608         dbg->control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP;
 609     }
 610 #endif
 611 #ifdef KVM_GUESTDBG_USE_HW
 612     if (kvm_arm_hw_debug_active(cs)) {
 613         dbg->control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_HW;
 614         kvm_arm_copy_hw_debug_data(&dbg->arch);
 615     }
 616 #endif
 617 }
 618
 619 void kvm_arch_init_irq_routing(KVMState *s)
 620 {
 621 }
 622
 623 int kvm_arch_irqchip_create(MachineState *ms, KVMState *s)
 624 {
 625      if (machine_kernel_irqchip_split(ms)) {
 626          perror("-machine kernel_irqchip=split is not supported on ARM.");
 627          exit(1);
 628     }
 629
 630     /* If we can create the VGIC using the newer device control API, we
 631      * let the device do this when it initializes itself, otherwise we
 632      * fall back to the old API */
 633     return kvm_check_extension(s, KVM_CAP_DEVICE_CTRL);
 634 }
 635
 636 int kvm_arm_vgic_probe(void)
 637 {
 638     if (kvm_create_device(kvm_state,
 639                           KVM_DEV_TYPE_ARM_VGIC_V3, true) == 0) {
 640         return 3;
 641     } else if (kvm_create_device(kvm_state,
 642                                  KVM_DEV_TYPE_ARM_VGIC_V2, true) == 0) {
 643         return 2;
 644     } else {
 645         return 0;
 646     }
 647 }
 648
 649 int kvm_arch_fixup_msi_route(struct kvm_irq_routing_entry *route,
 650                              uint64_t address, uint32_t data, PCIDevice *dev)
 651 {
 652     return 0;
 653 }
 654
 655 int kvm_arch_add_msi_route_post(struct kvm_irq_routing_entry *route,
 656                                 int vector, PCIDevice *dev)
 657 {
 658     return 0;
 659 }
 660
 661 int kvm_arch_release_virq_post(int virq)
 662 {
 663     return 0;
 664 }
 665
 666 int kvm_arch_msi_data_to_gsi(uint32_t data)
 667 {
 668     return (data - 32) & 0xffff;
 669 }