target-i386/kvm.c

   1 /*
   2  * QEMU KVM support
   3  *
   4  * Copyright (C) 2006-2008 Qumranet Technologies
   5  * Copyright IBM, Corp. 2008
   6  *
   7  * Authors:
   8  *  Anthony Liguori   <aliguori@us.ibm.com>
   9  *
  10  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  11  * See the COPYING file in the top-level directory.
  12  *
  13  */
  14
  15 #include <sys/types.h>
  16 #include <sys/ioctl.h>
  17 #include <sys/mman.h>
  18 #include <sys/utsname.h>
  19
  20 #include <linux/kvm.h>
  21 #include <linux/kvm_para.h>
  22
  23 #include "qemu-common.h"
  24 #include "sysemu.h"
  25 #include "kvm.h"
  26 #include "kvm_i386.h"
  27 #include "cpu.h"
  28 #include "gdbstub.h"
  29 #include "host-utils.h"
  30 #include "hw/pc.h"
  31 #include "hw/apic.h"
  32 #include "ioport.h"
  33 #include "hyperv.h"
  34 #include "hw/pci.h"
  35
  36 //#define DEBUG_KVM
  37
  38 #ifdef DEBUG_KVM
  39 #define DPRINTF(fmt, ...) \
  40     do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
  41 #else
  42 #define DPRINTF(fmt, ...) \
  43     do { } while (0)
  44 #endif
  45
  46 #define MSR_KVM_WALL_CLOCK  0x11
  47 #define MSR_KVM_SYSTEM_TIME 0x12
  48
  49 #ifndef BUS_MCEERR_AR
  50 #define BUS_MCEERR_AR 4
  51 #endif
  52 #ifndef BUS_MCEERR_AO
  53 #define BUS_MCEERR_AO 5
  54 #endif
  55
  56 const KVMCapabilityInfo kvm_arch_required_capabilities[] = {
  57     KVM_CAP_INFO(SET_TSS_ADDR),
  58     KVM_CAP_INFO(EXT_CPUID),
  59     KVM_CAP_INFO(MP_STATE),
  60     KVM_CAP_LAST_INFO
  61 };
  62
  63 static bool has_msr_star;
  64 static bool has_msr_hsave_pa;
  65 static bool has_msr_tsc_deadline;
  66 static bool has_msr_async_pf_en;
  67 static bool has_msr_pv_eoi_en;
  68 static bool has_msr_misc_enable;
  69 static int lm_capable_kernel;
  70
  71 bool kvm_allows_irq0_override(void)
  72 {
  73     return !kvm_irqchip_in_kernel() || kvm_has_gsi_routing();
  74 }
  75
  76 static struct kvm_cpuid2 *try_get_cpuid(KVMState *s, int max)
  77 {
  78     struct kvm_cpuid2 *cpuid;
  79     int r, size;
  80
  81     size = sizeof(*cpuid) + max * sizeof(*cpuid->entries);
  82     cpuid = (struct kvm_cpuid2 *)g_malloc0(size);
  83     cpuid->nent = max;
  84     r = kvm_ioctl(s, KVM_GET_SUPPORTED_CPUID, cpuid);
  85     if (r == 0 && cpuid->nent >= max) {
  86         r = -E2BIG;
  87     }
  88     if (r < 0) {
  89         if (r == -E2BIG) {
  90             g_free(cpuid);
  91             return NULL;
  92         } else {
  93             fprintf(stderr, "KVM_GET_SUPPORTED_CPUID failed: %s\n",
  94                     strerror(-r));
  95             exit(1);
  96         }
  97     }
  98     return cpuid;
  99 }
 100
 101 /* Run KVM_GET_SUPPORTED_CPUID ioctl(), allocating a buffer large enough
 102  * for all entries.
 103  */
 104 static struct kvm_cpuid2 *get_supported_cpuid(KVMState *s)
 105 {
 106     struct kvm_cpuid2 *cpuid;
 107     int max = 1;
 108     while ((cpuid = try_get_cpuid(s, max)) == NULL) {
 109         max *= 2;
 110     }
 111     return cpuid;
 112 }
 113
 114 struct kvm_para_features {
 115     int cap;
 116     int feature;
 117 } para_features[] = {
 118     { KVM_CAP_CLOCKSOURCE, KVM_FEATURE_CLOCKSOURCE },
 119     { KVM_CAP_NOP_IO_DELAY, KVM_FEATURE_NOP_IO_DELAY },
 120     { KVM_CAP_PV_MMU, KVM_FEATURE_MMU_OP },
 121     { KVM_CAP_ASYNC_PF, KVM_FEATURE_ASYNC_PF },
 122     { -1, -1 }
 123 };
 124
 125 static int get_para_features(KVMState *s)
 126 {
 127     int i, features = 0;
 128
 129     for (i = 0; i < ARRAY_SIZE(para_features) - 1; i++) {
 130         if (kvm_check_extension(s, para_features[i].cap)) {
 131             features |= (1 << para_features[i].feature);
 132         }
 133     }
 134
 135     return features;
 136 }
 137
 138
 139 /* Returns the value for a specific register on the cpuid entry
 140  */
 141 static uint32_t cpuid_entry_get_reg(struct kvm_cpuid_entry2 *entry, int reg)
 142 {
 143     uint32_t ret = 0;
 144     switch (reg) {
 145     case R_EAX:
 146         ret = entry->eax;
 147         break;
 148     case R_EBX:
 149         ret = entry->ebx;
 150         break;
 151     case R_ECX:
 152         ret = entry->ecx;
 153         break;
 154     case R_EDX:
 155         ret = entry->edx;
 156         break;
 157     }
 158     return ret;
 159 }
 160
 161 /* Find matching entry for function/index on kvm_cpuid2 struct
 162  */
 163 static struct kvm_cpuid_entry2 *cpuid_find_entry(struct kvm_cpuid2 *cpuid,
 164                                                  uint32_t function,
 165                                                  uint32_t index)
 166 {
 167     int i;
 168     for (i = 0; i < cpuid->nent; ++i) {
 169         if (cpuid->entries[i].function == function &&
 170             cpuid->entries[i].index == index) {
 171             return &cpuid->entries[i];
 172         }
 173     }
 174     /* not found: */
 175     return NULL;
 176 }
 177
 178 uint32_t kvm_arch_get_supported_cpuid(KVMState *s, uint32_t function,
 179                                       uint32_t index, int reg)
 180 {
 181     struct kvm_cpuid2 *cpuid;
 182     uint32_t ret = 0;
 183     uint32_t cpuid_1_edx;
 184     bool found = false;
 185
 186     cpuid = get_supported_cpuid(s);
 187
 188     struct kvm_cpuid_entry2 *entry = cpuid_find_entry(cpuid, function, index);
 189     if (entry) {
 190         found = true;
 191         ret = cpuid_entry_get_reg(entry, reg);
 192     }
 193
 194     /* Fixups for the data returned by KVM, below */
 195
 196     if (function == 1 && reg == R_EDX) {
 197         /* KVM before 2.6.30 misreports the following features */
 198         ret |= CPUID_MTRR | CPUID_PAT | CPUID_MCE | CPUID_MCA;
 199     } else if (function == 1 && reg == R_ECX) {
 200         /* We can set the hypervisor flag, even if KVM does not return it on
 201          * GET_SUPPORTED_CPUID
 202          */
 203         ret |= CPUID_EXT_HYPERVISOR;
 204         /* tsc-deadline flag is not returned by GET_SUPPORTED_CPUID, but it
 205          * can be enabled if the kernel has KVM_CAP_TSC_DEADLINE_TIMER,
 206          * and the irqchip is in the kernel.
 207          */
 208         if (kvm_irqchip_in_kernel() &&
 209                 kvm_check_extension(s, KVM_CAP_TSC_DEADLINE_TIMER)) {
 210             ret |= CPUID_EXT_TSC_DEADLINE_TIMER;
 211         }
 212     } else if (function == 0x80000001 && reg == R_EDX) {
 213         /* On Intel, kvm returns cpuid according to the Intel spec,
 214          * so add missing bits according to the AMD spec:
 215          */
 216         cpuid_1_edx = kvm_arch_get_supported_cpuid(s, 1, 0, R_EDX);
 217         ret |= cpuid_1_edx & CPUID_EXT2_AMD_ALIASES;
 218     }
 219
 220     g_free(cpuid);
 221
 222     /* fallback for older kernels */
 223     if ((function == KVM_CPUID_FEATURES) && !found) {
 224         ret = get_para_features(s);
 225     }
 226
 227     return ret;
 228 }
 229
 230 typedef struct HWPoisonPage {
 231     ram_addr_t ram_addr;
 232     QLIST_ENTRY(HWPoisonPage) list;
 233 } HWPoisonPage;
 234
 235 static QLIST_HEAD(, HWPoisonPage) hwpoison_page_list =
 236     QLIST_HEAD_INITIALIZER(hwpoison_page_list);
 237
 238 static void kvm_unpoison_all(void *param)
 239 {
 240     HWPoisonPage *page, *next_page;
 241
 242     QLIST_FOREACH_SAFE(page, &hwpoison_page_list, list, next_page) {
 243         QLIST_REMOVE(page, list);
 244         qemu_ram_remap(page->ram_addr, TARGET_PAGE_SIZE);
 245         g_free(page);
 246     }
 247 }
 248
 249 static void kvm_hwpoison_page_add(ram_addr_t ram_addr)
 250 {
 251     HWPoisonPage *page;
 252
 253     QLIST_FOREACH(page, &hwpoison_page_list, list) {
 254         if (page->ram_addr == ram_addr) {
 255             return;
 256         }
 257     }
 258     page = g_malloc(sizeof(HWPoisonPage));
 259     page->ram_addr = ram_addr;
 260     QLIST_INSERT_HEAD(&hwpoison_page_list, page, list);
 261 }
 262
 263 static int kvm_get_mce_cap_supported(KVMState *s, uint64_t *mce_cap,
 264                                      int *max_banks)
 265 {
 266     int r;
 267
 268     r = kvm_check_extension(s, KVM_CAP_MCE);
 269     if (r > 0) {
 270         *max_banks = r;
 271         return kvm_ioctl(s, KVM_X86_GET_MCE_CAP_SUPPORTED, mce_cap);
 272     }
 273     return -ENOSYS;
 274 }
 275
 276 static void kvm_mce_inject(CPUX86State *env, hwaddr paddr, int code)
 277 {
 278     uint64_t status = MCI_STATUS_VAL | MCI_STATUS_UC | MCI_STATUS_EN |
 279                       MCI_STATUS_MISCV | MCI_STATUS_ADDRV | MCI_STATUS_S;
 280     uint64_t mcg_status = MCG_STATUS_MCIP;
 281
 282     if (code == BUS_MCEERR_AR) {
 283         status |= MCI_STATUS_AR | 0x134;
 284         mcg_status |= MCG_STATUS_EIPV;
 285     } else {
 286         status |= 0xc0;
 287         mcg_status |= MCG_STATUS_RIPV;
 288     }
 289     cpu_x86_inject_mce(NULL, env, 9, status, mcg_status, paddr,
 290                        (MCM_ADDR_PHYS << 6) | 0xc,
 291                        cpu_x86_support_mca_broadcast(env) ?
 292                        MCE_INJECT_BROADCAST : 0);
 293 }
 294
 295 static void hardware_memory_error(void)
 296 {
 297     fprintf(stderr, "Hardware memory error!\n");
 298     exit(1);
 299 }
 300
 301 int kvm_arch_on_sigbus_vcpu(CPUX86State *env, int code, void *addr)
 302 {
 303     ram_addr_t ram_addr;
 304     hwaddr paddr;
 305
 306     if ((env->mcg_cap & MCG_SER_P) && addr
 307         && (code == BUS_MCEERR_AR || code == BUS_MCEERR_AO)) {
 308         if (qemu_ram_addr_from_host(addr, &ram_addr) ||
 309             !kvm_physical_memory_addr_from_host(env->kvm_state, addr, &paddr)) {
 310             fprintf(stderr, "Hardware memory error for memory used by "
 311                     "QEMU itself instead of guest system!\n");
 312             /* Hope we are lucky for AO MCE */
 313             if (code == BUS_MCEERR_AO) {
 314                 return 0;
 315             } else {
 316                 hardware_memory_error();
 317             }
 318         }
 319         kvm_hwpoison_page_add(ram_addr);
 320         kvm_mce_inject(env, paddr, code);
 321     } else {
 322         if (code == BUS_MCEERR_AO) {
 323             return 0;
 324         } else if (code == BUS_MCEERR_AR) {
 325             hardware_memory_error();
 326         } else {
 327             return 1;
 328         }
 329     }
 330     return 0;
 331 }
 332
 333 int kvm_arch_on_sigbus(int code, void *addr)
 334 {
 335     if ((first_cpu->mcg_cap & MCG_SER_P) && addr && code == BUS_MCEERR_AO) {
 336         ram_addr_t ram_addr;
 337         hwaddr paddr;
 338
 339         /* Hope we are lucky for AO MCE */
 340         if (qemu_ram_addr_from_host(addr, &ram_addr) ||
 341             !kvm_physical_memory_addr_from_host(first_cpu->kvm_state, addr,
 342                                                 &paddr)) {
 343             fprintf(stderr, "Hardware memory error for memory used by "
 344                     "QEMU itself instead of guest system!: %p\n", addr);
 345             return 0;
 346         }
 347         kvm_hwpoison_page_add(ram_addr);
 348         kvm_mce_inject(first_cpu, paddr, code);
 349     } else {
 350         if (code == BUS_MCEERR_AO) {
 351             return 0;
 352         } else if (code == BUS_MCEERR_AR) {
 353             hardware_memory_error();
 354         } else {
 355             return 1;
 356         }
 357     }
 358     return 0;
 359 }
 360
 361 static int kvm_inject_mce_oldstyle(CPUX86State *env)
 362 {
 363     if (!kvm_has_vcpu_events() && env->exception_injected == EXCP12_MCHK) {
 364         unsigned int bank, bank_num = env->mcg_cap & 0xff;
 365         struct kvm_x86_mce mce;
 366
 367         env->exception_injected = -1;
 368
 369         /*
 370          * There must be at least one bank in use if an MCE is pending.
 371          * Find it and use its values for the event injection.
 372          */
 373         for (bank = 0; bank < bank_num; bank++) {
 374             if (env->mce_banks[bank * 4 + 1] & MCI_STATUS_VAL) {
 375                 break;
 376             }
 377         }
 378         assert(bank < bank_num);
 379
 380         mce.bank = bank;
 381         mce.status = env->mce_banks[bank * 4 + 1];
 382         mce.mcg_status = env->mcg_status;
 383         mce.addr = env->mce_banks[bank * 4 + 2];
 384         mce.misc = env->mce_banks[bank * 4 + 3];
 385
 386         return kvm_vcpu_ioctl(env, KVM_X86_SET_MCE, &mce);
 387     }
 388     return 0;
 389 }
 390
 391 static void cpu_update_state(void *opaque, int running, RunState state)
 392 {
 393     CPUX86State *env = opaque;
 394
 395     if (running) {
 396         env->tsc_valid = false;
 397     }
 398 }
 399
 400 int kvm_arch_init_vcpu(CPUX86State *env)
 401 {
 402     struct {
 403         struct kvm_cpuid2 cpuid;
 404         struct kvm_cpuid_entry2 entries[100];
 405     } QEMU_PACKED cpuid_data;
 406     KVMState *s = env->kvm_state;
 407     uint32_t limit, i, j, cpuid_i;
 408     uint32_t unused;
 409     struct kvm_cpuid_entry2 *c;
 410     uint32_t signature[3];
 411     int r;
 412
 413     env->cpuid_features &= kvm_arch_get_supported_cpuid(s, 1, 0, R_EDX);
 414
 415     env->cpuid_ext_features &= kvm_arch_get_supported_cpuid(s, 1, 0, R_ECX);
 416
 417     env->cpuid_ext2_features &= kvm_arch_get_supported_cpuid(s, 0x80000001,
 418                                                              0, R_EDX);
 419     env->cpuid_ext3_features &= kvm_arch_get_supported_cpuid(s, 0x80000001,
 420                                                              0, R_ECX);
 421     env->cpuid_svm_features  &= kvm_arch_get_supported_cpuid(s, 0x8000000A,
 422                                                              0, R_EDX);
 423
 424     cpuid_i = 0;
 425
 426     /* Paravirtualization CPUIDs */
 427     c = &cpuid_data.entries[cpuid_i++];
 428     memset(c, 0, sizeof(*c));
 429     c->function = KVM_CPUID_SIGNATURE;
 430     if (!hyperv_enabled()) {
 431         memcpy(signature, "KVMKVMKVM\0\0\0", 12);
 432         c->eax = 0;
 433     } else {
 434         memcpy(signature, "Microsoft Hv", 12);
 435         c->eax = HYPERV_CPUID_MIN;
 436     }
 437     c->ebx = signature[0];
 438     c->ecx = signature[1];
 439     c->edx = signature[2];
 440
 441     c = &cpuid_data.entries[cpuid_i++];
 442     memset(c, 0, sizeof(*c));
 443     c->function = KVM_CPUID_FEATURES;
 444     c->eax = env->cpuid_kvm_features &
 445         kvm_arch_get_supported_cpuid(s, KVM_CPUID_FEATURES, 0, R_EAX);
 446
 447     if (hyperv_enabled()) {
 448         memcpy(signature, "Hv#1\0\0\0\0\0\0\0\0", 12);
 449         c->eax = signature[0];
 450
 451         c = &cpuid_data.entries[cpuid_i++];
 452         memset(c, 0, sizeof(*c));
 453         c->function = HYPERV_CPUID_VERSION;
 454         c->eax = 0x00001bbc;
 455         c->ebx = 0x00060001;
 456
 457         c = &cpuid_data.entries[cpuid_i++];
 458         memset(c, 0, sizeof(*c));
 459         c->function = HYPERV_CPUID_FEATURES;
 460         if (hyperv_relaxed_timing_enabled()) {
 461             c->eax |= HV_X64_MSR_HYPERCALL_AVAILABLE;
 462         }
 463         if (hyperv_vapic_recommended()) {
 464             c->eax |= HV_X64_MSR_HYPERCALL_AVAILABLE;
 465             c->eax |= HV_X64_MSR_APIC_ACCESS_AVAILABLE;
 466         }
 467
 468         c = &cpuid_data.entries[cpuid_i++];
 469         memset(c, 0, sizeof(*c));
 470         c->function = HYPERV_CPUID_ENLIGHTMENT_INFO;
 471         if (hyperv_relaxed_timing_enabled()) {
 472             c->eax |= HV_X64_RELAXED_TIMING_RECOMMENDED;
 473         }
 474         if (hyperv_vapic_recommended()) {
 475             c->eax |= HV_X64_APIC_ACCESS_RECOMMENDED;
 476         }
 477         c->ebx = hyperv_get_spinlock_retries();
 478
 479         c = &cpuid_data.entries[cpuid_i++];
 480         memset(c, 0, sizeof(*c));
 481         c->function = HYPERV_CPUID_IMPLEMENT_LIMITS;
 482         c->eax = 0x40;
 483         c->ebx = 0x40;
 484
 485         c = &cpuid_data.entries[cpuid_i++];
 486         memset(c, 0, sizeof(*c));
 487         c->function = KVM_CPUID_SIGNATURE_NEXT;
 488         memcpy(signature, "KVMKVMKVM\0\0\0", 12);
 489         c->eax = 0;
 490         c->ebx = signature[0];
 491         c->ecx = signature[1];
 492         c->edx = signature[2];
 493     }
 494
 495     has_msr_async_pf_en = c->eax & (1 << KVM_FEATURE_ASYNC_PF);
 496
 497     has_msr_pv_eoi_en = c->eax & (1 << KVM_FEATURE_PV_EOI);
 498
 499     cpu_x86_cpuid(env, 0, 0, &limit, &unused, &unused, &unused);
 500
 501     for (i = 0; i <= limit; i++) {
 502         c = &cpuid_data.entries[cpuid_i++];
 503
 504         switch (i) {
 505         case 2: {
 506             /* Keep reading function 2 till all the input is received */
 507             int times;
 508
 509             c->function = i;
 510             c->flags = KVM_CPUID_FLAG_STATEFUL_FUNC |
 511                        KVM_CPUID_FLAG_STATE_READ_NEXT;
 512             cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx);
 513             times = c->eax & 0xff;
 514
 515             for (j = 1; j < times; ++j) {
 516                 c = &cpuid_data.entries[cpuid_i++];
 517                 c->function = i;
 518                 c->flags = KVM_CPUID_FLAG_STATEFUL_FUNC;
 519                 cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx);
 520             }
 521             break;
 522         }
 523         case 4:
 524         case 0xb:
 525         case 0xd:
 526             for (j = 0; ; j++) {
 527                 if (i == 0xd && j == 64) {
 528                     break;
 529                 }
 530                 c->function = i;
 531                 c->flags = KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
 532                 c->index = j;
 533                 cpu_x86_cpuid(env, i, j, &c->eax, &c->ebx, &c->ecx, &c->edx);
 534
 535                 if (i == 4 && c->eax == 0) {
 536                     break;
 537                 }
 538                 if (i == 0xb && !(c->ecx & 0xff00)) {
 539                     break;
 540                 }
 541                 if (i == 0xd && c->eax == 0) {
 542                     continue;
 543                 }
 544                 c = &cpuid_data.entries[cpuid_i++];
 545             }
 546             break;
 547         default:
 548             c->function = i;
 549             c->flags = 0;
 550             cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx);
 551             break;
 552         }
 553     }
 554     cpu_x86_cpuid(env, 0x80000000, 0, &limit, &unused, &unused, &unused);
 555
 556     for (i = 0x80000000; i <= limit; i++) {
 557         c = &cpuid_data.entries[cpuid_i++];
 558
 559         c->function = i;
 560         c->flags = 0;
 561         cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx);
 562     }
 563
 564     /* Call Centaur's CPUID instructions they are supported. */
 565     if (env->cpuid_xlevel2 > 0) {
 566         env->cpuid_ext4_features &=
 567             kvm_arch_get_supported_cpuid(s, 0xC0000001, 0, R_EDX);
 568         cpu_x86_cpuid(env, 0xC0000000, 0, &limit, &unused, &unused, &unused);
 569
 570         for (i = 0xC0000000; i <= limit; i++) {
 571             c = &cpuid_data.entries[cpuid_i++];
 572
 573             c->function = i;
 574             c->flags = 0;
 575             cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx);
 576         }
 577     }
 578
 579     cpuid_data.cpuid.nent = cpuid_i;
 580
 581     if (((env->cpuid_version >> 8)&0xF) >= 6
 582         && (env->cpuid_features&(CPUID_MCE|CPUID_MCA)) == (CPUID_MCE|CPUID_MCA)
 583         && kvm_check_extension(env->kvm_state, KVM_CAP_MCE) > 0) {
 584         uint64_t mcg_cap;
 585         int banks;
 586         int ret;
 587
 588         ret = kvm_get_mce_cap_supported(env->kvm_state, &mcg_cap, &banks);
 589         if (ret < 0) {
 590             fprintf(stderr, "kvm_get_mce_cap_supported: %s", strerror(-ret));
 591             return ret;
 592         }
 593
 594         if (banks > MCE_BANKS_DEF) {
 595             banks = MCE_BANKS_DEF;
 596         }
 597         mcg_cap &= MCE_CAP_DEF;
 598         mcg_cap |= banks;
 599         ret = kvm_vcpu_ioctl(env, KVM_X86_SETUP_MCE, &mcg_cap);
 600         if (ret < 0) {
 601             fprintf(stderr, "KVM_X86_SETUP_MCE: %s", strerror(-ret));
 602             return ret;
 603         }
 604
 605         env->mcg_cap = mcg_cap;
 606     }
 607
 608     qemu_add_vm_change_state_handler(cpu_update_state, env);
 609
 610     cpuid_data.cpuid.padding = 0;
 611     r = kvm_vcpu_ioctl(env, KVM_SET_CPUID2, &cpuid_data);
 612     if (r) {
 613         return r;
 614     }
 615
 616     r = kvm_check_extension(env->kvm_state, KVM_CAP_TSC_CONTROL);
 617     if (r && env->tsc_khz) {
 618         r = kvm_vcpu_ioctl(env, KVM_SET_TSC_KHZ, env->tsc_khz);
 619         if (r < 0) {
 620             fprintf(stderr, "KVM_SET_TSC_KHZ failed\n");
 621             return r;
 622         }
 623     }
 624
 625     if (kvm_has_xsave()) {
 626         env->kvm_xsave_buf = qemu_memalign(4096, sizeof(struct kvm_xsave));
 627     }
 628
 629     return 0;
 630 }
 631
 632 void kvm_arch_reset_vcpu(CPUX86State *env)
 633 {
 634     X86CPU *cpu = x86_env_get_cpu(env);
 635
 636     env->exception_injected = -1;
 637     env->interrupt_injected = -1;
 638     env->xcr0 = 1;
 639     if (kvm_irqchip_in_kernel()) {
 640         env->mp_state = cpu_is_bsp(cpu) ? KVM_MP_STATE_RUNNABLE :
 641                                           KVM_MP_STATE_UNINITIALIZED;
 642     } else {
 643         env->mp_state = KVM_MP_STATE_RUNNABLE;
 644     }
 645 }
 646
 647 static int kvm_get_supported_msrs(KVMState *s)
 648 {
 649     static int kvm_supported_msrs;
 650     int ret = 0;
 651
 652     /* first time */
 653     if (kvm_supported_msrs == 0) {
 654         struct kvm_msr_list msr_list, *kvm_msr_list;
 655
 656         kvm_supported_msrs = -1;
 657
 658         /* Obtain MSR list from KVM.  These are the MSRs that we must
 659          * save/restore */
 660         msr_list.nmsrs = 0;
 661         ret = kvm_ioctl(s, KVM_GET_MSR_INDEX_LIST, &msr_list);
 662         if (ret < 0 && ret != -E2BIG) {
 663             return ret;
 664         }
 665         /* Old kernel modules had a bug and could write beyond the provided
 666            memory. Allocate at least a safe amount of 1K. */
 667         kvm_msr_list = g_malloc0(MAX(1024, sizeof(msr_list) +
 668                                               msr_list.nmsrs *
 669                                               sizeof(msr_list.indices[0])));
 670
 671         kvm_msr_list->nmsrs = msr_list.nmsrs;
 672         ret = kvm_ioctl(s, KVM_GET_MSR_INDEX_LIST, kvm_msr_list);
 673         if (ret >= 0) {
 674             int i;
 675
 676             for (i = 0; i < kvm_msr_list->nmsrs; i++) {
 677                 if (kvm_msr_list->indices[i] == MSR_STAR) {
 678                     has_msr_star = true;
 679                     continue;
 680                 }
 681                 if (kvm_msr_list->indices[i] == MSR_VM_HSAVE_PA) {
 682                     has_msr_hsave_pa = true;
 683                     continue;
 684                 }
 685                 if (kvm_msr_list->indices[i] == MSR_IA32_TSCDEADLINE) {
 686                     has_msr_tsc_deadline = true;
 687                     continue;
 688                 }
 689                 if (kvm_msr_list->indices[i] == MSR_IA32_MISC_ENABLE) {
 690                     has_msr_misc_enable = true;
 691                     continue;
 692                 }
 693             }
 694         }
 695
 696         g_free(kvm_msr_list);
 697     }
 698
 699     return ret;
 700 }
 701
 702 int kvm_arch_init(KVMState *s)
 703 {
 704     QemuOptsList *list = qemu_find_opts("machine");
 705     uint64_t identity_base = 0xfffbc000;
 706     uint64_t shadow_mem;
 707     int ret;
 708     struct utsname utsname;
 709
 710     ret = kvm_get_supported_msrs(s);
 711     if (ret < 0) {
 712         return ret;
 713     }
 714
 715     uname(&utsname);
 716     lm_capable_kernel = strcmp(utsname.machine, "x86_64") == 0;
 717
 718     /*
 719      * On older Intel CPUs, KVM uses vm86 mode to emulate 16-bit code directly.
 720      * In order to use vm86 mode, an EPT identity map and a TSS  are needed.
 721      * Since these must be part of guest physical memory, we need to allocate
 722      * them, both by setting their start addresses in the kernel and by
 723      * creating a corresponding e820 entry. We need 4 pages before the BIOS.
 724      *
 725      * Older KVM versions may not support setting the identity map base. In
 726      * that case we need to stick with the default, i.e. a 256K maximum BIOS
 727      * size.
 728      */
 729     if (kvm_check_extension(s, KVM_CAP_SET_IDENTITY_MAP_ADDR)) {
 730         /* Allows up to 16M BIOSes. */
 731         identity_base = 0xfeffc000;
 732
 733         ret = kvm_vm_ioctl(s, KVM_SET_IDENTITY_MAP_ADDR, &identity_base);
 734         if (ret < 0) {
 735             return ret;
 736         }
 737     }
 738
 739     /* Set TSS base one page after EPT identity map. */
 740     ret = kvm_vm_ioctl(s, KVM_SET_TSS_ADDR, identity_base + 0x1000);
 741     if (ret < 0) {
 742         return ret;
 743     }
 744
 745     /* Tell fw_cfg to notify the BIOS to reserve the range. */
 746     ret = e820_add_entry(identity_base, 0x4000, E820_RESERVED);
 747     if (ret < 0) {
 748         fprintf(stderr, "e820_add_entry() table is full\n");
 749         return ret;
 750     }
 751     qemu_register_reset(kvm_unpoison_all, NULL);
 752
 753     if (!QTAILQ_EMPTY(&list->head)) {
 754         shadow_mem = qemu_opt_get_size(QTAILQ_FIRST(&list->head),
 755                                        "kvm_shadow_mem", -1);
 756         if (shadow_mem != -1) {
 757             shadow_mem /= 4096;
 758             ret = kvm_vm_ioctl(s, KVM_SET_NR_MMU_PAGES, shadow_mem);
 759             if (ret < 0) {
 760                 return ret;
 761             }
 762         }
 763     }
 764     return 0;
 765 }
 766
 767 static void set_v8086_seg(struct kvm_segment *lhs, const SegmentCache *rhs)
 768 {
 769     lhs->selector = rhs->selector;
 770     lhs->base = rhs->base;
 771     lhs->limit = rhs->limit;
 772     lhs->type = 3;
 773     lhs->present = 1;
 774     lhs->dpl = 3;
 775     lhs->db = 0;
 776     lhs->s = 1;
 777     lhs->l = 0;
 778     lhs->g = 0;
 779     lhs->avl = 0;
 780     lhs->unusable = 0;
 781 }
 782
 783 static void set_seg(struct kvm_segment *lhs, const SegmentCache *rhs)
 784 {
 785     unsigned flags = rhs->flags;
 786     lhs->selector = rhs->selector;
 787     lhs->base = rhs->base;
 788     lhs->limit = rhs->limit;
 789     lhs->type = (flags >> DESC_TYPE_SHIFT) & 15;
 790     lhs->present = (flags & DESC_P_MASK) != 0;
 791     lhs->dpl = (flags >> DESC_DPL_SHIFT) & 3;
 792     lhs->db = (flags >> DESC_B_SHIFT) & 1;
 793     lhs->s = (flags & DESC_S_MASK) != 0;
 794     lhs->l = (flags >> DESC_L_SHIFT) & 1;
 795     lhs->g = (flags & DESC_G_MASK) != 0;
 796     lhs->avl = (flags & DESC_AVL_MASK) != 0;
 797     lhs->unusable = 0;
 798     lhs->padding = 0;
 799 }
 800
 801 static void get_seg(SegmentCache *lhs, const struct kvm_segment *rhs)
 802 {
 803     lhs->selector = rhs->selector;
 804     lhs->base = rhs->base;
 805     lhs->limit = rhs->limit;
 806     lhs->flags = (rhs->type << DESC_TYPE_SHIFT) |
 807                  (rhs->present * DESC_P_MASK) |
 808                  (rhs->dpl << DESC_DPL_SHIFT) |
 809                  (rhs->db << DESC_B_SHIFT) |
 810                  (rhs->s * DESC_S_MASK) |
 811                  (rhs->l << DESC_L_SHIFT) |
 812                  (rhs->g * DESC_G_MASK) |
 813                  (rhs->avl * DESC_AVL_MASK);
 814 }
 815
 816 static void kvm_getput_reg(__u64 *kvm_reg, target_ulong *qemu_reg, int set)
 817 {
 818     if (set) {
 819         *kvm_reg = *qemu_reg;
 820     } else {
 821         *qemu_reg = *kvm_reg;
 822     }
 823 }
 824
 825 static int kvm_getput_regs(CPUX86State *env, int set)
 826 {
 827     struct kvm_regs regs;
 828     int ret = 0;
 829
 830     if (!set) {
 831         ret = kvm_vcpu_ioctl(env, KVM_GET_REGS, &regs);
 832         if (ret < 0) {
 833             return ret;
 834         }
 835     }
 836
 837     kvm_getput_reg(&regs.rax, &env->regs[R_EAX], set);
 838     kvm_getput_reg(&regs.rbx, &env->regs[R_EBX], set);
 839     kvm_getput_reg(&regs.rcx, &env->regs[R_ECX], set);
 840     kvm_getput_reg(&regs.rdx, &env->regs[R_EDX], set);
 841     kvm_getput_reg(&regs.rsi, &env->regs[R_ESI], set);
 842     kvm_getput_reg(&regs.rdi, &env->regs[R_EDI], set);
 843     kvm_getput_reg(&regs.rsp, &env->regs[R_ESP], set);
 844     kvm_getput_reg(&regs.rbp, &env->regs[R_EBP], set);
 845 #ifdef TARGET_X86_64
 846     kvm_getput_reg(&regs.r8, &env->regs[8], set);
 847     kvm_getput_reg(&regs.r9, &env->regs[9], set);
 848     kvm_getput_reg(&regs.r10, &env->regs[10], set);
 849     kvm_getput_reg(&regs.r11, &env->regs[11], set);
 850     kvm_getput_reg(&regs.r12, &env->regs[12], set);
 851     kvm_getput_reg(&regs.r13, &env->regs[13], set);
 852     kvm_getput_reg(&regs.r14, &env->regs[14], set);
 853     kvm_getput_reg(&regs.r15, &env->regs[15], set);
 854 #endif
 855
 856     kvm_getput_reg(&regs.rflags, &env->eflags, set);
 857     kvm_getput_reg(&regs.rip, &env->eip, set);
 858
 859     if (set) {
 860         ret = kvm_vcpu_ioctl(env, KVM_SET_REGS, &regs);
 861     }
 862
 863     return ret;
 864 }
 865
 866 static int kvm_put_fpu(CPUX86State *env)
 867 {
 868     struct kvm_fpu fpu;
 869     int i;
 870
 871     memset(&fpu, 0, sizeof fpu);
 872     fpu.fsw = env->fpus & ~(7 << 11);
 873     fpu.fsw |= (env->fpstt & 7) << 11;
 874     fpu.fcw = env->fpuc;
 875     fpu.last_opcode = env->fpop;
 876     fpu.last_ip = env->fpip;
 877     fpu.last_dp = env->fpdp;
 878     for (i = 0; i < 8; ++i) {
 879         fpu.ftwx |= (!env->fptags[i]) << i;
 880     }
 881     memcpy(fpu.fpr, env->fpregs, sizeof env->fpregs);
 882     memcpy(fpu.xmm, env->xmm_regs, sizeof env->xmm_regs);
 883     fpu.mxcsr = env->mxcsr;
 884
 885     return kvm_vcpu_ioctl(env, KVM_SET_FPU, &fpu);
 886 }
 887
 888 #define XSAVE_FCW_FSW     0
 889 #define XSAVE_FTW_FOP     1
 890 #define XSAVE_CWD_RIP     2
 891 #define XSAVE_CWD_RDP     4
 892 #define XSAVE_MXCSR       6
 893 #define XSAVE_ST_SPACE    8
 894 #define XSAVE_XMM_SPACE   40
 895 #define XSAVE_XSTATE_BV   128
 896 #define XSAVE_YMMH_SPACE  144
 897
 898 static int kvm_put_xsave(CPUX86State *env)
 899 {
 900     struct kvm_xsave* xsave = env->kvm_xsave_buf;
 901     uint16_t cwd, swd, twd;
 902     int i, r;
 903
 904     if (!kvm_has_xsave()) {
 905         return kvm_put_fpu(env);
 906     }
 907
 908     memset(xsave, 0, sizeof(struct kvm_xsave));
 909     twd = 0;
 910     swd = env->fpus & ~(7 << 11);
 911     swd |= (env->fpstt & 7) << 11;
 912     cwd = env->fpuc;
 913     for (i = 0; i < 8; ++i) {
 914         twd |= (!env->fptags[i]) << i;
 915     }
 916     xsave->region[XSAVE_FCW_FSW] = (uint32_t)(swd << 16) + cwd;
 917     xsave->region[XSAVE_FTW_FOP] = (uint32_t)(env->fpop << 16) + twd;
 918     memcpy(&xsave->region[XSAVE_CWD_RIP], &env->fpip, sizeof(env->fpip));
 919     memcpy(&xsave->region[XSAVE_CWD_RDP], &env->fpdp, sizeof(env->fpdp));
 920     memcpy(&xsave->region[XSAVE_ST_SPACE], env->fpregs,
 921             sizeof env->fpregs);
 922     memcpy(&xsave->region[XSAVE_XMM_SPACE], env->xmm_regs,
 923             sizeof env->xmm_regs);
 924     xsave->region[XSAVE_MXCSR] = env->mxcsr;
 925     *(uint64_t *)&xsave->region[XSAVE_XSTATE_BV] = env->xstate_bv;
 926     memcpy(&xsave->region[XSAVE_YMMH_SPACE], env->ymmh_regs,
 927             sizeof env->ymmh_regs);
 928     r = kvm_vcpu_ioctl(env, KVM_SET_XSAVE, xsave);
 929     return r;
 930 }
 931
 932 static int kvm_put_xcrs(CPUX86State *env)
 933 {
 934     struct kvm_xcrs xcrs;
 935
 936     if (!kvm_has_xcrs()) {
 937         return 0;
 938     }
 939
 940     xcrs.nr_xcrs = 1;
 941     xcrs.flags = 0;
 942     xcrs.xcrs[0].xcr = 0;
 943     xcrs.xcrs[0].value = env->xcr0;
 944     return kvm_vcpu_ioctl(env, KVM_SET_XCRS, &xcrs);
 945 }
 946
 947 static int kvm_put_sregs(CPUX86State *env)
 948 {
 949     struct kvm_sregs sregs;
 950
 951     memset(sregs.interrupt_bitmap, 0, sizeof(sregs.interrupt_bitmap));
 952     if (env->interrupt_injected >= 0) {
 953         sregs.interrupt_bitmap[env->interrupt_injected / 64] |=
 954                 (uint64_t)1 << (env->interrupt_injected % 64);
 955     }
 956
 957     if ((env->eflags & VM_MASK)) {
 958         set_v8086_seg(&sregs.cs, &env->segs[R_CS]);
 959         set_v8086_seg(&sregs.ds, &env->segs[R_DS]);
 960         set_v8086_seg(&sregs.es, &env->segs[R_ES]);
 961         set_v8086_seg(&sregs.fs, &env->segs[R_FS]);
 962         set_v8086_seg(&sregs.gs, &env->segs[R_GS]);
 963         set_v8086_seg(&sregs.ss, &env->segs[R_SS]);
 964     } else {
 965         set_seg(&sregs.cs, &env->segs[R_CS]);
 966         set_seg(&sregs.ds, &env->segs[R_DS]);
 967         set_seg(&sregs.es, &env->segs[R_ES]);
 968         set_seg(&sregs.fs, &env->segs[R_FS]);
 969         set_seg(&sregs.gs, &env->segs[R_GS]);
 970         set_seg(&sregs.ss, &env->segs[R_SS]);
 971     }
 972
 973     set_seg(&sregs.tr, &env->tr);
 974     set_seg(&sregs.ldt, &env->ldt);
 975
 976     sregs.idt.limit = env->idt.limit;
 977     sregs.idt.base = env->idt.base;
 978     memset(sregs.idt.padding, 0, sizeof sregs.idt.padding);
 979     sregs.gdt.limit = env->gdt.limit;
 980     sregs.gdt.base = env->gdt.base;
 981     memset(sregs.gdt.padding, 0, sizeof sregs.gdt.padding);
 982
 983     sregs.cr0 = env->cr[0];
 984     sregs.cr2 = env->cr[2];
 985     sregs.cr3 = env->cr[3];
 986     sregs.cr4 = env->cr[4];
 987
 988     sregs.cr8 = cpu_get_apic_tpr(env->apic_state);
 989     sregs.apic_base = cpu_get_apic_base(env->apic_state);
 990
 991     sregs.efer = env->efer;
 992
 993     return kvm_vcpu_ioctl(env, KVM_SET_SREGS, &sregs);
 994 }
 995
 996 static void kvm_msr_entry_set(struct kvm_msr_entry *entry,
 997                               uint32_t index, uint64_t value)
 998 {
 999     entry->index = index;
1000     entry->data = value;
1001 }
1002
1003 static int kvm_put_msrs(CPUX86State *env, int level)
1004 {
1005     struct {
1006         struct kvm_msrs info;
1007         struct kvm_msr_entry entries[100];
1008     } msr_data;
1009     struct kvm_msr_entry *msrs = msr_data.entries;
1010     int n = 0;
1011
1012     kvm_msr_entry_set(&msrs[n++], MSR_IA32_SYSENTER_CS, env->sysenter_cs);
1013     kvm_msr_entry_set(&msrs[n++], MSR_IA32_SYSENTER_ESP, env->sysenter_esp);
1014     kvm_msr_entry_set(&msrs[n++], MSR_IA32_SYSENTER_EIP, env->sysenter_eip);
1015     kvm_msr_entry_set(&msrs[n++], MSR_PAT, env->pat);
1016     if (has_msr_star) {
1017         kvm_msr_entry_set(&msrs[n++], MSR_STAR, env->star);
1018     }
1019     if (has_msr_hsave_pa) {
1020         kvm_msr_entry_set(&msrs[n++], MSR_VM_HSAVE_PA, env->vm_hsave);
1021     }
1022     if (has_msr_tsc_deadline) {
1023         kvm_msr_entry_set(&msrs[n++], MSR_IA32_TSCDEADLINE, env->tsc_deadline);
1024     }
1025     if (has_msr_misc_enable) {
1026         kvm_msr_entry_set(&msrs[n++], MSR_IA32_MISC_ENABLE,
1027                           env->msr_ia32_misc_enable);
1028     }
1029 #ifdef TARGET_X86_64
1030     if (lm_capable_kernel) {
1031         kvm_msr_entry_set(&msrs[n++], MSR_CSTAR, env->cstar);
1032         kvm_msr_entry_set(&msrs[n++], MSR_KERNELGSBASE, env->kernelgsbase);
1033         kvm_msr_entry_set(&msrs[n++], MSR_FMASK, env->fmask);
1034         kvm_msr_entry_set(&msrs[n++], MSR_LSTAR, env->lstar);
1035     }
1036 #endif
1037     if (level == KVM_PUT_FULL_STATE) {
1038         /*
1039          * KVM is yet unable to synchronize TSC values of multiple VCPUs on
1040          * writeback. Until this is fixed, we only write the offset to SMP
1041          * guests after migration, desynchronizing the VCPUs, but avoiding
1042          * huge jump-backs that would occur without any writeback at all.
1043          */
1044         if (smp_cpus == 1 || env->tsc != 0) {
1045             kvm_msr_entry_set(&msrs[n++], MSR_IA32_TSC, env->tsc);
1046         }
1047     }
1048     /*
1049      * The following paravirtual MSRs have side effects on the guest or are
1050      * too heavy for normal writeback. Limit them to reset or full state
1051      * updates.
1052      */
1053     if (level >= KVM_PUT_RESET_STATE) {
1054         kvm_msr_entry_set(&msrs[n++], MSR_KVM_SYSTEM_TIME,
1055                           env->system_time_msr);
1056         kvm_msr_entry_set(&msrs[n++], MSR_KVM_WALL_CLOCK, env->wall_clock_msr);
1057         if (has_msr_async_pf_en) {
1058             kvm_msr_entry_set(&msrs[n++], MSR_KVM_ASYNC_PF_EN,
1059                               env->async_pf_en_msr);
1060         }
1061         if (has_msr_pv_eoi_en) {
1062             kvm_msr_entry_set(&msrs[n++], MSR_KVM_PV_EOI_EN,
1063                               env->pv_eoi_en_msr);
1064         }
1065         if (hyperv_hypercall_available()) {
1066             kvm_msr_entry_set(&msrs[n++], HV_X64_MSR_GUEST_OS_ID, 0);
1067             kvm_msr_entry_set(&msrs[n++], HV_X64_MSR_HYPERCALL, 0);
1068         }
1069         if (hyperv_vapic_recommended()) {
1070             kvm_msr_entry_set(&msrs[n++], HV_X64_MSR_APIC_ASSIST_PAGE, 0);
1071         }
1072     }
1073     if (env->mcg_cap) {
1074         int i;
1075
1076         kvm_msr_entry_set(&msrs[n++], MSR_MCG_STATUS, env->mcg_status);
1077         kvm_msr_entry_set(&msrs[n++], MSR_MCG_CTL, env->mcg_ctl);
1078         for (i = 0; i < (env->mcg_cap & 0xff) * 4; i++) {
1079             kvm_msr_entry_set(&msrs[n++], MSR_MC0_CTL + i, env->mce_banks[i]);
1080         }
1081     }
1082
1083     msr_data.info.nmsrs = n;
1084
1085     return kvm_vcpu_ioctl(env, KVM_SET_MSRS, &msr_data);
1086
1087 }
1088
1089
1090 static int kvm_get_fpu(CPUX86State *env)
1091 {
1092     struct kvm_fpu fpu;
1093     int i, ret;
1094
1095     ret = kvm_vcpu_ioctl(env, KVM_GET_FPU, &fpu);
1096     if (ret < 0) {
1097         return ret;
1098     }
1099
1100     env->fpstt = (fpu.fsw >> 11) & 7;
1101     env->fpus = fpu.fsw;
1102     env->fpuc = fpu.fcw;
1103     env->fpop = fpu.last_opcode;
1104     env->fpip = fpu.last_ip;
1105     env->fpdp = fpu.last_dp;
1106     for (i = 0; i < 8; ++i) {
1107         env->fptags[i] = !((fpu.ftwx >> i) & 1);
1108     }
1109     memcpy(env->fpregs, fpu.fpr, sizeof env->fpregs);
1110     memcpy(env->xmm_regs, fpu.xmm, sizeof env->xmm_regs);
1111     env->mxcsr = fpu.mxcsr;
1112
1113     return 0;
1114 }
1115
1116 static int kvm_get_xsave(CPUX86State *env)
1117 {
1118     struct kvm_xsave* xsave = env->kvm_xsave_buf;
1119     int ret, i;
1120     uint16_t cwd, swd, twd;
1121
1122     if (!kvm_has_xsave()) {
1123         return kvm_get_fpu(env);
1124     }
1125
1126     ret = kvm_vcpu_ioctl(env, KVM_GET_XSAVE, xsave);
1127     if (ret < 0) {
1128         return ret;
1129     }
1130
1131     cwd = (uint16_t)xsave->region[XSAVE_FCW_FSW];
1132     swd = (uint16_t)(xsave->region[XSAVE_FCW_FSW] >> 16);
1133     twd = (uint16_t)xsave->region[XSAVE_FTW_FOP];
1134     env->fpop = (uint16_t)(xsave->region[XSAVE_FTW_FOP] >> 16);
1135     env->fpstt = (swd >> 11) & 7;
1136     env->fpus = swd;
1137     env->fpuc = cwd;
1138     for (i = 0; i < 8; ++i) {
1139         env->fptags[i] = !((twd >> i) & 1);
1140     }
1141     memcpy(&env->fpip, &xsave->region[XSAVE_CWD_RIP], sizeof(env->fpip));
1142     memcpy(&env->fpdp, &xsave->region[XSAVE_CWD_RDP], sizeof(env->fpdp));
1143     env->mxcsr = xsave->region[XSAVE_MXCSR];
1144     memcpy(env->fpregs, &xsave->region[XSAVE_ST_SPACE],
1145             sizeof env->fpregs);
1146     memcpy(env->xmm_regs, &xsave->region[XSAVE_XMM_SPACE],
1147             sizeof env->xmm_regs);
1148     env->xstate_bv = *(uint64_t *)&xsave->region[XSAVE_XSTATE_BV];
1149     memcpy(env->ymmh_regs, &xsave->region[XSAVE_YMMH_SPACE],
1150             sizeof env->ymmh_regs);
1151     return 0;
1152 }
1153
1154 static int kvm_get_xcrs(CPUX86State *env)
1155 {
1156     int i, ret;
1157     struct kvm_xcrs xcrs;
1158
1159     if (!kvm_has_xcrs()) {
1160         return 0;
1161     }
1162
1163     ret = kvm_vcpu_ioctl(env, KVM_GET_XCRS, &xcrs);
1164     if (ret < 0) {
1165         return ret;
1166     }
1167
1168     for (i = 0; i < xcrs.nr_xcrs; i++) {
1169         /* Only support xcr0 now */
1170         if (xcrs.xcrs[0].xcr == 0) {
1171             env->xcr0 = xcrs.xcrs[0].value;
1172             break;
1173         }
1174     }
1175     return 0;
1176 }
1177
1178 static int kvm_get_sregs(CPUX86State *env)
1179 {
1180     struct kvm_sregs sregs;
1181     uint32_t hflags;
1182     int bit, i, ret;
1183
1184     ret = kvm_vcpu_ioctl(env, KVM_GET_SREGS, &sregs);
1185     if (ret < 0) {
1186         return ret;
1187     }
1188
1189     /* There can only be one pending IRQ set in the bitmap at a time, so try
1190        to find it and save its number instead (-1 for none). */
1191     env->interrupt_injected = -1;
1192     for (i = 0; i < ARRAY_SIZE(sregs.interrupt_bitmap); i++) {
1193         if (sregs.interrupt_bitmap[i]) {
1194             bit = ctz64(sregs.interrupt_bitmap[i]);
1195             env->interrupt_injected = i * 64 + bit;
1196             break;
1197         }
1198     }
1199
1200     get_seg(&env->segs[R_CS], &sregs.cs);
1201     get_seg(&env->segs[R_DS], &sregs.ds);
1202     get_seg(&env->segs[R_ES], &sregs.es);
1203     get_seg(&env->segs[R_FS], &sregs.fs);
1204     get_seg(&env->segs[R_GS], &sregs.gs);
1205     get_seg(&env->segs[R_SS], &sregs.ss);
1206
1207     get_seg(&env->tr, &sregs.tr);
1208     get_seg(&env->ldt, &sregs.ldt);
1209
1210     env->idt.limit = sregs.idt.limit;
1211     env->idt.base = sregs.idt.base;
1212     env->gdt.limit = sregs.gdt.limit;
1213     env->gdt.base = sregs.gdt.base;
1214
1215     env->cr[0] = sregs.cr0;
1216     env->cr[2] = sregs.cr2;
1217     env->cr[3] = sregs.cr3;
1218     env->cr[4] = sregs.cr4;
1219
1220     env->efer = sregs.efer;
1221
1222     /* changes to apic base and cr8/tpr are read back via kvm_arch_post_run */
1223
1224 #define HFLAG_COPY_MASK \
1225     ~( HF_CPL_MASK | HF_PE_MASK | HF_MP_MASK | HF_EM_MASK | \
1226        HF_TS_MASK | HF_TF_MASK | HF_VM_MASK | HF_IOPL_MASK | \
1227        HF_OSFXSR_MASK | HF_LMA_MASK | HF_CS32_MASK | \
1228        HF_SS32_MASK | HF_CS64_MASK | HF_ADDSEG_MASK)
1229
1230     hflags = (env->segs[R_CS].flags >> DESC_DPL_SHIFT) & HF_CPL_MASK;
1231     hflags |= (env->cr[0] & CR0_PE_MASK) << (HF_PE_SHIFT - CR0_PE_SHIFT);
1232     hflags |= (env->cr[0] << (HF_MP_SHIFT - CR0_MP_SHIFT)) &
1233                 (HF_MP_MASK | HF_EM_MASK | HF_TS_MASK);
1234     hflags |= (env->eflags & (HF_TF_MASK | HF_VM_MASK | HF_IOPL_MASK));
1235     hflags |= (env->cr[4] & CR4_OSFXSR_MASK) <<
1236                 (HF_OSFXSR_SHIFT - CR4_OSFXSR_SHIFT);
1237
1238     if (env->efer & MSR_EFER_LMA) {
1239         hflags |= HF_LMA_MASK;
1240     }
1241
1242     if ((hflags & HF_LMA_MASK) && (env->segs[R_CS].flags & DESC_L_MASK)) {
1243         hflags |= HF_CS32_MASK | HF_SS32_MASK | HF_CS64_MASK;
1244     } else {
1245         hflags |= (env->segs[R_CS].flags & DESC_B_MASK) >>
1246                     (DESC_B_SHIFT - HF_CS32_SHIFT);
1247         hflags |= (env->segs[R_SS].flags & DESC_B_MASK) >>
1248                     (DESC_B_SHIFT - HF_SS32_SHIFT);
1249         if (!(env->cr[0] & CR0_PE_MASK) || (env->eflags & VM_MASK) ||
1250             !(hflags & HF_CS32_MASK)) {
1251             hflags |= HF_ADDSEG_MASK;
1252         } else {
1253             hflags |= ((env->segs[R_DS].base | env->segs[R_ES].base |
1254                         env->segs[R_SS].base) != 0) << HF_ADDSEG_SHIFT;
1255         }
1256     }
1257     env->hflags = (env->hflags & HFLAG_COPY_MASK) | hflags;
1258
1259     return 0;
1260 }
1261
1262 static int kvm_get_msrs(CPUX86State *env)
1263 {
1264     struct {
1265         struct kvm_msrs info;
1266         struct kvm_msr_entry entries[100];
1267     } msr_data;
1268     struct kvm_msr_entry *msrs = msr_data.entries;
1269     int ret, i, n;
1270
1271     n = 0;
1272     msrs[n++].index = MSR_IA32_SYSENTER_CS;
1273     msrs[n++].index = MSR_IA32_SYSENTER_ESP;
1274     msrs[n++].index = MSR_IA32_SYSENTER_EIP;
1275     msrs[n++].index = MSR_PAT;
1276     if (has_msr_star) {
1277         msrs[n++].index = MSR_STAR;
1278     }
1279     if (has_msr_hsave_pa) {
1280         msrs[n++].index = MSR_VM_HSAVE_PA;
1281     }
1282     if (has_msr_tsc_deadline) {
1283         msrs[n++].index = MSR_IA32_TSCDEADLINE;
1284     }
1285     if (has_msr_misc_enable) {
1286         msrs[n++].index = MSR_IA32_MISC_ENABLE;
1287     }
1288
1289     if (!env->tsc_valid) {
1290         msrs[n++].index = MSR_IA32_TSC;
1291         env->tsc_valid = !runstate_is_running();
1292     }
1293
1294 #ifdef TARGET_X86_64
1295     if (lm_capable_kernel) {
1296         msrs[n++].index = MSR_CSTAR;
1297         msrs[n++].index = MSR_KERNELGSBASE;
1298         msrs[n++].index = MSR_FMASK;
1299         msrs[n++].index = MSR_LSTAR;
1300     }
1301 #endif
1302     msrs[n++].index = MSR_KVM_SYSTEM_TIME;
1303     msrs[n++].index = MSR_KVM_WALL_CLOCK;
1304     if (has_msr_async_pf_en) {
1305         msrs[n++].index = MSR_KVM_ASYNC_PF_EN;
1306     }
1307     if (has_msr_pv_eoi_en) {
1308         msrs[n++].index = MSR_KVM_PV_EOI_EN;
1309     }
1310
1311     if (env->mcg_cap) {
1312         msrs[n++].index = MSR_MCG_STATUS;
1313         msrs[n++].index = MSR_MCG_CTL;
1314         for (i = 0; i < (env->mcg_cap & 0xff) * 4; i++) {
1315             msrs[n++].index = MSR_MC0_CTL + i;
1316         }
1317     }
1318
1319     msr_data.info.nmsrs = n;
1320     ret = kvm_vcpu_ioctl(env, KVM_GET_MSRS, &msr_data);
1321     if (ret < 0) {
1322         return ret;
1323     }
1324
1325     for (i = 0; i < ret; i++) {
1326         switch (msrs[i].index) {
1327         case MSR_IA32_SYSENTER_CS:
1328             env->sysenter_cs = msrs[i].data;
1329             break;
1330         case MSR_IA32_SYSENTER_ESP:
1331             env->sysenter_esp = msrs[i].data;
1332             break;
1333         case MSR_IA32_SYSENTER_EIP:
1334             env->sysenter_eip = msrs[i].data;
1335             break;
1336         case MSR_PAT:
1337             env->pat = msrs[i].data;
1338             break;
1339         case MSR_STAR:
1340             env->star = msrs[i].data;
1341             break;
1342 #ifdef TARGET_X86_64
1343         case MSR_CSTAR:
1344             env->cstar = msrs[i].data;
1345             break;
1346         case MSR_KERNELGSBASE:
1347             env->kernelgsbase = msrs[i].data;
1348             break;
1349         case MSR_FMASK:
1350             env->fmask = msrs[i].data;
1351             break;
1352         case MSR_LSTAR:
1353             env->lstar = msrs[i].data;
1354             break;
1355 #endif
1356         case MSR_IA32_TSC:
1357             env->tsc = msrs[i].data;
1358             break;
1359         case MSR_IA32_TSCDEADLINE:
1360             env->tsc_deadline = msrs[i].data;
1361             break;
1362         case MSR_VM_HSAVE_PA:
1363             env->vm_hsave = msrs[i].data;
1364             break;
1365         case MSR_KVM_SYSTEM_TIME:
1366             env->system_time_msr = msrs[i].data;
1367             break;
1368         case MSR_KVM_WALL_CLOCK:
1369             env->wall_clock_msr = msrs[i].data;
1370             break;
1371         case MSR_MCG_STATUS:
1372             env->mcg_status = msrs[i].data;
1373             break;
1374         case MSR_MCG_CTL:
1375             env->mcg_ctl = msrs[i].data;
1376             break;
1377         case MSR_IA32_MISC_ENABLE:
1378             env->msr_ia32_misc_enable = msrs[i].data;
1379             break;
1380         default:
1381             if (msrs[i].index >= MSR_MC0_CTL &&
1382                 msrs[i].index < MSR_MC0_CTL + (env->mcg_cap & 0xff) * 4) {
1383                 env->mce_banks[msrs[i].index - MSR_MC0_CTL] = msrs[i].data;
1384             }
1385             break;
1386         case MSR_KVM_ASYNC_PF_EN:
1387             env->async_pf_en_msr = msrs[i].data;
1388             break;
1389         case MSR_KVM_PV_EOI_EN:
1390             env->pv_eoi_en_msr = msrs[i].data;
1391             break;
1392         }
1393     }
1394
1395     return 0;
1396 }
1397
1398 static int kvm_put_mp_state(CPUX86State *env)
1399 {
1400     struct kvm_mp_state mp_state = { .mp_state = env->mp_state };
1401
1402     return kvm_vcpu_ioctl(env, KVM_SET_MP_STATE, &mp_state);
1403 }
1404
1405 static int kvm_get_mp_state(CPUX86State *env)
1406 {
1407     struct kvm_mp_state mp_state;
1408     int ret;
1409
1410     ret = kvm_vcpu_ioctl(env, KVM_GET_MP_STATE, &mp_state);
1411     if (ret < 0) {
1412         return ret;
1413     }
1414     env->mp_state = mp_state.mp_state;
1415     if (kvm_irqchip_in_kernel()) {
1416         env->halted = (mp_state.mp_state == KVM_MP_STATE_HALTED);
1417     }
1418     return 0;
1419 }
1420
1421 static int kvm_get_apic(CPUX86State *env)
1422 {
1423     DeviceState *apic = env->apic_state;
1424     struct kvm_lapic_state kapic;
1425     int ret;
1426
1427     if (apic && kvm_irqchip_in_kernel()) {
1428         ret = kvm_vcpu_ioctl(env, KVM_GET_LAPIC, &kapic);
1429         if (ret < 0) {
1430             return ret;
1431         }
1432
1433         kvm_get_apic_state(apic, &kapic);
1434     }
1435     return 0;
1436 }
1437
1438 static int kvm_put_apic(CPUX86State *env)
1439 {
1440     DeviceState *apic = env->apic_state;
1441     struct kvm_lapic_state kapic;
1442
1443     if (apic && kvm_irqchip_in_kernel()) {
1444         kvm_put_apic_state(apic, &kapic);
1445
1446         return kvm_vcpu_ioctl(env, KVM_SET_LAPIC, &kapic);
1447     }
1448     return 0;
1449 }
1450
1451 static int kvm_put_vcpu_events(CPUX86State *env, int level)
1452 {
1453     struct kvm_vcpu_events events;
1454
1455     if (!kvm_has_vcpu_events()) {
1456         return 0;
1457     }
1458
1459     events.exception.injected = (env->exception_injected >= 0);
1460     events.exception.nr = env->exception_injected;
1461     events.exception.has_error_code = env->has_error_code;
1462     events.exception.error_code = env->error_code;
1463     events.exception.pad = 0;
1464
1465     events.interrupt.injected = (env->interrupt_injected >= 0);
1466     events.interrupt.nr = env->interrupt_injected;
1467     events.interrupt.soft = env->soft_interrupt;
1468
1469     events.nmi.injected = env->nmi_injected;
1470     events.nmi.pending = env->nmi_pending;
1471     events.nmi.masked = !!(env->hflags2 & HF2_NMI_MASK);
1472     events.nmi.pad = 0;
1473
1474     events.sipi_vector = env->sipi_vector;
1475
1476     events.flags = 0;
1477     if (level >= KVM_PUT_RESET_STATE) {
1478         events.flags |=
1479             KVM_VCPUEVENT_VALID_NMI_PENDING | KVM_VCPUEVENT_VALID_SIPI_VECTOR;
1480     }
1481
1482     return kvm_vcpu_ioctl(env, KVM_SET_VCPU_EVENTS, &events);
1483 }
1484
1485 static int kvm_get_vcpu_events(CPUX86State *env)
1486 {
1487     struct kvm_vcpu_events events;
1488     int ret;
1489
1490     if (!kvm_has_vcpu_events()) {
1491         return 0;
1492     }
1493
1494     ret = kvm_vcpu_ioctl(env, KVM_GET_VCPU_EVENTS, &events);
1495     if (ret < 0) {
1496        return ret;
1497     }
1498     env->exception_injected =
1499        events.exception.injected ? events.exception.nr : -1;
1500     env->has_error_code = events.exception.has_error_code;
1501     env->error_code = events.exception.error_code;
1502
1503     env->interrupt_injected =
1504         events.interrupt.injected ? events.interrupt.nr : -1;
1505     env->soft_interrupt = events.interrupt.soft;
1506
1507     env->nmi_injected = events.nmi.injected;
1508     env->nmi_pending = events.nmi.pending;
1509     if (events.nmi.masked) {
1510         env->hflags2 |= HF2_NMI_MASK;
1511     } else {
1512         env->hflags2 &= ~HF2_NMI_MASK;
1513     }
1514
1515     env->sipi_vector = events.sipi_vector;
1516
1517     return 0;
1518 }
1519
1520 static int kvm_guest_debug_workarounds(CPUX86State *env)
1521 {
1522     int ret = 0;
1523     unsigned long reinject_trap = 0;
1524
1525     if (!kvm_has_vcpu_events()) {
1526         if (env->exception_injected == 1) {
1527             reinject_trap = KVM_GUESTDBG_INJECT_DB;
1528         } else if (env->exception_injected == 3) {
1529             reinject_trap = KVM_GUESTDBG_INJECT_BP;
1530         }
1531         env->exception_injected = -1;
1532     }
1533
1534     /*
1535      * Kernels before KVM_CAP_X86_ROBUST_SINGLESTEP overwrote flags.TF
1536      * injected via SET_GUEST_DEBUG while updating GP regs. Work around this
1537      * by updating the debug state once again if single-stepping is on.
1538      * Another reason to call kvm_update_guest_debug here is a pending debug
1539      * trap raise by the guest. On kernels without SET_VCPU_EVENTS we have to
1540      * reinject them via SET_GUEST_DEBUG.
1541      */
1542     if (reinject_trap ||
1543         (!kvm_has_robust_singlestep() && env->singlestep_enabled)) {
1544         ret = kvm_update_guest_debug(env, reinject_trap);
1545     }
1546     return ret;
1547 }
1548
1549 static int kvm_put_debugregs(CPUX86State *env)
1550 {
1551     struct kvm_debugregs dbgregs;
1552     int i;
1553
1554     if (!kvm_has_debugregs()) {
1555         return 0;
1556     }
1557
1558     for (i = 0; i < 4; i++) {
1559         dbgregs.db[i] = env->dr[i];
1560     }
1561     dbgregs.dr6 = env->dr[6];
1562     dbgregs.dr7 = env->dr[7];
1563     dbgregs.flags = 0;
1564
1565     return kvm_vcpu_ioctl(env, KVM_SET_DEBUGREGS, &dbgregs);
1566 }
1567
1568 static int kvm_get_debugregs(CPUX86State *env)
1569 {
1570     struct kvm_debugregs dbgregs;
1571     int i, ret;
1572
1573     if (!kvm_has_debugregs()) {
1574         return 0;
1575     }
1576
1577     ret = kvm_vcpu_ioctl(env, KVM_GET_DEBUGREGS, &dbgregs);
1578     if (ret < 0) {
1579         return ret;
1580     }
1581     for (i = 0; i < 4; i++) {
1582         env->dr[i] = dbgregs.db[i];
1583     }
1584     env->dr[4] = env->dr[6] = dbgregs.dr6;
1585     env->dr[5] = env->dr[7] = dbgregs.dr7;
1586
1587     return 0;
1588 }
1589
1590 int kvm_arch_put_registers(CPUX86State *env, int level)
1591 {
1592     int ret;
1593
1594     assert(cpu_is_stopped(env) || qemu_cpu_is_self(env));
1595
1596     ret = kvm_getput_regs(env, 1);
1597     if (ret < 0) {
1598         return ret;
1599     }
1600     ret = kvm_put_xsave(env);
1601     if (ret < 0) {
1602         return ret;
1603     }
1604     ret = kvm_put_xcrs(env);
1605     if (ret < 0) {
1606         return ret;
1607     }
1608     ret = kvm_put_sregs(env);
1609     if (ret < 0) {
1610         return ret;
1611     }
1612     /* must be before kvm_put_msrs */
1613     ret = kvm_inject_mce_oldstyle(env);
1614     if (ret < 0) {
1615         return ret;
1616     }
1617     ret = kvm_put_msrs(env, level);
1618     if (ret < 0) {
1619         return ret;
1620     }
1621     if (level >= KVM_PUT_RESET_STATE) {
1622         ret = kvm_put_mp_state(env);
1623         if (ret < 0) {
1624             return ret;
1625         }
1626         ret = kvm_put_apic(env);
1627         if (ret < 0) {
1628             return ret;
1629         }
1630     }
1631     ret = kvm_put_vcpu_events(env, level);
1632     if (ret < 0) {
1633         return ret;
1634     }
1635     ret = kvm_put_debugregs(env);
1636     if (ret < 0) {
1637         return ret;
1638     }
1639     /* must be last */
1640     ret = kvm_guest_debug_workarounds(env);
1641     if (ret < 0) {
1642         return ret;
1643     }
1644     return 0;
1645 }
1646
1647 int kvm_arch_get_registers(CPUX86State *env)
1648 {
1649     int ret;
1650
1651     assert(cpu_is_stopped(env) || qemu_cpu_is_self(env));
1652
1653     ret = kvm_getput_regs(env, 0);
1654     if (ret < 0) {
1655         return ret;
1656     }
1657     ret = kvm_get_xsave(env);
1658     if (ret < 0) {
1659         return ret;
1660     }
1661     ret = kvm_get_xcrs(env);
1662     if (ret < 0) {
1663         return ret;
1664     }
1665     ret = kvm_get_sregs(env);
1666     if (ret < 0) {
1667         return ret;
1668     }
1669     ret = kvm_get_msrs(env);
1670     if (ret < 0) {
1671         return ret;
1672     }
1673     ret = kvm_get_mp_state(env);
1674     if (ret < 0) {
1675         return ret;
1676     }
1677     ret = kvm_get_apic(env);
1678     if (ret < 0) {
1679         return ret;
1680     }
1681     ret = kvm_get_vcpu_events(env);
1682     if (ret < 0) {
1683         return ret;
1684     }
1685     ret = kvm_get_debugregs(env);
1686     if (ret < 0) {
1687         return ret;
1688     }
1689     return 0;
1690 }
1691
1692 void kvm_arch_pre_run(CPUX86State *env, struct kvm_run *run)
1693 {
1694     int ret;
1695
1696     /* Inject NMI */
1697     if (env->interrupt_request & CPU_INTERRUPT_NMI) {
1698         env->interrupt_request &= ~CPU_INTERRUPT_NMI;
1699         DPRINTF("injected NMI\n");
1700         ret = kvm_vcpu_ioctl(env, KVM_NMI);
1701         if (ret < 0) {
1702             fprintf(stderr, "KVM: injection failed, NMI lost (%s)\n",
1703                     strerror(-ret));
1704         }
1705     }
1706
1707     if (!kvm_irqchip_in_kernel()) {
1708         /* Force the VCPU out of its inner loop to process any INIT requests
1709          * or pending TPR access reports. */
1710         if (env->interrupt_request &
1711             (CPU_INTERRUPT_INIT | CPU_INTERRUPT_TPR)) {
1712             env->exit_request = 1;
1713         }
1714
1715         /* Try to inject an interrupt if the guest can accept it */
1716         if (run->ready_for_interrupt_injection &&
1717             (env->interrupt_request & CPU_INTERRUPT_HARD) &&
1718             (env->eflags & IF_MASK)) {
1719             int irq;
1720
1721             env->interrupt_request &= ~CPU_INTERRUPT_HARD;
1722             irq = cpu_get_pic_interrupt(env);
1723             if (irq >= 0) {
1724                 struct kvm_interrupt intr;
1725
1726                 intr.irq = irq;
1727                 DPRINTF("injected interrupt %d\n", irq);
1728                 ret = kvm_vcpu_ioctl(env, KVM_INTERRUPT, &intr);
1729                 if (ret < 0) {
1730                     fprintf(stderr,
1731                             "KVM: injection failed, interrupt lost (%s)\n",
1732                             strerror(-ret));
1733                 }
1734             }
1735         }
1736
1737         /* If we have an interrupt but the guest is not ready to receive an
1738          * interrupt, request an interrupt window exit.  This will
1739          * cause a return to userspace as soon as the guest is ready to
1740          * receive interrupts. */
1741         if ((env->interrupt_request & CPU_INTERRUPT_HARD)) {
1742             run->request_interrupt_window = 1;
1743         } else {
1744             run->request_interrupt_window = 0;
1745         }
1746
1747         DPRINTF("setting tpr\n");
1748         run->cr8 = cpu_get_apic_tpr(env->apic_state);
1749     }
1750 }
1751
1752 void kvm_arch_post_run(CPUX86State *env, struct kvm_run *run)
1753 {
1754     if (run->if_flag) {
1755         env->eflags |= IF_MASK;
1756     } else {
1757         env->eflags &= ~IF_MASK;
1758     }
1759     cpu_set_apic_tpr(env->apic_state, run->cr8);
1760     cpu_set_apic_base(env->apic_state, run->apic_base);
1761 }
1762
1763 int kvm_arch_process_async_events(CPUX86State *env)
1764 {
1765     X86CPU *cpu = x86_env_get_cpu(env);
1766
1767     if (env->interrupt_request & CPU_INTERRUPT_MCE) {
1768         /* We must not raise CPU_INTERRUPT_MCE if it's not supported. */
1769         assert(env->mcg_cap);
1770
1771         env->interrupt_request &= ~CPU_INTERRUPT_MCE;
1772
1773         kvm_cpu_synchronize_state(env);
1774
1775         if (env->exception_injected == EXCP08_DBLE) {
1776             /* this means triple fault */
1777             qemu_system_reset_request();
1778             env->exit_request = 1;
1779             return 0;
1780         }
1781         env->exception_injected = EXCP12_MCHK;
1782         env->has_error_code = 0;
1783
1784         env->halted = 0;
1785         if (kvm_irqchip_in_kernel() && env->mp_state == KVM_MP_STATE_HALTED) {
1786             env->mp_state = KVM_MP_STATE_RUNNABLE;
1787         }
1788     }
1789
1790     if (kvm_irqchip_in_kernel()) {
1791         return 0;
1792     }
1793
1794     if (env->interrupt_request & CPU_INTERRUPT_POLL) {
1795         env->interrupt_request &= ~CPU_INTERRUPT_POLL;
1796         apic_poll_irq(env->apic_state);
1797     }
1798     if (((env->interrupt_request & CPU_INTERRUPT_HARD) &&
1799          (env->eflags & IF_MASK)) ||
1800         (env->interrupt_request & CPU_INTERRUPT_NMI)) {
1801         env->halted = 0;
1802     }
1803     if (env->interrupt_request & CPU_INTERRUPT_INIT) {
1804         kvm_cpu_synchronize_state(env);
1805         do_cpu_init(cpu);
1806     }
1807     if (env->interrupt_request & CPU_INTERRUPT_SIPI) {
1808         kvm_cpu_synchronize_state(env);
1809         do_cpu_sipi(cpu);
1810     }
1811     if (env->interrupt_request & CPU_INTERRUPT_TPR) {
1812         env->interrupt_request &= ~CPU_INTERRUPT_TPR;
1813         kvm_cpu_synchronize_state(env);
1814         apic_handle_tpr_access_report(env->apic_state, env->eip,
1815                                       env->tpr_access_type);
1816     }
1817
1818     return env->halted;
1819 }
1820
1821 static int kvm_handle_halt(CPUX86State *env)
1822 {
1823     if (!((env->interrupt_request & CPU_INTERRUPT_HARD) &&
1824           (env->eflags & IF_MASK)) &&
1825         !(env->interrupt_request & CPU_INTERRUPT_NMI)) {
1826         env->halted = 1;
1827         return EXCP_HLT;
1828     }
1829
1830     return 0;
1831 }
1832
1833 static int kvm_handle_tpr_access(CPUX86State *env)
1834 {
1835     struct kvm_run *run = env->kvm_run;
1836
1837     apic_handle_tpr_access_report(env->apic_state, run->tpr_access.rip,
1838                                   run->tpr_access.is_write ? TPR_ACCESS_WRITE
1839                                                            : TPR_ACCESS_READ);
1840     return 1;
1841 }
1842
1843 int kvm_arch_insert_sw_breakpoint(CPUX86State *env, struct kvm_sw_breakpoint *bp)
1844 {
1845     static const uint8_t int3 = 0xcc;
1846
1847     if (cpu_memory_rw_debug(env, bp->pc, (uint8_t *)&bp->saved_insn, 1, 0) ||
1848         cpu_memory_rw_debug(env, bp->pc, (uint8_t *)&int3, 1, 1)) {
1849         return -EINVAL;
1850     }
1851     return 0;
1852 }
1853
1854 int kvm_arch_remove_sw_breakpoint(CPUX86State *env, struct kvm_sw_breakpoint *bp)
1855 {
1856     uint8_t int3;
1857
1858     if (cpu_memory_rw_debug(env, bp->pc, &int3, 1, 0) || int3 != 0xcc ||
1859         cpu_memory_rw_debug(env, bp->pc, (uint8_t *)&bp->saved_insn, 1, 1)) {
1860         return -EINVAL;
1861     }
1862     return 0;
1863 }
1864
1865 static struct {
1866     target_ulong addr;
1867     int len;
1868     int type;
1869 } hw_breakpoint[4];
1870
1871 static int nb_hw_breakpoint;
1872
1873 static int find_hw_breakpoint(target_ulong addr, int len, int type)
1874 {
1875     int n;
1876
1877     for (n = 0; n < nb_hw_breakpoint; n++) {
1878         if (hw_breakpoint[n].addr == addr && hw_breakpoint[n].type == type &&
1879             (hw_breakpoint[n].len == len || len == -1)) {
1880             return n;
1881         }
1882     }
1883     return -1;
1884 }
1885
1886 int kvm_arch_insert_hw_breakpoint(target_ulong addr,
1887                                   target_ulong len, int type)
1888 {
1889     switch (type) {
1890     case GDB_BREAKPOINT_HW:
1891         len = 1;
1892         break;
1893     case GDB_WATCHPOINT_WRITE:
1894     case GDB_WATCHPOINT_ACCESS:
1895         switch (len) {
1896         case 1:
1897             break;
1898         case 2:
1899         case 4:
1900         case 8:
1901             if (addr & (len - 1)) {
1902                 return -EINVAL;
1903             }
1904             break;
1905         default:
1906             return -EINVAL;
1907         }
1908         break;
1909     default:
1910         return -ENOSYS;
1911     }
1912
1913     if (nb_hw_breakpoint == 4) {
1914         return -ENOBUFS;
1915     }
1916     if (find_hw_breakpoint(addr, len, type) >= 0) {
1917         return -EEXIST;
1918     }
1919     hw_breakpoint[nb_hw_breakpoint].addr = addr;
1920     hw_breakpoint[nb_hw_breakpoint].len = len;
1921     hw_breakpoint[nb_hw_breakpoint].type = type;
1922     nb_hw_breakpoint++;
1923
1924     return 0;
1925 }
1926
1927 int kvm_arch_remove_hw_breakpoint(target_ulong addr,
1928                                   target_ulong len, int type)
1929 {
1930     int n;
1931
1932     n = find_hw_breakpoint(addr, (type == GDB_BREAKPOINT_HW) ? 1 : len, type);
1933     if (n < 0) {
1934         return -ENOENT;
1935     }
1936     nb_hw_breakpoint--;
1937     hw_breakpoint[n] = hw_breakpoint[nb_hw_breakpoint];
1938
1939     return 0;
1940 }
1941
1942 void kvm_arch_remove_all_hw_breakpoints(void)
1943 {
1944     nb_hw_breakpoint = 0;
1945 }
1946
1947 static CPUWatchpoint hw_watchpoint;
1948
1949 static int kvm_handle_debug(struct kvm_debug_exit_arch *arch_info)
1950 {
1951     int ret = 0;
1952     int n;
1953
1954     if (arch_info->exception == 1) {
1955         if (arch_info->dr6 & (1 << 14)) {
1956             if (cpu_single_env->singlestep_enabled) {
1957                 ret = EXCP_DEBUG;
1958             }
1959         } else {
1960             for (n = 0; n < 4; n++) {
1961                 if (arch_info->dr6 & (1 << n)) {
1962                     switch ((arch_info->dr7 >> (16 + n*4)) & 0x3) {
1963                     case 0x0:
1964                         ret = EXCP_DEBUG;
1965                         break;
1966                     case 0x1:
1967                         ret = EXCP_DEBUG;
1968                         cpu_single_env->watchpoint_hit = &hw_watchpoint;
1969                         hw_watchpoint.vaddr = hw_breakpoint[n].addr;
1970                         hw_watchpoint.flags = BP_MEM_WRITE;
1971                         break;
1972                     case 0x3:
1973                         ret = EXCP_DEBUG;
1974                         cpu_single_env->watchpoint_hit = &hw_watchpoint;
1975                         hw_watchpoint.vaddr = hw_breakpoint[n].addr;
1976                         hw_watchpoint.flags = BP_MEM_ACCESS;
1977                         break;
1978                     }
1979                 }
1980             }
1981         }
1982     } else if (kvm_find_sw_breakpoint(cpu_single_env, arch_info->pc)) {
1983         ret = EXCP_DEBUG;
1984     }
1985     if (ret == 0) {
1986         cpu_synchronize_state(cpu_single_env);
1987         assert(cpu_single_env->exception_injected == -1);
1988
1989         /* pass to guest */
1990         cpu_single_env->exception_injected = arch_info->exception;
1991         cpu_single_env->has_error_code = 0;
1992     }
1993
1994     return ret;
1995 }
1996
1997 void kvm_arch_update_guest_debug(CPUX86State *env, struct kvm_guest_debug *dbg)
1998 {
1999     const uint8_t type_code[] = {
2000         [GDB_BREAKPOINT_HW] = 0x0,
2001         [GDB_WATCHPOINT_WRITE] = 0x1,
2002         [GDB_WATCHPOINT_ACCESS] = 0x3
2003     };
2004     const uint8_t len_code[] = {
2005         [1] = 0x0, [2] = 0x1, [4] = 0x3, [8] = 0x2
2006     };
2007     int n;
2008
2009     if (kvm_sw_breakpoints_active(env)) {
2010         dbg->control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP;
2011     }
2012     if (nb_hw_breakpoint > 0) {
2013         dbg->control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_HW_BP;
2014         dbg->arch.debugreg[7] = 0x0600;
2015         for (n = 0; n < nb_hw_breakpoint; n++) {
2016             dbg->arch.debugreg[n] = hw_breakpoint[n].addr;
2017             dbg->arch.debugreg[7] |= (2 << (n * 2)) |
2018                 (type_code[hw_breakpoint[n].type] << (16 + n*4)) |
2019                 ((uint32_t)len_code[hw_breakpoint[n].len] << (18 + n*4));
2020         }
2021     }
2022 }
2023
2024 static bool host_supports_vmx(void)
2025 {
2026     uint32_t ecx, unused;
2027
2028     host_cpuid(1, 0, &unused, &unused, &ecx, &unused);
2029     return ecx & CPUID_EXT_VMX;
2030 }
2031
2032 #define VMX_INVALID_GUEST_STATE 0x80000021
2033
2034 int kvm_arch_handle_exit(CPUX86State *env, struct kvm_run *run)
2035 {
2036     uint64_t code;
2037     int ret;
2038
2039     switch (run->exit_reason) {
2040     case KVM_EXIT_HLT:
2041         DPRINTF("handle_hlt\n");
2042         ret = kvm_handle_halt(env);
2043         break;
2044     case KVM_EXIT_SET_TPR:
2045         ret = 0;
2046         break;
2047     case KVM_EXIT_TPR_ACCESS:
2048         ret = kvm_handle_tpr_access(env);
2049         break;
2050     case KVM_EXIT_FAIL_ENTRY:
2051         code = run->fail_entry.hardware_entry_failure_reason;
2052         fprintf(stderr, "KVM: entry failed, hardware error 0x%" PRIx64 "\n",
2053                 code);
2054         if (host_supports_vmx() && code == VMX_INVALID_GUEST_STATE) {
2055             fprintf(stderr,
2056                     "\nIf you're running a guest on an Intel machine without "
2057                         "unrestricted mode\n"
2058                     "support, the failure can be most likely due to the guest "
2059                         "entering an invalid\n"
2060                     "state for Intel VT. For example, the guest maybe running "
2061                         "in big real mode\n"
2062                     "which is not supported on less recent Intel processors."
2063                         "\n\n");
2064         }
2065         ret = -1;
2066         break;
2067     case KVM_EXIT_EXCEPTION:
2068         fprintf(stderr, "KVM: exception %d exit (error code 0x%x)\n",
2069                 run->ex.exception, run->ex.error_code);
2070         ret = -1;
2071         break;
2072     case KVM_EXIT_DEBUG:
2073         DPRINTF("kvm_exit_debug\n");
2074         ret = kvm_handle_debug(&run->debug.arch);
2075         break;
2076     default:
2077         fprintf(stderr, "KVM: unknown exit reason %d\n", run->exit_reason);
2078         ret = -1;
2079         break;
2080     }
2081
2082     return ret;
2083 }
2084
2085 bool kvm_arch_stop_on_emulation_error(CPUX86State *env)
2086 {
2087     kvm_cpu_synchronize_state(env);
2088     return !(env->cr[0] & CR0_PE_MASK) ||
2089            ((env->segs[R_CS].selector  & 3) != 3);
2090 }
2091
2092 void kvm_arch_init_irq_routing(KVMState *s)
2093 {
2094     if (!kvm_check_extension(s, KVM_CAP_IRQ_ROUTING)) {
2095         /* If kernel can't do irq routing, interrupt source
2096          * override 0->2 cannot be set up as required by HPET.
2097          * So we have to disable it.
2098          */
2099         no_hpet = 1;
2100     }
2101     /* We know at this point that we're using the in-kernel
2102      * irqchip, so we can use irqfds, and on x86 we know
2103      * we can use msi via irqfd and GSI routing.
2104      */
2105     kvm_irqfds_allowed = true;
2106     kvm_msi_via_irqfd_allowed = true;
2107     kvm_gsi_routing_allowed = true;
2108 }
2109
2110 /* Classic KVM device assignment interface. Will remain x86 only. */
2111 int kvm_device_pci_assign(KVMState *s, PCIHostDeviceAddress *dev_addr,
2112                           uint32_t flags, uint32_t *dev_id)
2113 {
2114     struct kvm_assigned_pci_dev dev_data = {
2115         .segnr = dev_addr->domain,
2116         .busnr = dev_addr->bus,
2117         .devfn = PCI_DEVFN(dev_addr->slot, dev_addr->function),
2118         .flags = flags,
2119     };
2120     int ret;
2121
2122     dev_data.assigned_dev_id =
2123         (dev_addr->domain << 16) | (dev_addr->bus << 8) | dev_data.devfn;
2124
2125     ret = kvm_vm_ioctl(s, KVM_ASSIGN_PCI_DEVICE, &dev_data);
2126     if (ret < 0) {
2127         return ret;
2128     }
2129
2130     *dev_id = dev_data.assigned_dev_id;
2131
2132     return 0;
2133 }
2134
2135 int kvm_device_pci_deassign(KVMState *s, uint32_t dev_id)
2136 {
2137     struct kvm_assigned_pci_dev dev_data = {
2138         .assigned_dev_id = dev_id,
2139     };
2140
2141     return kvm_vm_ioctl(s, KVM_DEASSIGN_PCI_DEVICE, &dev_data);
2142 }
2143
2144 static int kvm_assign_irq_internal(KVMState *s, uint32_t dev_id,
2145                                    uint32_t irq_type, uint32_t guest_irq)
2146 {
2147     struct kvm_assigned_irq assigned_irq = {
2148         .assigned_dev_id = dev_id,
2149         .guest_irq = guest_irq,
2150         .flags = irq_type,
2151     };
2152
2153     if (kvm_check_extension(s, KVM_CAP_ASSIGN_DEV_IRQ)) {
2154         return kvm_vm_ioctl(s, KVM_ASSIGN_DEV_IRQ, &assigned_irq);
2155     } else {
2156         return kvm_vm_ioctl(s, KVM_ASSIGN_IRQ, &assigned_irq);
2157     }
2158 }
2159
2160 int kvm_device_intx_assign(KVMState *s, uint32_t dev_id, bool use_host_msi,
2161                            uint32_t guest_irq)
2162 {
2163     uint32_t irq_type = KVM_DEV_IRQ_GUEST_INTX |
2164         (use_host_msi ? KVM_DEV_IRQ_HOST_MSI : KVM_DEV_IRQ_HOST_INTX);
2165
2166     return kvm_assign_irq_internal(s, dev_id, irq_type, guest_irq);
2167 }
2168
2169 int kvm_device_intx_set_mask(KVMState *s, uint32_t dev_id, bool masked)
2170 {
2171     struct kvm_assigned_pci_dev dev_data = {
2172         .assigned_dev_id = dev_id,
2173         .flags = masked ? KVM_DEV_ASSIGN_MASK_INTX : 0,
2174     };
2175
2176     return kvm_vm_ioctl(s, KVM_ASSIGN_SET_INTX_MASK, &dev_data);
2177 }
2178
2179 static int kvm_deassign_irq_internal(KVMState *s, uint32_t dev_id,
2180                                      uint32_t type)
2181 {
2182     struct kvm_assigned_irq assigned_irq = {
2183         .assigned_dev_id = dev_id,
2184         .flags = type,
2185     };
2186
2187     return kvm_vm_ioctl(s, KVM_DEASSIGN_DEV_IRQ, &assigned_irq);
2188 }
2189
2190 int kvm_device_intx_deassign(KVMState *s, uint32_t dev_id, bool use_host_msi)
2191 {
2192     return kvm_deassign_irq_internal(s, dev_id, KVM_DEV_IRQ_GUEST_INTX |
2193         (use_host_msi ? KVM_DEV_IRQ_HOST_MSI : KVM_DEV_IRQ_HOST_INTX));
2194 }
2195
2196 int kvm_device_msi_assign(KVMState *s, uint32_t dev_id, int virq)
2197 {
2198     return kvm_assign_irq_internal(s, dev_id, KVM_DEV_IRQ_HOST_MSI |
2199                                               KVM_DEV_IRQ_GUEST_MSI, virq);
2200 }
2201
2202 int kvm_device_msi_deassign(KVMState *s, uint32_t dev_id)
2203 {
2204     return kvm_deassign_irq_internal(s, dev_id, KVM_DEV_IRQ_GUEST_MSI |
2205                                                 KVM_DEV_IRQ_HOST_MSI);
2206 }
2207
2208 bool kvm_device_msix_supported(KVMState *s)
2209 {
2210     /* The kernel lacks a corresponding KVM_CAP, so we probe by calling
2211      * KVM_ASSIGN_SET_MSIX_NR with an invalid parameter. */
2212     return kvm_vm_ioctl(s, KVM_ASSIGN_SET_MSIX_NR, NULL) == -EFAULT;
2213 }
2214
2215 int kvm_device_msix_init_vectors(KVMState *s, uint32_t dev_id,
2216                                  uint32_t nr_vectors)
2217 {
2218     struct kvm_assigned_msix_nr msix_nr = {
2219         .assigned_dev_id = dev_id,
2220         .entry_nr = nr_vectors,
2221     };
2222
2223     return kvm_vm_ioctl(s, KVM_ASSIGN_SET_MSIX_NR, &msix_nr);
2224 }
2225
2226 int kvm_device_msix_set_vector(KVMState *s, uint32_t dev_id, uint32_t vector,
2227                                int virq)
2228 {
2229     struct kvm_assigned_msix_entry msix_entry = {
2230         .assigned_dev_id = dev_id,
2231         .gsi = virq,
2232         .entry = vector,
2233     };
2234
2235     return kvm_vm_ioctl(s, KVM_ASSIGN_SET_MSIX_ENTRY, &msix_entry);
2236 }
2237
2238 int kvm_device_msix_assign(KVMState *s, uint32_t dev_id)
2239 {
2240     return kvm_assign_irq_internal(s, dev_id, KVM_DEV_IRQ_HOST_MSIX |
2241                                               KVM_DEV_IRQ_GUEST_MSIX, 0);
2242 }
2243
2244 int kvm_device_msix_deassign(KVMState *s, uint32_t dev_id)
2245 {
2246     return kvm_deassign_irq_internal(s, dev_id, KVM_DEV_IRQ_GUEST_MSIX |
2247                                                 KVM_DEV_IRQ_HOST_MSIX);
2248 }