target-i386/kvm.c

   1 /*
   2  * QEMU KVM support
   3  *
   4  * Copyright (C) 2006-2008 Qumranet Technologies
   5  * Copyright IBM, Corp. 2008
   6  *
   7  * Authors:
   8  *  Anthony Liguori   <aliguori@us.ibm.com>
   9  *
  10  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  11  * See the COPYING file in the top-level directory.
  12  *
  13  */
  14
  15 #include <sys/types.h>
  16 #include <sys/ioctl.h>
  17 #include <sys/mman.h>
  18 #include <sys/utsname.h>
  19
  20 #include <linux/kvm.h>
  21 #include <linux/kvm_para.h>
  22
  23 #include "qemu-common.h"
  24 #include "sysemu.h"
  25 #include "kvm.h"
  26 #include "kvm_i386.h"
  27 #include "cpu.h"
  28 #include "gdbstub.h"
  29 #include "host-utils.h"
  30 #include "hw/pc.h"
  31 #include "hw/apic.h"
  32 #include "ioport.h"
  33 #include "hyperv.h"
  34
  35 //#define DEBUG_KVM
  36
  37 #ifdef DEBUG_KVM
  38 #define DPRINTF(fmt, ...) \
  39     do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
  40 #else
  41 #define DPRINTF(fmt, ...) \
  42     do { } while (0)
  43 #endif
  44
  45 #define MSR_KVM_WALL_CLOCK  0x11
  46 #define MSR_KVM_SYSTEM_TIME 0x12
  47
  48 #ifndef BUS_MCEERR_AR
  49 #define BUS_MCEERR_AR 4
  50 #endif
  51 #ifndef BUS_MCEERR_AO
  52 #define BUS_MCEERR_AO 5
  53 #endif
  54
  55 const KVMCapabilityInfo kvm_arch_required_capabilities[] = {
  56     KVM_CAP_INFO(SET_TSS_ADDR),
  57     KVM_CAP_INFO(EXT_CPUID),
  58     KVM_CAP_INFO(MP_STATE),
  59     KVM_CAP_LAST_INFO
  60 };
  61
  62 static bool has_msr_star;
  63 static bool has_msr_hsave_pa;
  64 static bool has_msr_tsc_deadline;
  65 static bool has_msr_async_pf_en;
  66 static bool has_msr_pv_eoi_en;
  67 static bool has_msr_misc_enable;
  68 static int lm_capable_kernel;
  69
  70 bool kvm_allows_irq0_override(void)
  71 {
  72     return !kvm_irqchip_in_kernel() || kvm_has_gsi_routing();
  73 }
  74
  75 static struct kvm_cpuid2 *try_get_cpuid(KVMState *s, int max)
  76 {
  77     struct kvm_cpuid2 *cpuid;
  78     int r, size;
  79
  80     size = sizeof(*cpuid) + max * sizeof(*cpuid->entries);
  81     cpuid = (struct kvm_cpuid2 *)g_malloc0(size);
  82     cpuid->nent = max;
  83     r = kvm_ioctl(s, KVM_GET_SUPPORTED_CPUID, cpuid);
  84     if (r == 0 && cpuid->nent >= max) {
  85         r = -E2BIG;
  86     }
  87     if (r < 0) {
  88         if (r == -E2BIG) {
  89             g_free(cpuid);
  90             return NULL;
  91         } else {
  92             fprintf(stderr, "KVM_GET_SUPPORTED_CPUID failed: %s\n",
  93                     strerror(-r));
  94             exit(1);
  95         }
  96     }
  97     return cpuid;
  98 }
  99
 100 struct kvm_para_features {
 101     int cap;
 102     int feature;
 103 } para_features[] = {
 104     { KVM_CAP_CLOCKSOURCE, KVM_FEATURE_CLOCKSOURCE },
 105     { KVM_CAP_NOP_IO_DELAY, KVM_FEATURE_NOP_IO_DELAY },
 106     { KVM_CAP_PV_MMU, KVM_FEATURE_MMU_OP },
 107     { KVM_CAP_ASYNC_PF, KVM_FEATURE_ASYNC_PF },
 108     { -1, -1 }
 109 };
 110
 111 static int get_para_features(KVMState *s)
 112 {
 113     int i, features = 0;
 114
 115     for (i = 0; i < ARRAY_SIZE(para_features) - 1; i++) {
 116         if (kvm_check_extension(s, para_features[i].cap)) {
 117             features |= (1 << para_features[i].feature);
 118         }
 119     }
 120
 121     return features;
 122 }
 123
 124
 125 uint32_t kvm_arch_get_supported_cpuid(KVMState *s, uint32_t function,
 126                                       uint32_t index, int reg)
 127 {
 128     struct kvm_cpuid2 *cpuid;
 129     int i, max;
 130     uint32_t ret = 0;
 131     uint32_t cpuid_1_edx;
 132     int has_kvm_features = 0;
 133
 134     max = 1;
 135     while ((cpuid = try_get_cpuid(s, max)) == NULL) {
 136         max *= 2;
 137     }
 138
 139     for (i = 0; i < cpuid->nent; ++i) {
 140         if (cpuid->entries[i].function == function &&
 141             cpuid->entries[i].index == index) {
 142             if (cpuid->entries[i].function == KVM_CPUID_FEATURES) {
 143                 has_kvm_features = 1;
 144             }
 145             switch (reg) {
 146             case R_EAX:
 147                 ret = cpuid->entries[i].eax;
 148                 break;
 149             case R_EBX:
 150                 ret = cpuid->entries[i].ebx;
 151                 break;
 152             case R_ECX:
 153                 ret = cpuid->entries[i].ecx;
 154                 break;
 155             case R_EDX:
 156                 ret = cpuid->entries[i].edx;
 157                 switch (function) {
 158                 case 1:
 159                     /* KVM before 2.6.30 misreports the following features */
 160                     ret |= CPUID_MTRR | CPUID_PAT | CPUID_MCE | CPUID_MCA;
 161                     break;
 162                 case 0x80000001:
 163                     /* On Intel, kvm returns cpuid according to the Intel spec,
 164                      * so add missing bits according to the AMD spec:
 165                      */
 166                     cpuid_1_edx = kvm_arch_get_supported_cpuid(s, 1, 0, R_EDX);
 167                     ret |= cpuid_1_edx & 0x183f7ff;
 168                     break;
 169                 }
 170                 break;
 171             }
 172         }
 173     }
 174
 175     g_free(cpuid);
 176
 177     /* fallback for older kernels */
 178     if (!has_kvm_features && (function == KVM_CPUID_FEATURES)) {
 179         ret = get_para_features(s);
 180     }
 181
 182     return ret;
 183 }
 184
 185 typedef struct HWPoisonPage {
 186     ram_addr_t ram_addr;
 187     QLIST_ENTRY(HWPoisonPage) list;
 188 } HWPoisonPage;
 189
 190 static QLIST_HEAD(, HWPoisonPage) hwpoison_page_list =
 191     QLIST_HEAD_INITIALIZER(hwpoison_page_list);
 192
 193 static void kvm_unpoison_all(void *param)
 194 {
 195     HWPoisonPage *page, *next_page;
 196
 197     QLIST_FOREACH_SAFE(page, &hwpoison_page_list, list, next_page) {
 198         QLIST_REMOVE(page, list);
 199         qemu_ram_remap(page->ram_addr, TARGET_PAGE_SIZE);
 200         g_free(page);
 201     }
 202 }
 203
 204 static void kvm_hwpoison_page_add(ram_addr_t ram_addr)
 205 {
 206     HWPoisonPage *page;
 207
 208     QLIST_FOREACH(page, &hwpoison_page_list, list) {
 209         if (page->ram_addr == ram_addr) {
 210             return;
 211         }
 212     }
 213     page = g_malloc(sizeof(HWPoisonPage));
 214     page->ram_addr = ram_addr;
 215     QLIST_INSERT_HEAD(&hwpoison_page_list, page, list);
 216 }
 217
 218 static int kvm_get_mce_cap_supported(KVMState *s, uint64_t *mce_cap,
 219                                      int *max_banks)
 220 {
 221     int r;
 222
 223     r = kvm_check_extension(s, KVM_CAP_MCE);
 224     if (r > 0) {
 225         *max_banks = r;
 226         return kvm_ioctl(s, KVM_X86_GET_MCE_CAP_SUPPORTED, mce_cap);
 227     }
 228     return -ENOSYS;
 229 }
 230
 231 static void kvm_mce_inject(CPUX86State *env, target_phys_addr_t paddr, int code)
 232 {
 233     uint64_t status = MCI_STATUS_VAL | MCI_STATUS_UC | MCI_STATUS_EN |
 234                       MCI_STATUS_MISCV | MCI_STATUS_ADDRV | MCI_STATUS_S;
 235     uint64_t mcg_status = MCG_STATUS_MCIP;
 236
 237     if (code == BUS_MCEERR_AR) {
 238         status |= MCI_STATUS_AR | 0x134;
 239         mcg_status |= MCG_STATUS_EIPV;
 240     } else {
 241         status |= 0xc0;
 242         mcg_status |= MCG_STATUS_RIPV;
 243     }
 244     cpu_x86_inject_mce(NULL, env, 9, status, mcg_status, paddr,
 245                        (MCM_ADDR_PHYS << 6) | 0xc,
 246                        cpu_x86_support_mca_broadcast(env) ?
 247                        MCE_INJECT_BROADCAST : 0);
 248 }
 249
 250 static void hardware_memory_error(void)
 251 {
 252     fprintf(stderr, "Hardware memory error!\n");
 253     exit(1);
 254 }
 255
 256 int kvm_arch_on_sigbus_vcpu(CPUX86State *env, int code, void *addr)
 257 {
 258     ram_addr_t ram_addr;
 259     target_phys_addr_t paddr;
 260
 261     if ((env->mcg_cap & MCG_SER_P) && addr
 262         && (code == BUS_MCEERR_AR || code == BUS_MCEERR_AO)) {
 263         if (qemu_ram_addr_from_host(addr, &ram_addr) ||
 264             !kvm_physical_memory_addr_from_host(env->kvm_state, addr, &paddr)) {
 265             fprintf(stderr, "Hardware memory error for memory used by "
 266                     "QEMU itself instead of guest system!\n");
 267             /* Hope we are lucky for AO MCE */
 268             if (code == BUS_MCEERR_AO) {
 269                 return 0;
 270             } else {
 271                 hardware_memory_error();
 272             }
 273         }
 274         kvm_hwpoison_page_add(ram_addr);
 275         kvm_mce_inject(env, paddr, code);
 276     } else {
 277         if (code == BUS_MCEERR_AO) {
 278             return 0;
 279         } else if (code == BUS_MCEERR_AR) {
 280             hardware_memory_error();
 281         } else {
 282             return 1;
 283         }
 284     }
 285     return 0;
 286 }
 287
 288 int kvm_arch_on_sigbus(int code, void *addr)
 289 {
 290     if ((first_cpu->mcg_cap & MCG_SER_P) && addr && code == BUS_MCEERR_AO) {
 291         ram_addr_t ram_addr;
 292         target_phys_addr_t paddr;
 293
 294         /* Hope we are lucky for AO MCE */
 295         if (qemu_ram_addr_from_host(addr, &ram_addr) ||
 296             !kvm_physical_memory_addr_from_host(first_cpu->kvm_state, addr,
 297                                                 &paddr)) {
 298             fprintf(stderr, "Hardware memory error for memory used by "
 299                     "QEMU itself instead of guest system!: %p\n", addr);
 300             return 0;
 301         }
 302         kvm_hwpoison_page_add(ram_addr);
 303         kvm_mce_inject(first_cpu, paddr, code);
 304     } else {
 305         if (code == BUS_MCEERR_AO) {
 306             return 0;
 307         } else if (code == BUS_MCEERR_AR) {
 308             hardware_memory_error();
 309         } else {
 310             return 1;
 311         }
 312     }
 313     return 0;
 314 }
 315
 316 static int kvm_inject_mce_oldstyle(CPUX86State *env)
 317 {
 318     if (!kvm_has_vcpu_events() && env->exception_injected == EXCP12_MCHK) {
 319         unsigned int bank, bank_num = env->mcg_cap & 0xff;
 320         struct kvm_x86_mce mce;
 321
 322         env->exception_injected = -1;
 323
 324         /*
 325          * There must be at least one bank in use if an MCE is pending.
 326          * Find it and use its values for the event injection.
 327          */
 328         for (bank = 0; bank < bank_num; bank++) {
 329             if (env->mce_banks[bank * 4 + 1] & MCI_STATUS_VAL) {
 330                 break;
 331             }
 332         }
 333         assert(bank < bank_num);
 334
 335         mce.bank = bank;
 336         mce.status = env->mce_banks[bank * 4 + 1];
 337         mce.mcg_status = env->mcg_status;
 338         mce.addr = env->mce_banks[bank * 4 + 2];
 339         mce.misc = env->mce_banks[bank * 4 + 3];
 340
 341         return kvm_vcpu_ioctl(env, KVM_X86_SET_MCE, &mce);
 342     }
 343     return 0;
 344 }
 345
 346 static void cpu_update_state(void *opaque, int running, RunState state)
 347 {
 348     CPUX86State *env = opaque;
 349
 350     if (running) {
 351         env->tsc_valid = false;
 352     }
 353 }
 354
 355 int kvm_arch_init_vcpu(CPUX86State *env)
 356 {
 357     struct {
 358         struct kvm_cpuid2 cpuid;
 359         struct kvm_cpuid_entry2 entries[100];
 360     } QEMU_PACKED cpuid_data;
 361     KVMState *s = env->kvm_state;
 362     uint32_t limit, i, j, cpuid_i;
 363     uint32_t unused;
 364     struct kvm_cpuid_entry2 *c;
 365     uint32_t signature[3];
 366     int r;
 367
 368     env->cpuid_features &= kvm_arch_get_supported_cpuid(s, 1, 0, R_EDX);
 369
 370     i = env->cpuid_ext_features & CPUID_EXT_HYPERVISOR;
 371     j = env->cpuid_ext_features & CPUID_EXT_TSC_DEADLINE_TIMER;
 372     env->cpuid_ext_features &= kvm_arch_get_supported_cpuid(s, 1, 0, R_ECX);
 373     env->cpuid_ext_features |= i;
 374     if (j && kvm_irqchip_in_kernel() &&
 375         kvm_check_extension(s, KVM_CAP_TSC_DEADLINE_TIMER)) {
 376         env->cpuid_ext_features |= CPUID_EXT_TSC_DEADLINE_TIMER;
 377     }
 378
 379     env->cpuid_ext2_features &= kvm_arch_get_supported_cpuid(s, 0x80000001,
 380                                                              0, R_EDX);
 381     env->cpuid_ext3_features &= kvm_arch_get_supported_cpuid(s, 0x80000001,
 382                                                              0, R_ECX);
 383     env->cpuid_svm_features  &= kvm_arch_get_supported_cpuid(s, 0x8000000A,
 384                                                              0, R_EDX);
 385
 386     cpuid_i = 0;
 387
 388     /* Paravirtualization CPUIDs */
 389     c = &cpuid_data.entries[cpuid_i++];
 390     memset(c, 0, sizeof(*c));
 391     c->function = KVM_CPUID_SIGNATURE;
 392     if (!hyperv_enabled()) {
 393         memcpy(signature, "KVMKVMKVM\0\0\0", 12);
 394         c->eax = 0;
 395     } else {
 396         memcpy(signature, "Microsoft Hv", 12);
 397         c->eax = HYPERV_CPUID_MIN;
 398     }
 399     c->ebx = signature[0];
 400     c->ecx = signature[1];
 401     c->edx = signature[2];
 402
 403     c = &cpuid_data.entries[cpuid_i++];
 404     memset(c, 0, sizeof(*c));
 405     c->function = KVM_CPUID_FEATURES;
 406     c->eax = env->cpuid_kvm_features &
 407         kvm_arch_get_supported_cpuid(s, KVM_CPUID_FEATURES, 0, R_EAX);
 408
 409     if (hyperv_enabled()) {
 410         memcpy(signature, "Hv#1\0\0\0\0\0\0\0\0", 12);
 411         c->eax = signature[0];
 412
 413         c = &cpuid_data.entries[cpuid_i++];
 414         memset(c, 0, sizeof(*c));
 415         c->function = HYPERV_CPUID_VERSION;
 416         c->eax = 0x00001bbc;
 417         c->ebx = 0x00060001;
 418
 419         c = &cpuid_data.entries[cpuid_i++];
 420         memset(c, 0, sizeof(*c));
 421         c->function = HYPERV_CPUID_FEATURES;
 422         if (hyperv_relaxed_timing_enabled()) {
 423             c->eax |= HV_X64_MSR_HYPERCALL_AVAILABLE;
 424         }
 425         if (hyperv_vapic_recommended()) {
 426             c->eax |= HV_X64_MSR_HYPERCALL_AVAILABLE;
 427             c->eax |= HV_X64_MSR_APIC_ACCESS_AVAILABLE;
 428         }
 429
 430         c = &cpuid_data.entries[cpuid_i++];
 431         memset(c, 0, sizeof(*c));
 432         c->function = HYPERV_CPUID_ENLIGHTMENT_INFO;
 433         if (hyperv_relaxed_timing_enabled()) {
 434             c->eax |= HV_X64_RELAXED_TIMING_RECOMMENDED;
 435         }
 436         if (hyperv_vapic_recommended()) {
 437             c->eax |= HV_X64_APIC_ACCESS_RECOMMENDED;
 438         }
 439         c->ebx = hyperv_get_spinlock_retries();
 440
 441         c = &cpuid_data.entries[cpuid_i++];
 442         memset(c, 0, sizeof(*c));
 443         c->function = HYPERV_CPUID_IMPLEMENT_LIMITS;
 444         c->eax = 0x40;
 445         c->ebx = 0x40;
 446
 447         c = &cpuid_data.entries[cpuid_i++];
 448         memset(c, 0, sizeof(*c));
 449         c->function = KVM_CPUID_SIGNATURE_NEXT;
 450         memcpy(signature, "KVMKVMKVM\0\0\0", 12);
 451         c->eax = 0;
 452         c->ebx = signature[0];
 453         c->ecx = signature[1];
 454         c->edx = signature[2];
 455     }
 456
 457     has_msr_async_pf_en = c->eax & (1 << KVM_FEATURE_ASYNC_PF);
 458
 459     has_msr_pv_eoi_en = c->eax & (1 << KVM_FEATURE_PV_EOI);
 460
 461     cpu_x86_cpuid(env, 0, 0, &limit, &unused, &unused, &unused);
 462
 463     for (i = 0; i <= limit; i++) {
 464         c = &cpuid_data.entries[cpuid_i++];
 465
 466         switch (i) {
 467         case 2: {
 468             /* Keep reading function 2 till all the input is received */
 469             int times;
 470
 471             c->function = i;
 472             c->flags = KVM_CPUID_FLAG_STATEFUL_FUNC |
 473                        KVM_CPUID_FLAG_STATE_READ_NEXT;
 474             cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx);
 475             times = c->eax & 0xff;
 476
 477             for (j = 1; j < times; ++j) {
 478                 c = &cpuid_data.entries[cpuid_i++];
 479                 c->function = i;
 480                 c->flags = KVM_CPUID_FLAG_STATEFUL_FUNC;
 481                 cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx);
 482             }
 483             break;
 484         }
 485         case 4:
 486         case 0xb:
 487         case 0xd:
 488             for (j = 0; ; j++) {
 489                 if (i == 0xd && j == 64) {
 490                     break;
 491                 }
 492                 c->function = i;
 493                 c->flags = KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
 494                 c->index = j;
 495                 cpu_x86_cpuid(env, i, j, &c->eax, &c->ebx, &c->ecx, &c->edx);
 496
 497                 if (i == 4 && c->eax == 0) {
 498                     break;
 499                 }
 500                 if (i == 0xb && !(c->ecx & 0xff00)) {
 501                     break;
 502                 }
 503                 if (i == 0xd && c->eax == 0) {
 504                     continue;
 505                 }
 506                 c = &cpuid_data.entries[cpuid_i++];
 507             }
 508             break;
 509         default:
 510             c->function = i;
 511             c->flags = 0;
 512             cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx);
 513             break;
 514         }
 515     }
 516     cpu_x86_cpuid(env, 0x80000000, 0, &limit, &unused, &unused, &unused);
 517
 518     for (i = 0x80000000; i <= limit; i++) {
 519         c = &cpuid_data.entries[cpuid_i++];
 520
 521         c->function = i;
 522         c->flags = 0;
 523         cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx);
 524     }
 525
 526     /* Call Centaur's CPUID instructions they are supported. */
 527     if (env->cpuid_xlevel2 > 0) {
 528         env->cpuid_ext4_features &=
 529             kvm_arch_get_supported_cpuid(s, 0xC0000001, 0, R_EDX);
 530         cpu_x86_cpuid(env, 0xC0000000, 0, &limit, &unused, &unused, &unused);
 531
 532         for (i = 0xC0000000; i <= limit; i++) {
 533             c = &cpuid_data.entries[cpuid_i++];
 534
 535             c->function = i;
 536             c->flags = 0;
 537             cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx);
 538         }
 539     }
 540
 541     cpuid_data.cpuid.nent = cpuid_i;
 542
 543     if (((env->cpuid_version >> 8)&0xF) >= 6
 544         && (env->cpuid_features&(CPUID_MCE|CPUID_MCA)) == (CPUID_MCE|CPUID_MCA)
 545         && kvm_check_extension(env->kvm_state, KVM_CAP_MCE) > 0) {
 546         uint64_t mcg_cap;
 547         int banks;
 548         int ret;
 549
 550         ret = kvm_get_mce_cap_supported(env->kvm_state, &mcg_cap, &banks);
 551         if (ret < 0) {
 552             fprintf(stderr, "kvm_get_mce_cap_supported: %s", strerror(-ret));
 553             return ret;
 554         }
 555
 556         if (banks > MCE_BANKS_DEF) {
 557             banks = MCE_BANKS_DEF;
 558         }
 559         mcg_cap &= MCE_CAP_DEF;
 560         mcg_cap |= banks;
 561         ret = kvm_vcpu_ioctl(env, KVM_X86_SETUP_MCE, &mcg_cap);
 562         if (ret < 0) {
 563             fprintf(stderr, "KVM_X86_SETUP_MCE: %s", strerror(-ret));
 564             return ret;
 565         }
 566
 567         env->mcg_cap = mcg_cap;
 568     }
 569
 570     qemu_add_vm_change_state_handler(cpu_update_state, env);
 571
 572     cpuid_data.cpuid.padding = 0;
 573     r = kvm_vcpu_ioctl(env, KVM_SET_CPUID2, &cpuid_data);
 574     if (r) {
 575         return r;
 576     }
 577
 578     r = kvm_check_extension(env->kvm_state, KVM_CAP_TSC_CONTROL);
 579     if (r && env->tsc_khz) {
 580         r = kvm_vcpu_ioctl(env, KVM_SET_TSC_KHZ, env->tsc_khz);
 581         if (r < 0) {
 582             fprintf(stderr, "KVM_SET_TSC_KHZ failed\n");
 583             return r;
 584         }
 585     }
 586
 587     if (kvm_has_xsave()) {
 588         env->kvm_xsave_buf = qemu_memalign(4096, sizeof(struct kvm_xsave));
 589     }
 590
 591     return 0;
 592 }
 593
 594 void kvm_arch_reset_vcpu(CPUX86State *env)
 595 {
 596     X86CPU *cpu = x86_env_get_cpu(env);
 597
 598     env->exception_injected = -1;
 599     env->interrupt_injected = -1;
 600     env->xcr0 = 1;
 601     if (kvm_irqchip_in_kernel()) {
 602         env->mp_state = cpu_is_bsp(cpu) ? KVM_MP_STATE_RUNNABLE :
 603                                           KVM_MP_STATE_UNINITIALIZED;
 604     } else {
 605         env->mp_state = KVM_MP_STATE_RUNNABLE;
 606     }
 607 }
 608
 609 static int kvm_get_supported_msrs(KVMState *s)
 610 {
 611     static int kvm_supported_msrs;
 612     int ret = 0;
 613
 614     /* first time */
 615     if (kvm_supported_msrs == 0) {
 616         struct kvm_msr_list msr_list, *kvm_msr_list;
 617
 618         kvm_supported_msrs = -1;
 619
 620         /* Obtain MSR list from KVM.  These are the MSRs that we must
 621          * save/restore */
 622         msr_list.nmsrs = 0;
 623         ret = kvm_ioctl(s, KVM_GET_MSR_INDEX_LIST, &msr_list);
 624         if (ret < 0 && ret != -E2BIG) {
 625             return ret;
 626         }
 627         /* Old kernel modules had a bug and could write beyond the provided
 628            memory. Allocate at least a safe amount of 1K. */
 629         kvm_msr_list = g_malloc0(MAX(1024, sizeof(msr_list) +
 630                                               msr_list.nmsrs *
 631                                               sizeof(msr_list.indices[0])));
 632
 633         kvm_msr_list->nmsrs = msr_list.nmsrs;
 634         ret = kvm_ioctl(s, KVM_GET_MSR_INDEX_LIST, kvm_msr_list);
 635         if (ret >= 0) {
 636             int i;
 637
 638             for (i = 0; i < kvm_msr_list->nmsrs; i++) {
 639                 if (kvm_msr_list->indices[i] == MSR_STAR) {
 640                     has_msr_star = true;
 641                     continue;
 642                 }
 643                 if (kvm_msr_list->indices[i] == MSR_VM_HSAVE_PA) {
 644                     has_msr_hsave_pa = true;
 645                     continue;
 646                 }
 647                 if (kvm_msr_list->indices[i] == MSR_IA32_TSCDEADLINE) {
 648                     has_msr_tsc_deadline = true;
 649                     continue;
 650                 }
 651                 if (kvm_msr_list->indices[i] == MSR_IA32_MISC_ENABLE) {
 652                     has_msr_misc_enable = true;
 653                     continue;
 654                 }
 655             }
 656         }
 657
 658         g_free(kvm_msr_list);
 659     }
 660
 661     return ret;
 662 }
 663
 664 int kvm_arch_init(KVMState *s)
 665 {
 666     QemuOptsList *list = qemu_find_opts("machine");
 667     uint64_t identity_base = 0xfffbc000;
 668     uint64_t shadow_mem;
 669     int ret;
 670     struct utsname utsname;
 671
 672     ret = kvm_get_supported_msrs(s);
 673     if (ret < 0) {
 674         return ret;
 675     }
 676
 677     uname(&utsname);
 678     lm_capable_kernel = strcmp(utsname.machine, "x86_64") == 0;
 679
 680     /*
 681      * On older Intel CPUs, KVM uses vm86 mode to emulate 16-bit code directly.
 682      * In order to use vm86 mode, an EPT identity map and a TSS  are needed.
 683      * Since these must be part of guest physical memory, we need to allocate
 684      * them, both by setting their start addresses in the kernel and by
 685      * creating a corresponding e820 entry. We need 4 pages before the BIOS.
 686      *
 687      * Older KVM versions may not support setting the identity map base. In
 688      * that case we need to stick with the default, i.e. a 256K maximum BIOS
 689      * size.
 690      */
 691     if (kvm_check_extension(s, KVM_CAP_SET_IDENTITY_MAP_ADDR)) {
 692         /* Allows up to 16M BIOSes. */
 693         identity_base = 0xfeffc000;
 694
 695         ret = kvm_vm_ioctl(s, KVM_SET_IDENTITY_MAP_ADDR, &identity_base);
 696         if (ret < 0) {
 697             return ret;
 698         }
 699     }
 700
 701     /* Set TSS base one page after EPT identity map. */
 702     ret = kvm_vm_ioctl(s, KVM_SET_TSS_ADDR, identity_base + 0x1000);
 703     if (ret < 0) {
 704         return ret;
 705     }
 706
 707     /* Tell fw_cfg to notify the BIOS to reserve the range. */
 708     ret = e820_add_entry(identity_base, 0x4000, E820_RESERVED);
 709     if (ret < 0) {
 710         fprintf(stderr, "e820_add_entry() table is full\n");
 711         return ret;
 712     }
 713     qemu_register_reset(kvm_unpoison_all, NULL);
 714
 715     if (!QTAILQ_EMPTY(&list->head)) {
 716         shadow_mem = qemu_opt_get_size(QTAILQ_FIRST(&list->head),
 717                                        "kvm_shadow_mem", -1);
 718         if (shadow_mem != -1) {
 719             shadow_mem /= 4096;
 720             ret = kvm_vm_ioctl(s, KVM_SET_NR_MMU_PAGES, shadow_mem);
 721             if (ret < 0) {
 722                 return ret;
 723             }
 724         }
 725     }
 726     return 0;
 727 }
 728
 729 static void set_v8086_seg(struct kvm_segment *lhs, const SegmentCache *rhs)
 730 {
 731     lhs->selector = rhs->selector;
 732     lhs->base = rhs->base;
 733     lhs->limit = rhs->limit;
 734     lhs->type = 3;
 735     lhs->present = 1;
 736     lhs->dpl = 3;
 737     lhs->db = 0;
 738     lhs->s = 1;
 739     lhs->l = 0;
 740     lhs->g = 0;
 741     lhs->avl = 0;
 742     lhs->unusable = 0;
 743 }
 744
 745 static void set_seg(struct kvm_segment *lhs, const SegmentCache *rhs)
 746 {
 747     unsigned flags = rhs->flags;
 748     lhs->selector = rhs->selector;
 749     lhs->base = rhs->base;
 750     lhs->limit = rhs->limit;
 751     lhs->type = (flags >> DESC_TYPE_SHIFT) & 15;
 752     lhs->present = (flags & DESC_P_MASK) != 0;
 753     lhs->dpl = (flags >> DESC_DPL_SHIFT) & 3;
 754     lhs->db = (flags >> DESC_B_SHIFT) & 1;
 755     lhs->s = (flags & DESC_S_MASK) != 0;
 756     lhs->l = (flags >> DESC_L_SHIFT) & 1;
 757     lhs->g = (flags & DESC_G_MASK) != 0;
 758     lhs->avl = (flags & DESC_AVL_MASK) != 0;
 759     lhs->unusable = 0;
 760     lhs->padding = 0;
 761 }
 762
 763 static void get_seg(SegmentCache *lhs, const struct kvm_segment *rhs)
 764 {
 765     lhs->selector = rhs->selector;
 766     lhs->base = rhs->base;
 767     lhs->limit = rhs->limit;
 768     lhs->flags = (rhs->type << DESC_TYPE_SHIFT) |
 769                  (rhs->present * DESC_P_MASK) |
 770                  (rhs->dpl << DESC_DPL_SHIFT) |
 771                  (rhs->db << DESC_B_SHIFT) |
 772                  (rhs->s * DESC_S_MASK) |
 773                  (rhs->l << DESC_L_SHIFT) |
 774                  (rhs->g * DESC_G_MASK) |
 775                  (rhs->avl * DESC_AVL_MASK);
 776 }
 777
 778 static void kvm_getput_reg(__u64 *kvm_reg, target_ulong *qemu_reg, int set)
 779 {
 780     if (set) {
 781         *kvm_reg = *qemu_reg;
 782     } else {
 783         *qemu_reg = *kvm_reg;
 784     }
 785 }
 786
 787 static int kvm_getput_regs(CPUX86State *env, int set)
 788 {
 789     struct kvm_regs regs;
 790     int ret = 0;
 791
 792     if (!set) {
 793         ret = kvm_vcpu_ioctl(env, KVM_GET_REGS, &regs);
 794         if (ret < 0) {
 795             return ret;
 796         }
 797     }
 798
 799     kvm_getput_reg(&regs.rax, &env->regs[R_EAX], set);
 800     kvm_getput_reg(&regs.rbx, &env->regs[R_EBX], set);
 801     kvm_getput_reg(&regs.rcx, &env->regs[R_ECX], set);
 802     kvm_getput_reg(&regs.rdx, &env->regs[R_EDX], set);
 803     kvm_getput_reg(&regs.rsi, &env->regs[R_ESI], set);
 804     kvm_getput_reg(&regs.rdi, &env->regs[R_EDI], set);
 805     kvm_getput_reg(&regs.rsp, &env->regs[R_ESP], set);
 806     kvm_getput_reg(&regs.rbp, &env->regs[R_EBP], set);
 807 #ifdef TARGET_X86_64
 808     kvm_getput_reg(&regs.r8, &env->regs[8], set);
 809     kvm_getput_reg(&regs.r9, &env->regs[9], set);
 810     kvm_getput_reg(&regs.r10, &env->regs[10], set);
 811     kvm_getput_reg(&regs.r11, &env->regs[11], set);
 812     kvm_getput_reg(&regs.r12, &env->regs[12], set);
 813     kvm_getput_reg(&regs.r13, &env->regs[13], set);
 814     kvm_getput_reg(&regs.r14, &env->regs[14], set);
 815     kvm_getput_reg(&regs.r15, &env->regs[15], set);
 816 #endif
 817
 818     kvm_getput_reg(&regs.rflags, &env->eflags, set);
 819     kvm_getput_reg(&regs.rip, &env->eip, set);
 820
 821     if (set) {
 822         ret = kvm_vcpu_ioctl(env, KVM_SET_REGS, &regs);
 823     }
 824
 825     return ret;
 826 }
 827
 828 static int kvm_put_fpu(CPUX86State *env)
 829 {
 830     struct kvm_fpu fpu;
 831     int i;
 832
 833     memset(&fpu, 0, sizeof fpu);
 834     fpu.fsw = env->fpus & ~(7 << 11);
 835     fpu.fsw |= (env->fpstt & 7) << 11;
 836     fpu.fcw = env->fpuc;
 837     fpu.last_opcode = env->fpop;
 838     fpu.last_ip = env->fpip;
 839     fpu.last_dp = env->fpdp;
 840     for (i = 0; i < 8; ++i) {
 841         fpu.ftwx |= (!env->fptags[i]) << i;
 842     }
 843     memcpy(fpu.fpr, env->fpregs, sizeof env->fpregs);
 844     memcpy(fpu.xmm, env->xmm_regs, sizeof env->xmm_regs);
 845     fpu.mxcsr = env->mxcsr;
 846
 847     return kvm_vcpu_ioctl(env, KVM_SET_FPU, &fpu);
 848 }
 849
 850 #define XSAVE_FCW_FSW     0
 851 #define XSAVE_FTW_FOP     1
 852 #define XSAVE_CWD_RIP     2
 853 #define XSAVE_CWD_RDP     4
 854 #define XSAVE_MXCSR       6
 855 #define XSAVE_ST_SPACE    8
 856 #define XSAVE_XMM_SPACE   40
 857 #define XSAVE_XSTATE_BV   128
 858 #define XSAVE_YMMH_SPACE  144
 859
 860 static int kvm_put_xsave(CPUX86State *env)
 861 {
 862     struct kvm_xsave* xsave = env->kvm_xsave_buf;
 863     uint16_t cwd, swd, twd;
 864     int i, r;
 865
 866     if (!kvm_has_xsave()) {
 867         return kvm_put_fpu(env);
 868     }
 869
 870     memset(xsave, 0, sizeof(struct kvm_xsave));
 871     twd = 0;
 872     swd = env->fpus & ~(7 << 11);
 873     swd |= (env->fpstt & 7) << 11;
 874     cwd = env->fpuc;
 875     for (i = 0; i < 8; ++i) {
 876         twd |= (!env->fptags[i]) << i;
 877     }
 878     xsave->region[XSAVE_FCW_FSW] = (uint32_t)(swd << 16) + cwd;
 879     xsave->region[XSAVE_FTW_FOP] = (uint32_t)(env->fpop << 16) + twd;
 880     memcpy(&xsave->region[XSAVE_CWD_RIP], &env->fpip, sizeof(env->fpip));
 881     memcpy(&xsave->region[XSAVE_CWD_RDP], &env->fpdp, sizeof(env->fpdp));
 882     memcpy(&xsave->region[XSAVE_ST_SPACE], env->fpregs,
 883             sizeof env->fpregs);
 884     memcpy(&xsave->region[XSAVE_XMM_SPACE], env->xmm_regs,
 885             sizeof env->xmm_regs);
 886     xsave->region[XSAVE_MXCSR] = env->mxcsr;
 887     *(uint64_t *)&xsave->region[XSAVE_XSTATE_BV] = env->xstate_bv;
 888     memcpy(&xsave->region[XSAVE_YMMH_SPACE], env->ymmh_regs,
 889             sizeof env->ymmh_regs);
 890     r = kvm_vcpu_ioctl(env, KVM_SET_XSAVE, xsave);
 891     return r;
 892 }
 893
 894 static int kvm_put_xcrs(CPUX86State *env)
 895 {
 896     struct kvm_xcrs xcrs;
 897
 898     if (!kvm_has_xcrs()) {
 899         return 0;
 900     }
 901
 902     xcrs.nr_xcrs = 1;
 903     xcrs.flags = 0;
 904     xcrs.xcrs[0].xcr = 0;
 905     xcrs.xcrs[0].value = env->xcr0;
 906     return kvm_vcpu_ioctl(env, KVM_SET_XCRS, &xcrs);
 907 }
 908
 909 static int kvm_put_sregs(CPUX86State *env)
 910 {
 911     struct kvm_sregs sregs;
 912
 913     memset(sregs.interrupt_bitmap, 0, sizeof(sregs.interrupt_bitmap));
 914     if (env->interrupt_injected >= 0) {
 915         sregs.interrupt_bitmap[env->interrupt_injected / 64] |=
 916                 (uint64_t)1 << (env->interrupt_injected % 64);
 917     }
 918
 919     if ((env->eflags & VM_MASK)) {
 920         set_v8086_seg(&sregs.cs, &env->segs[R_CS]);
 921         set_v8086_seg(&sregs.ds, &env->segs[R_DS]);
 922         set_v8086_seg(&sregs.es, &env->segs[R_ES]);
 923         set_v8086_seg(&sregs.fs, &env->segs[R_FS]);
 924         set_v8086_seg(&sregs.gs, &env->segs[R_GS]);
 925         set_v8086_seg(&sregs.ss, &env->segs[R_SS]);
 926     } else {
 927         set_seg(&sregs.cs, &env->segs[R_CS]);
 928         set_seg(&sregs.ds, &env->segs[R_DS]);
 929         set_seg(&sregs.es, &env->segs[R_ES]);
 930         set_seg(&sregs.fs, &env->segs[R_FS]);
 931         set_seg(&sregs.gs, &env->segs[R_GS]);
 932         set_seg(&sregs.ss, &env->segs[R_SS]);
 933     }
 934
 935     set_seg(&sregs.tr, &env->tr);
 936     set_seg(&sregs.ldt, &env->ldt);
 937
 938     sregs.idt.limit = env->idt.limit;
 939     sregs.idt.base = env->idt.base;
 940     memset(sregs.idt.padding, 0, sizeof sregs.idt.padding);
 941     sregs.gdt.limit = env->gdt.limit;
 942     sregs.gdt.base = env->gdt.base;
 943     memset(sregs.gdt.padding, 0, sizeof sregs.gdt.padding);
 944
 945     sregs.cr0 = env->cr[0];
 946     sregs.cr2 = env->cr[2];
 947     sregs.cr3 = env->cr[3];
 948     sregs.cr4 = env->cr[4];
 949
 950     sregs.cr8 = cpu_get_apic_tpr(env->apic_state);
 951     sregs.apic_base = cpu_get_apic_base(env->apic_state);
 952
 953     sregs.efer = env->efer;
 954
 955     return kvm_vcpu_ioctl(env, KVM_SET_SREGS, &sregs);
 956 }
 957
 958 static void kvm_msr_entry_set(struct kvm_msr_entry *entry,
 959                               uint32_t index, uint64_t value)
 960 {
 961     entry->index = index;
 962     entry->data = value;
 963 }
 964
 965 static int kvm_put_msrs(CPUX86State *env, int level)
 966 {
 967     struct {
 968         struct kvm_msrs info;
 969         struct kvm_msr_entry entries[100];
 970     } msr_data;
 971     struct kvm_msr_entry *msrs = msr_data.entries;
 972     int n = 0;
 973
 974     kvm_msr_entry_set(&msrs[n++], MSR_IA32_SYSENTER_CS, env->sysenter_cs);
 975     kvm_msr_entry_set(&msrs[n++], MSR_IA32_SYSENTER_ESP, env->sysenter_esp);
 976     kvm_msr_entry_set(&msrs[n++], MSR_IA32_SYSENTER_EIP, env->sysenter_eip);
 977     kvm_msr_entry_set(&msrs[n++], MSR_PAT, env->pat);
 978     if (has_msr_star) {
 979         kvm_msr_entry_set(&msrs[n++], MSR_STAR, env->star);
 980     }
 981     if (has_msr_hsave_pa) {
 982         kvm_msr_entry_set(&msrs[n++], MSR_VM_HSAVE_PA, env->vm_hsave);
 983     }
 984     if (has_msr_tsc_deadline) {
 985         kvm_msr_entry_set(&msrs[n++], MSR_IA32_TSCDEADLINE, env->tsc_deadline);
 986     }
 987     if (has_msr_misc_enable) {
 988         kvm_msr_entry_set(&msrs[n++], MSR_IA32_MISC_ENABLE,
 989                           env->msr_ia32_misc_enable);
 990     }
 991 #ifdef TARGET_X86_64
 992     if (lm_capable_kernel) {
 993         kvm_msr_entry_set(&msrs[n++], MSR_CSTAR, env->cstar);
 994         kvm_msr_entry_set(&msrs[n++], MSR_KERNELGSBASE, env->kernelgsbase);
 995         kvm_msr_entry_set(&msrs[n++], MSR_FMASK, env->fmask);
 996         kvm_msr_entry_set(&msrs[n++], MSR_LSTAR, env->lstar);
 997     }
 998 #endif
 999     if (level == KVM_PUT_FULL_STATE) {
1000         /*
1001          * KVM is yet unable to synchronize TSC values of multiple VCPUs on
1002          * writeback. Until this is fixed, we only write the offset to SMP
1003          * guests after migration, desynchronizing the VCPUs, but avoiding
1004          * huge jump-backs that would occur without any writeback at all.
1005          */
1006         if (smp_cpus == 1 || env->tsc != 0) {
1007             kvm_msr_entry_set(&msrs[n++], MSR_IA32_TSC, env->tsc);
1008         }
1009     }
1010     /*
1011      * The following paravirtual MSRs have side effects on the guest or are
1012      * too heavy for normal writeback. Limit them to reset or full state
1013      * updates.
1014      */
1015     if (level >= KVM_PUT_RESET_STATE) {
1016         kvm_msr_entry_set(&msrs[n++], MSR_KVM_SYSTEM_TIME,
1017                           env->system_time_msr);
1018         kvm_msr_entry_set(&msrs[n++], MSR_KVM_WALL_CLOCK, env->wall_clock_msr);
1019         if (has_msr_async_pf_en) {
1020             kvm_msr_entry_set(&msrs[n++], MSR_KVM_ASYNC_PF_EN,
1021                               env->async_pf_en_msr);
1022         }
1023         if (has_msr_pv_eoi_en) {
1024             kvm_msr_entry_set(&msrs[n++], MSR_KVM_PV_EOI_EN,
1025                               env->pv_eoi_en_msr);
1026         }
1027         if (hyperv_hypercall_available()) {
1028             kvm_msr_entry_set(&msrs[n++], HV_X64_MSR_GUEST_OS_ID, 0);
1029             kvm_msr_entry_set(&msrs[n++], HV_X64_MSR_HYPERCALL, 0);
1030         }
1031         if (hyperv_vapic_recommended()) {
1032             kvm_msr_entry_set(&msrs[n++], HV_X64_MSR_APIC_ASSIST_PAGE, 0);
1033         }
1034     }
1035     if (env->mcg_cap) {
1036         int i;
1037
1038         kvm_msr_entry_set(&msrs[n++], MSR_MCG_STATUS, env->mcg_status);
1039         kvm_msr_entry_set(&msrs[n++], MSR_MCG_CTL, env->mcg_ctl);
1040         for (i = 0; i < (env->mcg_cap & 0xff) * 4; i++) {
1041             kvm_msr_entry_set(&msrs[n++], MSR_MC0_CTL + i, env->mce_banks[i]);
1042         }
1043     }
1044
1045     msr_data.info.nmsrs = n;
1046
1047     return kvm_vcpu_ioctl(env, KVM_SET_MSRS, &msr_data);
1048
1049 }
1050
1051
1052 static int kvm_get_fpu(CPUX86State *env)
1053 {
1054     struct kvm_fpu fpu;
1055     int i, ret;
1056
1057     ret = kvm_vcpu_ioctl(env, KVM_GET_FPU, &fpu);
1058     if (ret < 0) {
1059         return ret;
1060     }
1061
1062     env->fpstt = (fpu.fsw >> 11) & 7;
1063     env->fpus = fpu.fsw;
1064     env->fpuc = fpu.fcw;
1065     env->fpop = fpu.last_opcode;
1066     env->fpip = fpu.last_ip;
1067     env->fpdp = fpu.last_dp;
1068     for (i = 0; i < 8; ++i) {
1069         env->fptags[i] = !((fpu.ftwx >> i) & 1);
1070     }
1071     memcpy(env->fpregs, fpu.fpr, sizeof env->fpregs);
1072     memcpy(env->xmm_regs, fpu.xmm, sizeof env->xmm_regs);
1073     env->mxcsr = fpu.mxcsr;
1074
1075     return 0;
1076 }
1077
1078 static int kvm_get_xsave(CPUX86State *env)
1079 {
1080     struct kvm_xsave* xsave = env->kvm_xsave_buf;
1081     int ret, i;
1082     uint16_t cwd, swd, twd;
1083
1084     if (!kvm_has_xsave()) {
1085         return kvm_get_fpu(env);
1086     }
1087
1088     ret = kvm_vcpu_ioctl(env, KVM_GET_XSAVE, xsave);
1089     if (ret < 0) {
1090         return ret;
1091     }
1092
1093     cwd = (uint16_t)xsave->region[XSAVE_FCW_FSW];
1094     swd = (uint16_t)(xsave->region[XSAVE_FCW_FSW] >> 16);
1095     twd = (uint16_t)xsave->region[XSAVE_FTW_FOP];
1096     env->fpop = (uint16_t)(xsave->region[XSAVE_FTW_FOP] >> 16);
1097     env->fpstt = (swd >> 11) & 7;
1098     env->fpus = swd;
1099     env->fpuc = cwd;
1100     for (i = 0; i < 8; ++i) {
1101         env->fptags[i] = !((twd >> i) & 1);
1102     }
1103     memcpy(&env->fpip, &xsave->region[XSAVE_CWD_RIP], sizeof(env->fpip));
1104     memcpy(&env->fpdp, &xsave->region[XSAVE_CWD_RDP], sizeof(env->fpdp));
1105     env->mxcsr = xsave->region[XSAVE_MXCSR];
1106     memcpy(env->fpregs, &xsave->region[XSAVE_ST_SPACE],
1107             sizeof env->fpregs);
1108     memcpy(env->xmm_regs, &xsave->region[XSAVE_XMM_SPACE],
1109             sizeof env->xmm_regs);
1110     env->xstate_bv = *(uint64_t *)&xsave->region[XSAVE_XSTATE_BV];
1111     memcpy(env->ymmh_regs, &xsave->region[XSAVE_YMMH_SPACE],
1112             sizeof env->ymmh_regs);
1113     return 0;
1114 }
1115
1116 static int kvm_get_xcrs(CPUX86State *env)
1117 {
1118     int i, ret;
1119     struct kvm_xcrs xcrs;
1120
1121     if (!kvm_has_xcrs()) {
1122         return 0;
1123     }
1124
1125     ret = kvm_vcpu_ioctl(env, KVM_GET_XCRS, &xcrs);
1126     if (ret < 0) {
1127         return ret;
1128     }
1129
1130     for (i = 0; i < xcrs.nr_xcrs; i++) {
1131         /* Only support xcr0 now */
1132         if (xcrs.xcrs[0].xcr == 0) {
1133             env->xcr0 = xcrs.xcrs[0].value;
1134             break;
1135         }
1136     }
1137     return 0;
1138 }
1139
1140 static int kvm_get_sregs(CPUX86State *env)
1141 {
1142     struct kvm_sregs sregs;
1143     uint32_t hflags;
1144     int bit, i, ret;
1145
1146     ret = kvm_vcpu_ioctl(env, KVM_GET_SREGS, &sregs);
1147     if (ret < 0) {
1148         return ret;
1149     }
1150
1151     /* There can only be one pending IRQ set in the bitmap at a time, so try
1152        to find it and save its number instead (-1 for none). */
1153     env->interrupt_injected = -1;
1154     for (i = 0; i < ARRAY_SIZE(sregs.interrupt_bitmap); i++) {
1155         if (sregs.interrupt_bitmap[i]) {
1156             bit = ctz64(sregs.interrupt_bitmap[i]);
1157             env->interrupt_injected = i * 64 + bit;
1158             break;
1159         }
1160     }
1161
1162     get_seg(&env->segs[R_CS], &sregs.cs);
1163     get_seg(&env->segs[R_DS], &sregs.ds);
1164     get_seg(&env->segs[R_ES], &sregs.es);
1165     get_seg(&env->segs[R_FS], &sregs.fs);
1166     get_seg(&env->segs[R_GS], &sregs.gs);
1167     get_seg(&env->segs[R_SS], &sregs.ss);
1168
1169     get_seg(&env->tr, &sregs.tr);
1170     get_seg(&env->ldt, &sregs.ldt);
1171
1172     env->idt.limit = sregs.idt.limit;
1173     env->idt.base = sregs.idt.base;
1174     env->gdt.limit = sregs.gdt.limit;
1175     env->gdt.base = sregs.gdt.base;
1176
1177     env->cr[0] = sregs.cr0;
1178     env->cr[2] = sregs.cr2;
1179     env->cr[3] = sregs.cr3;
1180     env->cr[4] = sregs.cr4;
1181
1182     env->efer = sregs.efer;
1183
1184     /* changes to apic base and cr8/tpr are read back via kvm_arch_post_run */
1185
1186 #define HFLAG_COPY_MASK \
1187     ~( HF_CPL_MASK | HF_PE_MASK | HF_MP_MASK | HF_EM_MASK | \
1188        HF_TS_MASK | HF_TF_MASK | HF_VM_MASK | HF_IOPL_MASK | \
1189        HF_OSFXSR_MASK | HF_LMA_MASK | HF_CS32_MASK | \
1190        HF_SS32_MASK | HF_CS64_MASK | HF_ADDSEG_MASK)
1191
1192     hflags = (env->segs[R_CS].flags >> DESC_DPL_SHIFT) & HF_CPL_MASK;
1193     hflags |= (env->cr[0] & CR0_PE_MASK) << (HF_PE_SHIFT - CR0_PE_SHIFT);
1194     hflags |= (env->cr[0] << (HF_MP_SHIFT - CR0_MP_SHIFT)) &
1195                 (HF_MP_MASK | HF_EM_MASK | HF_TS_MASK);
1196     hflags |= (env->eflags & (HF_TF_MASK | HF_VM_MASK | HF_IOPL_MASK));
1197     hflags |= (env->cr[4] & CR4_OSFXSR_MASK) <<
1198                 (HF_OSFXSR_SHIFT - CR4_OSFXSR_SHIFT);
1199
1200     if (env->efer & MSR_EFER_LMA) {
1201         hflags |= HF_LMA_MASK;
1202     }
1203
1204     if ((hflags & HF_LMA_MASK) && (env->segs[R_CS].flags & DESC_L_MASK)) {
1205         hflags |= HF_CS32_MASK | HF_SS32_MASK | HF_CS64_MASK;
1206     } else {
1207         hflags |= (env->segs[R_CS].flags & DESC_B_MASK) >>
1208                     (DESC_B_SHIFT - HF_CS32_SHIFT);
1209         hflags |= (env->segs[R_SS].flags & DESC_B_MASK) >>
1210                     (DESC_B_SHIFT - HF_SS32_SHIFT);
1211         if (!(env->cr[0] & CR0_PE_MASK) || (env->eflags & VM_MASK) ||
1212             !(hflags & HF_CS32_MASK)) {
1213             hflags |= HF_ADDSEG_MASK;
1214         } else {
1215             hflags |= ((env->segs[R_DS].base | env->segs[R_ES].base |
1216                         env->segs[R_SS].base) != 0) << HF_ADDSEG_SHIFT;
1217         }
1218     }
1219     env->hflags = (env->hflags & HFLAG_COPY_MASK) | hflags;
1220
1221     return 0;
1222 }
1223
1224 static int kvm_get_msrs(CPUX86State *env)
1225 {
1226     struct {
1227         struct kvm_msrs info;
1228         struct kvm_msr_entry entries[100];
1229     } msr_data;
1230     struct kvm_msr_entry *msrs = msr_data.entries;
1231     int ret, i, n;
1232
1233     n = 0;
1234     msrs[n++].index = MSR_IA32_SYSENTER_CS;
1235     msrs[n++].index = MSR_IA32_SYSENTER_ESP;
1236     msrs[n++].index = MSR_IA32_SYSENTER_EIP;
1237     msrs[n++].index = MSR_PAT;
1238     if (has_msr_star) {
1239         msrs[n++].index = MSR_STAR;
1240     }
1241     if (has_msr_hsave_pa) {
1242         msrs[n++].index = MSR_VM_HSAVE_PA;
1243     }
1244     if (has_msr_tsc_deadline) {
1245         msrs[n++].index = MSR_IA32_TSCDEADLINE;
1246     }
1247     if (has_msr_misc_enable) {
1248         msrs[n++].index = MSR_IA32_MISC_ENABLE;
1249     }
1250
1251     if (!env->tsc_valid) {
1252         msrs[n++].index = MSR_IA32_TSC;
1253         env->tsc_valid = !runstate_is_running();
1254     }
1255
1256 #ifdef TARGET_X86_64
1257     if (lm_capable_kernel) {
1258         msrs[n++].index = MSR_CSTAR;
1259         msrs[n++].index = MSR_KERNELGSBASE;
1260         msrs[n++].index = MSR_FMASK;
1261         msrs[n++].index = MSR_LSTAR;
1262     }
1263 #endif
1264     msrs[n++].index = MSR_KVM_SYSTEM_TIME;
1265     msrs[n++].index = MSR_KVM_WALL_CLOCK;
1266     if (has_msr_async_pf_en) {
1267         msrs[n++].index = MSR_KVM_ASYNC_PF_EN;
1268     }
1269     if (has_msr_pv_eoi_en) {
1270         msrs[n++].index = MSR_KVM_PV_EOI_EN;
1271     }
1272
1273     if (env->mcg_cap) {
1274         msrs[n++].index = MSR_MCG_STATUS;
1275         msrs[n++].index = MSR_MCG_CTL;
1276         for (i = 0; i < (env->mcg_cap & 0xff) * 4; i++) {
1277             msrs[n++].index = MSR_MC0_CTL + i;
1278         }
1279     }
1280
1281     msr_data.info.nmsrs = n;
1282     ret = kvm_vcpu_ioctl(env, KVM_GET_MSRS, &msr_data);
1283     if (ret < 0) {
1284         return ret;
1285     }
1286
1287     for (i = 0; i < ret; i++) {
1288         switch (msrs[i].index) {
1289         case MSR_IA32_SYSENTER_CS:
1290             env->sysenter_cs = msrs[i].data;
1291             break;
1292         case MSR_IA32_SYSENTER_ESP:
1293             env->sysenter_esp = msrs[i].data;
1294             break;
1295         case MSR_IA32_SYSENTER_EIP:
1296             env->sysenter_eip = msrs[i].data;
1297             break;
1298         case MSR_PAT:
1299             env->pat = msrs[i].data;
1300             break;
1301         case MSR_STAR:
1302             env->star = msrs[i].data;
1303             break;
1304 #ifdef TARGET_X86_64
1305         case MSR_CSTAR:
1306             env->cstar = msrs[i].data;
1307             break;
1308         case MSR_KERNELGSBASE:
1309             env->kernelgsbase = msrs[i].data;
1310             break;
1311         case MSR_FMASK:
1312             env->fmask = msrs[i].data;
1313             break;
1314         case MSR_LSTAR:
1315             env->lstar = msrs[i].data;
1316             break;
1317 #endif
1318         case MSR_IA32_TSC:
1319             env->tsc = msrs[i].data;
1320             break;
1321         case MSR_IA32_TSCDEADLINE:
1322             env->tsc_deadline = msrs[i].data;
1323             break;
1324         case MSR_VM_HSAVE_PA:
1325             env->vm_hsave = msrs[i].data;
1326             break;
1327         case MSR_KVM_SYSTEM_TIME:
1328             env->system_time_msr = msrs[i].data;
1329             break;
1330         case MSR_KVM_WALL_CLOCK:
1331             env->wall_clock_msr = msrs[i].data;
1332             break;
1333         case MSR_MCG_STATUS:
1334             env->mcg_status = msrs[i].data;
1335             break;
1336         case MSR_MCG_CTL:
1337             env->mcg_ctl = msrs[i].data;
1338             break;
1339         case MSR_IA32_MISC_ENABLE:
1340             env->msr_ia32_misc_enable = msrs[i].data;
1341             break;
1342         default:
1343             if (msrs[i].index >= MSR_MC0_CTL &&
1344                 msrs[i].index < MSR_MC0_CTL + (env->mcg_cap & 0xff) * 4) {
1345                 env->mce_banks[msrs[i].index - MSR_MC0_CTL] = msrs[i].data;
1346             }
1347             break;
1348         case MSR_KVM_ASYNC_PF_EN:
1349             env->async_pf_en_msr = msrs[i].data;
1350             break;
1351         case MSR_KVM_PV_EOI_EN:
1352             env->pv_eoi_en_msr = msrs[i].data;
1353             break;
1354         }
1355     }
1356
1357     return 0;
1358 }
1359
1360 static int kvm_put_mp_state(CPUX86State *env)
1361 {
1362     struct kvm_mp_state mp_state = { .mp_state = env->mp_state };
1363
1364     return kvm_vcpu_ioctl(env, KVM_SET_MP_STATE, &mp_state);
1365 }
1366
1367 static int kvm_get_mp_state(CPUX86State *env)
1368 {
1369     struct kvm_mp_state mp_state;
1370     int ret;
1371
1372     ret = kvm_vcpu_ioctl(env, KVM_GET_MP_STATE, &mp_state);
1373     if (ret < 0) {
1374         return ret;
1375     }
1376     env->mp_state = mp_state.mp_state;
1377     if (kvm_irqchip_in_kernel()) {
1378         env->halted = (mp_state.mp_state == KVM_MP_STATE_HALTED);
1379     }
1380     return 0;
1381 }
1382
1383 static int kvm_get_apic(CPUX86State *env)
1384 {
1385     DeviceState *apic = env->apic_state;
1386     struct kvm_lapic_state kapic;
1387     int ret;
1388
1389     if (apic && kvm_irqchip_in_kernel()) {
1390         ret = kvm_vcpu_ioctl(env, KVM_GET_LAPIC, &kapic);
1391         if (ret < 0) {
1392             return ret;
1393         }
1394
1395         kvm_get_apic_state(apic, &kapic);
1396     }
1397     return 0;
1398 }
1399
1400 static int kvm_put_apic(CPUX86State *env)
1401 {
1402     DeviceState *apic = env->apic_state;
1403     struct kvm_lapic_state kapic;
1404
1405     if (apic && kvm_irqchip_in_kernel()) {
1406         kvm_put_apic_state(apic, &kapic);
1407
1408         return kvm_vcpu_ioctl(env, KVM_SET_LAPIC, &kapic);
1409     }
1410     return 0;
1411 }
1412
1413 static int kvm_put_vcpu_events(CPUX86State *env, int level)
1414 {
1415     struct kvm_vcpu_events events;
1416
1417     if (!kvm_has_vcpu_events()) {
1418         return 0;
1419     }
1420
1421     events.exception.injected = (env->exception_injected >= 0);
1422     events.exception.nr = env->exception_injected;
1423     events.exception.has_error_code = env->has_error_code;
1424     events.exception.error_code = env->error_code;
1425     events.exception.pad = 0;
1426
1427     events.interrupt.injected = (env->interrupt_injected >= 0);
1428     events.interrupt.nr = env->interrupt_injected;
1429     events.interrupt.soft = env->soft_interrupt;
1430
1431     events.nmi.injected = env->nmi_injected;
1432     events.nmi.pending = env->nmi_pending;
1433     events.nmi.masked = !!(env->hflags2 & HF2_NMI_MASK);
1434     events.nmi.pad = 0;
1435
1436     events.sipi_vector = env->sipi_vector;
1437
1438     events.flags = 0;
1439     if (level >= KVM_PUT_RESET_STATE) {
1440         events.flags |=
1441             KVM_VCPUEVENT_VALID_NMI_PENDING | KVM_VCPUEVENT_VALID_SIPI_VECTOR;
1442     }
1443
1444     return kvm_vcpu_ioctl(env, KVM_SET_VCPU_EVENTS, &events);
1445 }
1446
1447 static int kvm_get_vcpu_events(CPUX86State *env)
1448 {
1449     struct kvm_vcpu_events events;
1450     int ret;
1451
1452     if (!kvm_has_vcpu_events()) {
1453         return 0;
1454     }
1455
1456     ret = kvm_vcpu_ioctl(env, KVM_GET_VCPU_EVENTS, &events);
1457     if (ret < 0) {
1458        return ret;
1459     }
1460     env->exception_injected =
1461        events.exception.injected ? events.exception.nr : -1;
1462     env->has_error_code = events.exception.has_error_code;
1463     env->error_code = events.exception.error_code;
1464
1465     env->interrupt_injected =
1466         events.interrupt.injected ? events.interrupt.nr : -1;
1467     env->soft_interrupt = events.interrupt.soft;
1468
1469     env->nmi_injected = events.nmi.injected;
1470     env->nmi_pending = events.nmi.pending;
1471     if (events.nmi.masked) {
1472         env->hflags2 |= HF2_NMI_MASK;
1473     } else {
1474         env->hflags2 &= ~HF2_NMI_MASK;
1475     }
1476
1477     env->sipi_vector = events.sipi_vector;
1478
1479     return 0;
1480 }
1481
1482 static int kvm_guest_debug_workarounds(CPUX86State *env)
1483 {
1484     int ret = 0;
1485     unsigned long reinject_trap = 0;
1486
1487     if (!kvm_has_vcpu_events()) {
1488         if (env->exception_injected == 1) {
1489             reinject_trap = KVM_GUESTDBG_INJECT_DB;
1490         } else if (env->exception_injected == 3) {
1491             reinject_trap = KVM_GUESTDBG_INJECT_BP;
1492         }
1493         env->exception_injected = -1;
1494     }
1495
1496     /*
1497      * Kernels before KVM_CAP_X86_ROBUST_SINGLESTEP overwrote flags.TF
1498      * injected via SET_GUEST_DEBUG while updating GP regs. Work around this
1499      * by updating the debug state once again if single-stepping is on.
1500      * Another reason to call kvm_update_guest_debug here is a pending debug
1501      * trap raise by the guest. On kernels without SET_VCPU_EVENTS we have to
1502      * reinject them via SET_GUEST_DEBUG.
1503      */
1504     if (reinject_trap ||
1505         (!kvm_has_robust_singlestep() && env->singlestep_enabled)) {
1506         ret = kvm_update_guest_debug(env, reinject_trap);
1507     }
1508     return ret;
1509 }
1510
1511 static int kvm_put_debugregs(CPUX86State *env)
1512 {
1513     struct kvm_debugregs dbgregs;
1514     int i;
1515
1516     if (!kvm_has_debugregs()) {
1517         return 0;
1518     }
1519
1520     for (i = 0; i < 4; i++) {
1521         dbgregs.db[i] = env->dr[i];
1522     }
1523     dbgregs.dr6 = env->dr[6];
1524     dbgregs.dr7 = env->dr[7];
1525     dbgregs.flags = 0;
1526
1527     return kvm_vcpu_ioctl(env, KVM_SET_DEBUGREGS, &dbgregs);
1528 }
1529
1530 static int kvm_get_debugregs(CPUX86State *env)
1531 {
1532     struct kvm_debugregs dbgregs;
1533     int i, ret;
1534
1535     if (!kvm_has_debugregs()) {
1536         return 0;
1537     }
1538
1539     ret = kvm_vcpu_ioctl(env, KVM_GET_DEBUGREGS, &dbgregs);
1540     if (ret < 0) {
1541         return ret;
1542     }
1543     for (i = 0; i < 4; i++) {
1544         env->dr[i] = dbgregs.db[i];
1545     }
1546     env->dr[4] = env->dr[6] = dbgregs.dr6;
1547     env->dr[5] = env->dr[7] = dbgregs.dr7;
1548
1549     return 0;
1550 }
1551
1552 int kvm_arch_put_registers(CPUX86State *env, int level)
1553 {
1554     int ret;
1555
1556     assert(cpu_is_stopped(env) || qemu_cpu_is_self(env));
1557
1558     ret = kvm_getput_regs(env, 1);
1559     if (ret < 0) {
1560         return ret;
1561     }
1562     ret = kvm_put_xsave(env);
1563     if (ret < 0) {
1564         return ret;
1565     }
1566     ret = kvm_put_xcrs(env);
1567     if (ret < 0) {
1568         return ret;
1569     }
1570     ret = kvm_put_sregs(env);
1571     if (ret < 0) {
1572         return ret;
1573     }
1574     /* must be before kvm_put_msrs */
1575     ret = kvm_inject_mce_oldstyle(env);
1576     if (ret < 0) {
1577         return ret;
1578     }
1579     ret = kvm_put_msrs(env, level);
1580     if (ret < 0) {
1581         return ret;
1582     }
1583     if (level >= KVM_PUT_RESET_STATE) {
1584         ret = kvm_put_mp_state(env);
1585         if (ret < 0) {
1586             return ret;
1587         }
1588         ret = kvm_put_apic(env);
1589         if (ret < 0) {
1590             return ret;
1591         }
1592     }
1593     ret = kvm_put_vcpu_events(env, level);
1594     if (ret < 0) {
1595         return ret;
1596     }
1597     ret = kvm_put_debugregs(env);
1598     if (ret < 0) {
1599         return ret;
1600     }
1601     /* must be last */
1602     ret = kvm_guest_debug_workarounds(env);
1603     if (ret < 0) {
1604         return ret;
1605     }
1606     return 0;
1607 }
1608
1609 int kvm_arch_get_registers(CPUX86State *env)
1610 {
1611     int ret;
1612
1613     assert(cpu_is_stopped(env) || qemu_cpu_is_self(env));
1614
1615     ret = kvm_getput_regs(env, 0);
1616     if (ret < 0) {
1617         return ret;
1618     }
1619     ret = kvm_get_xsave(env);
1620     if (ret < 0) {
1621         return ret;
1622     }
1623     ret = kvm_get_xcrs(env);
1624     if (ret < 0) {
1625         return ret;
1626     }
1627     ret = kvm_get_sregs(env);
1628     if (ret < 0) {
1629         return ret;
1630     }
1631     ret = kvm_get_msrs(env);
1632     if (ret < 0) {
1633         return ret;
1634     }
1635     ret = kvm_get_mp_state(env);
1636     if (ret < 0) {
1637         return ret;
1638     }
1639     ret = kvm_get_apic(env);
1640     if (ret < 0) {
1641         return ret;
1642     }
1643     ret = kvm_get_vcpu_events(env);
1644     if (ret < 0) {
1645         return ret;
1646     }
1647     ret = kvm_get_debugregs(env);
1648     if (ret < 0) {
1649         return ret;
1650     }
1651     return 0;
1652 }
1653
1654 void kvm_arch_pre_run(CPUX86State *env, struct kvm_run *run)
1655 {
1656     int ret;
1657
1658     /* Inject NMI */
1659     if (env->interrupt_request & CPU_INTERRUPT_NMI) {
1660         env->interrupt_request &= ~CPU_INTERRUPT_NMI;
1661         DPRINTF("injected NMI\n");
1662         ret = kvm_vcpu_ioctl(env, KVM_NMI);
1663         if (ret < 0) {
1664             fprintf(stderr, "KVM: injection failed, NMI lost (%s)\n",
1665                     strerror(-ret));
1666         }
1667     }
1668
1669     if (!kvm_irqchip_in_kernel()) {
1670         /* Force the VCPU out of its inner loop to process any INIT requests
1671          * or pending TPR access reports. */
1672         if (env->interrupt_request &
1673             (CPU_INTERRUPT_INIT | CPU_INTERRUPT_TPR)) {
1674             env->exit_request = 1;
1675         }
1676
1677         /* Try to inject an interrupt if the guest can accept it */
1678         if (run->ready_for_interrupt_injection &&
1679             (env->interrupt_request & CPU_INTERRUPT_HARD) &&
1680             (env->eflags & IF_MASK)) {
1681             int irq;
1682
1683             env->interrupt_request &= ~CPU_INTERRUPT_HARD;
1684             irq = cpu_get_pic_interrupt(env);
1685             if (irq >= 0) {
1686                 struct kvm_interrupt intr;
1687
1688                 intr.irq = irq;
1689                 DPRINTF("injected interrupt %d\n", irq);
1690                 ret = kvm_vcpu_ioctl(env, KVM_INTERRUPT, &intr);
1691                 if (ret < 0) {
1692                     fprintf(stderr,
1693                             "KVM: injection failed, interrupt lost (%s)\n",
1694                             strerror(-ret));
1695                 }
1696             }
1697         }
1698
1699         /* If we have an interrupt but the guest is not ready to receive an
1700          * interrupt, request an interrupt window exit.  This will
1701          * cause a return to userspace as soon as the guest is ready to
1702          * receive interrupts. */
1703         if ((env->interrupt_request & CPU_INTERRUPT_HARD)) {
1704             run->request_interrupt_window = 1;
1705         } else {
1706             run->request_interrupt_window = 0;
1707         }
1708
1709         DPRINTF("setting tpr\n");
1710         run->cr8 = cpu_get_apic_tpr(env->apic_state);
1711     }
1712 }
1713
1714 void kvm_arch_post_run(CPUX86State *env, struct kvm_run *run)
1715 {
1716     if (run->if_flag) {
1717         env->eflags |= IF_MASK;
1718     } else {
1719         env->eflags &= ~IF_MASK;
1720     }
1721     cpu_set_apic_tpr(env->apic_state, run->cr8);
1722     cpu_set_apic_base(env->apic_state, run->apic_base);
1723 }
1724
1725 int kvm_arch_process_async_events(CPUX86State *env)
1726 {
1727     X86CPU *cpu = x86_env_get_cpu(env);
1728
1729     if (env->interrupt_request & CPU_INTERRUPT_MCE) {
1730         /* We must not raise CPU_INTERRUPT_MCE if it's not supported. */
1731         assert(env->mcg_cap);
1732
1733         env->interrupt_request &= ~CPU_INTERRUPT_MCE;
1734
1735         kvm_cpu_synchronize_state(env);
1736
1737         if (env->exception_injected == EXCP08_DBLE) {
1738             /* this means triple fault */
1739             qemu_system_reset_request();
1740             env->exit_request = 1;
1741             return 0;
1742         }
1743         env->exception_injected = EXCP12_MCHK;
1744         env->has_error_code = 0;
1745
1746         env->halted = 0;
1747         if (kvm_irqchip_in_kernel() && env->mp_state == KVM_MP_STATE_HALTED) {
1748             env->mp_state = KVM_MP_STATE_RUNNABLE;
1749         }
1750     }
1751
1752     if (kvm_irqchip_in_kernel()) {
1753         return 0;
1754     }
1755
1756     if (env->interrupt_request & CPU_INTERRUPT_POLL) {
1757         env->interrupt_request &= ~CPU_INTERRUPT_POLL;
1758         apic_poll_irq(env->apic_state);
1759     }
1760     if (((env->interrupt_request & CPU_INTERRUPT_HARD) &&
1761          (env->eflags & IF_MASK)) ||
1762         (env->interrupt_request & CPU_INTERRUPT_NMI)) {
1763         env->halted = 0;
1764     }
1765     if (env->interrupt_request & CPU_INTERRUPT_INIT) {
1766         kvm_cpu_synchronize_state(env);
1767         do_cpu_init(cpu);
1768     }
1769     if (env->interrupt_request & CPU_INTERRUPT_SIPI) {
1770         kvm_cpu_synchronize_state(env);
1771         do_cpu_sipi(cpu);
1772     }
1773     if (env->interrupt_request & CPU_INTERRUPT_TPR) {
1774         env->interrupt_request &= ~CPU_INTERRUPT_TPR;
1775         kvm_cpu_synchronize_state(env);
1776         apic_handle_tpr_access_report(env->apic_state, env->eip,
1777                                       env->tpr_access_type);
1778     }
1779
1780     return env->halted;
1781 }
1782
1783 static int kvm_handle_halt(CPUX86State *env)
1784 {
1785     if (!((env->interrupt_request & CPU_INTERRUPT_HARD) &&
1786           (env->eflags & IF_MASK)) &&
1787         !(env->interrupt_request & CPU_INTERRUPT_NMI)) {
1788         env->halted = 1;
1789         return EXCP_HLT;
1790     }
1791
1792     return 0;
1793 }
1794
1795 static int kvm_handle_tpr_access(CPUX86State *env)
1796 {
1797     struct kvm_run *run = env->kvm_run;
1798
1799     apic_handle_tpr_access_report(env->apic_state, run->tpr_access.rip,
1800                                   run->tpr_access.is_write ? TPR_ACCESS_WRITE
1801                                                            : TPR_ACCESS_READ);
1802     return 1;
1803 }
1804
1805 int kvm_arch_insert_sw_breakpoint(CPUX86State *env, struct kvm_sw_breakpoint *bp)
1806 {
1807     static const uint8_t int3 = 0xcc;
1808
1809     if (cpu_memory_rw_debug(env, bp->pc, (uint8_t *)&bp->saved_insn, 1, 0) ||
1810         cpu_memory_rw_debug(env, bp->pc, (uint8_t *)&int3, 1, 1)) {
1811         return -EINVAL;
1812     }
1813     return 0;
1814 }
1815
1816 int kvm_arch_remove_sw_breakpoint(CPUX86State *env, struct kvm_sw_breakpoint *bp)
1817 {
1818     uint8_t int3;
1819
1820     if (cpu_memory_rw_debug(env, bp->pc, &int3, 1, 0) || int3 != 0xcc ||
1821         cpu_memory_rw_debug(env, bp->pc, (uint8_t *)&bp->saved_insn, 1, 1)) {
1822         return -EINVAL;
1823     }
1824     return 0;
1825 }
1826
1827 static struct {
1828     target_ulong addr;
1829     int len;
1830     int type;
1831 } hw_breakpoint[4];
1832
1833 static int nb_hw_breakpoint;
1834
1835 static int find_hw_breakpoint(target_ulong addr, int len, int type)
1836 {
1837     int n;
1838
1839     for (n = 0; n < nb_hw_breakpoint; n++) {
1840         if (hw_breakpoint[n].addr == addr && hw_breakpoint[n].type == type &&
1841             (hw_breakpoint[n].len == len || len == -1)) {
1842             return n;
1843         }
1844     }
1845     return -1;
1846 }
1847
1848 int kvm_arch_insert_hw_breakpoint(target_ulong addr,
1849                                   target_ulong len, int type)
1850 {
1851     switch (type) {
1852     case GDB_BREAKPOINT_HW:
1853         len = 1;
1854         break;
1855     case GDB_WATCHPOINT_WRITE:
1856     case GDB_WATCHPOINT_ACCESS:
1857         switch (len) {
1858         case 1:
1859             break;
1860         case 2:
1861         case 4:
1862         case 8:
1863             if (addr & (len - 1)) {
1864                 return -EINVAL;
1865             }
1866             break;
1867         default:
1868             return -EINVAL;
1869         }
1870         break;
1871     default:
1872         return -ENOSYS;
1873     }
1874
1875     if (nb_hw_breakpoint == 4) {
1876         return -ENOBUFS;
1877     }
1878     if (find_hw_breakpoint(addr, len, type) >= 0) {
1879         return -EEXIST;
1880     }
1881     hw_breakpoint[nb_hw_breakpoint].addr = addr;
1882     hw_breakpoint[nb_hw_breakpoint].len = len;
1883     hw_breakpoint[nb_hw_breakpoint].type = type;
1884     nb_hw_breakpoint++;
1885
1886     return 0;
1887 }
1888
1889 int kvm_arch_remove_hw_breakpoint(target_ulong addr,
1890                                   target_ulong len, int type)
1891 {
1892     int n;
1893
1894     n = find_hw_breakpoint(addr, (type == GDB_BREAKPOINT_HW) ? 1 : len, type);
1895     if (n < 0) {
1896         return -ENOENT;
1897     }
1898     nb_hw_breakpoint--;
1899     hw_breakpoint[n] = hw_breakpoint[nb_hw_breakpoint];
1900
1901     return 0;
1902 }
1903
1904 void kvm_arch_remove_all_hw_breakpoints(void)
1905 {
1906     nb_hw_breakpoint = 0;
1907 }
1908
1909 static CPUWatchpoint hw_watchpoint;
1910
1911 static int kvm_handle_debug(struct kvm_debug_exit_arch *arch_info)
1912 {
1913     int ret = 0;
1914     int n;
1915
1916     if (arch_info->exception == 1) {
1917         if (arch_info->dr6 & (1 << 14)) {
1918             if (cpu_single_env->singlestep_enabled) {
1919                 ret = EXCP_DEBUG;
1920             }
1921         } else {
1922             for (n = 0; n < 4; n++) {
1923                 if (arch_info->dr6 & (1 << n)) {
1924                     switch ((arch_info->dr7 >> (16 + n*4)) & 0x3) {
1925                     case 0x0:
1926                         ret = EXCP_DEBUG;
1927                         break;
1928                     case 0x1:
1929                         ret = EXCP_DEBUG;
1930                         cpu_single_env->watchpoint_hit = &hw_watchpoint;
1931                         hw_watchpoint.vaddr = hw_breakpoint[n].addr;
1932                         hw_watchpoint.flags = BP_MEM_WRITE;
1933                         break;
1934                     case 0x3:
1935                         ret = EXCP_DEBUG;
1936                         cpu_single_env->watchpoint_hit = &hw_watchpoint;
1937                         hw_watchpoint.vaddr = hw_breakpoint[n].addr;
1938                         hw_watchpoint.flags = BP_MEM_ACCESS;
1939                         break;
1940                     }
1941                 }
1942             }
1943         }
1944     } else if (kvm_find_sw_breakpoint(cpu_single_env, arch_info->pc)) {
1945         ret = EXCP_DEBUG;
1946     }
1947     if (ret == 0) {
1948         cpu_synchronize_state(cpu_single_env);
1949         assert(cpu_single_env->exception_injected == -1);
1950
1951         /* pass to guest */
1952         cpu_single_env->exception_injected = arch_info->exception;
1953         cpu_single_env->has_error_code = 0;
1954     }
1955
1956     return ret;
1957 }
1958
1959 void kvm_arch_update_guest_debug(CPUX86State *env, struct kvm_guest_debug *dbg)
1960 {
1961     const uint8_t type_code[] = {
1962         [GDB_BREAKPOINT_HW] = 0x0,
1963         [GDB_WATCHPOINT_WRITE] = 0x1,
1964         [GDB_WATCHPOINT_ACCESS] = 0x3
1965     };
1966     const uint8_t len_code[] = {
1967         [1] = 0x0, [2] = 0x1, [4] = 0x3, [8] = 0x2
1968     };
1969     int n;
1970
1971     if (kvm_sw_breakpoints_active(env)) {
1972         dbg->control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP;
1973     }
1974     if (nb_hw_breakpoint > 0) {
1975         dbg->control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_HW_BP;
1976         dbg->arch.debugreg[7] = 0x0600;
1977         for (n = 0; n < nb_hw_breakpoint; n++) {
1978             dbg->arch.debugreg[n] = hw_breakpoint[n].addr;
1979             dbg->arch.debugreg[7] |= (2 << (n * 2)) |
1980                 (type_code[hw_breakpoint[n].type] << (16 + n*4)) |
1981                 ((uint32_t)len_code[hw_breakpoint[n].len] << (18 + n*4));
1982         }
1983     }
1984 }
1985
1986 static bool host_supports_vmx(void)
1987 {
1988     uint32_t ecx, unused;
1989
1990     host_cpuid(1, 0, &unused, &unused, &ecx, &unused);
1991     return ecx & CPUID_EXT_VMX;
1992 }
1993
1994 #define VMX_INVALID_GUEST_STATE 0x80000021
1995
1996 int kvm_arch_handle_exit(CPUX86State *env, struct kvm_run *run)
1997 {
1998     uint64_t code;
1999     int ret;
2000
2001     switch (run->exit_reason) {
2002     case KVM_EXIT_HLT:
2003         DPRINTF("handle_hlt\n");
2004         ret = kvm_handle_halt(env);
2005         break;
2006     case KVM_EXIT_SET_TPR:
2007         ret = 0;
2008         break;
2009     case KVM_EXIT_TPR_ACCESS:
2010         ret = kvm_handle_tpr_access(env);
2011         break;
2012     case KVM_EXIT_FAIL_ENTRY:
2013         code = run->fail_entry.hardware_entry_failure_reason;
2014         fprintf(stderr, "KVM: entry failed, hardware error 0x%" PRIx64 "\n",
2015                 code);
2016         if (host_supports_vmx() && code == VMX_INVALID_GUEST_STATE) {
2017             fprintf(stderr,
2018                     "\nIf you're running a guest on an Intel machine without "
2019                         "unrestricted mode\n"
2020                     "support, the failure can be most likely due to the guest "
2021                         "entering an invalid\n"
2022                     "state for Intel VT. For example, the guest maybe running "
2023                         "in big real mode\n"
2024                     "which is not supported on less recent Intel processors."
2025                         "\n\n");
2026         }
2027         ret = -1;
2028         break;
2029     case KVM_EXIT_EXCEPTION:
2030         fprintf(stderr, "KVM: exception %d exit (error code 0x%x)\n",
2031                 run->ex.exception, run->ex.error_code);
2032         ret = -1;
2033         break;
2034     case KVM_EXIT_DEBUG:
2035         DPRINTF("kvm_exit_debug\n");
2036         ret = kvm_handle_debug(&run->debug.arch);
2037         break;
2038     default:
2039         fprintf(stderr, "KVM: unknown exit reason %d\n", run->exit_reason);
2040         ret = -1;
2041         break;
2042     }
2043
2044     return ret;
2045 }
2046
2047 bool kvm_arch_stop_on_emulation_error(CPUX86State *env)
2048 {
2049     kvm_cpu_synchronize_state(env);
2050     return !(env->cr[0] & CR0_PE_MASK) ||
2051            ((env->segs[R_CS].selector  & 3) != 3);
2052 }
2053
2054 void kvm_arch_init_irq_routing(KVMState *s)
2055 {
2056     if (!kvm_check_extension(s, KVM_CAP_IRQ_ROUTING)) {
2057         /* If kernel can't do irq routing, interrupt source
2058          * override 0->2 cannot be set up as required by HPET.
2059          * So we have to disable it.
2060          */
2061         no_hpet = 1;
2062     }
2063     /* We know at this point that we're using the in-kernel
2064      * irqchip, so we can use irqfds, and on x86 we know
2065      * we can use msi via irqfd and GSI routing.
2066      */
2067     kvm_irqfds_allowed = true;
2068     kvm_msi_via_irqfd_allowed = true;
2069     kvm_gsi_routing_allowed = true;
2070 }