target-i386/kvm.c

   1 /*
   2  * QEMU KVM support
   3  *
   4  * Copyright (C) 2006-2008 Qumranet Technologies
   5  * Copyright IBM, Corp. 2008
   6  *
   7  * Authors:
   8  *  Anthony Liguori   <aliguori@us.ibm.com>
   9  *
  10  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  11  * See the COPYING file in the top-level directory.
  12  *
  13  */
  14
  15 #include <sys/types.h>
  16 #include <sys/ioctl.h>
  17 #include <sys/mman.h>
  18 #include <sys/utsname.h>
  19
  20 #include <linux/kvm.h>
  21
  22 #include "qemu-common.h"
  23 #include "sysemu.h"
  24 #include "kvm.h"
  25 #include "cpu.h"
  26 #include "gdbstub.h"
  27 #include "host-utils.h"
  28 #include "hw/pc.h"
  29 #include "hw/apic.h"
  30 #include "ioport.h"
  31 #include "kvm_x86.h"
  32
  33 #ifdef CONFIG_KVM_PARA
  34 #include <linux/kvm_para.h>
  35 #endif
  36 //
  37 //#define DEBUG_KVM
  38
  39 #ifdef DEBUG_KVM
  40 #define DPRINTF(fmt, ...) \
  41     do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
  42 #else
  43 #define DPRINTF(fmt, ...) \
  44     do { } while (0)
  45 #endif
  46
  47 #define MSR_KVM_WALL_CLOCK  0x11
  48 #define MSR_KVM_SYSTEM_TIME 0x12
  49
  50 #ifndef BUS_MCEERR_AR
  51 #define BUS_MCEERR_AR 4
  52 #endif
  53 #ifndef BUS_MCEERR_AO
  54 #define BUS_MCEERR_AO 5
  55 #endif
  56
  57 const KVMCapabilityInfo kvm_arch_required_capabilities[] = {
  58     KVM_CAP_INFO(SET_TSS_ADDR),
  59     KVM_CAP_INFO(EXT_CPUID),
  60     KVM_CAP_INFO(MP_STATE),
  61     KVM_CAP_LAST_INFO
  62 };
  63
  64 static bool has_msr_star;
  65 static bool has_msr_hsave_pa;
  66 #if defined(CONFIG_KVM_PARA) && defined(KVM_CAP_ASYNC_PF)
  67 static bool has_msr_async_pf_en;
  68 #endif
  69 static int lm_capable_kernel;
  70
  71 static struct kvm_cpuid2 *try_get_cpuid(KVMState *s, int max)
  72 {
  73     struct kvm_cpuid2 *cpuid;
  74     int r, size;
  75
  76     size = sizeof(*cpuid) + max * sizeof(*cpuid->entries);
  77     cpuid = (struct kvm_cpuid2 *)qemu_mallocz(size);
  78     cpuid->nent = max;
  79     r = kvm_ioctl(s, KVM_GET_SUPPORTED_CPUID, cpuid);
  80     if (r == 0 && cpuid->nent >= max) {
  81         r = -E2BIG;
  82     }
  83     if (r < 0) {
  84         if (r == -E2BIG) {
  85             qemu_free(cpuid);
  86             return NULL;
  87         } else {
  88             fprintf(stderr, "KVM_GET_SUPPORTED_CPUID failed: %s\n",
  89                     strerror(-r));
  90             exit(1);
  91         }
  92     }
  93     return cpuid;
  94 }
  95
  96 uint32_t kvm_arch_get_supported_cpuid(CPUState *env, uint32_t function,
  97                                       uint32_t index, int reg)
  98 {
  99     struct kvm_cpuid2 *cpuid;
 100     int i, max;
 101     uint32_t ret = 0;
 102     uint32_t cpuid_1_edx;
 103
 104     max = 1;
 105     while ((cpuid = try_get_cpuid(env->kvm_state, max)) == NULL) {
 106         max *= 2;
 107     }
 108
 109     for (i = 0; i < cpuid->nent; ++i) {
 110         if (cpuid->entries[i].function == function &&
 111             cpuid->entries[i].index == index) {
 112             switch (reg) {
 113             case R_EAX:
 114                 ret = cpuid->entries[i].eax;
 115                 break;
 116             case R_EBX:
 117                 ret = cpuid->entries[i].ebx;
 118                 break;
 119             case R_ECX:
 120                 ret = cpuid->entries[i].ecx;
 121                 break;
 122             case R_EDX:
 123                 ret = cpuid->entries[i].edx;
 124                 switch (function) {
 125                 case 1:
 126                     /* KVM before 2.6.30 misreports the following features */
 127                     ret |= CPUID_MTRR | CPUID_PAT | CPUID_MCE | CPUID_MCA;
 128                     break;
 129                 case 0x80000001:
 130                     /* On Intel, kvm returns cpuid according to the Intel spec,
 131                      * so add missing bits according to the AMD spec:
 132                      */
 133                     cpuid_1_edx = kvm_arch_get_supported_cpuid(env, 1, 0, R_EDX);
 134                     ret |= cpuid_1_edx & 0x183f7ff;
 135                     break;
 136                 }
 137                 break;
 138             }
 139         }
 140     }
 141
 142     qemu_free(cpuid);
 143
 144     return ret;
 145 }
 146
 147 #ifdef CONFIG_KVM_PARA
 148 struct kvm_para_features {
 149     int cap;
 150     int feature;
 151 } para_features[] = {
 152     { KVM_CAP_CLOCKSOURCE, KVM_FEATURE_CLOCKSOURCE },
 153     { KVM_CAP_NOP_IO_DELAY, KVM_FEATURE_NOP_IO_DELAY },
 154     { KVM_CAP_PV_MMU, KVM_FEATURE_MMU_OP },
 155 #ifdef KVM_CAP_ASYNC_PF
 156     { KVM_CAP_ASYNC_PF, KVM_FEATURE_ASYNC_PF },
 157 #endif
 158     { -1, -1 }
 159 };
 160
 161 static int get_para_features(CPUState *env)
 162 {
 163     int i, features = 0;
 164
 165     for (i = 0; i < ARRAY_SIZE(para_features) - 1; i++) {
 166         if (kvm_check_extension(env->kvm_state, para_features[i].cap)) {
 167             features |= (1 << para_features[i].feature);
 168         }
 169     }
 170 #ifdef KVM_CAP_ASYNC_PF
 171     has_msr_async_pf_en = features & (1 << KVM_FEATURE_ASYNC_PF);
 172 #endif
 173     return features;
 174 }
 175 #endif /* CONFIG_KVM_PARA */
 176
 177 #ifdef KVM_CAP_MCE
 178 static int kvm_get_mce_cap_supported(KVMState *s, uint64_t *mce_cap,
 179                                      int *max_banks)
 180 {
 181     int r;
 182
 183     r = kvm_check_extension(s, KVM_CAP_MCE);
 184     if (r > 0) {
 185         *max_banks = r;
 186         return kvm_ioctl(s, KVM_X86_GET_MCE_CAP_SUPPORTED, mce_cap);
 187     }
 188     return -ENOSYS;
 189 }
 190
 191 static int kvm_setup_mce(CPUState *env, uint64_t *mcg_cap)
 192 {
 193     return kvm_vcpu_ioctl(env, KVM_X86_SETUP_MCE, mcg_cap);
 194 }
 195
 196 static int kvm_set_mce(CPUState *env, struct kvm_x86_mce *m)
 197 {
 198     return kvm_vcpu_ioctl(env, KVM_X86_SET_MCE, m);
 199 }
 200
 201 static int kvm_get_msr(CPUState *env, struct kvm_msr_entry *msrs, int n)
 202 {
 203     struct kvm_msrs *kmsrs = qemu_malloc(sizeof *kmsrs + n * sizeof *msrs);
 204     int r;
 205
 206     kmsrs->nmsrs = n;
 207     memcpy(kmsrs->entries, msrs, n * sizeof *msrs);
 208     r = kvm_vcpu_ioctl(env, KVM_GET_MSRS, kmsrs);
 209     memcpy(msrs, kmsrs->entries, n * sizeof *msrs);
 210     free(kmsrs);
 211     return r;
 212 }
 213
 214 /* FIXME: kill this and kvm_get_msr, use env->mcg_status instead */
 215 static int kvm_mce_in_progress(CPUState *env)
 216 {
 217     struct kvm_msr_entry msr_mcg_status = {
 218         .index = MSR_MCG_STATUS,
 219     };
 220     int r;
 221
 222     r = kvm_get_msr(env, &msr_mcg_status, 1);
 223     if (r == -1 || r == 0) {
 224         fprintf(stderr, "Failed to get MCE status\n");
 225         return 0;
 226     }
 227     return !!(msr_mcg_status.data & MCG_STATUS_MCIP);
 228 }
 229
 230 struct kvm_x86_mce_data
 231 {
 232     CPUState *env;
 233     struct kvm_x86_mce *mce;
 234     int abort_on_error;
 235 };
 236
 237 static void kvm_do_inject_x86_mce(void *_data)
 238 {
 239     struct kvm_x86_mce_data *data = _data;
 240     int r;
 241
 242     /* If there is an MCE exception being processed, ignore this SRAO MCE */
 243     if ((data->env->mcg_cap & MCG_SER_P) &&
 244         !(data->mce->status & MCI_STATUS_AR)) {
 245         if (kvm_mce_in_progress(data->env)) {
 246             return;
 247         }
 248     }
 249
 250     r = kvm_set_mce(data->env, data->mce);
 251     if (r < 0) {
 252         perror("kvm_set_mce FAILED");
 253         if (data->abort_on_error) {
 254             abort();
 255         }
 256     }
 257 }
 258
 259 static void kvm_inject_x86_mce_on(CPUState *env, struct kvm_x86_mce *mce,
 260                                   int flag)
 261 {
 262     struct kvm_x86_mce_data data = {
 263         .env = env,
 264         .mce = mce,
 265         .abort_on_error = (flag & ABORT_ON_ERROR),
 266     };
 267
 268     if (!env->mcg_cap) {
 269         fprintf(stderr, "MCE support is not enabled!\n");
 270         return;
 271     }
 272
 273     run_on_cpu(env, kvm_do_inject_x86_mce, &data);
 274 }
 275
 276 static void kvm_mce_broadcast_rest(CPUState *env)
 277 {
 278     struct kvm_x86_mce mce = {
 279         .bank = 1,
 280         .status = MCI_STATUS_VAL | MCI_STATUS_UC,
 281         .mcg_status = MCG_STATUS_MCIP | MCG_STATUS_RIPV,
 282         .addr = 0,
 283         .misc = 0,
 284     };
 285     CPUState *cenv;
 286
 287     /* Broadcast MCA signal for processor version 06H_EH and above */
 288     if (cpu_x86_support_mca_broadcast(env)) {
 289         for (cenv = first_cpu; cenv != NULL; cenv = cenv->next_cpu) {
 290             if (cenv == env) {
 291                 continue;
 292             }
 293             kvm_inject_x86_mce_on(cenv, &mce, ABORT_ON_ERROR);
 294         }
 295     }
 296 }
 297
 298 static void kvm_mce_inj_srar_dataload(CPUState *env, target_phys_addr_t paddr)
 299 {
 300     struct kvm_x86_mce mce = {
 301         .bank = 9,
 302         .status = MCI_STATUS_VAL | MCI_STATUS_UC | MCI_STATUS_EN
 303                   | MCI_STATUS_MISCV | MCI_STATUS_ADDRV | MCI_STATUS_S
 304                   | MCI_STATUS_AR | 0x134,
 305         .mcg_status = MCG_STATUS_MCIP | MCG_STATUS_EIPV,
 306         .addr = paddr,
 307         .misc = (MCM_ADDR_PHYS << 6) | 0xc,
 308     };
 309     int r;
 310
 311     r = kvm_set_mce(env, &mce);
 312     if (r < 0) {
 313         fprintf(stderr, "kvm_set_mce: %s\n", strerror(errno));
 314         abort();
 315     }
 316     kvm_mce_broadcast_rest(env);
 317 }
 318
 319 static void kvm_mce_inj_srao_memscrub(CPUState *env, target_phys_addr_t paddr)
 320 {
 321     struct kvm_x86_mce mce = {
 322         .bank = 9,
 323         .status = MCI_STATUS_VAL | MCI_STATUS_UC | MCI_STATUS_EN
 324                   | MCI_STATUS_MISCV | MCI_STATUS_ADDRV | MCI_STATUS_S
 325                   | 0xc0,
 326         .mcg_status = MCG_STATUS_MCIP | MCG_STATUS_RIPV,
 327         .addr = paddr,
 328         .misc = (MCM_ADDR_PHYS << 6) | 0xc,
 329     };
 330     int r;
 331
 332     r = kvm_set_mce(env, &mce);
 333     if (r < 0) {
 334         fprintf(stderr, "kvm_set_mce: %s\n", strerror(errno));
 335         abort();
 336     }
 337     kvm_mce_broadcast_rest(env);
 338 }
 339
 340 static void kvm_mce_inj_srao_memscrub2(CPUState *env, target_phys_addr_t paddr)
 341 {
 342     struct kvm_x86_mce mce = {
 343         .bank = 9,
 344         .status = MCI_STATUS_VAL | MCI_STATUS_UC | MCI_STATUS_EN
 345                   | MCI_STATUS_MISCV | MCI_STATUS_ADDRV | MCI_STATUS_S
 346                   | 0xc0,
 347         .mcg_status = MCG_STATUS_MCIP | MCG_STATUS_RIPV,
 348         .addr = paddr,
 349         .misc = (MCM_ADDR_PHYS << 6) | 0xc,
 350     };
 351
 352     kvm_inject_x86_mce_on(env, &mce, ABORT_ON_ERROR);
 353     kvm_mce_broadcast_rest(env);
 354 }
 355 #endif /* KVM_CAP_MCE */
 356
 357 static void hardware_memory_error(void)
 358 {
 359     fprintf(stderr, "Hardware memory error!\n");
 360     exit(1);
 361 }
 362
 363 int kvm_arch_on_sigbus_vcpu(CPUState *env, int code, void *addr)
 364 {
 365 #ifdef KVM_CAP_MCE
 366     void *vaddr;
 367     ram_addr_t ram_addr;
 368     target_phys_addr_t paddr;
 369
 370     if ((env->mcg_cap & MCG_SER_P) && addr
 371         && (code == BUS_MCEERR_AR
 372             || code == BUS_MCEERR_AO)) {
 373         vaddr = (void *)addr;
 374         if (qemu_ram_addr_from_host(vaddr, &ram_addr) ||
 375             !kvm_physical_memory_addr_from_ram(env->kvm_state, ram_addr, &paddr)) {
 376             fprintf(stderr, "Hardware memory error for memory used by "
 377                     "QEMU itself instead of guest system!\n");
 378             /* Hope we are lucky for AO MCE */
 379             if (code == BUS_MCEERR_AO) {
 380                 return 0;
 381             } else {
 382                 hardware_memory_error();
 383             }
 384         }
 385
 386         if (code == BUS_MCEERR_AR) {
 387             /* Fake an Intel architectural Data Load SRAR UCR */
 388             kvm_mce_inj_srar_dataload(env, paddr);
 389         } else {
 390             /*
 391              * If there is an MCE excpetion being processed, ignore
 392              * this SRAO MCE
 393              */
 394             if (!kvm_mce_in_progress(env)) {
 395                 /* Fake an Intel architectural Memory scrubbing UCR */
 396                 kvm_mce_inj_srao_memscrub(env, paddr);
 397             }
 398         }
 399     } else
 400 #endif /* KVM_CAP_MCE */
 401     {
 402         if (code == BUS_MCEERR_AO) {
 403             return 0;
 404         } else if (code == BUS_MCEERR_AR) {
 405             hardware_memory_error();
 406         } else {
 407             return 1;
 408         }
 409     }
 410     return 0;
 411 }
 412
 413 int kvm_arch_on_sigbus(int code, void *addr)
 414 {
 415 #ifdef KVM_CAP_MCE
 416     if ((first_cpu->mcg_cap & MCG_SER_P) && addr && code == BUS_MCEERR_AO) {
 417         void *vaddr;
 418         ram_addr_t ram_addr;
 419         target_phys_addr_t paddr;
 420
 421         /* Hope we are lucky for AO MCE */
 422         vaddr = addr;
 423         if (qemu_ram_addr_from_host(vaddr, &ram_addr) ||
 424             !kvm_physical_memory_addr_from_ram(first_cpu->kvm_state, ram_addr,
 425                                                &paddr)) {
 426             fprintf(stderr, "Hardware memory error for memory used by "
 427                     "QEMU itself instead of guest system!: %p\n", addr);
 428             return 0;
 429         }
 430         kvm_mce_inj_srao_memscrub2(first_cpu, paddr);
 431     } else
 432 #endif /* KVM_CAP_MCE */
 433     {
 434         if (code == BUS_MCEERR_AO) {
 435             return 0;
 436         } else if (code == BUS_MCEERR_AR) {
 437             hardware_memory_error();
 438         } else {
 439             return 1;
 440         }
 441     }
 442     return 0;
 443 }
 444
 445 void kvm_inject_x86_mce(CPUState *cenv, int bank, uint64_t status,
 446                         uint64_t mcg_status, uint64_t addr, uint64_t misc,
 447                         int flag)
 448 {
 449 #ifdef KVM_CAP_MCE
 450     struct kvm_x86_mce mce = {
 451         .bank = bank,
 452         .status = status,
 453         .mcg_status = mcg_status,
 454         .addr = addr,
 455         .misc = misc,
 456     };
 457
 458     if (flag & MCE_BROADCAST) {
 459         kvm_mce_broadcast_rest(cenv);
 460     }
 461
 462     kvm_inject_x86_mce_on(cenv, &mce, flag);
 463 #else /* !KVM_CAP_MCE*/
 464     if (flag & ABORT_ON_ERROR) {
 465         abort();
 466     }
 467 #endif /* !KVM_CAP_MCE*/
 468 }
 469
 470 static int kvm_inject_mce_oldstyle(CPUState *env)
 471 {
 472 #ifdef KVM_CAP_MCE
 473     if (!kvm_has_vcpu_events() && env->exception_injected == EXCP12_MCHK) {
 474         unsigned int bank, bank_num = env->mcg_cap & 0xff;
 475         struct kvm_x86_mce mce;
 476
 477         env->exception_injected = -1;
 478
 479         /*
 480          * There must be at least one bank in use if an MCE is pending.
 481          * Find it and use its values for the event injection.
 482          */
 483         for (bank = 0; bank < bank_num; bank++) {
 484             if (env->mce_banks[bank * 4 + 1] & MCI_STATUS_VAL) {
 485                 break;
 486             }
 487         }
 488         assert(bank < bank_num);
 489
 490         mce.bank = bank;
 491         mce.status = env->mce_banks[bank * 4 + 1];
 492         mce.mcg_status = env->mcg_status;
 493         mce.addr = env->mce_banks[bank * 4 + 2];
 494         mce.misc = env->mce_banks[bank * 4 + 3];
 495
 496         return kvm_vcpu_ioctl(env, KVM_X86_SET_MCE, &mce);
 497     }
 498 #endif /* KVM_CAP_MCE */
 499     return 0;
 500 }
 501
 502 static void cpu_update_state(void *opaque, int running, int reason)
 503 {
 504     CPUState *env = opaque;
 505
 506     if (running) {
 507         env->tsc_valid = false;
 508     }
 509 }
 510
 511 int kvm_arch_init_vcpu(CPUState *env)
 512 {
 513     struct {
 514         struct kvm_cpuid2 cpuid;
 515         struct kvm_cpuid_entry2 entries[100];
 516     } __attribute__((packed)) cpuid_data;
 517     uint32_t limit, i, j, cpuid_i;
 518     uint32_t unused;
 519     struct kvm_cpuid_entry2 *c;
 520 #ifdef CONFIG_KVM_PARA
 521     uint32_t signature[3];
 522 #endif
 523
 524     env->cpuid_features &= kvm_arch_get_supported_cpuid(env, 1, 0, R_EDX);
 525
 526     i = env->cpuid_ext_features & CPUID_EXT_HYPERVISOR;
 527     env->cpuid_ext_features &= kvm_arch_get_supported_cpuid(env, 1, 0, R_ECX);
 528     env->cpuid_ext_features |= i;
 529
 530     env->cpuid_ext2_features &= kvm_arch_get_supported_cpuid(env, 0x80000001,
 531                                                              0, R_EDX);
 532     env->cpuid_ext3_features &= kvm_arch_get_supported_cpuid(env, 0x80000001,
 533                                                              0, R_ECX);
 534     env->cpuid_svm_features  &= kvm_arch_get_supported_cpuid(env, 0x8000000A,
 535                                                              0, R_EDX);
 536
 537
 538     cpuid_i = 0;
 539
 540 #ifdef CONFIG_KVM_PARA
 541     /* Paravirtualization CPUIDs */
 542     memcpy(signature, "KVMKVMKVM\0\0\0", 12);
 543     c = &cpuid_data.entries[cpuid_i++];
 544     memset(c, 0, sizeof(*c));
 545     c->function = KVM_CPUID_SIGNATURE;
 546     c->eax = 0;
 547     c->ebx = signature[0];
 548     c->ecx = signature[1];
 549     c->edx = signature[2];
 550
 551     c = &cpuid_data.entries[cpuid_i++];
 552     memset(c, 0, sizeof(*c));
 553     c->function = KVM_CPUID_FEATURES;
 554     c->eax = env->cpuid_kvm_features & get_para_features(env);
 555 #endif
 556
 557     cpu_x86_cpuid(env, 0, 0, &limit, &unused, &unused, &unused);
 558
 559     for (i = 0; i <= limit; i++) {
 560         c = &cpuid_data.entries[cpuid_i++];
 561
 562         switch (i) {
 563         case 2: {
 564             /* Keep reading function 2 till all the input is received */
 565             int times;
 566
 567             c->function = i;
 568             c->flags = KVM_CPUID_FLAG_STATEFUL_FUNC |
 569                        KVM_CPUID_FLAG_STATE_READ_NEXT;
 570             cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx);
 571             times = c->eax & 0xff;
 572
 573             for (j = 1; j < times; ++j) {
 574                 c = &cpuid_data.entries[cpuid_i++];
 575                 c->function = i;
 576                 c->flags = KVM_CPUID_FLAG_STATEFUL_FUNC;
 577                 cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx);
 578             }
 579             break;
 580         }
 581         case 4:
 582         case 0xb:
 583         case 0xd:
 584             for (j = 0; ; j++) {
 585                 c->function = i;
 586                 c->flags = KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
 587                 c->index = j;
 588                 cpu_x86_cpuid(env, i, j, &c->eax, &c->ebx, &c->ecx, &c->edx);
 589
 590                 if (i == 4 && c->eax == 0) {
 591                     break;
 592                 }
 593                 if (i == 0xb && !(c->ecx & 0xff00)) {
 594                     break;
 595                 }
 596                 if (i == 0xd && c->eax == 0) {
 597                     break;
 598                 }
 599                 c = &cpuid_data.entries[cpuid_i++];
 600             }
 601             break;
 602         default:
 603             c->function = i;
 604             c->flags = 0;
 605             cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx);
 606             break;
 607         }
 608     }
 609     cpu_x86_cpuid(env, 0x80000000, 0, &limit, &unused, &unused, &unused);
 610
 611     for (i = 0x80000000; i <= limit; i++) {
 612         c = &cpuid_data.entries[cpuid_i++];
 613
 614         c->function = i;
 615         c->flags = 0;
 616         cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx);
 617     }
 618
 619     cpuid_data.cpuid.nent = cpuid_i;
 620
 621 #ifdef KVM_CAP_MCE
 622     if (((env->cpuid_version >> 8)&0xF) >= 6
 623         && (env->cpuid_features&(CPUID_MCE|CPUID_MCA)) == (CPUID_MCE|CPUID_MCA)
 624         && kvm_check_extension(env->kvm_state, KVM_CAP_MCE) > 0) {
 625         uint64_t mcg_cap;
 626         int banks;
 627
 628         if (kvm_get_mce_cap_supported(env->kvm_state, &mcg_cap, &banks)) {
 629             perror("kvm_get_mce_cap_supported FAILED");
 630         } else {
 631             if (banks > MCE_BANKS_DEF)
 632                 banks = MCE_BANKS_DEF;
 633             mcg_cap &= MCE_CAP_DEF;
 634             mcg_cap |= banks;
 635             if (kvm_setup_mce(env, &mcg_cap)) {
 636                 perror("kvm_setup_mce FAILED");
 637             } else {
 638                 env->mcg_cap = mcg_cap;
 639             }
 640         }
 641     }
 642 #endif
 643
 644     qemu_add_vm_change_state_handler(cpu_update_state, env);
 645
 646     return kvm_vcpu_ioctl(env, KVM_SET_CPUID2, &cpuid_data);
 647 }
 648
 649 void kvm_arch_reset_vcpu(CPUState *env)
 650 {
 651     env->exception_injected = -1;
 652     env->interrupt_injected = -1;
 653     env->xcr0 = 1;
 654     if (kvm_irqchip_in_kernel()) {
 655         env->mp_state = cpu_is_bsp(env) ? KVM_MP_STATE_RUNNABLE :
 656                                           KVM_MP_STATE_UNINITIALIZED;
 657     } else {
 658         env->mp_state = KVM_MP_STATE_RUNNABLE;
 659     }
 660 }
 661
 662 static int kvm_get_supported_msrs(KVMState *s)
 663 {
 664     static int kvm_supported_msrs;
 665     int ret = 0;
 666
 667     /* first time */
 668     if (kvm_supported_msrs == 0) {
 669         struct kvm_msr_list msr_list, *kvm_msr_list;
 670
 671         kvm_supported_msrs = -1;
 672
 673         /* Obtain MSR list from KVM.  These are the MSRs that we must
 674          * save/restore */
 675         msr_list.nmsrs = 0;
 676         ret = kvm_ioctl(s, KVM_GET_MSR_INDEX_LIST, &msr_list);
 677         if (ret < 0 && ret != -E2BIG) {
 678             return ret;
 679         }
 680         /* Old kernel modules had a bug and could write beyond the provided
 681            memory. Allocate at least a safe amount of 1K. */
 682         kvm_msr_list = qemu_mallocz(MAX(1024, sizeof(msr_list) +
 683                                               msr_list.nmsrs *
 684                                               sizeof(msr_list.indices[0])));
 685
 686         kvm_msr_list->nmsrs = msr_list.nmsrs;
 687         ret = kvm_ioctl(s, KVM_GET_MSR_INDEX_LIST, kvm_msr_list);
 688         if (ret >= 0) {
 689             int i;
 690
 691             for (i = 0; i < kvm_msr_list->nmsrs; i++) {
 692                 if (kvm_msr_list->indices[i] == MSR_STAR) {
 693                     has_msr_star = true;
 694                     continue;
 695                 }
 696                 if (kvm_msr_list->indices[i] == MSR_VM_HSAVE_PA) {
 697                     has_msr_hsave_pa = true;
 698                     continue;
 699                 }
 700             }
 701         }
 702
 703         free(kvm_msr_list);
 704     }
 705
 706     return ret;
 707 }
 708
 709 int kvm_arch_init(KVMState *s)
 710 {
 711     uint64_t identity_base = 0xfffbc000;
 712     int ret;
 713     struct utsname utsname;
 714
 715     ret = kvm_get_supported_msrs(s);
 716     if (ret < 0) {
 717         return ret;
 718     }
 719
 720     uname(&utsname);
 721     lm_capable_kernel = strcmp(utsname.machine, "x86_64") == 0;
 722
 723     /*
 724      * On older Intel CPUs, KVM uses vm86 mode to emulate 16-bit code directly.
 725      * In order to use vm86 mode, an EPT identity map and a TSS  are needed.
 726      * Since these must be part of guest physical memory, we need to allocate
 727      * them, both by setting their start addresses in the kernel and by
 728      * creating a corresponding e820 entry. We need 4 pages before the BIOS.
 729      *
 730      * Older KVM versions may not support setting the identity map base. In
 731      * that case we need to stick with the default, i.e. a 256K maximum BIOS
 732      * size.
 733      */
 734 #ifdef KVM_CAP_SET_IDENTITY_MAP_ADDR
 735     if (kvm_check_extension(s, KVM_CAP_SET_IDENTITY_MAP_ADDR)) {
 736         /* Allows up to 16M BIOSes. */
 737         identity_base = 0xfeffc000;
 738
 739         ret = kvm_vm_ioctl(s, KVM_SET_IDENTITY_MAP_ADDR, &identity_base);
 740         if (ret < 0) {
 741             return ret;
 742         }
 743     }
 744 #endif
 745     /* Set TSS base one page after EPT identity map. */
 746     ret = kvm_vm_ioctl(s, KVM_SET_TSS_ADDR, identity_base + 0x1000);
 747     if (ret < 0) {
 748         return ret;
 749     }
 750
 751     /* Tell fw_cfg to notify the BIOS to reserve the range. */
 752     ret = e820_add_entry(identity_base, 0x4000, E820_RESERVED);
 753     if (ret < 0) {
 754         fprintf(stderr, "e820_add_entry() table is full\n");
 755         return ret;
 756     }
 757
 758     return 0;
 759 }
 760
 761 static void set_v8086_seg(struct kvm_segment *lhs, const SegmentCache *rhs)
 762 {
 763     lhs->selector = rhs->selector;
 764     lhs->base = rhs->base;
 765     lhs->limit = rhs->limit;
 766     lhs->type = 3;
 767     lhs->present = 1;
 768     lhs->dpl = 3;
 769     lhs->db = 0;
 770     lhs->s = 1;
 771     lhs->l = 0;
 772     lhs->g = 0;
 773     lhs->avl = 0;
 774     lhs->unusable = 0;
 775 }
 776
 777 static void set_seg(struct kvm_segment *lhs, const SegmentCache *rhs)
 778 {
 779     unsigned flags = rhs->flags;
 780     lhs->selector = rhs->selector;
 781     lhs->base = rhs->base;
 782     lhs->limit = rhs->limit;
 783     lhs->type = (flags >> DESC_TYPE_SHIFT) & 15;
 784     lhs->present = (flags & DESC_P_MASK) != 0;
 785     lhs->dpl = (flags >> DESC_DPL_SHIFT) & 3;
 786     lhs->db = (flags >> DESC_B_SHIFT) & 1;
 787     lhs->s = (flags & DESC_S_MASK) != 0;
 788     lhs->l = (flags >> DESC_L_SHIFT) & 1;
 789     lhs->g = (flags & DESC_G_MASK) != 0;
 790     lhs->avl = (flags & DESC_AVL_MASK) != 0;
 791     lhs->unusable = 0;
 792 }
 793
 794 static void get_seg(SegmentCache *lhs, const struct kvm_segment *rhs)
 795 {
 796     lhs->selector = rhs->selector;
 797     lhs->base = rhs->base;
 798     lhs->limit = rhs->limit;
 799     lhs->flags = (rhs->type << DESC_TYPE_SHIFT) |
 800                  (rhs->present * DESC_P_MASK) |
 801                  (rhs->dpl << DESC_DPL_SHIFT) |
 802                  (rhs->db << DESC_B_SHIFT) |
 803                  (rhs->s * DESC_S_MASK) |
 804                  (rhs->l << DESC_L_SHIFT) |
 805                  (rhs->g * DESC_G_MASK) |
 806                  (rhs->avl * DESC_AVL_MASK);
 807 }
 808
 809 static void kvm_getput_reg(__u64 *kvm_reg, target_ulong *qemu_reg, int set)
 810 {
 811     if (set) {
 812         *kvm_reg = *qemu_reg;
 813     } else {
 814         *qemu_reg = *kvm_reg;
 815     }
 816 }
 817
 818 static int kvm_getput_regs(CPUState *env, int set)
 819 {
 820     struct kvm_regs regs;
 821     int ret = 0;
 822
 823     if (!set) {
 824         ret = kvm_vcpu_ioctl(env, KVM_GET_REGS, &regs);
 825         if (ret < 0) {
 826             return ret;
 827         }
 828     }
 829
 830     kvm_getput_reg(&regs.rax, &env->regs[R_EAX], set);
 831     kvm_getput_reg(&regs.rbx, &env->regs[R_EBX], set);
 832     kvm_getput_reg(&regs.rcx, &env->regs[R_ECX], set);
 833     kvm_getput_reg(&regs.rdx, &env->regs[R_EDX], set);
 834     kvm_getput_reg(&regs.rsi, &env->regs[R_ESI], set);
 835     kvm_getput_reg(&regs.rdi, &env->regs[R_EDI], set);
 836     kvm_getput_reg(&regs.rsp, &env->regs[R_ESP], set);
 837     kvm_getput_reg(&regs.rbp, &env->regs[R_EBP], set);
 838 #ifdef TARGET_X86_64
 839     kvm_getput_reg(&regs.r8, &env->regs[8], set);
 840     kvm_getput_reg(&regs.r9, &env->regs[9], set);
 841     kvm_getput_reg(&regs.r10, &env->regs[10], set);
 842     kvm_getput_reg(&regs.r11, &env->regs[11], set);
 843     kvm_getput_reg(&regs.r12, &env->regs[12], set);
 844     kvm_getput_reg(&regs.r13, &env->regs[13], set);
 845     kvm_getput_reg(&regs.r14, &env->regs[14], set);
 846     kvm_getput_reg(&regs.r15, &env->regs[15], set);
 847 #endif
 848
 849     kvm_getput_reg(&regs.rflags, &env->eflags, set);
 850     kvm_getput_reg(&regs.rip, &env->eip, set);
 851
 852     if (set) {
 853         ret = kvm_vcpu_ioctl(env, KVM_SET_REGS, &regs);
 854     }
 855
 856     return ret;
 857 }
 858
 859 static int kvm_put_fpu(CPUState *env)
 860 {
 861     struct kvm_fpu fpu;
 862     int i;
 863
 864     memset(&fpu, 0, sizeof fpu);
 865     fpu.fsw = env->fpus & ~(7 << 11);
 866     fpu.fsw |= (env->fpstt & 7) << 11;
 867     fpu.fcw = env->fpuc;
 868     for (i = 0; i < 8; ++i) {
 869         fpu.ftwx |= (!env->fptags[i]) << i;
 870     }
 871     memcpy(fpu.fpr, env->fpregs, sizeof env->fpregs);
 872     memcpy(fpu.xmm, env->xmm_regs, sizeof env->xmm_regs);
 873     fpu.mxcsr = env->mxcsr;
 874
 875     return kvm_vcpu_ioctl(env, KVM_SET_FPU, &fpu);
 876 }
 877
 878 #ifdef KVM_CAP_XSAVE
 879 #define XSAVE_CWD_RIP     2
 880 #define XSAVE_CWD_RDP     4
 881 #define XSAVE_MXCSR       6
 882 #define XSAVE_ST_SPACE    8
 883 #define XSAVE_XMM_SPACE   40
 884 #define XSAVE_XSTATE_BV   128
 885 #define XSAVE_YMMH_SPACE  144
 886 #endif
 887
 888 static int kvm_put_xsave(CPUState *env)
 889 {
 890 #ifdef KVM_CAP_XSAVE
 891     int i, r;
 892     struct kvm_xsave* xsave;
 893     uint16_t cwd, swd, twd, fop;
 894
 895     if (!kvm_has_xsave()) {
 896         return kvm_put_fpu(env);
 897     }
 898
 899     xsave = qemu_memalign(4096, sizeof(struct kvm_xsave));
 900     memset(xsave, 0, sizeof(struct kvm_xsave));
 901     cwd = swd = twd = fop = 0;
 902     swd = env->fpus & ~(7 << 11);
 903     swd |= (env->fpstt & 7) << 11;
 904     cwd = env->fpuc;
 905     for (i = 0; i < 8; ++i) {
 906         twd |= (!env->fptags[i]) << i;
 907     }
 908     xsave->region[0] = (uint32_t)(swd << 16) + cwd;
 909     xsave->region[1] = (uint32_t)(fop << 16) + twd;
 910     memcpy(&xsave->region[XSAVE_ST_SPACE], env->fpregs,
 911             sizeof env->fpregs);
 912     memcpy(&xsave->region[XSAVE_XMM_SPACE], env->xmm_regs,
 913             sizeof env->xmm_regs);
 914     xsave->region[XSAVE_MXCSR] = env->mxcsr;
 915     *(uint64_t *)&xsave->region[XSAVE_XSTATE_BV] = env->xstate_bv;
 916     memcpy(&xsave->region[XSAVE_YMMH_SPACE], env->ymmh_regs,
 917             sizeof env->ymmh_regs);
 918     r = kvm_vcpu_ioctl(env, KVM_SET_XSAVE, xsave);
 919     qemu_free(xsave);
 920     return r;
 921 #else
 922     return kvm_put_fpu(env);
 923 #endif
 924 }
 925
 926 static int kvm_put_xcrs(CPUState *env)
 927 {
 928 #ifdef KVM_CAP_XCRS
 929     struct kvm_xcrs xcrs;
 930
 931     if (!kvm_has_xcrs()) {
 932         return 0;
 933     }
 934
 935     xcrs.nr_xcrs = 1;
 936     xcrs.flags = 0;
 937     xcrs.xcrs[0].xcr = 0;
 938     xcrs.xcrs[0].value = env->xcr0;
 939     return kvm_vcpu_ioctl(env, KVM_SET_XCRS, &xcrs);
 940 #else
 941     return 0;
 942 #endif
 943 }
 944
 945 static int kvm_put_sregs(CPUState *env)
 946 {
 947     struct kvm_sregs sregs;
 948
 949     memset(sregs.interrupt_bitmap, 0, sizeof(sregs.interrupt_bitmap));
 950     if (env->interrupt_injected >= 0) {
 951         sregs.interrupt_bitmap[env->interrupt_injected / 64] |=
 952                 (uint64_t)1 << (env->interrupt_injected % 64);
 953     }
 954
 955     if ((env->eflags & VM_MASK)) {
 956         set_v8086_seg(&sregs.cs, &env->segs[R_CS]);
 957         set_v8086_seg(&sregs.ds, &env->segs[R_DS]);
 958         set_v8086_seg(&sregs.es, &env->segs[R_ES]);
 959         set_v8086_seg(&sregs.fs, &env->segs[R_FS]);
 960         set_v8086_seg(&sregs.gs, &env->segs[R_GS]);
 961         set_v8086_seg(&sregs.ss, &env->segs[R_SS]);
 962     } else {
 963         set_seg(&sregs.cs, &env->segs[R_CS]);
 964         set_seg(&sregs.ds, &env->segs[R_DS]);
 965         set_seg(&sregs.es, &env->segs[R_ES]);
 966         set_seg(&sregs.fs, &env->segs[R_FS]);
 967         set_seg(&sregs.gs, &env->segs[R_GS]);
 968         set_seg(&sregs.ss, &env->segs[R_SS]);
 969     }
 970
 971     set_seg(&sregs.tr, &env->tr);
 972     set_seg(&sregs.ldt, &env->ldt);
 973
 974     sregs.idt.limit = env->idt.limit;
 975     sregs.idt.base = env->idt.base;
 976     sregs.gdt.limit = env->gdt.limit;
 977     sregs.gdt.base = env->gdt.base;
 978
 979     sregs.cr0 = env->cr[0];
 980     sregs.cr2 = env->cr[2];
 981     sregs.cr3 = env->cr[3];
 982     sregs.cr4 = env->cr[4];
 983
 984     sregs.cr8 = cpu_get_apic_tpr(env->apic_state);
 985     sregs.apic_base = cpu_get_apic_base(env->apic_state);
 986
 987     sregs.efer = env->efer;
 988
 989     return kvm_vcpu_ioctl(env, KVM_SET_SREGS, &sregs);
 990 }
 991
 992 static void kvm_msr_entry_set(struct kvm_msr_entry *entry,
 993                               uint32_t index, uint64_t value)
 994 {
 995     entry->index = index;
 996     entry->data = value;
 997 }
 998
 999 static int kvm_put_msrs(CPUState *env, int level)
1000 {
1001     struct {
1002         struct kvm_msrs info;
1003         struct kvm_msr_entry entries[100];
1004     } msr_data;
1005     struct kvm_msr_entry *msrs = msr_data.entries;
1006     int n = 0;
1007
1008     kvm_msr_entry_set(&msrs[n++], MSR_IA32_SYSENTER_CS, env->sysenter_cs);
1009     kvm_msr_entry_set(&msrs[n++], MSR_IA32_SYSENTER_ESP, env->sysenter_esp);
1010     kvm_msr_entry_set(&msrs[n++], MSR_IA32_SYSENTER_EIP, env->sysenter_eip);
1011     if (has_msr_star) {
1012         kvm_msr_entry_set(&msrs[n++], MSR_STAR, env->star);
1013     }
1014     if (has_msr_hsave_pa) {
1015         kvm_msr_entry_set(&msrs[n++], MSR_VM_HSAVE_PA, env->vm_hsave);
1016     }
1017 #ifdef TARGET_X86_64
1018     if (lm_capable_kernel) {
1019         kvm_msr_entry_set(&msrs[n++], MSR_CSTAR, env->cstar);
1020         kvm_msr_entry_set(&msrs[n++], MSR_KERNELGSBASE, env->kernelgsbase);
1021         kvm_msr_entry_set(&msrs[n++], MSR_FMASK, env->fmask);
1022         kvm_msr_entry_set(&msrs[n++], MSR_LSTAR, env->lstar);
1023     }
1024 #endif
1025     if (level == KVM_PUT_FULL_STATE) {
1026         /*
1027          * KVM is yet unable to synchronize TSC values of multiple VCPUs on
1028          * writeback. Until this is fixed, we only write the offset to SMP
1029          * guests after migration, desynchronizing the VCPUs, but avoiding
1030          * huge jump-backs that would occur without any writeback at all.
1031          */
1032         if (smp_cpus == 1 || env->tsc != 0) {
1033             kvm_msr_entry_set(&msrs[n++], MSR_IA32_TSC, env->tsc);
1034         }
1035     }
1036     /*
1037      * The following paravirtual MSRs have side effects on the guest or are
1038      * too heavy for normal writeback. Limit them to reset or full state
1039      * updates.
1040      */
1041     if (level >= KVM_PUT_RESET_STATE) {
1042         kvm_msr_entry_set(&msrs[n++], MSR_KVM_SYSTEM_TIME,
1043                           env->system_time_msr);
1044         kvm_msr_entry_set(&msrs[n++], MSR_KVM_WALL_CLOCK, env->wall_clock_msr);
1045 #if defined(CONFIG_KVM_PARA) && defined(KVM_CAP_ASYNC_PF)
1046         if (has_msr_async_pf_en) {
1047             kvm_msr_entry_set(&msrs[n++], MSR_KVM_ASYNC_PF_EN,
1048                               env->async_pf_en_msr);
1049         }
1050 #endif
1051     }
1052 #ifdef KVM_CAP_MCE
1053     if (env->mcg_cap) {
1054         int i;
1055
1056         if (level == KVM_PUT_RESET_STATE) {
1057             kvm_msr_entry_set(&msrs[n++], MSR_MCG_STATUS, env->mcg_status);
1058         } else if (level == KVM_PUT_FULL_STATE) {
1059             kvm_msr_entry_set(&msrs[n++], MSR_MCG_STATUS, env->mcg_status);
1060             kvm_msr_entry_set(&msrs[n++], MSR_MCG_CTL, env->mcg_ctl);
1061             for (i = 0; i < (env->mcg_cap & 0xff) * 4; i++) {
1062                 kvm_msr_entry_set(&msrs[n++], MSR_MC0_CTL + i, env->mce_banks[i]);
1063             }
1064         }
1065     }
1066 #endif
1067
1068     msr_data.info.nmsrs = n;
1069
1070     return kvm_vcpu_ioctl(env, KVM_SET_MSRS, &msr_data);
1071
1072 }
1073
1074
1075 static int kvm_get_fpu(CPUState *env)
1076 {
1077     struct kvm_fpu fpu;
1078     int i, ret;
1079
1080     ret = kvm_vcpu_ioctl(env, KVM_GET_FPU, &fpu);
1081     if (ret < 0) {
1082         return ret;
1083     }
1084
1085     env->fpstt = (fpu.fsw >> 11) & 7;
1086     env->fpus = fpu.fsw;
1087     env->fpuc = fpu.fcw;
1088     for (i = 0; i < 8; ++i) {
1089         env->fptags[i] = !((fpu.ftwx >> i) & 1);
1090     }
1091     memcpy(env->fpregs, fpu.fpr, sizeof env->fpregs);
1092     memcpy(env->xmm_regs, fpu.xmm, sizeof env->xmm_regs);
1093     env->mxcsr = fpu.mxcsr;
1094
1095     return 0;
1096 }
1097
1098 static int kvm_get_xsave(CPUState *env)
1099 {
1100 #ifdef KVM_CAP_XSAVE
1101     struct kvm_xsave* xsave;
1102     int ret, i;
1103     uint16_t cwd, swd, twd, fop;
1104
1105     if (!kvm_has_xsave()) {
1106         return kvm_get_fpu(env);
1107     }
1108
1109     xsave = qemu_memalign(4096, sizeof(struct kvm_xsave));
1110     ret = kvm_vcpu_ioctl(env, KVM_GET_XSAVE, xsave);
1111     if (ret < 0) {
1112         qemu_free(xsave);
1113         return ret;
1114     }
1115
1116     cwd = (uint16_t)xsave->region[0];
1117     swd = (uint16_t)(xsave->region[0] >> 16);
1118     twd = (uint16_t)xsave->region[1];
1119     fop = (uint16_t)(xsave->region[1] >> 16);
1120     env->fpstt = (swd >> 11) & 7;
1121     env->fpus = swd;
1122     env->fpuc = cwd;
1123     for (i = 0; i < 8; ++i) {
1124         env->fptags[i] = !((twd >> i) & 1);
1125     }
1126     env->mxcsr = xsave->region[XSAVE_MXCSR];
1127     memcpy(env->fpregs, &xsave->region[XSAVE_ST_SPACE],
1128             sizeof env->fpregs);
1129     memcpy(env->xmm_regs, &xsave->region[XSAVE_XMM_SPACE],
1130             sizeof env->xmm_regs);
1131     env->xstate_bv = *(uint64_t *)&xsave->region[XSAVE_XSTATE_BV];
1132     memcpy(env->ymmh_regs, &xsave->region[XSAVE_YMMH_SPACE],
1133             sizeof env->ymmh_regs);
1134     qemu_free(xsave);
1135     return 0;
1136 #else
1137     return kvm_get_fpu(env);
1138 #endif
1139 }
1140
1141 static int kvm_get_xcrs(CPUState *env)
1142 {
1143 #ifdef KVM_CAP_XCRS
1144     int i, ret;
1145     struct kvm_xcrs xcrs;
1146
1147     if (!kvm_has_xcrs()) {
1148         return 0;
1149     }
1150
1151     ret = kvm_vcpu_ioctl(env, KVM_GET_XCRS, &xcrs);
1152     if (ret < 0) {
1153         return ret;
1154     }
1155
1156     for (i = 0; i < xcrs.nr_xcrs; i++) {
1157         /* Only support xcr0 now */
1158         if (xcrs.xcrs[0].xcr == 0) {
1159             env->xcr0 = xcrs.xcrs[0].value;
1160             break;
1161         }
1162     }
1163     return 0;
1164 #else
1165     return 0;
1166 #endif
1167 }
1168
1169 static int kvm_get_sregs(CPUState *env)
1170 {
1171     struct kvm_sregs sregs;
1172     uint32_t hflags;
1173     int bit, i, ret;
1174
1175     ret = kvm_vcpu_ioctl(env, KVM_GET_SREGS, &sregs);
1176     if (ret < 0) {
1177         return ret;
1178     }
1179
1180     /* There can only be one pending IRQ set in the bitmap at a time, so try
1181        to find it and save its number instead (-1 for none). */
1182     env->interrupt_injected = -1;
1183     for (i = 0; i < ARRAY_SIZE(sregs.interrupt_bitmap); i++) {
1184         if (sregs.interrupt_bitmap[i]) {
1185             bit = ctz64(sregs.interrupt_bitmap[i]);
1186             env->interrupt_injected = i * 64 + bit;
1187             break;
1188         }
1189     }
1190
1191     get_seg(&env->segs[R_CS], &sregs.cs);
1192     get_seg(&env->segs[R_DS], &sregs.ds);
1193     get_seg(&env->segs[R_ES], &sregs.es);
1194     get_seg(&env->segs[R_FS], &sregs.fs);
1195     get_seg(&env->segs[R_GS], &sregs.gs);
1196     get_seg(&env->segs[R_SS], &sregs.ss);
1197
1198     get_seg(&env->tr, &sregs.tr);
1199     get_seg(&env->ldt, &sregs.ldt);
1200
1201     env->idt.limit = sregs.idt.limit;
1202     env->idt.base = sregs.idt.base;
1203     env->gdt.limit = sregs.gdt.limit;
1204     env->gdt.base = sregs.gdt.base;
1205
1206     env->cr[0] = sregs.cr0;
1207     env->cr[2] = sregs.cr2;
1208     env->cr[3] = sregs.cr3;
1209     env->cr[4] = sregs.cr4;
1210
1211     cpu_set_apic_base(env->apic_state, sregs.apic_base);
1212
1213     env->efer = sregs.efer;
1214     //cpu_set_apic_tpr(env->apic_state, sregs.cr8);
1215
1216 #define HFLAG_COPY_MASK \
1217     ~( HF_CPL_MASK | HF_PE_MASK | HF_MP_MASK | HF_EM_MASK | \
1218        HF_TS_MASK | HF_TF_MASK | HF_VM_MASK | HF_IOPL_MASK | \
1219        HF_OSFXSR_MASK | HF_LMA_MASK | HF_CS32_MASK | \
1220        HF_SS32_MASK | HF_CS64_MASK | HF_ADDSEG_MASK)
1221
1222     hflags = (env->segs[R_CS].flags >> DESC_DPL_SHIFT) & HF_CPL_MASK;
1223     hflags |= (env->cr[0] & CR0_PE_MASK) << (HF_PE_SHIFT - CR0_PE_SHIFT);
1224     hflags |= (env->cr[0] << (HF_MP_SHIFT - CR0_MP_SHIFT)) &
1225                 (HF_MP_MASK | HF_EM_MASK | HF_TS_MASK);
1226     hflags |= (env->eflags & (HF_TF_MASK | HF_VM_MASK | HF_IOPL_MASK));
1227     hflags |= (env->cr[4] & CR4_OSFXSR_MASK) <<
1228                 (HF_OSFXSR_SHIFT - CR4_OSFXSR_SHIFT);
1229
1230     if (env->efer & MSR_EFER_LMA) {
1231         hflags |= HF_LMA_MASK;
1232     }
1233
1234     if ((hflags & HF_LMA_MASK) && (env->segs[R_CS].flags & DESC_L_MASK)) {
1235         hflags |= HF_CS32_MASK | HF_SS32_MASK | HF_CS64_MASK;
1236     } else {
1237         hflags |= (env->segs[R_CS].flags & DESC_B_MASK) >>
1238                     (DESC_B_SHIFT - HF_CS32_SHIFT);
1239         hflags |= (env->segs[R_SS].flags & DESC_B_MASK) >>
1240                     (DESC_B_SHIFT - HF_SS32_SHIFT);
1241         if (!(env->cr[0] & CR0_PE_MASK) || (env->eflags & VM_MASK) ||
1242             !(hflags & HF_CS32_MASK)) {
1243             hflags |= HF_ADDSEG_MASK;
1244         } else {
1245             hflags |= ((env->segs[R_DS].base | env->segs[R_ES].base |
1246                         env->segs[R_SS].base) != 0) << HF_ADDSEG_SHIFT;
1247         }
1248     }
1249     env->hflags = (env->hflags & HFLAG_COPY_MASK) | hflags;
1250
1251     return 0;
1252 }
1253
1254 static int kvm_get_msrs(CPUState *env)
1255 {
1256     struct {
1257         struct kvm_msrs info;
1258         struct kvm_msr_entry entries[100];
1259     } msr_data;
1260     struct kvm_msr_entry *msrs = msr_data.entries;
1261     int ret, i, n;
1262
1263     n = 0;
1264     msrs[n++].index = MSR_IA32_SYSENTER_CS;
1265     msrs[n++].index = MSR_IA32_SYSENTER_ESP;
1266     msrs[n++].index = MSR_IA32_SYSENTER_EIP;
1267     if (has_msr_star) {
1268         msrs[n++].index = MSR_STAR;
1269     }
1270     if (has_msr_hsave_pa) {
1271         msrs[n++].index = MSR_VM_HSAVE_PA;
1272     }
1273
1274     if (!env->tsc_valid) {
1275         msrs[n++].index = MSR_IA32_TSC;
1276         env->tsc_valid = !vm_running;
1277     }
1278
1279 #ifdef TARGET_X86_64
1280     if (lm_capable_kernel) {
1281         msrs[n++].index = MSR_CSTAR;
1282         msrs[n++].index = MSR_KERNELGSBASE;
1283         msrs[n++].index = MSR_FMASK;
1284         msrs[n++].index = MSR_LSTAR;
1285     }
1286 #endif
1287     msrs[n++].index = MSR_KVM_SYSTEM_TIME;
1288     msrs[n++].index = MSR_KVM_WALL_CLOCK;
1289 #if defined(CONFIG_KVM_PARA) && defined(KVM_CAP_ASYNC_PF)
1290     if (has_msr_async_pf_en) {
1291         msrs[n++].index = MSR_KVM_ASYNC_PF_EN;
1292     }
1293 #endif
1294
1295 #ifdef KVM_CAP_MCE
1296     if (env->mcg_cap) {
1297         msrs[n++].index = MSR_MCG_STATUS;
1298         msrs[n++].index = MSR_MCG_CTL;
1299         for (i = 0; i < (env->mcg_cap & 0xff) * 4; i++) {
1300             msrs[n++].index = MSR_MC0_CTL + i;
1301         }
1302     }
1303 #endif
1304
1305     msr_data.info.nmsrs = n;
1306     ret = kvm_vcpu_ioctl(env, KVM_GET_MSRS, &msr_data);
1307     if (ret < 0) {
1308         return ret;
1309     }
1310
1311     for (i = 0; i < ret; i++) {
1312         switch (msrs[i].index) {
1313         case MSR_IA32_SYSENTER_CS:
1314             env->sysenter_cs = msrs[i].data;
1315             break;
1316         case MSR_IA32_SYSENTER_ESP:
1317             env->sysenter_esp = msrs[i].data;
1318             break;
1319         case MSR_IA32_SYSENTER_EIP:
1320             env->sysenter_eip = msrs[i].data;
1321             break;
1322         case MSR_STAR:
1323             env->star = msrs[i].data;
1324             break;
1325 #ifdef TARGET_X86_64
1326         case MSR_CSTAR:
1327             env->cstar = msrs[i].data;
1328             break;
1329         case MSR_KERNELGSBASE:
1330             env->kernelgsbase = msrs[i].data;
1331             break;
1332         case MSR_FMASK:
1333             env->fmask = msrs[i].data;
1334             break;
1335         case MSR_LSTAR:
1336             env->lstar = msrs[i].data;
1337             break;
1338 #endif
1339         case MSR_IA32_TSC:
1340             env->tsc = msrs[i].data;
1341             break;
1342         case MSR_VM_HSAVE_PA:
1343             env->vm_hsave = msrs[i].data;
1344             break;
1345         case MSR_KVM_SYSTEM_TIME:
1346             env->system_time_msr = msrs[i].data;
1347             break;
1348         case MSR_KVM_WALL_CLOCK:
1349             env->wall_clock_msr = msrs[i].data;
1350             break;
1351 #ifdef KVM_CAP_MCE
1352         case MSR_MCG_STATUS:
1353             env->mcg_status = msrs[i].data;
1354             break;
1355         case MSR_MCG_CTL:
1356             env->mcg_ctl = msrs[i].data;
1357             break;
1358 #endif
1359         default:
1360 #ifdef KVM_CAP_MCE
1361             if (msrs[i].index >= MSR_MC0_CTL &&
1362                 msrs[i].index < MSR_MC0_CTL + (env->mcg_cap & 0xff) * 4) {
1363                 env->mce_banks[msrs[i].index - MSR_MC0_CTL] = msrs[i].data;
1364             }
1365 #endif
1366             break;
1367 #if defined(CONFIG_KVM_PARA) && defined(KVM_CAP_ASYNC_PF)
1368         case MSR_KVM_ASYNC_PF_EN:
1369             env->async_pf_en_msr = msrs[i].data;
1370             break;
1371 #endif
1372         }
1373     }
1374
1375     return 0;
1376 }
1377
1378 static int kvm_put_mp_state(CPUState *env)
1379 {
1380     struct kvm_mp_state mp_state = { .mp_state = env->mp_state };
1381
1382     return kvm_vcpu_ioctl(env, KVM_SET_MP_STATE, &mp_state);
1383 }
1384
1385 static int kvm_get_mp_state(CPUState *env)
1386 {
1387     struct kvm_mp_state mp_state;
1388     int ret;
1389
1390     ret = kvm_vcpu_ioctl(env, KVM_GET_MP_STATE, &mp_state);
1391     if (ret < 0) {
1392         return ret;
1393     }
1394     env->mp_state = mp_state.mp_state;
1395     if (kvm_irqchip_in_kernel()) {
1396         env->halted = (mp_state.mp_state == KVM_MP_STATE_HALTED);
1397     }
1398     return 0;
1399 }
1400
1401 static int kvm_put_vcpu_events(CPUState *env, int level)
1402 {
1403 #ifdef KVM_CAP_VCPU_EVENTS
1404     struct kvm_vcpu_events events;
1405
1406     if (!kvm_has_vcpu_events()) {
1407         return 0;
1408     }
1409
1410     events.exception.injected = (env->exception_injected >= 0);
1411     events.exception.nr = env->exception_injected;
1412     events.exception.has_error_code = env->has_error_code;
1413     events.exception.error_code = env->error_code;
1414
1415     events.interrupt.injected = (env->interrupt_injected >= 0);
1416     events.interrupt.nr = env->interrupt_injected;
1417     events.interrupt.soft = env->soft_interrupt;
1418
1419     events.nmi.injected = env->nmi_injected;
1420     events.nmi.pending = env->nmi_pending;
1421     events.nmi.masked = !!(env->hflags2 & HF2_NMI_MASK);
1422
1423     events.sipi_vector = env->sipi_vector;
1424
1425     events.flags = 0;
1426     if (level >= KVM_PUT_RESET_STATE) {
1427         events.flags |=
1428             KVM_VCPUEVENT_VALID_NMI_PENDING | KVM_VCPUEVENT_VALID_SIPI_VECTOR;
1429     }
1430
1431     return kvm_vcpu_ioctl(env, KVM_SET_VCPU_EVENTS, &events);
1432 #else
1433     return 0;
1434 #endif
1435 }
1436
1437 static int kvm_get_vcpu_events(CPUState *env)
1438 {
1439 #ifdef KVM_CAP_VCPU_EVENTS
1440     struct kvm_vcpu_events events;
1441     int ret;
1442
1443     if (!kvm_has_vcpu_events()) {
1444         return 0;
1445     }
1446
1447     ret = kvm_vcpu_ioctl(env, KVM_GET_VCPU_EVENTS, &events);
1448     if (ret < 0) {
1449        return ret;
1450     }
1451     env->exception_injected =
1452        events.exception.injected ? events.exception.nr : -1;
1453     env->has_error_code = events.exception.has_error_code;
1454     env->error_code = events.exception.error_code;
1455
1456     env->interrupt_injected =
1457         events.interrupt.injected ? events.interrupt.nr : -1;
1458     env->soft_interrupt = events.interrupt.soft;
1459
1460     env->nmi_injected = events.nmi.injected;
1461     env->nmi_pending = events.nmi.pending;
1462     if (events.nmi.masked) {
1463         env->hflags2 |= HF2_NMI_MASK;
1464     } else {
1465         env->hflags2 &= ~HF2_NMI_MASK;
1466     }
1467
1468     env->sipi_vector = events.sipi_vector;
1469 #endif
1470
1471     return 0;
1472 }
1473
1474 static int kvm_guest_debug_workarounds(CPUState *env)
1475 {
1476     int ret = 0;
1477 #ifdef KVM_CAP_SET_GUEST_DEBUG
1478     unsigned long reinject_trap = 0;
1479
1480     if (!kvm_has_vcpu_events()) {
1481         if (env->exception_injected == 1) {
1482             reinject_trap = KVM_GUESTDBG_INJECT_DB;
1483         } else if (env->exception_injected == 3) {
1484             reinject_trap = KVM_GUESTDBG_INJECT_BP;
1485         }
1486         env->exception_injected = -1;
1487     }
1488
1489     /*
1490      * Kernels before KVM_CAP_X86_ROBUST_SINGLESTEP overwrote flags.TF
1491      * injected via SET_GUEST_DEBUG while updating GP regs. Work around this
1492      * by updating the debug state once again if single-stepping is on.
1493      * Another reason to call kvm_update_guest_debug here is a pending debug
1494      * trap raise by the guest. On kernels without SET_VCPU_EVENTS we have to
1495      * reinject them via SET_GUEST_DEBUG.
1496      */
1497     if (reinject_trap ||
1498         (!kvm_has_robust_singlestep() && env->singlestep_enabled)) {
1499         ret = kvm_update_guest_debug(env, reinject_trap);
1500     }
1501 #endif /* KVM_CAP_SET_GUEST_DEBUG */
1502     return ret;
1503 }
1504
1505 static int kvm_put_debugregs(CPUState *env)
1506 {
1507 #ifdef KVM_CAP_DEBUGREGS
1508     struct kvm_debugregs dbgregs;
1509     int i;
1510
1511     if (!kvm_has_debugregs()) {
1512         return 0;
1513     }
1514
1515     for (i = 0; i < 4; i++) {
1516         dbgregs.db[i] = env->dr[i];
1517     }
1518     dbgregs.dr6 = env->dr[6];
1519     dbgregs.dr7 = env->dr[7];
1520     dbgregs.flags = 0;
1521
1522     return kvm_vcpu_ioctl(env, KVM_SET_DEBUGREGS, &dbgregs);
1523 #else
1524     return 0;
1525 #endif
1526 }
1527
1528 static int kvm_get_debugregs(CPUState *env)
1529 {
1530 #ifdef KVM_CAP_DEBUGREGS
1531     struct kvm_debugregs dbgregs;
1532     int i, ret;
1533
1534     if (!kvm_has_debugregs()) {
1535         return 0;
1536     }
1537
1538     ret = kvm_vcpu_ioctl(env, KVM_GET_DEBUGREGS, &dbgregs);
1539     if (ret < 0) {
1540         return ret;
1541     }
1542     for (i = 0; i < 4; i++) {
1543         env->dr[i] = dbgregs.db[i];
1544     }
1545     env->dr[4] = env->dr[6] = dbgregs.dr6;
1546     env->dr[5] = env->dr[7] = dbgregs.dr7;
1547 #endif
1548
1549     return 0;
1550 }
1551
1552 int kvm_arch_put_registers(CPUState *env, int level)
1553 {
1554     int ret;
1555
1556     assert(cpu_is_stopped(env) || qemu_cpu_is_self(env));
1557
1558     ret = kvm_getput_regs(env, 1);
1559     if (ret < 0) {
1560         return ret;
1561     }
1562     ret = kvm_put_xsave(env);
1563     if (ret < 0) {
1564         return ret;
1565     }
1566     ret = kvm_put_xcrs(env);
1567     if (ret < 0) {
1568         return ret;
1569     }
1570     ret = kvm_put_sregs(env);
1571     if (ret < 0) {
1572         return ret;
1573     }
1574     /* must be before kvm_put_msrs */
1575     ret = kvm_inject_mce_oldstyle(env);
1576     if (ret < 0) {
1577         return ret;
1578     }
1579     ret = kvm_put_msrs(env, level);
1580     if (ret < 0) {
1581         return ret;
1582     }
1583     if (level >= KVM_PUT_RESET_STATE) {
1584         ret = kvm_put_mp_state(env);
1585         if (ret < 0) {
1586             return ret;
1587         }
1588     }
1589     ret = kvm_put_vcpu_events(env, level);
1590     if (ret < 0) {
1591         return ret;
1592     }
1593     ret = kvm_put_debugregs(env);
1594     if (ret < 0) {
1595         return ret;
1596     }
1597     /* must be last */
1598     ret = kvm_guest_debug_workarounds(env);
1599     if (ret < 0) {
1600         return ret;
1601     }
1602     return 0;
1603 }
1604
1605 int kvm_arch_get_registers(CPUState *env)
1606 {
1607     int ret;
1608
1609     assert(cpu_is_stopped(env) || qemu_cpu_is_self(env));
1610
1611     ret = kvm_getput_regs(env, 0);
1612     if (ret < 0) {
1613         return ret;
1614     }
1615     ret = kvm_get_xsave(env);
1616     if (ret < 0) {
1617         return ret;
1618     }
1619     ret = kvm_get_xcrs(env);
1620     if (ret < 0) {
1621         return ret;
1622     }
1623     ret = kvm_get_sregs(env);
1624     if (ret < 0) {
1625         return ret;
1626     }
1627     ret = kvm_get_msrs(env);
1628     if (ret < 0) {
1629         return ret;
1630     }
1631     ret = kvm_get_mp_state(env);
1632     if (ret < 0) {
1633         return ret;
1634     }
1635     ret = kvm_get_vcpu_events(env);
1636     if (ret < 0) {
1637         return ret;
1638     }
1639     ret = kvm_get_debugregs(env);
1640     if (ret < 0) {
1641         return ret;
1642     }
1643     return 0;
1644 }
1645
1646 void kvm_arch_pre_run(CPUState *env, struct kvm_run *run)
1647 {
1648     int ret;
1649
1650     /* Inject NMI */
1651     if (env->interrupt_request & CPU_INTERRUPT_NMI) {
1652         env->interrupt_request &= ~CPU_INTERRUPT_NMI;
1653         DPRINTF("injected NMI\n");
1654         ret = kvm_vcpu_ioctl(env, KVM_NMI);
1655         if (ret < 0) {
1656             fprintf(stderr, "KVM: injection failed, NMI lost (%s)\n",
1657                     strerror(-ret));
1658         }
1659     }
1660
1661     if (!kvm_irqchip_in_kernel()) {
1662         /* Force the VCPU out of its inner loop to process the INIT request */
1663         if (env->interrupt_request & CPU_INTERRUPT_INIT) {
1664             env->exit_request = 1;
1665         }
1666
1667         /* Try to inject an interrupt if the guest can accept it */
1668         if (run->ready_for_interrupt_injection &&
1669             (env->interrupt_request & CPU_INTERRUPT_HARD) &&
1670             (env->eflags & IF_MASK)) {
1671             int irq;
1672
1673             env->interrupt_request &= ~CPU_INTERRUPT_HARD;
1674             irq = cpu_get_pic_interrupt(env);
1675             if (irq >= 0) {
1676                 struct kvm_interrupt intr;
1677
1678                 intr.irq = irq;
1679                 DPRINTF("injected interrupt %d\n", irq);
1680                 ret = kvm_vcpu_ioctl(env, KVM_INTERRUPT, &intr);
1681                 if (ret < 0) {
1682                     fprintf(stderr,
1683                             "KVM: injection failed, interrupt lost (%s)\n",
1684                             strerror(-ret));
1685                 }
1686             }
1687         }
1688
1689         /* If we have an interrupt but the guest is not ready to receive an
1690          * interrupt, request an interrupt window exit.  This will
1691          * cause a return to userspace as soon as the guest is ready to
1692          * receive interrupts. */
1693         if ((env->interrupt_request & CPU_INTERRUPT_HARD)) {
1694             run->request_interrupt_window = 1;
1695         } else {
1696             run->request_interrupt_window = 0;
1697         }
1698
1699         DPRINTF("setting tpr\n");
1700         run->cr8 = cpu_get_apic_tpr(env->apic_state);
1701     }
1702 }
1703
1704 void kvm_arch_post_run(CPUState *env, struct kvm_run *run)
1705 {
1706     if (run->if_flag) {
1707         env->eflags |= IF_MASK;
1708     } else {
1709         env->eflags &= ~IF_MASK;
1710     }
1711     cpu_set_apic_tpr(env->apic_state, run->cr8);
1712     cpu_set_apic_base(env->apic_state, run->apic_base);
1713 }
1714
1715 int kvm_arch_process_async_events(CPUState *env)
1716 {
1717     if (env->interrupt_request & CPU_INTERRUPT_MCE) {
1718         /* We must not raise CPU_INTERRUPT_MCE if it's not supported. */
1719         assert(env->mcg_cap);
1720
1721         env->interrupt_request &= ~CPU_INTERRUPT_MCE;
1722
1723         kvm_cpu_synchronize_state(env);
1724
1725         if (env->exception_injected == EXCP08_DBLE) {
1726             /* this means triple fault */
1727             qemu_system_reset_request();
1728             env->exit_request = 1;
1729             return 0;
1730         }
1731         env->exception_injected = EXCP12_MCHK;
1732         env->has_error_code = 0;
1733
1734         env->halted = 0;
1735         if (kvm_irqchip_in_kernel() && env->mp_state == KVM_MP_STATE_HALTED) {
1736             env->mp_state = KVM_MP_STATE_RUNNABLE;
1737         }
1738     }
1739
1740     if (kvm_irqchip_in_kernel()) {
1741         return 0;
1742     }
1743
1744     if (env->interrupt_request & (CPU_INTERRUPT_HARD | CPU_INTERRUPT_NMI)) {
1745         env->halted = 0;
1746     }
1747     if (env->interrupt_request & CPU_INTERRUPT_INIT) {
1748         kvm_cpu_synchronize_state(env);
1749         do_cpu_init(env);
1750     }
1751     if (env->interrupt_request & CPU_INTERRUPT_SIPI) {
1752         kvm_cpu_synchronize_state(env);
1753         do_cpu_sipi(env);
1754     }
1755
1756     return env->halted;
1757 }
1758
1759 static int kvm_handle_halt(CPUState *env)
1760 {
1761     if (!((env->interrupt_request & CPU_INTERRUPT_HARD) &&
1762           (env->eflags & IF_MASK)) &&
1763         !(env->interrupt_request & CPU_INTERRUPT_NMI)) {
1764         env->halted = 1;
1765         return 0;
1766     }
1767
1768     return 1;
1769 }
1770
1771 static bool host_supports_vmx(void)
1772 {
1773     uint32_t ecx, unused;
1774
1775     host_cpuid(1, 0, &unused, &unused, &ecx, &unused);
1776     return ecx & CPUID_EXT_VMX;
1777 }
1778
1779 #define VMX_INVALID_GUEST_STATE 0x80000021
1780
1781 int kvm_arch_handle_exit(CPUState *env, struct kvm_run *run)
1782 {
1783     uint64_t code;
1784     int ret = 0;
1785
1786     switch (run->exit_reason) {
1787     case KVM_EXIT_HLT:
1788         DPRINTF("handle_hlt\n");
1789         ret = kvm_handle_halt(env);
1790         break;
1791     case KVM_EXIT_SET_TPR:
1792         ret = 1;
1793         break;
1794     case KVM_EXIT_FAIL_ENTRY:
1795         code = run->fail_entry.hardware_entry_failure_reason;
1796         fprintf(stderr, "KVM: entry failed, hardware error 0x%" PRIx64 "\n",
1797                 code);
1798         if (host_supports_vmx() && code == VMX_INVALID_GUEST_STATE) {
1799             fprintf(stderr,
1800                     "\nIf you're runnning a guest on an Intel machine without "
1801                         "unrestricted mode\n"
1802                     "support, the failure can be most likely due to the guest "
1803                         "entering an invalid\n"
1804                     "state for Intel VT. For example, the guest maybe running "
1805                         "in big real mode\n"
1806                     "which is not supported on less recent Intel processors."
1807                         "\n\n");
1808         }
1809         ret = -1;
1810         break;
1811     case KVM_EXIT_EXCEPTION:
1812         fprintf(stderr, "KVM: exception %d exit (error code 0x%x)\n",
1813                 run->ex.exception, run->ex.error_code);
1814         ret = -1;
1815         break;
1816     default:
1817         fprintf(stderr, "KVM: unknown exit reason %d\n", run->exit_reason);
1818         ret = -1;
1819         break;
1820     }
1821
1822     return ret;
1823 }
1824
1825 #ifdef KVM_CAP_SET_GUEST_DEBUG
1826 int kvm_arch_insert_sw_breakpoint(CPUState *env, struct kvm_sw_breakpoint *bp)
1827 {
1828     static const uint8_t int3 = 0xcc;
1829
1830     if (cpu_memory_rw_debug(env, bp->pc, (uint8_t *)&bp->saved_insn, 1, 0) ||
1831         cpu_memory_rw_debug(env, bp->pc, (uint8_t *)&int3, 1, 1)) {
1832         return -EINVAL;
1833     }
1834     return 0;
1835 }
1836
1837 int kvm_arch_remove_sw_breakpoint(CPUState *env, struct kvm_sw_breakpoint *bp)
1838 {
1839     uint8_t int3;
1840
1841     if (cpu_memory_rw_debug(env, bp->pc, &int3, 1, 0) || int3 != 0xcc ||
1842         cpu_memory_rw_debug(env, bp->pc, (uint8_t *)&bp->saved_insn, 1, 1)) {
1843         return -EINVAL;
1844     }
1845     return 0;
1846 }
1847
1848 static struct {
1849     target_ulong addr;
1850     int len;
1851     int type;
1852 } hw_breakpoint[4];
1853
1854 static int nb_hw_breakpoint;
1855
1856 static int find_hw_breakpoint(target_ulong addr, int len, int type)
1857 {
1858     int n;
1859
1860     for (n = 0; n < nb_hw_breakpoint; n++) {
1861         if (hw_breakpoint[n].addr == addr && hw_breakpoint[n].type == type &&
1862             (hw_breakpoint[n].len == len || len == -1)) {
1863             return n;
1864         }
1865     }
1866     return -1;
1867 }
1868
1869 int kvm_arch_insert_hw_breakpoint(target_ulong addr,
1870                                   target_ulong len, int type)
1871 {
1872     switch (type) {
1873     case GDB_BREAKPOINT_HW:
1874         len = 1;
1875         break;
1876     case GDB_WATCHPOINT_WRITE:
1877     case GDB_WATCHPOINT_ACCESS:
1878         switch (len) {
1879         case 1:
1880             break;
1881         case 2:
1882         case 4:
1883         case 8:
1884             if (addr & (len - 1)) {
1885                 return -EINVAL;
1886             }
1887             break;
1888         default:
1889             return -EINVAL;
1890         }
1891         break;
1892     default:
1893         return -ENOSYS;
1894     }
1895
1896     if (nb_hw_breakpoint == 4) {
1897         return -ENOBUFS;
1898     }
1899     if (find_hw_breakpoint(addr, len, type) >= 0) {
1900         return -EEXIST;
1901     }
1902     hw_breakpoint[nb_hw_breakpoint].addr = addr;
1903     hw_breakpoint[nb_hw_breakpoint].len = len;
1904     hw_breakpoint[nb_hw_breakpoint].type = type;
1905     nb_hw_breakpoint++;
1906
1907     return 0;
1908 }
1909
1910 int kvm_arch_remove_hw_breakpoint(target_ulong addr,
1911                                   target_ulong len, int type)
1912 {
1913     int n;
1914
1915     n = find_hw_breakpoint(addr, (type == GDB_BREAKPOINT_HW) ? 1 : len, type);
1916     if (n < 0) {
1917         return -ENOENT;
1918     }
1919     nb_hw_breakpoint--;
1920     hw_breakpoint[n] = hw_breakpoint[nb_hw_breakpoint];
1921
1922     return 0;
1923 }
1924
1925 void kvm_arch_remove_all_hw_breakpoints(void)
1926 {
1927     nb_hw_breakpoint = 0;
1928 }
1929
1930 static CPUWatchpoint hw_watchpoint;
1931
1932 int kvm_arch_debug(struct kvm_debug_exit_arch *arch_info)
1933 {
1934     int handle = 0;
1935     int n;
1936
1937     if (arch_info->exception == 1) {
1938         if (arch_info->dr6 & (1 << 14)) {
1939             if (cpu_single_env->singlestep_enabled) {
1940                 handle = 1;
1941             }
1942         } else {
1943             for (n = 0; n < 4; n++) {
1944                 if (arch_info->dr6 & (1 << n)) {
1945                     switch ((arch_info->dr7 >> (16 + n*4)) & 0x3) {
1946                     case 0x0:
1947                         handle = 1;
1948                         break;
1949                     case 0x1:
1950                         handle = 1;
1951                         cpu_single_env->watchpoint_hit = &hw_watchpoint;
1952                         hw_watchpoint.vaddr = hw_breakpoint[n].addr;
1953                         hw_watchpoint.flags = BP_MEM_WRITE;
1954                         break;
1955                     case 0x3:
1956                         handle = 1;
1957                         cpu_single_env->watchpoint_hit = &hw_watchpoint;
1958                         hw_watchpoint.vaddr = hw_breakpoint[n].addr;
1959                         hw_watchpoint.flags = BP_MEM_ACCESS;
1960                         break;
1961                     }
1962                 }
1963             }
1964         }
1965     } else if (kvm_find_sw_breakpoint(cpu_single_env, arch_info->pc)) {
1966         handle = 1;
1967     }
1968     if (!handle) {
1969         cpu_synchronize_state(cpu_single_env);
1970         assert(cpu_single_env->exception_injected == -1);
1971
1972         cpu_single_env->exception_injected = arch_info->exception;
1973         cpu_single_env->has_error_code = 0;
1974     }
1975
1976     return handle;
1977 }
1978
1979 void kvm_arch_update_guest_debug(CPUState *env, struct kvm_guest_debug *dbg)
1980 {
1981     const uint8_t type_code[] = {
1982         [GDB_BREAKPOINT_HW] = 0x0,
1983         [GDB_WATCHPOINT_WRITE] = 0x1,
1984         [GDB_WATCHPOINT_ACCESS] = 0x3
1985     };
1986     const uint8_t len_code[] = {
1987         [1] = 0x0, [2] = 0x1, [4] = 0x3, [8] = 0x2
1988     };
1989     int n;
1990
1991     if (kvm_sw_breakpoints_active(env)) {
1992         dbg->control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP;
1993     }
1994     if (nb_hw_breakpoint > 0) {
1995         dbg->control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_HW_BP;
1996         dbg->arch.debugreg[7] = 0x0600;
1997         for (n = 0; n < nb_hw_breakpoint; n++) {
1998             dbg->arch.debugreg[n] = hw_breakpoint[n].addr;
1999             dbg->arch.debugreg[7] |= (2 << (n * 2)) |
2000                 (type_code[hw_breakpoint[n].type] << (16 + n*4)) |
2001                 ((uint32_t)len_code[hw_breakpoint[n].len] << (18 + n*4));
2002         }
2003     }
2004 }
2005 #endif /* KVM_CAP_SET_GUEST_DEBUG */
2006
2007 bool kvm_arch_stop_on_emulation_error(CPUState *env)
2008 {
2009     return !(env->cr[0] & CR0_PE_MASK) ||
2010            ((env->segs[R_CS].selector  & 3) != 3);
2011 }