target/i386/nvmm/nvmm-all.c

   1 /*
   2  * Copyright (c) 2018-2019 Maxime Villard, All rights reserved.
   3  *
   4  * NetBSD Virtual Machine Monitor (NVMM) accelerator for QEMU.
   5  *
   6  * This work is licensed under the terms of the GNU GPL, version 2 or later.
   7  * See the COPYING file in the top-level directory.
   8  */
   9
  10 #include "qemu/osdep.h"
  11 #include "cpu.h"
  12 #include "exec/address-spaces.h"
  13 #include "exec/ioport.h"
  14 #include "qemu/accel.h"
  15 #include "sysemu/nvmm.h"
  16 #include "sysemu/cpus.h"
  17 #include "sysemu/runstate.h"
  18 #include "qemu/main-loop.h"
  19 #include "qemu/error-report.h"
  20 #include "qapi/error.h"
  21 #include "qemu/queue.h"
  22 #include "migration/blocker.h"
  23 #include "strings.h"
  24
  25 #include "nvmm-accel-ops.h"
  26
  27 #include <nvmm.h>
  28
  29 struct AccelCPUState {
  30     struct nvmm_vcpu vcpu;
  31     uint8_t tpr;
  32     bool stop;
  33
  34     /* Window-exiting for INTs/NMIs. */
  35     bool int_window_exit;
  36     bool nmi_window_exit;
  37
  38     /* The guest is in an interrupt shadow (POP SS, etc). */
  39     bool int_shadow;
  40 };
  41
  42 struct qemu_machine {
  43     struct nvmm_capability cap;
  44     struct nvmm_machine mach;
  45 };
  46
  47 /* -------------------------------------------------------------------------- */
  48
  49 static bool nvmm_allowed;
  50 static struct qemu_machine qemu_mach;
  51
  52 static struct nvmm_machine *
  53 get_nvmm_mach(void)
  54 {
  55     return &qemu_mach.mach;
  56 }
  57
  58 /* -------------------------------------------------------------------------- */
  59
  60 static void
  61 nvmm_set_segment(struct nvmm_x64_state_seg *nseg, const SegmentCache *qseg)
  62 {
  63     uint32_t attrib = qseg->flags;
  64
  65     nseg->selector = qseg->selector;
  66     nseg->limit = qseg->limit;
  67     nseg->base = qseg->base;
  68     nseg->attrib.type = __SHIFTOUT(attrib, DESC_TYPE_MASK);
  69     nseg->attrib.s = __SHIFTOUT(attrib, DESC_S_MASK);
  70     nseg->attrib.dpl = __SHIFTOUT(attrib, DESC_DPL_MASK);
  71     nseg->attrib.p = __SHIFTOUT(attrib, DESC_P_MASK);
  72     nseg->attrib.avl = __SHIFTOUT(attrib, DESC_AVL_MASK);
  73     nseg->attrib.l = __SHIFTOUT(attrib, DESC_L_MASK);
  74     nseg->attrib.def = __SHIFTOUT(attrib, DESC_B_MASK);
  75     nseg->attrib.g = __SHIFTOUT(attrib, DESC_G_MASK);
  76 }
  77
  78 static void
  79 nvmm_set_registers(CPUState *cpu)
  80 {
  81     CPUX86State *env = cpu_env(cpu);
  82     struct nvmm_machine *mach = get_nvmm_mach();
  83     AccelCPUState *qcpu = cpu->accel;
  84     struct nvmm_vcpu *vcpu = &qcpu->vcpu;
  85     struct nvmm_x64_state *state = vcpu->state;
  86     uint64_t bitmap;
  87     size_t i;
  88     int ret;
  89
  90     assert(cpu_is_stopped(cpu) || qemu_cpu_is_self(cpu));
  91
  92     /* GPRs. */
  93     state->gprs[NVMM_X64_GPR_RAX] = env->regs[R_EAX];
  94     state->gprs[NVMM_X64_GPR_RCX] = env->regs[R_ECX];
  95     state->gprs[NVMM_X64_GPR_RDX] = env->regs[R_EDX];
  96     state->gprs[NVMM_X64_GPR_RBX] = env->regs[R_EBX];
  97     state->gprs[NVMM_X64_GPR_RSP] = env->regs[R_ESP];
  98     state->gprs[NVMM_X64_GPR_RBP] = env->regs[R_EBP];
  99     state->gprs[NVMM_X64_GPR_RSI] = env->regs[R_ESI];
 100     state->gprs[NVMM_X64_GPR_RDI] = env->regs[R_EDI];
 101 #ifdef TARGET_X86_64
 102     state->gprs[NVMM_X64_GPR_R8]  = env->regs[R_R8];
 103     state->gprs[NVMM_X64_GPR_R9]  = env->regs[R_R9];
 104     state->gprs[NVMM_X64_GPR_R10] = env->regs[R_R10];
 105     state->gprs[NVMM_X64_GPR_R11] = env->regs[R_R11];
 106     state->gprs[NVMM_X64_GPR_R12] = env->regs[R_R12];
 107     state->gprs[NVMM_X64_GPR_R13] = env->regs[R_R13];
 108     state->gprs[NVMM_X64_GPR_R14] = env->regs[R_R14];
 109     state->gprs[NVMM_X64_GPR_R15] = env->regs[R_R15];
 110 #endif
 111
 112     /* RIP and RFLAGS. */
 113     state->gprs[NVMM_X64_GPR_RIP] = env->eip;
 114     state->gprs[NVMM_X64_GPR_RFLAGS] = env->eflags;
 115
 116     /* Segments. */
 117     nvmm_set_segment(&state->segs[NVMM_X64_SEG_CS], &env->segs[R_CS]);
 118     nvmm_set_segment(&state->segs[NVMM_X64_SEG_DS], &env->segs[R_DS]);
 119     nvmm_set_segment(&state->segs[NVMM_X64_SEG_ES], &env->segs[R_ES]);
 120     nvmm_set_segment(&state->segs[NVMM_X64_SEG_FS], &env->segs[R_FS]);
 121     nvmm_set_segment(&state->segs[NVMM_X64_SEG_GS], &env->segs[R_GS]);
 122     nvmm_set_segment(&state->segs[NVMM_X64_SEG_SS], &env->segs[R_SS]);
 123
 124     /* Special segments. */
 125     nvmm_set_segment(&state->segs[NVMM_X64_SEG_GDT], &env->gdt);
 126     nvmm_set_segment(&state->segs[NVMM_X64_SEG_LDT], &env->ldt);
 127     nvmm_set_segment(&state->segs[NVMM_X64_SEG_TR], &env->tr);
 128     nvmm_set_segment(&state->segs[NVMM_X64_SEG_IDT], &env->idt);
 129
 130     /* Control registers. */
 131     state->crs[NVMM_X64_CR_CR0] = env->cr[0];
 132     state->crs[NVMM_X64_CR_CR2] = env->cr[2];
 133     state->crs[NVMM_X64_CR_CR3] = env->cr[3];
 134     state->crs[NVMM_X64_CR_CR4] = env->cr[4];
 135     state->crs[NVMM_X64_CR_CR8] = qcpu->tpr;
 136     state->crs[NVMM_X64_CR_XCR0] = env->xcr0;
 137
 138     /* Debug registers. */
 139     state->drs[NVMM_X64_DR_DR0] = env->dr[0];
 140     state->drs[NVMM_X64_DR_DR1] = env->dr[1];
 141     state->drs[NVMM_X64_DR_DR2] = env->dr[2];
 142     state->drs[NVMM_X64_DR_DR3] = env->dr[3];
 143     state->drs[NVMM_X64_DR_DR6] = env->dr[6];
 144     state->drs[NVMM_X64_DR_DR7] = env->dr[7];
 145
 146     /* FPU. */
 147     state->fpu.fx_cw = env->fpuc;
 148     state->fpu.fx_sw = (env->fpus & ~0x3800) | ((env->fpstt & 0x7) << 11);
 149     state->fpu.fx_tw = 0;
 150     for (i = 0; i < 8; i++) {
 151         state->fpu.fx_tw |= (!env->fptags[i]) << i;
 152     }
 153     state->fpu.fx_opcode = env->fpop;
 154     state->fpu.fx_ip.fa_64 = env->fpip;
 155     state->fpu.fx_dp.fa_64 = env->fpdp;
 156     state->fpu.fx_mxcsr = env->mxcsr;
 157     state->fpu.fx_mxcsr_mask = 0x0000FFFF;
 158     assert(sizeof(state->fpu.fx_87_ac) == sizeof(env->fpregs));
 159     memcpy(state->fpu.fx_87_ac, env->fpregs, sizeof(env->fpregs));
 160     for (i = 0; i < CPU_NB_REGS; i++) {
 161         memcpy(&state->fpu.fx_xmm[i].xmm_bytes[0],
 162             &env->xmm_regs[i].ZMM_Q(0), 8);
 163         memcpy(&state->fpu.fx_xmm[i].xmm_bytes[8],
 164             &env->xmm_regs[i].ZMM_Q(1), 8);
 165     }
 166
 167     /* MSRs. */
 168     state->msrs[NVMM_X64_MSR_EFER] = env->efer;
 169     state->msrs[NVMM_X64_MSR_STAR] = env->star;
 170 #ifdef TARGET_X86_64
 171     state->msrs[NVMM_X64_MSR_LSTAR] = env->lstar;
 172     state->msrs[NVMM_X64_MSR_CSTAR] = env->cstar;
 173     state->msrs[NVMM_X64_MSR_SFMASK] = env->fmask;
 174     state->msrs[NVMM_X64_MSR_KERNELGSBASE] = env->kernelgsbase;
 175 #endif
 176     state->msrs[NVMM_X64_MSR_SYSENTER_CS]  = env->sysenter_cs;
 177     state->msrs[NVMM_X64_MSR_SYSENTER_ESP] = env->sysenter_esp;
 178     state->msrs[NVMM_X64_MSR_SYSENTER_EIP] = env->sysenter_eip;
 179     state->msrs[NVMM_X64_MSR_PAT] = env->pat;
 180     state->msrs[NVMM_X64_MSR_TSC] = env->tsc;
 181
 182     bitmap =
 183         NVMM_X64_STATE_SEGS |
 184         NVMM_X64_STATE_GPRS |
 185         NVMM_X64_STATE_CRS  |
 186         NVMM_X64_STATE_DRS  |
 187         NVMM_X64_STATE_MSRS |
 188         NVMM_X64_STATE_FPU;
 189
 190     ret = nvmm_vcpu_setstate(mach, vcpu, bitmap);
 191     if (ret == -1) {
 192         error_report("NVMM: Failed to set virtual processor context,"
 193             " error=%d", errno);
 194     }
 195 }
 196
 197 static void
 198 nvmm_get_segment(SegmentCache *qseg, const struct nvmm_x64_state_seg *nseg)
 199 {
 200     qseg->selector = nseg->selector;
 201     qseg->limit = nseg->limit;
 202     qseg->base = nseg->base;
 203
 204     qseg->flags =
 205         __SHIFTIN((uint32_t)nseg->attrib.type, DESC_TYPE_MASK) |
 206         __SHIFTIN((uint32_t)nseg->attrib.s, DESC_S_MASK) |
 207         __SHIFTIN((uint32_t)nseg->attrib.dpl, DESC_DPL_MASK) |
 208         __SHIFTIN((uint32_t)nseg->attrib.p, DESC_P_MASK) |
 209         __SHIFTIN((uint32_t)nseg->attrib.avl, DESC_AVL_MASK) |
 210         __SHIFTIN((uint32_t)nseg->attrib.l, DESC_L_MASK) |
 211         __SHIFTIN((uint32_t)nseg->attrib.def, DESC_B_MASK) |
 212         __SHIFTIN((uint32_t)nseg->attrib.g, DESC_G_MASK);
 213 }
 214
 215 static void
 216 nvmm_get_registers(CPUState *cpu)
 217 {
 218     CPUX86State *env = cpu_env(cpu);
 219     struct nvmm_machine *mach = get_nvmm_mach();
 220     AccelCPUState *qcpu = cpu->accel;
 221     struct nvmm_vcpu *vcpu = &qcpu->vcpu;
 222     X86CPU *x86_cpu = X86_CPU(cpu);
 223     struct nvmm_x64_state *state = vcpu->state;
 224     uint64_t bitmap, tpr;
 225     size_t i;
 226     int ret;
 227
 228     assert(cpu_is_stopped(cpu) || qemu_cpu_is_self(cpu));
 229
 230     bitmap =
 231         NVMM_X64_STATE_SEGS |
 232         NVMM_X64_STATE_GPRS |
 233         NVMM_X64_STATE_CRS  |
 234         NVMM_X64_STATE_DRS  |
 235         NVMM_X64_STATE_MSRS |
 236         NVMM_X64_STATE_FPU;
 237
 238     ret = nvmm_vcpu_getstate(mach, vcpu, bitmap);
 239     if (ret == -1) {
 240         error_report("NVMM: Failed to get virtual processor context,"
 241             " error=%d", errno);
 242     }
 243
 244     /* GPRs. */
 245     env->regs[R_EAX] = state->gprs[NVMM_X64_GPR_RAX];
 246     env->regs[R_ECX] = state->gprs[NVMM_X64_GPR_RCX];
 247     env->regs[R_EDX] = state->gprs[NVMM_X64_GPR_RDX];
 248     env->regs[R_EBX] = state->gprs[NVMM_X64_GPR_RBX];
 249     env->regs[R_ESP] = state->gprs[NVMM_X64_GPR_RSP];
 250     env->regs[R_EBP] = state->gprs[NVMM_X64_GPR_RBP];
 251     env->regs[R_ESI] = state->gprs[NVMM_X64_GPR_RSI];
 252     env->regs[R_EDI] = state->gprs[NVMM_X64_GPR_RDI];
 253 #ifdef TARGET_X86_64
 254     env->regs[R_R8]  = state->gprs[NVMM_X64_GPR_R8];
 255     env->regs[R_R9]  = state->gprs[NVMM_X64_GPR_R9];
 256     env->regs[R_R10] = state->gprs[NVMM_X64_GPR_R10];
 257     env->regs[R_R11] = state->gprs[NVMM_X64_GPR_R11];
 258     env->regs[R_R12] = state->gprs[NVMM_X64_GPR_R12];
 259     env->regs[R_R13] = state->gprs[NVMM_X64_GPR_R13];
 260     env->regs[R_R14] = state->gprs[NVMM_X64_GPR_R14];
 261     env->regs[R_R15] = state->gprs[NVMM_X64_GPR_R15];
 262 #endif
 263
 264     /* RIP and RFLAGS. */
 265     env->eip = state->gprs[NVMM_X64_GPR_RIP];
 266     env->eflags = state->gprs[NVMM_X64_GPR_RFLAGS];
 267
 268     /* Segments. */
 269     nvmm_get_segment(&env->segs[R_ES], &state->segs[NVMM_X64_SEG_ES]);
 270     nvmm_get_segment(&env->segs[R_CS], &state->segs[NVMM_X64_SEG_CS]);
 271     nvmm_get_segment(&env->segs[R_SS], &state->segs[NVMM_X64_SEG_SS]);
 272     nvmm_get_segment(&env->segs[R_DS], &state->segs[NVMM_X64_SEG_DS]);
 273     nvmm_get_segment(&env->segs[R_FS], &state->segs[NVMM_X64_SEG_FS]);
 274     nvmm_get_segment(&env->segs[R_GS], &state->segs[NVMM_X64_SEG_GS]);
 275
 276     /* Special segments. */
 277     nvmm_get_segment(&env->gdt, &state->segs[NVMM_X64_SEG_GDT]);
 278     nvmm_get_segment(&env->ldt, &state->segs[NVMM_X64_SEG_LDT]);
 279     nvmm_get_segment(&env->tr, &state->segs[NVMM_X64_SEG_TR]);
 280     nvmm_get_segment(&env->idt, &state->segs[NVMM_X64_SEG_IDT]);
 281
 282     /* Control registers. */
 283     env->cr[0] = state->crs[NVMM_X64_CR_CR0];
 284     env->cr[2] = state->crs[NVMM_X64_CR_CR2];
 285     env->cr[3] = state->crs[NVMM_X64_CR_CR3];
 286     env->cr[4] = state->crs[NVMM_X64_CR_CR4];
 287     tpr = state->crs[NVMM_X64_CR_CR8];
 288     if (tpr != qcpu->tpr) {
 289         qcpu->tpr = tpr;
 290         cpu_set_apic_tpr(x86_cpu->apic_state, tpr);
 291     }
 292     env->xcr0 = state->crs[NVMM_X64_CR_XCR0];
 293
 294     /* Debug registers. */
 295     env->dr[0] = state->drs[NVMM_X64_DR_DR0];
 296     env->dr[1] = state->drs[NVMM_X64_DR_DR1];
 297     env->dr[2] = state->drs[NVMM_X64_DR_DR2];
 298     env->dr[3] = state->drs[NVMM_X64_DR_DR3];
 299     env->dr[6] = state->drs[NVMM_X64_DR_DR6];
 300     env->dr[7] = state->drs[NVMM_X64_DR_DR7];
 301
 302     /* FPU. */
 303     env->fpuc = state->fpu.fx_cw;
 304     env->fpstt = (state->fpu.fx_sw >> 11) & 0x7;
 305     env->fpus = state->fpu.fx_sw & ~0x3800;
 306     for (i = 0; i < 8; i++) {
 307         env->fptags[i] = !((state->fpu.fx_tw >> i) & 1);
 308     }
 309     env->fpop = state->fpu.fx_opcode;
 310     env->fpip = state->fpu.fx_ip.fa_64;
 311     env->fpdp = state->fpu.fx_dp.fa_64;
 312     env->mxcsr = state->fpu.fx_mxcsr;
 313     assert(sizeof(state->fpu.fx_87_ac) == sizeof(env->fpregs));
 314     memcpy(env->fpregs, state->fpu.fx_87_ac, sizeof(env->fpregs));
 315     for (i = 0; i < CPU_NB_REGS; i++) {
 316         memcpy(&env->xmm_regs[i].ZMM_Q(0),
 317             &state->fpu.fx_xmm[i].xmm_bytes[0], 8);
 318         memcpy(&env->xmm_regs[i].ZMM_Q(1),
 319             &state->fpu.fx_xmm[i].xmm_bytes[8], 8);
 320     }
 321
 322     /* MSRs. */
 323     env->efer = state->msrs[NVMM_X64_MSR_EFER];
 324     env->star = state->msrs[NVMM_X64_MSR_STAR];
 325 #ifdef TARGET_X86_64
 326     env->lstar = state->msrs[NVMM_X64_MSR_LSTAR];
 327     env->cstar = state->msrs[NVMM_X64_MSR_CSTAR];
 328     env->fmask = state->msrs[NVMM_X64_MSR_SFMASK];
 329     env->kernelgsbase = state->msrs[NVMM_X64_MSR_KERNELGSBASE];
 330 #endif
 331     env->sysenter_cs  = state->msrs[NVMM_X64_MSR_SYSENTER_CS];
 332     env->sysenter_esp = state->msrs[NVMM_X64_MSR_SYSENTER_ESP];
 333     env->sysenter_eip = state->msrs[NVMM_X64_MSR_SYSENTER_EIP];
 334     env->pat = state->msrs[NVMM_X64_MSR_PAT];
 335     env->tsc = state->msrs[NVMM_X64_MSR_TSC];
 336
 337     x86_update_hflags(env);
 338 }
 339
 340 static bool
 341 nvmm_can_take_int(CPUState *cpu)
 342 {
 343     CPUX86State *env = cpu_env(cpu);
 344     AccelCPUState *qcpu = cpu->accel;
 345     struct nvmm_vcpu *vcpu = &qcpu->vcpu;
 346     struct nvmm_machine *mach = get_nvmm_mach();
 347
 348     if (qcpu->int_window_exit) {
 349         return false;
 350     }
 351
 352     if (qcpu->int_shadow || !(env->eflags & IF_MASK)) {
 353         struct nvmm_x64_state *state = vcpu->state;
 354
 355         /* Exit on interrupt window. */
 356         nvmm_vcpu_getstate(mach, vcpu, NVMM_X64_STATE_INTR);
 357         state->intr.int_window_exiting = 1;
 358         nvmm_vcpu_setstate(mach, vcpu, NVMM_X64_STATE_INTR);
 359
 360         return false;
 361     }
 362
 363     return true;
 364 }
 365
 366 static bool
 367 nvmm_can_take_nmi(CPUState *cpu)
 368 {
 369     AccelCPUState *qcpu = cpu->accel;
 370
 371     /*
 372      * Contrary to INTs, NMIs always schedule an exit when they are
 373      * completed. Therefore, if window-exiting is enabled, it means
 374      * NMIs are blocked.
 375      */
 376     if (qcpu->nmi_window_exit) {
 377         return false;
 378     }
 379
 380     return true;
 381 }
 382
 383 /*
 384  * Called before the VCPU is run. We inject events generated by the I/O
 385  * thread, and synchronize the guest TPR.
 386  */
 387 static void
 388 nvmm_vcpu_pre_run(CPUState *cpu)
 389 {
 390     CPUX86State *env = cpu_env(cpu);
 391     struct nvmm_machine *mach = get_nvmm_mach();
 392     AccelCPUState *qcpu = cpu->accel;
 393     struct nvmm_vcpu *vcpu = &qcpu->vcpu;
 394     X86CPU *x86_cpu = X86_CPU(cpu);
 395     struct nvmm_x64_state *state = vcpu->state;
 396     struct nvmm_vcpu_event *event = vcpu->event;
 397     bool has_event = false;
 398     bool sync_tpr = false;
 399     uint8_t tpr;
 400     int ret;
 401
 402     qemu_mutex_lock_iothread();
 403
 404     tpr = cpu_get_apic_tpr(x86_cpu->apic_state);
 405     if (tpr != qcpu->tpr) {
 406         qcpu->tpr = tpr;
 407         sync_tpr = true;
 408     }
 409
 410     /*
 411      * Force the VCPU out of its inner loop to process any INIT requests
 412      * or commit pending TPR access.
 413      */
 414     if (cpu->interrupt_request & (CPU_INTERRUPT_INIT | CPU_INTERRUPT_TPR)) {
 415         cpu->exit_request = 1;
 416     }
 417
 418     if (!has_event && (cpu->interrupt_request & CPU_INTERRUPT_NMI)) {
 419         if (nvmm_can_take_nmi(cpu)) {
 420             cpu->interrupt_request &= ~CPU_INTERRUPT_NMI;
 421             event->type = NVMM_VCPU_EVENT_INTR;
 422             event->vector = 2;
 423             has_event = true;
 424         }
 425     }
 426
 427     if (!has_event && (cpu->interrupt_request & CPU_INTERRUPT_HARD)) {
 428         if (nvmm_can_take_int(cpu)) {
 429             cpu->interrupt_request &= ~CPU_INTERRUPT_HARD;
 430             event->type = NVMM_VCPU_EVENT_INTR;
 431             event->vector = cpu_get_pic_interrupt(env);
 432             has_event = true;
 433         }
 434     }
 435
 436     /* Don't want SMIs. */
 437     if (cpu->interrupt_request & CPU_INTERRUPT_SMI) {
 438         cpu->interrupt_request &= ~CPU_INTERRUPT_SMI;
 439     }
 440
 441     if (sync_tpr) {
 442         ret = nvmm_vcpu_getstate(mach, vcpu, NVMM_X64_STATE_CRS);
 443         if (ret == -1) {
 444             error_report("NVMM: Failed to get CPU state,"
 445                 " error=%d", errno);
 446         }
 447
 448         state->crs[NVMM_X64_CR_CR8] = qcpu->tpr;
 449
 450         ret = nvmm_vcpu_setstate(mach, vcpu, NVMM_X64_STATE_CRS);
 451         if (ret == -1) {
 452             error_report("NVMM: Failed to set CPU state,"
 453                 " error=%d", errno);
 454         }
 455     }
 456
 457     if (has_event) {
 458         ret = nvmm_vcpu_inject(mach, vcpu);
 459         if (ret == -1) {
 460             error_report("NVMM: Failed to inject event,"
 461                 " error=%d", errno);
 462         }
 463     }
 464
 465     qemu_mutex_unlock_iothread();
 466 }
 467
 468 /*
 469  * Called after the VCPU ran. We synchronize the host view of the TPR and
 470  * RFLAGS.
 471  */
 472 static void
 473 nvmm_vcpu_post_run(CPUState *cpu, struct nvmm_vcpu_exit *exit)
 474 {
 475     AccelCPUState *qcpu = cpu->accel;
 476     X86CPU *x86_cpu = X86_CPU(cpu);
 477     CPUX86State *env = &x86_cpu->env;
 478     uint64_t tpr;
 479
 480     env->eflags = exit->exitstate.rflags;
 481     qcpu->int_shadow = exit->exitstate.int_shadow;
 482     qcpu->int_window_exit = exit->exitstate.int_window_exiting;
 483     qcpu->nmi_window_exit = exit->exitstate.nmi_window_exiting;
 484
 485     tpr = exit->exitstate.cr8;
 486     if (qcpu->tpr != tpr) {
 487         qcpu->tpr = tpr;
 488         qemu_mutex_lock_iothread();
 489         cpu_set_apic_tpr(x86_cpu->apic_state, qcpu->tpr);
 490         qemu_mutex_unlock_iothread();
 491     }
 492 }
 493
 494 /* -------------------------------------------------------------------------- */
 495
 496 static void
 497 nvmm_io_callback(struct nvmm_io *io)
 498 {
 499     MemTxAttrs attrs = { 0 };
 500     int ret;
 501
 502     ret = address_space_rw(&address_space_io, io->port, attrs, io->data,
 503         io->size, !io->in);
 504     if (ret != MEMTX_OK) {
 505         error_report("NVMM: I/O Transaction Failed "
 506             "[%s, port=%u, size=%zu]", (io->in ? "in" : "out"),
 507             io->port, io->size);
 508     }
 509
 510     /* Needed, otherwise infinite loop. */
 511     current_cpu->vcpu_dirty = false;
 512 }
 513
 514 static void
 515 nvmm_mem_callback(struct nvmm_mem *mem)
 516 {
 517     cpu_physical_memory_rw(mem->gpa, mem->data, mem->size, mem->write);
 518
 519     /* Needed, otherwise infinite loop. */
 520     current_cpu->vcpu_dirty = false;
 521 }
 522
 523 static struct nvmm_assist_callbacks nvmm_callbacks = {
 524     .io = nvmm_io_callback,
 525     .mem = nvmm_mem_callback
 526 };
 527
 528 /* -------------------------------------------------------------------------- */
 529
 530 static int
 531 nvmm_handle_mem(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu)
 532 {
 533     int ret;
 534
 535     ret = nvmm_assist_mem(mach, vcpu);
 536     if (ret == -1) {
 537         error_report("NVMM: Mem Assist Failed [gpa=%p]",
 538             (void *)vcpu->exit->u.mem.gpa);
 539     }
 540
 541     return ret;
 542 }
 543
 544 static int
 545 nvmm_handle_io(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu)
 546 {
 547     int ret;
 548
 549     ret = nvmm_assist_io(mach, vcpu);
 550     if (ret == -1) {
 551         error_report("NVMM: I/O Assist Failed [port=%d]",
 552             (int)vcpu->exit->u.io.port);
 553     }
 554
 555     return ret;
 556 }
 557
 558 static int
 559 nvmm_handle_rdmsr(struct nvmm_machine *mach, CPUState *cpu,
 560     struct nvmm_vcpu_exit *exit)
 561 {
 562     AccelCPUState *qcpu = cpu->accel;
 563     struct nvmm_vcpu *vcpu = &qcpu->vcpu;
 564     X86CPU *x86_cpu = X86_CPU(cpu);
 565     struct nvmm_x64_state *state = vcpu->state;
 566     uint64_t val;
 567     int ret;
 568
 569     switch (exit->u.rdmsr.msr) {
 570     case MSR_IA32_APICBASE:
 571         val = cpu_get_apic_base(x86_cpu->apic_state);
 572         break;
 573     case MSR_MTRRcap:
 574     case MSR_MTRRdefType:
 575     case MSR_MCG_CAP:
 576     case MSR_MCG_STATUS:
 577         val = 0;
 578         break;
 579     default: /* More MSRs to add? */
 580         val = 0;
 581         error_report("NVMM: Unexpected RDMSR 0x%x, ignored",
 582             exit->u.rdmsr.msr);
 583         break;
 584     }
 585
 586     ret = nvmm_vcpu_getstate(mach, vcpu, NVMM_X64_STATE_GPRS);
 587     if (ret == -1) {
 588         return -1;
 589     }
 590
 591     state->gprs[NVMM_X64_GPR_RAX] = (val & 0xFFFFFFFF);
 592     state->gprs[NVMM_X64_GPR_RDX] = (val >> 32);
 593     state->gprs[NVMM_X64_GPR_RIP] = exit->u.rdmsr.npc;
 594
 595     ret = nvmm_vcpu_setstate(mach, vcpu, NVMM_X64_STATE_GPRS);
 596     if (ret == -1) {
 597         return -1;
 598     }
 599
 600     return 0;
 601 }
 602
 603 static int
 604 nvmm_handle_wrmsr(struct nvmm_machine *mach, CPUState *cpu,
 605     struct nvmm_vcpu_exit *exit)
 606 {
 607     AccelCPUState *qcpu = cpu->accel;
 608     struct nvmm_vcpu *vcpu = &qcpu->vcpu;
 609     X86CPU *x86_cpu = X86_CPU(cpu);
 610     struct nvmm_x64_state *state = vcpu->state;
 611     uint64_t val;
 612     int ret;
 613
 614     val = exit->u.wrmsr.val;
 615
 616     switch (exit->u.wrmsr.msr) {
 617     case MSR_IA32_APICBASE:
 618         cpu_set_apic_base(x86_cpu->apic_state, val);
 619         break;
 620     case MSR_MTRRdefType:
 621     case MSR_MCG_STATUS:
 622         break;
 623     default: /* More MSRs to add? */
 624         error_report("NVMM: Unexpected WRMSR 0x%x [val=0x%lx], ignored",
 625             exit->u.wrmsr.msr, val);
 626         break;
 627     }
 628
 629     ret = nvmm_vcpu_getstate(mach, vcpu, NVMM_X64_STATE_GPRS);
 630     if (ret == -1) {
 631         return -1;
 632     }
 633
 634     state->gprs[NVMM_X64_GPR_RIP] = exit->u.wrmsr.npc;
 635
 636     ret = nvmm_vcpu_setstate(mach, vcpu, NVMM_X64_STATE_GPRS);
 637     if (ret == -1) {
 638         return -1;
 639     }
 640
 641     return 0;
 642 }
 643
 644 static int
 645 nvmm_handle_halted(struct nvmm_machine *mach, CPUState *cpu,
 646     struct nvmm_vcpu_exit *exit)
 647 {
 648     CPUX86State *env = cpu_env(cpu);
 649     int ret = 0;
 650
 651     qemu_mutex_lock_iothread();
 652
 653     if (!((cpu->interrupt_request & CPU_INTERRUPT_HARD) &&
 654           (env->eflags & IF_MASK)) &&
 655         !(cpu->interrupt_request & CPU_INTERRUPT_NMI)) {
 656         cpu->exception_index = EXCP_HLT;
 657         cpu->halted = true;
 658         ret = 1;
 659     }
 660
 661     qemu_mutex_unlock_iothread();
 662
 663     return ret;
 664 }
 665
 666 static int
 667 nvmm_inject_ud(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu)
 668 {
 669     struct nvmm_vcpu_event *event = vcpu->event;
 670
 671     event->type = NVMM_VCPU_EVENT_EXCP;
 672     event->vector = 6;
 673     event->u.excp.error = 0;
 674
 675     return nvmm_vcpu_inject(mach, vcpu);
 676 }
 677
 678 static int
 679 nvmm_vcpu_loop(CPUState *cpu)
 680 {
 681     struct nvmm_machine *mach = get_nvmm_mach();
 682     AccelCPUState *qcpu = cpu->accel;
 683     struct nvmm_vcpu *vcpu = &qcpu->vcpu;
 684     X86CPU *x86_cpu = X86_CPU(cpu);
 685     CPUX86State *env = &x86_cpu->env;
 686     struct nvmm_vcpu_exit *exit = vcpu->exit;
 687     int ret;
 688
 689     /*
 690      * Some asynchronous events must be handled outside of the inner
 691      * VCPU loop. They are handled here.
 692      */
 693     if (cpu->interrupt_request & CPU_INTERRUPT_INIT) {
 694         nvmm_cpu_synchronize_state(cpu);
 695         do_cpu_init(x86_cpu);
 696         /* set int/nmi windows back to the reset state */
 697     }
 698     if (cpu->interrupt_request & CPU_INTERRUPT_POLL) {
 699         cpu->interrupt_request &= ~CPU_INTERRUPT_POLL;
 700         apic_poll_irq(x86_cpu->apic_state);
 701     }
 702     if (((cpu->interrupt_request & CPU_INTERRUPT_HARD) &&
 703          (env->eflags & IF_MASK)) ||
 704         (cpu->interrupt_request & CPU_INTERRUPT_NMI)) {
 705         cpu->halted = false;
 706     }
 707     if (cpu->interrupt_request & CPU_INTERRUPT_SIPI) {
 708         nvmm_cpu_synchronize_state(cpu);
 709         do_cpu_sipi(x86_cpu);
 710     }
 711     if (cpu->interrupt_request & CPU_INTERRUPT_TPR) {
 712         cpu->interrupt_request &= ~CPU_INTERRUPT_TPR;
 713         nvmm_cpu_synchronize_state(cpu);
 714         apic_handle_tpr_access_report(x86_cpu->apic_state, env->eip,
 715             env->tpr_access_type);
 716     }
 717
 718     if (cpu->halted) {
 719         cpu->exception_index = EXCP_HLT;
 720         qatomic_set(&cpu->exit_request, false);
 721         return 0;
 722     }
 723
 724     qemu_mutex_unlock_iothread();
 725     cpu_exec_start(cpu);
 726
 727     /*
 728      * Inner VCPU loop.
 729      */
 730     do {
 731         if (cpu->vcpu_dirty) {
 732             nvmm_set_registers(cpu);
 733             cpu->vcpu_dirty = false;
 734         }
 735
 736         if (qcpu->stop) {
 737             cpu->exception_index = EXCP_INTERRUPT;
 738             qcpu->stop = false;
 739             ret = 1;
 740             break;
 741         }
 742
 743         nvmm_vcpu_pre_run(cpu);
 744
 745         if (qatomic_read(&cpu->exit_request)) {
 746 #if NVMM_USER_VERSION >= 2
 747             nvmm_vcpu_stop(vcpu);
 748 #else
 749             qemu_cpu_kick_self();
 750 #endif
 751         }
 752
 753         /* Read exit_request before the kernel reads the immediate exit flag */
 754         smp_rmb();
 755         ret = nvmm_vcpu_run(mach, vcpu);
 756         if (ret == -1) {
 757             error_report("NVMM: Failed to exec a virtual processor,"
 758                 " error=%d", errno);
 759             break;
 760         }
 761
 762         nvmm_vcpu_post_run(cpu, exit);
 763
 764         switch (exit->reason) {
 765         case NVMM_VCPU_EXIT_NONE:
 766             break;
 767 #if NVMM_USER_VERSION >= 2
 768         case NVMM_VCPU_EXIT_STOPPED:
 769             /*
 770              * The kernel cleared the immediate exit flag; cpu->exit_request
 771              * must be cleared after
 772              */
 773             smp_wmb();
 774             qcpu->stop = true;
 775             break;
 776 #endif
 777         case NVMM_VCPU_EXIT_MEMORY:
 778             ret = nvmm_handle_mem(mach, vcpu);
 779             break;
 780         case NVMM_VCPU_EXIT_IO:
 781             ret = nvmm_handle_io(mach, vcpu);
 782             break;
 783         case NVMM_VCPU_EXIT_INT_READY:
 784         case NVMM_VCPU_EXIT_NMI_READY:
 785         case NVMM_VCPU_EXIT_TPR_CHANGED:
 786             break;
 787         case NVMM_VCPU_EXIT_HALTED:
 788             ret = nvmm_handle_halted(mach, cpu, exit);
 789             break;
 790         case NVMM_VCPU_EXIT_SHUTDOWN:
 791             qemu_system_reset_request(SHUTDOWN_CAUSE_GUEST_RESET);
 792             cpu->exception_index = EXCP_INTERRUPT;
 793             ret = 1;
 794             break;
 795         case NVMM_VCPU_EXIT_RDMSR:
 796             ret = nvmm_handle_rdmsr(mach, cpu, exit);
 797             break;
 798         case NVMM_VCPU_EXIT_WRMSR:
 799             ret = nvmm_handle_wrmsr(mach, cpu, exit);
 800             break;
 801         case NVMM_VCPU_EXIT_MONITOR:
 802         case NVMM_VCPU_EXIT_MWAIT:
 803             ret = nvmm_inject_ud(mach, vcpu);
 804             break;
 805         default:
 806             error_report("NVMM: Unexpected VM exit code 0x%lx [hw=0x%lx]",
 807                 exit->reason, exit->u.inv.hwcode);
 808             nvmm_get_registers(cpu);
 809             qemu_mutex_lock_iothread();
 810             qemu_system_guest_panicked(cpu_get_crash_info(cpu));
 811             qemu_mutex_unlock_iothread();
 812             ret = -1;
 813             break;
 814         }
 815     } while (ret == 0);
 816
 817     cpu_exec_end(cpu);
 818     qemu_mutex_lock_iothread();
 819
 820     qatomic_set(&cpu->exit_request, false);
 821
 822     return ret < 0;
 823 }
 824
 825 /* -------------------------------------------------------------------------- */
 826
 827 static void
 828 do_nvmm_cpu_synchronize_state(CPUState *cpu, run_on_cpu_data arg)
 829 {
 830     nvmm_get_registers(cpu);
 831     cpu->vcpu_dirty = true;
 832 }
 833
 834 static void
 835 do_nvmm_cpu_synchronize_post_reset(CPUState *cpu, run_on_cpu_data arg)
 836 {
 837     nvmm_set_registers(cpu);
 838     cpu->vcpu_dirty = false;
 839 }
 840
 841 static void
 842 do_nvmm_cpu_synchronize_post_init(CPUState *cpu, run_on_cpu_data arg)
 843 {
 844     nvmm_set_registers(cpu);
 845     cpu->vcpu_dirty = false;
 846 }
 847
 848 static void
 849 do_nvmm_cpu_synchronize_pre_loadvm(CPUState *cpu, run_on_cpu_data arg)
 850 {
 851     cpu->vcpu_dirty = true;
 852 }
 853
 854 void nvmm_cpu_synchronize_state(CPUState *cpu)
 855 {
 856     if (!cpu->vcpu_dirty) {
 857         run_on_cpu(cpu, do_nvmm_cpu_synchronize_state, RUN_ON_CPU_NULL);
 858     }
 859 }
 860
 861 void nvmm_cpu_synchronize_post_reset(CPUState *cpu)
 862 {
 863     run_on_cpu(cpu, do_nvmm_cpu_synchronize_post_reset, RUN_ON_CPU_NULL);
 864 }
 865
 866 void nvmm_cpu_synchronize_post_init(CPUState *cpu)
 867 {
 868     run_on_cpu(cpu, do_nvmm_cpu_synchronize_post_init, RUN_ON_CPU_NULL);
 869 }
 870
 871 void nvmm_cpu_synchronize_pre_loadvm(CPUState *cpu)
 872 {
 873     run_on_cpu(cpu, do_nvmm_cpu_synchronize_pre_loadvm, RUN_ON_CPU_NULL);
 874 }
 875
 876 /* -------------------------------------------------------------------------- */
 877
 878 static Error *nvmm_migration_blocker;
 879
 880 /*
 881  * The nvmm_vcpu_stop() mechanism breaks races between entering the VMM
 882  * and another thread signaling the vCPU thread to exit.
 883  */
 884
 885 static void
 886 nvmm_ipi_signal(int sigcpu)
 887 {
 888     if (current_cpu) {
 889         AccelCPUState *qcpu = current_cpu->accel;
 890 #if NVMM_USER_VERSION >= 2
 891         struct nvmm_vcpu *vcpu = &qcpu->vcpu;
 892         nvmm_vcpu_stop(vcpu);
 893 #else
 894         qcpu->stop = true;
 895 #endif
 896     }
 897 }
 898
 899 static void
 900 nvmm_init_cpu_signals(void)
 901 {
 902     struct sigaction sigact;
 903     sigset_t set;
 904
 905     /* Install the IPI handler. */
 906     memset(&sigact, 0, sizeof(sigact));
 907     sigact.sa_handler = nvmm_ipi_signal;
 908     sigaction(SIG_IPI, &sigact, NULL);
 909
 910     /* Allow IPIs on the current thread. */
 911     sigprocmask(SIG_BLOCK, NULL, &set);
 912     sigdelset(&set, SIG_IPI);
 913     pthread_sigmask(SIG_SETMASK, &set, NULL);
 914 }
 915
 916 int
 917 nvmm_init_vcpu(CPUState *cpu)
 918 {
 919     struct nvmm_machine *mach = get_nvmm_mach();
 920     struct nvmm_vcpu_conf_cpuid cpuid;
 921     struct nvmm_vcpu_conf_tpr tpr;
 922     Error *local_error = NULL;
 923     AccelCPUState *qcpu;
 924     int ret, err;
 925
 926     nvmm_init_cpu_signals();
 927
 928     if (nvmm_migration_blocker == NULL) {
 929         error_setg(&nvmm_migration_blocker,
 930             "NVMM: Migration not supported");
 931
 932         if (migrate_add_blocker(nvmm_migration_blocker, &local_error) < 0) {
 933             error_report_err(local_error);
 934             error_free(nvmm_migration_blocker);
 935             return -EINVAL;
 936         }
 937     }
 938
 939     qcpu = g_new0(AccelCPUState, 1);
 940
 941     ret = nvmm_vcpu_create(mach, cpu->cpu_index, &qcpu->vcpu);
 942     if (ret == -1) {
 943         err = errno;
 944         error_report("NVMM: Failed to create a virtual processor,"
 945             " error=%d", err);
 946         g_free(qcpu);
 947         return -err;
 948     }
 949
 950     memset(&cpuid, 0, sizeof(cpuid));
 951     cpuid.mask = 1;
 952     cpuid.leaf = 0x00000001;
 953     cpuid.u.mask.set.edx = CPUID_MCE | CPUID_MCA | CPUID_MTRR;
 954     ret = nvmm_vcpu_configure(mach, &qcpu->vcpu, NVMM_VCPU_CONF_CPUID,
 955         &cpuid);
 956     if (ret == -1) {
 957         err = errno;
 958         error_report("NVMM: Failed to configure a virtual processor,"
 959             " error=%d", err);
 960         g_free(qcpu);
 961         return -err;
 962     }
 963
 964     ret = nvmm_vcpu_configure(mach, &qcpu->vcpu, NVMM_VCPU_CONF_CALLBACKS,
 965         &nvmm_callbacks);
 966     if (ret == -1) {
 967         err = errno;
 968         error_report("NVMM: Failed to configure a virtual processor,"
 969             " error=%d", err);
 970         g_free(qcpu);
 971         return -err;
 972     }
 973
 974     if (qemu_mach.cap.arch.vcpu_conf_support & NVMM_CAP_ARCH_VCPU_CONF_TPR) {
 975         memset(&tpr, 0, sizeof(tpr));
 976         tpr.exit_changed = 1;
 977         ret = nvmm_vcpu_configure(mach, &qcpu->vcpu, NVMM_VCPU_CONF_TPR, &tpr);
 978         if (ret == -1) {
 979             err = errno;
 980             error_report("NVMM: Failed to configure a virtual processor,"
 981                 " error=%d", err);
 982             g_free(qcpu);
 983             return -err;
 984         }
 985     }
 986
 987     cpu->vcpu_dirty = true;
 988     cpu->accel = qcpu;
 989
 990     return 0;
 991 }
 992
 993 int
 994 nvmm_vcpu_exec(CPUState *cpu)
 995 {
 996     int ret, fatal;
 997
 998     while (1) {
 999         if (cpu->exception_index >= EXCP_INTERRUPT) {
1000             ret = cpu->exception_index;
1001             cpu->exception_index = -1;
1002             break;
1003         }
1004
1005         fatal = nvmm_vcpu_loop(cpu);
1006
1007         if (fatal) {
1008             error_report("NVMM: Failed to execute a VCPU.");
1009             abort();
1010         }
1011     }
1012
1013     return ret;
1014 }
1015
1016 void
1017 nvmm_destroy_vcpu(CPUState *cpu)
1018 {
1019     struct nvmm_machine *mach = get_nvmm_mach();
1020     AccelCPUState *qcpu = cpu->accel;
1021
1022     nvmm_vcpu_destroy(mach, &qcpu->vcpu);
1023     g_free(cpu->accel);
1024 }
1025
1026 /* -------------------------------------------------------------------------- */
1027
1028 static void
1029 nvmm_update_mapping(hwaddr start_pa, ram_addr_t size, uintptr_t hva,
1030     bool add, bool rom, const char *name)
1031 {
1032     struct nvmm_machine *mach = get_nvmm_mach();
1033     int ret, prot;
1034
1035     if (add) {
1036         prot = PROT_READ | PROT_EXEC;
1037         if (!rom) {
1038             prot |= PROT_WRITE;
1039         }
1040         ret = nvmm_gpa_map(mach, hva, start_pa, size, prot);
1041     } else {
1042         ret = nvmm_gpa_unmap(mach, hva, start_pa, size);
1043     }
1044
1045     if (ret == -1) {
1046         error_report("NVMM: Failed to %s GPA range '%s' PA:%p, "
1047             "Size:%p bytes, HostVA:%p, error=%d",
1048             (add ? "map" : "unmap"), name, (void *)(uintptr_t)start_pa,
1049             (void *)size, (void *)hva, errno);
1050     }
1051 }
1052
1053 static void
1054 nvmm_process_section(MemoryRegionSection *section, int add)
1055 {
1056     MemoryRegion *mr = section->mr;
1057     hwaddr start_pa = section->offset_within_address_space;
1058     ram_addr_t size = int128_get64(section->size);
1059     unsigned int delta;
1060     uintptr_t hva;
1061
1062     if (!memory_region_is_ram(mr)) {
1063         return;
1064     }
1065
1066     /* Adjust start_pa and size so that they are page-aligned. */
1067     delta = qemu_real_host_page_size() - (start_pa & ~qemu_real_host_page_mask());
1068     delta &= ~qemu_real_host_page_mask();
1069     if (delta > size) {
1070         return;
1071     }
1072     start_pa += delta;
1073     size -= delta;
1074     size &= qemu_real_host_page_mask();
1075     if (!size || (start_pa & ~qemu_real_host_page_mask())) {
1076         return;
1077     }
1078
1079     hva = (uintptr_t)memory_region_get_ram_ptr(mr) +
1080         section->offset_within_region + delta;
1081
1082     nvmm_update_mapping(start_pa, size, hva, add,
1083         memory_region_is_rom(mr), mr->name);
1084 }
1085
1086 static void
1087 nvmm_region_add(MemoryListener *listener, MemoryRegionSection *section)
1088 {
1089     memory_region_ref(section->mr);
1090     nvmm_process_section(section, 1);
1091 }
1092
1093 static void
1094 nvmm_region_del(MemoryListener *listener, MemoryRegionSection *section)
1095 {
1096     nvmm_process_section(section, 0);
1097     memory_region_unref(section->mr);
1098 }
1099
1100 static void
1101 nvmm_transaction_begin(MemoryListener *listener)
1102 {
1103     /* nothing */
1104 }
1105
1106 static void
1107 nvmm_transaction_commit(MemoryListener *listener)
1108 {
1109     /* nothing */
1110 }
1111
1112 static void
1113 nvmm_log_sync(MemoryListener *listener, MemoryRegionSection *section)
1114 {
1115     MemoryRegion *mr = section->mr;
1116
1117     if (!memory_region_is_ram(mr)) {
1118         return;
1119     }
1120
1121     memory_region_set_dirty(mr, 0, int128_get64(section->size));
1122 }
1123
1124 static MemoryListener nvmm_memory_listener = {
1125     .name = "nvmm",
1126     .begin = nvmm_transaction_begin,
1127     .commit = nvmm_transaction_commit,
1128     .region_add = nvmm_region_add,
1129     .region_del = nvmm_region_del,
1130     .log_sync = nvmm_log_sync,
1131     .priority = MEMORY_LISTENER_PRIORITY_ACCEL,
1132 };
1133
1134 static void
1135 nvmm_ram_block_added(RAMBlockNotifier *n, void *host, size_t size,
1136                      size_t max_size)
1137 {
1138     struct nvmm_machine *mach = get_nvmm_mach();
1139     uintptr_t hva = (uintptr_t)host;
1140     int ret;
1141
1142     ret = nvmm_hva_map(mach, hva, max_size);
1143
1144     if (ret == -1) {
1145         error_report("NVMM: Failed to map HVA, HostVA:%p "
1146             "Size:%p bytes, error=%d",
1147             (void *)hva, (void *)size, errno);
1148     }
1149 }
1150
1151 static struct RAMBlockNotifier nvmm_ram_notifier = {
1152     .ram_block_added = nvmm_ram_block_added
1153 };
1154
1155 /* -------------------------------------------------------------------------- */
1156
1157 static int
1158 nvmm_accel_init(MachineState *ms)
1159 {
1160     int ret, err;
1161
1162     ret = nvmm_init();
1163     if (ret == -1) {
1164         err = errno;
1165         error_report("NVMM: Initialization failed, error=%d", errno);
1166         return -err;
1167     }
1168
1169     ret = nvmm_capability(&qemu_mach.cap);
1170     if (ret == -1) {
1171         err = errno;
1172         error_report("NVMM: Unable to fetch capability, error=%d", errno);
1173         return -err;
1174     }
1175     if (qemu_mach.cap.version < NVMM_KERN_VERSION) {
1176         error_report("NVMM: Unsupported version %u", qemu_mach.cap.version);
1177         return -EPROGMISMATCH;
1178     }
1179     if (qemu_mach.cap.state_size != sizeof(struct nvmm_x64_state)) {
1180         error_report("NVMM: Wrong state size %u", qemu_mach.cap.state_size);
1181         return -EPROGMISMATCH;
1182     }
1183
1184     ret = nvmm_machine_create(&qemu_mach.mach);
1185     if (ret == -1) {
1186         err = errno;
1187         error_report("NVMM: Machine creation failed, error=%d", errno);
1188         return -err;
1189     }
1190
1191     memory_listener_register(&nvmm_memory_listener, &address_space_memory);
1192     ram_block_notifier_add(&nvmm_ram_notifier);
1193
1194     printf("NetBSD Virtual Machine Monitor accelerator is operational\n");
1195     return 0;
1196 }
1197
1198 int
1199 nvmm_enabled(void)
1200 {
1201     return nvmm_allowed;
1202 }
1203
1204 static void
1205 nvmm_accel_class_init(ObjectClass *oc, void *data)
1206 {
1207     AccelClass *ac = ACCEL_CLASS(oc);
1208     ac->name = "NVMM";
1209     ac->init_machine = nvmm_accel_init;
1210     ac->allowed = &nvmm_allowed;
1211 }
1212
1213 static const TypeInfo nvmm_accel_type = {
1214     .name = ACCEL_CLASS_NAME("nvmm"),
1215     .parent = TYPE_ACCEL,
1216     .class_init = nvmm_accel_class_init,
1217 };
1218
1219 static void
1220 nvmm_type_init(void)
1221 {
1222     type_register_static(&nvmm_accel_type);
1223 }
1224
1225 type_init(nvmm_type_init);