]> git.proxmox.com Git - qemu.git/blob - target-i386/kvm.c
projects / qemu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
target-i386: Pass X86CPU to cpu_x86_inject_mce()
[qemu.git] / target-i386 / kvm.c
1 /*
2 * QEMU KVM support
3 *
4 * Copyright (C) 2006-2008 Qumranet Technologies
5 * Copyright IBM, Corp. 2008
6 *
7 * Authors:
8 * Anthony Liguori <aliguori@us.ibm.com>
9 *
10 * This work is licensed under the terms of the GNU GPL, version 2 or later.
11 * See the COPYING file in the top-level directory.
12 *
13 */
14
15 #include <sys/types.h>
16 #include <sys/ioctl.h>
17 #include <sys/mman.h>
18 #include <sys/utsname.h>
19
20 #include <linux/kvm.h>
21 #include <linux/kvm_para.h>
22
23 #include "qemu-common.h"
24 #include "sysemu.h"
25 #include "kvm.h"
26 #include "kvm_i386.h"
27 #include "cpu.h"
28 #include "gdbstub.h"
29 #include "host-utils.h"
30 #include "hw/pc.h"
31 #include "hw/apic.h"
32 #include "ioport.h"
33 #include "hyperv.h"
34 #include "hw/pci.h"
35
36 //#define DEBUG_KVM
37
38 #ifdef DEBUG_KVM
39 #define DPRINTF(fmt, ...) \
40 do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
41 #else
42 #define DPRINTF(fmt, ...) \
43 do { } while (0)
44 #endif
45
46 #define MSR_KVM_WALL_CLOCK 0x11
47 #define MSR_KVM_SYSTEM_TIME 0x12
48
49 #ifndef BUS_MCEERR_AR
50 #define BUS_MCEERR_AR 4
51 #endif
52 #ifndef BUS_MCEERR_AO
53 #define BUS_MCEERR_AO 5
54 #endif
55
56 const KVMCapabilityInfo kvm_arch_required_capabilities[] = {
57 KVM_CAP_INFO(SET_TSS_ADDR),
58 KVM_CAP_INFO(EXT_CPUID),
59 KVM_CAP_INFO(MP_STATE),
60 KVM_CAP_LAST_INFO
61 };
62
63 static bool has_msr_star;
64 static bool has_msr_hsave_pa;
65 static bool has_msr_tsc_deadline;
66 static bool has_msr_async_pf_en;
67 static bool has_msr_pv_eoi_en;
68 static bool has_msr_misc_enable;
69 static int lm_capable_kernel;
70
71 bool kvm_allows_irq0_override(void)
72 {
73 return !kvm_irqchip_in_kernel() || kvm_has_gsi_routing();
74 }
75
76 static struct kvm_cpuid2 *try_get_cpuid(KVMState *s, int max)
77 {
78 struct kvm_cpuid2 *cpuid;
79 int r, size;
80
81 size = sizeof(*cpuid) + max * sizeof(*cpuid->entries);
82 cpuid = (struct kvm_cpuid2 *)g_malloc0(size);
83 cpuid->nent = max;
84 r = kvm_ioctl(s, KVM_GET_SUPPORTED_CPUID, cpuid);
85 if (r == 0 && cpuid->nent >= max) {
86 r = -E2BIG;
87 }
88 if (r < 0) {
89 if (r == -E2BIG) {
90 g_free(cpuid);
91 return NULL;
92 } else {
93 fprintf(stderr, "KVM_GET_SUPPORTED_CPUID failed: %s\n",
94 strerror(-r));
95 exit(1);
96 }
97 }
98 return cpuid;
99 }
100
101 struct kvm_para_features {
102 int cap;
103 int feature;
104 } para_features[] = {
105 { KVM_CAP_CLOCKSOURCE, KVM_FEATURE_CLOCKSOURCE },
106 { KVM_CAP_NOP_IO_DELAY, KVM_FEATURE_NOP_IO_DELAY },
107 { KVM_CAP_PV_MMU, KVM_FEATURE_MMU_OP },
108 { KVM_CAP_ASYNC_PF, KVM_FEATURE_ASYNC_PF },
109 { -1, -1 }
110 };
111
112 static int get_para_features(KVMState *s)
113 {
114 int i, features = 0;
115
116 for (i = 0; i < ARRAY_SIZE(para_features) - 1; i++) {
117 if (kvm_check_extension(s, para_features[i].cap)) {
118 features |= (1 << para_features[i].feature);
119 }
120 }
121
122 return features;
123 }
124
125
126 uint32_t kvm_arch_get_supported_cpuid(KVMState *s, uint32_t function,
127 uint32_t index, int reg)
128 {
129 struct kvm_cpuid2 *cpuid;
130 int i, max;
131 uint32_t ret = 0;
132 uint32_t cpuid_1_edx;
133 int has_kvm_features = 0;
134
135 max = 1;
136 while ((cpuid = try_get_cpuid(s, max)) == NULL) {
137 max *= 2;
138 }
139
140 for (i = 0; i < cpuid->nent; ++i) {
141 if (cpuid->entries[i].function == function &&
142 cpuid->entries[i].index == index) {
143 if (cpuid->entries[i].function == KVM_CPUID_FEATURES) {
144 has_kvm_features = 1;
145 }
146 switch (reg) {
147 case R_EAX:
148 ret = cpuid->entries[i].eax;
149 break;
150 case R_EBX:
151 ret = cpuid->entries[i].ebx;
152 break;
153 case R_ECX:
154 ret = cpuid->entries[i].ecx;
155 break;
156 case R_EDX:
157 ret = cpuid->entries[i].edx;
158 switch (function) {
159 case 1:
160 /* KVM before 2.6.30 misreports the following features */
161 ret |= CPUID_MTRR | CPUID_PAT | CPUID_MCE | CPUID_MCA;
162 break;
163 case 0x80000001:
164 /* On Intel, kvm returns cpuid according to the Intel spec,
165 * so add missing bits according to the AMD spec:
166 */
167 cpuid_1_edx = kvm_arch_get_supported_cpuid(s, 1, 0, R_EDX);
168 ret |= cpuid_1_edx & CPUID_EXT2_AMD_ALIASES;
169 break;
170 }
171 break;
172 }
173 }
174 }
175
176 g_free(cpuid);
177
178 /* fallback for older kernels */
179 if (!has_kvm_features && (function == KVM_CPUID_FEATURES)) {
180 ret = get_para_features(s);
181 }
182
183 return ret;
184 }
185
186 typedef struct HWPoisonPage {
187 ram_addr_t ram_addr;
188 QLIST_ENTRY(HWPoisonPage) list;
189 } HWPoisonPage;
190
191 static QLIST_HEAD(, HWPoisonPage) hwpoison_page_list =
192 QLIST_HEAD_INITIALIZER(hwpoison_page_list);
193
194 static void kvm_unpoison_all(void *param)
195 {
196 HWPoisonPage *page, *next_page;
197
198 QLIST_FOREACH_SAFE(page, &hwpoison_page_list, list, next_page) {
199 QLIST_REMOVE(page, list);
200 qemu_ram_remap(page->ram_addr, TARGET_PAGE_SIZE);
201 g_free(page);
202 }
203 }
204
205 static void kvm_hwpoison_page_add(ram_addr_t ram_addr)
206 {
207 HWPoisonPage *page;
208
209 QLIST_FOREACH(page, &hwpoison_page_list, list) {
210 if (page->ram_addr == ram_addr) {
211 return;
212 }
213 }
214 page = g_malloc(sizeof(HWPoisonPage));
215 page->ram_addr = ram_addr;
216 QLIST_INSERT_HEAD(&hwpoison_page_list, page, list);
217 }
218
219 static int kvm_get_mce_cap_supported(KVMState *s, uint64_t *mce_cap,
220 int *max_banks)
221 {
222 int r;
223
224 r = kvm_check_extension(s, KVM_CAP_MCE);
225 if (r > 0) {
226 *max_banks = r;
227 return kvm_ioctl(s, KVM_X86_GET_MCE_CAP_SUPPORTED, mce_cap);
228 }
229 return -ENOSYS;
230 }
231
232 static void kvm_mce_inject(X86CPU *cpu, hwaddr paddr, int code)
233 {
234 CPUX86State *env = &cpu->env;
235 uint64_t status = MCI_STATUS_VAL | MCI_STATUS_UC | MCI_STATUS_EN |
236 MCI_STATUS_MISCV | MCI_STATUS_ADDRV | MCI_STATUS_S;
237 uint64_t mcg_status = MCG_STATUS_MCIP;
238
239 if (code == BUS_MCEERR_AR) {
240 status |= MCI_STATUS_AR | 0x134;
241 mcg_status |= MCG_STATUS_EIPV;
242 } else {
243 status |= 0xc0;
244 mcg_status |= MCG_STATUS_RIPV;
245 }
246 cpu_x86_inject_mce(NULL, cpu, 9, status, mcg_status, paddr,
247 (MCM_ADDR_PHYS << 6) | 0xc,
248 cpu_x86_support_mca_broadcast(env) ?
249 MCE_INJECT_BROADCAST : 0);
250 }
251
252 static void hardware_memory_error(void)
253 {
254 fprintf(stderr, "Hardware memory error!\n");
255 exit(1);
256 }
257
258 int kvm_arch_on_sigbus_vcpu(CPUX86State *env, int code, void *addr)
259 {
260 X86CPU *cpu = x86_env_get_cpu(env);
261 ram_addr_t ram_addr;
262 hwaddr paddr;
263
264 if ((env->mcg_cap & MCG_SER_P) && addr
265 && (code == BUS_MCEERR_AR || code == BUS_MCEERR_AO)) {
266 if (qemu_ram_addr_from_host(addr, &ram_addr) ||
267 !kvm_physical_memory_addr_from_host(env->kvm_state, addr, &paddr)) {
268 fprintf(stderr, "Hardware memory error for memory used by "
269 "QEMU itself instead of guest system!\n");
270 /* Hope we are lucky for AO MCE */
271 if (code == BUS_MCEERR_AO) {
272 return 0;
273 } else {
274 hardware_memory_error();
275 }
276 }
277 kvm_hwpoison_page_add(ram_addr);
278 kvm_mce_inject(cpu, paddr, code);
279 } else {
280 if (code == BUS_MCEERR_AO) {
281 return 0;
282 } else if (code == BUS_MCEERR_AR) {
283 hardware_memory_error();
284 } else {
285 return 1;
286 }
287 }
288 return 0;
289 }
290
291 int kvm_arch_on_sigbus(int code, void *addr)
292 {
293 if ((first_cpu->mcg_cap & MCG_SER_P) && addr && code == BUS_MCEERR_AO) {
294 ram_addr_t ram_addr;
295 hwaddr paddr;
296
297 /* Hope we are lucky for AO MCE */
298 if (qemu_ram_addr_from_host(addr, &ram_addr) ||
299 !kvm_physical_memory_addr_from_host(first_cpu->kvm_state, addr,
300 &paddr)) {
301 fprintf(stderr, "Hardware memory error for memory used by "
302 "QEMU itself instead of guest system!: %p\n", addr);
303 return 0;
304 }
305 kvm_hwpoison_page_add(ram_addr);
306 kvm_mce_inject(x86_env_get_cpu(first_cpu), paddr, code);
307 } else {
308 if (code == BUS_MCEERR_AO) {
309 return 0;
310 } else if (code == BUS_MCEERR_AR) {
311 hardware_memory_error();
312 } else {
313 return 1;
314 }
315 }
316 return 0;
317 }
318
319 static int kvm_inject_mce_oldstyle(CPUX86State *env)
320 {
321 if (!kvm_has_vcpu_events() && env->exception_injected == EXCP12_MCHK) {
322 unsigned int bank, bank_num = env->mcg_cap & 0xff;
323 struct kvm_x86_mce mce;
324
325 env->exception_injected = -1;
326
327 /*
328 * There must be at least one bank in use if an MCE is pending.
329 * Find it and use its values for the event injection.
330 */
331 for (bank = 0; bank < bank_num; bank++) {
332 if (env->mce_banks[bank * 4 + 1] & MCI_STATUS_VAL) {
333 break;
334 }
335 }
336 assert(bank < bank_num);
337
338 mce.bank = bank;
339 mce.status = env->mce_banks[bank * 4 + 1];
340 mce.mcg_status = env->mcg_status;
341 mce.addr = env->mce_banks[bank * 4 + 2];
342 mce.misc = env->mce_banks[bank * 4 + 3];
343
344 return kvm_vcpu_ioctl(env, KVM_X86_SET_MCE, &mce);
345 }
346 return 0;
347 }
348
349 static void cpu_update_state(void *opaque, int running, RunState state)
350 {
351 CPUX86State *env = opaque;
352
353 if (running) {
354 env->tsc_valid = false;
355 }
356 }
357
358 int kvm_arch_init_vcpu(CPUX86State *env)
359 {
360 struct {
361 struct kvm_cpuid2 cpuid;
362 struct kvm_cpuid_entry2 entries[100];
363 } QEMU_PACKED cpuid_data;
364 KVMState *s = env->kvm_state;
365 uint32_t limit, i, j, cpuid_i;
366 uint32_t unused;
367 struct kvm_cpuid_entry2 *c;
368 uint32_t signature[3];
369 int r;
370
371 env->cpuid_features &= kvm_arch_get_supported_cpuid(s, 1, 0, R_EDX);
372
373 i = env->cpuid_ext_features & CPUID_EXT_HYPERVISOR;
374 j = env->cpuid_ext_features & CPUID_EXT_TSC_DEADLINE_TIMER;
375 env->cpuid_ext_features &= kvm_arch_get_supported_cpuid(s, 1, 0, R_ECX);
376 env->cpuid_ext_features |= i;
377 if (j && kvm_irqchip_in_kernel() &&
378 kvm_check_extension(s, KVM_CAP_TSC_DEADLINE_TIMER)) {
379 env->cpuid_ext_features |= CPUID_EXT_TSC_DEADLINE_TIMER;
380 }
381
382 env->cpuid_ext2_features &= kvm_arch_get_supported_cpuid(s, 0x80000001,
383 0, R_EDX);
384 env->cpuid_ext3_features &= kvm_arch_get_supported_cpuid(s, 0x80000001,
385 0, R_ECX);
386 env->cpuid_svm_features &= kvm_arch_get_supported_cpuid(s, 0x8000000A,
387 0, R_EDX);
388
389 cpuid_i = 0;
390
391 /* Paravirtualization CPUIDs */
392 c = &cpuid_data.entries[cpuid_i++];
393 memset(c, 0, sizeof(*c));
394 c->function = KVM_CPUID_SIGNATURE;
395 if (!hyperv_enabled()) {
396 memcpy(signature, "KVMKVMKVM\0\0\0", 12);
397 c->eax = 0;
398 } else {
399 memcpy(signature, "Microsoft Hv", 12);
400 c->eax = HYPERV_CPUID_MIN;
401 }
402 c->ebx = signature[0];
403 c->ecx = signature[1];
404 c->edx = signature[2];
405
406 c = &cpuid_data.entries[cpuid_i++];
407 memset(c, 0, sizeof(*c));
408 c->function = KVM_CPUID_FEATURES;
409 c->eax = env->cpuid_kvm_features &
410 kvm_arch_get_supported_cpuid(s, KVM_CPUID_FEATURES, 0, R_EAX);
411
412 if (hyperv_enabled()) {
413 memcpy(signature, "Hv#1\0\0\0\0\0\0\0\0", 12);
414 c->eax = signature[0];
415
416 c = &cpuid_data.entries[cpuid_i++];
417 memset(c, 0, sizeof(*c));
418 c->function = HYPERV_CPUID_VERSION;
419 c->eax = 0x00001bbc;
420 c->ebx = 0x00060001;
421
422 c = &cpuid_data.entries[cpuid_i++];
423 memset(c, 0, sizeof(*c));
424 c->function = HYPERV_CPUID_FEATURES;
425 if (hyperv_relaxed_timing_enabled()) {
426 c->eax |= HV_X64_MSR_HYPERCALL_AVAILABLE;
427 }
428 if (hyperv_vapic_recommended()) {
429 c->eax |= HV_X64_MSR_HYPERCALL_AVAILABLE;
430 c->eax |= HV_X64_MSR_APIC_ACCESS_AVAILABLE;
431 }
432
433 c = &cpuid_data.entries[cpuid_i++];
434 memset(c, 0, sizeof(*c));
435 c->function = HYPERV_CPUID_ENLIGHTMENT_INFO;
436 if (hyperv_relaxed_timing_enabled()) {
437 c->eax |= HV_X64_RELAXED_TIMING_RECOMMENDED;
438 }
439 if (hyperv_vapic_recommended()) {
440 c->eax |= HV_X64_APIC_ACCESS_RECOMMENDED;
441 }
442 c->ebx = hyperv_get_spinlock_retries();
443
444 c = &cpuid_data.entries[cpuid_i++];
445 memset(c, 0, sizeof(*c));
446 c->function = HYPERV_CPUID_IMPLEMENT_LIMITS;
447 c->eax = 0x40;
448 c->ebx = 0x40;
449
450 c = &cpuid_data.entries[cpuid_i++];
451 memset(c, 0, sizeof(*c));
452 c->function = KVM_CPUID_SIGNATURE_NEXT;
453 memcpy(signature, "KVMKVMKVM\0\0\0", 12);
454 c->eax = 0;
455 c->ebx = signature[0];
456 c->ecx = signature[1];
457 c->edx = signature[2];
458 }
459
460 has_msr_async_pf_en = c->eax & (1 << KVM_FEATURE_ASYNC_PF);
461
462 has_msr_pv_eoi_en = c->eax & (1 << KVM_FEATURE_PV_EOI);
463
464 cpu_x86_cpuid(env, 0, 0, &limit, &unused, &unused, &unused);
465
466 for (i = 0; i <= limit; i++) {
467 c = &cpuid_data.entries[cpuid_i++];
468
469 switch (i) {
470 case 2: {
471 /* Keep reading function 2 till all the input is received */
472 int times;
473
474 c->function = i;
475 c->flags = KVM_CPUID_FLAG_STATEFUL_FUNC |
476 KVM_CPUID_FLAG_STATE_READ_NEXT;
477 cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx);
478 times = c->eax & 0xff;
479
480 for (j = 1; j < times; ++j) {
481 c = &cpuid_data.entries[cpuid_i++];
482 c->function = i;
483 c->flags = KVM_CPUID_FLAG_STATEFUL_FUNC;
484 cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx);
485 }
486 break;
487 }
488 case 4:
489 case 0xb:
490 case 0xd:
491 for (j = 0; ; j++) {
492 if (i == 0xd && j == 64) {
493 break;
494 }
495 c->function = i;
496 c->flags = KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
497 c->index = j;
498 cpu_x86_cpuid(env, i, j, &c->eax, &c->ebx, &c->ecx, &c->edx);
499
500 if (i == 4 && c->eax == 0) {
501 break;
502 }
503 if (i == 0xb && !(c->ecx & 0xff00)) {
504 break;
505 }
506 if (i == 0xd && c->eax == 0) {
507 continue;
508 }
509 c = &cpuid_data.entries[cpuid_i++];
510 }
511 break;
512 default:
513 c->function = i;
514 c->flags = 0;
515 cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx);
516 break;
517 }
518 }
519 cpu_x86_cpuid(env, 0x80000000, 0, &limit, &unused, &unused, &unused);
520
521 for (i = 0x80000000; i <= limit; i++) {
522 c = &cpuid_data.entries[cpuid_i++];
523
524 c->function = i;
525 c->flags = 0;
526 cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx);
527 }
528
529 /* Call Centaur's CPUID instructions they are supported. */
530 if (env->cpuid_xlevel2 > 0) {
531 env->cpuid_ext4_features &=
532 kvm_arch_get_supported_cpuid(s, 0xC0000001, 0, R_EDX);
533 cpu_x86_cpuid(env, 0xC0000000, 0, &limit, &unused, &unused, &unused);
534
535 for (i = 0xC0000000; i <= limit; i++) {
536 c = &cpuid_data.entries[cpuid_i++];
537
538 c->function = i;
539 c->flags = 0;
540 cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx);
541 }
542 }
543
544 cpuid_data.cpuid.nent = cpuid_i;
545
546 if (((env->cpuid_version >> 8)&0xF) >= 6
547 && (env->cpuid_features&(CPUID_MCE|CPUID_MCA)) == (CPUID_MCE|CPUID_MCA)
548 && kvm_check_extension(env->kvm_state, KVM_CAP_MCE) > 0) {
549 uint64_t mcg_cap;
550 int banks;
551 int ret;
552
553 ret = kvm_get_mce_cap_supported(env->kvm_state, &mcg_cap, &banks);
554 if (ret < 0) {
555 fprintf(stderr, "kvm_get_mce_cap_supported: %s", strerror(-ret));
556 return ret;
557 }
558
559 if (banks > MCE_BANKS_DEF) {
560 banks = MCE_BANKS_DEF;
561 }
562 mcg_cap &= MCE_CAP_DEF;
563 mcg_cap |= banks;
564 ret = kvm_vcpu_ioctl(env, KVM_X86_SETUP_MCE, &mcg_cap);
565 if (ret < 0) {
566 fprintf(stderr, "KVM_X86_SETUP_MCE: %s", strerror(-ret));
567 return ret;
568 }
569
570 env->mcg_cap = mcg_cap;
571 }
572
573 qemu_add_vm_change_state_handler(cpu_update_state, env);
574
575 cpuid_data.cpuid.padding = 0;
576 r = kvm_vcpu_ioctl(env, KVM_SET_CPUID2, &cpuid_data);
577 if (r) {
578 return r;
579 }
580
581 r = kvm_check_extension(env->kvm_state, KVM_CAP_TSC_CONTROL);
582 if (r && env->tsc_khz) {
583 r = kvm_vcpu_ioctl(env, KVM_SET_TSC_KHZ, env->tsc_khz);
584 if (r < 0) {
585 fprintf(stderr, "KVM_SET_TSC_KHZ failed\n");
586 return r;
587 }
588 }
589
590 if (kvm_has_xsave()) {
591 env->kvm_xsave_buf = qemu_memalign(4096, sizeof(struct kvm_xsave));
592 }
593
594 return 0;
595 }
596
597 void kvm_arch_reset_vcpu(CPUX86State *env)
598 {
599 X86CPU *cpu = x86_env_get_cpu(env);
600
601 env->exception_injected = -1;
602 env->interrupt_injected = -1;
603 env->xcr0 = 1;
604 if (kvm_irqchip_in_kernel()) {
605 env->mp_state = cpu_is_bsp(cpu) ? KVM_MP_STATE_RUNNABLE :
606 KVM_MP_STATE_UNINITIALIZED;
607 } else {
608 env->mp_state = KVM_MP_STATE_RUNNABLE;
609 }
610 }
611
612 static int kvm_get_supported_msrs(KVMState *s)
613 {
614 static int kvm_supported_msrs;
615 int ret = 0;
616
617 /* first time */
618 if (kvm_supported_msrs == 0) {
619 struct kvm_msr_list msr_list, *kvm_msr_list;
620
621 kvm_supported_msrs = -1;
622
623 /* Obtain MSR list from KVM. These are the MSRs that we must
624 * save/restore */
625 msr_list.nmsrs = 0;
626 ret = kvm_ioctl(s, KVM_GET_MSR_INDEX_LIST, &msr_list);
627 if (ret < 0 && ret != -E2BIG) {
628 return ret;
629 }
630 /* Old kernel modules had a bug and could write beyond the provided
631 memory. Allocate at least a safe amount of 1K. */
632 kvm_msr_list = g_malloc0(MAX(1024, sizeof(msr_list) +
633 msr_list.nmsrs *
634 sizeof(msr_list.indices[0])));
635
636 kvm_msr_list->nmsrs = msr_list.nmsrs;
637 ret = kvm_ioctl(s, KVM_GET_MSR_INDEX_LIST, kvm_msr_list);
638 if (ret >= 0) {
639 int i;
640
641 for (i = 0; i < kvm_msr_list->nmsrs; i++) {
642 if (kvm_msr_list->indices[i] == MSR_STAR) {
643 has_msr_star = true;
644 continue;
645 }
646 if (kvm_msr_list->indices[i] == MSR_VM_HSAVE_PA) {
647 has_msr_hsave_pa = true;
648 continue;
649 }
650 if (kvm_msr_list->indices[i] == MSR_IA32_TSCDEADLINE) {
651 has_msr_tsc_deadline = true;
652 continue;
653 }
654 if (kvm_msr_list->indices[i] == MSR_IA32_MISC_ENABLE) {
655 has_msr_misc_enable = true;
656 continue;
657 }
658 }
659 }
660
661 g_free(kvm_msr_list);
662 }
663
664 return ret;
665 }
666
667 int kvm_arch_init(KVMState *s)
668 {
669 QemuOptsList *list = qemu_find_opts("machine");
670 uint64_t identity_base = 0xfffbc000;
671 uint64_t shadow_mem;
672 int ret;
673 struct utsname utsname;
674
675 ret = kvm_get_supported_msrs(s);
676 if (ret < 0) {
677 return ret;
678 }
679
680 uname(&utsname);
681 lm_capable_kernel = strcmp(utsname.machine, "x86_64") == 0;
682
683 /*
684 * On older Intel CPUs, KVM uses vm86 mode to emulate 16-bit code directly.
685 * In order to use vm86 mode, an EPT identity map and a TSS are needed.
686 * Since these must be part of guest physical memory, we need to allocate
687 * them, both by setting their start addresses in the kernel and by
688 * creating a corresponding e820 entry. We need 4 pages before the BIOS.
689 *
690 * Older KVM versions may not support setting the identity map base. In
691 * that case we need to stick with the default, i.e. a 256K maximum BIOS
692 * size.
693 */
694 if (kvm_check_extension(s, KVM_CAP_SET_IDENTITY_MAP_ADDR)) {
695 /* Allows up to 16M BIOSes. */
696 identity_base = 0xfeffc000;
697
698 ret = kvm_vm_ioctl(s, KVM_SET_IDENTITY_MAP_ADDR, &identity_base);
699 if (ret < 0) {
700 return ret;
701 }
702 }
703
704 /* Set TSS base one page after EPT identity map. */
705 ret = kvm_vm_ioctl(s, KVM_SET_TSS_ADDR, identity_base + 0x1000);
706 if (ret < 0) {
707 return ret;
708 }
709
710 /* Tell fw_cfg to notify the BIOS to reserve the range. */
711 ret = e820_add_entry(identity_base, 0x4000, E820_RESERVED);
712 if (ret < 0) {
713 fprintf(stderr, "e820_add_entry() table is full\n");
714 return ret;
715 }
716 qemu_register_reset(kvm_unpoison_all, NULL);
717
718 if (!QTAILQ_EMPTY(&list->head)) {
719 shadow_mem = qemu_opt_get_size(QTAILQ_FIRST(&list->head),
720 "kvm_shadow_mem", -1);
721 if (shadow_mem != -1) {
722 shadow_mem /= 4096;
723 ret = kvm_vm_ioctl(s, KVM_SET_NR_MMU_PAGES, shadow_mem);
724 if (ret < 0) {
725 return ret;
726 }
727 }
728 }
729 return 0;
730 }
731
732 static void set_v8086_seg(struct kvm_segment *lhs, const SegmentCache *rhs)
733 {
734 lhs->selector = rhs->selector;
735 lhs->base = rhs->base;
736 lhs->limit = rhs->limit;
737 lhs->type = 3;
738 lhs->present = 1;
739 lhs->dpl = 3;
740 lhs->db = 0;
741 lhs->s = 1;
742 lhs->l = 0;
743 lhs->g = 0;
744 lhs->avl = 0;
745 lhs->unusable = 0;
746 }
747
748 static void set_seg(struct kvm_segment *lhs, const SegmentCache *rhs)
749 {
750 unsigned flags = rhs->flags;
751 lhs->selector = rhs->selector;
752 lhs->base = rhs->base;
753 lhs->limit = rhs->limit;
754 lhs->type = (flags >> DESC_TYPE_SHIFT) & 15;
755 lhs->present = (flags & DESC_P_MASK) != 0;
756 lhs->dpl = (flags >> DESC_DPL_SHIFT) & 3;
757 lhs->db = (flags >> DESC_B_SHIFT) & 1;
758 lhs->s = (flags & DESC_S_MASK) != 0;
759 lhs->l = (flags >> DESC_L_SHIFT) & 1;
760 lhs->g = (flags & DESC_G_MASK) != 0;
761 lhs->avl = (flags & DESC_AVL_MASK) != 0;
762 lhs->unusable = 0;
763 lhs->padding = 0;
764 }
765
766 static void get_seg(SegmentCache *lhs, const struct kvm_segment *rhs)
767 {
768 lhs->selector = rhs->selector;
769 lhs->base = rhs->base;
770 lhs->limit = rhs->limit;
771 lhs->flags = (rhs->type << DESC_TYPE_SHIFT) |
772 (rhs->present * DESC_P_MASK) |
773 (rhs->dpl << DESC_DPL_SHIFT) |
774 (rhs->db << DESC_B_SHIFT) |
775 (rhs->s * DESC_S_MASK) |
776 (rhs->l << DESC_L_SHIFT) |
777 (rhs->g * DESC_G_MASK) |
778 (rhs->avl * DESC_AVL_MASK);
779 }
780
781 static void kvm_getput_reg(__u64 *kvm_reg, target_ulong *qemu_reg, int set)
782 {
783 if (set) {
784 *kvm_reg = *qemu_reg;
785 } else {
786 *qemu_reg = *kvm_reg;
787 }
788 }
789
790 static int kvm_getput_regs(CPUX86State *env, int set)
791 {
792 struct kvm_regs regs;
793 int ret = 0;
794
795 if (!set) {
796 ret = kvm_vcpu_ioctl(env, KVM_GET_REGS, &regs);
797 if (ret < 0) {
798 return ret;
799 }
800 }
801
802 kvm_getput_reg(&regs.rax, &env->regs[R_EAX], set);
803 kvm_getput_reg(&regs.rbx, &env->regs[R_EBX], set);
804 kvm_getput_reg(&regs.rcx, &env->regs[R_ECX], set);
805 kvm_getput_reg(&regs.rdx, &env->regs[R_EDX], set);
806 kvm_getput_reg(&regs.rsi, &env->regs[R_ESI], set);
807 kvm_getput_reg(&regs.rdi, &env->regs[R_EDI], set);
808 kvm_getput_reg(&regs.rsp, &env->regs[R_ESP], set);
809 kvm_getput_reg(&regs.rbp, &env->regs[R_EBP], set);
810 #ifdef TARGET_X86_64
811 kvm_getput_reg(&regs.r8, &env->regs[8], set);
812 kvm_getput_reg(&regs.r9, &env->regs[9], set);
813 kvm_getput_reg(&regs.r10, &env->regs[10], set);
814 kvm_getput_reg(&regs.r11, &env->regs[11], set);
815 kvm_getput_reg(&regs.r12, &env->regs[12], set);
816 kvm_getput_reg(&regs.r13, &env->regs[13], set);
817 kvm_getput_reg(&regs.r14, &env->regs[14], set);
818 kvm_getput_reg(&regs.r15, &env->regs[15], set);
819 #endif
820
821 kvm_getput_reg(&regs.rflags, &env->eflags, set);
822 kvm_getput_reg(&regs.rip, &env->eip, set);
823
824 if (set) {
825 ret = kvm_vcpu_ioctl(env, KVM_SET_REGS, &regs);
826 }
827
828 return ret;
829 }
830
831 static int kvm_put_fpu(CPUX86State *env)
832 {
833 struct kvm_fpu fpu;
834 int i;
835
836 memset(&fpu, 0, sizeof fpu);
837 fpu.fsw = env->fpus & ~(7 << 11);
838 fpu.fsw |= (env->fpstt & 7) << 11;
839 fpu.fcw = env->fpuc;
840 fpu.last_opcode = env->fpop;
841 fpu.last_ip = env->fpip;
842 fpu.last_dp = env->fpdp;
843 for (i = 0; i < 8; ++i) {
844 fpu.ftwx |= (!env->fptags[i]) << i;
845 }
846 memcpy(fpu.fpr, env->fpregs, sizeof env->fpregs);
847 memcpy(fpu.xmm, env->xmm_regs, sizeof env->xmm_regs);
848 fpu.mxcsr = env->mxcsr;
849
850 return kvm_vcpu_ioctl(env, KVM_SET_FPU, &fpu);
851 }
852
853 #define XSAVE_FCW_FSW 0
854 #define XSAVE_FTW_FOP 1
855 #define XSAVE_CWD_RIP 2
856 #define XSAVE_CWD_RDP 4
857 #define XSAVE_MXCSR 6
858 #define XSAVE_ST_SPACE 8
859 #define XSAVE_XMM_SPACE 40
860 #define XSAVE_XSTATE_BV 128
861 #define XSAVE_YMMH_SPACE 144
862
863 static int kvm_put_xsave(CPUX86State *env)
864 {
865 struct kvm_xsave* xsave = env->kvm_xsave_buf;
866 uint16_t cwd, swd, twd;
867 int i, r;
868
869 if (!kvm_has_xsave()) {
870 return kvm_put_fpu(env);
871 }
872
873 memset(xsave, 0, sizeof(struct kvm_xsave));
874 twd = 0;
875 swd = env->fpus & ~(7 << 11);
876 swd |= (env->fpstt & 7) << 11;
877 cwd = env->fpuc;
878 for (i = 0; i < 8; ++i) {
879 twd |= (!env->fptags[i]) << i;
880 }
881 xsave->region[XSAVE_FCW_FSW] = (uint32_t)(swd << 16) + cwd;
882 xsave->region[XSAVE_FTW_FOP] = (uint32_t)(env->fpop << 16) + twd;
883 memcpy(&xsave->region[XSAVE_CWD_RIP], &env->fpip, sizeof(env->fpip));
884 memcpy(&xsave->region[XSAVE_CWD_RDP], &env->fpdp, sizeof(env->fpdp));
885 memcpy(&xsave->region[XSAVE_ST_SPACE], env->fpregs,
886 sizeof env->fpregs);
887 memcpy(&xsave->region[XSAVE_XMM_SPACE], env->xmm_regs,
888 sizeof env->xmm_regs);
889 xsave->region[XSAVE_MXCSR] = env->mxcsr;
890 *(uint64_t *)&xsave->region[XSAVE_XSTATE_BV] = env->xstate_bv;
891 memcpy(&xsave->region[XSAVE_YMMH_SPACE], env->ymmh_regs,
892 sizeof env->ymmh_regs);
893 r = kvm_vcpu_ioctl(env, KVM_SET_XSAVE, xsave);
894 return r;
895 }
896
897 static int kvm_put_xcrs(CPUX86State *env)
898 {
899 struct kvm_xcrs xcrs;
900
901 if (!kvm_has_xcrs()) {
902 return 0;
903 }
904
905 xcrs.nr_xcrs = 1;
906 xcrs.flags = 0;
907 xcrs.xcrs[0].xcr = 0;
908 xcrs.xcrs[0].value = env->xcr0;
909 return kvm_vcpu_ioctl(env, KVM_SET_XCRS, &xcrs);
910 }
911
912 static int kvm_put_sregs(CPUX86State *env)
913 {
914 struct kvm_sregs sregs;
915
916 memset(sregs.interrupt_bitmap, 0, sizeof(sregs.interrupt_bitmap));
917 if (env->interrupt_injected >= 0) {
918 sregs.interrupt_bitmap[env->interrupt_injected / 64] |=
919 (uint64_t)1 << (env->interrupt_injected % 64);
920 }
921
922 if ((env->eflags & VM_MASK)) {
923 set_v8086_seg(&sregs.cs, &env->segs[R_CS]);
924 set_v8086_seg(&sregs.ds, &env->segs[R_DS]);
925 set_v8086_seg(&sregs.es, &env->segs[R_ES]);
926 set_v8086_seg(&sregs.fs, &env->segs[R_FS]);
927 set_v8086_seg(&sregs.gs, &env->segs[R_GS]);
928 set_v8086_seg(&sregs.ss, &env->segs[R_SS]);
929 } else {
930 set_seg(&sregs.cs, &env->segs[R_CS]);
931 set_seg(&sregs.ds, &env->segs[R_DS]);
932 set_seg(&sregs.es, &env->segs[R_ES]);
933 set_seg(&sregs.fs, &env->segs[R_FS]);
934 set_seg(&sregs.gs, &env->segs[R_GS]);
935 set_seg(&sregs.ss, &env->segs[R_SS]);
936 }
937
938 set_seg(&sregs.tr, &env->tr);
939 set_seg(&sregs.ldt, &env->ldt);
940
941 sregs.idt.limit = env->idt.limit;
942 sregs.idt.base = env->idt.base;
943 memset(sregs.idt.padding, 0, sizeof sregs.idt.padding);
944 sregs.gdt.limit = env->gdt.limit;
945 sregs.gdt.base = env->gdt.base;
946 memset(sregs.gdt.padding, 0, sizeof sregs.gdt.padding);
947
948 sregs.cr0 = env->cr[0];
949 sregs.cr2 = env->cr[2];
950 sregs.cr3 = env->cr[3];
951 sregs.cr4 = env->cr[4];
952
953 sregs.cr8 = cpu_get_apic_tpr(env->apic_state);
954 sregs.apic_base = cpu_get_apic_base(env->apic_state);
955
956 sregs.efer = env->efer;
957
958 return kvm_vcpu_ioctl(env, KVM_SET_SREGS, &sregs);
959 }
960
961 static void kvm_msr_entry_set(struct kvm_msr_entry *entry,
962 uint32_t index, uint64_t value)
963 {
964 entry->index = index;
965 entry->data = value;
966 }
967
968 static int kvm_put_msrs(CPUX86State *env, int level)
969 {
970 struct {
971 struct kvm_msrs info;
972 struct kvm_msr_entry entries[100];
973 } msr_data;
974 struct kvm_msr_entry *msrs = msr_data.entries;
975 int n = 0;
976
977 kvm_msr_entry_set(&msrs[n++], MSR_IA32_SYSENTER_CS, env->sysenter_cs);
978 kvm_msr_entry_set(&msrs[n++], MSR_IA32_SYSENTER_ESP, env->sysenter_esp);
979 kvm_msr_entry_set(&msrs[n++], MSR_IA32_SYSENTER_EIP, env->sysenter_eip);
980 kvm_msr_entry_set(&msrs[n++], MSR_PAT, env->pat);
981 if (has_msr_star) {
982 kvm_msr_entry_set(&msrs[n++], MSR_STAR, env->star);
983 }
984 if (has_msr_hsave_pa) {
985 kvm_msr_entry_set(&msrs[n++], MSR_VM_HSAVE_PA, env->vm_hsave);
986 }
987 if (has_msr_tsc_deadline) {
988 kvm_msr_entry_set(&msrs[n++], MSR_IA32_TSCDEADLINE, env->tsc_deadline);
989 }
990 if (has_msr_misc_enable) {
991 kvm_msr_entry_set(&msrs[n++], MSR_IA32_MISC_ENABLE,
992 env->msr_ia32_misc_enable);
993 }
994 #ifdef TARGET_X86_64
995 if (lm_capable_kernel) {
996 kvm_msr_entry_set(&msrs[n++], MSR_CSTAR, env->cstar);
997 kvm_msr_entry_set(&msrs[n++], MSR_KERNELGSBASE, env->kernelgsbase);
998 kvm_msr_entry_set(&msrs[n++], MSR_FMASK, env->fmask);
999 kvm_msr_entry_set(&msrs[n++], MSR_LSTAR, env->lstar);
1000 }
1001 #endif
1002 if (level == KVM_PUT_FULL_STATE) {
1003 /*
1004 * KVM is yet unable to synchronize TSC values of multiple VCPUs on
1005 * writeback. Until this is fixed, we only write the offset to SMP
1006 * guests after migration, desynchronizing the VCPUs, but avoiding
1007 * huge jump-backs that would occur without any writeback at all.
1008 */
1009 if (smp_cpus == 1 || env->tsc != 0) {
1010 kvm_msr_entry_set(&msrs[n++], MSR_IA32_TSC, env->tsc);
1011 }
1012 }
1013 /*
1014 * The following paravirtual MSRs have side effects on the guest or are
1015 * too heavy for normal writeback. Limit them to reset or full state
1016 * updates.
1017 */
1018 if (level >= KVM_PUT_RESET_STATE) {
1019 kvm_msr_entry_set(&msrs[n++], MSR_KVM_SYSTEM_TIME,
1020 env->system_time_msr);
1021 kvm_msr_entry_set(&msrs[n++], MSR_KVM_WALL_CLOCK, env->wall_clock_msr);
1022 if (has_msr_async_pf_en) {
1023 kvm_msr_entry_set(&msrs[n++], MSR_KVM_ASYNC_PF_EN,
1024 env->async_pf_en_msr);
1025 }
1026 if (has_msr_pv_eoi_en) {
1027 kvm_msr_entry_set(&msrs[n++], MSR_KVM_PV_EOI_EN,
1028 env->pv_eoi_en_msr);
1029 }
1030 if (hyperv_hypercall_available()) {
1031 kvm_msr_entry_set(&msrs[n++], HV_X64_MSR_GUEST_OS_ID, 0);
1032 kvm_msr_entry_set(&msrs[n++], HV_X64_MSR_HYPERCALL, 0);
1033 }
1034 if (hyperv_vapic_recommended()) {
1035 kvm_msr_entry_set(&msrs[n++], HV_X64_MSR_APIC_ASSIST_PAGE, 0);
1036 }
1037 }
1038 if (env->mcg_cap) {
1039 int i;
1040
1041 kvm_msr_entry_set(&msrs[n++], MSR_MCG_STATUS, env->mcg_status);
1042 kvm_msr_entry_set(&msrs[n++], MSR_MCG_CTL, env->mcg_ctl);
1043 for (i = 0; i < (env->mcg_cap & 0xff) * 4; i++) {
1044 kvm_msr_entry_set(&msrs[n++], MSR_MC0_CTL + i, env->mce_banks[i]);
1045 }
1046 }
1047
1048 msr_data.info.nmsrs = n;
1049
1050 return kvm_vcpu_ioctl(env, KVM_SET_MSRS, &msr_data);
1051
1052 }
1053
1054
1055 static int kvm_get_fpu(CPUX86State *env)
1056 {
1057 struct kvm_fpu fpu;
1058 int i, ret;
1059
1060 ret = kvm_vcpu_ioctl(env, KVM_GET_FPU, &fpu);
1061 if (ret < 0) {
1062 return ret;
1063 }
1064
1065 env->fpstt = (fpu.fsw >> 11) & 7;
1066 env->fpus = fpu.fsw;
1067 env->fpuc = fpu.fcw;
1068 env->fpop = fpu.last_opcode;
1069 env->fpip = fpu.last_ip;
1070 env->fpdp = fpu.last_dp;
1071 for (i = 0; i < 8; ++i) {
1072 env->fptags[i] = !((fpu.ftwx >> i) & 1);
1073 }
1074 memcpy(env->fpregs, fpu.fpr, sizeof env->fpregs);
1075 memcpy(env->xmm_regs, fpu.xmm, sizeof env->xmm_regs);
1076 env->mxcsr = fpu.mxcsr;
1077
1078 return 0;
1079 }
1080
1081 static int kvm_get_xsave(CPUX86State *env)
1082 {
1083 struct kvm_xsave* xsave = env->kvm_xsave_buf;
1084 int ret, i;
1085 uint16_t cwd, swd, twd;
1086
1087 if (!kvm_has_xsave()) {
1088 return kvm_get_fpu(env);
1089 }
1090
1091 ret = kvm_vcpu_ioctl(env, KVM_GET_XSAVE, xsave);
1092 if (ret < 0) {
1093 return ret;
1094 }
1095
1096 cwd = (uint16_t)xsave->region[XSAVE_FCW_FSW];
1097 swd = (uint16_t)(xsave->region[XSAVE_FCW_FSW] >> 16);
1098 twd = (uint16_t)xsave->region[XSAVE_FTW_FOP];
1099 env->fpop = (uint16_t)(xsave->region[XSAVE_FTW_FOP] >> 16);
1100 env->fpstt = (swd >> 11) & 7;
1101 env->fpus = swd;
1102 env->fpuc = cwd;
1103 for (i = 0; i < 8; ++i) {
1104 env->fptags[i] = !((twd >> i) & 1);
1105 }
1106 memcpy(&env->fpip, &xsave->region[XSAVE_CWD_RIP], sizeof(env->fpip));
1107 memcpy(&env->fpdp, &xsave->region[XSAVE_CWD_RDP], sizeof(env->fpdp));
1108 env->mxcsr = xsave->region[XSAVE_MXCSR];
1109 memcpy(env->fpregs, &xsave->region[XSAVE_ST_SPACE],
1110 sizeof env->fpregs);
1111 memcpy(env->xmm_regs, &xsave->region[XSAVE_XMM_SPACE],
1112 sizeof env->xmm_regs);
1113 env->xstate_bv = *(uint64_t *)&xsave->region[XSAVE_XSTATE_BV];
1114 memcpy(env->ymmh_regs, &xsave->region[XSAVE_YMMH_SPACE],
1115 sizeof env->ymmh_regs);
1116 return 0;
1117 }
1118
1119 static int kvm_get_xcrs(CPUX86State *env)
1120 {
1121 int i, ret;
1122 struct kvm_xcrs xcrs;
1123
1124 if (!kvm_has_xcrs()) {
1125 return 0;
1126 }
1127
1128 ret = kvm_vcpu_ioctl(env, KVM_GET_XCRS, &xcrs);
1129 if (ret < 0) {
1130 return ret;
1131 }
1132
1133 for (i = 0; i < xcrs.nr_xcrs; i++) {
1134 /* Only support xcr0 now */
1135 if (xcrs.xcrs[0].xcr == 0) {
1136 env->xcr0 = xcrs.xcrs[0].value;
1137 break;
1138 }
1139 }
1140 return 0;
1141 }
1142
1143 static int kvm_get_sregs(CPUX86State *env)
1144 {
1145 struct kvm_sregs sregs;
1146 uint32_t hflags;
1147 int bit, i, ret;
1148
1149 ret = kvm_vcpu_ioctl(env, KVM_GET_SREGS, &sregs);
1150 if (ret < 0) {
1151 return ret;
1152 }
1153
1154 /* There can only be one pending IRQ set in the bitmap at a time, so try
1155 to find it and save its number instead (-1 for none). */
1156 env->interrupt_injected = -1;
1157 for (i = 0; i < ARRAY_SIZE(sregs.interrupt_bitmap); i++) {
1158 if (sregs.interrupt_bitmap[i]) {
1159 bit = ctz64(sregs.interrupt_bitmap[i]);
1160 env->interrupt_injected = i * 64 + bit;
1161 break;
1162 }
1163 }
1164
1165 get_seg(&env->segs[R_CS], &sregs.cs);
1166 get_seg(&env->segs[R_DS], &sregs.ds);
1167 get_seg(&env->segs[R_ES], &sregs.es);
1168 get_seg(&env->segs[R_FS], &sregs.fs);
1169 get_seg(&env->segs[R_GS], &sregs.gs);
1170 get_seg(&env->segs[R_SS], &sregs.ss);
1171
1172 get_seg(&env->tr, &sregs.tr);
1173 get_seg(&env->ldt, &sregs.ldt);
1174
1175 env->idt.limit = sregs.idt.limit;
1176 env->idt.base = sregs.idt.base;
1177 env->gdt.limit = sregs.gdt.limit;
1178 env->gdt.base = sregs.gdt.base;
1179
1180 env->cr[0] = sregs.cr0;
1181 env->cr[2] = sregs.cr2;
1182 env->cr[3] = sregs.cr3;
1183 env->cr[4] = sregs.cr4;
1184
1185 env->efer = sregs.efer;
1186
1187 /* changes to apic base and cr8/tpr are read back via kvm_arch_post_run */
1188
1189 #define HFLAG_COPY_MASK \
1190 ~( HF_CPL_MASK | HF_PE_MASK | HF_MP_MASK | HF_EM_MASK | \
1191 HF_TS_MASK | HF_TF_MASK | HF_VM_MASK | HF_IOPL_MASK | \
1192 HF_OSFXSR_MASK | HF_LMA_MASK | HF_CS32_MASK | \
1193 HF_SS32_MASK | HF_CS64_MASK | HF_ADDSEG_MASK)
1194
1195 hflags = (env->segs[R_CS].flags >> DESC_DPL_SHIFT) & HF_CPL_MASK;
1196 hflags |= (env->cr[0] & CR0_PE_MASK) << (HF_PE_SHIFT - CR0_PE_SHIFT);
1197 hflags |= (env->cr[0] << (HF_MP_SHIFT - CR0_MP_SHIFT)) &
1198 (HF_MP_MASK | HF_EM_MASK | HF_TS_MASK);
1199 hflags |= (env->eflags & (HF_TF_MASK | HF_VM_MASK | HF_IOPL_MASK));
1200 hflags |= (env->cr[4] & CR4_OSFXSR_MASK) <<
1201 (HF_OSFXSR_SHIFT - CR4_OSFXSR_SHIFT);
1202
1203 if (env->efer & MSR_EFER_LMA) {
1204 hflags |= HF_LMA_MASK;
1205 }
1206
1207 if ((hflags & HF_LMA_MASK) && (env->segs[R_CS].flags & DESC_L_MASK)) {
1208 hflags |= HF_CS32_MASK | HF_SS32_MASK | HF_CS64_MASK;
1209 } else {
1210 hflags |= (env->segs[R_CS].flags & DESC_B_MASK) >>
1211 (DESC_B_SHIFT - HF_CS32_SHIFT);
1212 hflags |= (env->segs[R_SS].flags & DESC_B_MASK) >>
1213 (DESC_B_SHIFT - HF_SS32_SHIFT);
1214 if (!(env->cr[0] & CR0_PE_MASK) || (env->eflags & VM_MASK) ||
1215 !(hflags & HF_CS32_MASK)) {
1216 hflags |= HF_ADDSEG_MASK;
1217 } else {
1218 hflags |= ((env->segs[R_DS].base | env->segs[R_ES].base |
1219 env->segs[R_SS].base) != 0) << HF_ADDSEG_SHIFT;
1220 }
1221 }
1222 env->hflags = (env->hflags & HFLAG_COPY_MASK) | hflags;
1223
1224 return 0;
1225 }
1226
1227 static int kvm_get_msrs(CPUX86State *env)
1228 {
1229 struct {
1230 struct kvm_msrs info;
1231 struct kvm_msr_entry entries[100];
1232 } msr_data;
1233 struct kvm_msr_entry *msrs = msr_data.entries;
1234 int ret, i, n;
1235
1236 n = 0;
1237 msrs[n++].index = MSR_IA32_SYSENTER_CS;
1238 msrs[n++].index = MSR_IA32_SYSENTER_ESP;
1239 msrs[n++].index = MSR_IA32_SYSENTER_EIP;
1240 msrs[n++].index = MSR_PAT;
1241 if (has_msr_star) {
1242 msrs[n++].index = MSR_STAR;
1243 }
1244 if (has_msr_hsave_pa) {
1245 msrs[n++].index = MSR_VM_HSAVE_PA;
1246 }
1247 if (has_msr_tsc_deadline) {
1248 msrs[n++].index = MSR_IA32_TSCDEADLINE;
1249 }
1250 if (has_msr_misc_enable) {
1251 msrs[n++].index = MSR_IA32_MISC_ENABLE;
1252 }
1253
1254 if (!env->tsc_valid) {
1255 msrs[n++].index = MSR_IA32_TSC;
1256 env->tsc_valid = !runstate_is_running();
1257 }
1258
1259 #ifdef TARGET_X86_64
1260 if (lm_capable_kernel) {
1261 msrs[n++].index = MSR_CSTAR;
1262 msrs[n++].index = MSR_KERNELGSBASE;
1263 msrs[n++].index = MSR_FMASK;
1264 msrs[n++].index = MSR_LSTAR;
1265 }
1266 #endif
1267 msrs[n++].index = MSR_KVM_SYSTEM_TIME;
1268 msrs[n++].index = MSR_KVM_WALL_CLOCK;
1269 if (has_msr_async_pf_en) {
1270 msrs[n++].index = MSR_KVM_ASYNC_PF_EN;
1271 }
1272 if (has_msr_pv_eoi_en) {
1273 msrs[n++].index = MSR_KVM_PV_EOI_EN;
1274 }
1275
1276 if (env->mcg_cap) {
1277 msrs[n++].index = MSR_MCG_STATUS;
1278 msrs[n++].index = MSR_MCG_CTL;
1279 for (i = 0; i < (env->mcg_cap & 0xff) * 4; i++) {
1280 msrs[n++].index = MSR_MC0_CTL + i;
1281 }
1282 }
1283
1284 msr_data.info.nmsrs = n;
1285 ret = kvm_vcpu_ioctl(env, KVM_GET_MSRS, &msr_data);
1286 if (ret < 0) {
1287 return ret;
1288 }
1289
1290 for (i = 0; i < ret; i++) {
1291 switch (msrs[i].index) {
1292 case MSR_IA32_SYSENTER_CS:
1293 env->sysenter_cs = msrs[i].data;
1294 break;
1295 case MSR_IA32_SYSENTER_ESP:
1296 env->sysenter_esp = msrs[i].data;
1297 break;
1298 case MSR_IA32_SYSENTER_EIP:
1299 env->sysenter_eip = msrs[i].data;
1300 break;
1301 case MSR_PAT:
1302 env->pat = msrs[i].data;
1303 break;
1304 case MSR_STAR:
1305 env->star = msrs[i].data;
1306 break;
1307 #ifdef TARGET_X86_64
1308 case MSR_CSTAR:
1309 env->cstar = msrs[i].data;
1310 break;
1311 case MSR_KERNELGSBASE:
1312 env->kernelgsbase = msrs[i].data;
1313 break;
1314 case MSR_FMASK:
1315 env->fmask = msrs[i].data;
1316 break;
1317 case MSR_LSTAR:
1318 env->lstar = msrs[i].data;
1319 break;
1320 #endif
1321 case MSR_IA32_TSC:
1322 env->tsc = msrs[i].data;
1323 break;
1324 case MSR_IA32_TSCDEADLINE:
1325 env->tsc_deadline = msrs[i].data;
1326 break;
1327 case MSR_VM_HSAVE_PA:
1328 env->vm_hsave = msrs[i].data;
1329 break;
1330 case MSR_KVM_SYSTEM_TIME:
1331 env->system_time_msr = msrs[i].data;
1332 break;
1333 case MSR_KVM_WALL_CLOCK:
1334 env->wall_clock_msr = msrs[i].data;
1335 break;
1336 case MSR_MCG_STATUS:
1337 env->mcg_status = msrs[i].data;
1338 break;
1339 case MSR_MCG_CTL:
1340 env->mcg_ctl = msrs[i].data;
1341 break;
1342 case MSR_IA32_MISC_ENABLE:
1343 env->msr_ia32_misc_enable = msrs[i].data;
1344 break;
1345 default:
1346 if (msrs[i].index >= MSR_MC0_CTL &&
1347 msrs[i].index < MSR_MC0_CTL + (env->mcg_cap & 0xff) * 4) {
1348 env->mce_banks[msrs[i].index - MSR_MC0_CTL] = msrs[i].data;
1349 }
1350 break;
1351 case MSR_KVM_ASYNC_PF_EN:
1352 env->async_pf_en_msr = msrs[i].data;
1353 break;
1354 case MSR_KVM_PV_EOI_EN:
1355 env->pv_eoi_en_msr = msrs[i].data;
1356 break;
1357 }
1358 }
1359
1360 return 0;
1361 }
1362
1363 static int kvm_put_mp_state(CPUX86State *env)
1364 {
1365 struct kvm_mp_state mp_state = { .mp_state = env->mp_state };
1366
1367 return kvm_vcpu_ioctl(env, KVM_SET_MP_STATE, &mp_state);
1368 }
1369
1370 static int kvm_get_mp_state(CPUX86State *env)
1371 {
1372 struct kvm_mp_state mp_state;
1373 int ret;
1374
1375 ret = kvm_vcpu_ioctl(env, KVM_GET_MP_STATE, &mp_state);
1376 if (ret < 0) {
1377 return ret;
1378 }
1379 env->mp_state = mp_state.mp_state;
1380 if (kvm_irqchip_in_kernel()) {
1381 env->halted = (mp_state.mp_state == KVM_MP_STATE_HALTED);
1382 }
1383 return 0;
1384 }
1385
1386 static int kvm_get_apic(CPUX86State *env)
1387 {
1388 DeviceState *apic = env->apic_state;
1389 struct kvm_lapic_state kapic;
1390 int ret;
1391
1392 if (apic && kvm_irqchip_in_kernel()) {
1393 ret = kvm_vcpu_ioctl(env, KVM_GET_LAPIC, &kapic);
1394 if (ret < 0) {
1395 return ret;
1396 }
1397
1398 kvm_get_apic_state(apic, &kapic);
1399 }
1400 return 0;
1401 }
1402
1403 static int kvm_put_apic(CPUX86State *env)
1404 {
1405 DeviceState *apic = env->apic_state;
1406 struct kvm_lapic_state kapic;
1407
1408 if (apic && kvm_irqchip_in_kernel()) {
1409 kvm_put_apic_state(apic, &kapic);
1410
1411 return kvm_vcpu_ioctl(env, KVM_SET_LAPIC, &kapic);
1412 }
1413 return 0;
1414 }
1415
1416 static int kvm_put_vcpu_events(CPUX86State *env, int level)
1417 {
1418 struct kvm_vcpu_events events;
1419
1420 if (!kvm_has_vcpu_events()) {
1421 return 0;
1422 }
1423
1424 events.exception.injected = (env->exception_injected >= 0);
1425 events.exception.nr = env->exception_injected;
1426 events.exception.has_error_code = env->has_error_code;
1427 events.exception.error_code = env->error_code;
1428 events.exception.pad = 0;
1429
1430 events.interrupt.injected = (env->interrupt_injected >= 0);
1431 events.interrupt.nr = env->interrupt_injected;
1432 events.interrupt.soft = env->soft_interrupt;
1433
1434 events.nmi.injected = env->nmi_injected;
1435 events.nmi.pending = env->nmi_pending;
1436 events.nmi.masked = !!(env->hflags2 & HF2_NMI_MASK);
1437 events.nmi.pad = 0;
1438
1439 events.sipi_vector = env->sipi_vector;
1440
1441 events.flags = 0;
1442 if (level >= KVM_PUT_RESET_STATE) {
1443 events.flags |=
1444 KVM_VCPUEVENT_VALID_NMI_PENDING | KVM_VCPUEVENT_VALID_SIPI_VECTOR;
1445 }
1446
1447 return kvm_vcpu_ioctl(env, KVM_SET_VCPU_EVENTS, &events);
1448 }
1449
1450 static int kvm_get_vcpu_events(CPUX86State *env)
1451 {
1452 struct kvm_vcpu_events events;
1453 int ret;
1454
1455 if (!kvm_has_vcpu_events()) {
1456 return 0;
1457 }
1458
1459 ret = kvm_vcpu_ioctl(env, KVM_GET_VCPU_EVENTS, &events);
1460 if (ret < 0) {
1461 return ret;
1462 }
1463 env->exception_injected =
1464 events.exception.injected ? events.exception.nr : -1;
1465 env->has_error_code = events.exception.has_error_code;
1466 env->error_code = events.exception.error_code;
1467
1468 env->interrupt_injected =
1469 events.interrupt.injected ? events.interrupt.nr : -1;
1470 env->soft_interrupt = events.interrupt.soft;
1471
1472 env->nmi_injected = events.nmi.injected;
1473 env->nmi_pending = events.nmi.pending;
1474 if (events.nmi.masked) {
1475 env->hflags2 |= HF2_NMI_MASK;
1476 } else {
1477 env->hflags2 &= ~HF2_NMI_MASK;
1478 }
1479
1480 env->sipi_vector = events.sipi_vector;
1481
1482 return 0;
1483 }
1484
1485 static int kvm_guest_debug_workarounds(CPUX86State *env)
1486 {
1487 int ret = 0;
1488 unsigned long reinject_trap = 0;
1489
1490 if (!kvm_has_vcpu_events()) {
1491 if (env->exception_injected == 1) {
1492 reinject_trap = KVM_GUESTDBG_INJECT_DB;
1493 } else if (env->exception_injected == 3) {
1494 reinject_trap = KVM_GUESTDBG_INJECT_BP;
1495 }
1496 env->exception_injected = -1;
1497 }
1498
1499 /*
1500 * Kernels before KVM_CAP_X86_ROBUST_SINGLESTEP overwrote flags.TF
1501 * injected via SET_GUEST_DEBUG while updating GP regs. Work around this
1502 * by updating the debug state once again if single-stepping is on.
1503 * Another reason to call kvm_update_guest_debug here is a pending debug
1504 * trap raise by the guest. On kernels without SET_VCPU_EVENTS we have to
1505 * reinject them via SET_GUEST_DEBUG.
1506 */
1507 if (reinject_trap ||
1508 (!kvm_has_robust_singlestep() && env->singlestep_enabled)) {
1509 ret = kvm_update_guest_debug(env, reinject_trap);
1510 }
1511 return ret;
1512 }
1513
1514 static int kvm_put_debugregs(CPUX86State *env)
1515 {
1516 struct kvm_debugregs dbgregs;
1517 int i;
1518
1519 if (!kvm_has_debugregs()) {
1520 return 0;
1521 }
1522
1523 for (i = 0; i < 4; i++) {
1524 dbgregs.db[i] = env->dr[i];
1525 }
1526 dbgregs.dr6 = env->dr[6];
1527 dbgregs.dr7 = env->dr[7];
1528 dbgregs.flags = 0;
1529
1530 return kvm_vcpu_ioctl(env, KVM_SET_DEBUGREGS, &dbgregs);
1531 }
1532
1533 static int kvm_get_debugregs(CPUX86State *env)
1534 {
1535 struct kvm_debugregs dbgregs;
1536 int i, ret;
1537
1538 if (!kvm_has_debugregs()) {
1539 return 0;
1540 }
1541
1542 ret = kvm_vcpu_ioctl(env, KVM_GET_DEBUGREGS, &dbgregs);
1543 if (ret < 0) {
1544 return ret;
1545 }
1546 for (i = 0; i < 4; i++) {
1547 env->dr[i] = dbgregs.db[i];
1548 }
1549 env->dr[4] = env->dr[6] = dbgregs.dr6;
1550 env->dr[5] = env->dr[7] = dbgregs.dr7;
1551
1552 return 0;
1553 }
1554
1555 int kvm_arch_put_registers(CPUX86State *env, int level)
1556 {
1557 CPUState *cpu = ENV_GET_CPU(env);
1558 int ret;
1559
1560 assert(cpu_is_stopped(cpu) || qemu_cpu_is_self(cpu));
1561
1562 ret = kvm_getput_regs(env, 1);
1563 if (ret < 0) {
1564 return ret;
1565 }
1566 ret = kvm_put_xsave(env);
1567 if (ret < 0) {
1568 return ret;
1569 }
1570 ret = kvm_put_xcrs(env);
1571 if (ret < 0) {
1572 return ret;
1573 }
1574 ret = kvm_put_sregs(env);
1575 if (ret < 0) {
1576 return ret;
1577 }
1578 /* must be before kvm_put_msrs */
1579 ret = kvm_inject_mce_oldstyle(env);
1580 if (ret < 0) {
1581 return ret;
1582 }
1583 ret = kvm_put_msrs(env, level);
1584 if (ret < 0) {
1585 return ret;
1586 }
1587 if (level >= KVM_PUT_RESET_STATE) {
1588 ret = kvm_put_mp_state(env);
1589 if (ret < 0) {
1590 return ret;
1591 }
1592 ret = kvm_put_apic(env);
1593 if (ret < 0) {
1594 return ret;
1595 }
1596 }
1597 ret = kvm_put_vcpu_events(env, level);
1598 if (ret < 0) {
1599 return ret;
1600 }
1601 ret = kvm_put_debugregs(env);
1602 if (ret < 0) {
1603 return ret;
1604 }
1605 /* must be last */
1606 ret = kvm_guest_debug_workarounds(env);
1607 if (ret < 0) {
1608 return ret;
1609 }
1610 return 0;
1611 }
1612
1613 int kvm_arch_get_registers(CPUX86State *env)
1614 {
1615 CPUState *cpu = ENV_GET_CPU(env);
1616 int ret;
1617
1618 assert(cpu_is_stopped(cpu) || qemu_cpu_is_self(cpu));
1619
1620 ret = kvm_getput_regs(env, 0);
1621 if (ret < 0) {
1622 return ret;
1623 }
1624 ret = kvm_get_xsave(env);
1625 if (ret < 0) {
1626 return ret;
1627 }
1628 ret = kvm_get_xcrs(env);
1629 if (ret < 0) {
1630 return ret;
1631 }
1632 ret = kvm_get_sregs(env);
1633 if (ret < 0) {
1634 return ret;
1635 }
1636 ret = kvm_get_msrs(env);
1637 if (ret < 0) {
1638 return ret;
1639 }
1640 ret = kvm_get_mp_state(env);
1641 if (ret < 0) {
1642 return ret;
1643 }
1644 ret = kvm_get_apic(env);
1645 if (ret < 0) {
1646 return ret;
1647 }
1648 ret = kvm_get_vcpu_events(env);
1649 if (ret < 0) {
1650 return ret;
1651 }
1652 ret = kvm_get_debugregs(env);
1653 if (ret < 0) {
1654 return ret;
1655 }
1656 return 0;
1657 }
1658
1659 void kvm_arch_pre_run(CPUX86State *env, struct kvm_run *run)
1660 {
1661 int ret;
1662
1663 /* Inject NMI */
1664 if (env->interrupt_request & CPU_INTERRUPT_NMI) {
1665 env->interrupt_request &= ~CPU_INTERRUPT_NMI;
1666 DPRINTF("injected NMI\n");
1667 ret = kvm_vcpu_ioctl(env, KVM_NMI);
1668 if (ret < 0) {
1669 fprintf(stderr, "KVM: injection failed, NMI lost (%s)\n",
1670 strerror(-ret));
1671 }
1672 }
1673
1674 if (!kvm_irqchip_in_kernel()) {
1675 /* Force the VCPU out of its inner loop to process any INIT requests
1676 * or pending TPR access reports. */
1677 if (env->interrupt_request &
1678 (CPU_INTERRUPT_INIT | CPU_INTERRUPT_TPR)) {
1679 env->exit_request = 1;
1680 }
1681
1682 /* Try to inject an interrupt if the guest can accept it */
1683 if (run->ready_for_interrupt_injection &&
1684 (env->interrupt_request & CPU_INTERRUPT_HARD) &&
1685 (env->eflags & IF_MASK)) {
1686 int irq;
1687
1688 env->interrupt_request &= ~CPU_INTERRUPT_HARD;
1689 irq = cpu_get_pic_interrupt(env);
1690 if (irq >= 0) {
1691 struct kvm_interrupt intr;
1692
1693 intr.irq = irq;
1694 DPRINTF("injected interrupt %d\n", irq);
1695 ret = kvm_vcpu_ioctl(env, KVM_INTERRUPT, &intr);
1696 if (ret < 0) {
1697 fprintf(stderr,
1698 "KVM: injection failed, interrupt lost (%s)\n",
1699 strerror(-ret));
1700 }
1701 }
1702 }
1703
1704 /* If we have an interrupt but the guest is not ready to receive an
1705 * interrupt, request an interrupt window exit. This will
1706 * cause a return to userspace as soon as the guest is ready to
1707 * receive interrupts. */
1708 if ((env->interrupt_request & CPU_INTERRUPT_HARD)) {
1709 run->request_interrupt_window = 1;
1710 } else {
1711 run->request_interrupt_window = 0;
1712 }
1713
1714 DPRINTF("setting tpr\n");
1715 run->cr8 = cpu_get_apic_tpr(env->apic_state);
1716 }
1717 }
1718
1719 void kvm_arch_post_run(CPUX86State *env, struct kvm_run *run)
1720 {
1721 if (run->if_flag) {
1722 env->eflags |= IF_MASK;
1723 } else {
1724 env->eflags &= ~IF_MASK;
1725 }
1726 cpu_set_apic_tpr(env->apic_state, run->cr8);
1727 cpu_set_apic_base(env->apic_state, run->apic_base);
1728 }
1729
1730 int kvm_arch_process_async_events(CPUX86State *env)
1731 {
1732 X86CPU *cpu = x86_env_get_cpu(env);
1733
1734 if (env->interrupt_request & CPU_INTERRUPT_MCE) {
1735 /* We must not raise CPU_INTERRUPT_MCE if it's not supported. */
1736 assert(env->mcg_cap);
1737
1738 env->interrupt_request &= ~CPU_INTERRUPT_MCE;
1739
1740 kvm_cpu_synchronize_state(env);
1741
1742 if (env->exception_injected == EXCP08_DBLE) {
1743 /* this means triple fault */
1744 qemu_system_reset_request();
1745 env->exit_request = 1;
1746 return 0;
1747 }
1748 env->exception_injected = EXCP12_MCHK;
1749 env->has_error_code = 0;
1750
1751 env->halted = 0;
1752 if (kvm_irqchip_in_kernel() && env->mp_state == KVM_MP_STATE_HALTED) {
1753 env->mp_state = KVM_MP_STATE_RUNNABLE;
1754 }
1755 }
1756
1757 if (kvm_irqchip_in_kernel()) {
1758 return 0;
1759 }
1760
1761 if (env->interrupt_request & CPU_INTERRUPT_POLL) {
1762 env->interrupt_request &= ~CPU_INTERRUPT_POLL;
1763 apic_poll_irq(env->apic_state);
1764 }
1765 if (((env->interrupt_request & CPU_INTERRUPT_HARD) &&
1766 (env->eflags & IF_MASK)) ||
1767 (env->interrupt_request & CPU_INTERRUPT_NMI)) {
1768 env->halted = 0;
1769 }
1770 if (env->interrupt_request & CPU_INTERRUPT_INIT) {
1771 kvm_cpu_synchronize_state(env);
1772 do_cpu_init(cpu);
1773 }
1774 if (env->interrupt_request & CPU_INTERRUPT_SIPI) {
1775 kvm_cpu_synchronize_state(env);
1776 do_cpu_sipi(cpu);
1777 }
1778 if (env->interrupt_request & CPU_INTERRUPT_TPR) {
1779 env->interrupt_request &= ~CPU_INTERRUPT_TPR;
1780 kvm_cpu_synchronize_state(env);
1781 apic_handle_tpr_access_report(env->apic_state, env->eip,
1782 env->tpr_access_type);
1783 }
1784
1785 return env->halted;
1786 }
1787
1788 static int kvm_handle_halt(CPUX86State *env)
1789 {
1790 if (!((env->interrupt_request & CPU_INTERRUPT_HARD) &&
1791 (env->eflags & IF_MASK)) &&
1792 !(env->interrupt_request & CPU_INTERRUPT_NMI)) {
1793 env->halted = 1;
1794 return EXCP_HLT;
1795 }
1796
1797 return 0;
1798 }
1799
1800 static int kvm_handle_tpr_access(CPUX86State *env)
1801 {
1802 struct kvm_run *run = env->kvm_run;
1803
1804 apic_handle_tpr_access_report(env->apic_state, run->tpr_access.rip,
1805 run->tpr_access.is_write ? TPR_ACCESS_WRITE
1806 : TPR_ACCESS_READ);
1807 return 1;
1808 }
1809
1810 int kvm_arch_insert_sw_breakpoint(CPUX86State *env, struct kvm_sw_breakpoint *bp)
1811 {
1812 static const uint8_t int3 = 0xcc;
1813
1814 if (cpu_memory_rw_debug(env, bp->pc, (uint8_t *)&bp->saved_insn, 1, 0) ||
1815 cpu_memory_rw_debug(env, bp->pc, (uint8_t *)&int3, 1, 1)) {
1816 return -EINVAL;
1817 }
1818 return 0;
1819 }
1820
1821 int kvm_arch_remove_sw_breakpoint(CPUX86State *env, struct kvm_sw_breakpoint *bp)
1822 {
1823 uint8_t int3;
1824
1825 if (cpu_memory_rw_debug(env, bp->pc, &int3, 1, 0) || int3 != 0xcc ||
1826 cpu_memory_rw_debug(env, bp->pc, (uint8_t *)&bp->saved_insn, 1, 1)) {
1827 return -EINVAL;
1828 }
1829 return 0;
1830 }
1831
1832 static struct {
1833 target_ulong addr;
1834 int len;
1835 int type;
1836 } hw_breakpoint[4];
1837
1838 static int nb_hw_breakpoint;
1839
1840 static int find_hw_breakpoint(target_ulong addr, int len, int type)
1841 {
1842 int n;
1843
1844 for (n = 0; n < nb_hw_breakpoint; n++) {
1845 if (hw_breakpoint[n].addr == addr && hw_breakpoint[n].type == type &&
1846 (hw_breakpoint[n].len == len || len == -1)) {
1847 return n;
1848 }
1849 }
1850 return -1;
1851 }
1852
1853 int kvm_arch_insert_hw_breakpoint(target_ulong addr,
1854 target_ulong len, int type)
1855 {
1856 switch (type) {
1857 case GDB_BREAKPOINT_HW:
1858 len = 1;
1859 break;
1860 case GDB_WATCHPOINT_WRITE:
1861 case GDB_WATCHPOINT_ACCESS:
1862 switch (len) {
1863 case 1:
1864 break;
1865 case 2:
1866 case 4:
1867 case 8:
1868 if (addr & (len - 1)) {
1869 return -EINVAL;
1870 }
1871 break;
1872 default:
1873 return -EINVAL;
1874 }
1875 break;
1876 default:
1877 return -ENOSYS;
1878 }
1879
1880 if (nb_hw_breakpoint == 4) {
1881 return -ENOBUFS;
1882 }
1883 if (find_hw_breakpoint(addr, len, type) >= 0) {
1884 return -EEXIST;
1885 }
1886 hw_breakpoint[nb_hw_breakpoint].addr = addr;
1887 hw_breakpoint[nb_hw_breakpoint].len = len;
1888 hw_breakpoint[nb_hw_breakpoint].type = type;
1889 nb_hw_breakpoint++;
1890
1891 return 0;
1892 }
1893
1894 int kvm_arch_remove_hw_breakpoint(target_ulong addr,
1895 target_ulong len, int type)
1896 {
1897 int n;
1898
1899 n = find_hw_breakpoint(addr, (type == GDB_BREAKPOINT_HW) ? 1 : len, type);
1900 if (n < 0) {
1901 return -ENOENT;
1902 }
1903 nb_hw_breakpoint--;
1904 hw_breakpoint[n] = hw_breakpoint[nb_hw_breakpoint];
1905
1906 return 0;
1907 }
1908
1909 void kvm_arch_remove_all_hw_breakpoints(void)
1910 {
1911 nb_hw_breakpoint = 0;
1912 }
1913
1914 static CPUWatchpoint hw_watchpoint;
1915
1916 static int kvm_handle_debug(struct kvm_debug_exit_arch *arch_info)
1917 {
1918 int ret = 0;
1919 int n;
1920
1921 if (arch_info->exception == 1) {
1922 if (arch_info->dr6 & (1 << 14)) {
1923 if (cpu_single_env->singlestep_enabled) {
1924 ret = EXCP_DEBUG;
1925 }
1926 } else {
1927 for (n = 0; n < 4; n++) {
1928 if (arch_info->dr6 & (1 << n)) {
1929 switch ((arch_info->dr7 >> (16 + n*4)) & 0x3) {
1930 case 0x0:
1931 ret = EXCP_DEBUG;
1932 break;
1933 case 0x1:
1934 ret = EXCP_DEBUG;
1935 cpu_single_env->watchpoint_hit = &hw_watchpoint;
1936 hw_watchpoint.vaddr = hw_breakpoint[n].addr;
1937 hw_watchpoint.flags = BP_MEM_WRITE;
1938 break;
1939 case 0x3:
1940 ret = EXCP_DEBUG;
1941 cpu_single_env->watchpoint_hit = &hw_watchpoint;
1942 hw_watchpoint.vaddr = hw_breakpoint[n].addr;
1943 hw_watchpoint.flags = BP_MEM_ACCESS;
1944 break;
1945 }
1946 }
1947 }
1948 }
1949 } else if (kvm_find_sw_breakpoint(cpu_single_env, arch_info->pc)) {
1950 ret = EXCP_DEBUG;
1951 }
1952 if (ret == 0) {
1953 cpu_synchronize_state(cpu_single_env);
1954 assert(cpu_single_env->exception_injected == -1);
1955
1956 /* pass to guest */
1957 cpu_single_env->exception_injected = arch_info->exception;
1958 cpu_single_env->has_error_code = 0;
1959 }
1960
1961 return ret;
1962 }
1963
1964 void kvm_arch_update_guest_debug(CPUX86State *env, struct kvm_guest_debug *dbg)
1965 {
1966 const uint8_t type_code[] = {
1967 [GDB_BREAKPOINT_HW] = 0x0,
1968 [GDB_WATCHPOINT_WRITE] = 0x1,
1969 [GDB_WATCHPOINT_ACCESS] = 0x3
1970 };
1971 const uint8_t len_code[] = {
1972 [1] = 0x0, [2] = 0x1, [4] = 0x3, [8] = 0x2
1973 };
1974 int n;
1975
1976 if (kvm_sw_breakpoints_active(env)) {
1977 dbg->control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP;
1978 }
1979 if (nb_hw_breakpoint > 0) {
1980 dbg->control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_HW_BP;
1981 dbg->arch.debugreg[7] = 0x0600;
1982 for (n = 0; n < nb_hw_breakpoint; n++) {
1983 dbg->arch.debugreg[n] = hw_breakpoint[n].addr;
1984 dbg->arch.debugreg[7] |= (2 << (n * 2)) |
1985 (type_code[hw_breakpoint[n].type] << (16 + n*4)) |
1986 ((uint32_t)len_code[hw_breakpoint[n].len] << (18 + n*4));
1987 }
1988 }
1989 }
1990
1991 static bool host_supports_vmx(void)
1992 {
1993 uint32_t ecx, unused;
1994
1995 host_cpuid(1, 0, &unused, &unused, &ecx, &unused);
1996 return ecx & CPUID_EXT_VMX;
1997 }
1998
1999 #define VMX_INVALID_GUEST_STATE 0x80000021
2000
2001 int kvm_arch_handle_exit(CPUX86State *env, struct kvm_run *run)
2002 {
2003 uint64_t code;
2004 int ret;
2005
2006 switch (run->exit_reason) {
2007 case KVM_EXIT_HLT:
2008 DPRINTF("handle_hlt\n");
2009 ret = kvm_handle_halt(env);
2010 break;
2011 case KVM_EXIT_SET_TPR:
2012 ret = 0;
2013 break;
2014 case KVM_EXIT_TPR_ACCESS:
2015 ret = kvm_handle_tpr_access(env);
2016 break;
2017 case KVM_EXIT_FAIL_ENTRY:
2018 code = run->fail_entry.hardware_entry_failure_reason;
2019 fprintf(stderr, "KVM: entry failed, hardware error 0x%" PRIx64 "\n",
2020 code);
2021 if (host_supports_vmx() && code == VMX_INVALID_GUEST_STATE) {
2022 fprintf(stderr,
2023 "\nIf you're running a guest on an Intel machine without "
2024 "unrestricted mode\n"
2025 "support, the failure can be most likely due to the guest "
2026 "entering an invalid\n"
2027 "state for Intel VT. For example, the guest maybe running "
2028 "in big real mode\n"
2029 "which is not supported on less recent Intel processors."
2030 "\n\n");
2031 }
2032 ret = -1;
2033 break;
2034 case KVM_EXIT_EXCEPTION:
2035 fprintf(stderr, "KVM: exception %d exit (error code 0x%x)\n",
2036 run->ex.exception, run->ex.error_code);
2037 ret = -1;
2038 break;
2039 case KVM_EXIT_DEBUG:
2040 DPRINTF("kvm_exit_debug\n");
2041 ret = kvm_handle_debug(&run->debug.arch);
2042 break;
2043 default:
2044 fprintf(stderr, "KVM: unknown exit reason %d\n", run->exit_reason);
2045 ret = -1;
2046 break;
2047 }
2048
2049 return ret;
2050 }
2051
2052 bool kvm_arch_stop_on_emulation_error(CPUX86State *env)
2053 {
2054 kvm_cpu_synchronize_state(env);
2055 return !(env->cr[0] & CR0_PE_MASK) ||
2056 ((env->segs[R_CS].selector & 3) != 3);
2057 }
2058
2059 void kvm_arch_init_irq_routing(KVMState *s)
2060 {
2061 if (!kvm_check_extension(s, KVM_CAP_IRQ_ROUTING)) {
2062 /* If kernel can't do irq routing, interrupt source
2063 * override 0->2 cannot be set up as required by HPET.
2064 * So we have to disable it.
2065 */
2066 no_hpet = 1;
2067 }
2068 /* We know at this point that we're using the in-kernel
2069 * irqchip, so we can use irqfds, and on x86 we know
2070 * we can use msi via irqfd and GSI routing.
2071 */
2072 kvm_irqfds_allowed = true;
2073 kvm_msi_via_irqfd_allowed = true;
2074 kvm_gsi_routing_allowed = true;
2075 }
2076
2077 /* Classic KVM device assignment interface. Will remain x86 only. */
2078 int kvm_device_pci_assign(KVMState *s, PCIHostDeviceAddress *dev_addr,
2079 uint32_t flags, uint32_t *dev_id)
2080 {
2081 struct kvm_assigned_pci_dev dev_data = {
2082 .segnr = dev_addr->domain,
2083 .busnr = dev_addr->bus,
2084 .devfn = PCI_DEVFN(dev_addr->slot, dev_addr->function),
2085 .flags = flags,
2086 };
2087 int ret;
2088
2089 dev_data.assigned_dev_id =
2090 (dev_addr->domain << 16) | (dev_addr->bus << 8) | dev_data.devfn;
2091
2092 ret = kvm_vm_ioctl(s, KVM_ASSIGN_PCI_DEVICE, &dev_data);
2093 if (ret < 0) {
2094 return ret;
2095 }
2096
2097 *dev_id = dev_data.assigned_dev_id;
2098
2099 return 0;
2100 }
2101
2102 int kvm_device_pci_deassign(KVMState *s, uint32_t dev_id)
2103 {
2104 struct kvm_assigned_pci_dev dev_data = {
2105 .assigned_dev_id = dev_id,
2106 };
2107
2108 return kvm_vm_ioctl(s, KVM_DEASSIGN_PCI_DEVICE, &dev_data);
2109 }
2110
2111 static int kvm_assign_irq_internal(KVMState *s, uint32_t dev_id,
2112 uint32_t irq_type, uint32_t guest_irq)
2113 {
2114 struct kvm_assigned_irq assigned_irq = {
2115 .assigned_dev_id = dev_id,
2116 .guest_irq = guest_irq,
2117 .flags = irq_type,
2118 };
2119
2120 if (kvm_check_extension(s, KVM_CAP_ASSIGN_DEV_IRQ)) {
2121 return kvm_vm_ioctl(s, KVM_ASSIGN_DEV_IRQ, &assigned_irq);
2122 } else {
2123 return kvm_vm_ioctl(s, KVM_ASSIGN_IRQ, &assigned_irq);
2124 }
2125 }
2126
2127 int kvm_device_intx_assign(KVMState *s, uint32_t dev_id, bool use_host_msi,
2128 uint32_t guest_irq)
2129 {
2130 uint32_t irq_type = KVM_DEV_IRQ_GUEST_INTX |
2131 (use_host_msi ? KVM_DEV_IRQ_HOST_MSI : KVM_DEV_IRQ_HOST_INTX);
2132
2133 return kvm_assign_irq_internal(s, dev_id, irq_type, guest_irq);
2134 }
2135
2136 int kvm_device_intx_set_mask(KVMState *s, uint32_t dev_id, bool masked)
2137 {
2138 struct kvm_assigned_pci_dev dev_data = {
2139 .assigned_dev_id = dev_id,
2140 .flags = masked ? KVM_DEV_ASSIGN_MASK_INTX : 0,
2141 };
2142
2143 return kvm_vm_ioctl(s, KVM_ASSIGN_SET_INTX_MASK, &dev_data);
2144 }
2145
2146 static int kvm_deassign_irq_internal(KVMState *s, uint32_t dev_id,
2147 uint32_t type)
2148 {
2149 struct kvm_assigned_irq assigned_irq = {
2150 .assigned_dev_id = dev_id,
2151 .flags = type,
2152 };
2153
2154 return kvm_vm_ioctl(s, KVM_DEASSIGN_DEV_IRQ, &assigned_irq);
2155 }
2156
2157 int kvm_device_intx_deassign(KVMState *s, uint32_t dev_id, bool use_host_msi)
2158 {
2159 return kvm_deassign_irq_internal(s, dev_id, KVM_DEV_IRQ_GUEST_INTX |
2160 (use_host_msi ? KVM_DEV_IRQ_HOST_MSI : KVM_DEV_IRQ_HOST_INTX));
2161 }
2162
2163 int kvm_device_msi_assign(KVMState *s, uint32_t dev_id, int virq)
2164 {
2165 return kvm_assign_irq_internal(s, dev_id, KVM_DEV_IRQ_HOST_MSI |
2166 KVM_DEV_IRQ_GUEST_MSI, virq);
2167 }
2168
2169 int kvm_device_msi_deassign(KVMState *s, uint32_t dev_id)
2170 {
2171 return kvm_deassign_irq_internal(s, dev_id, KVM_DEV_IRQ_GUEST_MSI |
2172 KVM_DEV_IRQ_HOST_MSI);
2173 }
2174
2175 bool kvm_device_msix_supported(KVMState *s)
2176 {
2177 /* The kernel lacks a corresponding KVM_CAP, so we probe by calling
2178 * KVM_ASSIGN_SET_MSIX_NR with an invalid parameter. */
2179 return kvm_vm_ioctl(s, KVM_ASSIGN_SET_MSIX_NR, NULL) == -EFAULT;
2180 }
2181
2182 int kvm_device_msix_init_vectors(KVMState *s, uint32_t dev_id,
2183 uint32_t nr_vectors)
2184 {
2185 struct kvm_assigned_msix_nr msix_nr = {
2186 .assigned_dev_id = dev_id,
2187 .entry_nr = nr_vectors,
2188 };
2189
2190 return kvm_vm_ioctl(s, KVM_ASSIGN_SET_MSIX_NR, &msix_nr);
2191 }
2192
2193 int kvm_device_msix_set_vector(KVMState *s, uint32_t dev_id, uint32_t vector,
2194 int virq)
2195 {
2196 struct kvm_assigned_msix_entry msix_entry = {
2197 .assigned_dev_id = dev_id,
2198 .gsi = virq,
2199 .entry = vector,
2200 };
2201
2202 return kvm_vm_ioctl(s, KVM_ASSIGN_SET_MSIX_ENTRY, &msix_entry);
2203 }
2204
2205 int kvm_device_msix_assign(KVMState *s, uint32_t dev_id)
2206 {
2207 return kvm_assign_irq_internal(s, dev_id, KVM_DEV_IRQ_HOST_MSIX |
2208 KVM_DEV_IRQ_GUEST_MSIX, 0);
2209 }
2210
2211 int kvm_device_msix_deassign(KVMState *s, uint32_t dev_id)
2212 {
2213 return kvm_deassign_irq_internal(s, dev_id, KVM_DEV_IRQ_GUEST_MSIX |
2214 KVM_DEV_IRQ_HOST_MSIX);
2215 }
Qemu copy for testing