]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blob - arch/x86/kvm/svm.c
projects / mirror_ubuntu-artful-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Revert "x86/svm: Add code to clear registers on VM exit"
[mirror_ubuntu-artful-kernel.git] / arch / x86 / kvm / svm.c
1 /*
2 * Kernel-based Virtual Machine driver for Linux
3 *
4 * AMD SVM support
5 *
6 * Copyright (C) 2006 Qumranet, Inc.
7 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
8 *
9 * Authors:
10 * Yaniv Kamay <yaniv@qumranet.com>
11 * Avi Kivity <avi@qumranet.com>
12 *
13 * This work is licensed under the terms of the GNU GPL, version 2. See
14 * the COPYING file in the top-level directory.
15 *
16 */
17
18 #define pr_fmt(fmt) "SVM: " fmt
19
20 #include <linux/kvm_host.h>
21
22 #include "irq.h"
23 #include "mmu.h"
24 #include "kvm_cache_regs.h"
25 #include "x86.h"
26 #include "cpuid.h"
27 #include "pmu.h"
28
29 #include <linux/module.h>
30 #include <linux/mod_devicetable.h>
31 #include <linux/kernel.h>
32 #include <linux/vmalloc.h>
33 #include <linux/highmem.h>
34 #include <linux/sched.h>
35 #include <linux/trace_events.h>
36 #include <linux/slab.h>
37 #include <linux/amd-iommu.h>
38 #include <linux/hashtable.h>
39 #include <linux/frame.h>
40
41 #include <asm/apic.h>
42 #include <asm/perf_event.h>
43 #include <asm/tlbflush.h>
44 #include <asm/desc.h>
45 #include <asm/debugreg.h>
46 #include <asm/kvm_para.h>
47 #include <asm/irq_remapping.h>
48
49 #include <asm/virtext.h>
50 #include "trace.h"
51
52 #define __ex(x) __kvm_handle_fault_on_reboot(x)
53
54 MODULE_AUTHOR("Qumranet");
55 MODULE_LICENSE("GPL");
56
57 static const struct x86_cpu_id svm_cpu_id[] = {
58 X86_FEATURE_MATCH(X86_FEATURE_SVM),
59 {}
60 };
61 MODULE_DEVICE_TABLE(x86cpu, svm_cpu_id);
62
63 #define IOPM_ALLOC_ORDER 2
64 #define MSRPM_ALLOC_ORDER 1
65
66 #define SEG_TYPE_LDT 2
67 #define SEG_TYPE_BUSY_TSS16 3
68
69 #define SVM_FEATURE_NPT (1 << 0)
70 #define SVM_FEATURE_LBRV (1 << 1)
71 #define SVM_FEATURE_SVML (1 << 2)
72 #define SVM_FEATURE_NRIP (1 << 3)
73 #define SVM_FEATURE_TSC_RATE (1 << 4)
74 #define SVM_FEATURE_VMCB_CLEAN (1 << 5)
75 #define SVM_FEATURE_FLUSH_ASID (1 << 6)
76 #define SVM_FEATURE_DECODE_ASSIST (1 << 7)
77 #define SVM_FEATURE_PAUSE_FILTER (1 << 10)
78
79 #define SVM_AVIC_DOORBELL 0xc001011b
80
81 #define NESTED_EXIT_HOST 0 /* Exit handled on host level */
82 #define NESTED_EXIT_DONE 1 /* Exit caused nested vmexit */
83 #define NESTED_EXIT_CONTINUE 2 /* Further checks needed */
84
85 #define DEBUGCTL_RESERVED_BITS (~(0x3fULL))
86
87 #define TSC_RATIO_RSVD 0xffffff0000000000ULL
88 #define TSC_RATIO_MIN 0x0000000000000001ULL
89 #define TSC_RATIO_MAX 0x000000ffffffffffULL
90
91 #define AVIC_HPA_MASK ~((0xFFFULL << 52) | 0xFFF)
92
93 /*
94 * 0xff is broadcast, so the max index allowed for physical APIC ID
95 * table is 0xfe. APIC IDs above 0xff are reserved.
96 */
97 #define AVIC_MAX_PHYSICAL_ID_COUNT 255
98
99 #define AVIC_UNACCEL_ACCESS_WRITE_MASK 1
100 #define AVIC_UNACCEL_ACCESS_OFFSET_MASK 0xFF0
101 #define AVIC_UNACCEL_ACCESS_VECTOR_MASK 0xFFFFFFFF
102
103 /* AVIC GATAG is encoded using VM and VCPU IDs */
104 #define AVIC_VCPU_ID_BITS 8
105 #define AVIC_VCPU_ID_MASK ((1 << AVIC_VCPU_ID_BITS) - 1)
106
107 #define AVIC_VM_ID_BITS 24
108 #define AVIC_VM_ID_NR (1 << AVIC_VM_ID_BITS)
109 #define AVIC_VM_ID_MASK ((1 << AVIC_VM_ID_BITS) - 1)
110
111 #define AVIC_GATAG(x, y) (((x & AVIC_VM_ID_MASK) << AVIC_VCPU_ID_BITS) | \
112 (y & AVIC_VCPU_ID_MASK))
113 #define AVIC_GATAG_TO_VMID(x) ((x >> AVIC_VCPU_ID_BITS) & AVIC_VM_ID_MASK)
114 #define AVIC_GATAG_TO_VCPUID(x) (x & AVIC_VCPU_ID_MASK)
115
116 static bool erratum_383_found __read_mostly;
117
118 static const u32 host_save_user_msrs[] = {
119 #ifdef CONFIG_X86_64
120 MSR_STAR, MSR_LSTAR, MSR_CSTAR, MSR_SYSCALL_MASK, MSR_KERNEL_GS_BASE,
121 MSR_FS_BASE,
122 #endif
123 MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
124 MSR_TSC_AUX,
125 };
126
127 #define NR_HOST_SAVE_USER_MSRS ARRAY_SIZE(host_save_user_msrs)
128
129 struct kvm_vcpu;
130
131 struct nested_state {
132 struct vmcb *hsave;
133 u64 hsave_msr;
134 u64 vm_cr_msr;
135 u64 vmcb;
136
137 /* These are the merged vectors */
138 u32 *msrpm;
139
140 /* gpa pointers to the real vectors */
141 u64 vmcb_msrpm;
142 u64 vmcb_iopm;
143
144 /* A VMEXIT is required but not yet emulated */
145 bool exit_required;
146
147 /* cache for intercepts of the guest */
148 u32 intercept_cr;
149 u32 intercept_dr;
150 u32 intercept_exceptions;
151 u64 intercept;
152
153 /* Nested Paging related state */
154 u64 nested_cr3;
155 };
156
157 #define MSRPM_OFFSETS 16
158 static u32 msrpm_offsets[MSRPM_OFFSETS] __read_mostly;
159
160 /*
161 * Set osvw_len to higher value when updated Revision Guides
162 * are published and we know what the new status bits are
163 */
164 static uint64_t osvw_len = 4, osvw_status;
165
166 struct vcpu_svm {
167 struct kvm_vcpu vcpu;
168 struct vmcb *vmcb;
169 unsigned long vmcb_pa;
170 struct svm_cpu_data *svm_data;
171 uint64_t asid_generation;
172 uint64_t sysenter_esp;
173 uint64_t sysenter_eip;
174 uint64_t tsc_aux;
175
176 u64 next_rip;
177
178 u64 spec_ctrl;
179
180 u64 host_user_msrs[NR_HOST_SAVE_USER_MSRS];
181 struct {
182 u16 fs;
183 u16 gs;
184 u16 ldt;
185 u64 gs_base;
186 } host;
187
188 u32 *msrpm;
189
190 ulong nmi_iret_rip;
191
192 struct nested_state nested;
193
194 bool nmi_singlestep;
195 u64 nmi_singlestep_guest_rflags;
196
197 unsigned int3_injected;
198 unsigned long int3_rip;
199
200 /* cached guest cpuid flags for faster access */
201 bool nrips_enabled : 1;
202
203 u32 ldr_reg;
204 struct page *avic_backing_page;
205 u64 *avic_physical_id_cache;
206 bool avic_is_running;
207
208 /*
209 * Per-vcpu list of struct amd_svm_iommu_ir:
210 * This is used mainly to store interrupt remapping information used
211 * when update the vcpu affinity. This avoids the need to scan for
212 * IRTE and try to match ga_tag in the IOMMU driver.
213 */
214 struct list_head ir_list;
215 spinlock_t ir_list_lock;
216 };
217
218 /*
219 * This is a wrapper of struct amd_iommu_ir_data.
220 */
221 struct amd_svm_iommu_ir {
222 struct list_head node; /* Used by SVM for per-vcpu ir_list */
223 void *data; /* Storing pointer to struct amd_ir_data */
224 };
225
226 #define AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK (0xFF)
227 #define AVIC_LOGICAL_ID_ENTRY_VALID_MASK (1 << 31)
228
229 #define AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK (0xFFULL)
230 #define AVIC_PHYSICAL_ID_ENTRY_BACKING_PAGE_MASK (0xFFFFFFFFFFULL << 12)
231 #define AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK (1ULL << 62)
232 #define AVIC_PHYSICAL_ID_ENTRY_VALID_MASK (1ULL << 63)
233
234 static DEFINE_PER_CPU(u64, current_tsc_ratio);
235 #define TSC_RATIO_DEFAULT 0x0100000000ULL
236
237 #define MSR_INVALID 0xffffffffU
238
239 static const struct svm_direct_access_msrs {
240 u32 index; /* Index of the MSR */
241 bool always; /* True if intercept is always on */
242 } direct_access_msrs[] = {
243 { .index = MSR_STAR, .always = true },
244 { .index = MSR_IA32_SYSENTER_CS, .always = true },
245 #ifdef CONFIG_X86_64
246 { .index = MSR_GS_BASE, .always = true },
247 { .index = MSR_FS_BASE, .always = true },
248 { .index = MSR_KERNEL_GS_BASE, .always = true },
249 { .index = MSR_LSTAR, .always = true },
250 { .index = MSR_CSTAR, .always = true },
251 { .index = MSR_SYSCALL_MASK, .always = true },
252 #endif
253 { .index = MSR_IA32_SPEC_CTRL, .always = true },
254 { .index = MSR_IA32_PRED_CMD, .always = true },
255 { .index = MSR_IA32_LASTBRANCHFROMIP, .always = false },
256 { .index = MSR_IA32_LASTBRANCHTOIP, .always = false },
257 { .index = MSR_IA32_LASTINTFROMIP, .always = false },
258 { .index = MSR_IA32_LASTINTTOIP, .always = false },
259 { .index = MSR_INVALID, .always = false },
260 };
261
262 /* enable NPT for AMD64 and X86 with PAE */
263 #if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
264 static bool npt_enabled = true;
265 #else
266 static bool npt_enabled;
267 #endif
268
269 /* allow nested paging (virtualized MMU) for all guests */
270 static int npt = true;
271 module_param(npt, int, S_IRUGO);
272
273 /* allow nested virtualization in KVM/SVM */
274 static int nested = true;
275 module_param(nested, int, S_IRUGO);
276
277 /* enable / disable AVIC */
278 static int avic;
279 #ifdef CONFIG_X86_LOCAL_APIC
280 module_param(avic, int, S_IRUGO);
281 #endif
282
283 /* enable/disable Virtual VMLOAD VMSAVE */
284 static int vls = true;
285 module_param(vls, int, 0444);
286
287 /* AVIC VM ID bit masks and lock */
288 static DECLARE_BITMAP(avic_vm_id_bitmap, AVIC_VM_ID_NR);
289 static DEFINE_SPINLOCK(avic_vm_id_lock);
290
291 static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0);
292 static void svm_flush_tlb(struct kvm_vcpu *vcpu);
293 static void svm_complete_interrupts(struct vcpu_svm *svm);
294
295 static int nested_svm_exit_handled(struct vcpu_svm *svm);
296 static int nested_svm_intercept(struct vcpu_svm *svm);
297 static int nested_svm_vmexit(struct vcpu_svm *svm);
298 static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr,
299 bool has_error_code, u32 error_code);
300
301 enum {
302 VMCB_INTERCEPTS, /* Intercept vectors, TSC offset,
303 pause filter count */
304 VMCB_PERM_MAP, /* IOPM Base and MSRPM Base */
305 VMCB_ASID, /* ASID */
306 VMCB_INTR, /* int_ctl, int_vector */
307 VMCB_NPT, /* npt_en, nCR3, gPAT */
308 VMCB_CR, /* CR0, CR3, CR4, EFER */
309 VMCB_DR, /* DR6, DR7 */
310 VMCB_DT, /* GDT, IDT */
311 VMCB_SEG, /* CS, DS, SS, ES, CPL */
312 VMCB_CR2, /* CR2 only */
313 VMCB_LBR, /* DBGCTL, BR_FROM, BR_TO, LAST_EX_FROM, LAST_EX_TO */
314 VMCB_AVIC, /* AVIC APIC_BAR, AVIC APIC_BACKING_PAGE,
315 * AVIC PHYSICAL_TABLE pointer,
316 * AVIC LOGICAL_TABLE pointer
317 */
318 VMCB_DIRTY_MAX,
319 };
320
321 /* TPR and CR2 are always written before VMRUN */
322 #define VMCB_ALWAYS_DIRTY_MASK ((1U << VMCB_INTR) | (1U << VMCB_CR2))
323
324 #define VMCB_AVIC_APIC_BAR_MASK 0xFFFFFFFFFF000ULL
325
326 static inline void mark_all_dirty(struct vmcb *vmcb)
327 {
328 vmcb->control.clean = 0;
329 }
330
331 static inline void mark_all_clean(struct vmcb *vmcb)
332 {
333 vmcb->control.clean = ((1 << VMCB_DIRTY_MAX) - 1)
334 & ~VMCB_ALWAYS_DIRTY_MASK;
335 }
336
337 static inline void mark_dirty(struct vmcb *vmcb, int bit)
338 {
339 vmcb->control.clean &= ~(1 << bit);
340 }
341
342 static inline struct vcpu_svm *to_svm(struct kvm_vcpu *vcpu)
343 {
344 return container_of(vcpu, struct vcpu_svm, vcpu);
345 }
346
347 static inline void avic_update_vapic_bar(struct vcpu_svm *svm, u64 data)
348 {
349 svm->vmcb->control.avic_vapic_bar = data & VMCB_AVIC_APIC_BAR_MASK;
350 mark_dirty(svm->vmcb, VMCB_AVIC);
351 }
352
353 static inline bool avic_vcpu_is_running(struct kvm_vcpu *vcpu)
354 {
355 struct vcpu_svm *svm = to_svm(vcpu);
356 u64 *entry = svm->avic_physical_id_cache;
357
358 if (!entry)
359 return false;
360
361 return (READ_ONCE(*entry) & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK);
362 }
363
364 static void recalc_intercepts(struct vcpu_svm *svm)
365 {
366 struct vmcb_control_area *c, *h;
367 struct nested_state *g;
368
369 mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
370
371 if (!is_guest_mode(&svm->vcpu))
372 return;
373
374 c = &svm->vmcb->control;
375 h = &svm->nested.hsave->control;
376 g = &svm->nested;
377
378 c->intercept_cr = h->intercept_cr | g->intercept_cr;
379 c->intercept_dr = h->intercept_dr | g->intercept_dr;
380 c->intercept_exceptions = h->intercept_exceptions | g->intercept_exceptions;
381 c->intercept = h->intercept | g->intercept;
382 }
383
384 static inline struct vmcb *get_host_vmcb(struct vcpu_svm *svm)
385 {
386 if (is_guest_mode(&svm->vcpu))
387 return svm->nested.hsave;
388 else
389 return svm->vmcb;
390 }
391
392 static inline void set_cr_intercept(struct vcpu_svm *svm, int bit)
393 {
394 struct vmcb *vmcb = get_host_vmcb(svm);
395
396 vmcb->control.intercept_cr |= (1U << bit);
397
398 recalc_intercepts(svm);
399 }
400
401 static inline void clr_cr_intercept(struct vcpu_svm *svm, int bit)
402 {
403 struct vmcb *vmcb = get_host_vmcb(svm);
404
405 vmcb->control.intercept_cr &= ~(1U << bit);
406
407 recalc_intercepts(svm);
408 }
409
410 static inline bool is_cr_intercept(struct vcpu_svm *svm, int bit)
411 {
412 struct vmcb *vmcb = get_host_vmcb(svm);
413
414 return vmcb->control.intercept_cr & (1U << bit);
415 }
416
417 static inline void set_dr_intercepts(struct vcpu_svm *svm)
418 {
419 struct vmcb *vmcb = get_host_vmcb(svm);
420
421 vmcb->control.intercept_dr = (1 << INTERCEPT_DR0_READ)
422 | (1 << INTERCEPT_DR1_READ)
423 | (1 << INTERCEPT_DR2_READ)
424 | (1 << INTERCEPT_DR3_READ)
425 | (1 << INTERCEPT_DR4_READ)
426 | (1 << INTERCEPT_DR5_READ)
427 | (1 << INTERCEPT_DR6_READ)
428 | (1 << INTERCEPT_DR7_READ)
429 | (1 << INTERCEPT_DR0_WRITE)
430 | (1 << INTERCEPT_DR1_WRITE)
431 | (1 << INTERCEPT_DR2_WRITE)
432 | (1 << INTERCEPT_DR3_WRITE)
433 | (1 << INTERCEPT_DR4_WRITE)
434 | (1 << INTERCEPT_DR5_WRITE)
435 | (1 << INTERCEPT_DR6_WRITE)
436 | (1 << INTERCEPT_DR7_WRITE);
437
438 recalc_intercepts(svm);
439 }
440
441 static inline void clr_dr_intercepts(struct vcpu_svm *svm)
442 {
443 struct vmcb *vmcb = get_host_vmcb(svm);
444
445 vmcb->control.intercept_dr = 0;
446
447 recalc_intercepts(svm);
448 }
449
450 static inline void set_exception_intercept(struct vcpu_svm *svm, int bit)
451 {
452 struct vmcb *vmcb = get_host_vmcb(svm);
453
454 vmcb->control.intercept_exceptions |= (1U << bit);
455
456 recalc_intercepts(svm);
457 }
458
459 static inline void clr_exception_intercept(struct vcpu_svm *svm, int bit)
460 {
461 struct vmcb *vmcb = get_host_vmcb(svm);
462
463 vmcb->control.intercept_exceptions &= ~(1U << bit);
464
465 recalc_intercepts(svm);
466 }
467
468 static inline void set_intercept(struct vcpu_svm *svm, int bit)
469 {
470 struct vmcb *vmcb = get_host_vmcb(svm);
471
472 vmcb->control.intercept |= (1ULL << bit);
473
474 recalc_intercepts(svm);
475 }
476
477 static inline void clr_intercept(struct vcpu_svm *svm, int bit)
478 {
479 struct vmcb *vmcb = get_host_vmcb(svm);
480
481 vmcb->control.intercept &= ~(1ULL << bit);
482
483 recalc_intercepts(svm);
484 }
485
486 static inline void enable_gif(struct vcpu_svm *svm)
487 {
488 svm->vcpu.arch.hflags |= HF_GIF_MASK;
489 }
490
491 static inline void disable_gif(struct vcpu_svm *svm)
492 {
493 svm->vcpu.arch.hflags &= ~HF_GIF_MASK;
494 }
495
496 static inline bool gif_set(struct vcpu_svm *svm)
497 {
498 return !!(svm->vcpu.arch.hflags & HF_GIF_MASK);
499 }
500
501 static unsigned long iopm_base;
502
503 struct kvm_ldttss_desc {
504 u16 limit0;
505 u16 base0;
506 unsigned base1:8, type:5, dpl:2, p:1;
507 unsigned limit1:4, zero0:3, g:1, base2:8;
508 u32 base3;
509 u32 zero1;
510 } __attribute__((packed));
511
512 struct svm_cpu_data {
513 int cpu;
514
515 u64 asid_generation;
516 u32 max_asid;
517 u32 next_asid;
518 struct kvm_ldttss_desc *tss_desc;
519
520 struct page *save_area;
521
522 struct vmcb *current_vmcb;
523 };
524
525 static DEFINE_PER_CPU(struct svm_cpu_data *, svm_data);
526
527 struct svm_init_data {
528 int cpu;
529 int r;
530 };
531
532 static const u32 msrpm_ranges[] = {0, 0xc0000000, 0xc0010000};
533
534 #define NUM_MSR_MAPS ARRAY_SIZE(msrpm_ranges)
535 #define MSRS_RANGE_SIZE 2048
536 #define MSRS_IN_RANGE (MSRS_RANGE_SIZE * 8 / 2)
537
538 static u32 svm_msrpm_offset(u32 msr)
539 {
540 u32 offset;
541 int i;
542
543 for (i = 0; i < NUM_MSR_MAPS; i++) {
544 if (msr < msrpm_ranges[i] ||
545 msr >= msrpm_ranges[i] + MSRS_IN_RANGE)
546 continue;
547
548 offset = (msr - msrpm_ranges[i]) / 4; /* 4 msrs per u8 */
549 offset += (i * MSRS_RANGE_SIZE); /* add range offset */
550
551 /* Now we have the u8 offset - but need the u32 offset */
552 return offset / 4;
553 }
554
555 /* MSR not in any range */
556 return MSR_INVALID;
557 }
558
559 #define MAX_INST_SIZE 15
560
561 static inline void clgi(void)
562 {
563 asm volatile (__ex(SVM_CLGI));
564 }
565
566 static inline void stgi(void)
567 {
568 asm volatile (__ex(SVM_STGI));
569 }
570
571 static inline void invlpga(unsigned long addr, u32 asid)
572 {
573 asm volatile (__ex(SVM_INVLPGA) : : "a"(addr), "c"(asid));
574 }
575
576 static int get_npt_level(void)
577 {
578 #ifdef CONFIG_X86_64
579 return PT64_ROOT_LEVEL;
580 #else
581 return PT32E_ROOT_LEVEL;
582 #endif
583 }
584
585 static void svm_set_efer(struct kvm_vcpu *vcpu, u64 efer)
586 {
587 vcpu->arch.efer = efer;
588 if (!npt_enabled && !(efer & EFER_LMA))
589 efer &= ~EFER_LME;
590
591 to_svm(vcpu)->vmcb->save.efer = efer | EFER_SVME;
592 mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR);
593 }
594
595 static int is_external_interrupt(u32 info)
596 {
597 info &= SVM_EVTINJ_TYPE_MASK | SVM_EVTINJ_VALID;
598 return info == (SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR);
599 }
600
601 static u32 svm_get_interrupt_shadow(struct kvm_vcpu *vcpu)
602 {
603 struct vcpu_svm *svm = to_svm(vcpu);
604 u32 ret = 0;
605
606 if (svm->vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK)
607 ret = KVM_X86_SHADOW_INT_STI | KVM_X86_SHADOW_INT_MOV_SS;
608 return ret;
609 }
610
611 static void svm_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
612 {
613 struct vcpu_svm *svm = to_svm(vcpu);
614
615 if (mask == 0)
616 svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK;
617 else
618 svm->vmcb->control.int_state |= SVM_INTERRUPT_SHADOW_MASK;
619
620 }
621
622 static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
623 {
624 struct vcpu_svm *svm = to_svm(vcpu);
625
626 if (svm->vmcb->control.next_rip != 0) {
627 WARN_ON_ONCE(!static_cpu_has(X86_FEATURE_NRIPS));
628 svm->next_rip = svm->vmcb->control.next_rip;
629 }
630
631 if (!svm->next_rip) {
632 if (emulate_instruction(vcpu, EMULTYPE_SKIP) !=
633 EMULATE_DONE)
634 printk(KERN_DEBUG "%s: NOP\n", __func__);
635 return;
636 }
637 if (svm->next_rip - kvm_rip_read(vcpu) > MAX_INST_SIZE)
638 printk(KERN_ERR "%s: ip 0x%lx next 0x%llx\n",
639 __func__, kvm_rip_read(vcpu), svm->next_rip);
640
641 kvm_rip_write(vcpu, svm->next_rip);
642 svm_set_interrupt_shadow(vcpu, 0);
643 }
644
645 static void svm_queue_exception(struct kvm_vcpu *vcpu)
646 {
647 struct vcpu_svm *svm = to_svm(vcpu);
648 unsigned nr = vcpu->arch.exception.nr;
649 bool has_error_code = vcpu->arch.exception.has_error_code;
650 bool reinject = vcpu->arch.exception.reinject;
651 u32 error_code = vcpu->arch.exception.error_code;
652
653 /*
654 * If we are within a nested VM we'd better #VMEXIT and let the guest
655 * handle the exception
656 */
657 if (!reinject &&
658 nested_svm_check_exception(svm, nr, has_error_code, error_code))
659 return;
660
661 if (nr == BP_VECTOR && !static_cpu_has(X86_FEATURE_NRIPS)) {
662 unsigned long rip, old_rip = kvm_rip_read(&svm->vcpu);
663
664 /*
665 * For guest debugging where we have to reinject #BP if some
666 * INT3 is guest-owned:
667 * Emulate nRIP by moving RIP forward. Will fail if injection
668 * raises a fault that is not intercepted. Still better than
669 * failing in all cases.
670 */
671 skip_emulated_instruction(&svm->vcpu);
672 rip = kvm_rip_read(&svm->vcpu);
673 svm->int3_rip = rip + svm->vmcb->save.cs.base;
674 svm->int3_injected = rip - old_rip;
675 }
676
677 svm->vmcb->control.event_inj = nr
678 | SVM_EVTINJ_VALID
679 | (has_error_code ? SVM_EVTINJ_VALID_ERR : 0)
680 | SVM_EVTINJ_TYPE_EXEPT;
681 svm->vmcb->control.event_inj_err = error_code;
682 }
683
684 static void svm_init_erratum_383(void)
685 {
686 u32 low, high;
687 int err;
688 u64 val;
689
690 if (!static_cpu_has_bug(X86_BUG_AMD_TLB_MMATCH))
691 return;
692
693 /* Use _safe variants to not break nested virtualization */
694 val = native_read_msr_safe(MSR_AMD64_DC_CFG, &err);
695 if (err)
696 return;
697
698 val |= (1ULL << 47);
699
700 low = lower_32_bits(val);
701 high = upper_32_bits(val);
702
703 native_write_msr_safe(MSR_AMD64_DC_CFG, low, high);
704
705 erratum_383_found = true;
706 }
707
708 static void svm_init_osvw(struct kvm_vcpu *vcpu)
709 {
710 /*
711 * Guests should see errata 400 and 415 as fixed (assuming that
712 * HLT and IO instructions are intercepted).
713 */
714 vcpu->arch.osvw.length = (osvw_len >= 3) ? (osvw_len) : 3;
715 vcpu->arch.osvw.status = osvw_status & ~(6ULL);
716
717 /*
718 * By increasing VCPU's osvw.length to 3 we are telling the guest that
719 * all osvw.status bits inside that length, including bit 0 (which is
720 * reserved for erratum 298), are valid. However, if host processor's
721 * osvw_len is 0 then osvw_status[0] carries no information. We need to
722 * be conservative here and therefore we tell the guest that erratum 298
723 * is present (because we really don't know).
724 */
725 if (osvw_len == 0 && boot_cpu_data.x86 == 0x10)
726 vcpu->arch.osvw.status |= 1;
727 }
728
729 static int has_svm(void)
730 {
731 const char *msg;
732
733 if (!cpu_has_svm(&msg)) {
734 printk(KERN_INFO "has_svm: %s\n", msg);
735 return 0;
736 }
737
738 return 1;
739 }
740
741 static void svm_hardware_disable(void)
742 {
743 /* Make sure we clean up behind us */
744 if (static_cpu_has(X86_FEATURE_TSCRATEMSR))
745 wrmsrl(MSR_AMD64_TSC_RATIO, TSC_RATIO_DEFAULT);
746
747 cpu_svm_disable();
748
749 amd_pmu_disable_virt();
750 }
751
752 static int svm_hardware_enable(void)
753 {
754
755 struct svm_cpu_data *sd;
756 uint64_t efer;
757 struct desc_struct *gdt;
758 int me = raw_smp_processor_id();
759
760 rdmsrl(MSR_EFER, efer);
761 if (efer & EFER_SVME)
762 return -EBUSY;
763
764 if (!has_svm()) {
765 pr_err("%s: err EOPNOTSUPP on %d\n", __func__, me);
766 return -EINVAL;
767 }
768 sd = per_cpu(svm_data, me);
769 if (!sd) {
770 pr_err("%s: svm_data is NULL on %d\n", __func__, me);
771 return -EINVAL;
772 }
773
774 sd->asid_generation = 1;
775 sd->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1;
776 sd->next_asid = sd->max_asid + 1;
777
778 gdt = get_current_gdt_rw();
779 sd->tss_desc = (struct kvm_ldttss_desc *)(gdt + GDT_ENTRY_TSS);
780
781 wrmsrl(MSR_EFER, efer | EFER_SVME);
782
783 wrmsrl(MSR_VM_HSAVE_PA, page_to_pfn(sd->save_area) << PAGE_SHIFT);
784
785 if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) {
786 wrmsrl(MSR_AMD64_TSC_RATIO, TSC_RATIO_DEFAULT);
787 __this_cpu_write(current_tsc_ratio, TSC_RATIO_DEFAULT);
788 }
789
790
791 /*
792 * Get OSVW bits.
793 *
794 * Note that it is possible to have a system with mixed processor
795 * revisions and therefore different OSVW bits. If bits are not the same
796 * on different processors then choose the worst case (i.e. if erratum
797 * is present on one processor and not on another then assume that the
798 * erratum is present everywhere).
799 */
800 if (cpu_has(&boot_cpu_data, X86_FEATURE_OSVW)) {
801 uint64_t len, status = 0;
802 int err;
803
804 len = native_read_msr_safe(MSR_AMD64_OSVW_ID_LENGTH, &err);
805 if (!err)
806 status = native_read_msr_safe(MSR_AMD64_OSVW_STATUS,
807 &err);
808
809 if (err)
810 osvw_status = osvw_len = 0;
811 else {
812 if (len < osvw_len)
813 osvw_len = len;
814 osvw_status |= status;
815 osvw_status &= (1ULL << osvw_len) - 1;
816 }
817 } else
818 osvw_status = osvw_len = 0;
819
820 svm_init_erratum_383();
821
822 amd_pmu_enable_virt();
823
824 return 0;
825 }
826
827 static void svm_cpu_uninit(int cpu)
828 {
829 struct svm_cpu_data *sd = per_cpu(svm_data, raw_smp_processor_id());
830
831 if (!sd)
832 return;
833
834 per_cpu(svm_data, raw_smp_processor_id()) = NULL;
835 __free_page(sd->save_area);
836 kfree(sd);
837 }
838
839 static int svm_cpu_init(int cpu)
840 {
841 struct svm_cpu_data *sd;
842 int r;
843
844 sd = kzalloc(sizeof(struct svm_cpu_data), GFP_KERNEL);
845 if (!sd)
846 return -ENOMEM;
847 sd->cpu = cpu;
848 sd->save_area = alloc_page(GFP_KERNEL);
849 r = -ENOMEM;
850 if (!sd->save_area)
851 goto err_1;
852
853 per_cpu(svm_data, cpu) = sd;
854
855 return 0;
856
857 err_1:
858 kfree(sd);
859 return r;
860
861 }
862
863 static bool valid_msr_intercept(u32 index)
864 {
865 int i;
866
867 for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++)
868 if (direct_access_msrs[i].index == index)
869 return true;
870
871 return false;
872 }
873
874 static void set_msr_interception(u32 *msrpm, unsigned msr,
875 int read, int write)
876 {
877 u8 bit_read, bit_write;
878 unsigned long tmp;
879 u32 offset;
880
881 /*
882 * If this warning triggers extend the direct_access_msrs list at the
883 * beginning of the file
884 */
885 WARN_ON(!valid_msr_intercept(msr));
886
887 offset = svm_msrpm_offset(msr);
888 bit_read = 2 * (msr & 0x0f);
889 bit_write = 2 * (msr & 0x0f) + 1;
890 tmp = msrpm[offset];
891
892 BUG_ON(offset == MSR_INVALID);
893
894 read ? clear_bit(bit_read, &tmp) : set_bit(bit_read, &tmp);
895 write ? clear_bit(bit_write, &tmp) : set_bit(bit_write, &tmp);
896
897 msrpm[offset] = tmp;
898 }
899
900 static void svm_vcpu_init_msrpm(u32 *msrpm)
901 {
902 int i;
903
904 memset(msrpm, 0xff, PAGE_SIZE * (1 << MSRPM_ALLOC_ORDER));
905
906 for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
907 if (!direct_access_msrs[i].always)
908 continue;
909
910 set_msr_interception(msrpm, direct_access_msrs[i].index, 1, 1);
911 }
912 }
913
914 static void add_msr_offset(u32 offset)
915 {
916 int i;
917
918 for (i = 0; i < MSRPM_OFFSETS; ++i) {
919
920 /* Offset already in list? */
921 if (msrpm_offsets[i] == offset)
922 return;
923
924 /* Slot used by another offset? */
925 if (msrpm_offsets[i] != MSR_INVALID)
926 continue;
927
928 /* Add offset to list */
929 msrpm_offsets[i] = offset;
930
931 return;
932 }
933
934 /*
935 * If this BUG triggers the msrpm_offsets table has an overflow. Just
936 * increase MSRPM_OFFSETS in this case.
937 */
938 BUG();
939 }
940
941 static void init_msrpm_offsets(void)
942 {
943 int i;
944
945 memset(msrpm_offsets, 0xff, sizeof(msrpm_offsets));
946
947 for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
948 u32 offset;
949
950 offset = svm_msrpm_offset(direct_access_msrs[i].index);
951 BUG_ON(offset == MSR_INVALID);
952
953 add_msr_offset(offset);
954 }
955 }
956
957 static void svm_enable_lbrv(struct vcpu_svm *svm)
958 {
959 u32 *msrpm = svm->msrpm;
960
961 svm->vmcb->control.virt_ext |= LBR_CTL_ENABLE_MASK;
962 set_msr_interception(msrpm, MSR_IA32_LASTBRANCHFROMIP, 1, 1);
963 set_msr_interception(msrpm, MSR_IA32_LASTBRANCHTOIP, 1, 1);
964 set_msr_interception(msrpm, MSR_IA32_LASTINTFROMIP, 1, 1);
965 set_msr_interception(msrpm, MSR_IA32_LASTINTTOIP, 1, 1);
966 }
967
968 static void svm_disable_lbrv(struct vcpu_svm *svm)
969 {
970 u32 *msrpm = svm->msrpm;
971
972 svm->vmcb->control.virt_ext &= ~LBR_CTL_ENABLE_MASK;
973 set_msr_interception(msrpm, MSR_IA32_LASTBRANCHFROMIP, 0, 0);
974 set_msr_interception(msrpm, MSR_IA32_LASTBRANCHTOIP, 0, 0);
975 set_msr_interception(msrpm, MSR_IA32_LASTINTFROMIP, 0, 0);
976 set_msr_interception(msrpm, MSR_IA32_LASTINTTOIP, 0, 0);
977 }
978
979 static void disable_nmi_singlestep(struct vcpu_svm *svm)
980 {
981 svm->nmi_singlestep = false;
982 if (!(svm->vcpu.guest_debug & KVM_GUESTDBG_SINGLESTEP)) {
983 /* Clear our flags if they were not set by the guest */
984 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF))
985 svm->vmcb->save.rflags &= ~X86_EFLAGS_TF;
986 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF))
987 svm->vmcb->save.rflags &= ~X86_EFLAGS_RF;
988 }
989 }
990
991 /* Note:
992 * This hash table is used to map VM_ID to a struct kvm_arch,
993 * when handling AMD IOMMU GALOG notification to schedule in
994 * a particular vCPU.
995 */
996 #define SVM_VM_DATA_HASH_BITS 8
997 static DEFINE_HASHTABLE(svm_vm_data_hash, SVM_VM_DATA_HASH_BITS);
998 static DEFINE_SPINLOCK(svm_vm_data_hash_lock);
999
1000 /* Note:
1001 * This function is called from IOMMU driver to notify
1002 * SVM to schedule in a particular vCPU of a particular VM.
1003 */
1004 static int avic_ga_log_notifier(u32 ga_tag)
1005 {
1006 unsigned long flags;
1007 struct kvm_arch *ka = NULL;
1008 struct kvm_vcpu *vcpu = NULL;
1009 u32 vm_id = AVIC_GATAG_TO_VMID(ga_tag);
1010 u32 vcpu_id = AVIC_GATAG_TO_VCPUID(ga_tag);
1011
1012 pr_debug("SVM: %s: vm_id=%#x, vcpu_id=%#x\n", __func__, vm_id, vcpu_id);
1013
1014 spin_lock_irqsave(&svm_vm_data_hash_lock, flags);
1015 hash_for_each_possible(svm_vm_data_hash, ka, hnode, vm_id) {
1016 struct kvm *kvm = container_of(ka, struct kvm, arch);
1017 struct kvm_arch *vm_data = &kvm->arch;
1018
1019 if (vm_data->avic_vm_id != vm_id)
1020 continue;
1021 vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id);
1022 break;
1023 }
1024 spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags);
1025
1026 if (!vcpu)
1027 return 0;
1028
1029 /* Note:
1030 * At this point, the IOMMU should have already set the pending
1031 * bit in the vAPIC backing page. So, we just need to schedule
1032 * in the vcpu.
1033 */
1034 if (vcpu->mode == OUTSIDE_GUEST_MODE)
1035 kvm_vcpu_wake_up(vcpu);
1036
1037 return 0;
1038 }
1039
1040 static __init int svm_hardware_setup(void)
1041 {
1042 int cpu;
1043 struct page *iopm_pages;
1044 void *iopm_va;
1045 int r;
1046
1047 iopm_pages = alloc_pages(GFP_KERNEL, IOPM_ALLOC_ORDER);
1048
1049 if (!iopm_pages)
1050 return -ENOMEM;
1051
1052 iopm_va = page_address(iopm_pages);
1053 memset(iopm_va, 0xff, PAGE_SIZE * (1 << IOPM_ALLOC_ORDER));
1054 iopm_base = page_to_pfn(iopm_pages) << PAGE_SHIFT;
1055
1056 init_msrpm_offsets();
1057
1058 if (boot_cpu_has(X86_FEATURE_NX))
1059 kvm_enable_efer_bits(EFER_NX);
1060
1061 if (boot_cpu_has(X86_FEATURE_FXSR_OPT))
1062 kvm_enable_efer_bits(EFER_FFXSR);
1063
1064 if (boot_cpu_has(X86_FEATURE_TSCRATEMSR)) {
1065 kvm_has_tsc_control = true;
1066 kvm_max_tsc_scaling_ratio = TSC_RATIO_MAX;
1067 kvm_tsc_scaling_ratio_frac_bits = 32;
1068 }
1069
1070 if (nested) {
1071 printk(KERN_INFO "kvm: Nested Virtualization enabled\n");
1072 kvm_enable_efer_bits(EFER_SVME | EFER_LMSLE);
1073 }
1074
1075 for_each_possible_cpu(cpu) {
1076 r = svm_cpu_init(cpu);
1077 if (r)
1078 goto err;
1079 }
1080
1081 if (!boot_cpu_has(X86_FEATURE_NPT))
1082 npt_enabled = false;
1083
1084 if (npt_enabled && !npt) {
1085 printk(KERN_INFO "kvm: Nested Paging disabled\n");
1086 npt_enabled = false;
1087 }
1088
1089 if (npt_enabled) {
1090 printk(KERN_INFO "kvm: Nested Paging enabled\n");
1091 kvm_enable_tdp();
1092 } else
1093 kvm_disable_tdp();
1094
1095 if (avic) {
1096 if (!npt_enabled ||
1097 !boot_cpu_has(X86_FEATURE_AVIC) ||
1098 !IS_ENABLED(CONFIG_X86_LOCAL_APIC)) {
1099 avic = false;
1100 } else {
1101 pr_info("AVIC enabled\n");
1102
1103 amd_iommu_register_ga_log_notifier(&avic_ga_log_notifier);
1104 }
1105 }
1106
1107 if (vls) {
1108 if (!npt_enabled ||
1109 !boot_cpu_has(X86_FEATURE_V_VMSAVE_VMLOAD) ||
1110 !IS_ENABLED(CONFIG_X86_64)) {
1111 vls = false;
1112 } else {
1113 pr_info("Virtual VMLOAD VMSAVE supported\n");
1114 }
1115 }
1116
1117 return 0;
1118
1119 err:
1120 __free_pages(iopm_pages, IOPM_ALLOC_ORDER);
1121 iopm_base = 0;
1122 return r;
1123 }
1124
1125 static __exit void svm_hardware_unsetup(void)
1126 {
1127 int cpu;
1128
1129 for_each_possible_cpu(cpu)
1130 svm_cpu_uninit(cpu);
1131
1132 __free_pages(pfn_to_page(iopm_base >> PAGE_SHIFT), IOPM_ALLOC_ORDER);
1133 iopm_base = 0;
1134 }
1135
1136 static void init_seg(struct vmcb_seg *seg)
1137 {
1138 seg->selector = 0;
1139 seg->attrib = SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK |
1140 SVM_SELECTOR_WRITE_MASK; /* Read/Write Data Segment */
1141 seg->limit = 0xffff;
1142 seg->base = 0;
1143 }
1144
1145 static void init_sys_seg(struct vmcb_seg *seg, uint32_t type)
1146 {
1147 seg->selector = 0;
1148 seg->attrib = SVM_SELECTOR_P_MASK | type;
1149 seg->limit = 0xffff;
1150 seg->base = 0;
1151 }
1152
1153 static void svm_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)
1154 {
1155 struct vcpu_svm *svm = to_svm(vcpu);
1156 u64 g_tsc_offset = 0;
1157
1158 if (is_guest_mode(vcpu)) {
1159 g_tsc_offset = svm->vmcb->control.tsc_offset -
1160 svm->nested.hsave->control.tsc_offset;
1161 svm->nested.hsave->control.tsc_offset = offset;
1162 } else
1163 trace_kvm_write_tsc_offset(vcpu->vcpu_id,
1164 svm->vmcb->control.tsc_offset,
1165 offset);
1166
1167 svm->vmcb->control.tsc_offset = offset + g_tsc_offset;
1168
1169 mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
1170 }
1171
1172 static void avic_init_vmcb(struct vcpu_svm *svm)
1173 {
1174 struct vmcb *vmcb = svm->vmcb;
1175 struct kvm_arch *vm_data = &svm->vcpu.kvm->arch;
1176 phys_addr_t bpa = page_to_phys(svm->avic_backing_page);
1177 phys_addr_t lpa = page_to_phys(vm_data->avic_logical_id_table_page);
1178 phys_addr_t ppa = page_to_phys(vm_data->avic_physical_id_table_page);
1179
1180 vmcb->control.avic_backing_page = bpa & AVIC_HPA_MASK;
1181 vmcb->control.avic_logical_id = lpa & AVIC_HPA_MASK;
1182 vmcb->control.avic_physical_id = ppa & AVIC_HPA_MASK;
1183 vmcb->control.avic_physical_id |= AVIC_MAX_PHYSICAL_ID_COUNT;
1184 vmcb->control.int_ctl |= AVIC_ENABLE_MASK;
1185 svm->vcpu.arch.apicv_active = true;
1186 }
1187
1188 static void init_vmcb(struct vcpu_svm *svm)
1189 {
1190 struct vmcb_control_area *control = &svm->vmcb->control;
1191 struct vmcb_save_area *save = &svm->vmcb->save;
1192
1193 svm->vcpu.arch.hflags = 0;
1194
1195 set_cr_intercept(svm, INTERCEPT_CR0_READ);
1196 set_cr_intercept(svm, INTERCEPT_CR3_READ);
1197 set_cr_intercept(svm, INTERCEPT_CR4_READ);
1198 set_cr_intercept(svm, INTERCEPT_CR0_WRITE);
1199 set_cr_intercept(svm, INTERCEPT_CR3_WRITE);
1200 set_cr_intercept(svm, INTERCEPT_CR4_WRITE);
1201 if (!kvm_vcpu_apicv_active(&svm->vcpu))
1202 set_cr_intercept(svm, INTERCEPT_CR8_WRITE);
1203
1204 set_dr_intercepts(svm);
1205
1206 set_exception_intercept(svm, PF_VECTOR);
1207 set_exception_intercept(svm, UD_VECTOR);
1208 set_exception_intercept(svm, MC_VECTOR);
1209 set_exception_intercept(svm, AC_VECTOR);
1210 set_exception_intercept(svm, DB_VECTOR);
1211
1212 set_intercept(svm, INTERCEPT_INTR);
1213 set_intercept(svm, INTERCEPT_NMI);
1214 set_intercept(svm, INTERCEPT_SMI);
1215 set_intercept(svm, INTERCEPT_SELECTIVE_CR0);
1216 set_intercept(svm, INTERCEPT_RDPMC);
1217 set_intercept(svm, INTERCEPT_CPUID);
1218 set_intercept(svm, INTERCEPT_INVD);
1219 set_intercept(svm, INTERCEPT_HLT);
1220 set_intercept(svm, INTERCEPT_INVLPG);
1221 set_intercept(svm, INTERCEPT_INVLPGA);
1222 set_intercept(svm, INTERCEPT_IOIO_PROT);
1223 set_intercept(svm, INTERCEPT_MSR_PROT);
1224 set_intercept(svm, INTERCEPT_TASK_SWITCH);
1225 set_intercept(svm, INTERCEPT_SHUTDOWN);
1226 set_intercept(svm, INTERCEPT_VMRUN);
1227 set_intercept(svm, INTERCEPT_VMMCALL);
1228 set_intercept(svm, INTERCEPT_VMLOAD);
1229 set_intercept(svm, INTERCEPT_VMSAVE);
1230 set_intercept(svm, INTERCEPT_STGI);
1231 set_intercept(svm, INTERCEPT_CLGI);
1232 set_intercept(svm, INTERCEPT_SKINIT);
1233 set_intercept(svm, INTERCEPT_WBINVD);
1234 set_intercept(svm, INTERCEPT_XSETBV);
1235
1236 if (!kvm_mwait_in_guest()) {
1237 set_intercept(svm, INTERCEPT_MONITOR);
1238 set_intercept(svm, INTERCEPT_MWAIT);
1239 }
1240
1241 control->iopm_base_pa = iopm_base;
1242 control->msrpm_base_pa = __pa(svm->msrpm);
1243 control->int_ctl = V_INTR_MASKING_MASK;
1244
1245 init_seg(&save->es);
1246 init_seg(&save->ss);
1247 init_seg(&save->ds);
1248 init_seg(&save->fs);
1249 init_seg(&save->gs);
1250
1251 save->cs.selector = 0xf000;
1252 save->cs.base = 0xffff0000;
1253 /* Executable/Readable Code Segment */
1254 save->cs.attrib = SVM_SELECTOR_READ_MASK | SVM_SELECTOR_P_MASK |
1255 SVM_SELECTOR_S_MASK | SVM_SELECTOR_CODE_MASK;
1256 save->cs.limit = 0xffff;
1257
1258 save->gdtr.limit = 0xffff;
1259 save->idtr.limit = 0xffff;
1260
1261 init_sys_seg(&save->ldtr, SEG_TYPE_LDT);
1262 init_sys_seg(&save->tr, SEG_TYPE_BUSY_TSS16);
1263
1264 svm_set_efer(&svm->vcpu, 0);
1265 save->dr6 = 0xffff0ff0;
1266 kvm_set_rflags(&svm->vcpu, 2);
1267 save->rip = 0x0000fff0;
1268 svm->vcpu.arch.regs[VCPU_REGS_RIP] = save->rip;
1269
1270 /*
1271 * svm_set_cr0() sets PG and WP and clears NW and CD on save->cr0.
1272 * It also updates the guest-visible cr0 value.
1273 */
1274 svm_set_cr0(&svm->vcpu, X86_CR0_NW | X86_CR0_CD | X86_CR0_ET);
1275 kvm_mmu_reset_context(&svm->vcpu);
1276
1277 save->cr4 = X86_CR4_PAE;
1278 /* rdx = ?? */
1279
1280 if (npt_enabled) {
1281 /* Setup VMCB for Nested Paging */
1282 control->nested_ctl = 1;
1283 clr_intercept(svm, INTERCEPT_INVLPG);
1284 clr_exception_intercept(svm, PF_VECTOR);
1285 clr_cr_intercept(svm, INTERCEPT_CR3_READ);
1286 clr_cr_intercept(svm, INTERCEPT_CR3_WRITE);
1287 save->g_pat = svm->vcpu.arch.pat;
1288 save->cr3 = 0;
1289 save->cr4 = 0;
1290 }
1291 svm->asid_generation = 0;
1292
1293 svm->nested.vmcb = 0;
1294 svm->vcpu.arch.hflags = 0;
1295
1296 if (boot_cpu_has(X86_FEATURE_PAUSEFILTER)) {
1297 control->pause_filter_count = 3000;
1298 set_intercept(svm, INTERCEPT_PAUSE);
1299 }
1300
1301 if (avic)
1302 avic_init_vmcb(svm);
1303
1304 /*
1305 * If hardware supports Virtual VMLOAD VMSAVE then enable it
1306 * in VMCB and clear intercepts to avoid #VMEXIT.
1307 */
1308 if (vls) {
1309 clr_intercept(svm, INTERCEPT_VMLOAD);
1310 clr_intercept(svm, INTERCEPT_VMSAVE);
1311 svm->vmcb->control.virt_ext |= VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK;
1312 }
1313
1314 mark_all_dirty(svm->vmcb);
1315
1316 enable_gif(svm);
1317
1318 }
1319
1320 static u64 *avic_get_physical_id_entry(struct kvm_vcpu *vcpu,
1321 unsigned int index)
1322 {
1323 u64 *avic_physical_id_table;
1324 struct kvm_arch *vm_data = &vcpu->kvm->arch;
1325
1326 if (index >= AVIC_MAX_PHYSICAL_ID_COUNT)
1327 return NULL;
1328
1329 avic_physical_id_table = page_address(vm_data->avic_physical_id_table_page);
1330
1331 return &avic_physical_id_table[index];
1332 }
1333
1334 /**
1335 * Note:
1336 * AVIC hardware walks the nested page table to check permissions,
1337 * but does not use the SPA address specified in the leaf page
1338 * table entry since it uses address in the AVIC_BACKING_PAGE pointer
1339 * field of the VMCB. Therefore, we set up the
1340 * APIC_ACCESS_PAGE_PRIVATE_MEMSLOT (4KB) here.
1341 */
1342 static int avic_init_access_page(struct kvm_vcpu *vcpu)
1343 {
1344 struct kvm *kvm = vcpu->kvm;
1345 int ret;
1346
1347 if (kvm->arch.apic_access_page_done)
1348 return 0;
1349
1350 ret = x86_set_memory_region(kvm,
1351 APIC_ACCESS_PAGE_PRIVATE_MEMSLOT,
1352 APIC_DEFAULT_PHYS_BASE,
1353 PAGE_SIZE);
1354 if (ret)
1355 return ret;
1356
1357 kvm->arch.apic_access_page_done = true;
1358 return 0;
1359 }
1360
1361 static int avic_init_backing_page(struct kvm_vcpu *vcpu)
1362 {
1363 int ret;
1364 u64 *entry, new_entry;
1365 int id = vcpu->vcpu_id;
1366 struct vcpu_svm *svm = to_svm(vcpu);
1367
1368 ret = avic_init_access_page(vcpu);
1369 if (ret)
1370 return ret;
1371
1372 if (id >= AVIC_MAX_PHYSICAL_ID_COUNT)
1373 return -EINVAL;
1374
1375 if (!svm->vcpu.arch.apic->regs)
1376 return -EINVAL;
1377
1378 svm->avic_backing_page = virt_to_page(svm->vcpu.arch.apic->regs);
1379
1380 /* Setting AVIC backing page address in the phy APIC ID table */
1381 entry = avic_get_physical_id_entry(vcpu, id);
1382 if (!entry)
1383 return -EINVAL;
1384
1385 new_entry = READ_ONCE(*entry);
1386 new_entry = (page_to_phys(svm->avic_backing_page) &
1387 AVIC_PHYSICAL_ID_ENTRY_BACKING_PAGE_MASK) |
1388 AVIC_PHYSICAL_ID_ENTRY_VALID_MASK;
1389 WRITE_ONCE(*entry, new_entry);
1390
1391 svm->avic_physical_id_cache = entry;
1392
1393 return 0;
1394 }
1395
1396 static inline int avic_get_next_vm_id(void)
1397 {
1398 int id;
1399
1400 spin_lock(&avic_vm_id_lock);
1401
1402 /* AVIC VM ID is one-based. */
1403 id = find_next_zero_bit(avic_vm_id_bitmap, AVIC_VM_ID_NR, 1);
1404 if (id <= AVIC_VM_ID_MASK)
1405 __set_bit(id, avic_vm_id_bitmap);
1406 else
1407 id = -EAGAIN;
1408
1409 spin_unlock(&avic_vm_id_lock);
1410 return id;
1411 }
1412
1413 static inline int avic_free_vm_id(int id)
1414 {
1415 if (id <= 0 || id > AVIC_VM_ID_MASK)
1416 return -EINVAL;
1417
1418 spin_lock(&avic_vm_id_lock);
1419 __clear_bit(id, avic_vm_id_bitmap);
1420 spin_unlock(&avic_vm_id_lock);
1421 return 0;
1422 }
1423
1424 static void avic_vm_destroy(struct kvm *kvm)
1425 {
1426 unsigned long flags;
1427 struct kvm_arch *vm_data = &kvm->arch;
1428
1429 if (!avic)
1430 return;
1431
1432 avic_free_vm_id(vm_data->avic_vm_id);
1433
1434 if (vm_data->avic_logical_id_table_page)
1435 __free_page(vm_data->avic_logical_id_table_page);
1436 if (vm_data->avic_physical_id_table_page)
1437 __free_page(vm_data->avic_physical_id_table_page);
1438
1439 spin_lock_irqsave(&svm_vm_data_hash_lock, flags);
1440 hash_del(&vm_data->hnode);
1441 spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags);
1442 }
1443
1444 static int avic_vm_init(struct kvm *kvm)
1445 {
1446 unsigned long flags;
1447 int vm_id, err = -ENOMEM;
1448 struct kvm_arch *vm_data = &kvm->arch;
1449 struct page *p_page;
1450 struct page *l_page;
1451
1452 if (!avic)
1453 return 0;
1454
1455 vm_id = avic_get_next_vm_id();
1456 if (vm_id < 0)
1457 return vm_id;
1458 vm_data->avic_vm_id = (u32)vm_id;
1459
1460 /* Allocating physical APIC ID table (4KB) */
1461 p_page = alloc_page(GFP_KERNEL);
1462 if (!p_page)
1463 goto free_avic;
1464
1465 vm_data->avic_physical_id_table_page = p_page;
1466 clear_page(page_address(p_page));
1467
1468 /* Allocating logical APIC ID table (4KB) */
1469 l_page = alloc_page(GFP_KERNEL);
1470 if (!l_page)
1471 goto free_avic;
1472
1473 vm_data->avic_logical_id_table_page = l_page;
1474 clear_page(page_address(l_page));
1475
1476 spin_lock_irqsave(&svm_vm_data_hash_lock, flags);
1477 hash_add(svm_vm_data_hash, &vm_data->hnode, vm_data->avic_vm_id);
1478 spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags);
1479
1480 return 0;
1481
1482 free_avic:
1483 avic_vm_destroy(kvm);
1484 return err;
1485 }
1486
1487 static inline int
1488 avic_update_iommu_vcpu_affinity(struct kvm_vcpu *vcpu, int cpu, bool r)
1489 {
1490 int ret = 0;
1491 unsigned long flags;
1492 struct amd_svm_iommu_ir *ir;
1493 struct vcpu_svm *svm = to_svm(vcpu);
1494
1495 if (!kvm_arch_has_assigned_device(vcpu->kvm))
1496 return 0;
1497
1498 /*
1499 * Here, we go through the per-vcpu ir_list to update all existing
1500 * interrupt remapping table entry targeting this vcpu.
1501 */
1502 spin_lock_irqsave(&svm->ir_list_lock, flags);
1503
1504 if (list_empty(&svm->ir_list))
1505 goto out;
1506
1507 list_for_each_entry(ir, &svm->ir_list, node) {
1508 ret = amd_iommu_update_ga(cpu, r, ir->data);
1509 if (ret)
1510 break;
1511 }
1512 out:
1513 spin_unlock_irqrestore(&svm->ir_list_lock, flags);
1514 return ret;
1515 }
1516
1517 static void avic_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1518 {
1519 u64 entry;
1520 /* ID = 0xff (broadcast), ID > 0xff (reserved) */
1521 int h_physical_id = kvm_cpu_get_apicid(cpu);
1522 struct vcpu_svm *svm = to_svm(vcpu);
1523
1524 if (!kvm_vcpu_apicv_active(vcpu))
1525 return;
1526
1527 if (WARN_ON(h_physical_id >= AVIC_MAX_PHYSICAL_ID_COUNT))
1528 return;
1529
1530 entry = READ_ONCE(*(svm->avic_physical_id_cache));
1531 WARN_ON(entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK);
1532
1533 entry &= ~AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK;
1534 entry |= (h_physical_id & AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK);
1535
1536 entry &= ~AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK;
1537 if (svm->avic_is_running)
1538 entry |= AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK;
1539
1540 WRITE_ONCE(*(svm->avic_physical_id_cache), entry);
1541 avic_update_iommu_vcpu_affinity(vcpu, h_physical_id,
1542 svm->avic_is_running);
1543 }
1544
1545 static void avic_vcpu_put(struct kvm_vcpu *vcpu)
1546 {
1547 u64 entry;
1548 struct vcpu_svm *svm = to_svm(vcpu);
1549
1550 if (!kvm_vcpu_apicv_active(vcpu))
1551 return;
1552
1553 entry = READ_ONCE(*(svm->avic_physical_id_cache));
1554 if (entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK)
1555 avic_update_iommu_vcpu_affinity(vcpu, -1, 0);
1556
1557 entry &= ~AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK;
1558 WRITE_ONCE(*(svm->avic_physical_id_cache), entry);
1559 }
1560
1561 /**
1562 * This function is called during VCPU halt/unhalt.
1563 */
1564 static void avic_set_running(struct kvm_vcpu *vcpu, bool is_run)
1565 {
1566 struct vcpu_svm *svm = to_svm(vcpu);
1567
1568 svm->avic_is_running = is_run;
1569 if (is_run)
1570 avic_vcpu_load(vcpu, vcpu->cpu);
1571 else
1572 avic_vcpu_put(vcpu);
1573 }
1574
1575 static void svm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
1576 {
1577 struct vcpu_svm *svm = to_svm(vcpu);
1578 u32 dummy;
1579 u32 eax = 1;
1580
1581 if (!init_event) {
1582 svm->vcpu.arch.apic_base = APIC_DEFAULT_PHYS_BASE |
1583 MSR_IA32_APICBASE_ENABLE;
1584 if (kvm_vcpu_is_reset_bsp(&svm->vcpu))
1585 svm->vcpu.arch.apic_base |= MSR_IA32_APICBASE_BSP;
1586 }
1587 init_vmcb(svm);
1588
1589 kvm_cpuid(vcpu, &eax, &dummy, &dummy, &dummy);
1590 kvm_register_write(vcpu, VCPU_REGS_RDX, eax);
1591
1592 if (kvm_vcpu_apicv_active(vcpu) && !init_event)
1593 avic_update_vapic_bar(svm, APIC_DEFAULT_PHYS_BASE);
1594 }
1595
1596 static struct kvm_vcpu *svm_create_vcpu(struct kvm *kvm, unsigned int id)
1597 {
1598 struct vcpu_svm *svm;
1599 struct page *page;
1600 struct page *msrpm_pages;
1601 struct page *hsave_page;
1602 struct page *nested_msrpm_pages;
1603 int err;
1604
1605 svm = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
1606 if (!svm) {
1607 err = -ENOMEM;
1608 goto out;
1609 }
1610
1611 err = kvm_vcpu_init(&svm->vcpu, kvm, id);
1612 if (err)
1613 goto free_svm;
1614
1615 err = -ENOMEM;
1616 page = alloc_page(GFP_KERNEL);
1617 if (!page)
1618 goto uninit;
1619
1620 msrpm_pages = alloc_pages(GFP_KERNEL, MSRPM_ALLOC_ORDER);
1621 if (!msrpm_pages)
1622 goto free_page1;
1623
1624 nested_msrpm_pages = alloc_pages(GFP_KERNEL, MSRPM_ALLOC_ORDER);
1625 if (!nested_msrpm_pages)
1626 goto free_page2;
1627
1628 hsave_page = alloc_page(GFP_KERNEL);
1629 if (!hsave_page)
1630 goto free_page3;
1631
1632 if (avic) {
1633 err = avic_init_backing_page(&svm->vcpu);
1634 if (err)
1635 goto free_page4;
1636
1637 INIT_LIST_HEAD(&svm->ir_list);
1638 spin_lock_init(&svm->ir_list_lock);
1639 }
1640
1641 /* We initialize this flag to true to make sure that the is_running
1642 * bit would be set the first time the vcpu is loaded.
1643 */
1644 svm->avic_is_running = true;
1645
1646 svm->nested.hsave = page_address(hsave_page);
1647
1648 svm->msrpm = page_address(msrpm_pages);
1649 svm_vcpu_init_msrpm(svm->msrpm);
1650
1651 svm->nested.msrpm = page_address(nested_msrpm_pages);
1652 svm_vcpu_init_msrpm(svm->nested.msrpm);
1653
1654 svm->vmcb = page_address(page);
1655 clear_page(svm->vmcb);
1656 svm->vmcb_pa = page_to_pfn(page) << PAGE_SHIFT;
1657 svm->asid_generation = 0;
1658 init_vmcb(svm);
1659
1660 svm_init_osvw(&svm->vcpu);
1661
1662 return &svm->vcpu;
1663
1664 free_page4:
1665 __free_page(hsave_page);
1666 free_page3:
1667 __free_pages(nested_msrpm_pages, MSRPM_ALLOC_ORDER);
1668 free_page2:
1669 __free_pages(msrpm_pages, MSRPM_ALLOC_ORDER);
1670 free_page1:
1671 __free_page(page);
1672 uninit:
1673 kvm_vcpu_uninit(&svm->vcpu);
1674 free_svm:
1675 kmem_cache_free(kvm_vcpu_cache, svm);
1676 out:
1677 return ERR_PTR(err);
1678 }
1679
1680 static void svm_free_vcpu(struct kvm_vcpu *vcpu)
1681 {
1682 struct vcpu_svm *svm = to_svm(vcpu);
1683
1684 __free_page(pfn_to_page(svm->vmcb_pa >> PAGE_SHIFT));
1685 __free_pages(virt_to_page(svm->msrpm), MSRPM_ALLOC_ORDER);
1686 __free_page(virt_to_page(svm->nested.hsave));
1687 __free_pages(virt_to_page(svm->nested.msrpm), MSRPM_ALLOC_ORDER);
1688 kvm_vcpu_uninit(vcpu);
1689 kmem_cache_free(kvm_vcpu_cache, svm);
1690
1691 /*
1692 * The VMCB could be recycled, causing a false negative in svm_vcpu_load;
1693 * block speculative execution.
1694 */
1695 if (ibpb_inuse)
1696 wrmsrl(MSR_IA32_PRED_CMD, FEATURE_SET_IBPB);
1697 }
1698
1699 static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1700 {
1701 struct vcpu_svm *svm = to_svm(vcpu);
1702 struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
1703 int i;
1704
1705 if (unlikely(cpu != vcpu->cpu)) {
1706 svm->asid_generation = 0;
1707 mark_all_dirty(svm->vmcb);
1708 }
1709
1710 #ifdef CONFIG_X86_64
1711 rdmsrl(MSR_GS_BASE, to_svm(vcpu)->host.gs_base);
1712 #endif
1713 savesegment(fs, svm->host.fs);
1714 savesegment(gs, svm->host.gs);
1715 svm->host.ldt = kvm_read_ldt();
1716
1717 for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
1718 rdmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
1719
1720 if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) {
1721 u64 tsc_ratio = vcpu->arch.tsc_scaling_ratio;
1722 if (tsc_ratio != __this_cpu_read(current_tsc_ratio)) {
1723 __this_cpu_write(current_tsc_ratio, tsc_ratio);
1724 wrmsrl(MSR_AMD64_TSC_RATIO, tsc_ratio);
1725 }
1726 }
1727 /* This assumes that the kernel never uses MSR_TSC_AUX */
1728 if (static_cpu_has(X86_FEATURE_RDTSCP))
1729 wrmsrl(MSR_TSC_AUX, svm->tsc_aux);
1730
1731 if (sd->current_vmcb != svm->vmcb) {
1732 sd->current_vmcb = svm->vmcb;
1733 if (ibpb_inuse)
1734 wrmsrl(MSR_IA32_PRED_CMD, FEATURE_SET_IBPB);
1735 }
1736
1737 avic_vcpu_load(vcpu, cpu);
1738 }
1739
1740 static void svm_vcpu_put(struct kvm_vcpu *vcpu)
1741 {
1742 struct vcpu_svm *svm = to_svm(vcpu);
1743 int i;
1744
1745 avic_vcpu_put(vcpu);
1746
1747 ++vcpu->stat.host_state_reload;
1748 kvm_load_ldt(svm->host.ldt);
1749 #ifdef CONFIG_X86_64
1750 loadsegment(fs, svm->host.fs);
1751 wrmsrl(MSR_KERNEL_GS_BASE, current->thread.gsbase);
1752 load_gs_index(svm->host.gs);
1753 #else
1754 #ifdef CONFIG_X86_32_LAZY_GS
1755 loadsegment(gs, svm->host.gs);
1756 #endif
1757 #endif
1758 for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
1759 wrmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
1760 }
1761
1762 static void svm_vcpu_blocking(struct kvm_vcpu *vcpu)
1763 {
1764 avic_set_running(vcpu, false);
1765 }
1766
1767 static void svm_vcpu_unblocking(struct kvm_vcpu *vcpu)
1768 {
1769 avic_set_running(vcpu, true);
1770 }
1771
1772 static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu)
1773 {
1774 struct vcpu_svm *svm = to_svm(vcpu);
1775 unsigned long rflags = svm->vmcb->save.rflags;
1776
1777 if (svm->nmi_singlestep) {
1778 /* Hide our flags if they were not set by the guest */
1779 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF))
1780 rflags &= ~X86_EFLAGS_TF;
1781 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF))
1782 rflags &= ~X86_EFLAGS_RF;
1783 }
1784 return rflags;
1785 }
1786
1787 static void svm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
1788 {
1789 if (to_svm(vcpu)->nmi_singlestep)
1790 rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
1791
1792 /*
1793 * Any change of EFLAGS.VM is accompanied by a reload of SS
1794 * (caused by either a task switch or an inter-privilege IRET),
1795 * so we do not need to update the CPL here.
1796 */
1797 to_svm(vcpu)->vmcb->save.rflags = rflags;
1798 }
1799
1800 static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
1801 {
1802 switch (reg) {
1803 case VCPU_EXREG_PDPTR:
1804 BUG_ON(!npt_enabled);
1805 load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu));
1806 break;
1807 default:
1808 BUG();
1809 }
1810 }
1811
1812 static void svm_set_vintr(struct vcpu_svm *svm)
1813 {
1814 set_intercept(svm, INTERCEPT_VINTR);
1815 }
1816
1817 static void svm_clear_vintr(struct vcpu_svm *svm)
1818 {
1819 clr_intercept(svm, INTERCEPT_VINTR);
1820 }
1821
1822 static struct vmcb_seg *svm_seg(struct kvm_vcpu *vcpu, int seg)
1823 {
1824 struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
1825
1826 switch (seg) {
1827 case VCPU_SREG_CS: return &save->cs;
1828 case VCPU_SREG_DS: return &save->ds;
1829 case VCPU_SREG_ES: return &save->es;
1830 case VCPU_SREG_FS: return &save->fs;
1831 case VCPU_SREG_GS: return &save->gs;
1832 case VCPU_SREG_SS: return &save->ss;
1833 case VCPU_SREG_TR: return &save->tr;
1834 case VCPU_SREG_LDTR: return &save->ldtr;
1835 }
1836 BUG();
1837 return NULL;
1838 }
1839
1840 static u64 svm_get_segment_base(struct kvm_vcpu *vcpu, int seg)
1841 {
1842 struct vmcb_seg *s = svm_seg(vcpu, seg);
1843
1844 return s->base;
1845 }
1846
1847 static void svm_get_segment(struct kvm_vcpu *vcpu,
1848 struct kvm_segment *var, int seg)
1849 {
1850 struct vmcb_seg *s = svm_seg(vcpu, seg);
1851
1852 var->base = s->base;
1853 var->limit = s->limit;
1854 var->selector = s->selector;
1855 var->type = s->attrib & SVM_SELECTOR_TYPE_MASK;
1856 var->s = (s->attrib >> SVM_SELECTOR_S_SHIFT) & 1;
1857 var->dpl = (s->attrib >> SVM_SELECTOR_DPL_SHIFT) & 3;
1858 var->present = (s->attrib >> SVM_SELECTOR_P_SHIFT) & 1;
1859 var->avl = (s->attrib >> SVM_SELECTOR_AVL_SHIFT) & 1;
1860 var->l = (s->attrib >> SVM_SELECTOR_L_SHIFT) & 1;
1861 var->db = (s->attrib >> SVM_SELECTOR_DB_SHIFT) & 1;
1862
1863 /*
1864 * AMD CPUs circa 2014 track the G bit for all segments except CS.
1865 * However, the SVM spec states that the G bit is not observed by the
1866 * CPU, and some VMware virtual CPUs drop the G bit for all segments.
1867 * So let's synthesize a legal G bit for all segments, this helps
1868 * running KVM nested. It also helps cross-vendor migration, because
1869 * Intel's vmentry has a check on the 'G' bit.
1870 */
1871 var->g = s->limit > 0xfffff;
1872
1873 /*
1874 * AMD's VMCB does not have an explicit unusable field, so emulate it
1875 * for cross vendor migration purposes by "not present"
1876 */
1877 var->unusable = !var->present;
1878
1879 switch (seg) {
1880 case VCPU_SREG_TR:
1881 /*
1882 * Work around a bug where the busy flag in the tr selector
1883 * isn't exposed
1884 */
1885 var->type |= 0x2;
1886 break;
1887 case VCPU_SREG_DS:
1888 case VCPU_SREG_ES:
1889 case VCPU_SREG_FS:
1890 case VCPU_SREG_GS:
1891 /*
1892 * The accessed bit must always be set in the segment
1893 * descriptor cache, although it can be cleared in the
1894 * descriptor, the cached bit always remains at 1. Since
1895 * Intel has a check on this, set it here to support
1896 * cross-vendor migration.
1897 */
1898 if (!var->unusable)
1899 var->type |= 0x1;
1900 break;
1901 case VCPU_SREG_SS:
1902 /*
1903 * On AMD CPUs sometimes the DB bit in the segment
1904 * descriptor is left as 1, although the whole segment has
1905 * been made unusable. Clear it here to pass an Intel VMX
1906 * entry check when cross vendor migrating.
1907 */
1908 if (var->unusable)
1909 var->db = 0;
1910 /* This is symmetric with svm_set_segment() */
1911 var->dpl = to_svm(vcpu)->vmcb->save.cpl;
1912 break;
1913 }
1914 }
1915
1916 static int svm_get_cpl(struct kvm_vcpu *vcpu)
1917 {
1918 struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
1919
1920 return save->cpl;
1921 }
1922
1923 static void svm_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1924 {
1925 struct vcpu_svm *svm = to_svm(vcpu);
1926
1927 dt->size = svm->vmcb->save.idtr.limit;
1928 dt->address = svm->vmcb->save.idtr.base;
1929 }
1930
1931 static void svm_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1932 {
1933 struct vcpu_svm *svm = to_svm(vcpu);
1934
1935 svm->vmcb->save.idtr.limit = dt->size;
1936 svm->vmcb->save.idtr.base = dt->address ;
1937 mark_dirty(svm->vmcb, VMCB_DT);
1938 }
1939
1940 static void svm_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1941 {
1942 struct vcpu_svm *svm = to_svm(vcpu);
1943
1944 dt->size = svm->vmcb->save.gdtr.limit;
1945 dt->address = svm->vmcb->save.gdtr.base;
1946 }
1947
1948 static void svm_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1949 {
1950 struct vcpu_svm *svm = to_svm(vcpu);
1951
1952 svm->vmcb->save.gdtr.limit = dt->size;
1953 svm->vmcb->save.gdtr.base = dt->address ;
1954 mark_dirty(svm->vmcb, VMCB_DT);
1955 }
1956
1957 static void svm_decache_cr0_guest_bits(struct kvm_vcpu *vcpu)
1958 {
1959 }
1960
1961 static void svm_decache_cr3(struct kvm_vcpu *vcpu)
1962 {
1963 }
1964
1965 static void svm_decache_cr4_guest_bits(struct kvm_vcpu *vcpu)
1966 {
1967 }
1968
1969 static void update_cr0_intercept(struct vcpu_svm *svm)
1970 {
1971 ulong gcr0 = svm->vcpu.arch.cr0;
1972 u64 *hcr0 = &svm->vmcb->save.cr0;
1973
1974 *hcr0 = (*hcr0 & ~SVM_CR0_SELECTIVE_MASK)
1975 | (gcr0 & SVM_CR0_SELECTIVE_MASK);
1976
1977 mark_dirty(svm->vmcb, VMCB_CR);
1978
1979 if (gcr0 == *hcr0) {
1980 clr_cr_intercept(svm, INTERCEPT_CR0_READ);
1981 clr_cr_intercept(svm, INTERCEPT_CR0_WRITE);
1982 } else {
1983 set_cr_intercept(svm, INTERCEPT_CR0_READ);
1984 set_cr_intercept(svm, INTERCEPT_CR0_WRITE);
1985 }
1986 }
1987
1988 static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
1989 {
1990 struct vcpu_svm *svm = to_svm(vcpu);
1991
1992 #ifdef CONFIG_X86_64
1993 if (vcpu->arch.efer & EFER_LME) {
1994 if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
1995 vcpu->arch.efer |= EFER_LMA;
1996 svm->vmcb->save.efer |= EFER_LMA | EFER_LME;
1997 }
1998
1999 if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) {
2000 vcpu->arch.efer &= ~EFER_LMA;
2001 svm->vmcb->save.efer &= ~(EFER_LMA | EFER_LME);
2002 }
2003 }
2004 #endif
2005 vcpu->arch.cr0 = cr0;
2006
2007 if (!npt_enabled)
2008 cr0 |= X86_CR0_PG | X86_CR0_WP;
2009
2010 /*
2011 * re-enable caching here because the QEMU bios
2012 * does not do it - this results in some delay at
2013 * reboot
2014 */
2015 if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))
2016 cr0 &= ~(X86_CR0_CD | X86_CR0_NW);
2017 svm->vmcb->save.cr0 = cr0;
2018 mark_dirty(svm->vmcb, VMCB_CR);
2019 update_cr0_intercept(svm);
2020 }
2021
2022 static int svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
2023 {
2024 unsigned long host_cr4_mce = cr4_read_shadow() & X86_CR4_MCE;
2025 unsigned long old_cr4 = to_svm(vcpu)->vmcb->save.cr4;
2026
2027 if (cr4 & X86_CR4_VMXE)
2028 return 1;
2029
2030 if (npt_enabled && ((old_cr4 ^ cr4) & X86_CR4_PGE))
2031 svm_flush_tlb(vcpu);
2032
2033 vcpu->arch.cr4 = cr4;
2034 if (!npt_enabled)
2035 cr4 |= X86_CR4_PAE;
2036 cr4 |= host_cr4_mce;
2037 to_svm(vcpu)->vmcb->save.cr4 = cr4;
2038 mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR);
2039 return 0;
2040 }
2041
2042 static void svm_set_segment(struct kvm_vcpu *vcpu,
2043 struct kvm_segment *var, int seg)
2044 {
2045 struct vcpu_svm *svm = to_svm(vcpu);
2046 struct vmcb_seg *s = svm_seg(vcpu, seg);
2047
2048 s->base = var->base;
2049 s->limit = var->limit;
2050 s->selector = var->selector;
2051 s->attrib = (var->type & SVM_SELECTOR_TYPE_MASK);
2052 s->attrib |= (var->s & 1) << SVM_SELECTOR_S_SHIFT;
2053 s->attrib |= (var->dpl & 3) << SVM_SELECTOR_DPL_SHIFT;
2054 s->attrib |= ((var->present & 1) && !var->unusable) << SVM_SELECTOR_P_SHIFT;
2055 s->attrib |= (var->avl & 1) << SVM_SELECTOR_AVL_SHIFT;
2056 s->attrib |= (var->l & 1) << SVM_SELECTOR_L_SHIFT;
2057 s->attrib |= (var->db & 1) << SVM_SELECTOR_DB_SHIFT;
2058 s->attrib |= (var->g & 1) << SVM_SELECTOR_G_SHIFT;
2059
2060 /*
2061 * This is always accurate, except if SYSRET returned to a segment
2062 * with SS.DPL != 3. Intel does not have this quirk, and always
2063 * forces SS.DPL to 3 on sysret, so we ignore that case; fixing it
2064 * would entail passing the CPL to userspace and back.
2065 */
2066 if (seg == VCPU_SREG_SS)
2067 /* This is symmetric with svm_get_segment() */
2068 svm->vmcb->save.cpl = (var->dpl & 3);
2069
2070 mark_dirty(svm->vmcb, VMCB_SEG);
2071 }
2072
2073 static void update_bp_intercept(struct kvm_vcpu *vcpu)
2074 {
2075 struct vcpu_svm *svm = to_svm(vcpu);
2076
2077 clr_exception_intercept(svm, BP_VECTOR);
2078
2079 if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) {
2080 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
2081 set_exception_intercept(svm, BP_VECTOR);
2082 } else
2083 vcpu->guest_debug = 0;
2084 }
2085
2086 static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd)
2087 {
2088 if (sd->next_asid > sd->max_asid) {
2089 ++sd->asid_generation;
2090 sd->next_asid = 1;
2091 svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID;
2092 }
2093
2094 svm->asid_generation = sd->asid_generation;
2095 svm->vmcb->control.asid = sd->next_asid++;
2096
2097 mark_dirty(svm->vmcb, VMCB_ASID);
2098 }
2099
2100 static u64 svm_get_dr6(struct kvm_vcpu *vcpu)
2101 {
2102 return to_svm(vcpu)->vmcb->save.dr6;
2103 }
2104
2105 static void svm_set_dr6(struct kvm_vcpu *vcpu, unsigned long value)
2106 {
2107 struct vcpu_svm *svm = to_svm(vcpu);
2108
2109 svm->vmcb->save.dr6 = value;
2110 mark_dirty(svm->vmcb, VMCB_DR);
2111 }
2112
2113 static void svm_sync_dirty_debug_regs(struct kvm_vcpu *vcpu)
2114 {
2115 struct vcpu_svm *svm = to_svm(vcpu);
2116
2117 get_debugreg(vcpu->arch.db[0], 0);
2118 get_debugreg(vcpu->arch.db[1], 1);
2119 get_debugreg(vcpu->arch.db[2], 2);
2120 get_debugreg(vcpu->arch.db[3], 3);
2121 vcpu->arch.dr6 = svm_get_dr6(vcpu);
2122 vcpu->arch.dr7 = svm->vmcb->save.dr7;
2123
2124 vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_WONT_EXIT;
2125 set_dr_intercepts(svm);
2126 }
2127
2128 static void svm_set_dr7(struct kvm_vcpu *vcpu, unsigned long value)
2129 {
2130 struct vcpu_svm *svm = to_svm(vcpu);
2131
2132 svm->vmcb->save.dr7 = value;
2133 mark_dirty(svm->vmcb, VMCB_DR);
2134 }
2135
2136 static int pf_interception(struct vcpu_svm *svm)
2137 {
2138 u64 fault_address = svm->vmcb->control.exit_info_2;
2139 u64 error_code = svm->vmcb->control.exit_info_1;
2140
2141 return kvm_handle_page_fault(&svm->vcpu, error_code, fault_address,
2142 svm->vmcb->control.insn_bytes,
2143 svm->vmcb->control.insn_len, !npt_enabled);
2144 }
2145
2146 static int db_interception(struct vcpu_svm *svm)
2147 {
2148 struct kvm_run *kvm_run = svm->vcpu.run;
2149
2150 if (!(svm->vcpu.guest_debug &
2151 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) &&
2152 !svm->nmi_singlestep) {
2153 kvm_queue_exception(&svm->vcpu, DB_VECTOR);
2154 return 1;
2155 }
2156
2157 if (svm->nmi_singlestep) {
2158 disable_nmi_singlestep(svm);
2159 }
2160
2161 if (svm->vcpu.guest_debug &
2162 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) {
2163 kvm_run->exit_reason = KVM_EXIT_DEBUG;
2164 kvm_run->debug.arch.pc =
2165 svm->vmcb->save.cs.base + svm->vmcb->save.rip;
2166 kvm_run->debug.arch.exception = DB_VECTOR;
2167 return 0;
2168 }
2169
2170 return 1;
2171 }
2172
2173 static int bp_interception(struct vcpu_svm *svm)
2174 {
2175 struct kvm_run *kvm_run = svm->vcpu.run;
2176
2177 kvm_run->exit_reason = KVM_EXIT_DEBUG;
2178 kvm_run->debug.arch.pc = svm->vmcb->save.cs.base + svm->vmcb->save.rip;
2179 kvm_run->debug.arch.exception = BP_VECTOR;
2180 return 0;
2181 }
2182
2183 static int ud_interception(struct vcpu_svm *svm)
2184 {
2185 int er;
2186
2187 er = emulate_instruction(&svm->vcpu, EMULTYPE_TRAP_UD);
2188 if (er != EMULATE_DONE)
2189 kvm_queue_exception(&svm->vcpu, UD_VECTOR);
2190 return 1;
2191 }
2192
2193 static int ac_interception(struct vcpu_svm *svm)
2194 {
2195 kvm_queue_exception_e(&svm->vcpu, AC_VECTOR, 0);
2196 return 1;
2197 }
2198
2199 static bool is_erratum_383(void)
2200 {
2201 int err, i;
2202 u64 value;
2203
2204 if (!erratum_383_found)
2205 return false;
2206
2207 value = native_read_msr_safe(MSR_IA32_MC0_STATUS, &err);
2208 if (err)
2209 return false;
2210
2211 /* Bit 62 may or may not be set for this mce */
2212 value &= ~(1ULL << 62);
2213
2214 if (value != 0xb600000000010015ULL)
2215 return false;
2216
2217 /* Clear MCi_STATUS registers */
2218 for (i = 0; i < 6; ++i)
2219 native_write_msr_safe(MSR_IA32_MCx_STATUS(i), 0, 0);
2220
2221 value = native_read_msr_safe(MSR_IA32_MCG_STATUS, &err);
2222 if (!err) {
2223 u32 low, high;
2224
2225 value &= ~(1ULL << 2);
2226 low = lower_32_bits(value);
2227 high = upper_32_bits(value);
2228
2229 native_write_msr_safe(MSR_IA32_MCG_STATUS, low, high);
2230 }
2231
2232 /* Flush tlb to evict multi-match entries */
2233 __flush_tlb_all();
2234
2235 return true;
2236 }
2237
2238 static void svm_handle_mce(struct vcpu_svm *svm)
2239 {
2240 if (is_erratum_383()) {
2241 /*
2242 * Erratum 383 triggered. Guest state is corrupt so kill the
2243 * guest.
2244 */
2245 pr_err("KVM: Guest triggered AMD Erratum 383\n");
2246
2247 kvm_make_request(KVM_REQ_TRIPLE_FAULT, &svm->vcpu);
2248
2249 return;
2250 }
2251
2252 /*
2253 * On an #MC intercept the MCE handler is not called automatically in
2254 * the host. So do it by hand here.
2255 */
2256 asm volatile (
2257 "int $0x12\n");
2258 /* not sure if we ever come back to this point */
2259
2260 return;
2261 }
2262
2263 static int mc_interception(struct vcpu_svm *svm)
2264 {
2265 return 1;
2266 }
2267
2268 static int shutdown_interception(struct vcpu_svm *svm)
2269 {
2270 struct kvm_run *kvm_run = svm->vcpu.run;
2271
2272 /*
2273 * VMCB is undefined after a SHUTDOWN intercept
2274 * so reinitialize it.
2275 */
2276 clear_page(svm->vmcb);
2277 init_vmcb(svm);
2278
2279 kvm_run->exit_reason = KVM_EXIT_SHUTDOWN;
2280 return 0;
2281 }
2282
2283 static int io_interception(struct vcpu_svm *svm)
2284 {
2285 struct kvm_vcpu *vcpu = &svm->vcpu;
2286 u32 io_info = svm->vmcb->control.exit_info_1; /* address size bug? */
2287 int size, in, string, ret;
2288 unsigned port;
2289
2290 ++svm->vcpu.stat.io_exits;
2291 string = (io_info & SVM_IOIO_STR_MASK) != 0;
2292 in = (io_info & SVM_IOIO_TYPE_MASK) != 0;
2293 if (string)
2294 return emulate_instruction(vcpu, 0) == EMULATE_DONE;
2295
2296 port = io_info >> 16;
2297 size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT;
2298 svm->next_rip = svm->vmcb->control.exit_info_2;
2299 ret = kvm_skip_emulated_instruction(&svm->vcpu);
2300
2301 /*
2302 * TODO: we might be squashing a KVM_GUESTDBG_SINGLESTEP-triggered
2303 * KVM_EXIT_DEBUG here.
2304 */
2305 if (in)
2306 return kvm_fast_pio_in(vcpu, size, port) && ret;
2307 else
2308 return kvm_fast_pio_out(vcpu, size, port) && ret;
2309 }
2310
2311 static int nmi_interception(struct vcpu_svm *svm)
2312 {
2313 return 1;
2314 }
2315
2316 static int intr_interception(struct vcpu_svm *svm)
2317 {
2318 ++svm->vcpu.stat.irq_exits;
2319 return 1;
2320 }
2321
2322 static int nop_on_interception(struct vcpu_svm *svm)
2323 {
2324 return 1;
2325 }
2326
2327 static int halt_interception(struct vcpu_svm *svm)
2328 {
2329 svm->next_rip = kvm_rip_read(&svm->vcpu) + 1;
2330 return kvm_emulate_halt(&svm->vcpu);
2331 }
2332
2333 static int vmmcall_interception(struct vcpu_svm *svm)
2334 {
2335 svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
2336 return kvm_emulate_hypercall(&svm->vcpu);
2337 }
2338
2339 static unsigned long nested_svm_get_tdp_cr3(struct kvm_vcpu *vcpu)
2340 {
2341 struct vcpu_svm *svm = to_svm(vcpu);
2342
2343 return svm->nested.nested_cr3;
2344 }
2345
2346 static u64 nested_svm_get_tdp_pdptr(struct kvm_vcpu *vcpu, int index)
2347 {
2348 struct vcpu_svm *svm = to_svm(vcpu);
2349 u64 cr3 = svm->nested.nested_cr3;
2350 u64 pdpte;
2351 int ret;
2352
2353 ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(cr3), &pdpte,
2354 offset_in_page(cr3) + index * 8, 8);
2355 if (ret)
2356 return 0;
2357 return pdpte;
2358 }
2359
2360 static void nested_svm_set_tdp_cr3(struct kvm_vcpu *vcpu,
2361 unsigned long root)
2362 {
2363 struct vcpu_svm *svm = to_svm(vcpu);
2364
2365 svm->vmcb->control.nested_cr3 = root;
2366 mark_dirty(svm->vmcb, VMCB_NPT);
2367 svm_flush_tlb(vcpu);
2368 }
2369
2370 static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu,
2371 struct x86_exception *fault)
2372 {
2373 struct vcpu_svm *svm = to_svm(vcpu);
2374
2375 if (svm->vmcb->control.exit_code != SVM_EXIT_NPF) {
2376 /*
2377 * TODO: track the cause of the nested page fault, and
2378 * correctly fill in the high bits of exit_info_1.
2379 */
2380 svm->vmcb->control.exit_code = SVM_EXIT_NPF;
2381 svm->vmcb->control.exit_code_hi = 0;
2382 svm->vmcb->control.exit_info_1 = (1ULL << 32);
2383 svm->vmcb->control.exit_info_2 = fault->address;
2384 }
2385
2386 svm->vmcb->control.exit_info_1 &= ~0xffffffffULL;
2387 svm->vmcb->control.exit_info_1 |= fault->error_code;
2388
2389 /*
2390 * The present bit is always zero for page structure faults on real
2391 * hardware.
2392 */
2393 if (svm->vmcb->control.exit_info_1 & (2ULL << 32))
2394 svm->vmcb->control.exit_info_1 &= ~1;
2395
2396 nested_svm_vmexit(svm);
2397 }
2398
2399 static void nested_svm_init_mmu_context(struct kvm_vcpu *vcpu)
2400 {
2401 WARN_ON(mmu_is_nested(vcpu));
2402 kvm_init_shadow_mmu(vcpu);
2403 vcpu->arch.mmu.set_cr3 = nested_svm_set_tdp_cr3;
2404 vcpu->arch.mmu.get_cr3 = nested_svm_get_tdp_cr3;
2405 vcpu->arch.mmu.get_pdptr = nested_svm_get_tdp_pdptr;
2406 vcpu->arch.mmu.inject_page_fault = nested_svm_inject_npf_exit;
2407 vcpu->arch.mmu.shadow_root_level = get_npt_level();
2408 reset_shadow_zero_bits_mask(vcpu, &vcpu->arch.mmu);
2409 vcpu->arch.walk_mmu = &vcpu->arch.nested_mmu;
2410 }
2411
2412 static void nested_svm_uninit_mmu_context(struct kvm_vcpu *vcpu)
2413 {
2414 vcpu->arch.walk_mmu = &vcpu->arch.mmu;
2415 }
2416
2417 static int nested_svm_check_permissions(struct vcpu_svm *svm)
2418 {
2419 if (!(svm->vcpu.arch.efer & EFER_SVME) ||
2420 !is_paging(&svm->vcpu)) {
2421 kvm_queue_exception(&svm->vcpu, UD_VECTOR);
2422 return 1;
2423 }
2424
2425 if (svm->vmcb->save.cpl) {
2426 kvm_inject_gp(&svm->vcpu, 0);
2427 return 1;
2428 }
2429
2430 return 0;
2431 }
2432
2433 static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr,
2434 bool has_error_code, u32 error_code)
2435 {
2436 int vmexit;
2437
2438 if (!is_guest_mode(&svm->vcpu))
2439 return 0;
2440
2441 vmexit = nested_svm_intercept(svm);
2442 if (vmexit != NESTED_EXIT_DONE)
2443 return 0;
2444
2445 svm->vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + nr;
2446 svm->vmcb->control.exit_code_hi = 0;
2447 svm->vmcb->control.exit_info_1 = error_code;
2448
2449 /*
2450 * FIXME: we should not write CR2 when L1 intercepts an L2 #PF exception.
2451 * The fix is to add the ancillary datum (CR2 or DR6) to structs
2452 * kvm_queued_exception and kvm_vcpu_events, so that CR2 and DR6 can be
2453 * written only when inject_pending_event runs (DR6 would written here
2454 * too). This should be conditional on a new capability---if the
2455 * capability is disabled, kvm_multiple_exception would write the
2456 * ancillary information to CR2 or DR6, for backwards ABI-compatibility.
2457 */
2458 if (svm->vcpu.arch.exception.nested_apf)
2459 svm->vmcb->control.exit_info_2 = svm->vcpu.arch.apf.nested_apf_token;
2460 else
2461 svm->vmcb->control.exit_info_2 = svm->vcpu.arch.cr2;
2462
2463 svm->nested.exit_required = true;
2464 return vmexit;
2465 }
2466
2467 /* This function returns true if it is save to enable the irq window */
2468 static inline bool nested_svm_intr(struct vcpu_svm *svm)
2469 {
2470 if (!is_guest_mode(&svm->vcpu))
2471 return true;
2472
2473 if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK))
2474 return true;
2475
2476 if (!(svm->vcpu.arch.hflags & HF_HIF_MASK))
2477 return false;
2478
2479 /*
2480 * if vmexit was already requested (by intercepted exception
2481 * for instance) do not overwrite it with "external interrupt"
2482 * vmexit.
2483 */
2484 if (svm->nested.exit_required)
2485 return false;
2486
2487 svm->vmcb->control.exit_code = SVM_EXIT_INTR;
2488 svm->vmcb->control.exit_info_1 = 0;
2489 svm->vmcb->control.exit_info_2 = 0;
2490
2491 if (svm->nested.intercept & 1ULL) {
2492 /*
2493 * The #vmexit can't be emulated here directly because this
2494 * code path runs with irqs and preemption disabled. A
2495 * #vmexit emulation might sleep. Only signal request for
2496 * the #vmexit here.
2497 */
2498 svm->nested.exit_required = true;
2499 trace_kvm_nested_intr_vmexit(svm->vmcb->save.rip);
2500 return false;
2501 }
2502
2503 return true;
2504 }
2505
2506 /* This function returns true if it is save to enable the nmi window */
2507 static inline bool nested_svm_nmi(struct vcpu_svm *svm)
2508 {
2509 if (!is_guest_mode(&svm->vcpu))
2510 return true;
2511
2512 if (!(svm->nested.intercept & (1ULL << INTERCEPT_NMI)))
2513 return true;
2514
2515 svm->vmcb->control.exit_code = SVM_EXIT_NMI;
2516 svm->nested.exit_required = true;
2517
2518 return false;
2519 }
2520
2521 static void *nested_svm_map(struct vcpu_svm *svm, u64 gpa, struct page **_page)
2522 {
2523 struct page *page;
2524
2525 might_sleep();
2526
2527 page = kvm_vcpu_gfn_to_page(&svm->vcpu, gpa >> PAGE_SHIFT);
2528 if (is_error_page(page))
2529 goto error;
2530
2531 *_page = page;
2532
2533 return kmap(page);
2534
2535 error:
2536 kvm_inject_gp(&svm->vcpu, 0);
2537
2538 return NULL;
2539 }
2540
2541 static void nested_svm_unmap(struct page *page)
2542 {
2543 kunmap(page);
2544 kvm_release_page_dirty(page);
2545 }
2546
2547 static int nested_svm_intercept_ioio(struct vcpu_svm *svm)
2548 {
2549 unsigned port, size, iopm_len;
2550 u16 val, mask;
2551 u8 start_bit;
2552 u64 gpa;
2553
2554 if (!(svm->nested.intercept & (1ULL << INTERCEPT_IOIO_PROT)))
2555 return NESTED_EXIT_HOST;
2556
2557 port = svm->vmcb->control.exit_info_1 >> 16;
2558 size = (svm->vmcb->control.exit_info_1 & SVM_IOIO_SIZE_MASK) >>
2559 SVM_IOIO_SIZE_SHIFT;
2560 gpa = svm->nested.vmcb_iopm + (port / 8);
2561 start_bit = port % 8;
2562 iopm_len = (start_bit + size > 8) ? 2 : 1;
2563 mask = (0xf >> (4 - size)) << start_bit;
2564 val = 0;
2565
2566 if (kvm_vcpu_read_guest(&svm->vcpu, gpa, &val, iopm_len))
2567 return NESTED_EXIT_DONE;
2568
2569 return (val & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST;
2570 }
2571
2572 static int nested_svm_exit_handled_msr(struct vcpu_svm *svm)
2573 {
2574 u32 offset, msr, value;
2575 int write, mask;
2576
2577 if (!(svm->nested.intercept & (1ULL << INTERCEPT_MSR_PROT)))
2578 return NESTED_EXIT_HOST;
2579
2580 msr = svm->vcpu.arch.regs[VCPU_REGS_RCX];
2581 offset = svm_msrpm_offset(msr);
2582 write = svm->vmcb->control.exit_info_1 & 1;
2583 mask = 1 << ((2 * (msr & 0xf)) + write);
2584
2585 if (offset == MSR_INVALID)
2586 return NESTED_EXIT_DONE;
2587
2588 /* Offset is in 32 bit units but need in 8 bit units */
2589 offset *= 4;
2590
2591 if (kvm_vcpu_read_guest(&svm->vcpu, svm->nested.vmcb_msrpm + offset, &value, 4))
2592 return NESTED_EXIT_DONE;
2593
2594 return (value & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST;
2595 }
2596
2597 /* DB exceptions for our internal use must not cause vmexit */
2598 static int nested_svm_intercept_db(struct vcpu_svm *svm)
2599 {
2600 unsigned long dr6;
2601
2602 /* if we're not singlestepping, it's not ours */
2603 if (!svm->nmi_singlestep)
2604 return NESTED_EXIT_DONE;
2605
2606 /* if it's not a singlestep exception, it's not ours */
2607 if (kvm_get_dr(&svm->vcpu, 6, &dr6))
2608 return NESTED_EXIT_DONE;
2609 if (!(dr6 & DR6_BS))
2610 return NESTED_EXIT_DONE;
2611
2612 /* if the guest is singlestepping, it should get the vmexit */
2613 if (svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF) {
2614 disable_nmi_singlestep(svm);
2615 return NESTED_EXIT_DONE;
2616 }
2617
2618 /* it's ours, the nested hypervisor must not see this one */
2619 return NESTED_EXIT_HOST;
2620 }
2621
2622 static int nested_svm_exit_special(struct vcpu_svm *svm)
2623 {
2624 u32 exit_code = svm->vmcb->control.exit_code;
2625
2626 switch (exit_code) {
2627 case SVM_EXIT_INTR:
2628 case SVM_EXIT_NMI:
2629 case SVM_EXIT_EXCP_BASE + MC_VECTOR:
2630 return NESTED_EXIT_HOST;
2631 case SVM_EXIT_NPF:
2632 /* For now we are always handling NPFs when using them */
2633 if (npt_enabled)
2634 return NESTED_EXIT_HOST;
2635 break;
2636 case SVM_EXIT_EXCP_BASE + PF_VECTOR:
2637 /* When we're shadowing, trap PFs, but not async PF */
2638 if (!npt_enabled && svm->vcpu.arch.apf.host_apf_reason == 0)
2639 return NESTED_EXIT_HOST;
2640 break;
2641 default:
2642 break;
2643 }
2644
2645 return NESTED_EXIT_CONTINUE;
2646 }
2647
2648 /*
2649 * If this function returns true, this #vmexit was already handled
2650 */
2651 static int nested_svm_intercept(struct vcpu_svm *svm)
2652 {
2653 u32 exit_code = svm->vmcb->control.exit_code;
2654 int vmexit = NESTED_EXIT_HOST;
2655
2656 switch (exit_code) {
2657 case SVM_EXIT_MSR:
2658 vmexit = nested_svm_exit_handled_msr(svm);
2659 break;
2660 case SVM_EXIT_IOIO:
2661 vmexit = nested_svm_intercept_ioio(svm);
2662 break;
2663 case SVM_EXIT_READ_CR0 ... SVM_EXIT_WRITE_CR8: {
2664 u32 bit = 1U << (exit_code - SVM_EXIT_READ_CR0);
2665 if (svm->nested.intercept_cr & bit)
2666 vmexit = NESTED_EXIT_DONE;
2667 break;
2668 }
2669 case SVM_EXIT_READ_DR0 ... SVM_EXIT_WRITE_DR7: {
2670 u32 bit = 1U << (exit_code - SVM_EXIT_READ_DR0);
2671 if (svm->nested.intercept_dr & bit)
2672 vmexit = NESTED_EXIT_DONE;
2673 break;
2674 }
2675 case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: {
2676 u32 excp_bits = 1 << (exit_code - SVM_EXIT_EXCP_BASE);
2677 if (svm->nested.intercept_exceptions & excp_bits) {
2678 if (exit_code == SVM_EXIT_EXCP_BASE + DB_VECTOR)
2679 vmexit = nested_svm_intercept_db(svm);
2680 else
2681 vmexit = NESTED_EXIT_DONE;
2682 }
2683 /* async page fault always cause vmexit */
2684 else if ((exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR) &&
2685 svm->vcpu.arch.exception.nested_apf != 0)
2686 vmexit = NESTED_EXIT_DONE;
2687 break;
2688 }
2689 case SVM_EXIT_ERR: {
2690 vmexit = NESTED_EXIT_DONE;
2691 break;
2692 }
2693 default: {
2694 u64 exit_bits = 1ULL << (exit_code - SVM_EXIT_INTR);
2695 if (svm->nested.intercept & exit_bits)
2696 vmexit = NESTED_EXIT_DONE;
2697 }
2698 }
2699
2700 return vmexit;
2701 }
2702
2703 static int nested_svm_exit_handled(struct vcpu_svm *svm)
2704 {
2705 int vmexit;
2706
2707 vmexit = nested_svm_intercept(svm);
2708
2709 if (vmexit == NESTED_EXIT_DONE)
2710 nested_svm_vmexit(svm);
2711
2712 return vmexit;
2713 }
2714
2715 static inline void copy_vmcb_control_area(struct vmcb *dst_vmcb, struct vmcb *from_vmcb)
2716 {
2717 struct vmcb_control_area *dst = &dst_vmcb->control;
2718 struct vmcb_control_area *from = &from_vmcb->control;
2719
2720 dst->intercept_cr = from->intercept_cr;
2721 dst->intercept_dr = from->intercept_dr;
2722 dst->intercept_exceptions = from->intercept_exceptions;
2723 dst->intercept = from->intercept;
2724 dst->iopm_base_pa = from->iopm_base_pa;
2725 dst->msrpm_base_pa = from->msrpm_base_pa;
2726 dst->tsc_offset = from->tsc_offset;
2727 dst->asid = from->asid;
2728 dst->tlb_ctl = from->tlb_ctl;
2729 dst->int_ctl = from->int_ctl;
2730 dst->int_vector = from->int_vector;
2731 dst->int_state = from->int_state;
2732 dst->exit_code = from->exit_code;
2733 dst->exit_code_hi = from->exit_code_hi;
2734 dst->exit_info_1 = from->exit_info_1;
2735 dst->exit_info_2 = from->exit_info_2;
2736 dst->exit_int_info = from->exit_int_info;
2737 dst->exit_int_info_err = from->exit_int_info_err;
2738 dst->nested_ctl = from->nested_ctl;
2739 dst->event_inj = from->event_inj;
2740 dst->event_inj_err = from->event_inj_err;
2741 dst->nested_cr3 = from->nested_cr3;
2742 dst->virt_ext = from->virt_ext;
2743 }
2744
2745 static int nested_svm_vmexit(struct vcpu_svm *svm)
2746 {
2747 struct vmcb *nested_vmcb;
2748 struct vmcb *hsave = svm->nested.hsave;
2749 struct vmcb *vmcb = svm->vmcb;
2750 struct page *page;
2751
2752 trace_kvm_nested_vmexit_inject(vmcb->control.exit_code,
2753 vmcb->control.exit_info_1,
2754 vmcb->control.exit_info_2,
2755 vmcb->control.exit_int_info,
2756 vmcb->control.exit_int_info_err,
2757 KVM_ISA_SVM);
2758
2759 nested_vmcb = nested_svm_map(svm, svm->nested.vmcb, &page);
2760 if (!nested_vmcb)
2761 return 1;
2762
2763 /* Exit Guest-Mode */
2764 leave_guest_mode(&svm->vcpu);
2765 svm->nested.vmcb = 0;
2766
2767 /* Give the current vmcb to the guest */
2768 disable_gif(svm);
2769
2770 nested_vmcb->save.es = vmcb->save.es;
2771 nested_vmcb->save.cs = vmcb->save.cs;
2772 nested_vmcb->save.ss = vmcb->save.ss;
2773 nested_vmcb->save.ds = vmcb->save.ds;
2774 nested_vmcb->save.gdtr = vmcb->save.gdtr;
2775 nested_vmcb->save.idtr = vmcb->save.idtr;
2776 nested_vmcb->save.efer = svm->vcpu.arch.efer;
2777 nested_vmcb->save.cr0 = kvm_read_cr0(&svm->vcpu);
2778 nested_vmcb->save.cr3 = kvm_read_cr3(&svm->vcpu);
2779 nested_vmcb->save.cr2 = vmcb->save.cr2;
2780 nested_vmcb->save.cr4 = svm->vcpu.arch.cr4;
2781 nested_vmcb->save.rflags = kvm_get_rflags(&svm->vcpu);
2782 nested_vmcb->save.rip = vmcb->save.rip;
2783 nested_vmcb->save.rsp = vmcb->save.rsp;
2784 nested_vmcb->save.rax = vmcb->save.rax;
2785 nested_vmcb->save.dr7 = vmcb->save.dr7;
2786 nested_vmcb->save.dr6 = vmcb->save.dr6;
2787 nested_vmcb->save.cpl = vmcb->save.cpl;
2788
2789 nested_vmcb->control.int_ctl = vmcb->control.int_ctl;
2790 nested_vmcb->control.int_vector = vmcb->control.int_vector;
2791 nested_vmcb->control.int_state = vmcb->control.int_state;
2792 nested_vmcb->control.exit_code = vmcb->control.exit_code;
2793 nested_vmcb->control.exit_code_hi = vmcb->control.exit_code_hi;
2794 nested_vmcb->control.exit_info_1 = vmcb->control.exit_info_1;
2795 nested_vmcb->control.exit_info_2 = vmcb->control.exit_info_2;
2796 nested_vmcb->control.exit_int_info = vmcb->control.exit_int_info;
2797 nested_vmcb->control.exit_int_info_err = vmcb->control.exit_int_info_err;
2798
2799 if (svm->nrips_enabled)
2800 nested_vmcb->control.next_rip = vmcb->control.next_rip;
2801
2802 /*
2803 * If we emulate a VMRUN/#VMEXIT in the same host #vmexit cycle we have
2804 * to make sure that we do not lose injected events. So check event_inj
2805 * here and copy it to exit_int_info if it is valid.
2806 * Exit_int_info and event_inj can't be both valid because the case
2807 * below only happens on a VMRUN instruction intercept which has
2808 * no valid exit_int_info set.
2809 */
2810 if (vmcb->control.event_inj & SVM_EVTINJ_VALID) {
2811 struct vmcb_control_area *nc = &nested_vmcb->control;
2812
2813 nc->exit_int_info = vmcb->control.event_inj;
2814 nc->exit_int_info_err = vmcb->control.event_inj_err;
2815 }
2816
2817 nested_vmcb->control.tlb_ctl = 0;
2818 nested_vmcb->control.event_inj = 0;
2819 nested_vmcb->control.event_inj_err = 0;
2820
2821 /* We always set V_INTR_MASKING and remember the old value in hflags */
2822 if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK))
2823 nested_vmcb->control.int_ctl &= ~V_INTR_MASKING_MASK;
2824
2825 /* Restore the original control entries */
2826 copy_vmcb_control_area(vmcb, hsave);
2827
2828 kvm_clear_exception_queue(&svm->vcpu);
2829 kvm_clear_interrupt_queue(&svm->vcpu);
2830
2831 svm->nested.nested_cr3 = 0;
2832
2833 /* Restore selected save entries */
2834 svm->vmcb->save.es = hsave->save.es;
2835 svm->vmcb->save.cs = hsave->save.cs;
2836 svm->vmcb->save.ss = hsave->save.ss;
2837 svm->vmcb->save.ds = hsave->save.ds;
2838 svm->vmcb->save.gdtr = hsave->save.gdtr;
2839 svm->vmcb->save.idtr = hsave->save.idtr;
2840 kvm_set_rflags(&svm->vcpu, hsave->save.rflags);
2841 svm_set_efer(&svm->vcpu, hsave->save.efer);
2842 svm_set_cr0(&svm->vcpu, hsave->save.cr0 | X86_CR0_PE);
2843 svm_set_cr4(&svm->vcpu, hsave->save.cr4);
2844 if (npt_enabled) {
2845 svm->vmcb->save.cr3 = hsave->save.cr3;
2846 svm->vcpu.arch.cr3 = hsave->save.cr3;
2847 } else {
2848 (void)kvm_set_cr3(&svm->vcpu, hsave->save.cr3);
2849 }
2850 kvm_register_write(&svm->vcpu, VCPU_REGS_RAX, hsave->save.rax);
2851 kvm_register_write(&svm->vcpu, VCPU_REGS_RSP, hsave->save.rsp);
2852 kvm_register_write(&svm->vcpu, VCPU_REGS_RIP, hsave->save.rip);
2853 svm->vmcb->save.dr7 = 0;
2854 svm->vmcb->save.cpl = 0;
2855 svm->vmcb->control.exit_int_info = 0;
2856
2857 mark_all_dirty(svm->vmcb);
2858
2859 nested_svm_unmap(page);
2860
2861 nested_svm_uninit_mmu_context(&svm->vcpu);
2862 kvm_mmu_reset_context(&svm->vcpu);
2863 kvm_mmu_load(&svm->vcpu);
2864
2865 return 0;
2866 }
2867
2868 static bool nested_svm_vmrun_msrpm(struct vcpu_svm *svm)
2869 {
2870 /*
2871 * This function merges the msr permission bitmaps of kvm and the
2872 * nested vmcb. It is optimized in that it only merges the parts where
2873 * the kvm msr permission bitmap may contain zero bits
2874 */
2875 int i;
2876
2877 if (!(svm->nested.intercept & (1ULL << INTERCEPT_MSR_PROT)))
2878 return true;
2879
2880 for (i = 0; i < MSRPM_OFFSETS; i++) {
2881 u32 value, p;
2882 u64 offset;
2883
2884 if (msrpm_offsets[i] == 0xffffffff)
2885 break;
2886
2887 p = msrpm_offsets[i];
2888 offset = svm->nested.vmcb_msrpm + (p * 4);
2889
2890 if (kvm_vcpu_read_guest(&svm->vcpu, offset, &value, 4))
2891 return false;
2892
2893 svm->nested.msrpm[p] = svm->msrpm[p] | value;
2894 }
2895
2896 svm->vmcb->control.msrpm_base_pa = __pa(svm->nested.msrpm);
2897
2898 return true;
2899 }
2900
2901 static bool nested_vmcb_checks(struct vmcb *vmcb)
2902 {
2903 if ((vmcb->control.intercept & (1ULL << INTERCEPT_VMRUN)) == 0)
2904 return false;
2905
2906 if (vmcb->control.asid == 0)
2907 return false;
2908
2909 if (vmcb->control.nested_ctl && !npt_enabled)
2910 return false;
2911
2912 return true;
2913 }
2914
2915 static bool nested_svm_vmrun(struct vcpu_svm *svm)
2916 {
2917 struct vmcb *nested_vmcb;
2918 struct vmcb *hsave = svm->nested.hsave;
2919 struct vmcb *vmcb = svm->vmcb;
2920 struct page *page;
2921 u64 vmcb_gpa;
2922
2923 vmcb_gpa = svm->vmcb->save.rax;
2924
2925 nested_vmcb = nested_svm_map(svm, svm->vmcb->save.rax, &page);
2926 if (!nested_vmcb)
2927 return false;
2928
2929 if (!nested_vmcb_checks(nested_vmcb)) {
2930 nested_vmcb->control.exit_code = SVM_EXIT_ERR;
2931 nested_vmcb->control.exit_code_hi = 0;
2932 nested_vmcb->control.exit_info_1 = 0;
2933 nested_vmcb->control.exit_info_2 = 0;
2934
2935 nested_svm_unmap(page);
2936
2937 return false;
2938 }
2939
2940 trace_kvm_nested_vmrun(svm->vmcb->save.rip, vmcb_gpa,
2941 nested_vmcb->save.rip,
2942 nested_vmcb->control.int_ctl,
2943 nested_vmcb->control.event_inj,
2944 nested_vmcb->control.nested_ctl);
2945
2946 trace_kvm_nested_intercepts(nested_vmcb->control.intercept_cr & 0xffff,
2947 nested_vmcb->control.intercept_cr >> 16,
2948 nested_vmcb->control.intercept_exceptions,
2949 nested_vmcb->control.intercept);
2950
2951 /* Clear internal status */
2952 kvm_clear_exception_queue(&svm->vcpu);
2953 kvm_clear_interrupt_queue(&svm->vcpu);
2954
2955 /*
2956 * Save the old vmcb, so we don't need to pick what we save, but can
2957 * restore everything when a VMEXIT occurs
2958 */
2959 hsave->save.es = vmcb->save.es;
2960 hsave->save.cs = vmcb->save.cs;
2961 hsave->save.ss = vmcb->save.ss;
2962 hsave->save.ds = vmcb->save.ds;
2963 hsave->save.gdtr = vmcb->save.gdtr;
2964 hsave->save.idtr = vmcb->save.idtr;
2965 hsave->save.efer = svm->vcpu.arch.efer;
2966 hsave->save.cr0 = kvm_read_cr0(&svm->vcpu);
2967 hsave->save.cr4 = svm->vcpu.arch.cr4;
2968 hsave->save.rflags = kvm_get_rflags(&svm->vcpu);
2969 hsave->save.rip = kvm_rip_read(&svm->vcpu);
2970 hsave->save.rsp = vmcb->save.rsp;
2971 hsave->save.rax = vmcb->save.rax;
2972 if (npt_enabled)
2973 hsave->save.cr3 = vmcb->save.cr3;
2974 else
2975 hsave->save.cr3 = kvm_read_cr3(&svm->vcpu);
2976
2977 copy_vmcb_control_area(hsave, vmcb);
2978
2979 if (kvm_get_rflags(&svm->vcpu) & X86_EFLAGS_IF)
2980 svm->vcpu.arch.hflags |= HF_HIF_MASK;
2981 else
2982 svm->vcpu.arch.hflags &= ~HF_HIF_MASK;
2983
2984 if (nested_vmcb->control.nested_ctl) {
2985 kvm_mmu_unload(&svm->vcpu);
2986 svm->nested.nested_cr3 = nested_vmcb->control.nested_cr3;
2987 nested_svm_init_mmu_context(&svm->vcpu);
2988 }
2989
2990 /* Load the nested guest state */
2991 svm->vmcb->save.es = nested_vmcb->save.es;
2992 svm->vmcb->save.cs = nested_vmcb->save.cs;
2993 svm->vmcb->save.ss = nested_vmcb->save.ss;
2994 svm->vmcb->save.ds = nested_vmcb->save.ds;
2995 svm->vmcb->save.gdtr = nested_vmcb->save.gdtr;
2996 svm->vmcb->save.idtr = nested_vmcb->save.idtr;
2997 kvm_set_rflags(&svm->vcpu, nested_vmcb->save.rflags);
2998 svm_set_efer(&svm->vcpu, nested_vmcb->save.efer);
2999 svm_set_cr0(&svm->vcpu, nested_vmcb->save.cr0);
3000 svm_set_cr4(&svm->vcpu, nested_vmcb->save.cr4);
3001 if (npt_enabled) {
3002 svm->vmcb->save.cr3 = nested_vmcb->save.cr3;
3003 svm->vcpu.arch.cr3 = nested_vmcb->save.cr3;
3004 } else
3005 (void)kvm_set_cr3(&svm->vcpu, nested_vmcb->save.cr3);
3006
3007 /* Guest paging mode is active - reset mmu */
3008 kvm_mmu_reset_context(&svm->vcpu);
3009
3010 svm->vmcb->save.cr2 = svm->vcpu.arch.cr2 = nested_vmcb->save.cr2;
3011 kvm_register_write(&svm->vcpu, VCPU_REGS_RAX, nested_vmcb->save.rax);
3012 kvm_register_write(&svm->vcpu, VCPU_REGS_RSP, nested_vmcb->save.rsp);
3013 kvm_register_write(&svm->vcpu, VCPU_REGS_RIP, nested_vmcb->save.rip);
3014
3015 /* In case we don't even reach vcpu_run, the fields are not updated */
3016 svm->vmcb->save.rax = nested_vmcb->save.rax;
3017 svm->vmcb->save.rsp = nested_vmcb->save.rsp;
3018 svm->vmcb->save.rip = nested_vmcb->save.rip;
3019 svm->vmcb->save.dr7 = nested_vmcb->save.dr7;
3020 svm->vmcb->save.dr6 = nested_vmcb->save.dr6;
3021 svm->vmcb->save.cpl = nested_vmcb->save.cpl;
3022
3023 svm->nested.vmcb_msrpm = nested_vmcb->control.msrpm_base_pa & ~0x0fffULL;
3024 svm->nested.vmcb_iopm = nested_vmcb->control.iopm_base_pa & ~0x0fffULL;
3025
3026 /* cache intercepts */
3027 svm->nested.intercept_cr = nested_vmcb->control.intercept_cr;
3028 svm->nested.intercept_dr = nested_vmcb->control.intercept_dr;
3029 svm->nested.intercept_exceptions = nested_vmcb->control.intercept_exceptions;
3030 svm->nested.intercept = nested_vmcb->control.intercept;
3031
3032 svm_flush_tlb(&svm->vcpu);
3033 svm->vmcb->control.int_ctl = nested_vmcb->control.int_ctl | V_INTR_MASKING_MASK;
3034 if (nested_vmcb->control.int_ctl & V_INTR_MASKING_MASK)
3035 svm->vcpu.arch.hflags |= HF_VINTR_MASK;
3036 else
3037 svm->vcpu.arch.hflags &= ~HF_VINTR_MASK;
3038
3039 if (svm->vcpu.arch.hflags & HF_VINTR_MASK) {
3040 /* We only want the cr8 intercept bits of the guest */
3041 clr_cr_intercept(svm, INTERCEPT_CR8_READ);
3042 clr_cr_intercept(svm, INTERCEPT_CR8_WRITE);
3043 }
3044
3045 /* We don't want to see VMMCALLs from a nested guest */
3046 clr_intercept(svm, INTERCEPT_VMMCALL);
3047
3048 svm->vmcb->control.virt_ext = nested_vmcb->control.virt_ext;
3049 svm->vmcb->control.int_vector = nested_vmcb->control.int_vector;
3050 svm->vmcb->control.int_state = nested_vmcb->control.int_state;
3051 svm->vmcb->control.tsc_offset += nested_vmcb->control.tsc_offset;
3052 svm->vmcb->control.event_inj = nested_vmcb->control.event_inj;
3053 svm->vmcb->control.event_inj_err = nested_vmcb->control.event_inj_err;
3054
3055 nested_svm_unmap(page);
3056
3057 /* Enter Guest-Mode */
3058 enter_guest_mode(&svm->vcpu);
3059
3060 /*
3061 * Merge guest and host intercepts - must be called with vcpu in
3062 * guest-mode to take affect here
3063 */
3064 recalc_intercepts(svm);
3065
3066 svm->nested.vmcb = vmcb_gpa;
3067
3068 enable_gif(svm);
3069
3070 mark_all_dirty(svm->vmcb);
3071
3072 return true;
3073 }
3074
3075 static void nested_svm_vmloadsave(struct vmcb *from_vmcb, struct vmcb *to_vmcb)
3076 {
3077 to_vmcb->save.fs = from_vmcb->save.fs;
3078 to_vmcb->save.gs = from_vmcb->save.gs;
3079 to_vmcb->save.tr = from_vmcb->save.tr;
3080 to_vmcb->save.ldtr = from_vmcb->save.ldtr;
3081 to_vmcb->save.kernel_gs_base = from_vmcb->save.kernel_gs_base;
3082 to_vmcb->save.star = from_vmcb->save.star;
3083 to_vmcb->save.lstar = from_vmcb->save.lstar;
3084 to_vmcb->save.cstar = from_vmcb->save.cstar;
3085 to_vmcb->save.sfmask = from_vmcb->save.sfmask;
3086 to_vmcb->save.sysenter_cs = from_vmcb->save.sysenter_cs;
3087 to_vmcb->save.sysenter_esp = from_vmcb->save.sysenter_esp;
3088 to_vmcb->save.sysenter_eip = from_vmcb->save.sysenter_eip;
3089 }
3090
3091 static int vmload_interception(struct vcpu_svm *svm)
3092 {
3093 struct vmcb *nested_vmcb;
3094 struct page *page;
3095 int ret;
3096
3097 if (nested_svm_check_permissions(svm))
3098 return 1;
3099
3100 nested_vmcb = nested_svm_map(svm, svm->vmcb->save.rax, &page);
3101 if (!nested_vmcb)
3102 return 1;
3103
3104 svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
3105 ret = kvm_skip_emulated_instruction(&svm->vcpu);
3106
3107 nested_svm_vmloadsave(nested_vmcb, svm->vmcb);
3108 nested_svm_unmap(page);
3109
3110 return ret;
3111 }
3112
3113 static int vmsave_interception(struct vcpu_svm *svm)
3114 {
3115 struct vmcb *nested_vmcb;
3116 struct page *page;
3117 int ret;
3118
3119 if (nested_svm_check_permissions(svm))
3120 return 1;
3121
3122 nested_vmcb = nested_svm_map(svm, svm->vmcb->save.rax, &page);
3123 if (!nested_vmcb)
3124 return 1;
3125
3126 svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
3127 ret = kvm_skip_emulated_instruction(&svm->vcpu);
3128
3129 nested_svm_vmloadsave(svm->vmcb, nested_vmcb);
3130 nested_svm_unmap(page);
3131
3132 return ret;
3133 }
3134
3135 static int vmrun_interception(struct vcpu_svm *svm)
3136 {
3137 if (nested_svm_check_permissions(svm))
3138 return 1;
3139
3140 /* Save rip after vmrun instruction */
3141 kvm_rip_write(&svm->vcpu, kvm_rip_read(&svm->vcpu) + 3);
3142
3143 if (!nested_svm_vmrun(svm))
3144 return 1;
3145
3146 if (!nested_svm_vmrun_msrpm(svm))
3147 goto failed;
3148
3149 return 1;
3150
3151 failed:
3152
3153 svm->vmcb->control.exit_code = SVM_EXIT_ERR;
3154 svm->vmcb->control.exit_code_hi = 0;
3155 svm->vmcb->control.exit_info_1 = 0;
3156 svm->vmcb->control.exit_info_2 = 0;
3157
3158 nested_svm_vmexit(svm);
3159
3160 return 1;
3161 }
3162
3163 static int stgi_interception(struct vcpu_svm *svm)
3164 {
3165 int ret;
3166
3167 if (nested_svm_check_permissions(svm))
3168 return 1;
3169
3170 svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
3171 ret = kvm_skip_emulated_instruction(&svm->vcpu);
3172 kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
3173
3174 enable_gif(svm);
3175
3176 return ret;
3177 }
3178
3179 static int clgi_interception(struct vcpu_svm *svm)
3180 {
3181 int ret;
3182
3183 if (nested_svm_check_permissions(svm))
3184 return 1;
3185
3186 svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
3187 ret = kvm_skip_emulated_instruction(&svm->vcpu);
3188
3189 disable_gif(svm);
3190
3191 /* After a CLGI no interrupts should come */
3192 if (!kvm_vcpu_apicv_active(&svm->vcpu)) {
3193 svm_clear_vintr(svm);
3194 svm->vmcb->control.int_ctl &= ~V_IRQ_MASK;
3195 mark_dirty(svm->vmcb, VMCB_INTR);
3196 }
3197
3198 return ret;
3199 }
3200
3201 static int invlpga_interception(struct vcpu_svm *svm)
3202 {
3203 struct kvm_vcpu *vcpu = &svm->vcpu;
3204
3205 trace_kvm_invlpga(svm->vmcb->save.rip, kvm_register_read(&svm->vcpu, VCPU_REGS_RCX),
3206 kvm_register_read(&svm->vcpu, VCPU_REGS_RAX));
3207
3208 /* Let's treat INVLPGA the same as INVLPG (can be optimized!) */
3209 kvm_mmu_invlpg(vcpu, kvm_register_read(&svm->vcpu, VCPU_REGS_RAX));
3210
3211 svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
3212 return kvm_skip_emulated_instruction(&svm->vcpu);
3213 }
3214
3215 static int skinit_interception(struct vcpu_svm *svm)
3216 {
3217 trace_kvm_skinit(svm->vmcb->save.rip, kvm_register_read(&svm->vcpu, VCPU_REGS_RAX));
3218
3219 kvm_queue_exception(&svm->vcpu, UD_VECTOR);
3220 return 1;
3221 }
3222
3223 static int wbinvd_interception(struct vcpu_svm *svm)
3224 {
3225 return kvm_emulate_wbinvd(&svm->vcpu);
3226 }
3227
3228 static int xsetbv_interception(struct vcpu_svm *svm)
3229 {
3230 u64 new_bv = kvm_read_edx_eax(&svm->vcpu);
3231 u32 index = kvm_register_read(&svm->vcpu, VCPU_REGS_RCX);
3232
3233 if (kvm_set_xcr(&svm->vcpu, index, new_bv) == 0) {
3234 svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
3235 return kvm_skip_emulated_instruction(&svm->vcpu);
3236 }
3237
3238 return 1;
3239 }
3240
3241 static int task_switch_interception(struct vcpu_svm *svm)
3242 {
3243 u16 tss_selector;
3244 int reason;
3245 int int_type = svm->vmcb->control.exit_int_info &
3246 SVM_EXITINTINFO_TYPE_MASK;
3247 int int_vec = svm->vmcb->control.exit_int_info & SVM_EVTINJ_VEC_MASK;
3248 uint32_t type =
3249 svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_TYPE_MASK;
3250 uint32_t idt_v =
3251 svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_VALID;
3252 bool has_error_code = false;
3253 u32 error_code = 0;
3254
3255 tss_selector = (u16)svm->vmcb->control.exit_info_1;
3256
3257 if (svm->vmcb->control.exit_info_2 &
3258 (1ULL << SVM_EXITINFOSHIFT_TS_REASON_IRET))
3259 reason = TASK_SWITCH_IRET;
3260 else if (svm->vmcb->control.exit_info_2 &
3261 (1ULL << SVM_EXITINFOSHIFT_TS_REASON_JMP))
3262 reason = TASK_SWITCH_JMP;
3263 else if (idt_v)
3264 reason = TASK_SWITCH_GATE;
3265 else
3266 reason = TASK_SWITCH_CALL;
3267
3268 if (reason == TASK_SWITCH_GATE) {
3269 switch (type) {
3270 case SVM_EXITINTINFO_TYPE_NMI:
3271 svm->vcpu.arch.nmi_injected = false;
3272 break;
3273 case SVM_EXITINTINFO_TYPE_EXEPT:
3274 if (svm->vmcb->control.exit_info_2 &
3275 (1ULL << SVM_EXITINFOSHIFT_TS_HAS_ERROR_CODE)) {
3276 has_error_code = true;
3277 error_code =
3278 (u32)svm->vmcb->control.exit_info_2;
3279 }
3280 kvm_clear_exception_queue(&svm->vcpu);
3281 break;
3282 case SVM_EXITINTINFO_TYPE_INTR:
3283 kvm_clear_interrupt_queue(&svm->vcpu);
3284 break;
3285 default:
3286 break;
3287 }
3288 }
3289
3290 if (reason != TASK_SWITCH_GATE ||
3291 int_type == SVM_EXITINTINFO_TYPE_SOFT ||
3292 (int_type == SVM_EXITINTINFO_TYPE_EXEPT &&
3293 (int_vec == OF_VECTOR || int_vec == BP_VECTOR)))
3294 skip_emulated_instruction(&svm->vcpu);
3295
3296 if (int_type != SVM_EXITINTINFO_TYPE_SOFT)
3297 int_vec = -1;
3298
3299 if (kvm_task_switch(&svm->vcpu, tss_selector, int_vec, reason,
3300 has_error_code, error_code) == EMULATE_FAIL) {
3301 svm->vcpu.run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
3302 svm->vcpu.run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
3303 svm->vcpu.run->internal.ndata = 0;
3304 return 0;
3305 }
3306 return 1;
3307 }
3308
3309 static int cpuid_interception(struct vcpu_svm *svm)
3310 {
3311 svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
3312 return kvm_emulate_cpuid(&svm->vcpu);
3313 }
3314
3315 static int iret_interception(struct vcpu_svm *svm)
3316 {
3317 ++svm->vcpu.stat.nmi_window_exits;
3318 clr_intercept(svm, INTERCEPT_IRET);
3319 svm->vcpu.arch.hflags |= HF_IRET_MASK;
3320 svm->nmi_iret_rip = kvm_rip_read(&svm->vcpu);
3321 kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
3322 return 1;
3323 }
3324
3325 static int invlpg_interception(struct vcpu_svm *svm)
3326 {
3327 if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
3328 return emulate_instruction(&svm->vcpu, 0) == EMULATE_DONE;
3329
3330 kvm_mmu_invlpg(&svm->vcpu, svm->vmcb->control.exit_info_1);
3331 return kvm_skip_emulated_instruction(&svm->vcpu);
3332 }
3333
3334 static int emulate_on_interception(struct vcpu_svm *svm)
3335 {
3336 return emulate_instruction(&svm->vcpu, 0) == EMULATE_DONE;
3337 }
3338
3339 static int rdpmc_interception(struct vcpu_svm *svm)
3340 {
3341 int err;
3342
3343 if (!static_cpu_has(X86_FEATURE_NRIPS))
3344 return emulate_on_interception(svm);
3345
3346 err = kvm_rdpmc(&svm->vcpu);
3347 return kvm_complete_insn_gp(&svm->vcpu, err);
3348 }
3349
3350 static bool check_selective_cr0_intercepted(struct vcpu_svm *svm,
3351 unsigned long val)
3352 {
3353 unsigned long cr0 = svm->vcpu.arch.cr0;
3354 bool ret = false;
3355 u64 intercept;
3356
3357 intercept = svm->nested.intercept;
3358
3359 if (!is_guest_mode(&svm->vcpu) ||
3360 (!(intercept & (1ULL << INTERCEPT_SELECTIVE_CR0))))
3361 return false;
3362
3363 cr0 &= ~SVM_CR0_SELECTIVE_MASK;
3364 val &= ~SVM_CR0_SELECTIVE_MASK;
3365
3366 if (cr0 ^ val) {
3367 svm->vmcb->control.exit_code = SVM_EXIT_CR0_SEL_WRITE;
3368 ret = (nested_svm_exit_handled(svm) == NESTED_EXIT_DONE);
3369 }
3370
3371 return ret;
3372 }
3373
3374 #define CR_VALID (1ULL << 63)
3375
3376 static int cr_interception(struct vcpu_svm *svm)
3377 {
3378 int reg, cr;
3379 unsigned long val;
3380 int err;
3381
3382 if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
3383 return emulate_on_interception(svm);
3384
3385 if (unlikely((svm->vmcb->control.exit_info_1 & CR_VALID) == 0))
3386 return emulate_on_interception(svm);
3387
3388 reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
3389 if (svm->vmcb->control.exit_code == SVM_EXIT_CR0_SEL_WRITE)
3390 cr = SVM_EXIT_WRITE_CR0 - SVM_EXIT_READ_CR0;
3391 else
3392 cr = svm->vmcb->control.exit_code - SVM_EXIT_READ_CR0;
3393
3394 err = 0;
3395 if (cr >= 16) { /* mov to cr */
3396 cr -= 16;
3397 val = kvm_register_read(&svm->vcpu, reg);
3398 switch (cr) {
3399 case 0:
3400 if (!check_selective_cr0_intercepted(svm, val))
3401 err = kvm_set_cr0(&svm->vcpu, val);
3402 else
3403 return 1;
3404
3405 break;
3406 case 3:
3407 err = kvm_set_cr3(&svm->vcpu, val);
3408 break;
3409 case 4:
3410 err = kvm_set_cr4(&svm->vcpu, val);
3411 break;
3412 case 8:
3413 err = kvm_set_cr8(&svm->vcpu, val);
3414 break;
3415 default:
3416 WARN(1, "unhandled write to CR%d", cr);
3417 kvm_queue_exception(&svm->vcpu, UD_VECTOR);
3418 return 1;
3419 }
3420 } else { /* mov from cr */
3421 switch (cr) {
3422 case 0:
3423 val = kvm_read_cr0(&svm->vcpu);
3424 break;
3425 case 2:
3426 val = svm->vcpu.arch.cr2;
3427 break;
3428 case 3:
3429 val = kvm_read_cr3(&svm->vcpu);
3430 break;
3431 case 4:
3432 val = kvm_read_cr4(&svm->vcpu);
3433 break;
3434 case 8:
3435 val = kvm_get_cr8(&svm->vcpu);
3436 break;
3437 default:
3438 WARN(1, "unhandled read from CR%d", cr);
3439 kvm_queue_exception(&svm->vcpu, UD_VECTOR);
3440 return 1;
3441 }
3442 kvm_register_write(&svm->vcpu, reg, val);
3443 }
3444 return kvm_complete_insn_gp(&svm->vcpu, err);
3445 }
3446
3447 static int dr_interception(struct vcpu_svm *svm)
3448 {
3449 int reg, dr;
3450 unsigned long val;
3451
3452 if (svm->vcpu.guest_debug == 0) {
3453 /*
3454 * No more DR vmexits; force a reload of the debug registers
3455 * and reenter on this instruction. The next vmexit will
3456 * retrieve the full state of the debug registers.
3457 */
3458 clr_dr_intercepts(svm);
3459 svm->vcpu.arch.switch_db_regs |= KVM_DEBUGREG_WONT_EXIT;
3460 return 1;
3461 }
3462
3463 if (!boot_cpu_has(X86_FEATURE_DECODEASSISTS))
3464 return emulate_on_interception(svm);
3465
3466 reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
3467 dr = svm->vmcb->control.exit_code - SVM_EXIT_READ_DR0;
3468
3469 if (dr >= 16) { /* mov to DRn */
3470 if (!kvm_require_dr(&svm->vcpu, dr - 16))
3471 return 1;
3472 val = kvm_register_read(&svm->vcpu, reg);
3473 kvm_set_dr(&svm->vcpu, dr - 16, val);
3474 } else {
3475 if (!kvm_require_dr(&svm->vcpu, dr))
3476 return 1;
3477 kvm_get_dr(&svm->vcpu, dr, &val);
3478 kvm_register_write(&svm->vcpu, reg, val);
3479 }
3480
3481 return kvm_skip_emulated_instruction(&svm->vcpu);
3482 }
3483
3484 static int cr8_write_interception(struct vcpu_svm *svm)
3485 {
3486 struct kvm_run *kvm_run = svm->vcpu.run;
3487 int r;
3488
3489 u8 cr8_prev = kvm_get_cr8(&svm->vcpu);
3490 /* instruction emulation calls kvm_set_cr8() */
3491 r = cr_interception(svm);
3492 if (lapic_in_kernel(&svm->vcpu))
3493 return r;
3494 if (cr8_prev <= kvm_get_cr8(&svm->vcpu))
3495 return r;
3496 kvm_run->exit_reason = KVM_EXIT_SET_TPR;
3497 return 0;
3498 }
3499
3500 static int svm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
3501 {
3502 struct vcpu_svm *svm = to_svm(vcpu);
3503
3504 switch (msr_info->index) {
3505 case MSR_IA32_TSC: {
3506 msr_info->data = svm->vmcb->control.tsc_offset +
3507 kvm_scale_tsc(vcpu, rdtsc());
3508
3509 break;
3510 }
3511 case MSR_STAR:
3512 msr_info->data = svm->vmcb->save.star;
3513 break;
3514 #ifdef CONFIG_X86_64
3515 case MSR_LSTAR:
3516 msr_info->data = svm->vmcb->save.lstar;
3517 break;
3518 case MSR_CSTAR:
3519 msr_info->data = svm->vmcb->save.cstar;
3520 break;
3521 case MSR_KERNEL_GS_BASE:
3522 msr_info->data = svm->vmcb->save.kernel_gs_base;
3523 break;
3524 case MSR_SYSCALL_MASK:
3525 msr_info->data = svm->vmcb->save.sfmask;
3526 break;
3527 #endif
3528 case MSR_IA32_SYSENTER_CS:
3529 msr_info->data = svm->vmcb->save.sysenter_cs;
3530 break;
3531 case MSR_IA32_SYSENTER_EIP:
3532 msr_info->data = svm->sysenter_eip;
3533 break;
3534 case MSR_IA32_SYSENTER_ESP:
3535 msr_info->data = svm->sysenter_esp;
3536 break;
3537 case MSR_TSC_AUX:
3538 if (!boot_cpu_has(X86_FEATURE_RDTSCP))
3539 return 1;
3540 msr_info->data = svm->tsc_aux;
3541 break;
3542 /*
3543 * Nobody will change the following 5 values in the VMCB so we can
3544 * safely return them on rdmsr. They will always be 0 until LBRV is
3545 * implemented.
3546 */
3547 case MSR_IA32_DEBUGCTLMSR:
3548 msr_info->data = svm->vmcb->save.dbgctl;
3549 break;
3550 case MSR_IA32_LASTBRANCHFROMIP:
3551 msr_info->data = svm->vmcb->save.br_from;
3552 break;
3553 case MSR_IA32_LASTBRANCHTOIP:
3554 msr_info->data = svm->vmcb->save.br_to;
3555 break;
3556 case MSR_IA32_LASTINTFROMIP:
3557 msr_info->data = svm->vmcb->save.last_excp_from;
3558 break;
3559 case MSR_IA32_LASTINTTOIP:
3560 msr_info->data = svm->vmcb->save.last_excp_to;
3561 break;
3562 case MSR_VM_HSAVE_PA:
3563 msr_info->data = svm->nested.hsave_msr;
3564 break;
3565 case MSR_VM_CR:
3566 msr_info->data = svm->nested.vm_cr_msr;
3567 break;
3568 case MSR_IA32_SPEC_CTRL:
3569 msr_info->data = svm->spec_ctrl;
3570 break;
3571 case MSR_IA32_UCODE_REV:
3572 msr_info->data = 0x01000065;
3573 break;
3574 case MSR_F15H_IC_CFG: {
3575
3576 int family, model;
3577
3578 family = guest_cpuid_family(vcpu);
3579 model = guest_cpuid_model(vcpu);
3580
3581 if (family < 0 || model < 0)
3582 return kvm_get_msr_common(vcpu, msr_info);
3583
3584 msr_info->data = 0;
3585
3586 if (family == 0x15 &&
3587 (model >= 0x2 && model < 0x20))
3588 msr_info->data = 0x1E;
3589 }
3590 break;
3591 default:
3592 return kvm_get_msr_common(vcpu, msr_info);
3593 }
3594 return 0;
3595 }
3596
3597 static int rdmsr_interception(struct vcpu_svm *svm)
3598 {
3599 u32 ecx = kvm_register_read(&svm->vcpu, VCPU_REGS_RCX);
3600 struct msr_data msr_info;
3601
3602 msr_info.index = ecx;
3603 msr_info.host_initiated = false;
3604 if (svm_get_msr(&svm->vcpu, &msr_info)) {
3605 trace_kvm_msr_read_ex(ecx);
3606 kvm_inject_gp(&svm->vcpu, 0);
3607 return 1;
3608 } else {
3609 trace_kvm_msr_read(ecx, msr_info.data);
3610
3611 kvm_register_write(&svm->vcpu, VCPU_REGS_RAX,
3612 msr_info.data & 0xffffffff);
3613 kvm_register_write(&svm->vcpu, VCPU_REGS_RDX,
3614 msr_info.data >> 32);
3615 svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
3616 return kvm_skip_emulated_instruction(&svm->vcpu);
3617 }
3618 }
3619
3620 static int svm_set_vm_cr(struct kvm_vcpu *vcpu, u64 data)
3621 {
3622 struct vcpu_svm *svm = to_svm(vcpu);
3623 int svm_dis, chg_mask;
3624
3625 if (data & ~SVM_VM_CR_VALID_MASK)
3626 return 1;
3627
3628 chg_mask = SVM_VM_CR_VALID_MASK;
3629
3630 if (svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK)
3631 chg_mask &= ~(SVM_VM_CR_SVM_LOCK_MASK | SVM_VM_CR_SVM_DIS_MASK);
3632
3633 svm->nested.vm_cr_msr &= ~chg_mask;
3634 svm->nested.vm_cr_msr |= (data & chg_mask);
3635
3636 svm_dis = svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK;
3637
3638 /* check for svm_disable while efer.svme is set */
3639 if (svm_dis && (vcpu->arch.efer & EFER_SVME))
3640 return 1;
3641
3642 return 0;
3643 }
3644
3645 static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr)
3646 {
3647 struct vcpu_svm *svm = to_svm(vcpu);
3648
3649 u32 ecx = msr->index;
3650 u64 data = msr->data;
3651 switch (ecx) {
3652 case MSR_IA32_TSC:
3653 kvm_write_tsc(vcpu, msr);
3654 break;
3655 case MSR_STAR:
3656 svm->vmcb->save.star = data;
3657 break;
3658 #ifdef CONFIG_X86_64
3659 case MSR_LSTAR:
3660 svm->vmcb->save.lstar = data;
3661 break;
3662 case MSR_CSTAR:
3663 svm->vmcb->save.cstar = data;
3664 break;
3665 case MSR_KERNEL_GS_BASE:
3666 svm->vmcb->save.kernel_gs_base = data;
3667 break;
3668 case MSR_SYSCALL_MASK:
3669 svm->vmcb->save.sfmask = data;
3670 break;
3671 #endif
3672 case MSR_IA32_SYSENTER_CS:
3673 svm->vmcb->save.sysenter_cs = data;
3674 break;
3675 case MSR_IA32_SYSENTER_EIP:
3676 svm->sysenter_eip = data;
3677 svm->vmcb->save.sysenter_eip = data;
3678 break;
3679 case MSR_IA32_SYSENTER_ESP:
3680 svm->sysenter_esp = data;
3681 svm->vmcb->save.sysenter_esp = data;
3682 break;
3683 case MSR_TSC_AUX:
3684 if (!boot_cpu_has(X86_FEATURE_RDTSCP))
3685 return 1;
3686
3687 /*
3688 * This is rare, so we update the MSR here instead of using
3689 * direct_access_msrs. Doing that would require a rdmsr in
3690 * svm_vcpu_put.
3691 */
3692 svm->tsc_aux = data;
3693 wrmsrl(MSR_TSC_AUX, svm->tsc_aux);
3694 break;
3695 case MSR_IA32_DEBUGCTLMSR:
3696 if (!boot_cpu_has(X86_FEATURE_LBRV)) {
3697 vcpu_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTL 0x%llx, nop\n",
3698 __func__, data);
3699 break;
3700 }
3701 if (data & DEBUGCTL_RESERVED_BITS)
3702 return 1;
3703
3704 svm->vmcb->save.dbgctl = data;
3705 mark_dirty(svm->vmcb, VMCB_LBR);
3706 if (data & (1ULL<<0))
3707 svm_enable_lbrv(svm);
3708 else
3709 svm_disable_lbrv(svm);
3710 break;
3711 case MSR_VM_HSAVE_PA:
3712 svm->nested.hsave_msr = data;
3713 break;
3714 case MSR_VM_CR:
3715 return svm_set_vm_cr(vcpu, data);
3716 case MSR_VM_IGNNE:
3717 vcpu_unimpl(vcpu, "unimplemented wrmsr: 0x%x data 0x%llx\n", ecx, data);
3718 break;
3719 case MSR_IA32_SPEC_CTRL:
3720 svm->spec_ctrl = data;
3721 break;
3722 case MSR_IA32_APICBASE:
3723 if (kvm_vcpu_apicv_active(vcpu))
3724 avic_update_vapic_bar(to_svm(vcpu), data);
3725 /* Follow through */
3726 default:
3727 return kvm_set_msr_common(vcpu, msr);
3728 }
3729 return 0;
3730 }
3731
3732 static int wrmsr_interception(struct vcpu_svm *svm)
3733 {
3734 struct msr_data msr;
3735 u32 ecx = kvm_register_read(&svm->vcpu, VCPU_REGS_RCX);
3736 u64 data = kvm_read_edx_eax(&svm->vcpu);
3737
3738 msr.data = data;
3739 msr.index = ecx;
3740 msr.host_initiated = false;
3741
3742 svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
3743 if (kvm_set_msr(&svm->vcpu, &msr)) {
3744 trace_kvm_msr_write_ex(ecx, data);
3745 kvm_inject_gp(&svm->vcpu, 0);
3746 return 1;
3747 } else {
3748 trace_kvm_msr_write(ecx, data);
3749 return kvm_skip_emulated_instruction(&svm->vcpu);
3750 }
3751 }
3752
3753 static int msr_interception(struct vcpu_svm *svm)
3754 {
3755 if (svm->vmcb->control.exit_info_1)
3756 return wrmsr_interception(svm);
3757 else
3758 return rdmsr_interception(svm);
3759 }
3760
3761 static int interrupt_window_interception(struct vcpu_svm *svm)
3762 {
3763 kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
3764 svm_clear_vintr(svm);
3765 svm->vmcb->control.int_ctl &= ~V_IRQ_MASK;
3766 mark_dirty(svm->vmcb, VMCB_INTR);
3767 ++svm->vcpu.stat.irq_window_exits;
3768 return 1;
3769 }
3770
3771 static int pause_interception(struct vcpu_svm *svm)
3772 {
3773 kvm_vcpu_on_spin(&(svm->vcpu));
3774 return 1;
3775 }
3776
3777 static int nop_interception(struct vcpu_svm *svm)
3778 {
3779 return kvm_skip_emulated_instruction(&(svm->vcpu));
3780 }
3781
3782 static int monitor_interception(struct vcpu_svm *svm)
3783 {
3784 printk_once(KERN_WARNING "kvm: MONITOR instruction emulated as NOP!\n");
3785 return nop_interception(svm);
3786 }
3787
3788 static int mwait_interception(struct vcpu_svm *svm)
3789 {
3790 printk_once(KERN_WARNING "kvm: MWAIT instruction emulated as NOP!\n");
3791 return nop_interception(svm);
3792 }
3793
3794 enum avic_ipi_failure_cause {
3795 AVIC_IPI_FAILURE_INVALID_INT_TYPE,
3796 AVIC_IPI_FAILURE_TARGET_NOT_RUNNING,
3797 AVIC_IPI_FAILURE_INVALID_TARGET,
3798 AVIC_IPI_FAILURE_INVALID_BACKING_PAGE,
3799 };
3800
3801 static int avic_incomplete_ipi_interception(struct vcpu_svm *svm)
3802 {
3803 u32 icrh = svm->vmcb->control.exit_info_1 >> 32;
3804 u32 icrl = svm->vmcb->control.exit_info_1;
3805 u32 id = svm->vmcb->control.exit_info_2 >> 32;
3806 u32 index = svm->vmcb->control.exit_info_2 & 0xFF;
3807 struct kvm_lapic *apic = svm->vcpu.arch.apic;
3808
3809 trace_kvm_avic_incomplete_ipi(svm->vcpu.vcpu_id, icrh, icrl, id, index);
3810
3811 switch (id) {
3812 case AVIC_IPI_FAILURE_INVALID_INT_TYPE:
3813 /*
3814 * AVIC hardware handles the generation of
3815 * IPIs when the specified Message Type is Fixed
3816 * (also known as fixed delivery mode) and
3817 * the Trigger Mode is edge-triggered. The hardware
3818 * also supports self and broadcast delivery modes
3819 * specified via the Destination Shorthand(DSH)
3820 * field of the ICRL. Logical and physical APIC ID
3821 * formats are supported. All other IPI types cause
3822 * a #VMEXIT, which needs to emulated.
3823 */
3824 kvm_lapic_reg_write(apic, APIC_ICR2, icrh);
3825 kvm_lapic_reg_write(apic, APIC_ICR, icrl);
3826 break;
3827 case AVIC_IPI_FAILURE_TARGET_NOT_RUNNING: {
3828 int i;
3829 struct kvm_vcpu *vcpu;
3830 struct kvm *kvm = svm->vcpu.kvm;
3831 struct kvm_lapic *apic = svm->vcpu.arch.apic;
3832
3833 /*
3834 * At this point, we expect that the AVIC HW has already
3835 * set the appropriate IRR bits on the valid target
3836 * vcpus. So, we just need to kick the appropriate vcpu.
3837 */
3838 kvm_for_each_vcpu(i, vcpu, kvm) {
3839 bool m = kvm_apic_match_dest(vcpu, apic,
3840 icrl & KVM_APIC_SHORT_MASK,
3841 GET_APIC_DEST_FIELD(icrh),
3842 icrl & KVM_APIC_DEST_MASK);
3843
3844 if (m && !avic_vcpu_is_running(vcpu))
3845 kvm_vcpu_wake_up(vcpu);
3846 }
3847 break;
3848 }
3849 case AVIC_IPI_FAILURE_INVALID_TARGET:
3850 break;
3851 case AVIC_IPI_FAILURE_INVALID_BACKING_PAGE:
3852 WARN_ONCE(1, "Invalid backing page\n");
3853 break;
3854 default:
3855 pr_err("Unknown IPI interception\n");
3856 }
3857
3858 return 1;
3859 }
3860
3861 static u32 *avic_get_logical_id_entry(struct kvm_vcpu *vcpu, u32 ldr, bool flat)
3862 {
3863 struct kvm_arch *vm_data = &vcpu->kvm->arch;
3864 int index;
3865 u32 *logical_apic_id_table;
3866 int dlid = GET_APIC_LOGICAL_ID(ldr);
3867
3868 if (!dlid)
3869 return NULL;
3870
3871 if (flat) { /* flat */
3872 index = ffs(dlid) - 1;
3873 if (index > 7)
3874 return NULL;
3875 } else { /* cluster */
3876 int cluster = (dlid & 0xf0) >> 4;
3877 int apic = ffs(dlid & 0x0f) - 1;
3878
3879 if ((apic < 0) || (apic > 7) ||
3880 (cluster >= 0xf))
3881 return NULL;
3882 index = (cluster << 2) + apic;
3883 }
3884
3885 logical_apic_id_table = (u32 *) page_address(vm_data->avic_logical_id_table_page);
3886
3887 return &logical_apic_id_table[index];
3888 }
3889
3890 static int avic_ldr_write(struct kvm_vcpu *vcpu, u8 g_physical_id, u32 ldr,
3891 bool valid)
3892 {
3893 bool flat;
3894 u32 *entry, new_entry;
3895
3896 flat = kvm_lapic_get_reg(vcpu->arch.apic, APIC_DFR) == APIC_DFR_FLAT;
3897 entry = avic_get_logical_id_entry(vcpu, ldr, flat);
3898 if (!entry)
3899 return -EINVAL;
3900
3901 new_entry = READ_ONCE(*entry);
3902 new_entry &= ~AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK;
3903 new_entry |= (g_physical_id & AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK);
3904 if (valid)
3905 new_entry |= AVIC_LOGICAL_ID_ENTRY_VALID_MASK;
3906 else
3907 new_entry &= ~AVIC_LOGICAL_ID_ENTRY_VALID_MASK;
3908 WRITE_ONCE(*entry, new_entry);
3909
3910 return 0;
3911 }
3912
3913 static int avic_handle_ldr_update(struct kvm_vcpu *vcpu)
3914 {
3915 int ret;
3916 struct vcpu_svm *svm = to_svm(vcpu);
3917 u32 ldr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_LDR);
3918
3919 if (!ldr)
3920 return 1;
3921
3922 ret = avic_ldr_write(vcpu, vcpu->vcpu_id, ldr, true);
3923 if (ret && svm->ldr_reg) {
3924 avic_ldr_write(vcpu, 0, svm->ldr_reg, false);
3925 svm->ldr_reg = 0;
3926 } else {
3927 svm->ldr_reg = ldr;
3928 }
3929 return ret;
3930 }
3931
3932 static int avic_handle_apic_id_update(struct kvm_vcpu *vcpu)
3933 {
3934 u64 *old, *new;
3935 struct vcpu_svm *svm = to_svm(vcpu);
3936 u32 apic_id_reg = kvm_lapic_get_reg(vcpu->arch.apic, APIC_ID);
3937 u32 id = (apic_id_reg >> 24) & 0xff;
3938
3939 if (vcpu->vcpu_id == id)
3940 return 0;
3941
3942 old = avic_get_physical_id_entry(vcpu, vcpu->vcpu_id);
3943 new = avic_get_physical_id_entry(vcpu, id);
3944 if (!new || !old)
3945 return 1;
3946
3947 /* We need to move physical_id_entry to new offset */
3948 *new = *old;
3949 *old = 0ULL;
3950 to_svm(vcpu)->avic_physical_id_cache = new;
3951
3952 /*
3953 * Also update the guest physical APIC ID in the logical
3954 * APIC ID table entry if already setup the LDR.
3955 */
3956 if (svm->ldr_reg)
3957 avic_handle_ldr_update(vcpu);
3958
3959 return 0;
3960 }
3961
3962 static int avic_handle_dfr_update(struct kvm_vcpu *vcpu)
3963 {
3964 struct vcpu_svm *svm = to_svm(vcpu);
3965 struct kvm_arch *vm_data = &vcpu->kvm->arch;
3966 u32 dfr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_DFR);
3967 u32 mod = (dfr >> 28) & 0xf;
3968
3969 /*
3970 * We assume that all local APICs are using the same type.
3971 * If this changes, we need to flush the AVIC logical
3972 * APID id table.
3973 */
3974 if (vm_data->ldr_mode == mod)
3975 return 0;
3976
3977 clear_page(page_address(vm_data->avic_logical_id_table_page));
3978 vm_data->ldr_mode = mod;
3979
3980 if (svm->ldr_reg)
3981 avic_handle_ldr_update(vcpu);
3982 return 0;
3983 }
3984
3985 static int avic_unaccel_trap_write(struct vcpu_svm *svm)
3986 {
3987 struct kvm_lapic *apic = svm->vcpu.arch.apic;
3988 u32 offset = svm->vmcb->control.exit_info_1 &
3989 AVIC_UNACCEL_ACCESS_OFFSET_MASK;
3990
3991 switch (offset) {
3992 case APIC_ID:
3993 if (avic_handle_apic_id_update(&svm->vcpu))
3994 return 0;
3995 break;
3996 case APIC_LDR:
3997 if (avic_handle_ldr_update(&svm->vcpu))
3998 return 0;
3999 break;
4000 case APIC_DFR:
4001 avic_handle_dfr_update(&svm->vcpu);
4002 break;
4003 default:
4004 break;
4005 }
4006
4007 kvm_lapic_reg_write(apic, offset, kvm_lapic_get_reg(apic, offset));
4008
4009 return 1;
4010 }
4011
4012 static bool is_avic_unaccelerated_access_trap(u32 offset)
4013 {
4014 bool ret = false;
4015
4016 switch (offset) {
4017 case APIC_ID:
4018 case APIC_EOI:
4019 case APIC_RRR:
4020 case APIC_LDR:
4021 case APIC_DFR:
4022 case APIC_SPIV:
4023 case APIC_ESR:
4024 case APIC_ICR:
4025 case APIC_LVTT:
4026 case APIC_LVTTHMR:
4027 case APIC_LVTPC:
4028 case APIC_LVT0:
4029 case APIC_LVT1:
4030 case APIC_LVTERR:
4031 case APIC_TMICT:
4032 case APIC_TDCR:
4033 ret = true;
4034 break;
4035 default:
4036 break;
4037 }
4038 return ret;
4039 }
4040
4041 static int avic_unaccelerated_access_interception(struct vcpu_svm *svm)
4042 {
4043 int ret = 0;
4044 u32 offset = svm->vmcb->control.exit_info_1 &
4045 AVIC_UNACCEL_ACCESS_OFFSET_MASK;
4046 u32 vector = svm->vmcb->control.exit_info_2 &
4047 AVIC_UNACCEL_ACCESS_VECTOR_MASK;
4048 bool write = (svm->vmcb->control.exit_info_1 >> 32) &
4049 AVIC_UNACCEL_ACCESS_WRITE_MASK;
4050 bool trap = is_avic_unaccelerated_access_trap(offset);
4051
4052 trace_kvm_avic_unaccelerated_access(svm->vcpu.vcpu_id, offset,
4053 trap, write, vector);
4054 if (trap) {
4055 /* Handling Trap */
4056 WARN_ONCE(!write, "svm: Handling trap read.\n");
4057 ret = avic_unaccel_trap_write(svm);
4058 } else {
4059 /* Handling Fault */
4060 ret = (emulate_instruction(&svm->vcpu, 0) == EMULATE_DONE);
4061 }
4062
4063 return ret;
4064 }
4065
4066 static int (*const svm_exit_handlers[])(struct vcpu_svm *svm) = {
4067 [SVM_EXIT_READ_CR0] = cr_interception,
4068 [SVM_EXIT_READ_CR3] = cr_interception,
4069 [SVM_EXIT_READ_CR4] = cr_interception,
4070 [SVM_EXIT_READ_CR8] = cr_interception,
4071 [SVM_EXIT_CR0_SEL_WRITE] = cr_interception,
4072 [SVM_EXIT_WRITE_CR0] = cr_interception,
4073 [SVM_EXIT_WRITE_CR3] = cr_interception,
4074 [SVM_EXIT_WRITE_CR4] = cr_interception,
4075 [SVM_EXIT_WRITE_CR8] = cr8_write_interception,
4076 [SVM_EXIT_READ_DR0] = dr_interception,
4077 [SVM_EXIT_READ_DR1] = dr_interception,
4078 [SVM_EXIT_READ_DR2] = dr_interception,
4079 [SVM_EXIT_READ_DR3] = dr_interception,
4080 [SVM_EXIT_READ_DR4] = dr_interception,
4081 [SVM_EXIT_READ_DR5] = dr_interception,
4082 [SVM_EXIT_READ_DR6] = dr_interception,
4083 [SVM_EXIT_READ_DR7] = dr_interception,
4084 [SVM_EXIT_WRITE_DR0] = dr_interception,
4085 [SVM_EXIT_WRITE_DR1] = dr_interception,
4086 [SVM_EXIT_WRITE_DR2] = dr_interception,
4087 [SVM_EXIT_WRITE_DR3] = dr_interception,
4088 [SVM_EXIT_WRITE_DR4] = dr_interception,
4089 [SVM_EXIT_WRITE_DR5] = dr_interception,
4090 [SVM_EXIT_WRITE_DR6] = dr_interception,
4091 [SVM_EXIT_WRITE_DR7] = dr_interception,
4092 [SVM_EXIT_EXCP_BASE + DB_VECTOR] = db_interception,
4093 [SVM_EXIT_EXCP_BASE + BP_VECTOR] = bp_interception,
4094 [SVM_EXIT_EXCP_BASE + UD_VECTOR] = ud_interception,
4095 [SVM_EXIT_EXCP_BASE + PF_VECTOR] = pf_interception,
4096 [SVM_EXIT_EXCP_BASE + MC_VECTOR] = mc_interception,
4097 [SVM_EXIT_EXCP_BASE + AC_VECTOR] = ac_interception,
4098 [SVM_EXIT_INTR] = intr_interception,
4099 [SVM_EXIT_NMI] = nmi_interception,
4100 [SVM_EXIT_SMI] = nop_on_interception,
4101 [SVM_EXIT_INIT] = nop_on_interception,
4102 [SVM_EXIT_VINTR] = interrupt_window_interception,
4103 [SVM_EXIT_RDPMC] = rdpmc_interception,
4104 [SVM_EXIT_CPUID] = cpuid_interception,
4105 [SVM_EXIT_IRET] = iret_interception,
4106 [SVM_EXIT_INVD] = emulate_on_interception,
4107 [SVM_EXIT_PAUSE] = pause_interception,
4108 [SVM_EXIT_HLT] = halt_interception,
4109 [SVM_EXIT_INVLPG] = invlpg_interception,
4110 [SVM_EXIT_INVLPGA] = invlpga_interception,
4111 [SVM_EXIT_IOIO] = io_interception,
4112 [SVM_EXIT_MSR] = msr_interception,
4113 [SVM_EXIT_TASK_SWITCH] = task_switch_interception,
4114 [SVM_EXIT_SHUTDOWN] = shutdown_interception,
4115 [SVM_EXIT_VMRUN] = vmrun_interception,
4116 [SVM_EXIT_VMMCALL] = vmmcall_interception,
4117 [SVM_EXIT_VMLOAD] = vmload_interception,
4118 [SVM_EXIT_VMSAVE] = vmsave_interception,
4119 [SVM_EXIT_STGI] = stgi_interception,
4120 [SVM_EXIT_CLGI] = clgi_interception,
4121 [SVM_EXIT_SKINIT] = skinit_interception,
4122 [SVM_EXIT_WBINVD] = wbinvd_interception,
4123 [SVM_EXIT_MONITOR] = monitor_interception,
4124 [SVM_EXIT_MWAIT] = mwait_interception,
4125 [SVM_EXIT_XSETBV] = xsetbv_interception,
4126 [SVM_EXIT_NPF] = pf_interception,
4127 [SVM_EXIT_RSM] = emulate_on_interception,
4128 [SVM_EXIT_AVIC_INCOMPLETE_IPI] = avic_incomplete_ipi_interception,
4129 [SVM_EXIT_AVIC_UNACCELERATED_ACCESS] = avic_unaccelerated_access_interception,
4130 };
4131
4132 static void dump_vmcb(struct kvm_vcpu *vcpu)
4133 {
4134 struct vcpu_svm *svm = to_svm(vcpu);
4135 struct vmcb_control_area *control = &svm->vmcb->control;
4136 struct vmcb_save_area *save = &svm->vmcb->save;
4137
4138 pr_err("VMCB Control Area:\n");
4139 pr_err("%-20s%04x\n", "cr_read:", control->intercept_cr & 0xffff);
4140 pr_err("%-20s%04x\n", "cr_write:", control->intercept_cr >> 16);
4141 pr_err("%-20s%04x\n", "dr_read:", control->intercept_dr & 0xffff);
4142 pr_err("%-20s%04x\n", "dr_write:", control->intercept_dr >> 16);
4143 pr_err("%-20s%08x\n", "exceptions:", control->intercept_exceptions);
4144 pr_err("%-20s%016llx\n", "intercepts:", control->intercept);
4145 pr_err("%-20s%d\n", "pause filter count:", control->pause_filter_count);
4146 pr_err("%-20s%016llx\n", "iopm_base_pa:", control->iopm_base_pa);
4147 pr_err("%-20s%016llx\n", "msrpm_base_pa:", control->msrpm_base_pa);
4148 pr_err("%-20s%016llx\n", "tsc_offset:", control->tsc_offset);
4149 pr_err("%-20s%d\n", "asid:", control->asid);
4150 pr_err("%-20s%d\n", "tlb_ctl:", control->tlb_ctl);
4151 pr_err("%-20s%08x\n", "int_ctl:", control->int_ctl);
4152 pr_err("%-20s%08x\n", "int_vector:", control->int_vector);
4153 pr_err("%-20s%08x\n", "int_state:", control->int_state);
4154 pr_err("%-20s%08x\n", "exit_code:", control->exit_code);
4155 pr_err("%-20s%016llx\n", "exit_info1:", control->exit_info_1);
4156 pr_err("%-20s%016llx\n", "exit_info2:", control->exit_info_2);
4157 pr_err("%-20s%08x\n", "exit_int_info:", control->exit_int_info);
4158 pr_err("%-20s%08x\n", "exit_int_info_err:", control->exit_int_info_err);
4159 pr_err("%-20s%lld\n", "nested_ctl:", control->nested_ctl);
4160 pr_err("%-20s%016llx\n", "nested_cr3:", control->nested_cr3);
4161 pr_err("%-20s%016llx\n", "avic_vapic_bar:", control->avic_vapic_bar);
4162 pr_err("%-20s%08x\n", "event_inj:", control->event_inj);
4163 pr_err("%-20s%08x\n", "event_inj_err:", control->event_inj_err);
4164 pr_err("%-20s%lld\n", "virt_ext:", control->virt_ext);
4165 pr_err("%-20s%016llx\n", "next_rip:", control->next_rip);
4166 pr_err("%-20s%016llx\n", "avic_backing_page:", control->avic_backing_page);
4167 pr_err("%-20s%016llx\n", "avic_logical_id:", control->avic_logical_id);
4168 pr_err("%-20s%016llx\n", "avic_physical_id:", control->avic_physical_id);
4169 pr_err("VMCB State Save Area:\n");
4170 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4171 "es:",
4172 save->es.selector, save->es.attrib,
4173 save->es.limit, save->es.base);
4174 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4175 "cs:",
4176 save->cs.selector, save->cs.attrib,
4177 save->cs.limit, save->cs.base);
4178 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4179 "ss:",
4180 save->ss.selector, save->ss.attrib,
4181 save->ss.limit, save->ss.base);
4182 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4183 "ds:",
4184 save->ds.selector, save->ds.attrib,
4185 save->ds.limit, save->ds.base);
4186 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4187 "fs:",
4188 save->fs.selector, save->fs.attrib,
4189 save->fs.limit, save->fs.base);
4190 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4191 "gs:",
4192 save->gs.selector, save->gs.attrib,
4193 save->gs.limit, save->gs.base);
4194 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4195 "gdtr:",
4196 save->gdtr.selector, save->gdtr.attrib,
4197 save->gdtr.limit, save->gdtr.base);
4198 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4199 "ldtr:",
4200 save->ldtr.selector, save->ldtr.attrib,
4201 save->ldtr.limit, save->ldtr.base);
4202 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4203 "idtr:",
4204 save->idtr.selector, save->idtr.attrib,
4205 save->idtr.limit, save->idtr.base);
4206 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4207 "tr:",
4208 save->tr.selector, save->tr.attrib,
4209 save->tr.limit, save->tr.base);
4210 pr_err("cpl: %d efer: %016llx\n",
4211 save->cpl, save->efer);
4212 pr_err("%-15s %016llx %-13s %016llx\n",
4213 "cr0:", save->cr0, "cr2:", save->cr2);
4214 pr_err("%-15s %016llx %-13s %016llx\n",
4215 "cr3:", save->cr3, "cr4:", save->cr4);
4216 pr_err("%-15s %016llx %-13s %016llx\n",
4217 "dr6:", save->dr6, "dr7:", save->dr7);
4218 pr_err("%-15s %016llx %-13s %016llx\n",
4219 "rip:", save->rip, "rflags:", save->rflags);
4220 pr_err("%-15s %016llx %-13s %016llx\n",
4221 "rsp:", save->rsp, "rax:", save->rax);
4222 pr_err("%-15s %016llx %-13s %016llx\n",
4223 "star:", save->star, "lstar:", save->lstar);
4224 pr_err("%-15s %016llx %-13s %016llx\n",
4225 "cstar:", save->cstar, "sfmask:", save->sfmask);
4226 pr_err("%-15s %016llx %-13s %016llx\n",
4227 "kernel_gs_base:", save->kernel_gs_base,
4228 "sysenter_cs:", save->sysenter_cs);
4229 pr_err("%-15s %016llx %-13s %016llx\n",
4230 "sysenter_esp:", save->sysenter_esp,
4231 "sysenter_eip:", save->sysenter_eip);
4232 pr_err("%-15s %016llx %-13s %016llx\n",
4233 "gpat:", save->g_pat, "dbgctl:", save->dbgctl);
4234 pr_err("%-15s %016llx %-13s %016llx\n",
4235 "br_from:", save->br_from, "br_to:", save->br_to);
4236 pr_err("%-15s %016llx %-13s %016llx\n",
4237 "excp_from:", save->last_excp_from,
4238 "excp_to:", save->last_excp_to);
4239 }
4240
4241 static void svm_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2)
4242 {
4243 struct vmcb_control_area *control = &to_svm(vcpu)->vmcb->control;
4244
4245 *info1 = control->exit_info_1;
4246 *info2 = control->exit_info_2;
4247 }
4248
4249 static int handle_exit(struct kvm_vcpu *vcpu)
4250 {
4251 struct vcpu_svm *svm = to_svm(vcpu);
4252 struct kvm_run *kvm_run = vcpu->run;
4253 u32 exit_code = svm->vmcb->control.exit_code;
4254
4255 trace_kvm_exit(exit_code, vcpu, KVM_ISA_SVM);
4256
4257 vcpu->arch.gpa_available = (exit_code == SVM_EXIT_NPF);
4258
4259 if (!is_cr_intercept(svm, INTERCEPT_CR0_WRITE))
4260 vcpu->arch.cr0 = svm->vmcb->save.cr0;
4261 if (npt_enabled)
4262 vcpu->arch.cr3 = svm->vmcb->save.cr3;
4263
4264 if (unlikely(svm->nested.exit_required)) {
4265 nested_svm_vmexit(svm);
4266 svm->nested.exit_required = false;
4267
4268 return 1;
4269 }
4270
4271 if (is_guest_mode(vcpu)) {
4272 int vmexit;
4273
4274 trace_kvm_nested_vmexit(svm->vmcb->save.rip, exit_code,
4275 svm->vmcb->control.exit_info_1,
4276 svm->vmcb->control.exit_info_2,
4277 svm->vmcb->control.exit_int_info,
4278 svm->vmcb->control.exit_int_info_err,
4279 KVM_ISA_SVM);
4280
4281 vmexit = nested_svm_exit_special(svm);
4282
4283 if (vmexit == NESTED_EXIT_CONTINUE)
4284 vmexit = nested_svm_exit_handled(svm);
4285
4286 if (vmexit == NESTED_EXIT_DONE)
4287 return 1;
4288 }
4289
4290 svm_complete_interrupts(svm);
4291
4292 if (svm->vmcb->control.exit_code == SVM_EXIT_ERR) {
4293 kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY;
4294 kvm_run->fail_entry.hardware_entry_failure_reason
4295 = svm->vmcb->control.exit_code;
4296 pr_err("KVM: FAILED VMRUN WITH VMCB:\n");
4297 dump_vmcb(vcpu);
4298 return 0;
4299 }
4300
4301 if (is_external_interrupt(svm->vmcb->control.exit_int_info) &&
4302 exit_code != SVM_EXIT_EXCP_BASE + PF_VECTOR &&
4303 exit_code != SVM_EXIT_NPF && exit_code != SVM_EXIT_TASK_SWITCH &&
4304 exit_code != SVM_EXIT_INTR && exit_code != SVM_EXIT_NMI)
4305 printk(KERN_ERR "%s: unexpected exit_int_info 0x%x "
4306 "exit_code 0x%x\n",
4307 __func__, svm->vmcb->control.exit_int_info,
4308 exit_code);
4309
4310 if (exit_code >= ARRAY_SIZE(svm_exit_handlers)
4311 || !svm_exit_handlers[exit_code]) {
4312 WARN_ONCE(1, "svm: unexpected exit reason 0x%x\n", exit_code);
4313 kvm_queue_exception(vcpu, UD_VECTOR);
4314 return 1;
4315 }
4316
4317 return svm_exit_handlers[exit_code](svm);
4318 }
4319
4320 static void reload_tss(struct kvm_vcpu *vcpu)
4321 {
4322 int cpu = raw_smp_processor_id();
4323
4324 struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
4325 sd->tss_desc->type = 9; /* available 32/64-bit TSS */
4326 load_TR_desc();
4327 }
4328
4329 static void pre_svm_run(struct vcpu_svm *svm)
4330 {
4331 int cpu = raw_smp_processor_id();
4332
4333 struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
4334
4335 /* FIXME: handle wraparound of asid_generation */
4336 if (svm->asid_generation != sd->asid_generation)
4337 new_asid(svm, sd);
4338 }
4339
4340 static void svm_inject_nmi(struct kvm_vcpu *vcpu)
4341 {
4342 struct vcpu_svm *svm = to_svm(vcpu);
4343
4344 svm->vmcb->control.event_inj = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI;
4345 vcpu->arch.hflags |= HF_NMI_MASK;
4346 set_intercept(svm, INTERCEPT_IRET);
4347 ++vcpu->stat.nmi_injections;
4348 }
4349
4350 static inline void svm_inject_irq(struct vcpu_svm *svm, int irq)
4351 {
4352 struct vmcb_control_area *control;
4353
4354 /* The following fields are ignored when AVIC is enabled */
4355 control = &svm->vmcb->control;
4356 control->int_vector = irq;
4357 control->int_ctl &= ~V_INTR_PRIO_MASK;
4358 control->int_ctl |= V_IRQ_MASK |
4359 ((/*control->int_vector >> 4*/ 0xf) << V_INTR_PRIO_SHIFT);
4360 mark_dirty(svm->vmcb, VMCB_INTR);
4361 }
4362
4363 static void svm_set_irq(struct kvm_vcpu *vcpu)
4364 {
4365 struct vcpu_svm *svm = to_svm(vcpu);
4366
4367 BUG_ON(!(gif_set(svm)));
4368
4369 trace_kvm_inj_virq(vcpu->arch.interrupt.nr);
4370 ++vcpu->stat.irq_injections;
4371
4372 svm->vmcb->control.event_inj = vcpu->arch.interrupt.nr |
4373 SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR;
4374 }
4375
4376 static inline bool svm_nested_virtualize_tpr(struct kvm_vcpu *vcpu)
4377 {
4378 return is_guest_mode(vcpu) && (vcpu->arch.hflags & HF_VINTR_MASK);
4379 }
4380
4381 static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
4382 {
4383 struct vcpu_svm *svm = to_svm(vcpu);
4384
4385 if (svm_nested_virtualize_tpr(vcpu) ||
4386 kvm_vcpu_apicv_active(vcpu))
4387 return;
4388
4389 clr_cr_intercept(svm, INTERCEPT_CR8_WRITE);
4390
4391 if (irr == -1)
4392 return;
4393
4394 if (tpr >= irr)
4395 set_cr_intercept(svm, INTERCEPT_CR8_WRITE);
4396 }
4397
4398 static void svm_set_virtual_x2apic_mode(struct kvm_vcpu *vcpu, bool set)
4399 {
4400 return;
4401 }
4402
4403 static bool svm_get_enable_apicv(void)
4404 {
4405 return avic;
4406 }
4407
4408 static void svm_hwapic_irr_update(struct kvm_vcpu *vcpu, int max_irr)
4409 {
4410 }
4411
4412 static void svm_hwapic_isr_update(struct kvm_vcpu *vcpu, int max_isr)
4413 {
4414 }
4415
4416 /* Note: Currently only used by Hyper-V. */
4417 static void svm_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu)
4418 {
4419 struct vcpu_svm *svm = to_svm(vcpu);
4420 struct vmcb *vmcb = svm->vmcb;
4421
4422 if (!avic)
4423 return;
4424
4425 vmcb->control.int_ctl &= ~AVIC_ENABLE_MASK;
4426 mark_dirty(vmcb, VMCB_INTR);
4427 }
4428
4429 static void svm_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap)
4430 {
4431 return;
4432 }
4433
4434 static void svm_deliver_avic_intr(struct kvm_vcpu *vcpu, int vec)
4435 {
4436 kvm_lapic_set_irr(vec, vcpu->arch.apic);
4437 smp_mb__after_atomic();
4438
4439 if (avic_vcpu_is_running(vcpu))
4440 wrmsrl(SVM_AVIC_DOORBELL,
4441 kvm_cpu_get_apicid(vcpu->cpu));
4442 else
4443 kvm_vcpu_wake_up(vcpu);
4444 }
4445
4446 static void svm_ir_list_del(struct vcpu_svm *svm, struct amd_iommu_pi_data *pi)
4447 {
4448 unsigned long flags;
4449 struct amd_svm_iommu_ir *cur;
4450
4451 spin_lock_irqsave(&svm->ir_list_lock, flags);
4452 list_for_each_entry(cur, &svm->ir_list, node) {
4453 if (cur->data != pi->ir_data)
4454 continue;
4455 list_del(&cur->node);
4456 kfree(cur);
4457 break;
4458 }
4459 spin_unlock_irqrestore(&svm->ir_list_lock, flags);
4460 }
4461
4462 static int svm_ir_list_add(struct vcpu_svm *svm, struct amd_iommu_pi_data *pi)
4463 {
4464 int ret = 0;
4465 unsigned long flags;
4466 struct amd_svm_iommu_ir *ir;
4467
4468 /**
4469 * In some cases, the existing irte is updaed and re-set,
4470 * so we need to check here if it's already been * added
4471 * to the ir_list.
4472 */
4473 if (pi->ir_data && (pi->prev_ga_tag != 0)) {
4474 struct kvm *kvm = svm->vcpu.kvm;
4475 u32 vcpu_id = AVIC_GATAG_TO_VCPUID(pi->prev_ga_tag);
4476 struct kvm_vcpu *prev_vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id);
4477 struct vcpu_svm *prev_svm;
4478
4479 if (!prev_vcpu) {
4480 ret = -EINVAL;
4481 goto out;
4482 }
4483
4484 prev_svm = to_svm(prev_vcpu);
4485 svm_ir_list_del(prev_svm, pi);
4486 }
4487
4488 /**
4489 * Allocating new amd_iommu_pi_data, which will get
4490 * add to the per-vcpu ir_list.
4491 */
4492 ir = kzalloc(sizeof(struct amd_svm_iommu_ir), GFP_KERNEL);
4493 if (!ir) {
4494 ret = -ENOMEM;
4495 goto out;
4496 }
4497 ir->data = pi->ir_data;
4498
4499 spin_lock_irqsave(&svm->ir_list_lock, flags);
4500 list_add(&ir->node, &svm->ir_list);
4501 spin_unlock_irqrestore(&svm->ir_list_lock, flags);
4502 out:
4503 return ret;
4504 }
4505
4506 /**
4507 * Note:
4508 * The HW cannot support posting multicast/broadcast
4509 * interrupts to a vCPU. So, we still use legacy interrupt
4510 * remapping for these kind of interrupts.
4511 *
4512 * For lowest-priority interrupts, we only support
4513 * those with single CPU as the destination, e.g. user
4514 * configures the interrupts via /proc/irq or uses
4515 * irqbalance to make the interrupts single-CPU.
4516 */
4517 static int
4518 get_pi_vcpu_info(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e,
4519 struct vcpu_data *vcpu_info, struct vcpu_svm **svm)
4520 {
4521 struct kvm_lapic_irq irq;
4522 struct kvm_vcpu *vcpu = NULL;
4523
4524 kvm_set_msi_irq(kvm, e, &irq);
4525
4526 if (!kvm_intr_is_single_vcpu(kvm, &irq, &vcpu)) {
4527 pr_debug("SVM: %s: use legacy intr remap mode for irq %u\n",
4528 __func__, irq.vector);
4529 return -1;
4530 }
4531
4532 pr_debug("SVM: %s: use GA mode for irq %u\n", __func__,
4533 irq.vector);
4534 *svm = to_svm(vcpu);
4535 vcpu_info->pi_desc_addr = page_to_phys((*svm)->avic_backing_page);
4536 vcpu_info->vector = irq.vector;
4537
4538 return 0;
4539 }
4540
4541 /*
4542 * svm_update_pi_irte - set IRTE for Posted-Interrupts
4543 *
4544 * @kvm: kvm
4545 * @host_irq: host irq of the interrupt
4546 * @guest_irq: gsi of the interrupt
4547 * @set: set or unset PI
4548 * returns 0 on success, < 0 on failure
4549 */
4550 static int svm_update_pi_irte(struct kvm *kvm, unsigned int host_irq,
4551 uint32_t guest_irq, bool set)
4552 {
4553 struct kvm_kernel_irq_routing_entry *e;
4554 struct kvm_irq_routing_table *irq_rt;
4555 int idx, ret = -EINVAL;
4556
4557 if (!kvm_arch_has_assigned_device(kvm) ||
4558 !irq_remapping_cap(IRQ_POSTING_CAP))
4559 return 0;
4560
4561 pr_debug("SVM: %s: host_irq=%#x, guest_irq=%#x, set=%#x\n",
4562 __func__, host_irq, guest_irq, set);
4563
4564 idx = srcu_read_lock(&kvm->irq_srcu);
4565 irq_rt = srcu_dereference(kvm->irq_routing, &kvm->irq_srcu);
4566 WARN_ON(guest_irq >= irq_rt->nr_rt_entries);
4567
4568 hlist_for_each_entry(e, &irq_rt->map[guest_irq], link) {
4569 struct vcpu_data vcpu_info;
4570 struct vcpu_svm *svm = NULL;
4571
4572 if (e->type != KVM_IRQ_ROUTING_MSI)
4573 continue;
4574
4575 /**
4576 * Here, we setup with legacy mode in the following cases:
4577 * 1. When cannot target interrupt to a specific vcpu.
4578 * 2. Unsetting posted interrupt.
4579 * 3. APIC virtialization is disabled for the vcpu.
4580 */
4581 if (!get_pi_vcpu_info(kvm, e, &vcpu_info, &svm) && set &&
4582 kvm_vcpu_apicv_active(&svm->vcpu)) {
4583 struct amd_iommu_pi_data pi;
4584
4585 /* Try to enable guest_mode in IRTE */
4586 pi.base = page_to_phys(svm->avic_backing_page) & AVIC_HPA_MASK;
4587 pi.ga_tag = AVIC_GATAG(kvm->arch.avic_vm_id,
4588 svm->vcpu.vcpu_id);
4589 pi.is_guest_mode = true;
4590 pi.vcpu_data = &vcpu_info;
4591 ret = irq_set_vcpu_affinity(host_irq, &pi);
4592
4593 /**
4594 * Here, we successfully setting up vcpu affinity in
4595 * IOMMU guest mode. Now, we need to store the posted
4596 * interrupt information in a per-vcpu ir_list so that
4597 * we can reference to them directly when we update vcpu
4598 * scheduling information in IOMMU irte.
4599 */
4600 if (!ret && pi.is_guest_mode)
4601 svm_ir_list_add(svm, &pi);
4602 } else {
4603 /* Use legacy mode in IRTE */
4604 struct amd_iommu_pi_data pi;
4605
4606 /**
4607 * Here, pi is used to:
4608 * - Tell IOMMU to use legacy mode for this interrupt.
4609 * - Retrieve ga_tag of prior interrupt remapping data.
4610 */
4611 pi.is_guest_mode = false;
4612 ret = irq_set_vcpu_affinity(host_irq, &pi);
4613
4614 /**
4615 * Check if the posted interrupt was previously
4616 * setup with the guest_mode by checking if the ga_tag
4617 * was cached. If so, we need to clean up the per-vcpu
4618 * ir_list.
4619 */
4620 if (!ret && pi.prev_ga_tag) {
4621 int id = AVIC_GATAG_TO_VCPUID(pi.prev_ga_tag);
4622 struct kvm_vcpu *vcpu;
4623
4624 vcpu = kvm_get_vcpu_by_id(kvm, id);
4625 if (vcpu)
4626 svm_ir_list_del(to_svm(vcpu), &pi);
4627 }
4628 }
4629
4630 if (!ret && svm) {
4631 trace_kvm_pi_irte_update(svm->vcpu.vcpu_id,
4632 host_irq, e->gsi,
4633 vcpu_info.vector,
4634 vcpu_info.pi_desc_addr, set);
4635 }
4636
4637 if (ret < 0) {
4638 pr_err("%s: failed to update PI IRTE\n", __func__);
4639 goto out;
4640 }
4641 }
4642
4643 ret = 0;
4644 out:
4645 srcu_read_unlock(&kvm->irq_srcu, idx);
4646 return ret;
4647 }
4648
4649 static int svm_nmi_allowed(struct kvm_vcpu *vcpu)
4650 {
4651 struct vcpu_svm *svm = to_svm(vcpu);
4652 struct vmcb *vmcb = svm->vmcb;
4653 int ret;
4654 ret = !(vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) &&
4655 !(svm->vcpu.arch.hflags & HF_NMI_MASK);
4656 ret = ret && gif_set(svm) && nested_svm_nmi(svm);
4657
4658 return ret;
4659 }
4660
4661 static bool svm_get_nmi_mask(struct kvm_vcpu *vcpu)
4662 {
4663 struct vcpu_svm *svm = to_svm(vcpu);
4664
4665 return !!(svm->vcpu.arch.hflags & HF_NMI_MASK);
4666 }
4667
4668 static void svm_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked)
4669 {
4670 struct vcpu_svm *svm = to_svm(vcpu);
4671
4672 if (masked) {
4673 svm->vcpu.arch.hflags |= HF_NMI_MASK;
4674 set_intercept(svm, INTERCEPT_IRET);
4675 } else {
4676 svm->vcpu.arch.hflags &= ~HF_NMI_MASK;
4677 clr_intercept(svm, INTERCEPT_IRET);
4678 }
4679 }
4680
4681 static int svm_interrupt_allowed(struct kvm_vcpu *vcpu)
4682 {
4683 struct vcpu_svm *svm = to_svm(vcpu);
4684 struct vmcb *vmcb = svm->vmcb;
4685 int ret;
4686
4687 if (!gif_set(svm) ||
4688 (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK))
4689 return 0;
4690
4691 ret = !!(kvm_get_rflags(vcpu) & X86_EFLAGS_IF);
4692
4693 if (is_guest_mode(vcpu))
4694 return ret && !(svm->vcpu.arch.hflags & HF_VINTR_MASK);
4695
4696 return ret;
4697 }
4698
4699 static void enable_irq_window(struct kvm_vcpu *vcpu)
4700 {
4701 struct vcpu_svm *svm = to_svm(vcpu);
4702
4703 if (kvm_vcpu_apicv_active(vcpu))
4704 return;
4705
4706 /*
4707 * In case GIF=0 we can't rely on the CPU to tell us when GIF becomes
4708 * 1, because that's a separate STGI/VMRUN intercept. The next time we
4709 * get that intercept, this function will be called again though and
4710 * we'll get the vintr intercept.
4711 */
4712 if (gif_set(svm) && nested_svm_intr(svm)) {
4713 svm_set_vintr(svm);
4714 svm_inject_irq(svm, 0x0);
4715 }
4716 }
4717
4718 static void enable_nmi_window(struct kvm_vcpu *vcpu)
4719 {
4720 struct vcpu_svm *svm = to_svm(vcpu);
4721
4722 if ((svm->vcpu.arch.hflags & (HF_NMI_MASK | HF_IRET_MASK))
4723 == HF_NMI_MASK)
4724 return; /* IRET will cause a vm exit */
4725
4726 if ((svm->vcpu.arch.hflags & HF_GIF_MASK) == 0)
4727 return; /* STGI will cause a vm exit */
4728
4729 if (svm->nested.exit_required)
4730 return; /* we're not going to run the guest yet */
4731
4732 /*
4733 * Something prevents NMI from been injected. Single step over possible
4734 * problem (IRET or exception injection or interrupt shadow)
4735 */
4736 svm->nmi_singlestep_guest_rflags = svm_get_rflags(vcpu);
4737 svm->nmi_singlestep = true;
4738 svm->vmcb->save.rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
4739 }
4740
4741 static int svm_set_tss_addr(struct kvm *kvm, unsigned int addr)
4742 {
4743 return 0;
4744 }
4745
4746 static void svm_flush_tlb(struct kvm_vcpu *vcpu)
4747 {
4748 struct vcpu_svm *svm = to_svm(vcpu);
4749
4750 if (static_cpu_has(X86_FEATURE_FLUSHBYASID))
4751 svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID;
4752 else
4753 svm->asid_generation--;
4754 }
4755
4756 static void svm_prepare_guest_switch(struct kvm_vcpu *vcpu)
4757 {
4758 }
4759
4760 static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu)
4761 {
4762 struct vcpu_svm *svm = to_svm(vcpu);
4763
4764 if (svm_nested_virtualize_tpr(vcpu))
4765 return;
4766
4767 if (!is_cr_intercept(svm, INTERCEPT_CR8_WRITE)) {
4768 int cr8 = svm->vmcb->control.int_ctl & V_TPR_MASK;
4769 kvm_set_cr8(vcpu, cr8);
4770 }
4771 }
4772
4773 static inline void sync_lapic_to_cr8(struct kvm_vcpu *vcpu)
4774 {
4775 struct vcpu_svm *svm = to_svm(vcpu);
4776 u64 cr8;
4777
4778 if (svm_nested_virtualize_tpr(vcpu) ||
4779 kvm_vcpu_apicv_active(vcpu))
4780 return;
4781
4782 cr8 = kvm_get_cr8(vcpu);
4783 svm->vmcb->control.int_ctl &= ~V_TPR_MASK;
4784 svm->vmcb->control.int_ctl |= cr8 & V_TPR_MASK;
4785 }
4786
4787 static void svm_complete_interrupts(struct vcpu_svm *svm)
4788 {
4789 u8 vector;
4790 int type;
4791 u32 exitintinfo = svm->vmcb->control.exit_int_info;
4792 unsigned int3_injected = svm->int3_injected;
4793
4794 svm->int3_injected = 0;
4795
4796 /*
4797 * If we've made progress since setting HF_IRET_MASK, we've
4798 * executed an IRET and can allow NMI injection.
4799 */
4800 if ((svm->vcpu.arch.hflags & HF_IRET_MASK)
4801 && kvm_rip_read(&svm->vcpu) != svm->nmi_iret_rip) {
4802 svm->vcpu.arch.hflags &= ~(HF_NMI_MASK | HF_IRET_MASK);
4803 kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
4804 }
4805
4806 svm->vcpu.arch.nmi_injected = false;
4807 kvm_clear_exception_queue(&svm->vcpu);
4808 kvm_clear_interrupt_queue(&svm->vcpu);
4809
4810 if (!(exitintinfo & SVM_EXITINTINFO_VALID))
4811 return;
4812
4813 kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
4814
4815 vector = exitintinfo & SVM_EXITINTINFO_VEC_MASK;
4816 type = exitintinfo & SVM_EXITINTINFO_TYPE_MASK;
4817
4818 switch (type) {
4819 case SVM_EXITINTINFO_TYPE_NMI:
4820 svm->vcpu.arch.nmi_injected = true;
4821 break;
4822 case SVM_EXITINTINFO_TYPE_EXEPT:
4823 /*
4824 * In case of software exceptions, do not reinject the vector,
4825 * but re-execute the instruction instead. Rewind RIP first
4826 * if we emulated INT3 before.
4827 */
4828 if (kvm_exception_is_soft(vector)) {
4829 if (vector == BP_VECTOR && int3_injected &&
4830 kvm_is_linear_rip(&svm->vcpu, svm->int3_rip))
4831 kvm_rip_write(&svm->vcpu,
4832 kvm_rip_read(&svm->vcpu) -
4833 int3_injected);
4834 break;
4835 }
4836 if (exitintinfo & SVM_EXITINTINFO_VALID_ERR) {
4837 u32 err = svm->vmcb->control.exit_int_info_err;
4838 kvm_requeue_exception_e(&svm->vcpu, vector, err);
4839
4840 } else
4841 kvm_requeue_exception(&svm->vcpu, vector);
4842 break;
4843 case SVM_EXITINTINFO_TYPE_INTR:
4844 kvm_queue_interrupt(&svm->vcpu, vector, false);
4845 break;
4846 default:
4847 break;
4848 }
4849 }
4850
4851 static void svm_cancel_injection(struct kvm_vcpu *vcpu)
4852 {
4853 struct vcpu_svm *svm = to_svm(vcpu);
4854 struct vmcb_control_area *control = &svm->vmcb->control;
4855
4856 control->exit_int_info = control->event_inj;
4857 control->exit_int_info_err = control->event_inj_err;
4858 control->event_inj = 0;
4859 svm_complete_interrupts(svm);
4860 }
4861
4862 static void svm_vcpu_run(struct kvm_vcpu *vcpu)
4863 {
4864 struct vcpu_svm *svm = to_svm(vcpu);
4865
4866 svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
4867 svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
4868 svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];
4869
4870 /*
4871 * A vmexit emulation is required before the vcpu can be executed
4872 * again.
4873 */
4874 if (unlikely(svm->nested.exit_required))
4875 return;
4876
4877 /*
4878 * Disable singlestep if we're injecting an interrupt/exception.
4879 * We don't want our modified rflags to be pushed on the stack where
4880 * we might not be able to easily reset them if we disabled NMI
4881 * singlestep later.
4882 */
4883 if (svm->nmi_singlestep && svm->vmcb->control.event_inj) {
4884 /*
4885 * Event injection happens before external interrupts cause a
4886 * vmexit and interrupts are disabled here, so smp_send_reschedule
4887 * is enough to force an immediate vmexit.
4888 */
4889 disable_nmi_singlestep(svm);
4890 smp_send_reschedule(vcpu->cpu);
4891 }
4892
4893 pre_svm_run(svm);
4894
4895 sync_lapic_to_cr8(vcpu);
4896
4897 svm->vmcb->save.cr2 = vcpu->arch.cr2;
4898
4899 clgi();
4900
4901 local_irq_enable();
4902
4903 if (ibrs_inuse && (svm->spec_ctrl != FEATURE_ENABLE_IBRS))
4904 wrmsrl(MSR_IA32_SPEC_CTRL, svm->spec_ctrl);
4905
4906 asm volatile (
4907 "push %%" _ASM_BP "; \n\t"
4908 "mov %c[rbx](%[svm]), %%" _ASM_BX " \n\t"
4909 "mov %c[rcx](%[svm]), %%" _ASM_CX " \n\t"
4910 "mov %c[rdx](%[svm]), %%" _ASM_DX " \n\t"
4911 "mov %c[rsi](%[svm]), %%" _ASM_SI " \n\t"
4912 "mov %c[rdi](%[svm]), %%" _ASM_DI " \n\t"
4913 "mov %c[rbp](%[svm]), %%" _ASM_BP " \n\t"
4914 #ifdef CONFIG_X86_64
4915 "mov %c[r8](%[svm]), %%r8 \n\t"
4916 "mov %c[r9](%[svm]), %%r9 \n\t"
4917 "mov %c[r10](%[svm]), %%r10 \n\t"
4918 "mov %c[r11](%[svm]), %%r11 \n\t"
4919 "mov %c[r12](%[svm]), %%r12 \n\t"
4920 "mov %c[r13](%[svm]), %%r13 \n\t"
4921 "mov %c[r14](%[svm]), %%r14 \n\t"
4922 "mov %c[r15](%[svm]), %%r15 \n\t"
4923 #endif
4924
4925 /* Enter guest mode */
4926 "push %%" _ASM_AX " \n\t"
4927 "mov %c[vmcb](%[svm]), %%" _ASM_AX " \n\t"
4928 __ex(SVM_VMLOAD) "\n\t"
4929 __ex(SVM_VMRUN) "\n\t"
4930 __ex(SVM_VMSAVE) "\n\t"
4931 "pop %%" _ASM_AX " \n\t"
4932
4933 /* Save guest registers, load host registers */
4934 "mov %%" _ASM_BX ", %c[rbx](%[svm]) \n\t"
4935 "mov %%" _ASM_CX ", %c[rcx](%[svm]) \n\t"
4936 "mov %%" _ASM_DX ", %c[rdx](%[svm]) \n\t"
4937 "mov %%" _ASM_SI ", %c[rsi](%[svm]) \n\t"
4938 "mov %%" _ASM_DI ", %c[rdi](%[svm]) \n\t"
4939 "mov %%" _ASM_BP ", %c[rbp](%[svm]) \n\t"
4940 #ifdef CONFIG_X86_64
4941 "mov %%r8, %c[r8](%[svm]) \n\t"
4942 "mov %%r9, %c[r9](%[svm]) \n\t"
4943 "mov %%r10, %c[r10](%[svm]) \n\t"
4944 "mov %%r11, %c[r11](%[svm]) \n\t"
4945 "mov %%r12, %c[r12](%[svm]) \n\t"
4946 "mov %%r13, %c[r13](%[svm]) \n\t"
4947 "mov %%r14, %c[r14](%[svm]) \n\t"
4948 "mov %%r15, %c[r15](%[svm]) \n\t"
4949 #endif
4950 "pop %%" _ASM_BP
4951 :
4952 : [svm]"a"(svm),
4953 [vmcb]"i"(offsetof(struct vcpu_svm, vmcb_pa)),
4954 [rbx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RBX])),
4955 [rcx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RCX])),
4956 [rdx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RDX])),
4957 [rsi]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RSI])),
4958 [rdi]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RDI])),
4959 [rbp]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RBP]))
4960 #ifdef CONFIG_X86_64
4961 , [r8]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R8])),
4962 [r9]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R9])),
4963 [r10]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R10])),
4964 [r11]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R11])),
4965 [r12]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R12])),
4966 [r13]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R13])),
4967 [r14]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R14])),
4968 [r15]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R15]))
4969 #endif
4970 : "cc", "memory"
4971 #ifdef CONFIG_X86_64
4972 , "rbx", "rcx", "rdx", "rsi", "rdi"
4973 , "r8", "r9", "r10", "r11" , "r12", "r13", "r14", "r15"
4974 #else
4975 , "ebx", "ecx", "edx", "esi", "edi"
4976 #endif
4977 );
4978
4979 if (ibrs_inuse) {
4980 rdmsrl(MSR_IA32_SPEC_CTRL, svm->spec_ctrl);
4981 if (svm->spec_ctrl != FEATURE_ENABLE_IBRS)
4982 wrmsrl(MSR_IA32_SPEC_CTRL, FEATURE_ENABLE_IBRS);
4983 }
4984
4985 stuff_RSB();
4986
4987 #ifdef CONFIG_X86_64
4988 wrmsrl(MSR_GS_BASE, svm->host.gs_base);
4989 #else
4990 loadsegment(fs, svm->host.fs);
4991 #ifndef CONFIG_X86_32_LAZY_GS
4992 loadsegment(gs, svm->host.gs);
4993 #endif
4994 #endif
4995
4996 reload_tss(vcpu);
4997
4998 local_irq_disable();
4999
5000 vcpu->arch.cr2 = svm->vmcb->save.cr2;
5001 vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax;
5002 vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp;
5003 vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip;
5004
5005 if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI))
5006 kvm_before_handle_nmi(&svm->vcpu);
5007
5008 stgi();
5009
5010 /* Any pending NMI will happen here */
5011
5012 if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI))
5013 kvm_after_handle_nmi(&svm->vcpu);
5014
5015 sync_cr8_to_lapic(vcpu);
5016
5017 svm->next_rip = 0;
5018
5019 svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING;
5020
5021 /* if exit due to PF check for async PF */
5022 if (svm->vmcb->control.exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR)
5023 svm->vcpu.arch.apf.host_apf_reason = kvm_read_and_reset_pf_reason();
5024
5025 if (npt_enabled) {
5026 vcpu->arch.regs_avail &= ~(1 << VCPU_EXREG_PDPTR);
5027 vcpu->arch.regs_dirty &= ~(1 << VCPU_EXREG_PDPTR);
5028 }
5029
5030 /*
5031 * We need to handle MC intercepts here before the vcpu has a chance to
5032 * change the physical cpu
5033 */
5034 if (unlikely(svm->vmcb->control.exit_code ==
5035 SVM_EXIT_EXCP_BASE + MC_VECTOR))
5036 svm_handle_mce(svm);
5037
5038 mark_all_clean(svm->vmcb);
5039 }
5040 STACK_FRAME_NON_STANDARD(svm_vcpu_run);
5041
5042 static void svm_set_cr3(struct kvm_vcpu *vcpu, unsigned long root)
5043 {
5044 struct vcpu_svm *svm = to_svm(vcpu);
5045
5046 svm->vmcb->save.cr3 = root;
5047 mark_dirty(svm->vmcb, VMCB_CR);
5048 svm_flush_tlb(vcpu);
5049 }
5050
5051 static void set_tdp_cr3(struct kvm_vcpu *vcpu, unsigned long root)
5052 {
5053 struct vcpu_svm *svm = to_svm(vcpu);
5054
5055 svm->vmcb->control.nested_cr3 = root;
5056 mark_dirty(svm->vmcb, VMCB_NPT);
5057
5058 /* Also sync guest cr3 here in case we live migrate */
5059 svm->vmcb->save.cr3 = kvm_read_cr3(vcpu);
5060 mark_dirty(svm->vmcb, VMCB_CR);
5061
5062 svm_flush_tlb(vcpu);
5063 }
5064
5065 static int is_disabled(void)
5066 {
5067 u64 vm_cr;
5068
5069 rdmsrl(MSR_VM_CR, vm_cr);
5070 if (vm_cr & (1 << SVM_VM_CR_SVM_DISABLE))
5071 return 1;
5072
5073 return 0;
5074 }
5075
5076 static void
5077 svm_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
5078 {
5079 /*
5080 * Patch in the VMMCALL instruction:
5081 */
5082 hypercall[0] = 0x0f;
5083 hypercall[1] = 0x01;
5084 hypercall[2] = 0xd9;
5085 }
5086
5087 static void svm_check_processor_compat(void *rtn)
5088 {
5089 *(int *)rtn = 0;
5090 }
5091
5092 static bool svm_cpu_has_accelerated_tpr(void)
5093 {
5094 return false;
5095 }
5096
5097 static bool svm_has_high_real_mode_segbase(void)
5098 {
5099 return true;
5100 }
5101
5102 static u64 svm_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
5103 {
5104 return 0;
5105 }
5106
5107 static void svm_cpuid_update(struct kvm_vcpu *vcpu)
5108 {
5109 struct vcpu_svm *svm = to_svm(vcpu);
5110 struct kvm_cpuid_entry2 *entry;
5111
5112 /* Update nrips enabled cache */
5113 svm->nrips_enabled = !!guest_cpuid_has_nrips(&svm->vcpu);
5114
5115 if (!kvm_vcpu_apicv_active(vcpu))
5116 return;
5117
5118 entry = kvm_find_cpuid_entry(vcpu, 1, 0);
5119 if (entry)
5120 entry->ecx &= ~bit(X86_FEATURE_X2APIC);
5121 }
5122
5123 static void svm_set_supported_cpuid(u32 func, struct kvm_cpuid_entry2 *entry)
5124 {
5125 switch (func) {
5126 case 0x1:
5127 if (avic)
5128 entry->ecx &= ~bit(X86_FEATURE_X2APIC);
5129 break;
5130 case 0x80000001:
5131 if (nested)
5132 entry->ecx |= (1 << 2); /* Set SVM bit */
5133 break;
5134 case 0x8000000A:
5135 entry->eax = 1; /* SVM revision 1 */
5136 entry->ebx = 8; /* Lets support 8 ASIDs in case we add proper
5137 ASID emulation to nested SVM */
5138 entry->ecx = 0; /* Reserved */
5139 entry->edx = 0; /* Per default do not support any
5140 additional features */
5141
5142 /* Support next_rip if host supports it */
5143 if (boot_cpu_has(X86_FEATURE_NRIPS))
5144 entry->edx |= SVM_FEATURE_NRIP;
5145
5146 /* Support NPT for the guest if enabled */
5147 if (npt_enabled)
5148 entry->edx |= SVM_FEATURE_NPT;
5149
5150 break;
5151 }
5152 }
5153
5154 static int svm_get_lpage_level(void)
5155 {
5156 return PT_PDPE_LEVEL;
5157 }
5158
5159 static bool svm_rdtscp_supported(void)
5160 {
5161 return boot_cpu_has(X86_FEATURE_RDTSCP);
5162 }
5163
5164 static bool svm_invpcid_supported(void)
5165 {
5166 return false;
5167 }
5168
5169 static bool svm_mpx_supported(void)
5170 {
5171 return false;
5172 }
5173
5174 static bool svm_xsaves_supported(void)
5175 {
5176 return false;
5177 }
5178
5179 static bool svm_has_wbinvd_exit(void)
5180 {
5181 return true;
5182 }
5183
5184 #define PRE_EX(exit) { .exit_code = (exit), \
5185 .stage = X86_ICPT_PRE_EXCEPT, }
5186 #define POST_EX(exit) { .exit_code = (exit), \
5187 .stage = X86_ICPT_POST_EXCEPT, }
5188 #define POST_MEM(exit) { .exit_code = (exit), \
5189 .stage = X86_ICPT_POST_MEMACCESS, }
5190
5191 static const struct __x86_intercept {
5192 u32 exit_code;
5193 enum x86_intercept_stage stage;
5194 } x86_intercept_map[] = {
5195 [x86_intercept_cr_read] = POST_EX(SVM_EXIT_READ_CR0),
5196 [x86_intercept_cr_write] = POST_EX(SVM_EXIT_WRITE_CR0),
5197 [x86_intercept_clts] = POST_EX(SVM_EXIT_WRITE_CR0),
5198 [x86_intercept_lmsw] = POST_EX(SVM_EXIT_WRITE_CR0),
5199 [x86_intercept_smsw] = POST_EX(SVM_EXIT_READ_CR0),
5200 [x86_intercept_dr_read] = POST_EX(SVM_EXIT_READ_DR0),
5201 [x86_intercept_dr_write] = POST_EX(SVM_EXIT_WRITE_DR0),
5202 [x86_intercept_sldt] = POST_EX(SVM_EXIT_LDTR_READ),
5203 [x86_intercept_str] = POST_EX(SVM_EXIT_TR_READ),
5204 [x86_intercept_lldt] = POST_EX(SVM_EXIT_LDTR_WRITE),
5205 [x86_intercept_ltr] = POST_EX(SVM_EXIT_TR_WRITE),
5206 [x86_intercept_sgdt] = POST_EX(SVM_EXIT_GDTR_READ),
5207 [x86_intercept_sidt] = POST_EX(SVM_EXIT_IDTR_READ),
5208 [x86_intercept_lgdt] = POST_EX(SVM_EXIT_GDTR_WRITE),
5209 [x86_intercept_lidt] = POST_EX(SVM_EXIT_IDTR_WRITE),
5210 [x86_intercept_vmrun] = POST_EX(SVM_EXIT_VMRUN),
5211 [x86_intercept_vmmcall] = POST_EX(SVM_EXIT_VMMCALL),
5212 [x86_intercept_vmload] = POST_EX(SVM_EXIT_VMLOAD),
5213 [x86_intercept_vmsave] = POST_EX(SVM_EXIT_VMSAVE),
5214 [x86_intercept_stgi] = POST_EX(SVM_EXIT_STGI),
5215 [x86_intercept_clgi] = POST_EX(SVM_EXIT_CLGI),
5216 [x86_intercept_skinit] = POST_EX(SVM_EXIT_SKINIT),
5217 [x86_intercept_invlpga] = POST_EX(SVM_EXIT_INVLPGA),
5218 [x86_intercept_rdtscp] = POST_EX(SVM_EXIT_RDTSCP),
5219 [x86_intercept_monitor] = POST_MEM(SVM_EXIT_MONITOR),
5220 [x86_intercept_mwait] = POST_EX(SVM_EXIT_MWAIT),
5221 [x86_intercept_invlpg] = POST_EX(SVM_EXIT_INVLPG),
5222 [x86_intercept_invd] = POST_EX(SVM_EXIT_INVD),
5223 [x86_intercept_wbinvd] = POST_EX(SVM_EXIT_WBINVD),
5224 [x86_intercept_wrmsr] = POST_EX(SVM_EXIT_MSR),
5225 [x86_intercept_rdtsc] = POST_EX(SVM_EXIT_RDTSC),
5226 [x86_intercept_rdmsr] = POST_EX(SVM_EXIT_MSR),
5227 [x86_intercept_rdpmc] = POST_EX(SVM_EXIT_RDPMC),
5228 [x86_intercept_cpuid] = PRE_EX(SVM_EXIT_CPUID),
5229 [x86_intercept_rsm] = PRE_EX(SVM_EXIT_RSM),
5230 [x86_intercept_pause] = PRE_EX(SVM_EXIT_PAUSE),
5231 [x86_intercept_pushf] = PRE_EX(SVM_EXIT_PUSHF),
5232 [x86_intercept_popf] = PRE_EX(SVM_EXIT_POPF),
5233 [x86_intercept_intn] = PRE_EX(SVM_EXIT_SWINT),
5234 [x86_intercept_iret] = PRE_EX(SVM_EXIT_IRET),
5235 [x86_intercept_icebp] = PRE_EX(SVM_EXIT_ICEBP),
5236 [x86_intercept_hlt] = POST_EX(SVM_EXIT_HLT),
5237 [x86_intercept_in] = POST_EX(SVM_EXIT_IOIO),
5238 [x86_intercept_ins] = POST_EX(SVM_EXIT_IOIO),
5239 [x86_intercept_out] = POST_EX(SVM_EXIT_IOIO),
5240 [x86_intercept_outs] = POST_EX(SVM_EXIT_IOIO),
5241 };
5242
5243 #undef PRE_EX
5244 #undef POST_EX
5245 #undef POST_MEM
5246
5247 static int svm_check_intercept(struct kvm_vcpu *vcpu,
5248 struct x86_instruction_info *info,
5249 enum x86_intercept_stage stage)
5250 {
5251 struct vcpu_svm *svm = to_svm(vcpu);
5252 int vmexit, ret = X86EMUL_CONTINUE;
5253 struct __x86_intercept icpt_info;
5254 struct vmcb *vmcb = svm->vmcb;
5255
5256 if (info->intercept >= ARRAY_SIZE(x86_intercept_map))
5257 goto out;
5258
5259 icpt_info = x86_intercept_map[info->intercept];
5260
5261 if (stage != icpt_info.stage)
5262 goto out;
5263
5264 switch (icpt_info.exit_code) {
5265 case SVM_EXIT_READ_CR0:
5266 if (info->intercept == x86_intercept_cr_read)
5267 icpt_info.exit_code += info->modrm_reg;
5268 break;
5269 case SVM_EXIT_WRITE_CR0: {
5270 unsigned long cr0, val;
5271 u64 intercept;
5272
5273 if (info->intercept == x86_intercept_cr_write)
5274 icpt_info.exit_code += info->modrm_reg;
5275
5276 if (icpt_info.exit_code != SVM_EXIT_WRITE_CR0 ||
5277 info->intercept == x86_intercept_clts)
5278 break;
5279
5280 intercept = svm->nested.intercept;
5281
5282 if (!(intercept & (1ULL << INTERCEPT_SELECTIVE_CR0)))
5283 break;
5284
5285 cr0 = vcpu->arch.cr0 & ~SVM_CR0_SELECTIVE_MASK;
5286 val = info->src_val & ~SVM_CR0_SELECTIVE_MASK;
5287
5288 if (info->intercept == x86_intercept_lmsw) {
5289 cr0 &= 0xfUL;
5290 val &= 0xfUL;
5291 /* lmsw can't clear PE - catch this here */
5292 if (cr0 & X86_CR0_PE)
5293 val |= X86_CR0_PE;
5294 }
5295
5296 if (cr0 ^ val)
5297 icpt_info.exit_code = SVM_EXIT_CR0_SEL_WRITE;
5298
5299 break;
5300 }
5301 case SVM_EXIT_READ_DR0:
5302 case SVM_EXIT_WRITE_DR0:
5303 icpt_info.exit_code += info->modrm_reg;
5304 break;
5305 case SVM_EXIT_MSR:
5306 if (info->intercept == x86_intercept_wrmsr)
5307 vmcb->control.exit_info_1 = 1;
5308 else
5309 vmcb->control.exit_info_1 = 0;
5310 break;
5311 case SVM_EXIT_PAUSE:
5312 /*
5313 * We get this for NOP only, but pause
5314 * is rep not, check this here
5315 */
5316 if (info->rep_prefix != REPE_PREFIX)
5317 goto out;
5318 case SVM_EXIT_IOIO: {
5319 u64 exit_info;
5320 u32 bytes;
5321
5322 if (info->intercept == x86_intercept_in ||
5323 info->intercept == x86_intercept_ins) {
5324 exit_info = ((info->src_val & 0xffff) << 16) |
5325 SVM_IOIO_TYPE_MASK;
5326 bytes = info->dst_bytes;
5327 } else {
5328 exit_info = (info->dst_val & 0xffff) << 16;
5329 bytes = info->src_bytes;
5330 }
5331
5332 if (info->intercept == x86_intercept_outs ||
5333 info->intercept == x86_intercept_ins)
5334 exit_info |= SVM_IOIO_STR_MASK;
5335
5336 if (info->rep_prefix)
5337 exit_info |= SVM_IOIO_REP_MASK;
5338
5339 bytes = min(bytes, 4u);
5340
5341 exit_info |= bytes << SVM_IOIO_SIZE_SHIFT;
5342
5343 exit_info |= (u32)info->ad_bytes << (SVM_IOIO_ASIZE_SHIFT - 1);
5344
5345 vmcb->control.exit_info_1 = exit_info;
5346 vmcb->control.exit_info_2 = info->next_rip;
5347
5348 break;
5349 }
5350 default:
5351 break;
5352 }
5353
5354 /* TODO: Advertise NRIPS to guest hypervisor unconditionally */
5355 if (static_cpu_has(X86_FEATURE_NRIPS))
5356 vmcb->control.next_rip = info->next_rip;
5357 vmcb->control.exit_code = icpt_info.exit_code;
5358 vmexit = nested_svm_exit_handled(svm);
5359
5360 ret = (vmexit == NESTED_EXIT_DONE) ? X86EMUL_INTERCEPTED
5361 : X86EMUL_CONTINUE;
5362
5363 out:
5364 return ret;
5365 }
5366
5367 static void svm_handle_external_intr(struct kvm_vcpu *vcpu)
5368 {
5369 local_irq_enable();
5370 /*
5371 * We must have an instruction with interrupts enabled, so
5372 * the timer interrupt isn't delayed by the interrupt shadow.
5373 */
5374 asm("nop");
5375 local_irq_disable();
5376 }
5377
5378 static void svm_sched_in(struct kvm_vcpu *vcpu, int cpu)
5379 {
5380 }
5381
5382 static inline void avic_post_state_restore(struct kvm_vcpu *vcpu)
5383 {
5384 if (avic_handle_apic_id_update(vcpu) != 0)
5385 return;
5386 if (avic_handle_dfr_update(vcpu) != 0)
5387 return;
5388 avic_handle_ldr_update(vcpu);
5389 }
5390
5391 static void svm_setup_mce(struct kvm_vcpu *vcpu)
5392 {
5393 /* [63:9] are reserved. */
5394 vcpu->arch.mcg_cap &= 0x1ff;
5395 }
5396
5397 static struct kvm_x86_ops svm_x86_ops __ro_after_init = {
5398 .cpu_has_kvm_support = has_svm,
5399 .disabled_by_bios = is_disabled,
5400 .hardware_setup = svm_hardware_setup,
5401 .hardware_unsetup = svm_hardware_unsetup,
5402 .check_processor_compatibility = svm_check_processor_compat,
5403 .hardware_enable = svm_hardware_enable,
5404 .hardware_disable = svm_hardware_disable,
5405 .cpu_has_accelerated_tpr = svm_cpu_has_accelerated_tpr,
5406 .cpu_has_high_real_mode_segbase = svm_has_high_real_mode_segbase,
5407
5408 .vcpu_create = svm_create_vcpu,
5409 .vcpu_free = svm_free_vcpu,
5410 .vcpu_reset = svm_vcpu_reset,
5411
5412 .vm_init = avic_vm_init,
5413 .vm_destroy = avic_vm_destroy,
5414
5415 .prepare_guest_switch = svm_prepare_guest_switch,
5416 .vcpu_load = svm_vcpu_load,
5417 .vcpu_put = svm_vcpu_put,
5418 .vcpu_blocking = svm_vcpu_blocking,
5419 .vcpu_unblocking = svm_vcpu_unblocking,
5420
5421 .update_bp_intercept = update_bp_intercept,
5422 .get_msr = svm_get_msr,
5423 .set_msr = svm_set_msr,
5424 .get_segment_base = svm_get_segment_base,
5425 .get_segment = svm_get_segment,
5426 .set_segment = svm_set_segment,
5427 .get_cpl = svm_get_cpl,
5428 .get_cs_db_l_bits = kvm_get_cs_db_l_bits,
5429 .decache_cr0_guest_bits = svm_decache_cr0_guest_bits,
5430 .decache_cr3 = svm_decache_cr3,
5431 .decache_cr4_guest_bits = svm_decache_cr4_guest_bits,
5432 .set_cr0 = svm_set_cr0,
5433 .set_cr3 = svm_set_cr3,
5434 .set_cr4 = svm_set_cr4,
5435 .set_efer = svm_set_efer,
5436 .get_idt = svm_get_idt,
5437 .set_idt = svm_set_idt,
5438 .get_gdt = svm_get_gdt,
5439 .set_gdt = svm_set_gdt,
5440 .get_dr6 = svm_get_dr6,
5441 .set_dr6 = svm_set_dr6,
5442 .set_dr7 = svm_set_dr7,
5443 .sync_dirty_debug_regs = svm_sync_dirty_debug_regs,
5444 .cache_reg = svm_cache_reg,
5445 .get_rflags = svm_get_rflags,
5446 .set_rflags = svm_set_rflags,
5447
5448 .tlb_flush = svm_flush_tlb,
5449
5450 .run = svm_vcpu_run,
5451 .handle_exit = handle_exit,
5452 .skip_emulated_instruction = skip_emulated_instruction,
5453 .set_interrupt_shadow = svm_set_interrupt_shadow,
5454 .get_interrupt_shadow = svm_get_interrupt_shadow,
5455 .patch_hypercall = svm_patch_hypercall,
5456 .set_irq = svm_set_irq,
5457 .set_nmi = svm_inject_nmi,
5458 .queue_exception = svm_queue_exception,
5459 .cancel_injection = svm_cancel_injection,
5460 .interrupt_allowed = svm_interrupt_allowed,
5461 .nmi_allowed = svm_nmi_allowed,
5462 .get_nmi_mask = svm_get_nmi_mask,
5463 .set_nmi_mask = svm_set_nmi_mask,
5464 .enable_nmi_window = enable_nmi_window,
5465 .enable_irq_window = enable_irq_window,
5466 .update_cr8_intercept = update_cr8_intercept,
5467 .set_virtual_x2apic_mode = svm_set_virtual_x2apic_mode,
5468 .get_enable_apicv = svm_get_enable_apicv,
5469 .refresh_apicv_exec_ctrl = svm_refresh_apicv_exec_ctrl,
5470 .load_eoi_exitmap = svm_load_eoi_exitmap,
5471 .hwapic_irr_update = svm_hwapic_irr_update,
5472 .hwapic_isr_update = svm_hwapic_isr_update,
5473 .apicv_post_state_restore = avic_post_state_restore,
5474
5475 .set_tss_addr = svm_set_tss_addr,
5476 .get_tdp_level = get_npt_level,
5477 .get_mt_mask = svm_get_mt_mask,
5478
5479 .get_exit_info = svm_get_exit_info,
5480
5481 .get_lpage_level = svm_get_lpage_level,
5482
5483 .cpuid_update = svm_cpuid_update,
5484
5485 .rdtscp_supported = svm_rdtscp_supported,
5486 .invpcid_supported = svm_invpcid_supported,
5487 .mpx_supported = svm_mpx_supported,
5488 .xsaves_supported = svm_xsaves_supported,
5489
5490 .set_supported_cpuid = svm_set_supported_cpuid,
5491
5492 .has_wbinvd_exit = svm_has_wbinvd_exit,
5493
5494 .write_tsc_offset = svm_write_tsc_offset,
5495
5496 .set_tdp_cr3 = set_tdp_cr3,
5497
5498 .check_intercept = svm_check_intercept,
5499 .handle_external_intr = svm_handle_external_intr,
5500
5501 .sched_in = svm_sched_in,
5502
5503 .pmu_ops = &amd_pmu_ops,
5504 .deliver_posted_interrupt = svm_deliver_avic_intr,
5505 .update_pi_irte = svm_update_pi_irte,
5506 .setup_mce = svm_setup_mce,
5507 };
5508
5509 static int __init svm_init(void)
5510 {
5511 return kvm_init(&svm_x86_ops, sizeof(struct vcpu_svm),
5512 __alignof__(struct vcpu_svm), THIS_MODULE);
5513 }
5514
5515 static void __exit svm_exit(void)
5516 {
5517 kvm_exit();
5518 }
5519
5520 module_init(svm_init)
5521 module_exit(svm_exit)
Ubuntu Artful kernel mirror