1 #define pr_fmt(fmt) "SVM: " fmt
3 #include <linux/kvm_host.h>
7 #include "kvm_cache_regs.h"
12 #include <linux/module.h>
13 #include <linux/mod_devicetable.h>
14 #include <linux/kernel.h>
15 #include <linux/vmalloc.h>
16 #include <linux/highmem.h>
17 #include <linux/amd-iommu.h>
18 #include <linux/sched.h>
19 #include <linux/trace_events.h>
20 #include <linux/slab.h>
21 #include <linux/hashtable.h>
22 #include <linux/frame.h>
23 #include <linux/psp-sev.h>
24 #include <linux/file.h>
25 #include <linux/pagemap.h>
26 #include <linux/swap.h>
27 #include <linux/rwsem.h>
30 #include <asm/perf_event.h>
31 #include <asm/tlbflush.h>
33 #include <asm/debugreg.h>
34 #include <asm/kvm_para.h>
35 #include <asm/irq_remapping.h>
37 #include <asm/spec-ctrl.h>
38 #include <asm/cpu_device_id.h>
40 #include <asm/virtext.h>
45 #define __ex(x) __kvm_handle_fault_on_reboot(x)
47 MODULE_AUTHOR("Qumranet");
48 MODULE_LICENSE("GPL");
51 static const struct x86_cpu_id svm_cpu_id
[] = {
52 X86_MATCH_FEATURE(X86_FEATURE_SVM
, NULL
),
55 MODULE_DEVICE_TABLE(x86cpu
, svm_cpu_id
);
58 #define IOPM_ALLOC_ORDER 2
59 #define MSRPM_ALLOC_ORDER 1
61 #define SEG_TYPE_LDT 2
62 #define SEG_TYPE_BUSY_TSS16 3
64 #define SVM_FEATURE_LBRV (1 << 1)
65 #define SVM_FEATURE_SVML (1 << 2)
66 #define SVM_FEATURE_TSC_RATE (1 << 4)
67 #define SVM_FEATURE_VMCB_CLEAN (1 << 5)
68 #define SVM_FEATURE_FLUSH_ASID (1 << 6)
69 #define SVM_FEATURE_DECODE_ASSIST (1 << 7)
70 #define SVM_FEATURE_PAUSE_FILTER (1 << 10)
72 #define DEBUGCTL_RESERVED_BITS (~(0x3fULL))
74 #define TSC_RATIO_RSVD 0xffffff0000000000ULL
75 #define TSC_RATIO_MIN 0x0000000000000001ULL
76 #define TSC_RATIO_MAX 0x000000ffffffffffULL
78 static bool erratum_383_found __read_mostly
;
80 u32 msrpm_offsets
[MSRPM_OFFSETS
] __read_mostly
;
83 * Set osvw_len to higher value when updated Revision Guides
84 * are published and we know what the new status bits are
86 static uint64_t osvw_len
= 4, osvw_status
;
88 static DEFINE_PER_CPU(u64
, current_tsc_ratio
);
89 #define TSC_RATIO_DEFAULT 0x0100000000ULL
91 static const struct svm_direct_access_msrs
{
92 u32 index
; /* Index of the MSR */
93 bool always
; /* True if intercept is always on */
94 } direct_access_msrs
[] = {
95 { .index
= MSR_STAR
, .always
= true },
96 { .index
= MSR_IA32_SYSENTER_CS
, .always
= true },
98 { .index
= MSR_GS_BASE
, .always
= true },
99 { .index
= MSR_FS_BASE
, .always
= true },
100 { .index
= MSR_KERNEL_GS_BASE
, .always
= true },
101 { .index
= MSR_LSTAR
, .always
= true },
102 { .index
= MSR_CSTAR
, .always
= true },
103 { .index
= MSR_SYSCALL_MASK
, .always
= true },
105 { .index
= MSR_IA32_SPEC_CTRL
, .always
= false },
106 { .index
= MSR_IA32_PRED_CMD
, .always
= false },
107 { .index
= MSR_IA32_LASTBRANCHFROMIP
, .always
= false },
108 { .index
= MSR_IA32_LASTBRANCHTOIP
, .always
= false },
109 { .index
= MSR_IA32_LASTINTFROMIP
, .always
= false },
110 { .index
= MSR_IA32_LASTINTTOIP
, .always
= false },
111 { .index
= MSR_INVALID
, .always
= false },
114 /* enable NPT for AMD64 and X86 with PAE */
115 #if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
116 bool npt_enabled
= true;
122 * These 2 parameters are used to config the controls for Pause-Loop Exiting:
123 * pause_filter_count: On processors that support Pause filtering(indicated
124 * by CPUID Fn8000_000A_EDX), the VMCB provides a 16 bit pause filter
125 * count value. On VMRUN this value is loaded into an internal counter.
126 * Each time a pause instruction is executed, this counter is decremented
127 * until it reaches zero at which time a #VMEXIT is generated if pause
128 * intercept is enabled. Refer to AMD APM Vol 2 Section 15.14.4 Pause
129 * Intercept Filtering for more details.
130 * This also indicate if ple logic enabled.
132 * pause_filter_thresh: In addition, some processor families support advanced
133 * pause filtering (indicated by CPUID Fn8000_000A_EDX) upper bound on
134 * the amount of time a guest is allowed to execute in a pause loop.
135 * In this mode, a 16-bit pause filter threshold field is added in the
136 * VMCB. The threshold value is a cycle count that is used to reset the
137 * pause counter. As with simple pause filtering, VMRUN loads the pause
138 * count value from VMCB into an internal counter. Then, on each pause
139 * instruction the hardware checks the elapsed number of cycles since
140 * the most recent pause instruction against the pause filter threshold.
141 * If the elapsed cycle count is greater than the pause filter threshold,
142 * then the internal pause count is reloaded from the VMCB and execution
143 * continues. If the elapsed cycle count is less than the pause filter
144 * threshold, then the internal pause count is decremented. If the count
145 * value is less than zero and PAUSE intercept is enabled, a #VMEXIT is
146 * triggered. If advanced pause filtering is supported and pause filter
147 * threshold field is set to zero, the filter will operate in the simpler,
151 static unsigned short pause_filter_thresh
= KVM_DEFAULT_PLE_GAP
;
152 module_param(pause_filter_thresh
, ushort
, 0444);
154 static unsigned short pause_filter_count
= KVM_SVM_DEFAULT_PLE_WINDOW
;
155 module_param(pause_filter_count
, ushort
, 0444);
157 /* Default doubles per-vcpu window every exit. */
158 static unsigned short pause_filter_count_grow
= KVM_DEFAULT_PLE_WINDOW_GROW
;
159 module_param(pause_filter_count_grow
, ushort
, 0444);
161 /* Default resets per-vcpu window every exit to pause_filter_count. */
162 static unsigned short pause_filter_count_shrink
= KVM_DEFAULT_PLE_WINDOW_SHRINK
;
163 module_param(pause_filter_count_shrink
, ushort
, 0444);
165 /* Default is to compute the maximum so we can never overflow. */
166 static unsigned short pause_filter_count_max
= KVM_SVM_DEFAULT_PLE_WINDOW_MAX
;
167 module_param(pause_filter_count_max
, ushort
, 0444);
169 /* allow nested paging (virtualized MMU) for all guests */
170 static int npt
= true;
171 module_param(npt
, int, S_IRUGO
);
173 /* allow nested virtualization in KVM/SVM */
174 static int nested
= true;
175 module_param(nested
, int, S_IRUGO
);
177 /* enable/disable Next RIP Save */
178 static int nrips
= true;
179 module_param(nrips
, int, 0444);
181 /* enable/disable Virtual VMLOAD VMSAVE */
182 static int vls
= true;
183 module_param(vls
, int, 0444);
185 /* enable/disable Virtual GIF */
186 static int vgif
= true;
187 module_param(vgif
, int, 0444);
189 /* enable/disable SEV support */
190 static int sev
= IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT_ACTIVE_BY_DEFAULT
);
191 module_param(sev
, int, 0444);
193 static bool __read_mostly dump_invalid_vmcb
= 0;
194 module_param(dump_invalid_vmcb
, bool, 0644);
196 static u8 rsm_ins_bytes
[] = "\x0f\xaa";
198 static void svm_complete_interrupts(struct vcpu_svm
*svm
);
200 static unsigned long iopm_base
;
202 struct kvm_ldttss_desc
{
205 unsigned base1
:8, type
:5, dpl
:2, p
:1;
206 unsigned limit1
:4, zero0
:3, g
:1, base2
:8;
209 } __attribute__((packed
));
211 DEFINE_PER_CPU(struct svm_cpu_data
*, svm_data
);
213 static const u32 msrpm_ranges
[] = {0, 0xc0000000, 0xc0010000};
215 #define NUM_MSR_MAPS ARRAY_SIZE(msrpm_ranges)
216 #define MSRS_RANGE_SIZE 2048
217 #define MSRS_IN_RANGE (MSRS_RANGE_SIZE * 8 / 2)
219 u32
svm_msrpm_offset(u32 msr
)
224 for (i
= 0; i
< NUM_MSR_MAPS
; i
++) {
225 if (msr
< msrpm_ranges
[i
] ||
226 msr
>= msrpm_ranges
[i
] + MSRS_IN_RANGE
)
229 offset
= (msr
- msrpm_ranges
[i
]) / 4; /* 4 msrs per u8 */
230 offset
+= (i
* MSRS_RANGE_SIZE
); /* add range offset */
232 /* Now we have the u8 offset - but need the u32 offset */
236 /* MSR not in any range */
240 #define MAX_INST_SIZE 15
242 static inline void clgi(void)
244 asm volatile (__ex("clgi"));
247 static inline void stgi(void)
249 asm volatile (__ex("stgi"));
252 static inline void invlpga(unsigned long addr
, u32 asid
)
254 asm volatile (__ex("invlpga %1, %0") : : "c"(asid
), "a"(addr
));
257 static int get_max_npt_level(void)
260 return PT64_ROOT_4LEVEL
;
262 return PT32E_ROOT_LEVEL
;
266 void svm_set_efer(struct kvm_vcpu
*vcpu
, u64 efer
)
268 struct vcpu_svm
*svm
= to_svm(vcpu
);
269 vcpu
->arch
.efer
= efer
;
272 /* Shadow paging assumes NX to be available. */
275 if (!(efer
& EFER_LMA
))
279 if (!(efer
& EFER_SVME
)) {
280 svm_leave_nested(svm
);
281 svm_set_gif(svm
, true);
284 svm
->vmcb
->save
.efer
= efer
| EFER_SVME
;
285 vmcb_mark_dirty(svm
->vmcb
, VMCB_CR
);
288 static int is_external_interrupt(u32 info
)
290 info
&= SVM_EVTINJ_TYPE_MASK
| SVM_EVTINJ_VALID
;
291 return info
== (SVM_EVTINJ_VALID
| SVM_EVTINJ_TYPE_INTR
);
294 static u32
svm_get_interrupt_shadow(struct kvm_vcpu
*vcpu
)
296 struct vcpu_svm
*svm
= to_svm(vcpu
);
299 if (svm
->vmcb
->control
.int_state
& SVM_INTERRUPT_SHADOW_MASK
)
300 ret
= KVM_X86_SHADOW_INT_STI
| KVM_X86_SHADOW_INT_MOV_SS
;
304 static void svm_set_interrupt_shadow(struct kvm_vcpu
*vcpu
, int mask
)
306 struct vcpu_svm
*svm
= to_svm(vcpu
);
309 svm
->vmcb
->control
.int_state
&= ~SVM_INTERRUPT_SHADOW_MASK
;
311 svm
->vmcb
->control
.int_state
|= SVM_INTERRUPT_SHADOW_MASK
;
315 static int skip_emulated_instruction(struct kvm_vcpu
*vcpu
)
317 struct vcpu_svm
*svm
= to_svm(vcpu
);
319 if (nrips
&& svm
->vmcb
->control
.next_rip
!= 0) {
320 WARN_ON_ONCE(!static_cpu_has(X86_FEATURE_NRIPS
));
321 svm
->next_rip
= svm
->vmcb
->control
.next_rip
;
324 if (!svm
->next_rip
) {
325 if (!kvm_emulate_instruction(vcpu
, EMULTYPE_SKIP
))
328 kvm_rip_write(vcpu
, svm
->next_rip
);
330 svm_set_interrupt_shadow(vcpu
, 0);
335 static void svm_queue_exception(struct kvm_vcpu
*vcpu
)
337 struct vcpu_svm
*svm
= to_svm(vcpu
);
338 unsigned nr
= vcpu
->arch
.exception
.nr
;
339 bool has_error_code
= vcpu
->arch
.exception
.has_error_code
;
340 u32 error_code
= vcpu
->arch
.exception
.error_code
;
342 kvm_deliver_exception_payload(&svm
->vcpu
);
344 if (nr
== BP_VECTOR
&& !nrips
) {
345 unsigned long rip
, old_rip
= kvm_rip_read(&svm
->vcpu
);
348 * For guest debugging where we have to reinject #BP if some
349 * INT3 is guest-owned:
350 * Emulate nRIP by moving RIP forward. Will fail if injection
351 * raises a fault that is not intercepted. Still better than
352 * failing in all cases.
354 (void)skip_emulated_instruction(&svm
->vcpu
);
355 rip
= kvm_rip_read(&svm
->vcpu
);
356 svm
->int3_rip
= rip
+ svm
->vmcb
->save
.cs
.base
;
357 svm
->int3_injected
= rip
- old_rip
;
360 svm
->vmcb
->control
.event_inj
= nr
362 | (has_error_code
? SVM_EVTINJ_VALID_ERR
: 0)
363 | SVM_EVTINJ_TYPE_EXEPT
;
364 svm
->vmcb
->control
.event_inj_err
= error_code
;
367 static void svm_init_erratum_383(void)
373 if (!static_cpu_has_bug(X86_BUG_AMD_TLB_MMATCH
))
376 /* Use _safe variants to not break nested virtualization */
377 val
= native_read_msr_safe(MSR_AMD64_DC_CFG
, &err
);
383 low
= lower_32_bits(val
);
384 high
= upper_32_bits(val
);
386 native_write_msr_safe(MSR_AMD64_DC_CFG
, low
, high
);
388 erratum_383_found
= true;
391 static void svm_init_osvw(struct kvm_vcpu
*vcpu
)
394 * Guests should see errata 400 and 415 as fixed (assuming that
395 * HLT and IO instructions are intercepted).
397 vcpu
->arch
.osvw
.length
= (osvw_len
>= 3) ? (osvw_len
) : 3;
398 vcpu
->arch
.osvw
.status
= osvw_status
& ~(6ULL);
401 * By increasing VCPU's osvw.length to 3 we are telling the guest that
402 * all osvw.status bits inside that length, including bit 0 (which is
403 * reserved for erratum 298), are valid. However, if host processor's
404 * osvw_len is 0 then osvw_status[0] carries no information. We need to
405 * be conservative here and therefore we tell the guest that erratum 298
406 * is present (because we really don't know).
408 if (osvw_len
== 0 && boot_cpu_data
.x86
== 0x10)
409 vcpu
->arch
.osvw
.status
|= 1;
412 static int has_svm(void)
416 if (!cpu_has_svm(&msg
)) {
417 printk(KERN_INFO
"has_svm: %s\n", msg
);
424 static void svm_hardware_disable(void)
426 /* Make sure we clean up behind us */
427 if (static_cpu_has(X86_FEATURE_TSCRATEMSR
))
428 wrmsrl(MSR_AMD64_TSC_RATIO
, TSC_RATIO_DEFAULT
);
432 amd_pmu_disable_virt();
435 static int svm_hardware_enable(void)
438 struct svm_cpu_data
*sd
;
440 struct desc_struct
*gdt
;
441 int me
= raw_smp_processor_id();
443 rdmsrl(MSR_EFER
, efer
);
444 if (efer
& EFER_SVME
)
448 pr_err("%s: err EOPNOTSUPP on %d\n", __func__
, me
);
451 sd
= per_cpu(svm_data
, me
);
453 pr_err("%s: svm_data is NULL on %d\n", __func__
, me
);
457 sd
->asid_generation
= 1;
458 sd
->max_asid
= cpuid_ebx(SVM_CPUID_FUNC
) - 1;
459 sd
->next_asid
= sd
->max_asid
+ 1;
460 sd
->min_asid
= max_sev_asid
+ 1;
462 gdt
= get_current_gdt_rw();
463 sd
->tss_desc
= (struct kvm_ldttss_desc
*)(gdt
+ GDT_ENTRY_TSS
);
465 wrmsrl(MSR_EFER
, efer
| EFER_SVME
);
467 wrmsrl(MSR_VM_HSAVE_PA
, page_to_pfn(sd
->save_area
) << PAGE_SHIFT
);
469 if (static_cpu_has(X86_FEATURE_TSCRATEMSR
)) {
470 wrmsrl(MSR_AMD64_TSC_RATIO
, TSC_RATIO_DEFAULT
);
471 __this_cpu_write(current_tsc_ratio
, TSC_RATIO_DEFAULT
);
478 * Note that it is possible to have a system with mixed processor
479 * revisions and therefore different OSVW bits. If bits are not the same
480 * on different processors then choose the worst case (i.e. if erratum
481 * is present on one processor and not on another then assume that the
482 * erratum is present everywhere).
484 if (cpu_has(&boot_cpu_data
, X86_FEATURE_OSVW
)) {
485 uint64_t len
, status
= 0;
488 len
= native_read_msr_safe(MSR_AMD64_OSVW_ID_LENGTH
, &err
);
490 status
= native_read_msr_safe(MSR_AMD64_OSVW_STATUS
,
494 osvw_status
= osvw_len
= 0;
498 osvw_status
|= status
;
499 osvw_status
&= (1ULL << osvw_len
) - 1;
502 osvw_status
= osvw_len
= 0;
504 svm_init_erratum_383();
506 amd_pmu_enable_virt();
511 static void svm_cpu_uninit(int cpu
)
513 struct svm_cpu_data
*sd
= per_cpu(svm_data
, raw_smp_processor_id());
518 per_cpu(svm_data
, raw_smp_processor_id()) = NULL
;
519 kfree(sd
->sev_vmcbs
);
520 __free_page(sd
->save_area
);
524 static int svm_cpu_init(int cpu
)
526 struct svm_cpu_data
*sd
;
528 sd
= kzalloc(sizeof(struct svm_cpu_data
), GFP_KERNEL
);
532 sd
->save_area
= alloc_page(GFP_KERNEL
);
536 if (svm_sev_enabled()) {
537 sd
->sev_vmcbs
= kmalloc_array(max_sev_asid
+ 1,
544 per_cpu(svm_data
, cpu
) = sd
;
549 __free_page(sd
->save_area
);
556 static bool valid_msr_intercept(u32 index
)
560 for (i
= 0; direct_access_msrs
[i
].index
!= MSR_INVALID
; i
++)
561 if (direct_access_msrs
[i
].index
== index
)
567 static bool msr_write_intercepted(struct kvm_vcpu
*vcpu
, unsigned msr
)
574 msrpm
= is_guest_mode(vcpu
) ? to_svm(vcpu
)->nested
.msrpm
:
577 offset
= svm_msrpm_offset(msr
);
578 bit_write
= 2 * (msr
& 0x0f) + 1;
581 BUG_ON(offset
== MSR_INVALID
);
583 return !!test_bit(bit_write
, &tmp
);
586 static void set_msr_interception(u32
*msrpm
, unsigned msr
,
589 u8 bit_read
, bit_write
;
594 * If this warning triggers extend the direct_access_msrs list at the
595 * beginning of the file
597 WARN_ON(!valid_msr_intercept(msr
));
599 offset
= svm_msrpm_offset(msr
);
600 bit_read
= 2 * (msr
& 0x0f);
601 bit_write
= 2 * (msr
& 0x0f) + 1;
604 BUG_ON(offset
== MSR_INVALID
);
606 read
? clear_bit(bit_read
, &tmp
) : set_bit(bit_read
, &tmp
);
607 write
? clear_bit(bit_write
, &tmp
) : set_bit(bit_write
, &tmp
);
612 static void svm_vcpu_init_msrpm(u32
*msrpm
)
616 memset(msrpm
, 0xff, PAGE_SIZE
* (1 << MSRPM_ALLOC_ORDER
));
618 for (i
= 0; direct_access_msrs
[i
].index
!= MSR_INVALID
; i
++) {
619 if (!direct_access_msrs
[i
].always
)
622 set_msr_interception(msrpm
, direct_access_msrs
[i
].index
, 1, 1);
626 static void add_msr_offset(u32 offset
)
630 for (i
= 0; i
< MSRPM_OFFSETS
; ++i
) {
632 /* Offset already in list? */
633 if (msrpm_offsets
[i
] == offset
)
636 /* Slot used by another offset? */
637 if (msrpm_offsets
[i
] != MSR_INVALID
)
640 /* Add offset to list */
641 msrpm_offsets
[i
] = offset
;
647 * If this BUG triggers the msrpm_offsets table has an overflow. Just
648 * increase MSRPM_OFFSETS in this case.
653 static void init_msrpm_offsets(void)
657 memset(msrpm_offsets
, 0xff, sizeof(msrpm_offsets
));
659 for (i
= 0; direct_access_msrs
[i
].index
!= MSR_INVALID
; i
++) {
662 offset
= svm_msrpm_offset(direct_access_msrs
[i
].index
);
663 BUG_ON(offset
== MSR_INVALID
);
665 add_msr_offset(offset
);
669 static void svm_enable_lbrv(struct vcpu_svm
*svm
)
671 u32
*msrpm
= svm
->msrpm
;
673 svm
->vmcb
->control
.virt_ext
|= LBR_CTL_ENABLE_MASK
;
674 set_msr_interception(msrpm
, MSR_IA32_LASTBRANCHFROMIP
, 1, 1);
675 set_msr_interception(msrpm
, MSR_IA32_LASTBRANCHTOIP
, 1, 1);
676 set_msr_interception(msrpm
, MSR_IA32_LASTINTFROMIP
, 1, 1);
677 set_msr_interception(msrpm
, MSR_IA32_LASTINTTOIP
, 1, 1);
680 static void svm_disable_lbrv(struct vcpu_svm
*svm
)
682 u32
*msrpm
= svm
->msrpm
;
684 svm
->vmcb
->control
.virt_ext
&= ~LBR_CTL_ENABLE_MASK
;
685 set_msr_interception(msrpm
, MSR_IA32_LASTBRANCHFROMIP
, 0, 0);
686 set_msr_interception(msrpm
, MSR_IA32_LASTBRANCHTOIP
, 0, 0);
687 set_msr_interception(msrpm
, MSR_IA32_LASTINTFROMIP
, 0, 0);
688 set_msr_interception(msrpm
, MSR_IA32_LASTINTTOIP
, 0, 0);
691 void disable_nmi_singlestep(struct vcpu_svm
*svm
)
693 svm
->nmi_singlestep
= false;
695 if (!(svm
->vcpu
.guest_debug
& KVM_GUESTDBG_SINGLESTEP
)) {
696 /* Clear our flags if they were not set by the guest */
697 if (!(svm
->nmi_singlestep_guest_rflags
& X86_EFLAGS_TF
))
698 svm
->vmcb
->save
.rflags
&= ~X86_EFLAGS_TF
;
699 if (!(svm
->nmi_singlestep_guest_rflags
& X86_EFLAGS_RF
))
700 svm
->vmcb
->save
.rflags
&= ~X86_EFLAGS_RF
;
704 static void grow_ple_window(struct kvm_vcpu
*vcpu
)
706 struct vcpu_svm
*svm
= to_svm(vcpu
);
707 struct vmcb_control_area
*control
= &svm
->vmcb
->control
;
708 int old
= control
->pause_filter_count
;
710 control
->pause_filter_count
= __grow_ple_window(old
,
712 pause_filter_count_grow
,
713 pause_filter_count_max
);
715 if (control
->pause_filter_count
!= old
) {
716 vmcb_mark_dirty(svm
->vmcb
, VMCB_INTERCEPTS
);
717 trace_kvm_ple_window_update(vcpu
->vcpu_id
,
718 control
->pause_filter_count
, old
);
722 static void shrink_ple_window(struct kvm_vcpu
*vcpu
)
724 struct vcpu_svm
*svm
= to_svm(vcpu
);
725 struct vmcb_control_area
*control
= &svm
->vmcb
->control
;
726 int old
= control
->pause_filter_count
;
728 control
->pause_filter_count
=
729 __shrink_ple_window(old
,
731 pause_filter_count_shrink
,
733 if (control
->pause_filter_count
!= old
) {
734 vmcb_mark_dirty(svm
->vmcb
, VMCB_INTERCEPTS
);
735 trace_kvm_ple_window_update(vcpu
->vcpu_id
,
736 control
->pause_filter_count
, old
);
741 * The default MMIO mask is a single bit (excluding the present bit),
742 * which could conflict with the memory encryption bit. Check for
743 * memory encryption support and override the default MMIO mask if
744 * memory encryption is enabled.
746 static __init
void svm_adjust_mmio_mask(void)
748 unsigned int enc_bit
, mask_bit
;
751 /* If there is no memory encryption support, use existing mask */
752 if (cpuid_eax(0x80000000) < 0x8000001f)
755 /* If memory encryption is not enabled, use existing mask */
756 rdmsrl(MSR_K8_SYSCFG
, msr
);
757 if (!(msr
& MSR_K8_SYSCFG_MEM_ENCRYPT
))
760 enc_bit
= cpuid_ebx(0x8000001f) & 0x3f;
761 mask_bit
= boot_cpu_data
.x86_phys_bits
;
763 /* Increment the mask bit if it is the same as the encryption bit */
764 if (enc_bit
== mask_bit
)
768 * If the mask bit location is below 52, then some bits above the
769 * physical addressing limit will always be reserved, so use the
770 * rsvd_bits() function to generate the mask. This mask, along with
771 * the present bit, will be used to generate a page fault with
774 * If the mask bit location is 52 (or above), then clear the mask.
776 mask
= (mask_bit
< 52) ? rsvd_bits(mask_bit
, 51) | PT_PRESENT_MASK
: 0;
778 kvm_mmu_set_mmio_spte_mask(mask
, PT_WRITABLE_MASK
| PT_USER_MASK
);
781 static void svm_hardware_teardown(void)
785 if (svm_sev_enabled())
786 sev_hardware_teardown();
788 for_each_possible_cpu(cpu
)
791 __free_pages(pfn_to_page(iopm_base
>> PAGE_SHIFT
), IOPM_ALLOC_ORDER
);
795 static __init
void svm_set_cpu_caps(void)
801 /* CPUID 0x80000001 and 0x8000000A (SVM features) */
803 kvm_cpu_cap_set(X86_FEATURE_SVM
);
806 kvm_cpu_cap_set(X86_FEATURE_NRIPS
);
809 kvm_cpu_cap_set(X86_FEATURE_NPT
);
812 /* CPUID 0x80000008 */
813 if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD
) ||
814 boot_cpu_has(X86_FEATURE_AMD_SSBD
))
815 kvm_cpu_cap_set(X86_FEATURE_VIRT_SSBD
);
818 static __init
int svm_hardware_setup(void)
821 struct page
*iopm_pages
;
825 iopm_pages
= alloc_pages(GFP_KERNEL
, IOPM_ALLOC_ORDER
);
830 iopm_va
= page_address(iopm_pages
);
831 memset(iopm_va
, 0xff, PAGE_SIZE
* (1 << IOPM_ALLOC_ORDER
));
832 iopm_base
= page_to_pfn(iopm_pages
) << PAGE_SHIFT
;
834 init_msrpm_offsets();
836 supported_xcr0
&= ~(XFEATURE_MASK_BNDREGS
| XFEATURE_MASK_BNDCSR
);
838 if (boot_cpu_has(X86_FEATURE_NX
))
839 kvm_enable_efer_bits(EFER_NX
);
841 if (boot_cpu_has(X86_FEATURE_FXSR_OPT
))
842 kvm_enable_efer_bits(EFER_FFXSR
);
844 if (boot_cpu_has(X86_FEATURE_TSCRATEMSR
)) {
845 kvm_has_tsc_control
= true;
846 kvm_max_tsc_scaling_ratio
= TSC_RATIO_MAX
;
847 kvm_tsc_scaling_ratio_frac_bits
= 32;
850 /* Check for pause filtering support */
851 if (!boot_cpu_has(X86_FEATURE_PAUSEFILTER
)) {
852 pause_filter_count
= 0;
853 pause_filter_thresh
= 0;
854 } else if (!boot_cpu_has(X86_FEATURE_PFTHRESHOLD
)) {
855 pause_filter_thresh
= 0;
859 printk(KERN_INFO
"kvm: Nested Virtualization enabled\n");
860 kvm_enable_efer_bits(EFER_SVME
| EFER_LMSLE
);
864 if (boot_cpu_has(X86_FEATURE_SEV
) &&
865 IS_ENABLED(CONFIG_KVM_AMD_SEV
)) {
866 r
= sev_hardware_setup();
874 svm_adjust_mmio_mask();
876 for_each_possible_cpu(cpu
) {
877 r
= svm_cpu_init(cpu
);
882 if (!boot_cpu_has(X86_FEATURE_NPT
))
885 if (npt_enabled
&& !npt
)
888 kvm_configure_mmu(npt_enabled
, get_max_npt_level(), PG_LEVEL_1G
);
889 pr_info("kvm: Nested Paging %sabled\n", npt_enabled
? "en" : "dis");
892 if (!boot_cpu_has(X86_FEATURE_NRIPS
))
898 !boot_cpu_has(X86_FEATURE_AVIC
) ||
899 !IS_ENABLED(CONFIG_X86_LOCAL_APIC
)) {
902 pr_info("AVIC enabled\n");
904 amd_iommu_register_ga_log_notifier(&avic_ga_log_notifier
);
910 !boot_cpu_has(X86_FEATURE_V_VMSAVE_VMLOAD
) ||
911 !IS_ENABLED(CONFIG_X86_64
)) {
914 pr_info("Virtual VMLOAD VMSAVE supported\n");
919 if (!boot_cpu_has(X86_FEATURE_VGIF
))
922 pr_info("Virtual GIF supported\n");
928 * It seems that on AMD processors PTE's accessed bit is
929 * being set by the CPU hardware before the NPF vmexit.
930 * This is not expected behaviour and our tests fail because
932 * A workaround here is to disable support for
933 * GUEST_MAXPHYADDR < HOST_MAXPHYADDR if NPT is enabled.
934 * In this case userspace can know if there is support using
935 * KVM_CAP_SMALLER_MAXPHYADDR extension and decide how to handle
937 * If future AMD CPU models change the behaviour described above,
938 * this variable can be changed accordingly
940 allow_smaller_maxphyaddr
= !npt_enabled
;
945 svm_hardware_teardown();
949 static void init_seg(struct vmcb_seg
*seg
)
952 seg
->attrib
= SVM_SELECTOR_P_MASK
| SVM_SELECTOR_S_MASK
|
953 SVM_SELECTOR_WRITE_MASK
; /* Read/Write Data Segment */
958 static void init_sys_seg(struct vmcb_seg
*seg
, uint32_t type
)
961 seg
->attrib
= SVM_SELECTOR_P_MASK
| type
;
966 static u64
svm_write_l1_tsc_offset(struct kvm_vcpu
*vcpu
, u64 offset
)
968 struct vcpu_svm
*svm
= to_svm(vcpu
);
969 u64 g_tsc_offset
= 0;
971 if (is_guest_mode(vcpu
)) {
972 /* Write L1's TSC offset. */
973 g_tsc_offset
= svm
->vmcb
->control
.tsc_offset
-
974 svm
->nested
.hsave
->control
.tsc_offset
;
975 svm
->nested
.hsave
->control
.tsc_offset
= offset
;
978 trace_kvm_write_tsc_offset(vcpu
->vcpu_id
,
979 svm
->vmcb
->control
.tsc_offset
- g_tsc_offset
,
982 svm
->vmcb
->control
.tsc_offset
= offset
+ g_tsc_offset
;
984 vmcb_mark_dirty(svm
->vmcb
, VMCB_INTERCEPTS
);
985 return svm
->vmcb
->control
.tsc_offset
;
988 static void init_vmcb(struct vcpu_svm
*svm
)
990 struct vmcb_control_area
*control
= &svm
->vmcb
->control
;
991 struct vmcb_save_area
*save
= &svm
->vmcb
->save
;
993 svm
->vcpu
.arch
.hflags
= 0;
995 set_cr_intercept(svm
, INTERCEPT_CR0_READ
);
996 set_cr_intercept(svm
, INTERCEPT_CR3_READ
);
997 set_cr_intercept(svm
, INTERCEPT_CR4_READ
);
998 set_cr_intercept(svm
, INTERCEPT_CR0_WRITE
);
999 set_cr_intercept(svm
, INTERCEPT_CR3_WRITE
);
1000 set_cr_intercept(svm
, INTERCEPT_CR4_WRITE
);
1001 if (!kvm_vcpu_apicv_active(&svm
->vcpu
))
1002 set_cr_intercept(svm
, INTERCEPT_CR8_WRITE
);
1004 set_dr_intercepts(svm
);
1006 set_exception_intercept(svm
, PF_VECTOR
);
1007 set_exception_intercept(svm
, UD_VECTOR
);
1008 set_exception_intercept(svm
, MC_VECTOR
);
1009 set_exception_intercept(svm
, AC_VECTOR
);
1010 set_exception_intercept(svm
, DB_VECTOR
);
1012 * Guest access to VMware backdoor ports could legitimately
1013 * trigger #GP because of TSS I/O permission bitmap.
1014 * We intercept those #GP and allow access to them anyway
1017 if (enable_vmware_backdoor
)
1018 set_exception_intercept(svm
, GP_VECTOR
);
1020 svm_set_intercept(svm
, INTERCEPT_INTR
);
1021 svm_set_intercept(svm
, INTERCEPT_NMI
);
1022 svm_set_intercept(svm
, INTERCEPT_SMI
);
1023 svm_set_intercept(svm
, INTERCEPT_SELECTIVE_CR0
);
1024 svm_set_intercept(svm
, INTERCEPT_RDPMC
);
1025 svm_set_intercept(svm
, INTERCEPT_CPUID
);
1026 svm_set_intercept(svm
, INTERCEPT_INVD
);
1027 svm_set_intercept(svm
, INTERCEPT_INVLPG
);
1028 svm_set_intercept(svm
, INTERCEPT_INVLPGA
);
1029 svm_set_intercept(svm
, INTERCEPT_IOIO_PROT
);
1030 svm_set_intercept(svm
, INTERCEPT_MSR_PROT
);
1031 svm_set_intercept(svm
, INTERCEPT_TASK_SWITCH
);
1032 svm_set_intercept(svm
, INTERCEPT_SHUTDOWN
);
1033 svm_set_intercept(svm
, INTERCEPT_VMRUN
);
1034 svm_set_intercept(svm
, INTERCEPT_VMMCALL
);
1035 svm_set_intercept(svm
, INTERCEPT_VMLOAD
);
1036 svm_set_intercept(svm
, INTERCEPT_VMSAVE
);
1037 svm_set_intercept(svm
, INTERCEPT_STGI
);
1038 svm_set_intercept(svm
, INTERCEPT_CLGI
);
1039 svm_set_intercept(svm
, INTERCEPT_SKINIT
);
1040 svm_set_intercept(svm
, INTERCEPT_WBINVD
);
1041 svm_set_intercept(svm
, INTERCEPT_XSETBV
);
1042 svm_set_intercept(svm
, INTERCEPT_RDPRU
);
1043 svm_set_intercept(svm
, INTERCEPT_RSM
);
1045 if (!kvm_mwait_in_guest(svm
->vcpu
.kvm
)) {
1046 svm_set_intercept(svm
, INTERCEPT_MONITOR
);
1047 svm_set_intercept(svm
, INTERCEPT_MWAIT
);
1050 if (!kvm_hlt_in_guest(svm
->vcpu
.kvm
))
1051 svm_set_intercept(svm
, INTERCEPT_HLT
);
1053 control
->iopm_base_pa
= __sme_set(iopm_base
);
1054 control
->msrpm_base_pa
= __sme_set(__pa(svm
->msrpm
));
1055 control
->int_ctl
= V_INTR_MASKING_MASK
;
1057 init_seg(&save
->es
);
1058 init_seg(&save
->ss
);
1059 init_seg(&save
->ds
);
1060 init_seg(&save
->fs
);
1061 init_seg(&save
->gs
);
1063 save
->cs
.selector
= 0xf000;
1064 save
->cs
.base
= 0xffff0000;
1065 /* Executable/Readable Code Segment */
1066 save
->cs
.attrib
= SVM_SELECTOR_READ_MASK
| SVM_SELECTOR_P_MASK
|
1067 SVM_SELECTOR_S_MASK
| SVM_SELECTOR_CODE_MASK
;
1068 save
->cs
.limit
= 0xffff;
1070 save
->gdtr
.limit
= 0xffff;
1071 save
->idtr
.limit
= 0xffff;
1073 init_sys_seg(&save
->ldtr
, SEG_TYPE_LDT
);
1074 init_sys_seg(&save
->tr
, SEG_TYPE_BUSY_TSS16
);
1076 svm_set_efer(&svm
->vcpu
, 0);
1077 save
->dr6
= 0xffff0ff0;
1078 kvm_set_rflags(&svm
->vcpu
, 2);
1079 save
->rip
= 0x0000fff0;
1080 svm
->vcpu
.arch
.regs
[VCPU_REGS_RIP
] = save
->rip
;
1083 * svm_set_cr0() sets PG and WP and clears NW and CD on save->cr0.
1084 * It also updates the guest-visible cr0 value.
1086 svm_set_cr0(&svm
->vcpu
, X86_CR0_NW
| X86_CR0_CD
| X86_CR0_ET
);
1087 kvm_mmu_reset_context(&svm
->vcpu
);
1089 save
->cr4
= X86_CR4_PAE
;
1093 /* Setup VMCB for Nested Paging */
1094 control
->nested_ctl
|= SVM_NESTED_CTL_NP_ENABLE
;
1095 svm_clr_intercept(svm
, INTERCEPT_INVLPG
);
1096 clr_exception_intercept(svm
, PF_VECTOR
);
1097 clr_cr_intercept(svm
, INTERCEPT_CR3_READ
);
1098 clr_cr_intercept(svm
, INTERCEPT_CR3_WRITE
);
1099 save
->g_pat
= svm
->vcpu
.arch
.pat
;
1103 svm
->asid_generation
= 0;
1105 svm
->nested
.vmcb
= 0;
1106 svm
->vcpu
.arch
.hflags
= 0;
1108 if (!kvm_pause_in_guest(svm
->vcpu
.kvm
)) {
1109 control
->pause_filter_count
= pause_filter_count
;
1110 if (pause_filter_thresh
)
1111 control
->pause_filter_thresh
= pause_filter_thresh
;
1112 svm_set_intercept(svm
, INTERCEPT_PAUSE
);
1114 svm_clr_intercept(svm
, INTERCEPT_PAUSE
);
1117 if (kvm_vcpu_apicv_active(&svm
->vcpu
))
1118 avic_init_vmcb(svm
);
1121 * If hardware supports Virtual VMLOAD VMSAVE then enable it
1122 * in VMCB and clear intercepts to avoid #VMEXIT.
1125 svm_clr_intercept(svm
, INTERCEPT_VMLOAD
);
1126 svm_clr_intercept(svm
, INTERCEPT_VMSAVE
);
1127 svm
->vmcb
->control
.virt_ext
|= VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK
;
1131 svm_clr_intercept(svm
, INTERCEPT_STGI
);
1132 svm_clr_intercept(svm
, INTERCEPT_CLGI
);
1133 svm
->vmcb
->control
.int_ctl
|= V_GIF_ENABLE_MASK
;
1136 if (sev_guest(svm
->vcpu
.kvm
)) {
1137 svm
->vmcb
->control
.nested_ctl
|= SVM_NESTED_CTL_SEV_ENABLE
;
1138 clr_exception_intercept(svm
, UD_VECTOR
);
1141 vmcb_mark_all_dirty(svm
->vmcb
);
1147 static void svm_vcpu_reset(struct kvm_vcpu
*vcpu
, bool init_event
)
1149 struct vcpu_svm
*svm
= to_svm(vcpu
);
1154 svm
->virt_spec_ctrl
= 0;
1157 svm
->vcpu
.arch
.apic_base
= APIC_DEFAULT_PHYS_BASE
|
1158 MSR_IA32_APICBASE_ENABLE
;
1159 if (kvm_vcpu_is_reset_bsp(&svm
->vcpu
))
1160 svm
->vcpu
.arch
.apic_base
|= MSR_IA32_APICBASE_BSP
;
1164 kvm_cpuid(vcpu
, &eax
, &dummy
, &dummy
, &dummy
, false);
1165 kvm_rdx_write(vcpu
, eax
);
1167 if (kvm_vcpu_apicv_active(vcpu
) && !init_event
)
1168 avic_update_vapic_bar(svm
, APIC_DEFAULT_PHYS_BASE
);
1171 static int svm_create_vcpu(struct kvm_vcpu
*vcpu
)
1173 struct vcpu_svm
*svm
;
1175 struct page
*msrpm_pages
;
1176 struct page
*hsave_page
;
1177 struct page
*nested_msrpm_pages
;
1180 BUILD_BUG_ON(offsetof(struct vcpu_svm
, vcpu
) != 0);
1184 page
= alloc_page(GFP_KERNEL_ACCOUNT
);
1188 msrpm_pages
= alloc_pages(GFP_KERNEL_ACCOUNT
, MSRPM_ALLOC_ORDER
);
1192 nested_msrpm_pages
= alloc_pages(GFP_KERNEL_ACCOUNT
, MSRPM_ALLOC_ORDER
);
1193 if (!nested_msrpm_pages
)
1196 hsave_page
= alloc_page(GFP_KERNEL_ACCOUNT
);
1200 err
= avic_init_vcpu(svm
);
1204 /* We initialize this flag to true to make sure that the is_running
1205 * bit would be set the first time the vcpu is loaded.
1207 if (irqchip_in_kernel(vcpu
->kvm
) && kvm_apicv_activated(vcpu
->kvm
))
1208 svm
->avic_is_running
= true;
1210 svm
->nested
.hsave
= page_address(hsave_page
);
1211 clear_page(svm
->nested
.hsave
);
1213 svm
->msrpm
= page_address(msrpm_pages
);
1214 svm_vcpu_init_msrpm(svm
->msrpm
);
1216 svm
->nested
.msrpm
= page_address(nested_msrpm_pages
);
1217 svm_vcpu_init_msrpm(svm
->nested
.msrpm
);
1219 svm
->vmcb
= page_address(page
);
1220 clear_page(svm
->vmcb
);
1221 svm
->vmcb_pa
= __sme_set(page_to_pfn(page
) << PAGE_SHIFT
);
1222 svm
->asid_generation
= 0;
1225 svm_init_osvw(vcpu
);
1226 vcpu
->arch
.microcode_version
= 0x01000065;
1231 __free_page(hsave_page
);
1233 __free_pages(nested_msrpm_pages
, MSRPM_ALLOC_ORDER
);
1235 __free_pages(msrpm_pages
, MSRPM_ALLOC_ORDER
);
1242 static void svm_clear_current_vmcb(struct vmcb
*vmcb
)
1246 for_each_online_cpu(i
)
1247 cmpxchg(&per_cpu(svm_data
, i
)->current_vmcb
, vmcb
, NULL
);
1250 static void svm_free_vcpu(struct kvm_vcpu
*vcpu
)
1252 struct vcpu_svm
*svm
= to_svm(vcpu
);
1255 * The vmcb page can be recycled, causing a false negative in
1256 * svm_vcpu_load(). So, ensure that no logical CPU has this
1257 * vmcb page recorded as its current vmcb.
1259 svm_clear_current_vmcb(svm
->vmcb
);
1261 __free_page(pfn_to_page(__sme_clr(svm
->vmcb_pa
) >> PAGE_SHIFT
));
1262 __free_pages(virt_to_page(svm
->msrpm
), MSRPM_ALLOC_ORDER
);
1263 __free_page(virt_to_page(svm
->nested
.hsave
));
1264 __free_pages(virt_to_page(svm
->nested
.msrpm
), MSRPM_ALLOC_ORDER
);
1267 static void svm_vcpu_load(struct kvm_vcpu
*vcpu
, int cpu
)
1269 struct vcpu_svm
*svm
= to_svm(vcpu
);
1270 struct svm_cpu_data
*sd
= per_cpu(svm_data
, cpu
);
1273 if (unlikely(cpu
!= vcpu
->cpu
)) {
1274 svm
->asid_generation
= 0;
1275 vmcb_mark_all_dirty(svm
->vmcb
);
1278 #ifdef CONFIG_X86_64
1279 rdmsrl(MSR_GS_BASE
, to_svm(vcpu
)->host
.gs_base
);
1281 savesegment(fs
, svm
->host
.fs
);
1282 savesegment(gs
, svm
->host
.gs
);
1283 svm
->host
.ldt
= kvm_read_ldt();
1285 for (i
= 0; i
< NR_HOST_SAVE_USER_MSRS
; i
++)
1286 rdmsrl(host_save_user_msrs
[i
], svm
->host_user_msrs
[i
]);
1288 if (static_cpu_has(X86_FEATURE_TSCRATEMSR
)) {
1289 u64 tsc_ratio
= vcpu
->arch
.tsc_scaling_ratio
;
1290 if (tsc_ratio
!= __this_cpu_read(current_tsc_ratio
)) {
1291 __this_cpu_write(current_tsc_ratio
, tsc_ratio
);
1292 wrmsrl(MSR_AMD64_TSC_RATIO
, tsc_ratio
);
1295 /* This assumes that the kernel never uses MSR_TSC_AUX */
1296 if (static_cpu_has(X86_FEATURE_RDTSCP
))
1297 wrmsrl(MSR_TSC_AUX
, svm
->tsc_aux
);
1299 if (sd
->current_vmcb
!= svm
->vmcb
) {
1300 sd
->current_vmcb
= svm
->vmcb
;
1301 indirect_branch_prediction_barrier();
1303 avic_vcpu_load(vcpu
, cpu
);
1306 static void svm_vcpu_put(struct kvm_vcpu
*vcpu
)
1308 struct vcpu_svm
*svm
= to_svm(vcpu
);
1311 avic_vcpu_put(vcpu
);
1313 ++vcpu
->stat
.host_state_reload
;
1314 kvm_load_ldt(svm
->host
.ldt
);
1315 #ifdef CONFIG_X86_64
1316 loadsegment(fs
, svm
->host
.fs
);
1317 wrmsrl(MSR_KERNEL_GS_BASE
, current
->thread
.gsbase
);
1318 load_gs_index(svm
->host
.gs
);
1320 #ifdef CONFIG_X86_32_LAZY_GS
1321 loadsegment(gs
, svm
->host
.gs
);
1324 for (i
= 0; i
< NR_HOST_SAVE_USER_MSRS
; i
++)
1325 wrmsrl(host_save_user_msrs
[i
], svm
->host_user_msrs
[i
]);
1328 static unsigned long svm_get_rflags(struct kvm_vcpu
*vcpu
)
1330 struct vcpu_svm
*svm
= to_svm(vcpu
);
1331 unsigned long rflags
= svm
->vmcb
->save
.rflags
;
1333 if (svm
->nmi_singlestep
) {
1334 /* Hide our flags if they were not set by the guest */
1335 if (!(svm
->nmi_singlestep_guest_rflags
& X86_EFLAGS_TF
))
1336 rflags
&= ~X86_EFLAGS_TF
;
1337 if (!(svm
->nmi_singlestep_guest_rflags
& X86_EFLAGS_RF
))
1338 rflags
&= ~X86_EFLAGS_RF
;
1343 static void svm_set_rflags(struct kvm_vcpu
*vcpu
, unsigned long rflags
)
1345 if (to_svm(vcpu
)->nmi_singlestep
)
1346 rflags
|= (X86_EFLAGS_TF
| X86_EFLAGS_RF
);
1349 * Any change of EFLAGS.VM is accompanied by a reload of SS
1350 * (caused by either a task switch or an inter-privilege IRET),
1351 * so we do not need to update the CPL here.
1353 to_svm(vcpu
)->vmcb
->save
.rflags
= rflags
;
1356 static void svm_cache_reg(struct kvm_vcpu
*vcpu
, enum kvm_reg reg
)
1359 case VCPU_EXREG_PDPTR
:
1360 BUG_ON(!npt_enabled
);
1361 load_pdptrs(vcpu
, vcpu
->arch
.walk_mmu
, kvm_read_cr3(vcpu
));
1368 static void svm_set_vintr(struct vcpu_svm
*svm
)
1370 struct vmcb_control_area
*control
;
1372 /* The following fields are ignored when AVIC is enabled */
1373 WARN_ON(kvm_vcpu_apicv_active(&svm
->vcpu
));
1374 svm_set_intercept(svm
, INTERCEPT_VINTR
);
1377 * This is just a dummy VINTR to actually cause a vmexit to happen.
1378 * Actual injection of virtual interrupts happens through EVENTINJ.
1380 control
= &svm
->vmcb
->control
;
1381 control
->int_vector
= 0x0;
1382 control
->int_ctl
&= ~V_INTR_PRIO_MASK
;
1383 control
->int_ctl
|= V_IRQ_MASK
|
1384 ((/*control->int_vector >> 4*/ 0xf) << V_INTR_PRIO_SHIFT
);
1385 vmcb_mark_dirty(svm
->vmcb
, VMCB_INTR
);
1388 static void svm_clear_vintr(struct vcpu_svm
*svm
)
1390 const u32 mask
= V_TPR_MASK
| V_GIF_ENABLE_MASK
| V_GIF_MASK
| V_INTR_MASKING_MASK
;
1391 svm_clr_intercept(svm
, INTERCEPT_VINTR
);
1393 /* Drop int_ctl fields related to VINTR injection. */
1394 svm
->vmcb
->control
.int_ctl
&= mask
;
1395 if (is_guest_mode(&svm
->vcpu
)) {
1396 svm
->nested
.hsave
->control
.int_ctl
&= mask
;
1398 WARN_ON((svm
->vmcb
->control
.int_ctl
& V_TPR_MASK
) !=
1399 (svm
->nested
.ctl
.int_ctl
& V_TPR_MASK
));
1400 svm
->vmcb
->control
.int_ctl
|= svm
->nested
.ctl
.int_ctl
& ~mask
;
1403 vmcb_mark_dirty(svm
->vmcb
, VMCB_INTR
);
1406 static struct vmcb_seg
*svm_seg(struct kvm_vcpu
*vcpu
, int seg
)
1408 struct vmcb_save_area
*save
= &to_svm(vcpu
)->vmcb
->save
;
1411 case VCPU_SREG_CS
: return &save
->cs
;
1412 case VCPU_SREG_DS
: return &save
->ds
;
1413 case VCPU_SREG_ES
: return &save
->es
;
1414 case VCPU_SREG_FS
: return &save
->fs
;
1415 case VCPU_SREG_GS
: return &save
->gs
;
1416 case VCPU_SREG_SS
: return &save
->ss
;
1417 case VCPU_SREG_TR
: return &save
->tr
;
1418 case VCPU_SREG_LDTR
: return &save
->ldtr
;
1424 static u64
svm_get_segment_base(struct kvm_vcpu
*vcpu
, int seg
)
1426 struct vmcb_seg
*s
= svm_seg(vcpu
, seg
);
1431 static void svm_get_segment(struct kvm_vcpu
*vcpu
,
1432 struct kvm_segment
*var
, int seg
)
1434 struct vmcb_seg
*s
= svm_seg(vcpu
, seg
);
1436 var
->base
= s
->base
;
1437 var
->limit
= s
->limit
;
1438 var
->selector
= s
->selector
;
1439 var
->type
= s
->attrib
& SVM_SELECTOR_TYPE_MASK
;
1440 var
->s
= (s
->attrib
>> SVM_SELECTOR_S_SHIFT
) & 1;
1441 var
->dpl
= (s
->attrib
>> SVM_SELECTOR_DPL_SHIFT
) & 3;
1442 var
->present
= (s
->attrib
>> SVM_SELECTOR_P_SHIFT
) & 1;
1443 var
->avl
= (s
->attrib
>> SVM_SELECTOR_AVL_SHIFT
) & 1;
1444 var
->l
= (s
->attrib
>> SVM_SELECTOR_L_SHIFT
) & 1;
1445 var
->db
= (s
->attrib
>> SVM_SELECTOR_DB_SHIFT
) & 1;
1448 * AMD CPUs circa 2014 track the G bit for all segments except CS.
1449 * However, the SVM spec states that the G bit is not observed by the
1450 * CPU, and some VMware virtual CPUs drop the G bit for all segments.
1451 * So let's synthesize a legal G bit for all segments, this helps
1452 * running KVM nested. It also helps cross-vendor migration, because
1453 * Intel's vmentry has a check on the 'G' bit.
1455 var
->g
= s
->limit
> 0xfffff;
1458 * AMD's VMCB does not have an explicit unusable field, so emulate it
1459 * for cross vendor migration purposes by "not present"
1461 var
->unusable
= !var
->present
;
1466 * Work around a bug where the busy flag in the tr selector
1476 * The accessed bit must always be set in the segment
1477 * descriptor cache, although it can be cleared in the
1478 * descriptor, the cached bit always remains at 1. Since
1479 * Intel has a check on this, set it here to support
1480 * cross-vendor migration.
1487 * On AMD CPUs sometimes the DB bit in the segment
1488 * descriptor is left as 1, although the whole segment has
1489 * been made unusable. Clear it here to pass an Intel VMX
1490 * entry check when cross vendor migrating.
1494 /* This is symmetric with svm_set_segment() */
1495 var
->dpl
= to_svm(vcpu
)->vmcb
->save
.cpl
;
1500 static int svm_get_cpl(struct kvm_vcpu
*vcpu
)
1502 struct vmcb_save_area
*save
= &to_svm(vcpu
)->vmcb
->save
;
1507 static void svm_get_idt(struct kvm_vcpu
*vcpu
, struct desc_ptr
*dt
)
1509 struct vcpu_svm
*svm
= to_svm(vcpu
);
1511 dt
->size
= svm
->vmcb
->save
.idtr
.limit
;
1512 dt
->address
= svm
->vmcb
->save
.idtr
.base
;
1515 static void svm_set_idt(struct kvm_vcpu
*vcpu
, struct desc_ptr
*dt
)
1517 struct vcpu_svm
*svm
= to_svm(vcpu
);
1519 svm
->vmcb
->save
.idtr
.limit
= dt
->size
;
1520 svm
->vmcb
->save
.idtr
.base
= dt
->address
;
1521 vmcb_mark_dirty(svm
->vmcb
, VMCB_DT
);
1524 static void svm_get_gdt(struct kvm_vcpu
*vcpu
, struct desc_ptr
*dt
)
1526 struct vcpu_svm
*svm
= to_svm(vcpu
);
1528 dt
->size
= svm
->vmcb
->save
.gdtr
.limit
;
1529 dt
->address
= svm
->vmcb
->save
.gdtr
.base
;
1532 static void svm_set_gdt(struct kvm_vcpu
*vcpu
, struct desc_ptr
*dt
)
1534 struct vcpu_svm
*svm
= to_svm(vcpu
);
1536 svm
->vmcb
->save
.gdtr
.limit
= dt
->size
;
1537 svm
->vmcb
->save
.gdtr
.base
= dt
->address
;
1538 vmcb_mark_dirty(svm
->vmcb
, VMCB_DT
);
1541 static void update_cr0_intercept(struct vcpu_svm
*svm
)
1543 ulong gcr0
= svm
->vcpu
.arch
.cr0
;
1544 u64
*hcr0
= &svm
->vmcb
->save
.cr0
;
1546 *hcr0
= (*hcr0
& ~SVM_CR0_SELECTIVE_MASK
)
1547 | (gcr0
& SVM_CR0_SELECTIVE_MASK
);
1549 vmcb_mark_dirty(svm
->vmcb
, VMCB_CR
);
1551 if (gcr0
== *hcr0
) {
1552 clr_cr_intercept(svm
, INTERCEPT_CR0_READ
);
1553 clr_cr_intercept(svm
, INTERCEPT_CR0_WRITE
);
1555 set_cr_intercept(svm
, INTERCEPT_CR0_READ
);
1556 set_cr_intercept(svm
, INTERCEPT_CR0_WRITE
);
1560 void svm_set_cr0(struct kvm_vcpu
*vcpu
, unsigned long cr0
)
1562 struct vcpu_svm
*svm
= to_svm(vcpu
);
1564 #ifdef CONFIG_X86_64
1565 if (vcpu
->arch
.efer
& EFER_LME
) {
1566 if (!is_paging(vcpu
) && (cr0
& X86_CR0_PG
)) {
1567 vcpu
->arch
.efer
|= EFER_LMA
;
1568 svm
->vmcb
->save
.efer
|= EFER_LMA
| EFER_LME
;
1571 if (is_paging(vcpu
) && !(cr0
& X86_CR0_PG
)) {
1572 vcpu
->arch
.efer
&= ~EFER_LMA
;
1573 svm
->vmcb
->save
.efer
&= ~(EFER_LMA
| EFER_LME
);
1577 vcpu
->arch
.cr0
= cr0
;
1580 cr0
|= X86_CR0_PG
| X86_CR0_WP
;
1583 * re-enable caching here because the QEMU bios
1584 * does not do it - this results in some delay at
1587 if (kvm_check_has_quirk(vcpu
->kvm
, KVM_X86_QUIRK_CD_NW_CLEARED
))
1588 cr0
&= ~(X86_CR0_CD
| X86_CR0_NW
);
1589 svm
->vmcb
->save
.cr0
= cr0
;
1590 vmcb_mark_dirty(svm
->vmcb
, VMCB_CR
);
1591 update_cr0_intercept(svm
);
1594 int svm_set_cr4(struct kvm_vcpu
*vcpu
, unsigned long cr4
)
1596 unsigned long host_cr4_mce
= cr4_read_shadow() & X86_CR4_MCE
;
1597 unsigned long old_cr4
= to_svm(vcpu
)->vmcb
->save
.cr4
;
1599 if (cr4
& X86_CR4_VMXE
)
1602 if (npt_enabled
&& ((old_cr4
^ cr4
) & X86_CR4_PGE
))
1603 svm_flush_tlb(vcpu
);
1605 vcpu
->arch
.cr4
= cr4
;
1608 cr4
|= host_cr4_mce
;
1609 to_svm(vcpu
)->vmcb
->save
.cr4
= cr4
;
1610 vmcb_mark_dirty(to_svm(vcpu
)->vmcb
, VMCB_CR
);
1614 static void svm_set_segment(struct kvm_vcpu
*vcpu
,
1615 struct kvm_segment
*var
, int seg
)
1617 struct vcpu_svm
*svm
= to_svm(vcpu
);
1618 struct vmcb_seg
*s
= svm_seg(vcpu
, seg
);
1620 s
->base
= var
->base
;
1621 s
->limit
= var
->limit
;
1622 s
->selector
= var
->selector
;
1623 s
->attrib
= (var
->type
& SVM_SELECTOR_TYPE_MASK
);
1624 s
->attrib
|= (var
->s
& 1) << SVM_SELECTOR_S_SHIFT
;
1625 s
->attrib
|= (var
->dpl
& 3) << SVM_SELECTOR_DPL_SHIFT
;
1626 s
->attrib
|= ((var
->present
& 1) && !var
->unusable
) << SVM_SELECTOR_P_SHIFT
;
1627 s
->attrib
|= (var
->avl
& 1) << SVM_SELECTOR_AVL_SHIFT
;
1628 s
->attrib
|= (var
->l
& 1) << SVM_SELECTOR_L_SHIFT
;
1629 s
->attrib
|= (var
->db
& 1) << SVM_SELECTOR_DB_SHIFT
;
1630 s
->attrib
|= (var
->g
& 1) << SVM_SELECTOR_G_SHIFT
;
1633 * This is always accurate, except if SYSRET returned to a segment
1634 * with SS.DPL != 3. Intel does not have this quirk, and always
1635 * forces SS.DPL to 3 on sysret, so we ignore that case; fixing it
1636 * would entail passing the CPL to userspace and back.
1638 if (seg
== VCPU_SREG_SS
)
1639 /* This is symmetric with svm_get_segment() */
1640 svm
->vmcb
->save
.cpl
= (var
->dpl
& 3);
1642 vmcb_mark_dirty(svm
->vmcb
, VMCB_SEG
);
1645 static void update_exception_bitmap(struct kvm_vcpu
*vcpu
)
1647 struct vcpu_svm
*svm
= to_svm(vcpu
);
1649 clr_exception_intercept(svm
, BP_VECTOR
);
1651 if (vcpu
->guest_debug
& KVM_GUESTDBG_ENABLE
) {
1652 if (vcpu
->guest_debug
& KVM_GUESTDBG_USE_SW_BP
)
1653 set_exception_intercept(svm
, BP_VECTOR
);
1657 static void new_asid(struct vcpu_svm
*svm
, struct svm_cpu_data
*sd
)
1659 if (sd
->next_asid
> sd
->max_asid
) {
1660 ++sd
->asid_generation
;
1661 sd
->next_asid
= sd
->min_asid
;
1662 svm
->vmcb
->control
.tlb_ctl
= TLB_CONTROL_FLUSH_ALL_ASID
;
1665 svm
->asid_generation
= sd
->asid_generation
;
1666 svm
->vmcb
->control
.asid
= sd
->next_asid
++;
1668 vmcb_mark_dirty(svm
->vmcb
, VMCB_ASID
);
1671 static void svm_set_dr6(struct vcpu_svm
*svm
, unsigned long value
)
1673 struct vmcb
*vmcb
= svm
->vmcb
;
1675 if (unlikely(value
!= vmcb
->save
.dr6
)) {
1676 vmcb
->save
.dr6
= value
;
1677 vmcb_mark_dirty(vmcb
, VMCB_DR
);
1681 static void svm_sync_dirty_debug_regs(struct kvm_vcpu
*vcpu
)
1683 struct vcpu_svm
*svm
= to_svm(vcpu
);
1685 get_debugreg(vcpu
->arch
.db
[0], 0);
1686 get_debugreg(vcpu
->arch
.db
[1], 1);
1687 get_debugreg(vcpu
->arch
.db
[2], 2);
1688 get_debugreg(vcpu
->arch
.db
[3], 3);
1690 * We cannot reset svm->vmcb->save.dr6 to DR6_FIXED_1|DR6_RTM here,
1691 * because db_interception might need it. We can do it before vmentry.
1693 vcpu
->arch
.dr6
= svm
->vmcb
->save
.dr6
;
1694 vcpu
->arch
.dr7
= svm
->vmcb
->save
.dr7
;
1695 vcpu
->arch
.switch_db_regs
&= ~KVM_DEBUGREG_WONT_EXIT
;
1696 set_dr_intercepts(svm
);
1699 static void svm_set_dr7(struct kvm_vcpu
*vcpu
, unsigned long value
)
1701 struct vcpu_svm
*svm
= to_svm(vcpu
);
1703 svm
->vmcb
->save
.dr7
= value
;
1704 vmcb_mark_dirty(svm
->vmcb
, VMCB_DR
);
1707 static int pf_interception(struct vcpu_svm
*svm
)
1709 u64 fault_address
= __sme_clr(svm
->vmcb
->control
.exit_info_2
);
1710 u64 error_code
= svm
->vmcb
->control
.exit_info_1
;
1712 return kvm_handle_page_fault(&svm
->vcpu
, error_code
, fault_address
,
1713 static_cpu_has(X86_FEATURE_DECODEASSISTS
) ?
1714 svm
->vmcb
->control
.insn_bytes
: NULL
,
1715 svm
->vmcb
->control
.insn_len
);
1718 static int npf_interception(struct vcpu_svm
*svm
)
1720 u64 fault_address
= __sme_clr(svm
->vmcb
->control
.exit_info_2
);
1721 u64 error_code
= svm
->vmcb
->control
.exit_info_1
;
1723 trace_kvm_page_fault(fault_address
, error_code
);
1724 return kvm_mmu_page_fault(&svm
->vcpu
, fault_address
, error_code
,
1725 static_cpu_has(X86_FEATURE_DECODEASSISTS
) ?
1726 svm
->vmcb
->control
.insn_bytes
: NULL
,
1727 svm
->vmcb
->control
.insn_len
);
1730 static int db_interception(struct vcpu_svm
*svm
)
1732 struct kvm_run
*kvm_run
= svm
->vcpu
.run
;
1733 struct kvm_vcpu
*vcpu
= &svm
->vcpu
;
1735 if (!(svm
->vcpu
.guest_debug
&
1736 (KVM_GUESTDBG_SINGLESTEP
| KVM_GUESTDBG_USE_HW_BP
)) &&
1737 !svm
->nmi_singlestep
) {
1738 u32 payload
= (svm
->vmcb
->save
.dr6
^ DR6_RTM
) & ~DR6_FIXED_1
;
1739 kvm_queue_exception_p(&svm
->vcpu
, DB_VECTOR
, payload
);
1743 if (svm
->nmi_singlestep
) {
1744 disable_nmi_singlestep(svm
);
1745 /* Make sure we check for pending NMIs upon entry */
1746 kvm_make_request(KVM_REQ_EVENT
, vcpu
);
1749 if (svm
->vcpu
.guest_debug
&
1750 (KVM_GUESTDBG_SINGLESTEP
| KVM_GUESTDBG_USE_HW_BP
)) {
1751 kvm_run
->exit_reason
= KVM_EXIT_DEBUG
;
1752 kvm_run
->debug
.arch
.dr6
= svm
->vmcb
->save
.dr6
;
1753 kvm_run
->debug
.arch
.dr7
= svm
->vmcb
->save
.dr7
;
1754 kvm_run
->debug
.arch
.pc
=
1755 svm
->vmcb
->save
.cs
.base
+ svm
->vmcb
->save
.rip
;
1756 kvm_run
->debug
.arch
.exception
= DB_VECTOR
;
1763 static int bp_interception(struct vcpu_svm
*svm
)
1765 struct kvm_run
*kvm_run
= svm
->vcpu
.run
;
1767 kvm_run
->exit_reason
= KVM_EXIT_DEBUG
;
1768 kvm_run
->debug
.arch
.pc
= svm
->vmcb
->save
.cs
.base
+ svm
->vmcb
->save
.rip
;
1769 kvm_run
->debug
.arch
.exception
= BP_VECTOR
;
1773 static int ud_interception(struct vcpu_svm
*svm
)
1775 return handle_ud(&svm
->vcpu
);
1778 static int ac_interception(struct vcpu_svm
*svm
)
1780 kvm_queue_exception_e(&svm
->vcpu
, AC_VECTOR
, 0);
1784 static int gp_interception(struct vcpu_svm
*svm
)
1786 struct kvm_vcpu
*vcpu
= &svm
->vcpu
;
1787 u32 error_code
= svm
->vmcb
->control
.exit_info_1
;
1789 WARN_ON_ONCE(!enable_vmware_backdoor
);
1792 * VMware backdoor emulation on #GP interception only handles IN{S},
1793 * OUT{S}, and RDPMC, none of which generate a non-zero error code.
1796 kvm_queue_exception_e(vcpu
, GP_VECTOR
, error_code
);
1799 return kvm_emulate_instruction(vcpu
, EMULTYPE_VMWARE_GP
);
1802 static bool is_erratum_383(void)
1807 if (!erratum_383_found
)
1810 value
= native_read_msr_safe(MSR_IA32_MC0_STATUS
, &err
);
1814 /* Bit 62 may or may not be set for this mce */
1815 value
&= ~(1ULL << 62);
1817 if (value
!= 0xb600000000010015ULL
)
1820 /* Clear MCi_STATUS registers */
1821 for (i
= 0; i
< 6; ++i
)
1822 native_write_msr_safe(MSR_IA32_MCx_STATUS(i
), 0, 0);
1824 value
= native_read_msr_safe(MSR_IA32_MCG_STATUS
, &err
);
1828 value
&= ~(1ULL << 2);
1829 low
= lower_32_bits(value
);
1830 high
= upper_32_bits(value
);
1832 native_write_msr_safe(MSR_IA32_MCG_STATUS
, low
, high
);
1835 /* Flush tlb to evict multi-match entries */
1842 * Trigger machine check on the host. We assume all the MSRs are already set up
1843 * by the CPU and that we still run on the same CPU as the MCE occurred on.
1844 * We pass a fake environment to the machine check handler because we want
1845 * the guest to be always treated like user space, no matter what context
1846 * it used internally.
1848 static void kvm_machine_check(void)
1850 #if defined(CONFIG_X86_MCE)
1851 struct pt_regs regs
= {
1852 .cs
= 3, /* Fake ring 3 no matter what the guest ran on */
1853 .flags
= X86_EFLAGS_IF
,
1856 do_machine_check(®s
);
1860 static void svm_handle_mce(struct vcpu_svm
*svm
)
1862 if (is_erratum_383()) {
1864 * Erratum 383 triggered. Guest state is corrupt so kill the
1867 pr_err("KVM: Guest triggered AMD Erratum 383\n");
1869 kvm_make_request(KVM_REQ_TRIPLE_FAULT
, &svm
->vcpu
);
1875 * On an #MC intercept the MCE handler is not called automatically in
1876 * the host. So do it by hand here.
1878 kvm_machine_check();
1881 static int mc_interception(struct vcpu_svm
*svm
)
1886 static int shutdown_interception(struct vcpu_svm
*svm
)
1888 struct kvm_run
*kvm_run
= svm
->vcpu
.run
;
1891 * VMCB is undefined after a SHUTDOWN intercept
1892 * so reinitialize it.
1894 clear_page(svm
->vmcb
);
1897 kvm_run
->exit_reason
= KVM_EXIT_SHUTDOWN
;
1901 static int io_interception(struct vcpu_svm
*svm
)
1903 struct kvm_vcpu
*vcpu
= &svm
->vcpu
;
1904 u32 io_info
= svm
->vmcb
->control
.exit_info_1
; /* address size bug? */
1905 int size
, in
, string
;
1908 ++svm
->vcpu
.stat
.io_exits
;
1909 string
= (io_info
& SVM_IOIO_STR_MASK
) != 0;
1910 in
= (io_info
& SVM_IOIO_TYPE_MASK
) != 0;
1912 return kvm_emulate_instruction(vcpu
, 0);
1914 port
= io_info
>> 16;
1915 size
= (io_info
& SVM_IOIO_SIZE_MASK
) >> SVM_IOIO_SIZE_SHIFT
;
1916 svm
->next_rip
= svm
->vmcb
->control
.exit_info_2
;
1918 return kvm_fast_pio(&svm
->vcpu
, size
, port
, in
);
1921 static int nmi_interception(struct vcpu_svm
*svm
)
1926 static int intr_interception(struct vcpu_svm
*svm
)
1928 ++svm
->vcpu
.stat
.irq_exits
;
1932 static int nop_on_interception(struct vcpu_svm
*svm
)
1937 static int halt_interception(struct vcpu_svm
*svm
)
1939 return kvm_emulate_halt(&svm
->vcpu
);
1942 static int vmmcall_interception(struct vcpu_svm
*svm
)
1944 return kvm_emulate_hypercall(&svm
->vcpu
);
1947 static int vmload_interception(struct vcpu_svm
*svm
)
1949 struct vmcb
*nested_vmcb
;
1950 struct kvm_host_map map
;
1953 if (nested_svm_check_permissions(svm
))
1956 ret
= kvm_vcpu_map(&svm
->vcpu
, gpa_to_gfn(svm
->vmcb
->save
.rax
), &map
);
1959 kvm_inject_gp(&svm
->vcpu
, 0);
1963 nested_vmcb
= map
.hva
;
1965 ret
= kvm_skip_emulated_instruction(&svm
->vcpu
);
1967 nested_svm_vmloadsave(nested_vmcb
, svm
->vmcb
);
1968 kvm_vcpu_unmap(&svm
->vcpu
, &map
, true);
1973 static int vmsave_interception(struct vcpu_svm
*svm
)
1975 struct vmcb
*nested_vmcb
;
1976 struct kvm_host_map map
;
1979 if (nested_svm_check_permissions(svm
))
1982 ret
= kvm_vcpu_map(&svm
->vcpu
, gpa_to_gfn(svm
->vmcb
->save
.rax
), &map
);
1985 kvm_inject_gp(&svm
->vcpu
, 0);
1989 nested_vmcb
= map
.hva
;
1991 ret
= kvm_skip_emulated_instruction(&svm
->vcpu
);
1993 nested_svm_vmloadsave(svm
->vmcb
, nested_vmcb
);
1994 kvm_vcpu_unmap(&svm
->vcpu
, &map
, true);
1999 static int vmrun_interception(struct vcpu_svm
*svm
)
2001 if (nested_svm_check_permissions(svm
))
2004 return nested_svm_vmrun(svm
);
2007 void svm_set_gif(struct vcpu_svm
*svm
, bool value
)
2011 * If VGIF is enabled, the STGI intercept is only added to
2012 * detect the opening of the SMI/NMI window; remove it now.
2013 * Likewise, clear the VINTR intercept, we will set it
2014 * again while processing KVM_REQ_EVENT if needed.
2016 if (vgif_enabled(svm
))
2017 svm_clr_intercept(svm
, INTERCEPT_STGI
);
2018 if (svm_is_intercept(svm
, INTERCEPT_VINTR
))
2019 svm_clear_vintr(svm
);
2022 if (svm
->vcpu
.arch
.smi_pending
||
2023 svm
->vcpu
.arch
.nmi_pending
||
2024 kvm_cpu_has_injectable_intr(&svm
->vcpu
))
2025 kvm_make_request(KVM_REQ_EVENT
, &svm
->vcpu
);
2030 * After a CLGI no interrupts should come. But if vGIF is
2031 * in use, we still rely on the VINTR intercept (rather than
2032 * STGI) to detect an open interrupt window.
2034 if (!vgif_enabled(svm
))
2035 svm_clear_vintr(svm
);
2039 static int stgi_interception(struct vcpu_svm
*svm
)
2043 if (nested_svm_check_permissions(svm
))
2046 ret
= kvm_skip_emulated_instruction(&svm
->vcpu
);
2047 svm_set_gif(svm
, true);
2051 static int clgi_interception(struct vcpu_svm
*svm
)
2055 if (nested_svm_check_permissions(svm
))
2058 ret
= kvm_skip_emulated_instruction(&svm
->vcpu
);
2059 svm_set_gif(svm
, false);
2063 static int invlpga_interception(struct vcpu_svm
*svm
)
2065 struct kvm_vcpu
*vcpu
= &svm
->vcpu
;
2067 trace_kvm_invlpga(svm
->vmcb
->save
.rip
, kvm_rcx_read(&svm
->vcpu
),
2068 kvm_rax_read(&svm
->vcpu
));
2070 /* Let's treat INVLPGA the same as INVLPG (can be optimized!) */
2071 kvm_mmu_invlpg(vcpu
, kvm_rax_read(&svm
->vcpu
));
2073 return kvm_skip_emulated_instruction(&svm
->vcpu
);
2076 static int skinit_interception(struct vcpu_svm
*svm
)
2078 trace_kvm_skinit(svm
->vmcb
->save
.rip
, kvm_rax_read(&svm
->vcpu
));
2080 kvm_queue_exception(&svm
->vcpu
, UD_VECTOR
);
2084 static int wbinvd_interception(struct vcpu_svm
*svm
)
2086 return kvm_emulate_wbinvd(&svm
->vcpu
);
2089 static int xsetbv_interception(struct vcpu_svm
*svm
)
2091 u64 new_bv
= kvm_read_edx_eax(&svm
->vcpu
);
2092 u32 index
= kvm_rcx_read(&svm
->vcpu
);
2094 if (kvm_set_xcr(&svm
->vcpu
, index
, new_bv
) == 0) {
2095 return kvm_skip_emulated_instruction(&svm
->vcpu
);
2101 static int rdpru_interception(struct vcpu_svm
*svm
)
2103 kvm_queue_exception(&svm
->vcpu
, UD_VECTOR
);
2107 static int task_switch_interception(struct vcpu_svm
*svm
)
2111 int int_type
= svm
->vmcb
->control
.exit_int_info
&
2112 SVM_EXITINTINFO_TYPE_MASK
;
2113 int int_vec
= svm
->vmcb
->control
.exit_int_info
& SVM_EVTINJ_VEC_MASK
;
2115 svm
->vmcb
->control
.exit_int_info
& SVM_EXITINTINFO_TYPE_MASK
;
2117 svm
->vmcb
->control
.exit_int_info
& SVM_EXITINTINFO_VALID
;
2118 bool has_error_code
= false;
2121 tss_selector
= (u16
)svm
->vmcb
->control
.exit_info_1
;
2123 if (svm
->vmcb
->control
.exit_info_2
&
2124 (1ULL << SVM_EXITINFOSHIFT_TS_REASON_IRET
))
2125 reason
= TASK_SWITCH_IRET
;
2126 else if (svm
->vmcb
->control
.exit_info_2
&
2127 (1ULL << SVM_EXITINFOSHIFT_TS_REASON_JMP
))
2128 reason
= TASK_SWITCH_JMP
;
2130 reason
= TASK_SWITCH_GATE
;
2132 reason
= TASK_SWITCH_CALL
;
2134 if (reason
== TASK_SWITCH_GATE
) {
2136 case SVM_EXITINTINFO_TYPE_NMI
:
2137 svm
->vcpu
.arch
.nmi_injected
= false;
2139 case SVM_EXITINTINFO_TYPE_EXEPT
:
2140 if (svm
->vmcb
->control
.exit_info_2
&
2141 (1ULL << SVM_EXITINFOSHIFT_TS_HAS_ERROR_CODE
)) {
2142 has_error_code
= true;
2144 (u32
)svm
->vmcb
->control
.exit_info_2
;
2146 kvm_clear_exception_queue(&svm
->vcpu
);
2148 case SVM_EXITINTINFO_TYPE_INTR
:
2149 kvm_clear_interrupt_queue(&svm
->vcpu
);
2156 if (reason
!= TASK_SWITCH_GATE
||
2157 int_type
== SVM_EXITINTINFO_TYPE_SOFT
||
2158 (int_type
== SVM_EXITINTINFO_TYPE_EXEPT
&&
2159 (int_vec
== OF_VECTOR
|| int_vec
== BP_VECTOR
))) {
2160 if (!skip_emulated_instruction(&svm
->vcpu
))
2164 if (int_type
!= SVM_EXITINTINFO_TYPE_SOFT
)
2167 return kvm_task_switch(&svm
->vcpu
, tss_selector
, int_vec
, reason
,
2168 has_error_code
, error_code
);
2171 static int cpuid_interception(struct vcpu_svm
*svm
)
2173 return kvm_emulate_cpuid(&svm
->vcpu
);
2176 static int iret_interception(struct vcpu_svm
*svm
)
2178 ++svm
->vcpu
.stat
.nmi_window_exits
;
2179 svm_clr_intercept(svm
, INTERCEPT_IRET
);
2180 svm
->vcpu
.arch
.hflags
|= HF_IRET_MASK
;
2181 svm
->nmi_iret_rip
= kvm_rip_read(&svm
->vcpu
);
2182 kvm_make_request(KVM_REQ_EVENT
, &svm
->vcpu
);
2186 static int invlpg_interception(struct vcpu_svm
*svm
)
2188 if (!static_cpu_has(X86_FEATURE_DECODEASSISTS
))
2189 return kvm_emulate_instruction(&svm
->vcpu
, 0);
2191 kvm_mmu_invlpg(&svm
->vcpu
, svm
->vmcb
->control
.exit_info_1
);
2192 return kvm_skip_emulated_instruction(&svm
->vcpu
);
2195 static int emulate_on_interception(struct vcpu_svm
*svm
)
2197 return kvm_emulate_instruction(&svm
->vcpu
, 0);
2200 static int rsm_interception(struct vcpu_svm
*svm
)
2202 return kvm_emulate_instruction_from_buffer(&svm
->vcpu
, rsm_ins_bytes
, 2);
2205 static int rdpmc_interception(struct vcpu_svm
*svm
)
2210 return emulate_on_interception(svm
);
2212 err
= kvm_rdpmc(&svm
->vcpu
);
2213 return kvm_complete_insn_gp(&svm
->vcpu
, err
);
2216 static bool check_selective_cr0_intercepted(struct vcpu_svm
*svm
,
2219 unsigned long cr0
= svm
->vcpu
.arch
.cr0
;
2223 intercept
= svm
->nested
.ctl
.intercept
;
2225 if (!is_guest_mode(&svm
->vcpu
) ||
2226 (!(intercept
& (1ULL << INTERCEPT_SELECTIVE_CR0
))))
2229 cr0
&= ~SVM_CR0_SELECTIVE_MASK
;
2230 val
&= ~SVM_CR0_SELECTIVE_MASK
;
2233 svm
->vmcb
->control
.exit_code
= SVM_EXIT_CR0_SEL_WRITE
;
2234 ret
= (nested_svm_exit_handled(svm
) == NESTED_EXIT_DONE
);
2240 #define CR_VALID (1ULL << 63)
2242 static int cr_interception(struct vcpu_svm
*svm
)
2248 if (!static_cpu_has(X86_FEATURE_DECODEASSISTS
))
2249 return emulate_on_interception(svm
);
2251 if (unlikely((svm
->vmcb
->control
.exit_info_1
& CR_VALID
) == 0))
2252 return emulate_on_interception(svm
);
2254 reg
= svm
->vmcb
->control
.exit_info_1
& SVM_EXITINFO_REG_MASK
;
2255 if (svm
->vmcb
->control
.exit_code
== SVM_EXIT_CR0_SEL_WRITE
)
2256 cr
= SVM_EXIT_WRITE_CR0
- SVM_EXIT_READ_CR0
;
2258 cr
= svm
->vmcb
->control
.exit_code
- SVM_EXIT_READ_CR0
;
2261 if (cr
>= 16) { /* mov to cr */
2263 val
= kvm_register_read(&svm
->vcpu
, reg
);
2266 if (!check_selective_cr0_intercepted(svm
, val
))
2267 err
= kvm_set_cr0(&svm
->vcpu
, val
);
2273 err
= kvm_set_cr3(&svm
->vcpu
, val
);
2276 err
= kvm_set_cr4(&svm
->vcpu
, val
);
2279 err
= kvm_set_cr8(&svm
->vcpu
, val
);
2282 WARN(1, "unhandled write to CR%d", cr
);
2283 kvm_queue_exception(&svm
->vcpu
, UD_VECTOR
);
2286 } else { /* mov from cr */
2289 val
= kvm_read_cr0(&svm
->vcpu
);
2292 val
= svm
->vcpu
.arch
.cr2
;
2295 val
= kvm_read_cr3(&svm
->vcpu
);
2298 val
= kvm_read_cr4(&svm
->vcpu
);
2301 val
= kvm_get_cr8(&svm
->vcpu
);
2304 WARN(1, "unhandled read from CR%d", cr
);
2305 kvm_queue_exception(&svm
->vcpu
, UD_VECTOR
);
2308 kvm_register_write(&svm
->vcpu
, reg
, val
);
2310 return kvm_complete_insn_gp(&svm
->vcpu
, err
);
2313 static int dr_interception(struct vcpu_svm
*svm
)
2318 if (svm
->vcpu
.guest_debug
== 0) {
2320 * No more DR vmexits; force a reload of the debug registers
2321 * and reenter on this instruction. The next vmexit will
2322 * retrieve the full state of the debug registers.
2324 clr_dr_intercepts(svm
);
2325 svm
->vcpu
.arch
.switch_db_regs
|= KVM_DEBUGREG_WONT_EXIT
;
2329 if (!boot_cpu_has(X86_FEATURE_DECODEASSISTS
))
2330 return emulate_on_interception(svm
);
2332 reg
= svm
->vmcb
->control
.exit_info_1
& SVM_EXITINFO_REG_MASK
;
2333 dr
= svm
->vmcb
->control
.exit_code
- SVM_EXIT_READ_DR0
;
2335 if (dr
>= 16) { /* mov to DRn */
2336 if (!kvm_require_dr(&svm
->vcpu
, dr
- 16))
2338 val
= kvm_register_read(&svm
->vcpu
, reg
);
2339 kvm_set_dr(&svm
->vcpu
, dr
- 16, val
);
2341 if (!kvm_require_dr(&svm
->vcpu
, dr
))
2343 kvm_get_dr(&svm
->vcpu
, dr
, &val
);
2344 kvm_register_write(&svm
->vcpu
, reg
, val
);
2347 return kvm_skip_emulated_instruction(&svm
->vcpu
);
2350 static int cr8_write_interception(struct vcpu_svm
*svm
)
2352 struct kvm_run
*kvm_run
= svm
->vcpu
.run
;
2355 u8 cr8_prev
= kvm_get_cr8(&svm
->vcpu
);
2356 /* instruction emulation calls kvm_set_cr8() */
2357 r
= cr_interception(svm
);
2358 if (lapic_in_kernel(&svm
->vcpu
))
2360 if (cr8_prev
<= kvm_get_cr8(&svm
->vcpu
))
2362 kvm_run
->exit_reason
= KVM_EXIT_SET_TPR
;
2366 static int svm_get_msr_feature(struct kvm_msr_entry
*msr
)
2370 switch (msr
->index
) {
2371 case MSR_F10H_DECFG
:
2372 if (boot_cpu_has(X86_FEATURE_LFENCE_RDTSC
))
2373 msr
->data
|= MSR_F10H_DECFG_LFENCE_SERIALIZE
;
2375 case MSR_IA32_PERF_CAPABILITIES
:
2378 return KVM_MSR_RET_INVALID
;
2384 static int svm_get_msr(struct kvm_vcpu
*vcpu
, struct msr_data
*msr_info
)
2386 struct vcpu_svm
*svm
= to_svm(vcpu
);
2388 switch (msr_info
->index
) {
2390 msr_info
->data
= svm
->vmcb
->save
.star
;
2392 #ifdef CONFIG_X86_64
2394 msr_info
->data
= svm
->vmcb
->save
.lstar
;
2397 msr_info
->data
= svm
->vmcb
->save
.cstar
;
2399 case MSR_KERNEL_GS_BASE
:
2400 msr_info
->data
= svm
->vmcb
->save
.kernel_gs_base
;
2402 case MSR_SYSCALL_MASK
:
2403 msr_info
->data
= svm
->vmcb
->save
.sfmask
;
2406 case MSR_IA32_SYSENTER_CS
:
2407 msr_info
->data
= svm
->vmcb
->save
.sysenter_cs
;
2409 case MSR_IA32_SYSENTER_EIP
:
2410 msr_info
->data
= svm
->sysenter_eip
;
2412 case MSR_IA32_SYSENTER_ESP
:
2413 msr_info
->data
= svm
->sysenter_esp
;
2416 if (!boot_cpu_has(X86_FEATURE_RDTSCP
))
2418 msr_info
->data
= svm
->tsc_aux
;
2421 * Nobody will change the following 5 values in the VMCB so we can
2422 * safely return them on rdmsr. They will always be 0 until LBRV is
2425 case MSR_IA32_DEBUGCTLMSR
:
2426 msr_info
->data
= svm
->vmcb
->save
.dbgctl
;
2428 case MSR_IA32_LASTBRANCHFROMIP
:
2429 msr_info
->data
= svm
->vmcb
->save
.br_from
;
2431 case MSR_IA32_LASTBRANCHTOIP
:
2432 msr_info
->data
= svm
->vmcb
->save
.br_to
;
2434 case MSR_IA32_LASTINTFROMIP
:
2435 msr_info
->data
= svm
->vmcb
->save
.last_excp_from
;
2437 case MSR_IA32_LASTINTTOIP
:
2438 msr_info
->data
= svm
->vmcb
->save
.last_excp_to
;
2440 case MSR_VM_HSAVE_PA
:
2441 msr_info
->data
= svm
->nested
.hsave_msr
;
2444 msr_info
->data
= svm
->nested
.vm_cr_msr
;
2446 case MSR_IA32_SPEC_CTRL
:
2447 if (!msr_info
->host_initiated
&&
2448 !guest_cpuid_has(vcpu
, X86_FEATURE_SPEC_CTRL
) &&
2449 !guest_cpuid_has(vcpu
, X86_FEATURE_AMD_STIBP
) &&
2450 !guest_cpuid_has(vcpu
, X86_FEATURE_AMD_IBRS
) &&
2451 !guest_cpuid_has(vcpu
, X86_FEATURE_AMD_SSBD
))
2454 msr_info
->data
= svm
->spec_ctrl
;
2456 case MSR_AMD64_VIRT_SPEC_CTRL
:
2457 if (!msr_info
->host_initiated
&&
2458 !guest_cpuid_has(vcpu
, X86_FEATURE_VIRT_SSBD
))
2461 msr_info
->data
= svm
->virt_spec_ctrl
;
2463 case MSR_F15H_IC_CFG
: {
2467 family
= guest_cpuid_family(vcpu
);
2468 model
= guest_cpuid_model(vcpu
);
2470 if (family
< 0 || model
< 0)
2471 return kvm_get_msr_common(vcpu
, msr_info
);
2475 if (family
== 0x15 &&
2476 (model
>= 0x2 && model
< 0x20))
2477 msr_info
->data
= 0x1E;
2480 case MSR_F10H_DECFG
:
2481 msr_info
->data
= svm
->msr_decfg
;
2484 return kvm_get_msr_common(vcpu
, msr_info
);
2489 static int rdmsr_interception(struct vcpu_svm
*svm
)
2491 return kvm_emulate_rdmsr(&svm
->vcpu
);
2494 static int svm_set_vm_cr(struct kvm_vcpu
*vcpu
, u64 data
)
2496 struct vcpu_svm
*svm
= to_svm(vcpu
);
2497 int svm_dis
, chg_mask
;
2499 if (data
& ~SVM_VM_CR_VALID_MASK
)
2502 chg_mask
= SVM_VM_CR_VALID_MASK
;
2504 if (svm
->nested
.vm_cr_msr
& SVM_VM_CR_SVM_DIS_MASK
)
2505 chg_mask
&= ~(SVM_VM_CR_SVM_LOCK_MASK
| SVM_VM_CR_SVM_DIS_MASK
);
2507 svm
->nested
.vm_cr_msr
&= ~chg_mask
;
2508 svm
->nested
.vm_cr_msr
|= (data
& chg_mask
);
2510 svm_dis
= svm
->nested
.vm_cr_msr
& SVM_VM_CR_SVM_DIS_MASK
;
2512 /* check for svm_disable while efer.svme is set */
2513 if (svm_dis
&& (vcpu
->arch
.efer
& EFER_SVME
))
2519 static int svm_set_msr(struct kvm_vcpu
*vcpu
, struct msr_data
*msr
)
2521 struct vcpu_svm
*svm
= to_svm(vcpu
);
2523 u32 ecx
= msr
->index
;
2524 u64 data
= msr
->data
;
2526 case MSR_IA32_CR_PAT
:
2527 if (!kvm_mtrr_valid(vcpu
, MSR_IA32_CR_PAT
, data
))
2529 vcpu
->arch
.pat
= data
;
2530 svm
->vmcb
->save
.g_pat
= data
;
2531 vmcb_mark_dirty(svm
->vmcb
, VMCB_NPT
);
2533 case MSR_IA32_SPEC_CTRL
:
2534 if (!msr
->host_initiated
&&
2535 !guest_cpuid_has(vcpu
, X86_FEATURE_SPEC_CTRL
) &&
2536 !guest_cpuid_has(vcpu
, X86_FEATURE_AMD_STIBP
) &&
2537 !guest_cpuid_has(vcpu
, X86_FEATURE_AMD_IBRS
) &&
2538 !guest_cpuid_has(vcpu
, X86_FEATURE_AMD_SSBD
))
2541 if (kvm_spec_ctrl_test_value(data
))
2544 svm
->spec_ctrl
= data
;
2550 * When it's written (to non-zero) for the first time, pass
2554 * The handling of the MSR bitmap for L2 guests is done in
2555 * nested_svm_vmrun_msrpm.
2556 * We update the L1 MSR bit as well since it will end up
2557 * touching the MSR anyway now.
2559 set_msr_interception(svm
->msrpm
, MSR_IA32_SPEC_CTRL
, 1, 1);
2561 case MSR_IA32_PRED_CMD
:
2562 if (!msr
->host_initiated
&&
2563 !guest_cpuid_has(vcpu
, X86_FEATURE_AMD_IBPB
))
2566 if (data
& ~PRED_CMD_IBPB
)
2568 if (!boot_cpu_has(X86_FEATURE_AMD_IBPB
))
2573 wrmsrl(MSR_IA32_PRED_CMD
, PRED_CMD_IBPB
);
2574 set_msr_interception(svm
->msrpm
, MSR_IA32_PRED_CMD
, 0, 1);
2576 case MSR_AMD64_VIRT_SPEC_CTRL
:
2577 if (!msr
->host_initiated
&&
2578 !guest_cpuid_has(vcpu
, X86_FEATURE_VIRT_SSBD
))
2581 if (data
& ~SPEC_CTRL_SSBD
)
2584 svm
->virt_spec_ctrl
= data
;
2587 svm
->vmcb
->save
.star
= data
;
2589 #ifdef CONFIG_X86_64
2591 svm
->vmcb
->save
.lstar
= data
;
2594 svm
->vmcb
->save
.cstar
= data
;
2596 case MSR_KERNEL_GS_BASE
:
2597 svm
->vmcb
->save
.kernel_gs_base
= data
;
2599 case MSR_SYSCALL_MASK
:
2600 svm
->vmcb
->save
.sfmask
= data
;
2603 case MSR_IA32_SYSENTER_CS
:
2604 svm
->vmcb
->save
.sysenter_cs
= data
;
2606 case MSR_IA32_SYSENTER_EIP
:
2607 svm
->sysenter_eip
= data
;
2608 svm
->vmcb
->save
.sysenter_eip
= data
;
2610 case MSR_IA32_SYSENTER_ESP
:
2611 svm
->sysenter_esp
= data
;
2612 svm
->vmcb
->save
.sysenter_esp
= data
;
2615 if (!boot_cpu_has(X86_FEATURE_RDTSCP
))
2619 * This is rare, so we update the MSR here instead of using
2620 * direct_access_msrs. Doing that would require a rdmsr in
2623 svm
->tsc_aux
= data
;
2624 wrmsrl(MSR_TSC_AUX
, svm
->tsc_aux
);
2626 case MSR_IA32_DEBUGCTLMSR
:
2627 if (!boot_cpu_has(X86_FEATURE_LBRV
)) {
2628 vcpu_unimpl(vcpu
, "%s: MSR_IA32_DEBUGCTL 0x%llx, nop\n",
2632 if (data
& DEBUGCTL_RESERVED_BITS
)
2635 svm
->vmcb
->save
.dbgctl
= data
;
2636 vmcb_mark_dirty(svm
->vmcb
, VMCB_LBR
);
2637 if (data
& (1ULL<<0))
2638 svm_enable_lbrv(svm
);
2640 svm_disable_lbrv(svm
);
2642 case MSR_VM_HSAVE_PA
:
2643 svm
->nested
.hsave_msr
= data
;
2646 return svm_set_vm_cr(vcpu
, data
);
2648 vcpu_unimpl(vcpu
, "unimplemented wrmsr: 0x%x data 0x%llx\n", ecx
, data
);
2650 case MSR_F10H_DECFG
: {
2651 struct kvm_msr_entry msr_entry
;
2653 msr_entry
.index
= msr
->index
;
2654 if (svm_get_msr_feature(&msr_entry
))
2657 /* Check the supported bits */
2658 if (data
& ~msr_entry
.data
)
2661 /* Don't allow the guest to change a bit, #GP */
2662 if (!msr
->host_initiated
&& (data
^ msr_entry
.data
))
2665 svm
->msr_decfg
= data
;
2668 case MSR_IA32_APICBASE
:
2669 if (kvm_vcpu_apicv_active(vcpu
))
2670 avic_update_vapic_bar(to_svm(vcpu
), data
);
2673 return kvm_set_msr_common(vcpu
, msr
);
2678 static int wrmsr_interception(struct vcpu_svm
*svm
)
2680 return kvm_emulate_wrmsr(&svm
->vcpu
);
2683 static int msr_interception(struct vcpu_svm
*svm
)
2685 if (svm
->vmcb
->control
.exit_info_1
)
2686 return wrmsr_interception(svm
);
2688 return rdmsr_interception(svm
);
2691 static int interrupt_window_interception(struct vcpu_svm
*svm
)
2693 kvm_make_request(KVM_REQ_EVENT
, &svm
->vcpu
);
2694 svm_clear_vintr(svm
);
2697 * For AVIC, the only reason to end up here is ExtINTs.
2698 * In this case AVIC was temporarily disabled for
2699 * requesting the IRQ window and we have to re-enable it.
2701 svm_toggle_avic_for_irq_window(&svm
->vcpu
, true);
2703 ++svm
->vcpu
.stat
.irq_window_exits
;
2707 static int pause_interception(struct vcpu_svm
*svm
)
2709 struct kvm_vcpu
*vcpu
= &svm
->vcpu
;
2710 bool in_kernel
= (svm_get_cpl(vcpu
) == 0);
2712 if (!kvm_pause_in_guest(vcpu
->kvm
))
2713 grow_ple_window(vcpu
);
2715 kvm_vcpu_on_spin(vcpu
, in_kernel
);
2719 static int nop_interception(struct vcpu_svm
*svm
)
2721 return kvm_skip_emulated_instruction(&(svm
->vcpu
));
2724 static int monitor_interception(struct vcpu_svm
*svm
)
2726 printk_once(KERN_WARNING
"kvm: MONITOR instruction emulated as NOP!\n");
2727 return nop_interception(svm
);
2730 static int mwait_interception(struct vcpu_svm
*svm
)
2732 printk_once(KERN_WARNING
"kvm: MWAIT instruction emulated as NOP!\n");
2733 return nop_interception(svm
);
2736 static int (*const svm_exit_handlers
[])(struct vcpu_svm
*svm
) = {
2737 [SVM_EXIT_READ_CR0
] = cr_interception
,
2738 [SVM_EXIT_READ_CR3
] = cr_interception
,
2739 [SVM_EXIT_READ_CR4
] = cr_interception
,
2740 [SVM_EXIT_READ_CR8
] = cr_interception
,
2741 [SVM_EXIT_CR0_SEL_WRITE
] = cr_interception
,
2742 [SVM_EXIT_WRITE_CR0
] = cr_interception
,
2743 [SVM_EXIT_WRITE_CR3
] = cr_interception
,
2744 [SVM_EXIT_WRITE_CR4
] = cr_interception
,
2745 [SVM_EXIT_WRITE_CR8
] = cr8_write_interception
,
2746 [SVM_EXIT_READ_DR0
] = dr_interception
,
2747 [SVM_EXIT_READ_DR1
] = dr_interception
,
2748 [SVM_EXIT_READ_DR2
] = dr_interception
,
2749 [SVM_EXIT_READ_DR3
] = dr_interception
,
2750 [SVM_EXIT_READ_DR4
] = dr_interception
,
2751 [SVM_EXIT_READ_DR5
] = dr_interception
,
2752 [SVM_EXIT_READ_DR6
] = dr_interception
,
2753 [SVM_EXIT_READ_DR7
] = dr_interception
,
2754 [SVM_EXIT_WRITE_DR0
] = dr_interception
,
2755 [SVM_EXIT_WRITE_DR1
] = dr_interception
,
2756 [SVM_EXIT_WRITE_DR2
] = dr_interception
,
2757 [SVM_EXIT_WRITE_DR3
] = dr_interception
,
2758 [SVM_EXIT_WRITE_DR4
] = dr_interception
,
2759 [SVM_EXIT_WRITE_DR5
] = dr_interception
,
2760 [SVM_EXIT_WRITE_DR6
] = dr_interception
,
2761 [SVM_EXIT_WRITE_DR7
] = dr_interception
,
2762 [SVM_EXIT_EXCP_BASE
+ DB_VECTOR
] = db_interception
,
2763 [SVM_EXIT_EXCP_BASE
+ BP_VECTOR
] = bp_interception
,
2764 [SVM_EXIT_EXCP_BASE
+ UD_VECTOR
] = ud_interception
,
2765 [SVM_EXIT_EXCP_BASE
+ PF_VECTOR
] = pf_interception
,
2766 [SVM_EXIT_EXCP_BASE
+ MC_VECTOR
] = mc_interception
,
2767 [SVM_EXIT_EXCP_BASE
+ AC_VECTOR
] = ac_interception
,
2768 [SVM_EXIT_EXCP_BASE
+ GP_VECTOR
] = gp_interception
,
2769 [SVM_EXIT_INTR
] = intr_interception
,
2770 [SVM_EXIT_NMI
] = nmi_interception
,
2771 [SVM_EXIT_SMI
] = nop_on_interception
,
2772 [SVM_EXIT_INIT
] = nop_on_interception
,
2773 [SVM_EXIT_VINTR
] = interrupt_window_interception
,
2774 [SVM_EXIT_RDPMC
] = rdpmc_interception
,
2775 [SVM_EXIT_CPUID
] = cpuid_interception
,
2776 [SVM_EXIT_IRET
] = iret_interception
,
2777 [SVM_EXIT_INVD
] = emulate_on_interception
,
2778 [SVM_EXIT_PAUSE
] = pause_interception
,
2779 [SVM_EXIT_HLT
] = halt_interception
,
2780 [SVM_EXIT_INVLPG
] = invlpg_interception
,
2781 [SVM_EXIT_INVLPGA
] = invlpga_interception
,
2782 [SVM_EXIT_IOIO
] = io_interception
,
2783 [SVM_EXIT_MSR
] = msr_interception
,
2784 [SVM_EXIT_TASK_SWITCH
] = task_switch_interception
,
2785 [SVM_EXIT_SHUTDOWN
] = shutdown_interception
,
2786 [SVM_EXIT_VMRUN
] = vmrun_interception
,
2787 [SVM_EXIT_VMMCALL
] = vmmcall_interception
,
2788 [SVM_EXIT_VMLOAD
] = vmload_interception
,
2789 [SVM_EXIT_VMSAVE
] = vmsave_interception
,
2790 [SVM_EXIT_STGI
] = stgi_interception
,
2791 [SVM_EXIT_CLGI
] = clgi_interception
,
2792 [SVM_EXIT_SKINIT
] = skinit_interception
,
2793 [SVM_EXIT_WBINVD
] = wbinvd_interception
,
2794 [SVM_EXIT_MONITOR
] = monitor_interception
,
2795 [SVM_EXIT_MWAIT
] = mwait_interception
,
2796 [SVM_EXIT_XSETBV
] = xsetbv_interception
,
2797 [SVM_EXIT_RDPRU
] = rdpru_interception
,
2798 [SVM_EXIT_NPF
] = npf_interception
,
2799 [SVM_EXIT_RSM
] = rsm_interception
,
2800 [SVM_EXIT_AVIC_INCOMPLETE_IPI
] = avic_incomplete_ipi_interception
,
2801 [SVM_EXIT_AVIC_UNACCELERATED_ACCESS
] = avic_unaccelerated_access_interception
,
2804 static void dump_vmcb(struct kvm_vcpu
*vcpu
)
2806 struct vcpu_svm
*svm
= to_svm(vcpu
);
2807 struct vmcb_control_area
*control
= &svm
->vmcb
->control
;
2808 struct vmcb_save_area
*save
= &svm
->vmcb
->save
;
2810 if (!dump_invalid_vmcb
) {
2811 pr_warn_ratelimited("set kvm_amd.dump_invalid_vmcb=1 to dump internal KVM state.\n");
2815 pr_err("VMCB Control Area:\n");
2816 pr_err("%-20s%04x\n", "cr_read:", control
->intercept_cr
& 0xffff);
2817 pr_err("%-20s%04x\n", "cr_write:", control
->intercept_cr
>> 16);
2818 pr_err("%-20s%04x\n", "dr_read:", control
->intercept_dr
& 0xffff);
2819 pr_err("%-20s%04x\n", "dr_write:", control
->intercept_dr
>> 16);
2820 pr_err("%-20s%08x\n", "exceptions:", control
->intercept_exceptions
);
2821 pr_err("%-20s%016llx\n", "intercepts:", control
->intercept
);
2822 pr_err("%-20s%d\n", "pause filter count:", control
->pause_filter_count
);
2823 pr_err("%-20s%d\n", "pause filter threshold:",
2824 control
->pause_filter_thresh
);
2825 pr_err("%-20s%016llx\n", "iopm_base_pa:", control
->iopm_base_pa
);
2826 pr_err("%-20s%016llx\n", "msrpm_base_pa:", control
->msrpm_base_pa
);
2827 pr_err("%-20s%016llx\n", "tsc_offset:", control
->tsc_offset
);
2828 pr_err("%-20s%d\n", "asid:", control
->asid
);
2829 pr_err("%-20s%d\n", "tlb_ctl:", control
->tlb_ctl
);
2830 pr_err("%-20s%08x\n", "int_ctl:", control
->int_ctl
);
2831 pr_err("%-20s%08x\n", "int_vector:", control
->int_vector
);
2832 pr_err("%-20s%08x\n", "int_state:", control
->int_state
);
2833 pr_err("%-20s%08x\n", "exit_code:", control
->exit_code
);
2834 pr_err("%-20s%016llx\n", "exit_info1:", control
->exit_info_1
);
2835 pr_err("%-20s%016llx\n", "exit_info2:", control
->exit_info_2
);
2836 pr_err("%-20s%08x\n", "exit_int_info:", control
->exit_int_info
);
2837 pr_err("%-20s%08x\n", "exit_int_info_err:", control
->exit_int_info_err
);
2838 pr_err("%-20s%lld\n", "nested_ctl:", control
->nested_ctl
);
2839 pr_err("%-20s%016llx\n", "nested_cr3:", control
->nested_cr3
);
2840 pr_err("%-20s%016llx\n", "avic_vapic_bar:", control
->avic_vapic_bar
);
2841 pr_err("%-20s%08x\n", "event_inj:", control
->event_inj
);
2842 pr_err("%-20s%08x\n", "event_inj_err:", control
->event_inj_err
);
2843 pr_err("%-20s%lld\n", "virt_ext:", control
->virt_ext
);
2844 pr_err("%-20s%016llx\n", "next_rip:", control
->next_rip
);
2845 pr_err("%-20s%016llx\n", "avic_backing_page:", control
->avic_backing_page
);
2846 pr_err("%-20s%016llx\n", "avic_logical_id:", control
->avic_logical_id
);
2847 pr_err("%-20s%016llx\n", "avic_physical_id:", control
->avic_physical_id
);
2848 pr_err("VMCB State Save Area:\n");
2849 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
2851 save
->es
.selector
, save
->es
.attrib
,
2852 save
->es
.limit
, save
->es
.base
);
2853 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
2855 save
->cs
.selector
, save
->cs
.attrib
,
2856 save
->cs
.limit
, save
->cs
.base
);
2857 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
2859 save
->ss
.selector
, save
->ss
.attrib
,
2860 save
->ss
.limit
, save
->ss
.base
);
2861 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
2863 save
->ds
.selector
, save
->ds
.attrib
,
2864 save
->ds
.limit
, save
->ds
.base
);
2865 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
2867 save
->fs
.selector
, save
->fs
.attrib
,
2868 save
->fs
.limit
, save
->fs
.base
);
2869 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
2871 save
->gs
.selector
, save
->gs
.attrib
,
2872 save
->gs
.limit
, save
->gs
.base
);
2873 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
2875 save
->gdtr
.selector
, save
->gdtr
.attrib
,
2876 save
->gdtr
.limit
, save
->gdtr
.base
);
2877 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
2879 save
->ldtr
.selector
, save
->ldtr
.attrib
,
2880 save
->ldtr
.limit
, save
->ldtr
.base
);
2881 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
2883 save
->idtr
.selector
, save
->idtr
.attrib
,
2884 save
->idtr
.limit
, save
->idtr
.base
);
2885 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
2887 save
->tr
.selector
, save
->tr
.attrib
,
2888 save
->tr
.limit
, save
->tr
.base
);
2889 pr_err("cpl: %d efer: %016llx\n",
2890 save
->cpl
, save
->efer
);
2891 pr_err("%-15s %016llx %-13s %016llx\n",
2892 "cr0:", save
->cr0
, "cr2:", save
->cr2
);
2893 pr_err("%-15s %016llx %-13s %016llx\n",
2894 "cr3:", save
->cr3
, "cr4:", save
->cr4
);
2895 pr_err("%-15s %016llx %-13s %016llx\n",
2896 "dr6:", save
->dr6
, "dr7:", save
->dr7
);
2897 pr_err("%-15s %016llx %-13s %016llx\n",
2898 "rip:", save
->rip
, "rflags:", save
->rflags
);
2899 pr_err("%-15s %016llx %-13s %016llx\n",
2900 "rsp:", save
->rsp
, "rax:", save
->rax
);
2901 pr_err("%-15s %016llx %-13s %016llx\n",
2902 "star:", save
->star
, "lstar:", save
->lstar
);
2903 pr_err("%-15s %016llx %-13s %016llx\n",
2904 "cstar:", save
->cstar
, "sfmask:", save
->sfmask
);
2905 pr_err("%-15s %016llx %-13s %016llx\n",
2906 "kernel_gs_base:", save
->kernel_gs_base
,
2907 "sysenter_cs:", save
->sysenter_cs
);
2908 pr_err("%-15s %016llx %-13s %016llx\n",
2909 "sysenter_esp:", save
->sysenter_esp
,
2910 "sysenter_eip:", save
->sysenter_eip
);
2911 pr_err("%-15s %016llx %-13s %016llx\n",
2912 "gpat:", save
->g_pat
, "dbgctl:", save
->dbgctl
);
2913 pr_err("%-15s %016llx %-13s %016llx\n",
2914 "br_from:", save
->br_from
, "br_to:", save
->br_to
);
2915 pr_err("%-15s %016llx %-13s %016llx\n",
2916 "excp_from:", save
->last_excp_from
,
2917 "excp_to:", save
->last_excp_to
);
2920 static void svm_get_exit_info(struct kvm_vcpu
*vcpu
, u64
*info1
, u64
*info2
)
2922 struct vmcb_control_area
*control
= &to_svm(vcpu
)->vmcb
->control
;
2924 *info1
= control
->exit_info_1
;
2925 *info2
= control
->exit_info_2
;
2928 static int handle_exit(struct kvm_vcpu
*vcpu
, fastpath_t exit_fastpath
)
2930 struct vcpu_svm
*svm
= to_svm(vcpu
);
2931 struct kvm_run
*kvm_run
= vcpu
->run
;
2932 u32 exit_code
= svm
->vmcb
->control
.exit_code
;
2934 trace_kvm_exit(exit_code
, vcpu
, KVM_ISA_SVM
);
2936 if (!is_cr_intercept(svm
, INTERCEPT_CR0_WRITE
))
2937 vcpu
->arch
.cr0
= svm
->vmcb
->save
.cr0
;
2939 vcpu
->arch
.cr3
= svm
->vmcb
->save
.cr3
;
2941 svm_complete_interrupts(svm
);
2943 if (is_guest_mode(vcpu
)) {
2946 trace_kvm_nested_vmexit(svm
->vmcb
->save
.rip
, exit_code
,
2947 svm
->vmcb
->control
.exit_info_1
,
2948 svm
->vmcb
->control
.exit_info_2
,
2949 svm
->vmcb
->control
.exit_int_info
,
2950 svm
->vmcb
->control
.exit_int_info_err
,
2953 vmexit
= nested_svm_exit_special(svm
);
2955 if (vmexit
== NESTED_EXIT_CONTINUE
)
2956 vmexit
= nested_svm_exit_handled(svm
);
2958 if (vmexit
== NESTED_EXIT_DONE
)
2962 if (svm
->vmcb
->control
.exit_code
== SVM_EXIT_ERR
) {
2963 kvm_run
->exit_reason
= KVM_EXIT_FAIL_ENTRY
;
2964 kvm_run
->fail_entry
.hardware_entry_failure_reason
2965 = svm
->vmcb
->control
.exit_code
;
2966 kvm_run
->fail_entry
.cpu
= vcpu
->arch
.last_vmentry_cpu
;
2971 if (is_external_interrupt(svm
->vmcb
->control
.exit_int_info
) &&
2972 exit_code
!= SVM_EXIT_EXCP_BASE
+ PF_VECTOR
&&
2973 exit_code
!= SVM_EXIT_NPF
&& exit_code
!= SVM_EXIT_TASK_SWITCH
&&
2974 exit_code
!= SVM_EXIT_INTR
&& exit_code
!= SVM_EXIT_NMI
)
2975 printk(KERN_ERR
"%s: unexpected exit_int_info 0x%x "
2977 __func__
, svm
->vmcb
->control
.exit_int_info
,
2980 if (exit_fastpath
!= EXIT_FASTPATH_NONE
)
2983 if (exit_code
>= ARRAY_SIZE(svm_exit_handlers
)
2984 || !svm_exit_handlers
[exit_code
]) {
2985 vcpu_unimpl(vcpu
, "svm: unexpected exit reason 0x%x\n", exit_code
);
2987 vcpu
->run
->exit_reason
= KVM_EXIT_INTERNAL_ERROR
;
2988 vcpu
->run
->internal
.suberror
=
2989 KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON
;
2990 vcpu
->run
->internal
.ndata
= 2;
2991 vcpu
->run
->internal
.data
[0] = exit_code
;
2992 vcpu
->run
->internal
.data
[1] = vcpu
->arch
.last_vmentry_cpu
;
2996 #ifdef CONFIG_RETPOLINE
2997 if (exit_code
== SVM_EXIT_MSR
)
2998 return msr_interception(svm
);
2999 else if (exit_code
== SVM_EXIT_VINTR
)
3000 return interrupt_window_interception(svm
);
3001 else if (exit_code
== SVM_EXIT_INTR
)
3002 return intr_interception(svm
);
3003 else if (exit_code
== SVM_EXIT_HLT
)
3004 return halt_interception(svm
);
3005 else if (exit_code
== SVM_EXIT_NPF
)
3006 return npf_interception(svm
);
3008 return svm_exit_handlers
[exit_code
](svm
);
3011 static void reload_tss(struct kvm_vcpu
*vcpu
)
3013 struct svm_cpu_data
*sd
= per_cpu(svm_data
, vcpu
->cpu
);
3015 sd
->tss_desc
->type
= 9; /* available 32/64-bit TSS */
3019 static void pre_svm_run(struct vcpu_svm
*svm
)
3021 struct svm_cpu_data
*sd
= per_cpu(svm_data
, svm
->vcpu
.cpu
);
3023 if (sev_guest(svm
->vcpu
.kvm
))
3024 return pre_sev_run(svm
, svm
->vcpu
.cpu
);
3026 /* FIXME: handle wraparound of asid_generation */
3027 if (svm
->asid_generation
!= sd
->asid_generation
)
3031 static void svm_inject_nmi(struct kvm_vcpu
*vcpu
)
3033 struct vcpu_svm
*svm
= to_svm(vcpu
);
3035 svm
->vmcb
->control
.event_inj
= SVM_EVTINJ_VALID
| SVM_EVTINJ_TYPE_NMI
;
3036 vcpu
->arch
.hflags
|= HF_NMI_MASK
;
3037 svm_set_intercept(svm
, INTERCEPT_IRET
);
3038 ++vcpu
->stat
.nmi_injections
;
3041 static void svm_set_irq(struct kvm_vcpu
*vcpu
)
3043 struct vcpu_svm
*svm
= to_svm(vcpu
);
3045 BUG_ON(!(gif_set(svm
)));
3047 trace_kvm_inj_virq(vcpu
->arch
.interrupt
.nr
);
3048 ++vcpu
->stat
.irq_injections
;
3050 svm
->vmcb
->control
.event_inj
= vcpu
->arch
.interrupt
.nr
|
3051 SVM_EVTINJ_VALID
| SVM_EVTINJ_TYPE_INTR
;
3054 static void update_cr8_intercept(struct kvm_vcpu
*vcpu
, int tpr
, int irr
)
3056 struct vcpu_svm
*svm
= to_svm(vcpu
);
3058 if (nested_svm_virtualize_tpr(vcpu
))
3061 clr_cr_intercept(svm
, INTERCEPT_CR8_WRITE
);
3067 set_cr_intercept(svm
, INTERCEPT_CR8_WRITE
);
3070 bool svm_nmi_blocked(struct kvm_vcpu
*vcpu
)
3072 struct vcpu_svm
*svm
= to_svm(vcpu
);
3073 struct vmcb
*vmcb
= svm
->vmcb
;
3079 if (is_guest_mode(vcpu
) && nested_exit_on_nmi(svm
))
3082 ret
= (vmcb
->control
.int_state
& SVM_INTERRUPT_SHADOW_MASK
) ||
3083 (svm
->vcpu
.arch
.hflags
& HF_NMI_MASK
);
3088 static int svm_nmi_allowed(struct kvm_vcpu
*vcpu
, bool for_injection
)
3090 struct vcpu_svm
*svm
= to_svm(vcpu
);
3091 if (svm
->nested
.nested_run_pending
)
3094 /* An NMI must not be injected into L2 if it's supposed to VM-Exit. */
3095 if (for_injection
&& is_guest_mode(vcpu
) && nested_exit_on_nmi(svm
))
3098 return !svm_nmi_blocked(vcpu
);
3101 static bool svm_get_nmi_mask(struct kvm_vcpu
*vcpu
)
3103 struct vcpu_svm
*svm
= to_svm(vcpu
);
3105 return !!(svm
->vcpu
.arch
.hflags
& HF_NMI_MASK
);
3108 static void svm_set_nmi_mask(struct kvm_vcpu
*vcpu
, bool masked
)
3110 struct vcpu_svm
*svm
= to_svm(vcpu
);
3113 svm
->vcpu
.arch
.hflags
|= HF_NMI_MASK
;
3114 svm_set_intercept(svm
, INTERCEPT_IRET
);
3116 svm
->vcpu
.arch
.hflags
&= ~HF_NMI_MASK
;
3117 svm_clr_intercept(svm
, INTERCEPT_IRET
);
3121 bool svm_interrupt_blocked(struct kvm_vcpu
*vcpu
)
3123 struct vcpu_svm
*svm
= to_svm(vcpu
);
3124 struct vmcb
*vmcb
= svm
->vmcb
;
3129 if (is_guest_mode(vcpu
)) {
3130 /* As long as interrupts are being delivered... */
3131 if ((svm
->nested
.ctl
.int_ctl
& V_INTR_MASKING_MASK
)
3132 ? !(svm
->nested
.hsave
->save
.rflags
& X86_EFLAGS_IF
)
3133 : !(kvm_get_rflags(vcpu
) & X86_EFLAGS_IF
))
3136 /* ... vmexits aren't blocked by the interrupt shadow */
3137 if (nested_exit_on_intr(svm
))
3140 if (!(kvm_get_rflags(vcpu
) & X86_EFLAGS_IF
))
3144 return (vmcb
->control
.int_state
& SVM_INTERRUPT_SHADOW_MASK
);
3147 static int svm_interrupt_allowed(struct kvm_vcpu
*vcpu
, bool for_injection
)
3149 struct vcpu_svm
*svm
= to_svm(vcpu
);
3150 if (svm
->nested
.nested_run_pending
)
3154 * An IRQ must not be injected into L2 if it's supposed to VM-Exit,
3155 * e.g. if the IRQ arrived asynchronously after checking nested events.
3157 if (for_injection
&& is_guest_mode(vcpu
) && nested_exit_on_intr(svm
))
3160 return !svm_interrupt_blocked(vcpu
);
3163 static void enable_irq_window(struct kvm_vcpu
*vcpu
)
3165 struct vcpu_svm
*svm
= to_svm(vcpu
);
3168 * In case GIF=0 we can't rely on the CPU to tell us when GIF becomes
3169 * 1, because that's a separate STGI/VMRUN intercept. The next time we
3170 * get that intercept, this function will be called again though and
3171 * we'll get the vintr intercept. However, if the vGIF feature is
3172 * enabled, the STGI interception will not occur. Enable the irq
3173 * window under the assumption that the hardware will set the GIF.
3175 if (vgif_enabled(svm
) || gif_set(svm
)) {
3177 * IRQ window is not needed when AVIC is enabled,
3178 * unless we have pending ExtINT since it cannot be injected
3179 * via AVIC. In such case, we need to temporarily disable AVIC,
3180 * and fallback to injecting IRQ via V_IRQ.
3182 svm_toggle_avic_for_irq_window(vcpu
, false);
3187 static void enable_nmi_window(struct kvm_vcpu
*vcpu
)
3189 struct vcpu_svm
*svm
= to_svm(vcpu
);
3191 if ((svm
->vcpu
.arch
.hflags
& (HF_NMI_MASK
| HF_IRET_MASK
))
3193 return; /* IRET will cause a vm exit */
3195 if (!gif_set(svm
)) {
3196 if (vgif_enabled(svm
))
3197 svm_set_intercept(svm
, INTERCEPT_STGI
);
3198 return; /* STGI will cause a vm exit */
3202 * Something prevents NMI from been injected. Single step over possible
3203 * problem (IRET or exception injection or interrupt shadow)
3205 svm
->nmi_singlestep_guest_rflags
= svm_get_rflags(vcpu
);
3206 svm
->nmi_singlestep
= true;
3207 svm
->vmcb
->save
.rflags
|= (X86_EFLAGS_TF
| X86_EFLAGS_RF
);
3210 static int svm_set_tss_addr(struct kvm
*kvm
, unsigned int addr
)
3215 static int svm_set_identity_map_addr(struct kvm
*kvm
, u64 ident_addr
)
3220 void svm_flush_tlb(struct kvm_vcpu
*vcpu
)
3222 struct vcpu_svm
*svm
= to_svm(vcpu
);
3225 * Flush only the current ASID even if the TLB flush was invoked via
3226 * kvm_flush_remote_tlbs(). Although flushing remote TLBs requires all
3227 * ASIDs to be flushed, KVM uses a single ASID for L1 and L2, and
3228 * unconditionally does a TLB flush on both nested VM-Enter and nested
3229 * VM-Exit (via kvm_mmu_reset_context()).
3231 if (static_cpu_has(X86_FEATURE_FLUSHBYASID
))
3232 svm
->vmcb
->control
.tlb_ctl
= TLB_CONTROL_FLUSH_ASID
;
3234 svm
->asid_generation
--;
3237 static void svm_flush_tlb_gva(struct kvm_vcpu
*vcpu
, gva_t gva
)
3239 struct vcpu_svm
*svm
= to_svm(vcpu
);
3241 invlpga(gva
, svm
->vmcb
->control
.asid
);
3244 static void svm_prepare_guest_switch(struct kvm_vcpu
*vcpu
)
3248 static inline void sync_cr8_to_lapic(struct kvm_vcpu
*vcpu
)
3250 struct vcpu_svm
*svm
= to_svm(vcpu
);
3252 if (nested_svm_virtualize_tpr(vcpu
))
3255 if (!is_cr_intercept(svm
, INTERCEPT_CR8_WRITE
)) {
3256 int cr8
= svm
->vmcb
->control
.int_ctl
& V_TPR_MASK
;
3257 kvm_set_cr8(vcpu
, cr8
);
3261 static inline void sync_lapic_to_cr8(struct kvm_vcpu
*vcpu
)
3263 struct vcpu_svm
*svm
= to_svm(vcpu
);
3266 if (nested_svm_virtualize_tpr(vcpu
) ||
3267 kvm_vcpu_apicv_active(vcpu
))
3270 cr8
= kvm_get_cr8(vcpu
);
3271 svm
->vmcb
->control
.int_ctl
&= ~V_TPR_MASK
;
3272 svm
->vmcb
->control
.int_ctl
|= cr8
& V_TPR_MASK
;
3275 static void svm_complete_interrupts(struct vcpu_svm
*svm
)
3279 u32 exitintinfo
= svm
->vmcb
->control
.exit_int_info
;
3280 unsigned int3_injected
= svm
->int3_injected
;
3282 svm
->int3_injected
= 0;
3285 * If we've made progress since setting HF_IRET_MASK, we've
3286 * executed an IRET and can allow NMI injection.
3288 if ((svm
->vcpu
.arch
.hflags
& HF_IRET_MASK
)
3289 && kvm_rip_read(&svm
->vcpu
) != svm
->nmi_iret_rip
) {
3290 svm
->vcpu
.arch
.hflags
&= ~(HF_NMI_MASK
| HF_IRET_MASK
);
3291 kvm_make_request(KVM_REQ_EVENT
, &svm
->vcpu
);
3294 svm
->vcpu
.arch
.nmi_injected
= false;
3295 kvm_clear_exception_queue(&svm
->vcpu
);
3296 kvm_clear_interrupt_queue(&svm
->vcpu
);
3298 if (!(exitintinfo
& SVM_EXITINTINFO_VALID
))
3301 kvm_make_request(KVM_REQ_EVENT
, &svm
->vcpu
);
3303 vector
= exitintinfo
& SVM_EXITINTINFO_VEC_MASK
;
3304 type
= exitintinfo
& SVM_EXITINTINFO_TYPE_MASK
;
3307 case SVM_EXITINTINFO_TYPE_NMI
:
3308 svm
->vcpu
.arch
.nmi_injected
= true;
3310 case SVM_EXITINTINFO_TYPE_EXEPT
:
3312 * In case of software exceptions, do not reinject the vector,
3313 * but re-execute the instruction instead. Rewind RIP first
3314 * if we emulated INT3 before.
3316 if (kvm_exception_is_soft(vector
)) {
3317 if (vector
== BP_VECTOR
&& int3_injected
&&
3318 kvm_is_linear_rip(&svm
->vcpu
, svm
->int3_rip
))
3319 kvm_rip_write(&svm
->vcpu
,
3320 kvm_rip_read(&svm
->vcpu
) -
3324 if (exitintinfo
& SVM_EXITINTINFO_VALID_ERR
) {
3325 u32 err
= svm
->vmcb
->control
.exit_int_info_err
;
3326 kvm_requeue_exception_e(&svm
->vcpu
, vector
, err
);
3329 kvm_requeue_exception(&svm
->vcpu
, vector
);
3331 case SVM_EXITINTINFO_TYPE_INTR
:
3332 kvm_queue_interrupt(&svm
->vcpu
, vector
, false);
3339 static void svm_cancel_injection(struct kvm_vcpu
*vcpu
)
3341 struct vcpu_svm
*svm
= to_svm(vcpu
);
3342 struct vmcb_control_area
*control
= &svm
->vmcb
->control
;
3344 control
->exit_int_info
= control
->event_inj
;
3345 control
->exit_int_info_err
= control
->event_inj_err
;
3346 control
->event_inj
= 0;
3347 svm_complete_interrupts(svm
);
3350 static fastpath_t
svm_exit_handlers_fastpath(struct kvm_vcpu
*vcpu
)
3352 if (!is_guest_mode(vcpu
) &&
3353 to_svm(vcpu
)->vmcb
->control
.exit_code
== SVM_EXIT_MSR
&&
3354 to_svm(vcpu
)->vmcb
->control
.exit_info_1
)
3355 return handle_fastpath_set_msr_irqoff(vcpu
);
3357 return EXIT_FASTPATH_NONE
;
3360 void __svm_vcpu_run(unsigned long vmcb_pa
, unsigned long *regs
);
3362 static noinstr
void svm_vcpu_enter_exit(struct kvm_vcpu
*vcpu
,
3363 struct vcpu_svm
*svm
)
3366 * VMENTER enables interrupts (host state), but the kernel state is
3367 * interrupts disabled when this is invoked. Also tell RCU about
3368 * it. This is the same logic as for exit_to_user_mode().
3370 * This ensures that e.g. latency analysis on the host observes
3371 * guest mode as interrupt enabled.
3373 * guest_enter_irqoff() informs context tracking about the
3374 * transition to guest mode and if enabled adjusts RCU state
3377 instrumentation_begin();
3378 trace_hardirqs_on_prepare();
3379 lockdep_hardirqs_on_prepare(CALLER_ADDR0
);
3380 instrumentation_end();
3382 guest_enter_irqoff();
3383 lockdep_hardirqs_on(CALLER_ADDR0
);
3385 __svm_vcpu_run(svm
->vmcb_pa
, (unsigned long *)&svm
->vcpu
.arch
.regs
);
3387 #ifdef CONFIG_X86_64
3388 native_wrmsrl(MSR_GS_BASE
, svm
->host
.gs_base
);
3390 loadsegment(fs
, svm
->host
.fs
);
3391 #ifndef CONFIG_X86_32_LAZY_GS
3392 loadsegment(gs
, svm
->host
.gs
);
3397 * VMEXIT disables interrupts (host state), but tracing and lockdep
3398 * have them in state 'on' as recorded before entering guest mode.
3399 * Same as enter_from_user_mode().
3401 * guest_exit_irqoff() restores host context and reinstates RCU if
3402 * enabled and required.
3404 * This needs to be done before the below as native_read_msr()
3405 * contains a tracepoint and x86_spec_ctrl_restore_host() calls
3406 * into world and some more.
3408 lockdep_hardirqs_off(CALLER_ADDR0
);
3409 guest_exit_irqoff();
3411 instrumentation_begin();
3412 trace_hardirqs_off_finish();
3413 instrumentation_end();
3416 static __no_kcsan fastpath_t
svm_vcpu_run(struct kvm_vcpu
*vcpu
)
3418 fastpath_t exit_fastpath
;
3419 struct vcpu_svm
*svm
= to_svm(vcpu
);
3421 svm
->vmcb
->save
.rax
= vcpu
->arch
.regs
[VCPU_REGS_RAX
];
3422 svm
->vmcb
->save
.rsp
= vcpu
->arch
.regs
[VCPU_REGS_RSP
];
3423 svm
->vmcb
->save
.rip
= vcpu
->arch
.regs
[VCPU_REGS_RIP
];
3426 * Disable singlestep if we're injecting an interrupt/exception.
3427 * We don't want our modified rflags to be pushed on the stack where
3428 * we might not be able to easily reset them if we disabled NMI
3431 if (svm
->nmi_singlestep
&& svm
->vmcb
->control
.event_inj
) {
3433 * Event injection happens before external interrupts cause a
3434 * vmexit and interrupts are disabled here, so smp_send_reschedule
3435 * is enough to force an immediate vmexit.
3437 disable_nmi_singlestep(svm
);
3438 smp_send_reschedule(vcpu
->cpu
);
3443 sync_lapic_to_cr8(vcpu
);
3445 svm
->vmcb
->save
.cr2
= vcpu
->arch
.cr2
;
3448 * Run with all-zero DR6 unless needed, so that we can get the exact cause
3451 if (unlikely(svm
->vcpu
.arch
.switch_db_regs
& KVM_DEBUGREG_WONT_EXIT
))
3452 svm_set_dr6(svm
, vcpu
->arch
.dr6
);
3454 svm_set_dr6(svm
, DR6_FIXED_1
| DR6_RTM
);
3457 kvm_load_guest_xsave_state(vcpu
);
3459 if (lapic_in_kernel(vcpu
) &&
3460 vcpu
->arch
.apic
->lapic_timer
.timer_advance_ns
)
3461 kvm_wait_lapic_expire(vcpu
);
3464 * If this vCPU has touched SPEC_CTRL, restore the guest's value if
3465 * it's non-zero. Since vmentry is serialising on affected CPUs, there
3466 * is no need to worry about the conditional branch over the wrmsr
3467 * being speculatively taken.
3469 x86_spec_ctrl_set_guest(svm
->spec_ctrl
, svm
->virt_spec_ctrl
);
3471 svm_vcpu_enter_exit(vcpu
, svm
);
3474 * We do not use IBRS in the kernel. If this vCPU has used the
3475 * SPEC_CTRL MSR it may have left it on; save the value and
3476 * turn it off. This is much more efficient than blindly adding
3477 * it to the atomic save/restore list. Especially as the former
3478 * (Saving guest MSRs on vmexit) doesn't even exist in KVM.
3480 * For non-nested case:
3481 * If the L01 MSR bitmap does not intercept the MSR, then we need to
3485 * If the L02 MSR bitmap does not intercept the MSR, then we need to
3488 if (unlikely(!msr_write_intercepted(vcpu
, MSR_IA32_SPEC_CTRL
)))
3489 svm
->spec_ctrl
= native_read_msr(MSR_IA32_SPEC_CTRL
);
3493 x86_spec_ctrl_restore_host(svm
->spec_ctrl
, svm
->virt_spec_ctrl
);
3495 vcpu
->arch
.cr2
= svm
->vmcb
->save
.cr2
;
3496 vcpu
->arch
.regs
[VCPU_REGS_RAX
] = svm
->vmcb
->save
.rax
;
3497 vcpu
->arch
.regs
[VCPU_REGS_RSP
] = svm
->vmcb
->save
.rsp
;
3498 vcpu
->arch
.regs
[VCPU_REGS_RIP
] = svm
->vmcb
->save
.rip
;
3500 if (unlikely(svm
->vmcb
->control
.exit_code
== SVM_EXIT_NMI
))
3501 kvm_before_interrupt(&svm
->vcpu
);
3503 kvm_load_host_xsave_state(vcpu
);
3506 /* Any pending NMI will happen here */
3507 exit_fastpath
= svm_exit_handlers_fastpath(vcpu
);
3509 if (unlikely(svm
->vmcb
->control
.exit_code
== SVM_EXIT_NMI
))
3510 kvm_after_interrupt(&svm
->vcpu
);
3512 sync_cr8_to_lapic(vcpu
);
3515 if (is_guest_mode(&svm
->vcpu
)) {
3516 sync_nested_vmcb_control(svm
);
3517 svm
->nested
.nested_run_pending
= 0;
3520 svm
->vmcb
->control
.tlb_ctl
= TLB_CONTROL_DO_NOTHING
;
3522 /* if exit due to PF check for async PF */
3523 if (svm
->vmcb
->control
.exit_code
== SVM_EXIT_EXCP_BASE
+ PF_VECTOR
)
3524 svm
->vcpu
.arch
.apf
.host_apf_flags
=
3525 kvm_read_and_reset_apf_flags();
3528 vcpu
->arch
.regs_avail
&= ~(1 << VCPU_EXREG_PDPTR
);
3529 vcpu
->arch
.regs_dirty
&= ~(1 << VCPU_EXREG_PDPTR
);
3533 * We need to handle MC intercepts here before the vcpu has a chance to
3534 * change the physical cpu
3536 if (unlikely(svm
->vmcb
->control
.exit_code
==
3537 SVM_EXIT_EXCP_BASE
+ MC_VECTOR
))
3538 svm_handle_mce(svm
);
3540 vmcb_mark_all_clean(svm
->vmcb
);
3541 return exit_fastpath
;
3544 static void svm_load_mmu_pgd(struct kvm_vcpu
*vcpu
, unsigned long root
,
3547 struct vcpu_svm
*svm
= to_svm(vcpu
);
3550 cr3
= __sme_set(root
);
3552 svm
->vmcb
->control
.nested_cr3
= cr3
;
3553 vmcb_mark_dirty(svm
->vmcb
, VMCB_NPT
);
3555 /* Loading L2's CR3 is handled by enter_svm_guest_mode. */
3556 if (!test_bit(VCPU_EXREG_CR3
, (ulong
*)&vcpu
->arch
.regs_avail
))
3558 cr3
= vcpu
->arch
.cr3
;
3561 svm
->vmcb
->save
.cr3
= cr3
;
3562 vmcb_mark_dirty(svm
->vmcb
, VMCB_CR
);
3565 static int is_disabled(void)
3569 rdmsrl(MSR_VM_CR
, vm_cr
);
3570 if (vm_cr
& (1 << SVM_VM_CR_SVM_DISABLE
))
3577 svm_patch_hypercall(struct kvm_vcpu
*vcpu
, unsigned char *hypercall
)
3580 * Patch in the VMMCALL instruction:
3582 hypercall
[0] = 0x0f;
3583 hypercall
[1] = 0x01;
3584 hypercall
[2] = 0xd9;
3587 static int __init
svm_check_processor_compat(void)
3592 static bool svm_cpu_has_accelerated_tpr(void)
3597 static bool svm_has_emulated_msr(u32 index
)
3600 case MSR_IA32_MCG_EXT_CTL
:
3601 case MSR_IA32_VMX_BASIC
... MSR_IA32_VMX_VMFUNC
:
3610 static u64
svm_get_mt_mask(struct kvm_vcpu
*vcpu
, gfn_t gfn
, bool is_mmio
)
3615 static void svm_vcpu_after_set_cpuid(struct kvm_vcpu
*vcpu
)
3617 struct vcpu_svm
*svm
= to_svm(vcpu
);
3619 vcpu
->arch
.xsaves_enabled
= guest_cpuid_has(vcpu
, X86_FEATURE_XSAVE
) &&
3620 boot_cpu_has(X86_FEATURE_XSAVE
) &&
3621 boot_cpu_has(X86_FEATURE_XSAVES
);
3623 /* Update nrips enabled cache */
3624 svm
->nrips_enabled
= kvm_cpu_cap_has(X86_FEATURE_NRIPS
) &&
3625 guest_cpuid_has(&svm
->vcpu
, X86_FEATURE_NRIPS
);
3627 if (!kvm_vcpu_apicv_active(vcpu
))
3631 * AVIC does not work with an x2APIC mode guest. If the X2APIC feature
3632 * is exposed to the guest, disable AVIC.
3634 if (guest_cpuid_has(vcpu
, X86_FEATURE_X2APIC
))
3635 kvm_request_apicv_update(vcpu
->kvm
, false,
3636 APICV_INHIBIT_REASON_X2APIC
);
3639 * Currently, AVIC does not work with nested virtualization.
3640 * So, we disable AVIC when cpuid for SVM is set in the L1 guest.
3642 if (nested
&& guest_cpuid_has(vcpu
, X86_FEATURE_SVM
))
3643 kvm_request_apicv_update(vcpu
->kvm
, false,
3644 APICV_INHIBIT_REASON_NESTED
);
3647 static bool svm_has_wbinvd_exit(void)
3652 #define PRE_EX(exit) { .exit_code = (exit), \
3653 .stage = X86_ICPT_PRE_EXCEPT, }
3654 #define POST_EX(exit) { .exit_code = (exit), \
3655 .stage = X86_ICPT_POST_EXCEPT, }
3656 #define POST_MEM(exit) { .exit_code = (exit), \
3657 .stage = X86_ICPT_POST_MEMACCESS, }
3659 static const struct __x86_intercept
{
3661 enum x86_intercept_stage stage
;
3662 } x86_intercept_map
[] = {
3663 [x86_intercept_cr_read
] = POST_EX(SVM_EXIT_READ_CR0
),
3664 [x86_intercept_cr_write
] = POST_EX(SVM_EXIT_WRITE_CR0
),
3665 [x86_intercept_clts
] = POST_EX(SVM_EXIT_WRITE_CR0
),
3666 [x86_intercept_lmsw
] = POST_EX(SVM_EXIT_WRITE_CR0
),
3667 [x86_intercept_smsw
] = POST_EX(SVM_EXIT_READ_CR0
),
3668 [x86_intercept_dr_read
] = POST_EX(SVM_EXIT_READ_DR0
),
3669 [x86_intercept_dr_write
] = POST_EX(SVM_EXIT_WRITE_DR0
),
3670 [x86_intercept_sldt
] = POST_EX(SVM_EXIT_LDTR_READ
),
3671 [x86_intercept_str
] = POST_EX(SVM_EXIT_TR_READ
),
3672 [x86_intercept_lldt
] = POST_EX(SVM_EXIT_LDTR_WRITE
),
3673 [x86_intercept_ltr
] = POST_EX(SVM_EXIT_TR_WRITE
),
3674 [x86_intercept_sgdt
] = POST_EX(SVM_EXIT_GDTR_READ
),
3675 [x86_intercept_sidt
] = POST_EX(SVM_EXIT_IDTR_READ
),
3676 [x86_intercept_lgdt
] = POST_EX(SVM_EXIT_GDTR_WRITE
),
3677 [x86_intercept_lidt
] = POST_EX(SVM_EXIT_IDTR_WRITE
),
3678 [x86_intercept_vmrun
] = POST_EX(SVM_EXIT_VMRUN
),
3679 [x86_intercept_vmmcall
] = POST_EX(SVM_EXIT_VMMCALL
),
3680 [x86_intercept_vmload
] = POST_EX(SVM_EXIT_VMLOAD
),
3681 [x86_intercept_vmsave
] = POST_EX(SVM_EXIT_VMSAVE
),
3682 [x86_intercept_stgi
] = POST_EX(SVM_EXIT_STGI
),
3683 [x86_intercept_clgi
] = POST_EX(SVM_EXIT_CLGI
),
3684 [x86_intercept_skinit
] = POST_EX(SVM_EXIT_SKINIT
),
3685 [x86_intercept_invlpga
] = POST_EX(SVM_EXIT_INVLPGA
),
3686 [x86_intercept_rdtscp
] = POST_EX(SVM_EXIT_RDTSCP
),
3687 [x86_intercept_monitor
] = POST_MEM(SVM_EXIT_MONITOR
),
3688 [x86_intercept_mwait
] = POST_EX(SVM_EXIT_MWAIT
),
3689 [x86_intercept_invlpg
] = POST_EX(SVM_EXIT_INVLPG
),
3690 [x86_intercept_invd
] = POST_EX(SVM_EXIT_INVD
),
3691 [x86_intercept_wbinvd
] = POST_EX(SVM_EXIT_WBINVD
),
3692 [x86_intercept_wrmsr
] = POST_EX(SVM_EXIT_MSR
),
3693 [x86_intercept_rdtsc
] = POST_EX(SVM_EXIT_RDTSC
),
3694 [x86_intercept_rdmsr
] = POST_EX(SVM_EXIT_MSR
),
3695 [x86_intercept_rdpmc
] = POST_EX(SVM_EXIT_RDPMC
),
3696 [x86_intercept_cpuid
] = PRE_EX(SVM_EXIT_CPUID
),
3697 [x86_intercept_rsm
] = PRE_EX(SVM_EXIT_RSM
),
3698 [x86_intercept_pause
] = PRE_EX(SVM_EXIT_PAUSE
),
3699 [x86_intercept_pushf
] = PRE_EX(SVM_EXIT_PUSHF
),
3700 [x86_intercept_popf
] = PRE_EX(SVM_EXIT_POPF
),
3701 [x86_intercept_intn
] = PRE_EX(SVM_EXIT_SWINT
),
3702 [x86_intercept_iret
] = PRE_EX(SVM_EXIT_IRET
),
3703 [x86_intercept_icebp
] = PRE_EX(SVM_EXIT_ICEBP
),
3704 [x86_intercept_hlt
] = POST_EX(SVM_EXIT_HLT
),
3705 [x86_intercept_in
] = POST_EX(SVM_EXIT_IOIO
),
3706 [x86_intercept_ins
] = POST_EX(SVM_EXIT_IOIO
),
3707 [x86_intercept_out
] = POST_EX(SVM_EXIT_IOIO
),
3708 [x86_intercept_outs
] = POST_EX(SVM_EXIT_IOIO
),
3709 [x86_intercept_xsetbv
] = PRE_EX(SVM_EXIT_XSETBV
),
3716 static int svm_check_intercept(struct kvm_vcpu
*vcpu
,
3717 struct x86_instruction_info
*info
,
3718 enum x86_intercept_stage stage
,
3719 struct x86_exception
*exception
)
3721 struct vcpu_svm
*svm
= to_svm(vcpu
);
3722 int vmexit
, ret
= X86EMUL_CONTINUE
;
3723 struct __x86_intercept icpt_info
;
3724 struct vmcb
*vmcb
= svm
->vmcb
;
3726 if (info
->intercept
>= ARRAY_SIZE(x86_intercept_map
))
3729 icpt_info
= x86_intercept_map
[info
->intercept
];
3731 if (stage
!= icpt_info
.stage
)
3734 switch (icpt_info
.exit_code
) {
3735 case SVM_EXIT_READ_CR0
:
3736 if (info
->intercept
== x86_intercept_cr_read
)
3737 icpt_info
.exit_code
+= info
->modrm_reg
;
3739 case SVM_EXIT_WRITE_CR0
: {
3740 unsigned long cr0
, val
;
3743 if (info
->intercept
== x86_intercept_cr_write
)
3744 icpt_info
.exit_code
+= info
->modrm_reg
;
3746 if (icpt_info
.exit_code
!= SVM_EXIT_WRITE_CR0
||
3747 info
->intercept
== x86_intercept_clts
)
3750 intercept
= svm
->nested
.ctl
.intercept
;
3752 if (!(intercept
& (1ULL << INTERCEPT_SELECTIVE_CR0
)))
3755 cr0
= vcpu
->arch
.cr0
& ~SVM_CR0_SELECTIVE_MASK
;
3756 val
= info
->src_val
& ~SVM_CR0_SELECTIVE_MASK
;
3758 if (info
->intercept
== x86_intercept_lmsw
) {
3761 /* lmsw can't clear PE - catch this here */
3762 if (cr0
& X86_CR0_PE
)
3767 icpt_info
.exit_code
= SVM_EXIT_CR0_SEL_WRITE
;
3771 case SVM_EXIT_READ_DR0
:
3772 case SVM_EXIT_WRITE_DR0
:
3773 icpt_info
.exit_code
+= info
->modrm_reg
;
3776 if (info
->intercept
== x86_intercept_wrmsr
)
3777 vmcb
->control
.exit_info_1
= 1;
3779 vmcb
->control
.exit_info_1
= 0;
3781 case SVM_EXIT_PAUSE
:
3783 * We get this for NOP only, but pause
3784 * is rep not, check this here
3786 if (info
->rep_prefix
!= REPE_PREFIX
)
3789 case SVM_EXIT_IOIO
: {
3793 if (info
->intercept
== x86_intercept_in
||
3794 info
->intercept
== x86_intercept_ins
) {
3795 exit_info
= ((info
->src_val
& 0xffff) << 16) |
3797 bytes
= info
->dst_bytes
;
3799 exit_info
= (info
->dst_val
& 0xffff) << 16;
3800 bytes
= info
->src_bytes
;
3803 if (info
->intercept
== x86_intercept_outs
||
3804 info
->intercept
== x86_intercept_ins
)
3805 exit_info
|= SVM_IOIO_STR_MASK
;
3807 if (info
->rep_prefix
)
3808 exit_info
|= SVM_IOIO_REP_MASK
;
3810 bytes
= min(bytes
, 4u);
3812 exit_info
|= bytes
<< SVM_IOIO_SIZE_SHIFT
;
3814 exit_info
|= (u32
)info
->ad_bytes
<< (SVM_IOIO_ASIZE_SHIFT
- 1);
3816 vmcb
->control
.exit_info_1
= exit_info
;
3817 vmcb
->control
.exit_info_2
= info
->next_rip
;
3825 /* TODO: Advertise NRIPS to guest hypervisor unconditionally */
3826 if (static_cpu_has(X86_FEATURE_NRIPS
))
3827 vmcb
->control
.next_rip
= info
->next_rip
;
3828 vmcb
->control
.exit_code
= icpt_info
.exit_code
;
3829 vmexit
= nested_svm_exit_handled(svm
);
3831 ret
= (vmexit
== NESTED_EXIT_DONE
) ? X86EMUL_INTERCEPTED
3838 static void svm_handle_exit_irqoff(struct kvm_vcpu
*vcpu
)
3842 static void svm_sched_in(struct kvm_vcpu
*vcpu
, int cpu
)
3844 if (!kvm_pause_in_guest(vcpu
->kvm
))
3845 shrink_ple_window(vcpu
);
3848 static void svm_setup_mce(struct kvm_vcpu
*vcpu
)
3850 /* [63:9] are reserved. */
3851 vcpu
->arch
.mcg_cap
&= 0x1ff;
3854 bool svm_smi_blocked(struct kvm_vcpu
*vcpu
)
3856 struct vcpu_svm
*svm
= to_svm(vcpu
);
3858 /* Per APM Vol.2 15.22.2 "Response to SMI" */
3862 return is_smm(vcpu
);
3865 static int svm_smi_allowed(struct kvm_vcpu
*vcpu
, bool for_injection
)
3867 struct vcpu_svm
*svm
= to_svm(vcpu
);
3868 if (svm
->nested
.nested_run_pending
)
3871 /* An SMI must not be injected into L2 if it's supposed to VM-Exit. */
3872 if (for_injection
&& is_guest_mode(vcpu
) && nested_exit_on_smi(svm
))
3875 return !svm_smi_blocked(vcpu
);
3878 static int svm_pre_enter_smm(struct kvm_vcpu
*vcpu
, char *smstate
)
3880 struct vcpu_svm
*svm
= to_svm(vcpu
);
3883 if (is_guest_mode(vcpu
)) {
3884 /* FED8h - SVM Guest */
3885 put_smstate(u64
, smstate
, 0x7ed8, 1);
3886 /* FEE0h - SVM Guest VMCB Physical Address */
3887 put_smstate(u64
, smstate
, 0x7ee0, svm
->nested
.vmcb
);
3889 svm
->vmcb
->save
.rax
= vcpu
->arch
.regs
[VCPU_REGS_RAX
];
3890 svm
->vmcb
->save
.rsp
= vcpu
->arch
.regs
[VCPU_REGS_RSP
];
3891 svm
->vmcb
->save
.rip
= vcpu
->arch
.regs
[VCPU_REGS_RIP
];
3893 ret
= nested_svm_vmexit(svm
);
3900 static int svm_pre_leave_smm(struct kvm_vcpu
*vcpu
, const char *smstate
)
3902 struct vcpu_svm
*svm
= to_svm(vcpu
);
3903 struct vmcb
*nested_vmcb
;
3904 struct kvm_host_map map
;
3909 guest
= GET_SMSTATE(u64
, smstate
, 0x7ed8);
3910 vmcb
= GET_SMSTATE(u64
, smstate
, 0x7ee0);
3913 if (kvm_vcpu_map(&svm
->vcpu
, gpa_to_gfn(vmcb
), &map
) == -EINVAL
)
3915 nested_vmcb
= map
.hva
;
3916 ret
= enter_svm_guest_mode(svm
, vmcb
, nested_vmcb
);
3917 kvm_vcpu_unmap(&svm
->vcpu
, &map
, true);
3923 static void enable_smi_window(struct kvm_vcpu
*vcpu
)
3925 struct vcpu_svm
*svm
= to_svm(vcpu
);
3927 if (!gif_set(svm
)) {
3928 if (vgif_enabled(svm
))
3929 svm_set_intercept(svm
, INTERCEPT_STGI
);
3930 /* STGI will cause a vm exit */
3932 /* We must be in SMM; RSM will cause a vmexit anyway. */
3936 static bool svm_need_emulation_on_page_fault(struct kvm_vcpu
*vcpu
)
3938 unsigned long cr4
= kvm_read_cr4(vcpu
);
3939 bool smep
= cr4
& X86_CR4_SMEP
;
3940 bool smap
= cr4
& X86_CR4_SMAP
;
3941 bool is_user
= svm_get_cpl(vcpu
) == 3;
3944 * If RIP is invalid, go ahead with emulation which will cause an
3945 * internal error exit.
3947 if (!kvm_vcpu_gfn_to_memslot(vcpu
, kvm_rip_read(vcpu
) >> PAGE_SHIFT
))
3951 * Detect and workaround Errata 1096 Fam_17h_00_0Fh.
3954 * When CPU raise #NPF on guest data access and vCPU CR4.SMAP=1, it is
3955 * possible that CPU microcode implementing DecodeAssist will fail
3956 * to read bytes of instruction which caused #NPF. In this case,
3957 * GuestIntrBytes field of the VMCB on a VMEXIT will incorrectly
3958 * return 0 instead of the correct guest instruction bytes.
3960 * This happens because CPU microcode reading instruction bytes
3961 * uses a special opcode which attempts to read data using CPL=0
3962 * priviledges. The microcode reads CS:RIP and if it hits a SMAP
3963 * fault, it gives up and returns no instruction bytes.
3966 * We reach here in case CPU supports DecodeAssist, raised #NPF and
3967 * returned 0 in GuestIntrBytes field of the VMCB.
3968 * First, errata can only be triggered in case vCPU CR4.SMAP=1.
3969 * Second, if vCPU CR4.SMEP=1, errata could only be triggered
3970 * in case vCPU CPL==3 (Because otherwise guest would have triggered
3971 * a SMEP fault instead of #NPF).
3972 * Otherwise, vCPU CR4.SMEP=0, errata could be triggered by any vCPU CPL.
3973 * As most guests enable SMAP if they have also enabled SMEP, use above
3974 * logic in order to attempt minimize false-positive of detecting errata
3975 * while still preserving all cases semantic correctness.
3978 * To determine what instruction the guest was executing, the hypervisor
3979 * will have to decode the instruction at the instruction pointer.
3981 * In non SEV guest, hypervisor will be able to read the guest
3982 * memory to decode the instruction pointer when insn_len is zero
3983 * so we return true to indicate that decoding is possible.
3985 * But in the SEV guest, the guest memory is encrypted with the
3986 * guest specific key and hypervisor will not be able to decode the
3987 * instruction pointer so we will not able to workaround it. Lets
3988 * print the error and request to kill the guest.
3990 if (smap
&& (!smep
|| is_user
)) {
3991 if (!sev_guest(vcpu
->kvm
))
3994 pr_err_ratelimited("KVM: SEV Guest triggered AMD Erratum 1096\n");
3995 kvm_make_request(KVM_REQ_TRIPLE_FAULT
, vcpu
);
4001 static bool svm_apic_init_signal_blocked(struct kvm_vcpu
*vcpu
)
4003 struct vcpu_svm
*svm
= to_svm(vcpu
);
4006 * TODO: Last condition latch INIT signals on vCPU when
4007 * vCPU is in guest-mode and vmcb12 defines intercept on INIT.
4008 * To properly emulate the INIT intercept,
4009 * svm_check_nested_events() should call nested_svm_vmexit()
4010 * if an INIT signal is pending.
4012 return !gif_set(svm
) ||
4013 (svm
->vmcb
->control
.intercept
& (1ULL << INTERCEPT_INIT
));
4016 static void svm_vm_destroy(struct kvm
*kvm
)
4018 avic_vm_destroy(kvm
);
4019 sev_vm_destroy(kvm
);
4022 static int svm_vm_init(struct kvm
*kvm
)
4024 if (!pause_filter_count
|| !pause_filter_thresh
)
4025 kvm
->arch
.pause_in_guest
= true;
4028 int ret
= avic_vm_init(kvm
);
4033 kvm_apicv_init(kvm
, avic
);
4037 static struct kvm_x86_ops svm_x86_ops __initdata
= {
4038 .hardware_unsetup
= svm_hardware_teardown
,
4039 .hardware_enable
= svm_hardware_enable
,
4040 .hardware_disable
= svm_hardware_disable
,
4041 .cpu_has_accelerated_tpr
= svm_cpu_has_accelerated_tpr
,
4042 .has_emulated_msr
= svm_has_emulated_msr
,
4044 .vcpu_create
= svm_create_vcpu
,
4045 .vcpu_free
= svm_free_vcpu
,
4046 .vcpu_reset
= svm_vcpu_reset
,
4048 .vm_size
= sizeof(struct kvm_svm
),
4049 .vm_init
= svm_vm_init
,
4050 .vm_destroy
= svm_vm_destroy
,
4052 .prepare_guest_switch
= svm_prepare_guest_switch
,
4053 .vcpu_load
= svm_vcpu_load
,
4054 .vcpu_put
= svm_vcpu_put
,
4055 .vcpu_blocking
= svm_vcpu_blocking
,
4056 .vcpu_unblocking
= svm_vcpu_unblocking
,
4058 .update_exception_bitmap
= update_exception_bitmap
,
4059 .get_msr_feature
= svm_get_msr_feature
,
4060 .get_msr
= svm_get_msr
,
4061 .set_msr
= svm_set_msr
,
4062 .get_segment_base
= svm_get_segment_base
,
4063 .get_segment
= svm_get_segment
,
4064 .set_segment
= svm_set_segment
,
4065 .get_cpl
= svm_get_cpl
,
4066 .get_cs_db_l_bits
= kvm_get_cs_db_l_bits
,
4067 .set_cr0
= svm_set_cr0
,
4068 .set_cr4
= svm_set_cr4
,
4069 .set_efer
= svm_set_efer
,
4070 .get_idt
= svm_get_idt
,
4071 .set_idt
= svm_set_idt
,
4072 .get_gdt
= svm_get_gdt
,
4073 .set_gdt
= svm_set_gdt
,
4074 .set_dr7
= svm_set_dr7
,
4075 .sync_dirty_debug_regs
= svm_sync_dirty_debug_regs
,
4076 .cache_reg
= svm_cache_reg
,
4077 .get_rflags
= svm_get_rflags
,
4078 .set_rflags
= svm_set_rflags
,
4080 .tlb_flush_all
= svm_flush_tlb
,
4081 .tlb_flush_current
= svm_flush_tlb
,
4082 .tlb_flush_gva
= svm_flush_tlb_gva
,
4083 .tlb_flush_guest
= svm_flush_tlb
,
4085 .run
= svm_vcpu_run
,
4086 .handle_exit
= handle_exit
,
4087 .skip_emulated_instruction
= skip_emulated_instruction
,
4088 .update_emulated_instruction
= NULL
,
4089 .set_interrupt_shadow
= svm_set_interrupt_shadow
,
4090 .get_interrupt_shadow
= svm_get_interrupt_shadow
,
4091 .patch_hypercall
= svm_patch_hypercall
,
4092 .set_irq
= svm_set_irq
,
4093 .set_nmi
= svm_inject_nmi
,
4094 .queue_exception
= svm_queue_exception
,
4095 .cancel_injection
= svm_cancel_injection
,
4096 .interrupt_allowed
= svm_interrupt_allowed
,
4097 .nmi_allowed
= svm_nmi_allowed
,
4098 .get_nmi_mask
= svm_get_nmi_mask
,
4099 .set_nmi_mask
= svm_set_nmi_mask
,
4100 .enable_nmi_window
= enable_nmi_window
,
4101 .enable_irq_window
= enable_irq_window
,
4102 .update_cr8_intercept
= update_cr8_intercept
,
4103 .set_virtual_apic_mode
= svm_set_virtual_apic_mode
,
4104 .refresh_apicv_exec_ctrl
= svm_refresh_apicv_exec_ctrl
,
4105 .check_apicv_inhibit_reasons
= svm_check_apicv_inhibit_reasons
,
4106 .pre_update_apicv_exec_ctrl
= svm_pre_update_apicv_exec_ctrl
,
4107 .load_eoi_exitmap
= svm_load_eoi_exitmap
,
4108 .hwapic_irr_update
= svm_hwapic_irr_update
,
4109 .hwapic_isr_update
= svm_hwapic_isr_update
,
4110 .sync_pir_to_irr
= kvm_lapic_find_highest_irr
,
4111 .apicv_post_state_restore
= avic_post_state_restore
,
4113 .set_tss_addr
= svm_set_tss_addr
,
4114 .set_identity_map_addr
= svm_set_identity_map_addr
,
4115 .get_mt_mask
= svm_get_mt_mask
,
4117 .get_exit_info
= svm_get_exit_info
,
4119 .vcpu_after_set_cpuid
= svm_vcpu_after_set_cpuid
,
4121 .has_wbinvd_exit
= svm_has_wbinvd_exit
,
4123 .write_l1_tsc_offset
= svm_write_l1_tsc_offset
,
4125 .load_mmu_pgd
= svm_load_mmu_pgd
,
4127 .check_intercept
= svm_check_intercept
,
4128 .handle_exit_irqoff
= svm_handle_exit_irqoff
,
4130 .request_immediate_exit
= __kvm_request_immediate_exit
,
4132 .sched_in
= svm_sched_in
,
4134 .pmu_ops
= &amd_pmu_ops
,
4135 .nested_ops
= &svm_nested_ops
,
4137 .deliver_posted_interrupt
= svm_deliver_avic_intr
,
4138 .dy_apicv_has_pending_interrupt
= svm_dy_apicv_has_pending_interrupt
,
4139 .update_pi_irte
= svm_update_pi_irte
,
4140 .setup_mce
= svm_setup_mce
,
4142 .smi_allowed
= svm_smi_allowed
,
4143 .pre_enter_smm
= svm_pre_enter_smm
,
4144 .pre_leave_smm
= svm_pre_leave_smm
,
4145 .enable_smi_window
= enable_smi_window
,
4147 .mem_enc_op
= svm_mem_enc_op
,
4148 .mem_enc_reg_region
= svm_register_enc_region
,
4149 .mem_enc_unreg_region
= svm_unregister_enc_region
,
4151 .need_emulation_on_page_fault
= svm_need_emulation_on_page_fault
,
4153 .apic_init_signal_blocked
= svm_apic_init_signal_blocked
,
4156 static struct kvm_x86_init_ops svm_init_ops __initdata
= {
4157 .cpu_has_kvm_support
= has_svm
,
4158 .disabled_by_bios
= is_disabled
,
4159 .hardware_setup
= svm_hardware_setup
,
4160 .check_processor_compatibility
= svm_check_processor_compat
,
4162 .runtime_ops
= &svm_x86_ops
,
4165 static int __init
svm_init(void)
4167 return kvm_init(&svm_init_ops
, sizeof(struct vcpu_svm
),
4168 __alignof__(struct vcpu_svm
), THIS_MODULE
);
4171 static void __exit
svm_exit(void)
4176 module_init(svm_init
)
4177 module_exit(svm_exit
)